]>
Commit | Line | Data |
---|---|---|
8c660648 | 1 | /* Instruction scheduling pass. |
cbe34bb5 | 2 | Copyright (C) 1992-2017 Free Software Foundation, Inc. |
8c660648 JL |
3 | Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by, |
4 | and currently maintained by, Jim Wilson (wilson@cygnus.com) | |
5 | ||
1322177d | 6 | This file is part of GCC. |
5d14e356 | 7 | |
1322177d LB |
8 | GCC is free software; you can redistribute it and/or modify it under |
9 | the terms of the GNU General Public License as published by the Free | |
9dcd6f09 | 10 | Software Foundation; either version 3, or (at your option) any later |
1322177d | 11 | version. |
5d14e356 | 12 | |
1322177d LB |
13 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
14 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
5d14e356 RK |
15 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
16 | for more details. | |
17 | ||
18 | You should have received a copy of the GNU General Public License | |
9dcd6f09 NC |
19 | along with GCC; see the file COPYING3. If not see |
20 | <http://www.gnu.org/licenses/>. */ | |
8c660648 | 21 | |
b4ead7d4 | 22 | /* Instruction scheduling pass. This file, along with sched-deps.c, |
151e9aac | 23 | contains the generic parts. The actual entry point for |
b4ead7d4 | 24 | the normal instruction scheduling pass is found in sched-rgn.c. |
8c660648 JL |
25 | |
26 | We compute insn priorities based on data dependencies. Flow | |
27 | analysis only creates a fraction of the data-dependencies we must | |
28 | observe: namely, only those dependencies which the combiner can be | |
29 | expected to use. For this pass, we must therefore create the | |
30 | remaining dependencies we need to observe: register dependencies, | |
31 | memory dependencies, dependencies to keep function calls in order, | |
32 | and the dependence between a conditional branch and the setting of | |
33 | condition codes are all dealt with here. | |
34 | ||
35 | The scheduler first traverses the data flow graph, starting with | |
36 | the last instruction, and proceeding to the first, assigning values | |
37 | to insn_priority as it goes. This sorts the instructions | |
38 | topologically by data dependence. | |
39 | ||
40 | Once priorities have been established, we order the insns using | |
41 | list scheduling. This works as follows: starting with a list of | |
42 | all the ready insns, and sorted according to priority number, we | |
43 | schedule the insn from the end of the list by placing its | |
44 | predecessors in the list according to their priority order. We | |
45 | consider this insn scheduled by setting the pointer to the "end" of | |
46 | the list to point to the previous insn. When an insn has no | |
47 | predecessors, we either queue it until sufficient time has elapsed | |
48 | or add it to the ready list. As the instructions are scheduled or | |
49 | when stalls are introduced, the queue advances and dumps insns into | |
50 | the ready list. When all insns down to the lowest priority have | |
51 | been scheduled, the critical path of the basic block has been made | |
52 | as short as possible. The remaining insns are then scheduled in | |
53 | remaining slots. | |
54 | ||
8c660648 JL |
55 | The following list shows the order in which we want to break ties |
56 | among insns in the ready list: | |
57 | ||
58 | 1. choose insn with the longest path to end of bb, ties | |
59 | broken by | |
60 | 2. choose insn with least contribution to register pressure, | |
61 | ties broken by | |
62 | 3. prefer in-block upon interblock motion, ties broken by | |
63 | 4. prefer useful upon speculative motion, ties broken by | |
64 | 5. choose insn with largest control flow probability, ties | |
65 | broken by | |
66 | 6. choose insn with the least dependences upon the previously | |
67 | scheduled insn, or finally | |
2db45993 JL |
68 | 7 choose the insn which has the most insns dependent on it. |
69 | 8. choose insn with lowest UID. | |
8c660648 JL |
70 | |
71 | Memory references complicate matters. Only if we can be certain | |
72 | that memory references are not part of the data dependency graph | |
73 | (via true, anti, or output dependence), can we move operations past | |
74 | memory references. To first approximation, reads can be done | |
75 | independently, while writes introduce dependencies. Better | |
76 | approximations will yield fewer dependencies. | |
77 | ||
78 | Before reload, an extended analysis of interblock data dependences | |
79 | is required for interblock scheduling. This is performed in | |
151e9aac | 80 | compute_block_dependences (). |
8c660648 JL |
81 | |
82 | Dependencies set up by memory references are treated in exactly the | |
b198261f MK |
83 | same way as other dependencies, by using insn backward dependences |
84 | INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences | |
151e9aac | 85 | INSN_FORW_DEPS for the purpose of forward list scheduling. |
8c660648 JL |
86 | |
87 | Having optimized the critical path, we may have also unduly | |
88 | extended the lifetimes of some registers. If an operation requires | |
89 | that constants be loaded into registers, it is certainly desirable | |
90 | to load those constants as early as necessary, but no earlier. | |
91 | I.e., it will not do to load up a bunch of registers at the | |
92 | beginning of a basic block only to use them at the end, if they | |
93 | could be loaded later, since this may result in excessive register | |
94 | utilization. | |
95 | ||
96 | Note that since branches are never in basic blocks, but only end | |
97 | basic blocks, this pass will not move branches. But that is ok, | |
98 | since we can use GNU's delayed branch scheduling pass to take care | |
99 | of this case. | |
100 | ||
101 | Also note that no further optimizations based on algebraic | |
102 | identities are performed, so this pass would be a good one to | |
103 | perform instruction splitting, such as breaking up a multiply | |
104 | instruction into shifts and adds where that is profitable. | |
105 | ||
106 | Given the memory aliasing analysis that this pass should perform, | |
107 | it should be possible to remove redundant stores to memory, and to | |
108 | load values from registers instead of hitting memory. | |
109 | ||
110 | Before reload, speculative insns are moved only if a 'proof' exists | |
111 | that no exception will be caused by this, and if no live registers | |
112 | exist that inhibit the motion (live registers constraints are not | |
113 | represented by data dependence edges). | |
114 | ||
115 | This pass must update information that subsequent passes expect to | |
116 | be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths, | |
a813c111 | 117 | reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END. |
8c660648 JL |
118 | |
119 | The information in the line number notes is carefully retained by | |
120 | this pass. Notes that refer to the starting and ending of | |
121 | exception regions are also carefully retained by this pass. All | |
122 | other NOTE insns are grouped in their same relative order at the | |
b4ead7d4 | 123 | beginning of basic blocks and regions that have been scheduled. */ |
8c660648 | 124 | \f |
8c660648 | 125 | #include "config.h" |
5835e573 | 126 | #include "system.h" |
4977bab6 | 127 | #include "coretypes.h" |
c7131fb2 | 128 | #include "backend.h" |
957060b5 | 129 | #include "target.h" |
8c660648 | 130 | #include "rtl.h" |
957060b5 | 131 | #include "cfghooks.h" |
c7131fb2 | 132 | #include "df.h" |
4d0cdd0c | 133 | #include "memmodel.h" |
6baf1cc8 | 134 | #include "tm_p.h" |
957060b5 | 135 | #include "insn-config.h" |
8c660648 | 136 | #include "regs.h" |
957060b5 AM |
137 | #include "ira.h" |
138 | #include "recog.h" | |
8c660648 | 139 | #include "insn-attr.h" |
60393bbc AM |
140 | #include "cfgrtl.h" |
141 | #include "cfgbuild.h" | |
1708fd40 | 142 | #include "sched-int.h" |
677f3fa8 | 143 | #include "common/common-target.h" |
496d7bb0 | 144 | #include "params.h" |
6fb5fa3c | 145 | #include "dbgcnt.h" |
e855c69d | 146 | #include "cfgloop.h" |
7ee2468b | 147 | #include "dumpfile.h" |
013a8899 | 148 | #include "print-rtl.h" |
8c660648 | 149 | |
8c660648 JL |
150 | #ifdef INSN_SCHEDULING |
151 | ||
f20f2613 VM |
152 | /* True if we do register pressure relief through live-range |
153 | shrinkage. */ | |
154 | static bool live_range_shrinkage_p; | |
155 | ||
156 | /* Switch on live range shrinkage. */ | |
157 | void | |
158 | initialize_live_range_shrinkage (void) | |
159 | { | |
160 | live_range_shrinkage_p = true; | |
161 | } | |
162 | ||
163 | /* Switch off live range shrinkage. */ | |
164 | void | |
165 | finish_live_range_shrinkage (void) | |
166 | { | |
167 | live_range_shrinkage_p = false; | |
168 | } | |
169 | ||
8c660648 JL |
170 | /* issue_rate is the number of insns that can be scheduled in the same |
171 | machine cycle. It can be defined in the config/mach/mach.h file, | |
172 | otherwise we set it to 1. */ | |
173 | ||
e855c69d | 174 | int issue_rate; |
8c660648 | 175 | |
26965010 BS |
176 | /* This can be set to true by a backend if the scheduler should not |
177 | enable a DCE pass. */ | |
178 | bool sched_no_dce; | |
179 | ||
06d7e8e7 BS |
180 | /* The current initiation interval used when modulo scheduling. */ |
181 | static int modulo_ii; | |
182 | ||
183 | /* The maximum number of stages we are prepared to handle. */ | |
184 | static int modulo_max_stages; | |
185 | ||
186 | /* The number of insns that exist in each iteration of the loop. We use this | |
187 | to detect when we've scheduled all insns from the first iteration. */ | |
188 | static int modulo_n_insns; | |
189 | ||
190 | /* The current count of insns in the first iteration of the loop that have | |
191 | already been scheduled. */ | |
192 | static int modulo_insns_scheduled; | |
193 | ||
194 | /* The maximum uid of insns from the first iteration of the loop. */ | |
195 | static int modulo_iter0_max_uid; | |
196 | ||
197 | /* The number of times we should attempt to backtrack when modulo scheduling. | |
198 | Decreased each time we have to backtrack. */ | |
199 | static int modulo_backtracks_left; | |
200 | ||
201 | /* The stage in which the last insn from the original loop was | |
202 | scheduled. */ | |
203 | static int modulo_last_stage; | |
204 | ||
cc132865 | 205 | /* sched-verbose controls the amount of debugging output the |
409f8483 | 206 | scheduler prints. It is controlled by -fsched-verbose=N: |
4111f1c9 NB |
207 | N=0: no debugging output. |
208 | N=1: default value. | |
8c660648 JL |
209 | N=2: bb's probabilities, detailed ready list info, unit/insn info. |
210 | N=3: rtl at abort point, control-flow, regions info. | |
cc132865 | 211 | N=5: dependences info. */ |
b4ead7d4 | 212 | int sched_verbose = 0; |
8c660648 | 213 | |
4111f1c9 | 214 | /* Debugging file. All printouts are sent to dump. */ |
c62c2659 | 215 | FILE *sched_dump = 0; |
a88f02e7 | 216 | |
b8698a0f | 217 | /* This is a placeholder for the scheduler parameters common |
e855c69d AB |
218 | to all schedulers. */ |
219 | struct common_sched_info_def *common_sched_info; | |
f66d83e1 | 220 | |
e855c69d | 221 | #define INSN_TICK(INSN) (HID (INSN)->tick) |
26965010 BS |
222 | #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick) |
223 | #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate) | |
e855c69d | 224 | #define INTER_TICK(INSN) (HID (INSN)->inter_tick) |
26965010 BS |
225 | #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn) |
226 | #define SHADOW_P(INSN) (HID (INSN)->shadow_p) | |
e2724e63 | 227 | #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec) |
ffc1ded5 | 228 | /* Cached cost of the instruction. Use insn_sched_cost to get cost of the |
1a83e602 BS |
229 | insn. -1 here means that the field is not initialized. */ |
230 | #define INSN_COST(INSN) (HID (INSN)->cost) | |
63f54b1a MK |
231 | |
232 | /* If INSN_TICK of an instruction is equal to INVALID_TICK, | |
233 | then it should be recalculated from scratch. */ | |
234 | #define INVALID_TICK (-(max_insn_queue_index + 1)) | |
235 | /* The minimal value of the INSN_TICK of an instruction. */ | |
236 | #define MIN_TICK (-max_insn_queue_index) | |
8c660648 | 237 | |
63387a85 MK |
238 | /* Original order of insns in the ready list. |
239 | Used to keep order of normal insns while separating DEBUG_INSNs. */ | |
240 | #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order) | |
241 | ||
b75ae7f4 MK |
242 | /* The deciding reason for INSN's place in the ready list. */ |
243 | #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win) | |
244 | ||
8c660648 JL |
245 | /* List of important notes we must keep around. This is a pointer to the |
246 | last element in the list. */ | |
ce1ce33a | 247 | rtx_insn *note_list; |
8c660648 | 248 | |
496d7bb0 MK |
249 | static struct spec_info_def spec_info_var; |
250 | /* Description of the speculative part of the scheduling. | |
251 | If NULL - no speculation. */ | |
e855c69d | 252 | spec_info_t spec_info = NULL; |
496d7bb0 MK |
253 | |
254 | /* True, if recovery block was added during scheduling of current block. | |
255 | Used to determine, if we need to fix INSN_TICKs. */ | |
e2f6ff94 MK |
256 | static bool haifa_recovery_bb_recently_added_p; |
257 | ||
258 | /* True, if recovery block was added during this scheduling pass. | |
259 | Used to determine if we should have empty memory pools of dependencies | |
260 | after finishing current region. */ | |
261 | bool haifa_recovery_bb_ever_added_p; | |
496d7bb0 | 262 | |
917f1b7e | 263 | /* Counters of different types of speculative instructions. */ |
496d7bb0 MK |
264 | static int nr_begin_data, nr_be_in_data, nr_begin_control, nr_be_in_control; |
265 | ||
496d7bb0 | 266 | /* Array used in {unlink, restore}_bb_notes. */ |
9b517712 | 267 | static rtx_insn **bb_header = 0; |
496d7bb0 | 268 | |
496d7bb0 MK |
269 | /* Basic block after which recovery blocks will be created. */ |
270 | static basic_block before_recovery; | |
271 | ||
e855c69d AB |
272 | /* Basic block just before the EXIT_BLOCK and after recovery, if we have |
273 | created it. */ | |
274 | basic_block after_recovery; | |
275 | ||
276 | /* FALSE if we add bb to another region, so we don't need to initialize it. */ | |
277 | bool adding_bb_to_current_region_p = true; | |
278 | ||
8c660648 JL |
279 | /* Queues, etc. */ |
280 | ||
281 | /* An instruction is ready to be scheduled when all insns preceding it | |
282 | have already been scheduled. It is important to ensure that all | |
283 | insns which use its result will not be executed until its result | |
284 | has been computed. An insn is maintained in one of four structures: | |
285 | ||
286 | (P) the "Pending" set of insns which cannot be scheduled until | |
287 | their dependencies have been satisfied. | |
288 | (Q) the "Queued" set of insns that can be scheduled when sufficient | |
289 | time has passed. | |
290 | (R) the "Ready" list of unscheduled, uncommitted insns. | |
291 | (S) the "Scheduled" list of insns. | |
292 | ||
293 | Initially, all insns are either "Pending" or "Ready" depending on | |
294 | whether their dependencies are satisfied. | |
295 | ||
296 | Insns move from the "Ready" list to the "Scheduled" list as they | |
297 | are committed to the schedule. As this occurs, the insns in the | |
298 | "Pending" list have their dependencies satisfied and move to either | |
299 | the "Ready" list or the "Queued" set depending on whether | |
300 | sufficient time has passed to make them ready. As time passes, | |
fa0aee89 | 301 | insns move from the "Queued" set to the "Ready" list. |
8c660648 | 302 | |
b198261f MK |
303 | The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the |
304 | unscheduled insns, i.e., those that are ready, queued, and pending. | |
8c660648 JL |
305 | The "Queued" set (Q) is implemented by the variable `insn_queue'. |
306 | The "Ready" list (R) is implemented by the variables `ready' and | |
307 | `n_ready'. | |
308 | The "Scheduled" list (S) is the new insn chain built by this pass. | |
309 | ||
310 | The transition (R->S) is implemented in the scheduling loop in | |
311 | `schedule_block' when the best insn to schedule is chosen. | |
8c660648 JL |
312 | The transitions (P->R and P->Q) are implemented in `schedule_insn' as |
313 | insns move from the ready list to the scheduled list. | |
314 | The transition (Q->R) is implemented in 'queue_to_insn' as time | |
315 | passes or stalls are introduced. */ | |
316 | ||
317 | /* Implement a circular buffer to delay instructions until sufficient | |
fa0aee89 | 318 | time has passed. For the new pipeline description interface, |
63f54b1a | 319 | MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less |
fa0aee89 PB |
320 | than maximal time of instruction execution computed by genattr.c on |
321 | the base maximal time of functional unit reservations and getting a | |
322 | result. This is the longest time an insn may be queued. */ | |
fae15c93 | 323 | |
3dc99c19 | 324 | static rtx_insn_list **insn_queue; |
8c660648 JL |
325 | static int q_ptr = 0; |
326 | static int q_size = 0; | |
fa0aee89 PB |
327 | #define NEXT_Q(X) (((X)+1) & max_insn_queue_index) |
328 | #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index) | |
fae15c93 | 329 | |
63f54b1a MK |
330 | #define QUEUE_SCHEDULED (-3) |
331 | #define QUEUE_NOWHERE (-2) | |
332 | #define QUEUE_READY (-1) | |
333 | /* QUEUE_SCHEDULED - INSN is scheduled. | |
334 | QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in | |
335 | queue or ready list. | |
336 | QUEUE_READY - INSN is in ready list. | |
337 | N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */ | |
b8698a0f | 338 | |
e855c69d | 339 | #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index) |
63f54b1a | 340 | |
fae15c93 VM |
341 | /* The following variable value refers for all current and future |
342 | reservations of the processor units. */ | |
343 | state_t curr_state; | |
344 | ||
345 | /* The following variable value is size of memory representing all | |
fa0aee89 | 346 | current and future reservations of the processor units. */ |
e855c69d | 347 | size_t dfa_state_size; |
fae15c93 VM |
348 | |
349 | /* The following array is used to find the best insn from ready when | |
350 | the automaton pipeline interface is used. */ | |
4960a0cb | 351 | signed char *ready_try = NULL; |
8c660648 | 352 | |
e855c69d | 353 | /* The ready list. */ |
b5b8b0ac | 354 | struct ready_list ready = {NULL, 0, 0, 0, 0}; |
176f9a7b | 355 | |
e855c69d AB |
356 | /* The pointer to the ready list (to be removed). */ |
357 | static struct ready_list *readyp = &ready; | |
63f54b1a MK |
358 | |
359 | /* Scheduling clock. */ | |
360 | static int clock_var; | |
361 | ||
26965010 BS |
362 | /* Clock at which the previous instruction was issued. */ |
363 | static int last_clock_var; | |
364 | ||
365 | /* Set to true if, when queuing a shadow insn, we discover that it would be | |
366 | scheduled too late. */ | |
367 | static bool must_backtrack; | |
368 | ||
369 | /* The following variable value is number of essential insns issued on | |
370 | the current cycle. An insn is essential one if it changes the | |
371 | processors state. */ | |
372 | int cycle_issued_insns; | |
373 | ||
86014d07 BS |
374 | /* This records the actual schedule. It is built up during the main phase |
375 | of schedule_block, and afterwards used to reorder the insns in the RTL. */ | |
ce1ce33a | 376 | static vec<rtx_insn *> scheduled_insns; |
86014d07 | 377 | |
9678086d | 378 | static int may_trap_exp (const_rtx, int); |
15aab9c0 VM |
379 | |
380 | /* Nonzero iff the address is comprised from at most 1 register. */ | |
381 | #define CONST_BASED_ADDRESS_P(x) \ | |
f8cfc6aa | 382 | (REG_P (x) \ |
15aab9c0 VM |
383 | || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \ |
384 | || (GET_CODE (x) == LO_SUM)) \ | |
385 | && (CONSTANT_P (XEXP (x, 0)) \ | |
386 | || CONSTANT_P (XEXP (x, 1))))) | |
387 | ||
388 | /* Returns a class that insn with GET_DEST(insn)=x may belong to, | |
389 | as found by analyzing insn's expression. */ | |
390 | ||
e855c69d AB |
391 | \f |
392 | static int haifa_luid_for_non_insn (rtx x); | |
393 | ||
394 | /* Haifa version of sched_info hooks common to all headers. */ | |
b8698a0f | 395 | const struct common_sched_info_def haifa_common_sched_info = |
e855c69d AB |
396 | { |
397 | NULL, /* fix_recovery_cfg */ | |
398 | NULL, /* add_block */ | |
399 | NULL, /* estimate_number_of_insns */ | |
400 | haifa_luid_for_non_insn, /* luid_for_non_insn */ | |
401 | SCHED_PASS_UNKNOWN /* sched_pass_id */ | |
402 | }; | |
403 | ||
e855c69d | 404 | /* Mapping from instruction UID to its Logical UID. */ |
7de76362 | 405 | vec<int> sched_luids; |
e855c69d AB |
406 | |
407 | /* Next LUID to assign to an instruction. */ | |
408 | int sched_max_luid = 1; | |
409 | ||
410 | /* Haifa Instruction Data. */ | |
7de76362 | 411 | vec<haifa_insn_data_def> h_i_d; |
e855c69d AB |
412 | |
413 | void (* sched_init_only_bb) (basic_block, basic_block); | |
414 | ||
415 | /* Split block function. Different schedulers might use different functions | |
416 | to handle their internal data consistent. */ | |
417 | basic_block (* sched_split_block) (basic_block, rtx); | |
418 | ||
419 | /* Create empty basic block after the specified block. */ | |
420 | basic_block (* sched_create_empty_bb) (basic_block); | |
421 | ||
7a1aca9c RS |
422 | /* Return the number of cycles until INSN is expected to be ready. |
423 | Return zero if it already is. */ | |
424 | static int | |
f57aa6b0 | 425 | insn_delay (rtx_insn *insn) |
7a1aca9c RS |
426 | { |
427 | return MAX (INSN_TICK (insn) - clock_var, 0); | |
428 | } | |
429 | ||
15aab9c0 | 430 | static int |
9678086d | 431 | may_trap_exp (const_rtx x, int is_store) |
15aab9c0 VM |
432 | { |
433 | enum rtx_code code; | |
434 | ||
435 | if (x == 0) | |
436 | return TRAP_FREE; | |
437 | code = GET_CODE (x); | |
438 | if (is_store) | |
439 | { | |
440 | if (code == MEM && may_trap_p (x)) | |
441 | return TRAP_RISKY; | |
442 | else | |
443 | return TRAP_FREE; | |
444 | } | |
445 | if (code == MEM) | |
446 | { | |
447 | /* The insn uses memory: a volatile load. */ | |
448 | if (MEM_VOLATILE_P (x)) | |
449 | return IRISKY; | |
450 | /* An exception-free load. */ | |
451 | if (!may_trap_p (x)) | |
452 | return IFREE; | |
453 | /* A load with 1 base register, to be further checked. */ | |
454 | if (CONST_BASED_ADDRESS_P (XEXP (x, 0))) | |
455 | return PFREE_CANDIDATE; | |
456 | /* No info on the load, to be further checked. */ | |
457 | return PRISKY_CANDIDATE; | |
458 | } | |
459 | else | |
460 | { | |
461 | const char *fmt; | |
462 | int i, insn_class = TRAP_FREE; | |
463 | ||
464 | /* Neither store nor load, check if it may cause a trap. */ | |
465 | if (may_trap_p (x)) | |
466 | return TRAP_RISKY; | |
467 | /* Recursive step: walk the insn... */ | |
468 | fmt = GET_RTX_FORMAT (code); | |
469 | for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--) | |
470 | { | |
471 | if (fmt[i] == 'e') | |
472 | { | |
473 | int tmp_class = may_trap_exp (XEXP (x, i), is_store); | |
474 | insn_class = WORST_CLASS (insn_class, tmp_class); | |
475 | } | |
476 | else if (fmt[i] == 'E') | |
477 | { | |
478 | int j; | |
479 | for (j = 0; j < XVECLEN (x, i); j++) | |
480 | { | |
481 | int tmp_class = may_trap_exp (XVECEXP (x, i, j), is_store); | |
482 | insn_class = WORST_CLASS (insn_class, tmp_class); | |
483 | if (insn_class == TRAP_RISKY || insn_class == IRISKY) | |
484 | break; | |
485 | } | |
486 | } | |
487 | if (insn_class == TRAP_RISKY || insn_class == IRISKY) | |
488 | break; | |
489 | } | |
490 | return insn_class; | |
491 | } | |
492 | } | |
493 | ||
ac4a7e21 AM |
494 | /* Classifies rtx X of an insn for the purpose of verifying that X can be |
495 | executed speculatively (and consequently the insn can be moved | |
496 | speculatively), by examining X, returning: | |
15aab9c0 VM |
497 | TRAP_RISKY: store, or risky non-load insn (e.g. division by variable). |
498 | TRAP_FREE: non-load insn. | |
a98ebe2e | 499 | IFREE: load from a globally safe location. |
15aab9c0 VM |
500 | IRISKY: volatile load. |
501 | PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for | |
502 | being either PFREE or PRISKY. */ | |
503 | ||
ac4a7e21 AM |
504 | static int |
505 | haifa_classify_rtx (const_rtx x) | |
15aab9c0 | 506 | { |
15aab9c0 VM |
507 | int tmp_class = TRAP_FREE; |
508 | int insn_class = TRAP_FREE; | |
509 | enum rtx_code code; | |
510 | ||
ac4a7e21 | 511 | if (GET_CODE (x) == PARALLEL) |
15aab9c0 | 512 | { |
ac4a7e21 | 513 | int i, len = XVECLEN (x, 0); |
15aab9c0 VM |
514 | |
515 | for (i = len - 1; i >= 0; i--) | |
516 | { | |
ac4a7e21 | 517 | tmp_class = haifa_classify_rtx (XVECEXP (x, 0, i)); |
15aab9c0 VM |
518 | insn_class = WORST_CLASS (insn_class, tmp_class); |
519 | if (insn_class == TRAP_RISKY || insn_class == IRISKY) | |
520 | break; | |
521 | } | |
522 | } | |
523 | else | |
524 | { | |
ac4a7e21 | 525 | code = GET_CODE (x); |
15aab9c0 VM |
526 | switch (code) |
527 | { | |
528 | case CLOBBER: | |
529 | /* Test if it is a 'store'. */ | |
ac4a7e21 | 530 | tmp_class = may_trap_exp (XEXP (x, 0), 1); |
15aab9c0 VM |
531 | break; |
532 | case SET: | |
533 | /* Test if it is a store. */ | |
ac4a7e21 | 534 | tmp_class = may_trap_exp (SET_DEST (x), 1); |
15aab9c0 VM |
535 | if (tmp_class == TRAP_RISKY) |
536 | break; | |
537 | /* Test if it is a load. */ | |
538 | tmp_class = | |
539 | WORST_CLASS (tmp_class, | |
ac4a7e21 | 540 | may_trap_exp (SET_SRC (x), 0)); |
15aab9c0 VM |
541 | break; |
542 | case COND_EXEC: | |
ac4a7e21 AM |
543 | tmp_class = haifa_classify_rtx (COND_EXEC_CODE (x)); |
544 | if (tmp_class == TRAP_RISKY) | |
545 | break; | |
546 | tmp_class = WORST_CLASS (tmp_class, | |
547 | may_trap_exp (COND_EXEC_TEST (x), 0)); | |
548 | break; | |
15aab9c0 VM |
549 | case TRAP_IF: |
550 | tmp_class = TRAP_RISKY; | |
551 | break; | |
552 | default:; | |
553 | } | |
554 | insn_class = tmp_class; | |
555 | } | |
556 | ||
557 | return insn_class; | |
558 | } | |
559 | ||
ac4a7e21 AM |
560 | int |
561 | haifa_classify_insn (const_rtx insn) | |
562 | { | |
563 | return haifa_classify_rtx (PATTERN (insn)); | |
564 | } | |
06d7e8e7 BS |
565 | \f |
566 | /* After the scheduler initialization function has been called, this function | |
567 | can be called to enable modulo scheduling. II is the initiation interval | |
568 | we should use, it affects the delays for delay_pairs that were recorded as | |
569 | separated by a given number of stages. | |
570 | ||
571 | MAX_STAGES provides us with a limit | |
572 | after which we give up scheduling; the caller must have unrolled at least | |
573 | as many copies of the loop body and recorded delay_pairs for them. | |
574 | ||
575 | INSNS is the number of real (non-debug) insns in one iteration of | |
576 | the loop. MAX_UID can be used to test whether an insn belongs to | |
577 | the first iteration of the loop; all of them have a uid lower than | |
578 | MAX_UID. */ | |
579 | void | |
580 | set_modulo_params (int ii, int max_stages, int insns, int max_uid) | |
581 | { | |
582 | modulo_ii = ii; | |
583 | modulo_max_stages = max_stages; | |
584 | modulo_n_insns = insns; | |
585 | modulo_iter0_max_uid = max_uid; | |
586 | modulo_backtracks_left = PARAM_VALUE (PARAM_MAX_MODULO_BACKTRACK_ATTEMPTS); | |
587 | } | |
ac4a7e21 | 588 | |
26965010 BS |
589 | /* A structure to record a pair of insns where the first one is a real |
590 | insn that has delay slots, and the second is its delayed shadow. | |
591 | I1 is scheduled normally and will emit an assembly instruction, | |
592 | while I2 describes the side effect that takes place at the | |
593 | transition between cycles CYCLES and (CYCLES + 1) after I1. */ | |
594 | struct delay_pair | |
595 | { | |
596 | struct delay_pair *next_same_i1; | |
ce1ce33a | 597 | rtx_insn *i1, *i2; |
26965010 | 598 | int cycles; |
06d7e8e7 BS |
599 | /* When doing modulo scheduling, we a delay_pair can also be used to |
600 | show that I1 and I2 are the same insn in a different stage. If that | |
601 | is the case, STAGES will be nonzero. */ | |
602 | int stages; | |
26965010 BS |
603 | }; |
604 | ||
4a8fb1a1 LC |
605 | /* Helpers for delay hashing. */ |
606 | ||
8d67ee55 | 607 | struct delay_i1_hasher : nofree_ptr_hash <delay_pair> |
4a8fb1a1 | 608 | { |
67f58944 TS |
609 | typedef void *compare_type; |
610 | static inline hashval_t hash (const delay_pair *); | |
611 | static inline bool equal (const delay_pair *, const void *); | |
4a8fb1a1 LC |
612 | }; |
613 | ||
614 | /* Returns a hash value for X, based on hashing just I1. */ | |
615 | ||
616 | inline hashval_t | |
67f58944 | 617 | delay_i1_hasher::hash (const delay_pair *x) |
4a8fb1a1 LC |
618 | { |
619 | return htab_hash_pointer (x->i1); | |
620 | } | |
621 | ||
622 | /* Return true if I1 of pair X is the same as that of pair Y. */ | |
623 | ||
624 | inline bool | |
67f58944 | 625 | delay_i1_hasher::equal (const delay_pair *x, const void *y) |
4a8fb1a1 LC |
626 | { |
627 | return x->i1 == y; | |
628 | } | |
629 | ||
95fbe13e | 630 | struct delay_i2_hasher : free_ptr_hash <delay_pair> |
4a8fb1a1 | 631 | { |
67f58944 TS |
632 | typedef void *compare_type; |
633 | static inline hashval_t hash (const delay_pair *); | |
634 | static inline bool equal (const delay_pair *, const void *); | |
4a8fb1a1 LC |
635 | }; |
636 | ||
637 | /* Returns a hash value for X, based on hashing just I2. */ | |
638 | ||
639 | inline hashval_t | |
67f58944 | 640 | delay_i2_hasher::hash (const delay_pair *x) |
4a8fb1a1 LC |
641 | { |
642 | return htab_hash_pointer (x->i2); | |
643 | } | |
644 | ||
645 | /* Return true if I2 of pair X is the same as that of pair Y. */ | |
646 | ||
647 | inline bool | |
67f58944 | 648 | delay_i2_hasher::equal (const delay_pair *x, const void *y) |
4a8fb1a1 LC |
649 | { |
650 | return x->i2 == y; | |
651 | } | |
652 | ||
26965010 BS |
653 | /* Two hash tables to record delay_pairs, one indexed by I1 and the other |
654 | indexed by I2. */ | |
c203e8a7 TS |
655 | static hash_table<delay_i1_hasher> *delay_htab; |
656 | static hash_table<delay_i2_hasher> *delay_htab_i2; | |
26965010 | 657 | |
06d7e8e7 BS |
658 | /* Called through htab_traverse. Walk the hashtable using I2 as |
659 | index, and delete all elements involving an UID higher than | |
660 | that pointed to by *DATA. */ | |
4a8fb1a1 LC |
661 | int |
662 | haifa_htab_i2_traverse (delay_pair **slot, int *data) | |
06d7e8e7 | 663 | { |
4a8fb1a1 LC |
664 | int maxuid = *data; |
665 | struct delay_pair *p = *slot; | |
06d7e8e7 BS |
666 | if (INSN_UID (p->i2) >= maxuid || INSN_UID (p->i1) >= maxuid) |
667 | { | |
c203e8a7 | 668 | delay_htab_i2->clear_slot (slot); |
06d7e8e7 BS |
669 | } |
670 | return 1; | |
671 | } | |
672 | ||
673 | /* Called through htab_traverse. Walk the hashtable using I2 as | |
674 | index, and delete all elements involving an UID higher than | |
675 | that pointed to by *DATA. */ | |
4a8fb1a1 LC |
676 | int |
677 | haifa_htab_i1_traverse (delay_pair **pslot, int *data) | |
06d7e8e7 | 678 | { |
4a8fb1a1 | 679 | int maxuid = *data; |
06d7e8e7 BS |
680 | struct delay_pair *p, *first, **pprev; |
681 | ||
682 | if (INSN_UID ((*pslot)->i1) >= maxuid) | |
683 | { | |
c203e8a7 | 684 | delay_htab->clear_slot (pslot); |
06d7e8e7 BS |
685 | return 1; |
686 | } | |
687 | pprev = &first; | |
688 | for (p = *pslot; p; p = p->next_same_i1) | |
689 | { | |
690 | if (INSN_UID (p->i2) < maxuid) | |
691 | { | |
692 | *pprev = p; | |
693 | pprev = &p->next_same_i1; | |
694 | } | |
695 | } | |
696 | *pprev = NULL; | |
697 | if (first == NULL) | |
c203e8a7 | 698 | delay_htab->clear_slot (pslot); |
06d7e8e7 BS |
699 | else |
700 | *pslot = first; | |
701 | return 1; | |
702 | } | |
703 | ||
704 | /* Discard all delay pairs which involve an insn with an UID higher | |
705 | than MAX_UID. */ | |
706 | void | |
707 | discard_delay_pairs_above (int max_uid) | |
708 | { | |
c203e8a7 TS |
709 | delay_htab->traverse <int *, haifa_htab_i1_traverse> (&max_uid); |
710 | delay_htab_i2->traverse <int *, haifa_htab_i2_traverse> (&max_uid); | |
26965010 BS |
711 | } |
712 | ||
06d7e8e7 BS |
713 | /* This function can be called by a port just before it starts the final |
714 | scheduling pass. It records the fact that an instruction with delay | |
715 | slots has been split into two insns, I1 and I2. The first one will be | |
716 | scheduled normally and initiates the operation. The second one is a | |
717 | shadow which must follow a specific number of cycles after I1; its only | |
718 | purpose is to show the side effect that occurs at that cycle in the RTL. | |
719 | If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN, | |
720 | while I2 retains the original insn type. | |
721 | ||
722 | There are two ways in which the number of cycles can be specified, | |
723 | involving the CYCLES and STAGES arguments to this function. If STAGES | |
724 | is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor | |
725 | which is multiplied by MODULO_II to give the number of cycles. This is | |
726 | only useful if the caller also calls set_modulo_params to enable modulo | |
727 | scheduling. */ | |
26965010 BS |
728 | |
729 | void | |
ce1ce33a | 730 | record_delay_slot_pair (rtx_insn *i1, rtx_insn *i2, int cycles, int stages) |
26965010 BS |
731 | { |
732 | struct delay_pair *p = XNEW (struct delay_pair); | |
733 | struct delay_pair **slot; | |
734 | ||
735 | p->i1 = i1; | |
736 | p->i2 = i2; | |
737 | p->cycles = cycles; | |
06d7e8e7 | 738 | p->stages = stages; |
26965010 | 739 | |
c203e8a7 | 740 | if (!delay_htab) |
26965010 | 741 | { |
c203e8a7 TS |
742 | delay_htab = new hash_table<delay_i1_hasher> (10); |
743 | delay_htab_i2 = new hash_table<delay_i2_hasher> (10); | |
26965010 | 744 | } |
c203e8a7 | 745 | slot = delay_htab->find_slot_with_hash (i1, htab_hash_pointer (i1), INSERT); |
26965010 BS |
746 | p->next_same_i1 = *slot; |
747 | *slot = p; | |
c203e8a7 | 748 | slot = delay_htab_i2->find_slot (p, INSERT); |
26965010 BS |
749 | *slot = p; |
750 | } | |
751 | ||
e2724e63 BS |
752 | /* Examine the delay pair hashtable to see if INSN is a shadow for another, |
753 | and return the other insn if so. Return NULL otherwise. */ | |
f57aa6b0 DM |
754 | rtx_insn * |
755 | real_insn_for_shadow (rtx_insn *insn) | |
e2724e63 BS |
756 | { |
757 | struct delay_pair *pair; | |
758 | ||
c203e8a7 | 759 | if (!delay_htab) |
f57aa6b0 | 760 | return NULL; |
e2724e63 | 761 | |
c203e8a7 | 762 | pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn)); |
e2724e63 | 763 | if (!pair || pair->stages > 0) |
f57aa6b0 | 764 | return NULL; |
e2724e63 BS |
765 | return pair->i1; |
766 | } | |
767 | ||
26965010 BS |
768 | /* For a pair P of insns, return the fixed distance in cycles from the first |
769 | insn after which the second must be scheduled. */ | |
770 | static int | |
771 | pair_delay (struct delay_pair *p) | |
772 | { | |
06d7e8e7 BS |
773 | if (p->stages == 0) |
774 | return p->cycles; | |
775 | else | |
776 | return p->stages * modulo_ii; | |
26965010 BS |
777 | } |
778 | ||
779 | /* Given an insn INSN, add a dependence on its delayed shadow if it | |
780 | has one. Also try to find situations where shadows depend on each other | |
781 | and add dependencies to the real insns to limit the amount of backtracking | |
782 | needed. */ | |
783 | void | |
ce1ce33a | 784 | add_delay_dependencies (rtx_insn *insn) |
26965010 BS |
785 | { |
786 | struct delay_pair *pair; | |
787 | sd_iterator_def sd_it; | |
788 | dep_t dep; | |
789 | ||
c203e8a7 | 790 | if (!delay_htab) |
26965010 BS |
791 | return; |
792 | ||
c203e8a7 | 793 | pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn)); |
26965010 BS |
794 | if (!pair) |
795 | return; | |
796 | add_dependence (insn, pair->i1, REG_DEP_ANTI); | |
06d7e8e7 BS |
797 | if (pair->stages) |
798 | return; | |
26965010 BS |
799 | |
800 | FOR_EACH_DEP (pair->i2, SD_LIST_BACK, sd_it, dep) | |
801 | { | |
9b517712 | 802 | rtx_insn *pro = DEP_PRO (dep); |
26965010 | 803 | struct delay_pair *other_pair |
c203e8a7 | 804 | = delay_htab_i2->find_with_hash (pro, htab_hash_pointer (pro)); |
06d7e8e7 | 805 | if (!other_pair || other_pair->stages) |
26965010 BS |
806 | continue; |
807 | if (pair_delay (other_pair) >= pair_delay (pair)) | |
808 | { | |
809 | if (sched_verbose >= 4) | |
810 | { | |
811 | fprintf (sched_dump, ";;\tadding dependence %d <- %d\n", | |
812 | INSN_UID (other_pair->i1), | |
813 | INSN_UID (pair->i1)); | |
814 | fprintf (sched_dump, ";;\tpair1 %d <- %d, cost %d\n", | |
815 | INSN_UID (pair->i1), | |
816 | INSN_UID (pair->i2), | |
817 | pair_delay (pair)); | |
818 | fprintf (sched_dump, ";;\tpair2 %d <- %d, cost %d\n", | |
819 | INSN_UID (other_pair->i1), | |
820 | INSN_UID (other_pair->i2), | |
821 | pair_delay (other_pair)); | |
822 | } | |
823 | add_dependence (pair->i1, other_pair->i1, REG_DEP_ANTI); | |
824 | } | |
825 | } | |
826 | } | |
827 | \f | |
8c660648 | 828 | /* Forward declarations. */ |
fae15c93 | 829 | |
0e0ce50d | 830 | static int priority (rtx_insn *); |
340c7904 | 831 | static int autopref_rank_for_schedule (const rtx_insn *, const rtx_insn *); |
1d088dee | 832 | static int rank_for_schedule (const void *, const void *); |
ce1ce33a DM |
833 | static void swap_sort (rtx_insn **, int); |
834 | static void queue_insn (rtx_insn *, int, const char *); | |
835 | static int schedule_insn (rtx_insn *); | |
f57aa6b0 | 836 | static void adjust_priority (rtx_insn *); |
1d088dee | 837 | static void advance_one_cycle (void); |
e855c69d AB |
838 | static void extend_h_i_d (void); |
839 | ||
8c660648 | 840 | |
8c660648 JL |
841 | /* Notes handling mechanism: |
842 | ========================= | |
843 | Generally, NOTES are saved before scheduling and restored after scheduling. | |
07c02828 | 844 | The scheduler distinguishes between two types of notes: |
8c660648 | 845 | |
6039a0c7 | 846 | (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes: |
8c660648 JL |
847 | Before scheduling a region, a pointer to the note is added to the insn |
848 | that follows or precedes it. (This happens as part of the data dependence | |
849 | computation). After scheduling an insn, the pointer contained in it is | |
850 | used for regenerating the corresponding note (in reemit_notes). | |
851 | ||
6039a0c7 | 852 | (2) All other notes (e.g. INSN_DELETED): Before scheduling a block, |
8c660648 JL |
853 | these notes are put in a list (in rm_other_notes() and |
854 | unlink_other_notes ()). After scheduling the block, these notes are | |
855 | inserted at the beginning of the block (in schedule_block()). */ | |
856 | ||
ce1ce33a DM |
857 | static void ready_add (struct ready_list *, rtx_insn *, bool); |
858 | static rtx_insn *ready_remove_first (struct ready_list *); | |
859 | static rtx_insn *ready_remove_first_dispatch (struct ready_list *ready); | |
3fe41456 | 860 | |
1d088dee | 861 | static void queue_to_ready (struct ready_list *); |
569fa502 | 862 | static int early_queue_to_ready (state_t, struct ready_list *); |
176f9a7b | 863 | |
fae15c93 | 864 | /* The following functions are used to implement multi-pass scheduling |
fa0aee89 | 865 | on the first cycle. */ |
ce1ce33a | 866 | static rtx_insn *ready_remove (struct ready_list *, int); |
90831096 | 867 | static void ready_remove_insn (rtx_insn *); |
fae15c93 | 868 | |
f57aa6b0 | 869 | static void fix_inter_tick (rtx_insn *, rtx_insn *); |
ce1ce33a DM |
870 | static int fix_tick_ready (rtx_insn *); |
871 | static void change_queue_index (rtx_insn *, int); | |
63f54b1a | 872 | |
496d7bb0 MK |
873 | /* The following functions are used to implement scheduling of data/control |
874 | speculative instructions. */ | |
875 | ||
876 | static void extend_h_i_d (void); | |
f57aa6b0 DM |
877 | static void init_h_i_d (rtx_insn *); |
878 | static int haifa_speculate_insn (rtx_insn *, ds_t, rtx *); | |
ce1ce33a | 879 | static void generate_recovery_code (rtx_insn *); |
90831096 | 880 | static void process_insn_forw_deps_be_in_spec (rtx_insn *, rtx_insn *, ds_t); |
ce1ce33a | 881 | static void begin_speculative_block (rtx_insn *); |
6144a836 | 882 | static void add_to_speculative_block (rtx_insn *); |
e855c69d | 883 | static void init_before_recovery (basic_block *); |
ce1ce33a | 884 | static void create_check_block_twin (rtx_insn *, bool); |
496d7bb0 | 885 | static void fix_recovery_deps (basic_block); |
f57aa6b0 DM |
886 | static bool haifa_change_pattern (rtx_insn *, rtx); |
887 | static void dump_new_block_header (int, basic_block, rtx_insn *, rtx_insn *); | |
496d7bb0 | 888 | static void restore_bb_notes (basic_block); |
f57aa6b0 DM |
889 | static void fix_jump_move (rtx_insn *); |
890 | static void move_block_after_check (rtx_insn *); | |
9771b263 | 891 | static void move_succs (vec<edge, va_gc> **, basic_block); |
ce1ce33a | 892 | static void sched_remove_insn (rtx_insn *); |
6144a836 | 893 | static void clear_priorities (rtx_insn *, rtx_vec_t *); |
916fa4f0 | 894 | static void calc_priorities (rtx_vec_t); |
0e0ce50d | 895 | static void add_jump_dependencies (rtx_insn *, rtx_insn *); |
496d7bb0 | 896 | |
8c660648 JL |
897 | #endif /* INSN_SCHEDULING */ |
898 | \f | |
1708fd40 | 899 | /* Point to state used for the current scheduling pass. */ |
e855c69d | 900 | struct haifa_sched_info *current_sched_info; |
8c660648 JL |
901 | \f |
902 | #ifndef INSN_SCHEDULING | |
903 | void | |
10d22567 | 904 | schedule_insns (void) |
8c660648 JL |
905 | { |
906 | } | |
907 | #else | |
cbb13457 | 908 | |
ce18efcb VM |
909 | /* Do register pressure sensitive insn scheduling if the flag is set |
910 | up. */ | |
60867e8c | 911 | enum sched_pressure_algorithm sched_pressure; |
ce18efcb | 912 | |
1756cb66 | 913 | /* Map regno -> its pressure class. The map defined only when |
7a1aca9c | 914 | SCHED_PRESSURE != SCHED_PRESSURE_NONE. */ |
1756cb66 | 915 | enum reg_class *sched_regno_pressure_class; |
ce18efcb | 916 | |
1756cb66 | 917 | /* The current register pressure. Only elements corresponding pressure |
ce18efcb VM |
918 | classes are defined. */ |
919 | static int curr_reg_pressure[N_REG_CLASSES]; | |
920 | ||
921 | /* Saved value of the previous array. */ | |
922 | static int saved_reg_pressure[N_REG_CLASSES]; | |
923 | ||
924 | /* Register living at given scheduling point. */ | |
925 | static bitmap curr_reg_live; | |
926 | ||
927 | /* Saved value of the previous array. */ | |
928 | static bitmap saved_reg_live; | |
929 | ||
930 | /* Registers mentioned in the current region. */ | |
931 | static bitmap region_ref_regs; | |
932 | ||
f380f608 RS |
933 | /* Temporary bitmap used for SCHED_PRESSURE_MODEL. */ |
934 | static bitmap tmp_bitmap; | |
935 | ||
4dd9ac6c MK |
936 | /* Effective number of available registers of a given class (see comment |
937 | in sched_pressure_start_bb). */ | |
938 | static int sched_class_regs_num[N_REG_CLASSES]; | |
6d5c37d0 | 939 | /* Number of call_saved_regs and fixed_regs. Helpers for calculating of |
4dd9ac6c | 940 | sched_class_regs_num. */ |
6d5c37d0 PH |
941 | static int call_saved_regs_num[N_REG_CLASSES]; |
942 | static int fixed_regs_num[N_REG_CLASSES]; | |
4dd9ac6c | 943 | |
ce18efcb VM |
944 | /* Initiate register pressure relative info for scheduling the current |
945 | region. Currently it is only clearing register mentioned in the | |
946 | current region. */ | |
947 | void | |
948 | sched_init_region_reg_pressure_info (void) | |
949 | { | |
950 | bitmap_clear (region_ref_regs); | |
951 | } | |
952 | ||
7a1aca9c RS |
953 | /* PRESSURE[CL] describes the pressure on register class CL. Update it |
954 | for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO. | |
955 | LIVE tracks the set of live registers; if it is null, assume that | |
956 | every birth or death is genuine. */ | |
957 | static inline void | |
958 | mark_regno_birth_or_death (bitmap live, int *pressure, int regno, bool birth_p) | |
ce18efcb | 959 | { |
1756cb66 | 960 | enum reg_class pressure_class; |
ce18efcb | 961 | |
1756cb66 | 962 | pressure_class = sched_regno_pressure_class[regno]; |
ce18efcb VM |
963 | if (regno >= FIRST_PSEUDO_REGISTER) |
964 | { | |
1756cb66 | 965 | if (pressure_class != NO_REGS) |
ce18efcb VM |
966 | { |
967 | if (birth_p) | |
968 | { | |
7a1aca9c RS |
969 | if (!live || bitmap_set_bit (live, regno)) |
970 | pressure[pressure_class] | |
971 | += (ira_reg_class_max_nregs | |
972 | [pressure_class][PSEUDO_REGNO_MODE (regno)]); | |
ce18efcb VM |
973 | } |
974 | else | |
975 | { | |
7a1aca9c RS |
976 | if (!live || bitmap_clear_bit (live, regno)) |
977 | pressure[pressure_class] | |
978 | -= (ira_reg_class_max_nregs | |
979 | [pressure_class][PSEUDO_REGNO_MODE (regno)]); | |
ce18efcb VM |
980 | } |
981 | } | |
982 | } | |
1756cb66 | 983 | else if (pressure_class != NO_REGS |
ce18efcb VM |
984 | && ! TEST_HARD_REG_BIT (ira_no_alloc_regs, regno)) |
985 | { | |
986 | if (birth_p) | |
987 | { | |
7a1aca9c RS |
988 | if (!live || bitmap_set_bit (live, regno)) |
989 | pressure[pressure_class]++; | |
ce18efcb VM |
990 | } |
991 | else | |
992 | { | |
7a1aca9c RS |
993 | if (!live || bitmap_clear_bit (live, regno)) |
994 | pressure[pressure_class]--; | |
ce18efcb VM |
995 | } |
996 | } | |
997 | } | |
998 | ||
999 | /* Initiate current register pressure related info from living | |
1000 | registers given by LIVE. */ | |
1001 | static void | |
1002 | initiate_reg_pressure_info (bitmap live) | |
1003 | { | |
1004 | int i; | |
1005 | unsigned int j; | |
1006 | bitmap_iterator bi; | |
1007 | ||
1756cb66 VM |
1008 | for (i = 0; i < ira_pressure_classes_num; i++) |
1009 | curr_reg_pressure[ira_pressure_classes[i]] = 0; | |
ce18efcb VM |
1010 | bitmap_clear (curr_reg_live); |
1011 | EXECUTE_IF_SET_IN_BITMAP (live, 0, j, bi) | |
7a1aca9c RS |
1012 | if (sched_pressure == SCHED_PRESSURE_MODEL |
1013 | || current_nr_blocks == 1 | |
1014 | || bitmap_bit_p (region_ref_regs, j)) | |
1015 | mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, j, true); | |
ce18efcb VM |
1016 | } |
1017 | ||
1018 | /* Mark registers in X as mentioned in the current region. */ | |
1019 | static void | |
1020 | setup_ref_regs (rtx x) | |
1021 | { | |
07a737f3 | 1022 | int i, j; |
ce18efcb VM |
1023 | const RTX_CODE code = GET_CODE (x); |
1024 | const char *fmt; | |
1025 | ||
1026 | if (REG_P (x)) | |
1027 | { | |
07a737f3 | 1028 | bitmap_set_range (region_ref_regs, REGNO (x), REG_NREGS (x)); |
ce18efcb VM |
1029 | return; |
1030 | } | |
1031 | fmt = GET_RTX_FORMAT (code); | |
1032 | for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--) | |
1033 | if (fmt[i] == 'e') | |
1034 | setup_ref_regs (XEXP (x, i)); | |
1035 | else if (fmt[i] == 'E') | |
1036 | { | |
1037 | for (j = 0; j < XVECLEN (x, i); j++) | |
1038 | setup_ref_regs (XVECEXP (x, i, j)); | |
1039 | } | |
1040 | } | |
1041 | ||
1042 | /* Initiate current register pressure related info at the start of | |
1043 | basic block BB. */ | |
1044 | static void | |
1045 | initiate_bb_reg_pressure_info (basic_block bb) | |
1046 | { | |
6ab31ade | 1047 | unsigned int i ATTRIBUTE_UNUSED; |
dc01c3d1 | 1048 | rtx_insn *insn; |
ce18efcb VM |
1049 | |
1050 | if (current_nr_blocks > 1) | |
1051 | FOR_BB_INSNS (bb, insn) | |
69b45d71 | 1052 | if (NONDEBUG_INSN_P (insn)) |
ce18efcb VM |
1053 | setup_ref_regs (PATTERN (insn)); |
1054 | initiate_reg_pressure_info (df_get_live_in (bb)); | |
ce18efcb VM |
1055 | if (bb_has_eh_pred (bb)) |
1056 | for (i = 0; ; ++i) | |
1057 | { | |
1058 | unsigned int regno = EH_RETURN_DATA_REGNO (i); | |
b8698a0f | 1059 | |
ce18efcb VM |
1060 | if (regno == INVALID_REGNUM) |
1061 | break; | |
1062 | if (! bitmap_bit_p (df_get_live_in (bb), regno)) | |
7a1aca9c RS |
1063 | mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, |
1064 | regno, true); | |
ce18efcb | 1065 | } |
ce18efcb VM |
1066 | } |
1067 | ||
1068 | /* Save current register pressure related info. */ | |
1069 | static void | |
1070 | save_reg_pressure (void) | |
1071 | { | |
1072 | int i; | |
b8698a0f | 1073 | |
1756cb66 VM |
1074 | for (i = 0; i < ira_pressure_classes_num; i++) |
1075 | saved_reg_pressure[ira_pressure_classes[i]] | |
1076 | = curr_reg_pressure[ira_pressure_classes[i]]; | |
ce18efcb VM |
1077 | bitmap_copy (saved_reg_live, curr_reg_live); |
1078 | } | |
1079 | ||
1080 | /* Restore saved register pressure related info. */ | |
1081 | static void | |
1082 | restore_reg_pressure (void) | |
1083 | { | |
1084 | int i; | |
b8698a0f | 1085 | |
1756cb66 VM |
1086 | for (i = 0; i < ira_pressure_classes_num; i++) |
1087 | curr_reg_pressure[ira_pressure_classes[i]] | |
1088 | = saved_reg_pressure[ira_pressure_classes[i]]; | |
ce18efcb VM |
1089 | bitmap_copy (curr_reg_live, saved_reg_live); |
1090 | } | |
1091 | ||
1092 | /* Return TRUE if the register is dying after its USE. */ | |
1093 | static bool | |
1094 | dying_use_p (struct reg_use_data *use) | |
1095 | { | |
1096 | struct reg_use_data *next; | |
1097 | ||
1098 | for (next = use->next_regno_use; next != use; next = next->next_regno_use) | |
bb956f3e AO |
1099 | if (NONDEBUG_INSN_P (next->insn) |
1100 | && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED) | |
ce18efcb VM |
1101 | return false; |
1102 | return true; | |
1103 | } | |
1104 | ||
1105 | /* Print info about the current register pressure and its excess for | |
1756cb66 | 1106 | each pressure class. */ |
ce18efcb VM |
1107 | static void |
1108 | print_curr_reg_pressure (void) | |
1109 | { | |
1110 | int i; | |
1111 | enum reg_class cl; | |
1112 | ||
1113 | fprintf (sched_dump, ";;\t"); | |
1756cb66 | 1114 | for (i = 0; i < ira_pressure_classes_num; i++) |
ce18efcb | 1115 | { |
1756cb66 | 1116 | cl = ira_pressure_classes[i]; |
ce18efcb VM |
1117 | gcc_assert (curr_reg_pressure[cl] >= 0); |
1118 | fprintf (sched_dump, " %s:%d(%d)", reg_class_names[cl], | |
1119 | curr_reg_pressure[cl], | |
4dd9ac6c | 1120 | curr_reg_pressure[cl] - sched_class_regs_num[cl]); |
ce18efcb VM |
1121 | } |
1122 | fprintf (sched_dump, "\n"); | |
1123 | } | |
e2724e63 BS |
1124 | \f |
1125 | /* Determine if INSN has a condition that is clobbered if a register | |
1126 | in SET_REGS is modified. */ | |
1127 | static bool | |
ce1ce33a | 1128 | cond_clobbered_p (rtx_insn *insn, HARD_REG_SET set_regs) |
e2724e63 BS |
1129 | { |
1130 | rtx pat = PATTERN (insn); | |
1131 | gcc_assert (GET_CODE (pat) == COND_EXEC); | |
1132 | if (TEST_HARD_REG_BIT (set_regs, REGNO (XEXP (COND_EXEC_TEST (pat), 0)))) | |
1133 | { | |
1134 | sd_iterator_def sd_it; | |
1135 | dep_t dep; | |
1136 | haifa_change_pattern (insn, ORIG_PAT (insn)); | |
1137 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) | |
1138 | DEP_STATUS (dep) &= ~DEP_CANCELLED; | |
1139 | TODO_SPEC (insn) = HARD_DEP; | |
1140 | if (sched_verbose >= 2) | |
1141 | fprintf (sched_dump, | |
1142 | ";;\t\tdequeue insn %s because of clobbered condition\n", | |
1143 | (*current_sched_info->print_insn) (insn, 0)); | |
1144 | return true; | |
1145 | } | |
1146 | ||
1147 | return false; | |
1148 | } | |
1149 | ||
1a83e602 BS |
1150 | /* This function should be called after modifying the pattern of INSN, |
1151 | to update scheduler data structures as needed. */ | |
1152 | static void | |
f57aa6b0 | 1153 | update_insn_after_change (rtx_insn *insn) |
1a83e602 BS |
1154 | { |
1155 | sd_iterator_def sd_it; | |
1156 | dep_t dep; | |
1157 | ||
1158 | dfa_clear_single_insn_cache (insn); | |
1159 | ||
1160 | sd_it = sd_iterator_start (insn, | |
1161 | SD_LIST_FORW | SD_LIST_BACK | SD_LIST_RES_BACK); | |
1162 | while (sd_iterator_cond (&sd_it, &dep)) | |
1163 | { | |
1164 | DEP_COST (dep) = UNKNOWN_DEP_COST; | |
1165 | sd_iterator_next (&sd_it); | |
1166 | } | |
1167 | ||
1168 | /* Invalidate INSN_COST, so it'll be recalculated. */ | |
1169 | INSN_COST (insn) = -1; | |
1170 | /* Invalidate INSN_TICK, so it'll be recalculated. */ | |
1171 | INSN_TICK (insn) = INVALID_TICK; | |
340c7904 MK |
1172 | |
1173 | /* Invalidate autoprefetch data entry. */ | |
1174 | INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status | |
1175 | = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED; | |
1176 | INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status | |
1177 | = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED; | |
1a83e602 BS |
1178 | } |
1179 | ||
1a83e602 BS |
1180 | |
1181 | /* Two VECs, one to hold dependencies for which pattern replacements | |
1182 | need to be applied or restored at the start of the next cycle, and | |
1183 | another to hold an integer that is either one, to apply the | |
1184 | corresponding replacement, or zero to restore it. */ | |
9771b263 DN |
1185 | static vec<dep_t> next_cycle_replace_deps; |
1186 | static vec<int> next_cycle_apply; | |
1a83e602 BS |
1187 | |
1188 | static void apply_replacement (dep_t, bool); | |
1189 | static void restore_pattern (dep_t, bool); | |
1190 | ||
e2724e63 BS |
1191 | /* Look at the remaining dependencies for insn NEXT, and compute and return |
1192 | the TODO_SPEC value we should use for it. This is called after one of | |
1a83e602 BS |
1193 | NEXT's dependencies has been resolved. |
1194 | We also perform pattern replacements for predication, and for broken | |
1195 | replacement dependencies. The latter is only done if FOR_BACKTRACK is | |
1196 | false. */ | |
ce18efcb | 1197 | |
e2724e63 | 1198 | static ds_t |
f57aa6b0 | 1199 | recompute_todo_spec (rtx_insn *next, bool for_backtrack) |
e2724e63 BS |
1200 | { |
1201 | ds_t new_ds; | |
1202 | sd_iterator_def sd_it; | |
1a83e602 | 1203 | dep_t dep, modify_dep = NULL; |
e2724e63 BS |
1204 | int n_spec = 0; |
1205 | int n_control = 0; | |
1a83e602 | 1206 | int n_replace = 0; |
e2724e63 BS |
1207 | bool first_p = true; |
1208 | ||
1209 | if (sd_lists_empty_p (next, SD_LIST_BACK)) | |
1210 | /* NEXT has all its dependencies resolved. */ | |
1211 | return 0; | |
1212 | ||
1213 | if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK)) | |
1214 | return HARD_DEP; | |
1215 | ||
61a17dca JG |
1216 | /* If NEXT is intended to sit adjacent to this instruction, we don't |
1217 | want to try to break any dependencies. Treat it as a HARD_DEP. */ | |
1218 | if (SCHED_GROUP_P (next)) | |
1219 | return HARD_DEP; | |
1220 | ||
e2724e63 BS |
1221 | /* Now we've got NEXT with speculative deps only. |
1222 | 1. Look at the deps to see what we have to do. | |
1223 | 2. Check if we can do 'todo'. */ | |
1224 | new_ds = 0; | |
1225 | ||
1226 | FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep) | |
1227 | { | |
9b517712 | 1228 | rtx_insn *pro = DEP_PRO (dep); |
e2724e63 BS |
1229 | ds_t ds = DEP_STATUS (dep) & SPECULATIVE; |
1230 | ||
1a83e602 | 1231 | if (DEBUG_INSN_P (pro) && !DEBUG_INSN_P (next)) |
e2724e63 BS |
1232 | continue; |
1233 | ||
1234 | if (ds) | |
1235 | { | |
1236 | n_spec++; | |
1237 | if (first_p) | |
1238 | { | |
1239 | first_p = false; | |
1240 | ||
1241 | new_ds = ds; | |
1242 | } | |
1243 | else | |
1244 | new_ds = ds_merge (new_ds, ds); | |
1245 | } | |
1a83e602 | 1246 | else if (DEP_TYPE (dep) == REG_DEP_CONTROL) |
e2724e63 | 1247 | { |
1a83e602 BS |
1248 | if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED) |
1249 | { | |
1250 | n_control++; | |
1251 | modify_dep = dep; | |
1252 | } | |
1253 | DEP_STATUS (dep) &= ~DEP_CANCELLED; | |
1254 | } | |
1255 | else if (DEP_REPLACE (dep) != NULL) | |
1256 | { | |
1257 | if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED) | |
1258 | { | |
1259 | n_replace++; | |
1260 | modify_dep = dep; | |
1261 | } | |
e2724e63 BS |
1262 | DEP_STATUS (dep) &= ~DEP_CANCELLED; |
1263 | } | |
1264 | } | |
1265 | ||
1a83e602 BS |
1266 | if (n_replace > 0 && n_control == 0 && n_spec == 0) |
1267 | { | |
1268 | if (!dbg_cnt (sched_breakdep)) | |
1269 | return HARD_DEP; | |
1270 | FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep) | |
1271 | { | |
1272 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
1273 | if (desc != NULL) | |
1274 | { | |
1275 | if (desc->insn == next && !for_backtrack) | |
1276 | { | |
1277 | gcc_assert (n_replace == 1); | |
1278 | apply_replacement (dep, true); | |
1279 | } | |
1280 | DEP_STATUS (dep) |= DEP_CANCELLED; | |
1281 | } | |
1282 | } | |
1283 | return 0; | |
1284 | } | |
1285 | ||
1286 | else if (n_control == 1 && n_replace == 0 && n_spec == 0) | |
e2724e63 | 1287 | { |
f57aa6b0 DM |
1288 | rtx_insn *pro, *other; |
1289 | rtx new_pat; | |
e2724e63 BS |
1290 | rtx cond = NULL_RTX; |
1291 | bool success; | |
ce1ce33a | 1292 | rtx_insn *prev = NULL; |
e2724e63 BS |
1293 | int i; |
1294 | unsigned regno; | |
1295 | ||
1296 | if ((current_sched_info->flags & DO_PREDICATION) == 0 | |
1297 | || (ORIG_PAT (next) != NULL_RTX | |
1298 | && PREDICATED_PAT (next) == NULL_RTX)) | |
1299 | return HARD_DEP; | |
1300 | ||
1a83e602 | 1301 | pro = DEP_PRO (modify_dep); |
e2724e63 BS |
1302 | other = real_insn_for_shadow (pro); |
1303 | if (other != NULL_RTX) | |
1304 | pro = other; | |
1305 | ||
1306 | cond = sched_get_reverse_condition_uncached (pro); | |
1307 | regno = REGNO (XEXP (cond, 0)); | |
1308 | ||
1309 | /* Find the last scheduled insn that modifies the condition register. | |
89ff14c2 BS |
1310 | We can stop looking once we find the insn we depend on through the |
1311 | REG_DEP_CONTROL; if the condition register isn't modified after it, | |
1312 | we know that it still has the right value. */ | |
1313 | if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED) | |
9771b263 | 1314 | FOR_EACH_VEC_ELT_REVERSE (scheduled_insns, i, prev) |
89ff14c2 BS |
1315 | { |
1316 | HARD_REG_SET t; | |
1317 | ||
356bf593 | 1318 | find_all_hard_reg_sets (prev, &t, true); |
89ff14c2 BS |
1319 | if (TEST_HARD_REG_BIT (t, regno)) |
1320 | return HARD_DEP; | |
1321 | if (prev == pro) | |
1322 | break; | |
1323 | } | |
e2724e63 BS |
1324 | if (ORIG_PAT (next) == NULL_RTX) |
1325 | { | |
1326 | ORIG_PAT (next) = PATTERN (next); | |
1327 | ||
1328 | new_pat = gen_rtx_COND_EXEC (VOIDmode, cond, PATTERN (next)); | |
1329 | success = haifa_change_pattern (next, new_pat); | |
1330 | if (!success) | |
1331 | return HARD_DEP; | |
1332 | PREDICATED_PAT (next) = new_pat; | |
1333 | } | |
1334 | else if (PATTERN (next) != PREDICATED_PAT (next)) | |
1335 | { | |
1336 | bool success = haifa_change_pattern (next, | |
1337 | PREDICATED_PAT (next)); | |
1338 | gcc_assert (success); | |
1339 | } | |
1a83e602 | 1340 | DEP_STATUS (modify_dep) |= DEP_CANCELLED; |
e2724e63 BS |
1341 | return DEP_CONTROL; |
1342 | } | |
1343 | ||
1344 | if (PREDICATED_PAT (next) != NULL_RTX) | |
1345 | { | |
1346 | int tick = INSN_TICK (next); | |
1347 | bool success = haifa_change_pattern (next, | |
1348 | ORIG_PAT (next)); | |
1349 | INSN_TICK (next) = tick; | |
1350 | gcc_assert (success); | |
1351 | } | |
1352 | ||
1353 | /* We can't handle the case where there are both speculative and control | |
1354 | dependencies, so we return HARD_DEP in such a case. Also fail if | |
1355 | we have speculative dependencies with not enough points, or more than | |
1356 | one control dependency. */ | |
1a83e602 | 1357 | if ((n_spec > 0 && (n_control > 0 || n_replace > 0)) |
e2724e63 BS |
1358 | || (n_spec > 0 |
1359 | /* Too few points? */ | |
1360 | && ds_weak (new_ds) < spec_info->data_weakness_cutoff) | |
1a83e602 BS |
1361 | || n_control > 0 |
1362 | || n_replace > 0) | |
e2724e63 BS |
1363 | return HARD_DEP; |
1364 | ||
1365 | return new_ds; | |
1366 | } | |
1367 | \f | |
9b69cf83 | 1368 | /* Pointer to the last instruction scheduled. */ |
ce1ce33a | 1369 | static rtx_insn *last_scheduled_insn; |
8c660648 | 1370 | |
5c24671b AO |
1371 | /* Pointer to the last nondebug instruction scheduled within the |
1372 | block, or the prev_head of the scheduling block. Used by | |
1373 | rank_for_schedule, so that insns independent of the last scheduled | |
1374 | insn will be preferred over dependent instructions. */ | |
90831096 | 1375 | static rtx_insn *last_nondebug_scheduled_insn; |
5c24671b | 1376 | |
9b69cf83 BS |
1377 | /* Pointer that iterates through the list of unscheduled insns if we |
1378 | have a dbg_cnt enabled. It always points at an insn prior to the | |
1379 | first unscheduled one. */ | |
ce1ce33a | 1380 | static rtx_insn *nonscheduled_insns_begin; |
9b69cf83 | 1381 | |
b198261f | 1382 | /* Compute cost of executing INSN. |
496d7bb0 MK |
1383 | This is the number of cycles between instruction issue and |
1384 | instruction results. */ | |
5fd8300b | 1385 | int |
ffc1ded5 | 1386 | insn_sched_cost (rtx_insn *insn) |
8c660648 | 1387 | { |
e855c69d AB |
1388 | int cost; |
1389 | ||
b16abbcb BC |
1390 | if (sched_fusion) |
1391 | return 0; | |
1392 | ||
e855c69d AB |
1393 | if (sel_sched_p ()) |
1394 | { | |
1395 | if (recog_memoized (insn) < 0) | |
1396 | return 0; | |
1397 | ||
1398 | cost = insn_default_latency (insn); | |
1399 | if (cost < 0) | |
1400 | cost = 0; | |
1401 | ||
1402 | return cost; | |
1403 | } | |
1404 | ||
1405 | cost = INSN_COST (insn); | |
8c660648 | 1406 | |
fae15c93 | 1407 | if (cost < 0) |
8c660648 | 1408 | { |
fae15c93 VM |
1409 | /* A USE insn, or something else we don't need to |
1410 | understand. We can't pass these directly to | |
1411 | result_ready_cost or insn_default_latency because it will | |
1412 | trigger a fatal error for unrecognizable insns. */ | |
1413 | if (recog_memoized (insn) < 0) | |
8c660648 | 1414 | { |
fae15c93 VM |
1415 | INSN_COST (insn) = 0; |
1416 | return 0; | |
8c660648 JL |
1417 | } |
1418 | else | |
1419 | { | |
fa0aee89 | 1420 | cost = insn_default_latency (insn); |
fae15c93 VM |
1421 | if (cost < 0) |
1422 | cost = 0; | |
1d088dee | 1423 | |
8c660648 JL |
1424 | INSN_COST (insn) = cost; |
1425 | } | |
1426 | } | |
1427 | ||
b198261f MK |
1428 | return cost; |
1429 | } | |
1430 | ||
1431 | /* Compute cost of dependence LINK. | |
1432 | This is the number of cycles between instruction issue and | |
078a70a1 AN |
1433 | instruction results. |
1434 | ??? We also use this function to call recog_memoized on all insns. */ | |
b198261f | 1435 | int |
e855c69d | 1436 | dep_cost_1 (dep_t link, dw_t dw) |
b198261f | 1437 | { |
9b517712 DM |
1438 | rtx_insn *insn = DEP_PRO (link); |
1439 | rtx_insn *used = DEP_CON (link); | |
b198261f | 1440 | int cost; |
8c660648 | 1441 | |
a19af9c4 BS |
1442 | if (DEP_COST (link) != UNKNOWN_DEP_COST) |
1443 | return DEP_COST (link); | |
1444 | ||
c203e8a7 | 1445 | if (delay_htab) |
26965010 BS |
1446 | { |
1447 | struct delay_pair *delay_entry; | |
1448 | delay_entry | |
c203e8a7 | 1449 | = delay_htab_i2->find_with_hash (used, htab_hash_pointer (used)); |
26965010 BS |
1450 | if (delay_entry) |
1451 | { | |
1452 | if (delay_entry->i1 == insn) | |
1453 | { | |
1454 | DEP_COST (link) = pair_delay (delay_entry); | |
1455 | return DEP_COST (link); | |
1456 | } | |
1457 | } | |
1458 | } | |
1459 | ||
fae15c93 VM |
1460 | /* A USE insn should never require the value used to be computed. |
1461 | This allows the computation of a function's result and parameter | |
ce18efcb | 1462 | values to overlap the return and call. We don't care about the |
dd5a833e | 1463 | dependence cost when only decreasing register pressure. */ |
fae15c93 | 1464 | if (recog_memoized (used) < 0) |
078a70a1 AN |
1465 | { |
1466 | cost = 0; | |
1467 | recog_memoized (insn); | |
1468 | } | |
fae15c93 | 1469 | else |
8c660648 | 1470 | { |
e2f6ff94 | 1471 | enum reg_note dep_type = DEP_TYPE (link); |
b198261f | 1472 | |
ffc1ded5 | 1473 | cost = insn_sched_cost (insn); |
496d7bb0 | 1474 | |
fa0aee89 | 1475 | if (INSN_CODE (insn) >= 0) |
197043f5 | 1476 | { |
496d7bb0 | 1477 | if (dep_type == REG_DEP_ANTI) |
fa0aee89 | 1478 | cost = 0; |
496d7bb0 | 1479 | else if (dep_type == REG_DEP_OUTPUT) |
fae15c93 | 1480 | { |
fa0aee89 PB |
1481 | cost = (insn_default_latency (insn) |
1482 | - insn_default_latency (used)); | |
1483 | if (cost <= 0) | |
1484 | cost = 1; | |
fae15c93 | 1485 | } |
fa0aee89 PB |
1486 | else if (bypass_p (insn)) |
1487 | cost = insn_latency (insn, used); | |
197043f5 | 1488 | } |
b8698a0f | 1489 | |
b8ec5764 | 1490 | |
b505225b TS |
1491 | if (targetm.sched.adjust_cost) |
1492 | cost = targetm.sched.adjust_cost (used, (int) dep_type, insn, cost, | |
1493 | dw); | |
fae15c93 VM |
1494 | |
1495 | if (cost < 0) | |
1496 | cost = 0; | |
1497 | } | |
1d088dee | 1498 | |
a19af9c4 | 1499 | DEP_COST (link) = cost; |
8c660648 JL |
1500 | return cost; |
1501 | } | |
1502 | ||
e855c69d AB |
1503 | /* Compute cost of dependence LINK. |
1504 | This is the number of cycles between instruction issue and | |
1505 | instruction results. */ | |
1506 | int | |
1507 | dep_cost (dep_t link) | |
1508 | { | |
1509 | return dep_cost_1 (link, 0); | |
1510 | } | |
1511 | ||
1512 | /* Use this sel-sched.c friendly function in reorder2 instead of increasing | |
1513 | INSN_PRIORITY explicitly. */ | |
1514 | void | |
f57aa6b0 | 1515 | increase_insn_priority (rtx_insn *insn, int amount) |
e855c69d AB |
1516 | { |
1517 | if (!sel_sched_p ()) | |
1518 | { | |
1519 | /* We're dealing with haifa-sched.c INSN_PRIORITY. */ | |
1520 | if (INSN_PRIORITY_KNOWN (insn)) | |
1521 | INSN_PRIORITY (insn) += amount; | |
1522 | } | |
1523 | else | |
1524 | { | |
b8698a0f | 1525 | /* In sel-sched.c INSN_PRIORITY is not kept up to date. |
e855c69d AB |
1526 | Use EXPR_PRIORITY instead. */ |
1527 | sel_add_to_insn_priority (insn, amount); | |
1528 | } | |
1529 | } | |
1530 | ||
916fa4f0 MK |
1531 | /* Return 'true' if DEP should be included in priority calculations. */ |
1532 | static bool | |
1533 | contributes_to_priority_p (dep_t dep) | |
1534 | { | |
b5b8b0ac AO |
1535 | if (DEBUG_INSN_P (DEP_CON (dep)) |
1536 | || DEBUG_INSN_P (DEP_PRO (dep))) | |
1537 | return false; | |
1538 | ||
916fa4f0 MK |
1539 | /* Critical path is meaningful in block boundaries only. */ |
1540 | if (!current_sched_info->contributes_to_priority (DEP_CON (dep), | |
1541 | DEP_PRO (dep))) | |
1542 | return false; | |
1543 | ||
1a83e602 BS |
1544 | if (DEP_REPLACE (dep) != NULL) |
1545 | return false; | |
1546 | ||
916fa4f0 MK |
1547 | /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set, |
1548 | then speculative instructions will less likely be | |
1549 | scheduled. That is because the priority of | |
1550 | their producers will increase, and, thus, the | |
1551 | producers will more likely be scheduled, thus, | |
1552 | resolving the dependence. */ | |
e855c69d | 1553 | if (sched_deps_info->generate_spec_deps |
916fa4f0 MK |
1554 | && !(spec_info->flags & COUNT_SPEC_IN_CRITICAL_PATH) |
1555 | && (DEP_STATUS (dep) & SPECULATIVE)) | |
1556 | return false; | |
8c660648 | 1557 | |
916fa4f0 MK |
1558 | return true; |
1559 | } | |
1560 | ||
7a1aca9c | 1561 | /* Compute the number of nondebug deps in list LIST for INSN. */ |
b5b8b0ac AO |
1562 | |
1563 | static int | |
90831096 | 1564 | dep_list_size (rtx_insn *insn, sd_list_types_def list) |
b5b8b0ac AO |
1565 | { |
1566 | sd_iterator_def sd_it; | |
1567 | dep_t dep; | |
1568 | int dbgcount = 0, nodbgcount = 0; | |
1569 | ||
1570 | if (!MAY_HAVE_DEBUG_INSNS) | |
7a1aca9c | 1571 | return sd_lists_size (insn, list); |
b5b8b0ac | 1572 | |
7a1aca9c | 1573 | FOR_EACH_DEP (insn, list, sd_it, dep) |
b5b8b0ac AO |
1574 | { |
1575 | if (DEBUG_INSN_P (DEP_CON (dep))) | |
1576 | dbgcount++; | |
f49b295a | 1577 | else if (!DEBUG_INSN_P (DEP_PRO (dep))) |
b5b8b0ac AO |
1578 | nodbgcount++; |
1579 | } | |
1580 | ||
7a1aca9c | 1581 | gcc_assert (dbgcount + nodbgcount == sd_lists_size (insn, list)); |
b5b8b0ac AO |
1582 | |
1583 | return nodbgcount; | |
1584 | } | |
1585 | ||
b16abbcb BC |
1586 | bool sched_fusion; |
1587 | ||
916fa4f0 | 1588 | /* Compute the priority number for INSN. */ |
8c660648 | 1589 | static int |
0e0ce50d | 1590 | priority (rtx_insn *insn) |
8c660648 | 1591 | { |
2c3c49de | 1592 | if (! INSN_P (insn)) |
8c660648 JL |
1593 | return 0; |
1594 | ||
c80b4100 | 1595 | /* We should not be interested in priority of an already scheduled insn. */ |
916fa4f0 MK |
1596 | gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED); |
1597 | ||
1598 | if (!INSN_PRIORITY_KNOWN (insn)) | |
8c660648 | 1599 | { |
e855c69d | 1600 | int this_priority = -1; |
21e4c9a8 | 1601 | |
b16abbcb BC |
1602 | if (sched_fusion) |
1603 | { | |
1604 | int this_fusion_priority; | |
1605 | ||
1606 | targetm.sched.fusion_priority (insn, FUSION_MAX_PRIORITY, | |
1607 | &this_fusion_priority, &this_priority); | |
1608 | INSN_FUSION_PRIORITY (insn) = this_fusion_priority; | |
1609 | } | |
1610 | else if (dep_list_size (insn, SD_LIST_FORW) == 0) | |
ffc1ded5 SB |
1611 | /* ??? We should set INSN_PRIORITY to insn_sched_cost when and insn |
1612 | has some forward deps but all of them are ignored by | |
b198261f MK |
1613 | contributes_to_priority hook. At the moment we set priority of |
1614 | such insn to 0. */ | |
ffc1ded5 | 1615 | this_priority = insn_sched_cost (insn); |
8c660648 | 1616 | else |
21e4c9a8 | 1617 | { |
dc01c3d1 | 1618 | rtx_insn *prev_first, *twin; |
496d7bb0 | 1619 | basic_block rec; |
8c660648 | 1620 | |
496d7bb0 MK |
1621 | /* For recovery check instructions we calculate priority slightly |
1622 | different than that of normal instructions. Instead of walking | |
b198261f MK |
1623 | through INSN_FORW_DEPS (check) list, we walk through |
1624 | INSN_FORW_DEPS list of each instruction in the corresponding | |
b8698a0f | 1625 | recovery block. */ |
8c660648 | 1626 | |
e855c69d AB |
1627 | /* Selective scheduling does not define RECOVERY_BLOCK macro. */ |
1628 | rec = sel_sched_p () ? NULL : RECOVERY_BLOCK (insn); | |
fefa31b5 | 1629 | if (!rec || rec == EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 MK |
1630 | { |
1631 | prev_first = PREV_INSN (insn); | |
1632 | twin = insn; | |
1633 | } | |
1634 | else | |
1635 | { | |
1636 | prev_first = NEXT_INSN (BB_HEAD (rec)); | |
1637 | twin = PREV_INSN (BB_END (rec)); | |
1638 | } | |
8c660648 | 1639 | |
496d7bb0 MK |
1640 | do |
1641 | { | |
e2f6ff94 MK |
1642 | sd_iterator_def sd_it; |
1643 | dep_t dep; | |
1644 | ||
1645 | FOR_EACH_DEP (twin, SD_LIST_FORW, sd_it, dep) | |
496d7bb0 | 1646 | { |
0e0ce50d | 1647 | rtx_insn *next; |
496d7bb0 | 1648 | int next_priority; |
b198261f MK |
1649 | |
1650 | next = DEP_CON (dep); | |
1651 | ||
496d7bb0 MK |
1652 | if (BLOCK_FOR_INSN (next) != rec) |
1653 | { | |
b198261f MK |
1654 | int cost; |
1655 | ||
916fa4f0 | 1656 | if (!contributes_to_priority_p (dep)) |
496d7bb0 | 1657 | continue; |
b198261f MK |
1658 | |
1659 | if (twin == insn) | |
1660 | cost = dep_cost (dep); | |
1661 | else | |
1662 | { | |
1663 | struct _dep _dep1, *dep1 = &_dep1; | |
1664 | ||
1665 | init_dep (dep1, insn, next, REG_DEP_ANTI); | |
1666 | ||
1667 | cost = dep_cost (dep1); | |
1668 | } | |
1669 | ||
1670 | next_priority = cost + priority (next); | |
496d7bb0 MK |
1671 | |
1672 | if (next_priority > this_priority) | |
1673 | this_priority = next_priority; | |
1674 | } | |
1675 | } | |
b8698a0f | 1676 | |
496d7bb0 | 1677 | twin = PREV_INSN (twin); |
21e4c9a8 | 1678 | } |
496d7bb0 | 1679 | while (twin != prev_first); |
21e4c9a8 | 1680 | } |
e855c69d AB |
1681 | |
1682 | if (this_priority < 0) | |
1683 | { | |
1684 | gcc_assert (this_priority == -1); | |
1685 | ||
ffc1ded5 | 1686 | this_priority = insn_sched_cost (insn); |
e855c69d AB |
1687 | } |
1688 | ||
8c660648 | 1689 | INSN_PRIORITY (insn) = this_priority; |
916fa4f0 | 1690 | INSN_PRIORITY_STATUS (insn) = 1; |
8c660648 | 1691 | } |
21e4c9a8 BS |
1692 | |
1693 | return INSN_PRIORITY (insn); | |
8c660648 JL |
1694 | } |
1695 | \f | |
8c660648 | 1696 | /* Macros and functions for keeping the priority queue sorted, and |
d91edf86 | 1697 | dealing with queuing and dequeuing of instructions. */ |
8c660648 | 1698 | |
7a1aca9c RS |
1699 | /* For each pressure class CL, set DEATH[CL] to the number of registers |
1700 | in that class that die in INSN. */ | |
1701 | ||
1702 | static void | |
f57aa6b0 | 1703 | calculate_reg_deaths (rtx_insn *insn, int *death) |
7a1aca9c RS |
1704 | { |
1705 | int i; | |
1706 | struct reg_use_data *use; | |
1707 | ||
1708 | for (i = 0; i < ira_pressure_classes_num; i++) | |
1709 | death[ira_pressure_classes[i]] = 0; | |
1710 | for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use) | |
1711 | if (dying_use_p (use)) | |
1712 | mark_regno_birth_or_death (0, death, use->regno, true); | |
1713 | } | |
1714 | ||
ce18efcb VM |
1715 | /* Setup info about the current register pressure impact of scheduling |
1716 | INSN at the current scheduling point. */ | |
1717 | static void | |
f57aa6b0 | 1718 | setup_insn_reg_pressure_info (rtx_insn *insn) |
ce18efcb VM |
1719 | { |
1720 | int i, change, before, after, hard_regno; | |
1721 | int excess_cost_change; | |
ef4bddc2 | 1722 | machine_mode mode; |
ce18efcb VM |
1723 | enum reg_class cl; |
1724 | struct reg_pressure_data *pressure_info; | |
1725 | int *max_reg_pressure; | |
ce18efcb VM |
1726 | static int death[N_REG_CLASSES]; |
1727 | ||
c11b0b3b AO |
1728 | gcc_checking_assert (!DEBUG_INSN_P (insn)); |
1729 | ||
ce18efcb | 1730 | excess_cost_change = 0; |
7a1aca9c | 1731 | calculate_reg_deaths (insn, death); |
ce18efcb VM |
1732 | pressure_info = INSN_REG_PRESSURE (insn); |
1733 | max_reg_pressure = INSN_MAX_REG_PRESSURE (insn); | |
1734 | gcc_assert (pressure_info != NULL && max_reg_pressure != NULL); | |
1756cb66 | 1735 | for (i = 0; i < ira_pressure_classes_num; i++) |
ce18efcb | 1736 | { |
1756cb66 | 1737 | cl = ira_pressure_classes[i]; |
ce18efcb VM |
1738 | gcc_assert (curr_reg_pressure[cl] >= 0); |
1739 | change = (int) pressure_info[i].set_increase - death[cl]; | |
4dd9ac6c | 1740 | before = MAX (0, max_reg_pressure[i] - sched_class_regs_num[cl]); |
ce18efcb | 1741 | after = MAX (0, max_reg_pressure[i] + change |
4dd9ac6c | 1742 | - sched_class_regs_num[cl]); |
ce18efcb VM |
1743 | hard_regno = ira_class_hard_regs[cl][0]; |
1744 | gcc_assert (hard_regno >= 0); | |
1745 | mode = reg_raw_mode[hard_regno]; | |
1746 | excess_cost_change += ((after - before) | |
1747 | * (ira_memory_move_cost[mode][cl][0] | |
1748 | + ira_memory_move_cost[mode][cl][1])); | |
1749 | } | |
1750 | INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn) = excess_cost_change; | |
1751 | } | |
7a1aca9c RS |
1752 | \f |
1753 | /* This is the first page of code related to SCHED_PRESSURE_MODEL. | |
1754 | It tries to make the scheduler take register pressure into account | |
1755 | without introducing too many unnecessary stalls. It hooks into the | |
1756 | main scheduling algorithm at several points: | |
1757 | ||
1758 | - Before scheduling starts, model_start_schedule constructs a | |
1759 | "model schedule" for the current block. This model schedule is | |
1760 | chosen solely to keep register pressure down. It does not take the | |
1761 | target's pipeline or the original instruction order into account, | |
1762 | except as a tie-breaker. It also doesn't work to a particular | |
1763 | pressure limit. | |
1764 | ||
1765 | This model schedule gives us an idea of what pressure can be | |
1766 | achieved for the block and gives us an example of a schedule that | |
1767 | keeps to that pressure. It also makes the final schedule less | |
1768 | dependent on the original instruction order. This is important | |
1769 | because the original order can either be "wide" (many values live | |
1770 | at once, such as in user-scheduled code) or "narrow" (few values | |
1771 | live at once, such as after loop unrolling, where several | |
1772 | iterations are executed sequentially). | |
1773 | ||
1774 | We do not apply this model schedule to the rtx stream. We simply | |
1775 | record it in model_schedule. We also compute the maximum pressure, | |
1776 | MP, that was seen during this schedule. | |
1777 | ||
1778 | - Instructions are added to the ready queue even if they require | |
1779 | a stall. The length of the stall is instead computed as: | |
1780 | ||
1781 | MAX (INSN_TICK (INSN) - clock_var, 0) | |
1782 | ||
1783 | (= insn_delay). This allows rank_for_schedule to choose between | |
1784 | introducing a deliberate stall or increasing pressure. | |
1785 | ||
1786 | - Before sorting the ready queue, model_set_excess_costs assigns | |
1787 | a pressure-based cost to each ready instruction in the queue. | |
1788 | This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE | |
1789 | (ECC for short) and is effectively measured in cycles. | |
1790 | ||
1791 | - rank_for_schedule ranks instructions based on: | |
1792 | ||
1793 | ECC (insn) + insn_delay (insn) | |
1794 | ||
1795 | then as: | |
1796 | ||
1797 | insn_delay (insn) | |
1798 | ||
1799 | So, for example, an instruction X1 with an ECC of 1 that can issue | |
1800 | now will win over an instruction X0 with an ECC of zero that would | |
1801 | introduce a stall of one cycle. However, an instruction X2 with an | |
1802 | ECC of 2 that can issue now will lose to both X0 and X1. | |
1803 | ||
1804 | - When an instruction is scheduled, model_recompute updates the model | |
1805 | schedule with the new pressures (some of which might now exceed the | |
1806 | original maximum pressure MP). model_update_limit_points then searches | |
1807 | for the new point of maximum pressure, if not already known. */ | |
1808 | ||
1809 | /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL | |
1810 | from surrounding debug information. */ | |
1811 | #define MODEL_BAR \ | |
1812 | ";;\t\t+------------------------------------------------------\n" | |
1813 | ||
1814 | /* Information about the pressure on a particular register class at a | |
1815 | particular point of the model schedule. */ | |
1816 | struct model_pressure_data { | |
1817 | /* The pressure at this point of the model schedule, or -1 if the | |
1818 | point is associated with an instruction that has already been | |
1819 | scheduled. */ | |
1820 | int ref_pressure; | |
1821 | ||
1822 | /* The maximum pressure during or after this point of the model schedule. */ | |
1823 | int max_pressure; | |
1824 | }; | |
1825 | ||
1826 | /* Per-instruction information that is used while building the model | |
1827 | schedule. Here, "schedule" refers to the model schedule rather | |
1828 | than the main schedule. */ | |
1829 | struct model_insn_info { | |
1830 | /* The instruction itself. */ | |
40a243d4 | 1831 | rtx_insn *insn; |
7a1aca9c RS |
1832 | |
1833 | /* If this instruction is in model_worklist, these fields link to the | |
1834 | previous (higher-priority) and next (lower-priority) instructions | |
1835 | in the list. */ | |
1836 | struct model_insn_info *prev; | |
1837 | struct model_insn_info *next; | |
1838 | ||
1839 | /* While constructing the schedule, QUEUE_INDEX describes whether an | |
1840 | instruction has already been added to the schedule (QUEUE_SCHEDULED), | |
1841 | is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE). | |
1842 | old_queue records the value that QUEUE_INDEX had before scheduling | |
1843 | started, so that we can restore it once the schedule is complete. */ | |
1844 | int old_queue; | |
1845 | ||
1846 | /* The relative importance of an unscheduled instruction. Higher | |
1847 | values indicate greater importance. */ | |
1848 | unsigned int model_priority; | |
1849 | ||
1850 | /* The length of the longest path of satisfied true dependencies | |
1851 | that leads to this instruction. */ | |
1852 | unsigned int depth; | |
1853 | ||
1854 | /* The length of the longest path of dependencies of any kind | |
1855 | that leads from this instruction. */ | |
1856 | unsigned int alap; | |
1857 | ||
1858 | /* The number of predecessor nodes that must still be scheduled. */ | |
1859 | int unscheduled_preds; | |
1860 | }; | |
1861 | ||
1862 | /* Information about the pressure limit for a particular register class. | |
1863 | This structure is used when applying a model schedule to the main | |
1864 | schedule. */ | |
1865 | struct model_pressure_limit { | |
1866 | /* The maximum register pressure seen in the original model schedule. */ | |
1867 | int orig_pressure; | |
1868 | ||
1869 | /* The maximum register pressure seen in the current model schedule | |
1870 | (which excludes instructions that have already been scheduled). */ | |
1871 | int pressure; | |
1872 | ||
1873 | /* The point of the current model schedule at which PRESSURE is first | |
1874 | reached. It is set to -1 if the value needs to be recomputed. */ | |
1875 | int point; | |
1876 | }; | |
1877 | ||
1878 | /* Describes a particular way of measuring register pressure. */ | |
1879 | struct model_pressure_group { | |
1880 | /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */ | |
1881 | struct model_pressure_limit limits[N_REG_CLASSES]; | |
1882 | ||
1883 | /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure | |
1884 | on register class ira_pressure_classes[PCI] at point POINT of the | |
1885 | current model schedule. A POINT of model_num_insns describes the | |
1886 | pressure at the end of the schedule. */ | |
1887 | struct model_pressure_data *model; | |
1888 | }; | |
1889 | ||
1890 | /* Index POINT gives the instruction at point POINT of the model schedule. | |
1891 | This array doesn't change during main scheduling. */ | |
f57aa6b0 | 1892 | static vec<rtx_insn *> model_schedule; |
7a1aca9c RS |
1893 | |
1894 | /* The list of instructions in the model worklist, sorted in order of | |
1895 | decreasing priority. */ | |
1896 | static struct model_insn_info *model_worklist; | |
1897 | ||
1898 | /* Index I describes the instruction with INSN_LUID I. */ | |
1899 | static struct model_insn_info *model_insns; | |
1900 | ||
1901 | /* The number of instructions in the model schedule. */ | |
1902 | static int model_num_insns; | |
1903 | ||
1904 | /* The index of the first instruction in model_schedule that hasn't yet been | |
1905 | added to the main schedule, or model_num_insns if all of them have. */ | |
1906 | static int model_curr_point; | |
1907 | ||
1908 | /* Describes the pressure before each instruction in the model schedule. */ | |
1909 | static struct model_pressure_group model_before_pressure; | |
1910 | ||
1911 | /* The first unused model_priority value (as used in model_insn_info). */ | |
1912 | static unsigned int model_next_priority; | |
1913 | ||
1914 | ||
1915 | /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP | |
1916 | at point POINT of the model schedule. */ | |
1917 | #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \ | |
1918 | (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)]) | |
1919 | ||
1920 | /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or | |
1921 | after point POINT of the model schedule. */ | |
1922 | #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \ | |
1923 | (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure) | |
1924 | ||
1925 | /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT | |
1926 | of the model schedule. */ | |
1927 | #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \ | |
1928 | (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure) | |
1929 | ||
1930 | /* Information about INSN that is used when creating the model schedule. */ | |
1931 | #define MODEL_INSN_INFO(INSN) \ | |
1932 | (&model_insns[INSN_LUID (INSN)]) | |
1933 | ||
1934 | /* The instruction at point POINT of the model schedule. */ | |
1935 | #define MODEL_INSN(POINT) \ | |
9771b263 | 1936 | (model_schedule[POINT]) |
7a1aca9c RS |
1937 | |
1938 | ||
1939 | /* Return INSN's index in the model schedule, or model_num_insns if it | |
1940 | doesn't belong to that schedule. */ | |
1941 | ||
1942 | static int | |
f57aa6b0 | 1943 | model_index (rtx_insn *insn) |
7a1aca9c RS |
1944 | { |
1945 | if (INSN_MODEL_INDEX (insn) == 0) | |
1946 | return model_num_insns; | |
1947 | return INSN_MODEL_INDEX (insn) - 1; | |
1948 | } | |
1949 | ||
1950 | /* Make sure that GROUP->limits is up-to-date for the current point | |
1951 | of the model schedule. */ | |
1952 | ||
1953 | static void | |
1954 | model_update_limit_points_in_group (struct model_pressure_group *group) | |
1955 | { | |
1956 | int pci, max_pressure, point; | |
1957 | ||
1958 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
1959 | { | |
1960 | /* We may have passed the final point at which the pressure in | |
1961 | group->limits[pci].pressure was reached. Update the limit if so. */ | |
1962 | max_pressure = MODEL_MAX_PRESSURE (group, model_curr_point, pci); | |
1963 | group->limits[pci].pressure = max_pressure; | |
1964 | ||
1965 | /* Find the point at which MAX_PRESSURE is first reached. We need | |
1966 | to search in three cases: | |
1967 | ||
1968 | - We've already moved past the previous pressure point. | |
1969 | In this case we search forward from model_curr_point. | |
1970 | ||
1971 | - We scheduled the previous point of maximum pressure ahead of | |
1972 | its position in the model schedule, but doing so didn't bring | |
1973 | the pressure point earlier. In this case we search forward | |
1974 | from that previous pressure point. | |
1975 | ||
1976 | - Scheduling an instruction early caused the maximum pressure | |
1977 | to decrease. In this case we will have set the pressure | |
1978 | point to -1, and we search forward from model_curr_point. */ | |
1979 | point = MAX (group->limits[pci].point, model_curr_point); | |
1980 | while (point < model_num_insns | |
1981 | && MODEL_REF_PRESSURE (group, point, pci) < max_pressure) | |
1982 | point++; | |
1983 | group->limits[pci].point = point; | |
1984 | ||
1985 | gcc_assert (MODEL_REF_PRESSURE (group, point, pci) == max_pressure); | |
1986 | gcc_assert (MODEL_MAX_PRESSURE (group, point, pci) == max_pressure); | |
1987 | } | |
1988 | } | |
1989 | ||
1990 | /* Make sure that all register-pressure limits are up-to-date for the | |
1991 | current position in the model schedule. */ | |
1992 | ||
1993 | static void | |
1994 | model_update_limit_points (void) | |
1995 | { | |
1996 | model_update_limit_points_in_group (&model_before_pressure); | |
1997 | } | |
1998 | ||
1999 | /* Return the model_index of the last unscheduled use in chain USE | |
2000 | outside of USE's instruction. Return -1 if there are no other uses, | |
2001 | or model_num_insns if the register is live at the end of the block. */ | |
2002 | ||
2003 | static int | |
2004 | model_last_use_except (struct reg_use_data *use) | |
2005 | { | |
2006 | struct reg_use_data *next; | |
2007 | int last, index; | |
2008 | ||
2009 | last = -1; | |
2010 | for (next = use->next_regno_use; next != use; next = next->next_regno_use) | |
2011 | if (NONDEBUG_INSN_P (next->insn) | |
2012 | && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED) | |
2013 | { | |
2014 | index = model_index (next->insn); | |
2015 | if (index == model_num_insns) | |
2016 | return model_num_insns; | |
2017 | if (last < index) | |
2018 | last = index; | |
2019 | } | |
2020 | return last; | |
2021 | } | |
2022 | ||
2023 | /* An instruction with model_index POINT has just been scheduled, and it | |
2024 | adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1. | |
2025 | Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and | |
2026 | MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */ | |
2027 | ||
2028 | static void | |
2029 | model_start_update_pressure (struct model_pressure_group *group, | |
2030 | int point, int pci, int delta) | |
2031 | { | |
2032 | int next_max_pressure; | |
2033 | ||
2034 | if (point == model_num_insns) | |
2035 | { | |
2036 | /* The instruction wasn't part of the model schedule; it was moved | |
2037 | from a different block. Update the pressure for the end of | |
2038 | the model schedule. */ | |
2039 | MODEL_REF_PRESSURE (group, point, pci) += delta; | |
2040 | MODEL_MAX_PRESSURE (group, point, pci) += delta; | |
2041 | } | |
2042 | else | |
2043 | { | |
2044 | /* Record that this instruction has been scheduled. Nothing now | |
2045 | changes between POINT and POINT + 1, so get the maximum pressure | |
2046 | from the latter. If the maximum pressure decreases, the new | |
2047 | pressure point may be before POINT. */ | |
2048 | MODEL_REF_PRESSURE (group, point, pci) = -1; | |
2049 | next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci); | |
2050 | if (MODEL_MAX_PRESSURE (group, point, pci) > next_max_pressure) | |
2051 | { | |
2052 | MODEL_MAX_PRESSURE (group, point, pci) = next_max_pressure; | |
2053 | if (group->limits[pci].point == point) | |
2054 | group->limits[pci].point = -1; | |
2055 | } | |
2056 | } | |
2057 | } | |
2058 | ||
2059 | /* Record that scheduling a later instruction has changed the pressure | |
2060 | at point POINT of the model schedule by DELTA (which might be 0). | |
2061 | Update GROUP accordingly. Return nonzero if these changes might | |
2062 | trigger changes to previous points as well. */ | |
2063 | ||
2064 | static int | |
2065 | model_update_pressure (struct model_pressure_group *group, | |
2066 | int point, int pci, int delta) | |
2067 | { | |
2068 | int ref_pressure, max_pressure, next_max_pressure; | |
2069 | ||
2070 | /* If POINT hasn't yet been scheduled, update its pressure. */ | |
2071 | ref_pressure = MODEL_REF_PRESSURE (group, point, pci); | |
2072 | if (ref_pressure >= 0 && delta != 0) | |
2073 | { | |
2074 | ref_pressure += delta; | |
2075 | MODEL_REF_PRESSURE (group, point, pci) = ref_pressure; | |
2076 | ||
2077 | /* Check whether the maximum pressure in the overall schedule | |
2078 | has increased. (This means that the MODEL_MAX_PRESSURE of | |
4dd9ac6c | 2079 | every point <= POINT will need to increase too; see below.) */ |
7a1aca9c RS |
2080 | if (group->limits[pci].pressure < ref_pressure) |
2081 | group->limits[pci].pressure = ref_pressure; | |
2082 | ||
2083 | /* If we are at maximum pressure, and the maximum pressure | |
2084 | point was previously unknown or later than POINT, | |
2085 | bring it forward. */ | |
2086 | if (group->limits[pci].pressure == ref_pressure | |
2087 | && !IN_RANGE (group->limits[pci].point, 0, point)) | |
2088 | group->limits[pci].point = point; | |
2089 | ||
2090 | /* If POINT used to be the point of maximum pressure, but isn't | |
2091 | any longer, we need to recalculate it using a forward walk. */ | |
2092 | if (group->limits[pci].pressure > ref_pressure | |
2093 | && group->limits[pci].point == point) | |
2094 | group->limits[pci].point = -1; | |
2095 | } | |
2096 | ||
2097 | /* Update the maximum pressure at POINT. Changes here might also | |
2098 | affect the maximum pressure at POINT - 1. */ | |
2099 | next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci); | |
2100 | max_pressure = MAX (ref_pressure, next_max_pressure); | |
2101 | if (MODEL_MAX_PRESSURE (group, point, pci) != max_pressure) | |
2102 | { | |
2103 | MODEL_MAX_PRESSURE (group, point, pci) = max_pressure; | |
2104 | return 1; | |
2105 | } | |
2106 | return 0; | |
2107 | } | |
2108 | ||
2109 | /* INSN has just been scheduled. Update the model schedule accordingly. */ | |
2110 | ||
2111 | static void | |
f57aa6b0 | 2112 | model_recompute (rtx_insn *insn) |
7a1aca9c RS |
2113 | { |
2114 | struct { | |
2115 | int last_use; | |
2116 | int regno; | |
2117 | } uses[FIRST_PSEUDO_REGISTER + MAX_RECOG_OPERANDS]; | |
2118 | struct reg_use_data *use; | |
2119 | struct reg_pressure_data *reg_pressure; | |
2120 | int delta[N_REG_CLASSES]; | |
2121 | int pci, point, mix, new_last, cl, ref_pressure, queue; | |
2122 | unsigned int i, num_uses, num_pending_births; | |
2123 | bool print_p; | |
2124 | ||
2125 | /* The destinations of INSN were previously live from POINT onwards, but are | |
2126 | now live from model_curr_point onwards. Set up DELTA accordingly. */ | |
2127 | point = model_index (insn); | |
2128 | reg_pressure = INSN_REG_PRESSURE (insn); | |
2129 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2130 | { | |
2131 | cl = ira_pressure_classes[pci]; | |
2132 | delta[cl] = reg_pressure[pci].set_increase; | |
2133 | } | |
2134 | ||
2135 | /* Record which registers previously died at POINT, but which now die | |
2136 | before POINT. Adjust DELTA so that it represents the effect of | |
2137 | this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of | |
2138 | registers that will be born in the range [model_curr_point, POINT). */ | |
2139 | num_uses = 0; | |
2140 | num_pending_births = 0; | |
f380f608 | 2141 | bitmap_clear (tmp_bitmap); |
7a1aca9c RS |
2142 | for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use) |
2143 | { | |
2144 | new_last = model_last_use_except (use); | |
f380f608 | 2145 | if (new_last < point && bitmap_set_bit (tmp_bitmap, use->regno)) |
7a1aca9c RS |
2146 | { |
2147 | gcc_assert (num_uses < ARRAY_SIZE (uses)); | |
2148 | uses[num_uses].last_use = new_last; | |
2149 | uses[num_uses].regno = use->regno; | |
2150 | /* This register is no longer live after POINT - 1. */ | |
2151 | mark_regno_birth_or_death (NULL, delta, use->regno, false); | |
2152 | num_uses++; | |
2153 | if (new_last >= 0) | |
2154 | num_pending_births++; | |
2155 | } | |
2156 | } | |
2157 | ||
2158 | /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT. | |
2159 | Also set each group pressure limit for POINT. */ | |
2160 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2161 | { | |
2162 | cl = ira_pressure_classes[pci]; | |
2163 | model_start_update_pressure (&model_before_pressure, | |
2164 | point, pci, delta[cl]); | |
2165 | } | |
2166 | ||
2167 | /* Walk the model schedule backwards, starting immediately before POINT. */ | |
2168 | print_p = false; | |
2169 | if (point != model_curr_point) | |
2170 | do | |
2171 | { | |
2172 | point--; | |
2173 | insn = MODEL_INSN (point); | |
2174 | queue = QUEUE_INDEX (insn); | |
2175 | ||
2176 | if (queue != QUEUE_SCHEDULED) | |
2177 | { | |
2178 | /* DELTA describes the effect of the move on the register pressure | |
2179 | after POINT. Make it describe the effect on the pressure | |
2180 | before POINT. */ | |
2181 | i = 0; | |
2182 | while (i < num_uses) | |
2183 | { | |
2184 | if (uses[i].last_use == point) | |
2185 | { | |
2186 | /* This register is now live again. */ | |
2187 | mark_regno_birth_or_death (NULL, delta, | |
2188 | uses[i].regno, true); | |
2189 | ||
2190 | /* Remove this use from the array. */ | |
2191 | uses[i] = uses[num_uses - 1]; | |
2192 | num_uses--; | |
2193 | num_pending_births--; | |
2194 | } | |
2195 | else | |
2196 | i++; | |
2197 | } | |
2198 | ||
2199 | if (sched_verbose >= 5) | |
2200 | { | |
7a1aca9c RS |
2201 | if (!print_p) |
2202 | { | |
2203 | fprintf (sched_dump, MODEL_BAR); | |
2204 | fprintf (sched_dump, ";;\t\t| New pressure for model" | |
2205 | " schedule\n"); | |
2206 | fprintf (sched_dump, MODEL_BAR); | |
2207 | print_p = true; | |
2208 | } | |
2209 | ||
7a1aca9c | 2210 | fprintf (sched_dump, ";;\t\t| %3d %4d %-30s ", |
7eba871a SB |
2211 | point, INSN_UID (insn), |
2212 | str_pattern_slim (PATTERN (insn))); | |
7a1aca9c RS |
2213 | for (pci = 0; pci < ira_pressure_classes_num; pci++) |
2214 | { | |
2215 | cl = ira_pressure_classes[pci]; | |
2216 | ref_pressure = MODEL_REF_PRESSURE (&model_before_pressure, | |
2217 | point, pci); | |
2218 | fprintf (sched_dump, " %s:[%d->%d]", | |
2219 | reg_class_names[ira_pressure_classes[pci]], | |
2220 | ref_pressure, ref_pressure + delta[cl]); | |
2221 | } | |
2222 | fprintf (sched_dump, "\n"); | |
2223 | } | |
2224 | } | |
2225 | ||
2226 | /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1 | |
2227 | might have changed as well. */ | |
2228 | mix = num_pending_births; | |
2229 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2230 | { | |
2231 | cl = ira_pressure_classes[pci]; | |
2232 | mix |= delta[cl]; | |
2233 | mix |= model_update_pressure (&model_before_pressure, | |
2234 | point, pci, delta[cl]); | |
2235 | } | |
2236 | } | |
2237 | while (mix && point > model_curr_point); | |
2238 | ||
2239 | if (print_p) | |
2240 | fprintf (sched_dump, MODEL_BAR); | |
2241 | } | |
1a83e602 BS |
2242 | |
2243 | /* After DEP, which was cancelled, has been resolved for insn NEXT, | |
2244 | check whether the insn's pattern needs restoring. */ | |
2245 | static bool | |
f57aa6b0 | 2246 | must_restore_pattern_p (rtx_insn *next, dep_t dep) |
1a83e602 BS |
2247 | { |
2248 | if (QUEUE_INDEX (next) == QUEUE_SCHEDULED) | |
2249 | return false; | |
2250 | ||
2251 | if (DEP_TYPE (dep) == REG_DEP_CONTROL) | |
2252 | { | |
2253 | gcc_assert (ORIG_PAT (next) != NULL_RTX); | |
2254 | gcc_assert (next == DEP_CON (dep)); | |
2255 | } | |
2256 | else | |
2257 | { | |
2258 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
2259 | if (desc->insn != next) | |
2260 | { | |
2261 | gcc_assert (*desc->loc == desc->orig); | |
2262 | return false; | |
2263 | } | |
2264 | } | |
2265 | return true; | |
2266 | } | |
7a1aca9c RS |
2267 | \f |
2268 | /* model_spill_cost (CL, P, P') returns the cost of increasing the | |
2269 | pressure on CL from P to P'. We use this to calculate a "base ECC", | |
2270 | baseECC (CL, X), for each pressure class CL and each instruction X. | |
2271 | Supposing X changes the pressure on CL from P to P', and that the | |
2272 | maximum pressure on CL in the current model schedule is MP', then: | |
2273 | ||
2274 | * if X occurs before or at the next point of maximum pressure in | |
2275 | the model schedule and P' > MP', then: | |
2276 | ||
2277 | baseECC (CL, X) = model_spill_cost (CL, MP, P') | |
2278 | ||
2279 | The idea is that the pressure after scheduling a fixed set of | |
2280 | instructions -- in this case, the set up to and including the | |
2281 | next maximum pressure point -- is going to be the same regardless | |
2282 | of the order; we simply want to keep the intermediate pressure | |
2283 | under control. Thus X has a cost of zero unless scheduling it | |
2284 | now would exceed MP'. | |
2285 | ||
2286 | If all increases in the set are by the same amount, no zero-cost | |
2287 | instruction will ever cause the pressure to exceed MP'. However, | |
2288 | if X is instead moved past an instruction X' with pressure in the | |
2289 | range (MP' - (P' - P), MP'), the pressure at X' will increase | |
2290 | beyond MP'. Since baseECC is very much a heuristic anyway, | |
2291 | it doesn't seem worth the overhead of tracking cases like these. | |
2292 | ||
2293 | The cost of exceeding MP' is always based on the original maximum | |
2294 | pressure MP. This is so that going 2 registers over the original | |
2295 | limit has the same cost regardless of whether it comes from two | |
2296 | separate +1 deltas or from a single +2 delta. | |
2297 | ||
2298 | * if X occurs after the next point of maximum pressure in the model | |
2299 | schedule and P' > P, then: | |
2300 | ||
2301 | baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P)) | |
2302 | ||
2303 | That is, if we move X forward across a point of maximum pressure, | |
2304 | and if X increases the pressure by P' - P, then we conservatively | |
2305 | assume that scheduling X next would increase the maximum pressure | |
2306 | by P' - P. Again, the cost of doing this is based on the original | |
2307 | maximum pressure MP, for the same reason as above. | |
2308 | ||
2309 | * if P' < P, P > MP, and X occurs at or after the next point of | |
2310 | maximum pressure, then: | |
2311 | ||
2312 | baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P) | |
ce18efcb | 2313 | |
7a1aca9c RS |
2314 | That is, if we have already exceeded the original maximum pressure MP, |
2315 | and if X might reduce the maximum pressure again -- or at least push | |
2316 | it further back, and thus allow more scheduling freedom -- it is given | |
2317 | a negative cost to reflect the improvement. | |
2318 | ||
2319 | * otherwise, | |
2320 | ||
2321 | baseECC (CL, X) = 0 | |
2322 | ||
2323 | In this case, X is not expected to affect the maximum pressure MP', | |
2324 | so it has zero cost. | |
2325 | ||
2326 | We then create a combined value baseECC (X) that is the sum of | |
2327 | baseECC (CL, X) for each pressure class CL. | |
2328 | ||
2329 | baseECC (X) could itself be used as the ECC value described above. | |
2330 | However, this is often too conservative, in the sense that it | |
2331 | tends to make high-priority instructions that increase pressure | |
2332 | wait too long in cases where introducing a spill would be better. | |
2333 | For this reason the final ECC is a priority-adjusted form of | |
2334 | baseECC (X). Specifically, we calculate: | |
2335 | ||
2336 | P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X) | |
2337 | baseP = MAX { P (X) | baseECC (X) <= 0 } | |
2338 | ||
2339 | Then: | |
2340 | ||
2341 | ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0) | |
2342 | ||
2343 | Thus an instruction's effect on pressure is ignored if it has a high | |
2344 | enough priority relative to the ones that don't increase pressure. | |
2345 | Negative values of baseECC (X) do not increase the priority of X | |
2346 | itself, but they do make it harder for other instructions to | |
2347 | increase the pressure further. | |
2348 | ||
2349 | This pressure cost is deliberately timid. The intention has been | |
2350 | to choose a heuristic that rarely interferes with the normal list | |
2351 | scheduler in cases where that scheduler would produce good code. | |
2352 | We simply want to curb some of its worst excesses. */ | |
2353 | ||
2354 | /* Return the cost of increasing the pressure in class CL from FROM to TO. | |
2355 | ||
2356 | Here we use the very simplistic cost model that every register above | |
4dd9ac6c | 2357 | sched_class_regs_num[CL] has a spill cost of 1. We could use other |
7a1aca9c RS |
2358 | measures instead, such as one based on MEMORY_MOVE_COST. However: |
2359 | ||
2360 | (1) In order for an instruction to be scheduled, the higher cost | |
2361 | would need to be justified in a single saving of that many stalls. | |
2362 | This is overly pessimistic, because the benefit of spilling is | |
2363 | often to avoid a sequence of several short stalls rather than | |
2364 | a single long one. | |
2365 | ||
2366 | (2) The cost is still arbitrary. Because we are not allocating | |
2367 | registers during scheduling, we have no way of knowing for | |
2368 | sure how many memory accesses will be required by each spill, | |
2369 | where the spills will be placed within the block, or even | |
2370 | which block(s) will contain the spills. | |
2371 | ||
2372 | So a higher cost than 1 is often too conservative in practice, | |
2373 | forcing blocks to contain unnecessary stalls instead of spill code. | |
2374 | The simple cost below seems to be the best compromise. It reduces | |
2375 | the interference with the normal list scheduler, which helps make | |
2376 | it more suitable for a default-on option. */ | |
2377 | ||
2378 | static int | |
2379 | model_spill_cost (int cl, int from, int to) | |
2380 | { | |
4dd9ac6c | 2381 | from = MAX (from, sched_class_regs_num[cl]); |
7a1aca9c RS |
2382 | return MAX (to, from) - from; |
2383 | } | |
2384 | ||
2385 | /* Return baseECC (ira_pressure_classes[PCI], POINT), given that | |
2386 | P = curr_reg_pressure[ira_pressure_classes[PCI]] and that | |
2387 | P' = P + DELTA. */ | |
2388 | ||
2389 | static int | |
2390 | model_excess_group_cost (struct model_pressure_group *group, | |
2391 | int point, int pci, int delta) | |
2392 | { | |
2393 | int pressure, cl; | |
2394 | ||
2395 | cl = ira_pressure_classes[pci]; | |
2396 | if (delta < 0 && point >= group->limits[pci].point) | |
2397 | { | |
2398 | pressure = MAX (group->limits[pci].orig_pressure, | |
2399 | curr_reg_pressure[cl] + delta); | |
2400 | return -model_spill_cost (cl, pressure, curr_reg_pressure[cl]); | |
2401 | } | |
2402 | ||
2403 | if (delta > 0) | |
2404 | { | |
2405 | if (point > group->limits[pci].point) | |
2406 | pressure = group->limits[pci].pressure + delta; | |
2407 | else | |
2408 | pressure = curr_reg_pressure[cl] + delta; | |
2409 | ||
2410 | if (pressure > group->limits[pci].pressure) | |
2411 | return model_spill_cost (cl, group->limits[pci].orig_pressure, | |
2412 | pressure); | |
2413 | } | |
2414 | ||
2415 | return 0; | |
2416 | } | |
2417 | ||
2418 | /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump | |
2419 | if PRINT_P. */ | |
2420 | ||
2421 | static int | |
f57aa6b0 | 2422 | model_excess_cost (rtx_insn *insn, bool print_p) |
7a1aca9c RS |
2423 | { |
2424 | int point, pci, cl, cost, this_cost, delta; | |
2425 | struct reg_pressure_data *insn_reg_pressure; | |
2426 | int insn_death[N_REG_CLASSES]; | |
2427 | ||
2428 | calculate_reg_deaths (insn, insn_death); | |
2429 | point = model_index (insn); | |
2430 | insn_reg_pressure = INSN_REG_PRESSURE (insn); | |
2431 | cost = 0; | |
2432 | ||
2433 | if (print_p) | |
2434 | fprintf (sched_dump, ";;\t\t| %3d %4d | %4d %+3d |", point, | |
2435 | INSN_UID (insn), INSN_PRIORITY (insn), insn_delay (insn)); | |
2436 | ||
2437 | /* Sum up the individual costs for each register class. */ | |
2438 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2439 | { | |
2440 | cl = ira_pressure_classes[pci]; | |
2441 | delta = insn_reg_pressure[pci].set_increase - insn_death[cl]; | |
2442 | this_cost = model_excess_group_cost (&model_before_pressure, | |
2443 | point, pci, delta); | |
2444 | cost += this_cost; | |
2445 | if (print_p) | |
2446 | fprintf (sched_dump, " %s:[%d base cost %d]", | |
2447 | reg_class_names[cl], delta, this_cost); | |
2448 | } | |
2449 | ||
2450 | if (print_p) | |
2451 | fprintf (sched_dump, "\n"); | |
2452 | ||
2453 | return cost; | |
2454 | } | |
2455 | ||
2456 | /* Dump the next points of maximum pressure for GROUP. */ | |
2457 | ||
2458 | static void | |
2459 | model_dump_pressure_points (struct model_pressure_group *group) | |
2460 | { | |
2461 | int pci, cl; | |
2462 | ||
2463 | fprintf (sched_dump, ";;\t\t| pressure points"); | |
2464 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
2465 | { | |
2466 | cl = ira_pressure_classes[pci]; | |
2467 | fprintf (sched_dump, " %s:[%d->%d at ", reg_class_names[cl], | |
2468 | curr_reg_pressure[cl], group->limits[pci].pressure); | |
2469 | if (group->limits[pci].point < model_num_insns) | |
2470 | fprintf (sched_dump, "%d:%d]", group->limits[pci].point, | |
2471 | INSN_UID (MODEL_INSN (group->limits[pci].point))); | |
2472 | else | |
2473 | fprintf (sched_dump, "end]"); | |
2474 | } | |
2475 | fprintf (sched_dump, "\n"); | |
2476 | } | |
2477 | ||
2478 | /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */ | |
2479 | ||
2480 | static void | |
ce1ce33a | 2481 | model_set_excess_costs (rtx_insn **insns, int count) |
7a1aca9c RS |
2482 | { |
2483 | int i, cost, priority_base, priority; | |
2484 | bool print_p; | |
2485 | ||
2486 | /* Record the baseECC value for each instruction in the model schedule, | |
4dd9ac6c | 2487 | except that negative costs are converted to zero ones now rather than |
7a1aca9c RS |
2488 | later. Do not assign a cost to debug instructions, since they must |
2489 | not change code-generation decisions. Experiments suggest we also | |
2490 | get better results by not assigning a cost to instructions from | |
2491 | a different block. | |
2492 | ||
2493 | Set PRIORITY_BASE to baseP in the block comment above. This is the | |
2494 | maximum priority of the "cheap" instructions, which should always | |
2495 | include the next model instruction. */ | |
2496 | priority_base = 0; | |
2497 | print_p = false; | |
2498 | for (i = 0; i < count; i++) | |
2499 | if (INSN_MODEL_INDEX (insns[i])) | |
2500 | { | |
2501 | if (sched_verbose >= 6 && !print_p) | |
2502 | { | |
2503 | fprintf (sched_dump, MODEL_BAR); | |
2504 | fprintf (sched_dump, ";;\t\t| Pressure costs for ready queue\n"); | |
2505 | model_dump_pressure_points (&model_before_pressure); | |
2506 | fprintf (sched_dump, MODEL_BAR); | |
2507 | print_p = true; | |
2508 | } | |
2509 | cost = model_excess_cost (insns[i], print_p); | |
2510 | if (cost <= 0) | |
2511 | { | |
2512 | priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]) - cost; | |
2513 | priority_base = MAX (priority_base, priority); | |
2514 | cost = 0; | |
2515 | } | |
2516 | INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = cost; | |
2517 | } | |
2518 | if (print_p) | |
2519 | fprintf (sched_dump, MODEL_BAR); | |
2520 | ||
2521 | /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each | |
2522 | instruction. */ | |
2523 | for (i = 0; i < count; i++) | |
2524 | { | |
2525 | cost = INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]); | |
2526 | priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]); | |
2527 | if (cost > 0 && priority > priority_base) | |
2528 | { | |
2529 | cost += priority_base - priority; | |
2530 | INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = MAX (cost, 0); | |
2531 | } | |
2532 | } | |
2533 | } | |
2534 | \f | |
4bd12f3d MK |
2535 | |
2536 | /* Enum of rank_for_schedule heuristic decisions. */ | |
2537 | enum rfs_decision { | |
696d5fa1 | 2538 | RFS_LIVE_RANGE_SHRINK1, RFS_LIVE_RANGE_SHRINK2, |
4bd12f3d MK |
2539 | RFS_SCHED_GROUP, RFS_PRESSURE_DELAY, RFS_PRESSURE_TICK, |
2540 | RFS_FEEDS_BACKTRACK_INSN, RFS_PRIORITY, RFS_SPECULATION, | |
2541 | RFS_SCHED_RANK, RFS_LAST_INSN, RFS_PRESSURE_INDEX, | |
b16abbcb | 2542 | RFS_DEP_COUNT, RFS_TIE, RFS_FUSION, RFS_N }; |
4bd12f3d MK |
2543 | |
2544 | /* Corresponding strings for print outs. */ | |
2545 | static const char *rfs_str[RFS_N] = { | |
696d5fa1 | 2546 | "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2", |
4bd12f3d MK |
2547 | "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK", |
2548 | "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION", | |
2549 | "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX", | |
b16abbcb | 2550 | "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION" }; |
4bd12f3d MK |
2551 | |
2552 | /* Statistical breakdown of rank_for_schedule decisions. */ | |
50686850 | 2553 | struct rank_for_schedule_stats_t { unsigned stats[RFS_N]; }; |
4bd12f3d MK |
2554 | static rank_for_schedule_stats_t rank_for_schedule_stats; |
2555 | ||
b75ae7f4 MK |
2556 | /* Return the result of comparing insns TMP and TMP2 and update |
2557 | Rank_For_Schedule statistics. */ | |
4bd12f3d | 2558 | static int |
b75ae7f4 | 2559 | rfs_result (enum rfs_decision decision, int result, rtx tmp, rtx tmp2) |
4bd12f3d MK |
2560 | { |
2561 | ++rank_for_schedule_stats.stats[decision]; | |
b75ae7f4 MK |
2562 | if (result < 0) |
2563 | INSN_LAST_RFS_WIN (tmp) = decision; | |
2564 | else if (result > 0) | |
2565 | INSN_LAST_RFS_WIN (tmp2) = decision; | |
2566 | else | |
2567 | gcc_unreachable (); | |
4bd12f3d MK |
2568 | return result; |
2569 | } | |
2570 | ||
63387a85 MK |
2571 | /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while |
2572 | keeping normal insns in original order. */ | |
2573 | ||
2574 | static int | |
2575 | rank_for_schedule_debug (const void *x, const void *y) | |
2576 | { | |
2577 | rtx_insn *tmp = *(rtx_insn * const *) y; | |
2578 | rtx_insn *tmp2 = *(rtx_insn * const *) x; | |
2579 | ||
2580 | /* Schedule debug insns as early as possible. */ | |
2581 | if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2)) | |
696d5fa1 | 2582 | return -1; |
63387a85 | 2583 | else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2)) |
696d5fa1 | 2584 | return 1; |
63387a85 | 2585 | else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2)) |
696d5fa1 | 2586 | return INSN_LUID (tmp) - INSN_LUID (tmp2); |
63387a85 MK |
2587 | else |
2588 | return INSN_RFS_DEBUG_ORIG_ORDER (tmp2) - INSN_RFS_DEBUG_ORIG_ORDER (tmp); | |
2589 | } | |
2590 | ||
8c660648 JL |
2591 | /* Returns a positive value if x is preferred; returns a negative value if |
2592 | y is preferred. Should never return 0, since that will make the sort | |
2593 | unstable. */ | |
2594 | ||
2595 | static int | |
1d088dee | 2596 | rank_for_schedule (const void *x, const void *y) |
8c660648 | 2597 | { |
ce1ce33a DM |
2598 | rtx_insn *tmp = *(rtx_insn * const *) y; |
2599 | rtx_insn *tmp2 = *(rtx_insn * const *) x; | |
b198261f | 2600 | int tmp_class, tmp2_class; |
f20f2613 | 2601 | int val, priority_val, info_val, diff; |
8c660648 | 2602 | |
f20f2613 VM |
2603 | if (live_range_shrinkage_p) |
2604 | { | |
2605 | /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse | |
2606 | code. */ | |
2607 | gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED); | |
2608 | if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) < 0 | |
2609 | || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) < 0) | |
2610 | && (diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) | |
2611 | - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2))) != 0) | |
b75ae7f4 | 2612 | return rfs_result (RFS_LIVE_RANGE_SHRINK1, diff, tmp, tmp2); |
f20f2613 VM |
2613 | /* Sort by INSN_LUID (original insn order), so that we make the |
2614 | sort stable. This minimizes instruction movement, thus | |
2615 | minimizing sched's effect on debugging and cross-jumping. */ | |
4bd12f3d | 2616 | return rfs_result (RFS_LIVE_RANGE_SHRINK2, |
b75ae7f4 | 2617 | INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2); |
f20f2613 VM |
2618 | } |
2619 | ||
58fb7809 | 2620 | /* The insn in a schedule group should be issued the first. */ |
b8698a0f | 2621 | if (flag_sched_group_heuristic && |
ee4764a8 | 2622 | SCHED_GROUP_P (tmp) != SCHED_GROUP_P (tmp2)) |
b75ae7f4 MK |
2623 | return rfs_result (RFS_SCHED_GROUP, SCHED_GROUP_P (tmp2) ? 1 : -1, |
2624 | tmp, tmp2); | |
58fb7809 | 2625 | |
916fa4f0 MK |
2626 | /* Make sure that priority of TMP and TMP2 are initialized. */ |
2627 | gcc_assert (INSN_PRIORITY_KNOWN (tmp) && INSN_PRIORITY_KNOWN (tmp2)); | |
2628 | ||
b16abbcb BC |
2629 | if (sched_fusion) |
2630 | { | |
2631 | /* The instruction that has the same fusion priority as the last | |
2632 | instruction is the instruction we picked next. If that is not | |
2633 | the case, we sort ready list firstly by fusion priority, then | |
2634 | by priority, and at last by INSN_LUID. */ | |
2635 | int a = INSN_FUSION_PRIORITY (tmp); | |
2636 | int b = INSN_FUSION_PRIORITY (tmp2); | |
2637 | int last = -1; | |
2638 | ||
2639 | if (last_nondebug_scheduled_insn | |
2640 | && !NOTE_P (last_nondebug_scheduled_insn) | |
2641 | && BLOCK_FOR_INSN (tmp) | |
2642 | == BLOCK_FOR_INSN (last_nondebug_scheduled_insn)) | |
2643 | last = INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn); | |
2644 | ||
2645 | if (a != last && b != last) | |
2646 | { | |
2647 | if (a == b) | |
2648 | { | |
2649 | a = INSN_PRIORITY (tmp); | |
2650 | b = INSN_PRIORITY (tmp2); | |
2651 | } | |
2652 | if (a != b) | |
2653 | return rfs_result (RFS_FUSION, b - a, tmp, tmp2); | |
2654 | else | |
2655 | return rfs_result (RFS_FUSION, | |
2656 | INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2); | |
2657 | } | |
2658 | else if (a == b) | |
2659 | { | |
2660 | gcc_assert (last_nondebug_scheduled_insn | |
2661 | && !NOTE_P (last_nondebug_scheduled_insn)); | |
2662 | last = INSN_PRIORITY (last_nondebug_scheduled_insn); | |
2663 | ||
2664 | a = abs (INSN_PRIORITY (tmp) - last); | |
2665 | b = abs (INSN_PRIORITY (tmp2) - last); | |
2666 | if (a != b) | |
2667 | return rfs_result (RFS_FUSION, a - b, tmp, tmp2); | |
2668 | else | |
2669 | return rfs_result (RFS_FUSION, | |
2670 | INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2); | |
2671 | } | |
2672 | else if (a == last) | |
2673 | return rfs_result (RFS_FUSION, -1, tmp, tmp2); | |
2674 | else | |
2675 | return rfs_result (RFS_FUSION, 1, tmp, tmp2); | |
2676 | } | |
2677 | ||
7a1aca9c | 2678 | if (sched_pressure != SCHED_PRESSURE_NONE) |
ce18efcb | 2679 | { |
ce18efcb VM |
2680 | /* Prefer insn whose scheduling results in the smallest register |
2681 | pressure excess. */ | |
2682 | if ((diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) | |
7a1aca9c | 2683 | + insn_delay (tmp) |
ce18efcb | 2684 | - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) |
7a1aca9c | 2685 | - insn_delay (tmp2)))) |
b75ae7f4 | 2686 | return rfs_result (RFS_PRESSURE_DELAY, diff, tmp, tmp2); |
ce18efcb VM |
2687 | } |
2688 | ||
7a1aca9c | 2689 | if (sched_pressure != SCHED_PRESSURE_NONE |
88366b18 MK |
2690 | && (INSN_TICK (tmp2) > clock_var || INSN_TICK (tmp) > clock_var) |
2691 | && INSN_TICK (tmp2) != INSN_TICK (tmp)) | |
ce18efcb | 2692 | { |
88366b18 | 2693 | diff = INSN_TICK (tmp) - INSN_TICK (tmp2); |
b75ae7f4 | 2694 | return rfs_result (RFS_PRESSURE_TICK, diff, tmp, tmp2); |
ce18efcb | 2695 | } |
26965010 BS |
2696 | |
2697 | /* If we are doing backtracking in this schedule, prefer insns that | |
2698 | have forward dependencies with negative cost against an insn that | |
2699 | was already scheduled. */ | |
2700 | if (current_sched_info->flags & DO_BACKTRACKING) | |
2701 | { | |
2702 | priority_val = FEEDS_BACKTRACK_INSN (tmp2) - FEEDS_BACKTRACK_INSN (tmp); | |
2703 | if (priority_val) | |
b75ae7f4 | 2704 | return rfs_result (RFS_FEEDS_BACKTRACK_INSN, priority_val, tmp, tmp2); |
26965010 BS |
2705 | } |
2706 | ||
63de6c74 | 2707 | /* Prefer insn with higher priority. */ |
8c660648 | 2708 | priority_val = INSN_PRIORITY (tmp2) - INSN_PRIORITY (tmp); |
30028c85 | 2709 | |
ee4764a8 | 2710 | if (flag_sched_critical_path_heuristic && priority_val) |
b75ae7f4 | 2711 | return rfs_result (RFS_PRIORITY, priority_val, tmp, tmp2); |
b8698a0f | 2712 | |
340c7904 MK |
2713 | if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) >= 0) |
2714 | { | |
2715 | int autopref = autopref_rank_for_schedule (tmp, tmp2); | |
2716 | if (autopref != 0) | |
2717 | return autopref; | |
2718 | } | |
2719 | ||
496d7bb0 | 2720 | /* Prefer speculative insn with greater dependencies weakness. */ |
ee4764a8 | 2721 | if (flag_sched_spec_insn_heuristic && spec_info) |
496d7bb0 MK |
2722 | { |
2723 | ds_t ds1, ds2; | |
2724 | dw_t dw1, dw2; | |
2725 | int dw; | |
2726 | ||
2727 | ds1 = TODO_SPEC (tmp) & SPECULATIVE; | |
2728 | if (ds1) | |
e855c69d | 2729 | dw1 = ds_weak (ds1); |
496d7bb0 MK |
2730 | else |
2731 | dw1 = NO_DEP_WEAK; | |
b8698a0f | 2732 | |
496d7bb0 MK |
2733 | ds2 = TODO_SPEC (tmp2) & SPECULATIVE; |
2734 | if (ds2) | |
e855c69d | 2735 | dw2 = ds_weak (ds2); |
496d7bb0 MK |
2736 | else |
2737 | dw2 = NO_DEP_WEAK; | |
2738 | ||
2739 | dw = dw2 - dw1; | |
2740 | if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8)) | |
b75ae7f4 | 2741 | return rfs_result (RFS_SPECULATION, dw, tmp, tmp2); |
496d7bb0 MK |
2742 | } |
2743 | ||
1708fd40 | 2744 | info_val = (*current_sched_info->rank) (tmp, tmp2); |
c3284718 | 2745 | if (flag_sched_rank_heuristic && info_val) |
b75ae7f4 | 2746 | return rfs_result (RFS_SCHED_RANK, info_val, tmp, tmp2); |
8c660648 | 2747 | |
b5b8b0ac AO |
2748 | /* Compare insns based on their relation to the last scheduled |
2749 | non-debug insn. */ | |
5c24671b | 2750 | if (flag_sched_last_insn_heuristic && last_nondebug_scheduled_insn) |
8c660648 | 2751 | { |
e2f6ff94 MK |
2752 | dep_t dep1; |
2753 | dep_t dep2; | |
90831096 | 2754 | rtx_insn *last = last_nondebug_scheduled_insn; |
e2f6ff94 | 2755 | |
8c660648 JL |
2756 | /* Classify the instructions into three classes: |
2757 | 1) Data dependent on last schedule insn. | |
2758 | 2) Anti/Output dependent on last scheduled insn. | |
2759 | 3) Independent of last scheduled insn, or has latency of one. | |
2760 | Choose the insn from the highest numbered class if different. */ | |
b5b8b0ac | 2761 | dep1 = sd_find_dep_between (last, tmp, true); |
b198261f | 2762 | |
e2f6ff94 | 2763 | if (dep1 == NULL || dep_cost (dep1) == 1) |
8c660648 | 2764 | tmp_class = 3; |
b198261f | 2765 | else if (/* Data dependence. */ |
e2f6ff94 | 2766 | DEP_TYPE (dep1) == REG_DEP_TRUE) |
8c660648 JL |
2767 | tmp_class = 1; |
2768 | else | |
2769 | tmp_class = 2; | |
2770 | ||
b5b8b0ac | 2771 | dep2 = sd_find_dep_between (last, tmp2, true); |
b198261f | 2772 | |
e2f6ff94 | 2773 | if (dep2 == NULL || dep_cost (dep2) == 1) |
8c660648 | 2774 | tmp2_class = 3; |
b198261f | 2775 | else if (/* Data dependence. */ |
e2f6ff94 | 2776 | DEP_TYPE (dep2) == REG_DEP_TRUE) |
8c660648 JL |
2777 | tmp2_class = 1; |
2778 | else | |
2779 | tmp2_class = 2; | |
2780 | ||
2781 | if ((val = tmp2_class - tmp_class)) | |
b75ae7f4 | 2782 | return rfs_result (RFS_LAST_INSN, val, tmp, tmp2); |
8c660648 JL |
2783 | } |
2784 | ||
7a1aca9c | 2785 | /* Prefer instructions that occur earlier in the model schedule. */ |
88366b18 MK |
2786 | if (sched_pressure == SCHED_PRESSURE_MODEL |
2787 | && INSN_BB (tmp) == target_bb && INSN_BB (tmp2) == target_bb) | |
7a1aca9c | 2788 | { |
7a1aca9c | 2789 | diff = model_index (tmp) - model_index (tmp2); |
4bd12f3d | 2790 | gcc_assert (diff != 0); |
b75ae7f4 | 2791 | return rfs_result (RFS_PRESSURE_INDEX, diff, tmp, tmp2); |
7a1aca9c RS |
2792 | } |
2793 | ||
7a403706 | 2794 | /* Prefer the insn which has more later insns that depend on it. |
2db45993 JL |
2795 | This gives the scheduler more freedom when scheduling later |
2796 | instructions at the expense of added register pressure. */ | |
2db45993 | 2797 | |
7a1aca9c RS |
2798 | val = (dep_list_size (tmp2, SD_LIST_FORW) |
2799 | - dep_list_size (tmp, SD_LIST_FORW)); | |
2db45993 | 2800 | |
ee4764a8 | 2801 | if (flag_sched_dep_count_heuristic && val != 0) |
b75ae7f4 | 2802 | return rfs_result (RFS_DEP_COUNT, val, tmp, tmp2); |
7a403706 | 2803 | |
8c660648 JL |
2804 | /* If insns are equally good, sort by INSN_LUID (original insn order), |
2805 | so that we make the sort stable. This minimizes instruction movement, | |
2806 | thus minimizing sched's effect on debugging and cross-jumping. */ | |
b75ae7f4 | 2807 | return rfs_result (RFS_TIE, INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2); |
8c660648 JL |
2808 | } |
2809 | ||
2810 | /* Resort the array A in which only element at index N may be out of order. */ | |
2811 | ||
cbb13457 | 2812 | HAIFA_INLINE static void |
ce1ce33a | 2813 | swap_sort (rtx_insn **a, int n) |
8c660648 | 2814 | { |
ce1ce33a | 2815 | rtx_insn *insn = a[n - 1]; |
8c660648 JL |
2816 | int i = n - 2; |
2817 | ||
2818 | while (i >= 0 && rank_for_schedule (a + i, &insn) >= 0) | |
2819 | { | |
2820 | a[i + 1] = a[i]; | |
2821 | i -= 1; | |
2822 | } | |
2823 | a[i + 1] = insn; | |
2824 | } | |
2825 | ||
8c660648 JL |
2826 | /* Add INSN to the insn queue so that it can be executed at least |
2827 | N_CYCLES after the currently executing insn. Preserve insns | |
ec4efea9 BS |
2828 | chain for debugging purposes. REASON will be printed in debugging |
2829 | output. */ | |
8c660648 | 2830 | |
cbb13457 | 2831 | HAIFA_INLINE static void |
ce1ce33a | 2832 | queue_insn (rtx_insn *insn, int n_cycles, const char *reason) |
8c660648 JL |
2833 | { |
2834 | int next_q = NEXT_Q_AFTER (q_ptr, n_cycles); | |
3dc99c19 | 2835 | rtx_insn_list *link = alloc_INSN_LIST (insn, insn_queue[next_q]); |
26965010 | 2836 | int new_tick; |
63f54b1a MK |
2837 | |
2838 | gcc_assert (n_cycles <= max_insn_queue_index); | |
b5b8b0ac | 2839 | gcc_assert (!DEBUG_INSN_P (insn)); |
63f54b1a | 2840 | |
8c660648 JL |
2841 | insn_queue[next_q] = link; |
2842 | q_size += 1; | |
2843 | ||
2844 | if (sched_verbose >= 2) | |
2845 | { | |
1708fd40 BS |
2846 | fprintf (sched_dump, ";;\t\tReady-->Q: insn %s: ", |
2847 | (*current_sched_info->print_insn) (insn, 0)); | |
8c660648 | 2848 | |
ec4efea9 | 2849 | fprintf (sched_dump, "queued for %d cycles (%s).\n", n_cycles, reason); |
8c660648 | 2850 | } |
e855c69d | 2851 | |
63f54b1a | 2852 | QUEUE_INDEX (insn) = next_q; |
26965010 BS |
2853 | |
2854 | if (current_sched_info->flags & DO_BACKTRACKING) | |
2855 | { | |
2856 | new_tick = clock_var + n_cycles; | |
2857 | if (INSN_TICK (insn) == INVALID_TICK || INSN_TICK (insn) < new_tick) | |
2858 | INSN_TICK (insn) = new_tick; | |
2859 | ||
2860 | if (INSN_EXACT_TICK (insn) != INVALID_TICK | |
2861 | && INSN_EXACT_TICK (insn) < clock_var + n_cycles) | |
2862 | { | |
2863 | must_backtrack = true; | |
2864 | if (sched_verbose >= 2) | |
2865 | fprintf (sched_dump, ";;\t\tcausing a backtrack.\n"); | |
2866 | } | |
2867 | } | |
63f54b1a MK |
2868 | } |
2869 | ||
2870 | /* Remove INSN from queue. */ | |
2871 | static void | |
f57aa6b0 | 2872 | queue_remove (rtx_insn *insn) |
63f54b1a MK |
2873 | { |
2874 | gcc_assert (QUEUE_INDEX (insn) >= 0); | |
2875 | remove_free_INSN_LIST_elem (insn, &insn_queue[QUEUE_INDEX (insn)]); | |
2876 | q_size--; | |
2877 | QUEUE_INDEX (insn) = QUEUE_NOWHERE; | |
176f9a7b BS |
2878 | } |
2879 | ||
2880 | /* Return a pointer to the bottom of the ready list, i.e. the insn | |
2881 | with the lowest priority. */ | |
2882 | ||
ce1ce33a | 2883 | rtx_insn ** |
1d088dee | 2884 | ready_lastpos (struct ready_list *ready) |
176f9a7b | 2885 | { |
63f54b1a | 2886 | gcc_assert (ready->n_ready >= 1); |
176f9a7b BS |
2887 | return ready->vec + ready->first - ready->n_ready + 1; |
2888 | } | |
2889 | ||
63f54b1a | 2890 | /* Add an element INSN to the ready list so that it ends up with the |
917f1b7e | 2891 | lowest/highest priority depending on FIRST_P. */ |
176f9a7b | 2892 | |
63f54b1a | 2893 | HAIFA_INLINE static void |
ce1ce33a | 2894 | ready_add (struct ready_list *ready, rtx_insn *insn, bool first_p) |
176f9a7b | 2895 | { |
63f54b1a | 2896 | if (!first_p) |
176f9a7b | 2897 | { |
63f54b1a MK |
2898 | if (ready->first == ready->n_ready) |
2899 | { | |
2900 | memmove (ready->vec + ready->veclen - ready->n_ready, | |
2901 | ready_lastpos (ready), | |
2902 | ready->n_ready * sizeof (rtx)); | |
2903 | ready->first = ready->veclen - 1; | |
2904 | } | |
2905 | ready->vec[ready->first - ready->n_ready] = insn; | |
176f9a7b | 2906 | } |
63f54b1a MK |
2907 | else |
2908 | { | |
2909 | if (ready->first == ready->veclen - 1) | |
2910 | { | |
2911 | if (ready->n_ready) | |
2912 | /* ready_lastpos() fails when called with (ready->n_ready == 0). */ | |
2913 | memmove (ready->vec + ready->veclen - ready->n_ready - 1, | |
2914 | ready_lastpos (ready), | |
2915 | ready->n_ready * sizeof (rtx)); | |
2916 | ready->first = ready->veclen - 2; | |
2917 | } | |
2918 | ready->vec[++(ready->first)] = insn; | |
2919 | } | |
2920 | ||
176f9a7b | 2921 | ready->n_ready++; |
b5b8b0ac AO |
2922 | if (DEBUG_INSN_P (insn)) |
2923 | ready->n_debug++; | |
63f54b1a MK |
2924 | |
2925 | gcc_assert (QUEUE_INDEX (insn) != QUEUE_READY); | |
2926 | QUEUE_INDEX (insn) = QUEUE_READY; | |
26965010 BS |
2927 | |
2928 | if (INSN_EXACT_TICK (insn) != INVALID_TICK | |
2929 | && INSN_EXACT_TICK (insn) < clock_var) | |
2930 | { | |
2931 | must_backtrack = true; | |
2932 | } | |
176f9a7b | 2933 | } |
8c660648 | 2934 | |
176f9a7b BS |
2935 | /* Remove the element with the highest priority from the ready list and |
2936 | return it. */ | |
2937 | ||
ce1ce33a | 2938 | HAIFA_INLINE static rtx_insn * |
1d088dee | 2939 | ready_remove_first (struct ready_list *ready) |
176f9a7b | 2940 | { |
ce1ce33a | 2941 | rtx_insn *t; |
b8698a0f | 2942 | |
535a42b1 | 2943 | gcc_assert (ready->n_ready); |
176f9a7b BS |
2944 | t = ready->vec[ready->first--]; |
2945 | ready->n_ready--; | |
b5b8b0ac AO |
2946 | if (DEBUG_INSN_P (t)) |
2947 | ready->n_debug--; | |
176f9a7b BS |
2948 | /* If the queue becomes empty, reset it. */ |
2949 | if (ready->n_ready == 0) | |
2950 | ready->first = ready->veclen - 1; | |
63f54b1a MK |
2951 | |
2952 | gcc_assert (QUEUE_INDEX (t) == QUEUE_READY); | |
2953 | QUEUE_INDEX (t) = QUEUE_NOWHERE; | |
2954 | ||
176f9a7b BS |
2955 | return t; |
2956 | } | |
2957 | ||
fae15c93 VM |
2958 | /* The following code implements multi-pass scheduling for the first |
2959 | cycle. In other words, we will try to choose ready insn which | |
2960 | permits to start maximum number of insns on the same cycle. */ | |
2961 | ||
2962 | /* Return a pointer to the element INDEX from the ready. INDEX for | |
2963 | insn with the highest priority is 0, and the lowest priority has | |
2964 | N_READY - 1. */ | |
2965 | ||
ce1ce33a | 2966 | rtx_insn * |
1d088dee | 2967 | ready_element (struct ready_list *ready, int index) |
fae15c93 | 2968 | { |
535a42b1 | 2969 | gcc_assert (ready->n_ready && index < ready->n_ready); |
b8698a0f | 2970 | |
fae15c93 VM |
2971 | return ready->vec[ready->first - index]; |
2972 | } | |
2973 | ||
2974 | /* Remove the element INDEX from the ready list and return it. INDEX | |
2975 | for insn with the highest priority is 0, and the lowest priority | |
2976 | has N_READY - 1. */ | |
2977 | ||
ce1ce33a | 2978 | HAIFA_INLINE static rtx_insn * |
1d088dee | 2979 | ready_remove (struct ready_list *ready, int index) |
fae15c93 | 2980 | { |
ce1ce33a | 2981 | rtx_insn *t; |
fae15c93 VM |
2982 | int i; |
2983 | ||
2984 | if (index == 0) | |
2985 | return ready_remove_first (ready); | |
535a42b1 | 2986 | gcc_assert (ready->n_ready && index < ready->n_ready); |
fae15c93 VM |
2987 | t = ready->vec[ready->first - index]; |
2988 | ready->n_ready--; | |
b5b8b0ac AO |
2989 | if (DEBUG_INSN_P (t)) |
2990 | ready->n_debug--; | |
fae15c93 VM |
2991 | for (i = index; i < ready->n_ready; i++) |
2992 | ready->vec[ready->first - i] = ready->vec[ready->first - i - 1]; | |
63f54b1a | 2993 | QUEUE_INDEX (t) = QUEUE_NOWHERE; |
fae15c93 VM |
2994 | return t; |
2995 | } | |
2996 | ||
63f54b1a MK |
2997 | /* Remove INSN from the ready list. */ |
2998 | static void | |
90831096 | 2999 | ready_remove_insn (rtx_insn *insn) |
63f54b1a MK |
3000 | { |
3001 | int i; | |
3002 | ||
3003 | for (i = 0; i < readyp->n_ready; i++) | |
3004 | if (ready_element (readyp, i) == insn) | |
3005 | { | |
3006 | ready_remove (readyp, i); | |
3007 | return; | |
3008 | } | |
3009 | gcc_unreachable (); | |
3010 | } | |
fae15c93 | 3011 | |
4bd12f3d MK |
3012 | /* Calculate difference of two statistics set WAS and NOW. |
3013 | Result returned in WAS. */ | |
3014 | static void | |
3015 | rank_for_schedule_stats_diff (rank_for_schedule_stats_t *was, | |
3016 | const rank_for_schedule_stats_t *now) | |
3017 | { | |
3018 | for (int i = 0; i < RFS_N; ++i) | |
3019 | was->stats[i] = now->stats[i] - was->stats[i]; | |
3020 | } | |
3021 | ||
3022 | /* Print rank_for_schedule statistics. */ | |
3023 | static void | |
3024 | print_rank_for_schedule_stats (const char *prefix, | |
b75ae7f4 MK |
3025 | const rank_for_schedule_stats_t *stats, |
3026 | struct ready_list *ready) | |
4bd12f3d MK |
3027 | { |
3028 | for (int i = 0; i < RFS_N; ++i) | |
3029 | if (stats->stats[i]) | |
b75ae7f4 MK |
3030 | { |
3031 | fprintf (sched_dump, "%s%20s: %u", prefix, rfs_str[i], stats->stats[i]); | |
3032 | ||
3033 | if (ready != NULL) | |
3034 | /* Print out insns that won due to RFS_<I>. */ | |
3035 | { | |
3036 | rtx_insn **p = ready_lastpos (ready); | |
3037 | ||
3038 | fprintf (sched_dump, ":"); | |
3039 | /* Start with 1 since least-priority insn didn't have any wins. */ | |
3040 | for (int j = 1; j < ready->n_ready; ++j) | |
3041 | if (INSN_LAST_RFS_WIN (p[j]) == i) | |
3042 | fprintf (sched_dump, " %s", | |
3043 | (*current_sched_info->print_insn) (p[j], 0)); | |
3044 | } | |
3045 | fprintf (sched_dump, "\n"); | |
3046 | } | |
4bd12f3d MK |
3047 | } |
3048 | ||
696d5fa1 MK |
3049 | /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end |
3050 | of array. */ | |
3051 | static void | |
3052 | ready_sort_debug (struct ready_list *ready) | |
176f9a7b | 3053 | { |
ce18efcb | 3054 | int i; |
ce1ce33a | 3055 | rtx_insn **first = ready_lastpos (ready); |
ce18efcb | 3056 | |
63387a85 | 3057 | for (i = 0; i < ready->n_ready; ++i) |
696d5fa1 MK |
3058 | if (!DEBUG_INSN_P (first[i])) |
3059 | INSN_RFS_DEBUG_ORIG_ORDER (first[i]) = i; | |
63387a85 | 3060 | |
696d5fa1 MK |
3061 | qsort (first, ready->n_ready, sizeof (rtx), rank_for_schedule_debug); |
3062 | } | |
63387a85 | 3063 | |
696d5fa1 MK |
3064 | /* Sort non-debug insns in the ready list READY by ascending priority. |
3065 | Assumes that all debug insns are separated from the real insns. */ | |
3066 | static void | |
3067 | ready_sort_real (struct ready_list *ready) | |
3068 | { | |
3069 | int i; | |
3070 | rtx_insn **first = ready_lastpos (ready); | |
3071 | int n_ready_real = ready->n_ready - ready->n_debug; | |
3072 | ||
3073 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED) | |
3074 | for (i = 0; i < n_ready_real; ++i) | |
3075 | setup_insn_reg_pressure_info (first[i]); | |
3076 | else if (sched_pressure == SCHED_PRESSURE_MODEL | |
3077 | && model_curr_point < model_num_insns) | |
3078 | model_set_excess_costs (first, n_ready_real); | |
4bd12f3d MK |
3079 | |
3080 | rank_for_schedule_stats_t stats1; | |
3081 | if (sched_verbose >= 4) | |
3082 | stats1 = rank_for_schedule_stats; | |
3083 | ||
696d5fa1 MK |
3084 | if (n_ready_real == 2) |
3085 | swap_sort (first, n_ready_real); | |
3086 | else if (n_ready_real > 2) | |
f9f43fb3 | 3087 | qsort (first, n_ready_real, sizeof (rtx), rank_for_schedule); |
4bd12f3d MK |
3088 | |
3089 | if (sched_verbose >= 4) | |
3090 | { | |
3091 | rank_for_schedule_stats_diff (&stats1, &rank_for_schedule_stats); | |
b75ae7f4 | 3092 | print_rank_for_schedule_stats (";;\t\t", &stats1, ready); |
4bd12f3d | 3093 | } |
8c660648 JL |
3094 | } |
3095 | ||
696d5fa1 MK |
3096 | /* Sort the ready list READY by ascending priority. */ |
3097 | static void | |
3098 | ready_sort (struct ready_list *ready) | |
3099 | { | |
3100 | if (ready->n_debug > 0) | |
3101 | ready_sort_debug (ready); | |
3102 | else | |
3103 | ready_sort_real (ready); | |
3104 | } | |
3105 | ||
8c660648 | 3106 | /* PREV is an insn that is ready to execute. Adjust its priority if that |
c46a37c4 | 3107 | will help shorten or lengthen register lifetimes as appropriate. Also |
fa10beec | 3108 | provide a hook for the target to tweak itself. */ |
8c660648 | 3109 | |
cbb13457 | 3110 | HAIFA_INLINE static void |
f57aa6b0 | 3111 | adjust_priority (rtx_insn *prev) |
8c660648 | 3112 | { |
c46a37c4 RH |
3113 | /* ??? There used to be code here to try and estimate how an insn |
3114 | affected register lifetimes, but it did it by looking at REG_DEAD | |
7a403706 | 3115 | notes, which we removed in schedule_region. Nor did it try to |
c46a37c4 | 3116 | take into account register pressure or anything useful like that. |
8c660648 | 3117 | |
c46a37c4 | 3118 | Revisit when we have a machine model to work with and not before. */ |
197043f5 | 3119 | |
c237e94a ZW |
3120 | if (targetm.sched.adjust_priority) |
3121 | INSN_PRIORITY (prev) = | |
5fd9b178 | 3122 | targetm.sched.adjust_priority (prev, INSN_PRIORITY (prev)); |
8c660648 JL |
3123 | } |
3124 | ||
e855c69d AB |
3125 | /* Advance DFA state STATE on one cycle. */ |
3126 | void | |
3127 | advance_state (state_t state) | |
fae15c93 | 3128 | { |
1c3d0d93 MK |
3129 | if (targetm.sched.dfa_pre_advance_cycle) |
3130 | targetm.sched.dfa_pre_advance_cycle (); | |
3131 | ||
fa0aee89 | 3132 | if (targetm.sched.dfa_pre_cycle_insn) |
e855c69d | 3133 | state_transition (state, |
fa0aee89 PB |
3134 | targetm.sched.dfa_pre_cycle_insn ()); |
3135 | ||
e855c69d | 3136 | state_transition (state, NULL); |
b8698a0f | 3137 | |
fa0aee89 | 3138 | if (targetm.sched.dfa_post_cycle_insn) |
e855c69d | 3139 | state_transition (state, |
fa0aee89 | 3140 | targetm.sched.dfa_post_cycle_insn ()); |
1c3d0d93 MK |
3141 | |
3142 | if (targetm.sched.dfa_post_advance_cycle) | |
3143 | targetm.sched.dfa_post_advance_cycle (); | |
fae15c93 VM |
3144 | } |
3145 | ||
e855c69d AB |
3146 | /* Advance time on one cycle. */ |
3147 | HAIFA_INLINE static void | |
3148 | advance_one_cycle (void) | |
3149 | { | |
3150 | advance_state (curr_state); | |
d5c67efd | 3151 | if (sched_verbose >= 4) |
f50d71ec | 3152 | fprintf (sched_dump, ";;\tAdvance the current state.\n"); |
e855c69d AB |
3153 | } |
3154 | ||
ce18efcb VM |
3155 | /* Update register pressure after scheduling INSN. */ |
3156 | static void | |
f57aa6b0 | 3157 | update_register_pressure (rtx_insn *insn) |
ce18efcb VM |
3158 | { |
3159 | struct reg_use_data *use; | |
3160 | struct reg_set_data *set; | |
3161 | ||
c11b0b3b AO |
3162 | gcc_checking_assert (!DEBUG_INSN_P (insn)); |
3163 | ||
ce18efcb | 3164 | for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use) |
7a1aca9c RS |
3165 | if (dying_use_p (use)) |
3166 | mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, | |
3167 | use->regno, false); | |
ce18efcb | 3168 | for (set = INSN_REG_SET_LIST (insn); set != NULL; set = set->next_insn_set) |
7a1aca9c RS |
3169 | mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, |
3170 | set->regno, true); | |
ce18efcb VM |
3171 | } |
3172 | ||
3173 | /* Set up or update (if UPDATE_P) max register pressure (see its | |
3174 | meaning in sched-int.h::_haifa_insn_data) for all current BB insns | |
3175 | after insn AFTER. */ | |
3176 | static void | |
dc01c3d1 | 3177 | setup_insn_max_reg_pressure (rtx_insn *after, bool update_p) |
ce18efcb VM |
3178 | { |
3179 | int i, p; | |
3180 | bool eq_p; | |
f57aa6b0 | 3181 | rtx_insn *insn; |
ce18efcb VM |
3182 | static int max_reg_pressure[N_REG_CLASSES]; |
3183 | ||
3184 | save_reg_pressure (); | |
1756cb66 VM |
3185 | for (i = 0; i < ira_pressure_classes_num; i++) |
3186 | max_reg_pressure[ira_pressure_classes[i]] | |
3187 | = curr_reg_pressure[ira_pressure_classes[i]]; | |
ce18efcb | 3188 | for (insn = NEXT_INSN (after); |
0495bafc VM |
3189 | insn != NULL_RTX && ! BARRIER_P (insn) |
3190 | && BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (after); | |
ce18efcb VM |
3191 | insn = NEXT_INSN (insn)) |
3192 | if (NONDEBUG_INSN_P (insn)) | |
3193 | { | |
3194 | eq_p = true; | |
1756cb66 | 3195 | for (i = 0; i < ira_pressure_classes_num; i++) |
ce18efcb | 3196 | { |
1756cb66 | 3197 | p = max_reg_pressure[ira_pressure_classes[i]]; |
ce18efcb VM |
3198 | if (INSN_MAX_REG_PRESSURE (insn)[i] != p) |
3199 | { | |
3200 | eq_p = false; | |
3201 | INSN_MAX_REG_PRESSURE (insn)[i] | |
1756cb66 | 3202 | = max_reg_pressure[ira_pressure_classes[i]]; |
ce18efcb VM |
3203 | } |
3204 | } | |
3205 | if (update_p && eq_p) | |
3206 | break; | |
3207 | update_register_pressure (insn); | |
1756cb66 VM |
3208 | for (i = 0; i < ira_pressure_classes_num; i++) |
3209 | if (max_reg_pressure[ira_pressure_classes[i]] | |
3210 | < curr_reg_pressure[ira_pressure_classes[i]]) | |
3211 | max_reg_pressure[ira_pressure_classes[i]] | |
3212 | = curr_reg_pressure[ira_pressure_classes[i]]; | |
ce18efcb VM |
3213 | } |
3214 | restore_reg_pressure (); | |
3215 | } | |
3216 | ||
3217 | /* Update the current register pressure after scheduling INSN. Update | |
3218 | also max register pressure for unscheduled insns of the current | |
3219 | BB. */ | |
3220 | static void | |
f57aa6b0 | 3221 | update_reg_and_insn_max_reg_pressure (rtx_insn *insn) |
ce18efcb VM |
3222 | { |
3223 | int i; | |
3224 | int before[N_REG_CLASSES]; | |
3225 | ||
1756cb66 VM |
3226 | for (i = 0; i < ira_pressure_classes_num; i++) |
3227 | before[i] = curr_reg_pressure[ira_pressure_classes[i]]; | |
ce18efcb | 3228 | update_register_pressure (insn); |
1756cb66 VM |
3229 | for (i = 0; i < ira_pressure_classes_num; i++) |
3230 | if (curr_reg_pressure[ira_pressure_classes[i]] != before[i]) | |
ce18efcb | 3231 | break; |
1756cb66 | 3232 | if (i < ira_pressure_classes_num) |
ce18efcb VM |
3233 | setup_insn_max_reg_pressure (insn, true); |
3234 | } | |
3235 | ||
3236 | /* Set up register pressure at the beginning of basic block BB whose | |
3237 | insns starting after insn AFTER. Set up also max register pressure | |
3238 | for all insns of the basic block. */ | |
3239 | void | |
dc01c3d1 | 3240 | sched_setup_bb_reg_pressure_info (basic_block bb, rtx_insn *after) |
ce18efcb | 3241 | { |
60867e8c | 3242 | gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED); |
ce18efcb VM |
3243 | initiate_bb_reg_pressure_info (bb); |
3244 | setup_insn_max_reg_pressure (after, false); | |
3245 | } | |
51564684 | 3246 | \f |
e2724e63 BS |
3247 | /* If doing predication while scheduling, verify whether INSN, which |
3248 | has just been scheduled, clobbers the conditions of any | |
3249 | instructions that must be predicated in order to break their | |
3250 | dependencies. If so, remove them from the queues so that they will | |
3251 | only be scheduled once their control dependency is resolved. */ | |
3252 | ||
3253 | static void | |
90831096 | 3254 | check_clobbered_conditions (rtx_insn *insn) |
e2724e63 BS |
3255 | { |
3256 | HARD_REG_SET t; | |
3257 | int i; | |
3258 | ||
3259 | if ((current_sched_info->flags & DO_PREDICATION) == 0) | |
3260 | return; | |
3261 | ||
356bf593 | 3262 | find_all_hard_reg_sets (insn, &t, true); |
e2724e63 BS |
3263 | |
3264 | restart: | |
3265 | for (i = 0; i < ready.n_ready; i++) | |
3266 | { | |
ce1ce33a | 3267 | rtx_insn *x = ready_element (&ready, i); |
e2724e63 BS |
3268 | if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t)) |
3269 | { | |
3270 | ready_remove_insn (x); | |
3271 | goto restart; | |
3272 | } | |
3273 | } | |
3274 | for (i = 0; i <= max_insn_queue_index; i++) | |
3275 | { | |
96b70414 | 3276 | rtx_insn_list *link; |
e2724e63 BS |
3277 | int q = NEXT_Q_AFTER (q_ptr, i); |
3278 | ||
3279 | restart_queue: | |
96b70414 | 3280 | for (link = insn_queue[q]; link; link = link->next ()) |
e2724e63 | 3281 | { |
96b70414 | 3282 | rtx_insn *x = link->insn (); |
e2724e63 BS |
3283 | if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t)) |
3284 | { | |
3285 | queue_remove (x); | |
3286 | goto restart_queue; | |
3287 | } | |
3288 | } | |
3289 | } | |
3290 | } | |
3291 | \f | |
7a1aca9c RS |
3292 | /* Return (in order): |
3293 | ||
3294 | - positive if INSN adversely affects the pressure on one | |
3295 | register class | |
3296 | ||
3297 | - negative if INSN reduces the pressure on one register class | |
3298 | ||
3299 | - 0 if INSN doesn't affect the pressure on any register class. */ | |
3300 | ||
3301 | static int | |
3302 | model_classify_pressure (struct model_insn_info *insn) | |
3303 | { | |
3304 | struct reg_pressure_data *reg_pressure; | |
3305 | int death[N_REG_CLASSES]; | |
3306 | int pci, cl, sum; | |
3307 | ||
3308 | calculate_reg_deaths (insn->insn, death); | |
3309 | reg_pressure = INSN_REG_PRESSURE (insn->insn); | |
3310 | sum = 0; | |
3311 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3312 | { | |
3313 | cl = ira_pressure_classes[pci]; | |
3314 | if (death[cl] < reg_pressure[pci].set_increase) | |
3315 | return 1; | |
3316 | sum += reg_pressure[pci].set_increase - death[cl]; | |
3317 | } | |
3318 | return sum; | |
3319 | } | |
3320 | ||
3321 | /* Return true if INSN1 should come before INSN2 in the model schedule. */ | |
3322 | ||
3323 | static int | |
3324 | model_order_p (struct model_insn_info *insn1, struct model_insn_info *insn2) | |
3325 | { | |
3326 | unsigned int height1, height2; | |
3327 | unsigned int priority1, priority2; | |
3328 | ||
3329 | /* Prefer instructions with a higher model priority. */ | |
3330 | if (insn1->model_priority != insn2->model_priority) | |
3331 | return insn1->model_priority > insn2->model_priority; | |
3332 | ||
3333 | /* Combine the length of the longest path of satisfied true dependencies | |
3334 | that leads to each instruction (depth) with the length of the longest | |
3335 | path of any dependencies that leads from the instruction (alap). | |
3336 | Prefer instructions with the greatest combined length. If the combined | |
3337 | lengths are equal, prefer instructions with the greatest depth. | |
3338 | ||
3339 | The idea is that, if we have a set S of "equal" instructions that each | |
3340 | have ALAP value X, and we pick one such instruction I, any true-dependent | |
3341 | successors of I that have ALAP value X - 1 should be preferred over S. | |
3342 | This encourages the schedule to be "narrow" rather than "wide". | |
3343 | However, if I is a low-priority instruction that we decided to | |
3344 | schedule because of its model_classify_pressure, and if there | |
3345 | is a set of higher-priority instructions T, the aforementioned | |
3346 | successors of I should not have the edge over T. */ | |
3347 | height1 = insn1->depth + insn1->alap; | |
3348 | height2 = insn2->depth + insn2->alap; | |
3349 | if (height1 != height2) | |
3350 | return height1 > height2; | |
3351 | if (insn1->depth != insn2->depth) | |
3352 | return insn1->depth > insn2->depth; | |
3353 | ||
3354 | /* We have no real preference between INSN1 an INSN2 as far as attempts | |
3355 | to reduce pressure go. Prefer instructions with higher priorities. */ | |
3356 | priority1 = INSN_PRIORITY (insn1->insn); | |
3357 | priority2 = INSN_PRIORITY (insn2->insn); | |
3358 | if (priority1 != priority2) | |
3359 | return priority1 > priority2; | |
3360 | ||
3361 | /* Use the original rtl sequence as a tie-breaker. */ | |
3362 | return insn1 < insn2; | |
3363 | } | |
3364 | ||
3365 | /* Add INSN to the model worklist immediately after PREV. Add it to the | |
3366 | beginning of the list if PREV is null. */ | |
3367 | ||
3368 | static void | |
3369 | model_add_to_worklist_at (struct model_insn_info *insn, | |
3370 | struct model_insn_info *prev) | |
3371 | { | |
3372 | gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_NOWHERE); | |
3373 | QUEUE_INDEX (insn->insn) = QUEUE_READY; | |
3374 | ||
3375 | insn->prev = prev; | |
3376 | if (prev) | |
3377 | { | |
3378 | insn->next = prev->next; | |
3379 | prev->next = insn; | |
3380 | } | |
3381 | else | |
3382 | { | |
3383 | insn->next = model_worklist; | |
3384 | model_worklist = insn; | |
3385 | } | |
3386 | if (insn->next) | |
3387 | insn->next->prev = insn; | |
3388 | } | |
3389 | ||
3390 | /* Remove INSN from the model worklist. */ | |
3391 | ||
3392 | static void | |
3393 | model_remove_from_worklist (struct model_insn_info *insn) | |
3394 | { | |
3395 | gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_READY); | |
3396 | QUEUE_INDEX (insn->insn) = QUEUE_NOWHERE; | |
3397 | ||
3398 | if (insn->prev) | |
3399 | insn->prev->next = insn->next; | |
3400 | else | |
3401 | model_worklist = insn->next; | |
3402 | if (insn->next) | |
3403 | insn->next->prev = insn->prev; | |
3404 | } | |
3405 | ||
3406 | /* Add INSN to the model worklist. Start looking for a suitable position | |
3407 | between neighbors PREV and NEXT, testing at most MAX_SCHED_READY_INSNS | |
3408 | insns either side. A null PREV indicates the beginning of the list and | |
3409 | a null NEXT indicates the end. */ | |
3410 | ||
3411 | static void | |
3412 | model_add_to_worklist (struct model_insn_info *insn, | |
3413 | struct model_insn_info *prev, | |
3414 | struct model_insn_info *next) | |
3415 | { | |
3416 | int count; | |
3417 | ||
3418 | count = MAX_SCHED_READY_INSNS; | |
3419 | if (count > 0 && prev && model_order_p (insn, prev)) | |
3420 | do | |
3421 | { | |
3422 | count--; | |
3423 | prev = prev->prev; | |
3424 | } | |
3425 | while (count > 0 && prev && model_order_p (insn, prev)); | |
3426 | else | |
3427 | while (count > 0 && next && model_order_p (next, insn)) | |
3428 | { | |
3429 | count--; | |
3430 | prev = next; | |
3431 | next = next->next; | |
3432 | } | |
3433 | model_add_to_worklist_at (insn, prev); | |
3434 | } | |
3435 | ||
3436 | /* INSN may now have a higher priority (in the model_order_p sense) | |
3437 | than before. Move it up the worklist if necessary. */ | |
3438 | ||
3439 | static void | |
3440 | model_promote_insn (struct model_insn_info *insn) | |
3441 | { | |
3442 | struct model_insn_info *prev; | |
3443 | int count; | |
3444 | ||
3445 | prev = insn->prev; | |
3446 | count = MAX_SCHED_READY_INSNS; | |
3447 | while (count > 0 && prev && model_order_p (insn, prev)) | |
3448 | { | |
3449 | count--; | |
3450 | prev = prev->prev; | |
3451 | } | |
3452 | if (prev != insn->prev) | |
3453 | { | |
3454 | model_remove_from_worklist (insn); | |
3455 | model_add_to_worklist_at (insn, prev); | |
3456 | } | |
3457 | } | |
3458 | ||
3459 | /* Add INSN to the end of the model schedule. */ | |
3460 | ||
3461 | static void | |
f57aa6b0 | 3462 | model_add_to_schedule (rtx_insn *insn) |
7a1aca9c RS |
3463 | { |
3464 | unsigned int point; | |
3465 | ||
3466 | gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE); | |
3467 | QUEUE_INDEX (insn) = QUEUE_SCHEDULED; | |
3468 | ||
9771b263 DN |
3469 | point = model_schedule.length (); |
3470 | model_schedule.quick_push (insn); | |
7a1aca9c RS |
3471 | INSN_MODEL_INDEX (insn) = point + 1; |
3472 | } | |
3473 | ||
3474 | /* Analyze the instructions that are to be scheduled, setting up | |
3475 | MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready | |
3476 | instructions to model_worklist. */ | |
3477 | ||
3478 | static void | |
3479 | model_analyze_insns (void) | |
3480 | { | |
40a243d4 | 3481 | rtx_insn *start, *end, *iter; |
7a1aca9c RS |
3482 | sd_iterator_def sd_it; |
3483 | dep_t dep; | |
3484 | struct model_insn_info *insn, *con; | |
3485 | ||
3486 | model_num_insns = 0; | |
3487 | start = PREV_INSN (current_sched_info->next_tail); | |
3488 | end = current_sched_info->prev_head; | |
3489 | for (iter = start; iter != end; iter = PREV_INSN (iter)) | |
3490 | if (NONDEBUG_INSN_P (iter)) | |
3491 | { | |
3492 | insn = MODEL_INSN_INFO (iter); | |
3493 | insn->insn = iter; | |
3494 | FOR_EACH_DEP (iter, SD_LIST_FORW, sd_it, dep) | |
3495 | { | |
3496 | con = MODEL_INSN_INFO (DEP_CON (dep)); | |
3497 | if (con->insn && insn->alap < con->alap + 1) | |
3498 | insn->alap = con->alap + 1; | |
3499 | } | |
3500 | ||
3501 | insn->old_queue = QUEUE_INDEX (iter); | |
3502 | QUEUE_INDEX (iter) = QUEUE_NOWHERE; | |
3503 | ||
3504 | insn->unscheduled_preds = dep_list_size (iter, SD_LIST_HARD_BACK); | |
3505 | if (insn->unscheduled_preds == 0) | |
3506 | model_add_to_worklist (insn, NULL, model_worklist); | |
3507 | ||
3508 | model_num_insns++; | |
3509 | } | |
3510 | } | |
3511 | ||
3512 | /* The global state describes the register pressure at the start of the | |
3513 | model schedule. Initialize GROUP accordingly. */ | |
3514 | ||
3515 | static void | |
3516 | model_init_pressure_group (struct model_pressure_group *group) | |
3517 | { | |
3518 | int pci, cl; | |
3519 | ||
3520 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3521 | { | |
3522 | cl = ira_pressure_classes[pci]; | |
3523 | group->limits[pci].pressure = curr_reg_pressure[cl]; | |
3524 | group->limits[pci].point = 0; | |
3525 | } | |
3526 | /* Use index model_num_insns to record the state after the last | |
3527 | instruction in the model schedule. */ | |
3528 | group->model = XNEWVEC (struct model_pressure_data, | |
3529 | (model_num_insns + 1) * ira_pressure_classes_num); | |
3530 | } | |
3531 | ||
3532 | /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE. | |
3533 | Update the maximum pressure for the whole schedule. */ | |
3534 | ||
3535 | static void | |
3536 | model_record_pressure (struct model_pressure_group *group, | |
3537 | int point, int pci, int pressure) | |
3538 | { | |
3539 | MODEL_REF_PRESSURE (group, point, pci) = pressure; | |
3540 | if (group->limits[pci].pressure < pressure) | |
3541 | { | |
3542 | group->limits[pci].pressure = pressure; | |
3543 | group->limits[pci].point = point; | |
3544 | } | |
3545 | } | |
3546 | ||
3547 | /* INSN has just been added to the end of the model schedule. Record its | |
3548 | register-pressure information. */ | |
3549 | ||
3550 | static void | |
3551 | model_record_pressures (struct model_insn_info *insn) | |
3552 | { | |
3553 | struct reg_pressure_data *reg_pressure; | |
3554 | int point, pci, cl, delta; | |
3555 | int death[N_REG_CLASSES]; | |
3556 | ||
3557 | point = model_index (insn->insn); | |
3558 | if (sched_verbose >= 2) | |
3559 | { | |
7a1aca9c RS |
3560 | if (point == 0) |
3561 | { | |
3562 | fprintf (sched_dump, "\n;;\tModel schedule:\n;;\n"); | |
3563 | fprintf (sched_dump, ";;\t| idx insn | mpri hght dpth prio |\n"); | |
3564 | } | |
7a1aca9c RS |
3565 | fprintf (sched_dump, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ", |
3566 | point, INSN_UID (insn->insn), insn->model_priority, | |
3567 | insn->depth + insn->alap, insn->depth, | |
7eba871a SB |
3568 | INSN_PRIORITY (insn->insn), |
3569 | str_pattern_slim (PATTERN (insn->insn))); | |
7a1aca9c RS |
3570 | } |
3571 | calculate_reg_deaths (insn->insn, death); | |
3572 | reg_pressure = INSN_REG_PRESSURE (insn->insn); | |
3573 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3574 | { | |
3575 | cl = ira_pressure_classes[pci]; | |
3576 | delta = reg_pressure[pci].set_increase - death[cl]; | |
3577 | if (sched_verbose >= 2) | |
3578 | fprintf (sched_dump, " %s:[%d,%+d]", reg_class_names[cl], | |
3579 | curr_reg_pressure[cl], delta); | |
3580 | model_record_pressure (&model_before_pressure, point, pci, | |
3581 | curr_reg_pressure[cl]); | |
3582 | } | |
3583 | if (sched_verbose >= 2) | |
3584 | fprintf (sched_dump, "\n"); | |
3585 | } | |
3586 | ||
3587 | /* All instructions have been added to the model schedule. Record the | |
3588 | final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */ | |
3589 | ||
3590 | static void | |
3591 | model_record_final_pressures (struct model_pressure_group *group) | |
3592 | { | |
3593 | int point, pci, max_pressure, ref_pressure, cl; | |
3594 | ||
3595 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3596 | { | |
3597 | /* Record the final pressure for this class. */ | |
3598 | cl = ira_pressure_classes[pci]; | |
3599 | point = model_num_insns; | |
3600 | ref_pressure = curr_reg_pressure[cl]; | |
3601 | model_record_pressure (group, point, pci, ref_pressure); | |
3602 | ||
3603 | /* Record the original maximum pressure. */ | |
3604 | group->limits[pci].orig_pressure = group->limits[pci].pressure; | |
3605 | ||
3606 | /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */ | |
3607 | max_pressure = ref_pressure; | |
3608 | MODEL_MAX_PRESSURE (group, point, pci) = max_pressure; | |
3609 | while (point > 0) | |
3610 | { | |
3611 | point--; | |
3612 | ref_pressure = MODEL_REF_PRESSURE (group, point, pci); | |
3613 | max_pressure = MAX (max_pressure, ref_pressure); | |
3614 | MODEL_MAX_PRESSURE (group, point, pci) = max_pressure; | |
3615 | } | |
3616 | } | |
3617 | } | |
3618 | ||
3619 | /* Update all successors of INSN, given that INSN has just been scheduled. */ | |
3620 | ||
3621 | static void | |
3622 | model_add_successors_to_worklist (struct model_insn_info *insn) | |
3623 | { | |
3624 | sd_iterator_def sd_it; | |
3625 | struct model_insn_info *con; | |
3626 | dep_t dep; | |
3627 | ||
3628 | FOR_EACH_DEP (insn->insn, SD_LIST_FORW, sd_it, dep) | |
3629 | { | |
3630 | con = MODEL_INSN_INFO (DEP_CON (dep)); | |
3631 | /* Ignore debug instructions, and instructions from other blocks. */ | |
3632 | if (con->insn) | |
3633 | { | |
3634 | con->unscheduled_preds--; | |
3635 | ||
3636 | /* Update the depth field of each true-dependent successor. | |
3637 | Increasing the depth gives them a higher priority than | |
3638 | before. */ | |
3639 | if (DEP_TYPE (dep) == REG_DEP_TRUE && con->depth < insn->depth + 1) | |
3640 | { | |
3641 | con->depth = insn->depth + 1; | |
3642 | if (QUEUE_INDEX (con->insn) == QUEUE_READY) | |
3643 | model_promote_insn (con); | |
3644 | } | |
3645 | ||
3646 | /* If this is a true dependency, or if there are no remaining | |
3647 | dependencies for CON (meaning that CON only had non-true | |
3648 | dependencies), make sure that CON is on the worklist. | |
3649 | We don't bother otherwise because it would tend to fill the | |
3650 | worklist with a lot of low-priority instructions that are not | |
3651 | yet ready to issue. */ | |
3652 | if ((con->depth > 0 || con->unscheduled_preds == 0) | |
3653 | && QUEUE_INDEX (con->insn) == QUEUE_NOWHERE) | |
3654 | model_add_to_worklist (con, insn, insn->next); | |
3655 | } | |
3656 | } | |
3657 | } | |
3658 | ||
3659 | /* Give INSN a higher priority than any current instruction, then give | |
3660 | unscheduled predecessors of INSN a higher priority still. If any of | |
3661 | those predecessors are not on the model worklist, do the same for its | |
3662 | predecessors, and so on. */ | |
3663 | ||
3664 | static void | |
3665 | model_promote_predecessors (struct model_insn_info *insn) | |
3666 | { | |
3667 | struct model_insn_info *pro, *first; | |
3668 | sd_iterator_def sd_it; | |
3669 | dep_t dep; | |
3670 | ||
3671 | if (sched_verbose >= 7) | |
3672 | fprintf (sched_dump, ";;\t+--- priority of %d = %d, priority of", | |
3673 | INSN_UID (insn->insn), model_next_priority); | |
3674 | insn->model_priority = model_next_priority++; | |
3675 | model_remove_from_worklist (insn); | |
3676 | model_add_to_worklist_at (insn, NULL); | |
3677 | ||
3678 | first = NULL; | |
3679 | for (;;) | |
3680 | { | |
3681 | FOR_EACH_DEP (insn->insn, SD_LIST_HARD_BACK, sd_it, dep) | |
3682 | { | |
3683 | pro = MODEL_INSN_INFO (DEP_PRO (dep)); | |
3684 | /* The first test is to ignore debug instructions, and instructions | |
3685 | from other blocks. */ | |
3686 | if (pro->insn | |
3687 | && pro->model_priority != model_next_priority | |
3688 | && QUEUE_INDEX (pro->insn) != QUEUE_SCHEDULED) | |
3689 | { | |
3690 | pro->model_priority = model_next_priority; | |
3691 | if (sched_verbose >= 7) | |
3692 | fprintf (sched_dump, " %d", INSN_UID (pro->insn)); | |
3693 | if (QUEUE_INDEX (pro->insn) == QUEUE_READY) | |
3694 | { | |
3695 | /* PRO is already in the worklist, but it now has | |
3696 | a higher priority than before. Move it at the | |
3697 | appropriate place. */ | |
3698 | model_remove_from_worklist (pro); | |
3699 | model_add_to_worklist (pro, NULL, model_worklist); | |
3700 | } | |
3701 | else | |
3702 | { | |
3703 | /* PRO isn't in the worklist. Recursively process | |
3704 | its predecessors until we find one that is. */ | |
3705 | pro->next = first; | |
3706 | first = pro; | |
3707 | } | |
3708 | } | |
3709 | } | |
3710 | if (!first) | |
3711 | break; | |
3712 | insn = first; | |
3713 | first = insn->next; | |
3714 | } | |
3715 | if (sched_verbose >= 7) | |
3716 | fprintf (sched_dump, " = %d\n", model_next_priority); | |
3717 | model_next_priority++; | |
3718 | } | |
3719 | ||
3720 | /* Pick one instruction from model_worklist and process it. */ | |
3721 | ||
3722 | static void | |
3723 | model_choose_insn (void) | |
3724 | { | |
3725 | struct model_insn_info *insn, *fallback; | |
3726 | int count; | |
3727 | ||
3728 | if (sched_verbose >= 7) | |
3729 | { | |
3730 | fprintf (sched_dump, ";;\t+--- worklist:\n"); | |
3731 | insn = model_worklist; | |
3732 | count = MAX_SCHED_READY_INSNS; | |
3733 | while (count > 0 && insn) | |
3734 | { | |
3735 | fprintf (sched_dump, ";;\t+--- %d [%d, %d, %d, %d]\n", | |
3736 | INSN_UID (insn->insn), insn->model_priority, | |
3737 | insn->depth + insn->alap, insn->depth, | |
3738 | INSN_PRIORITY (insn->insn)); | |
3739 | count--; | |
3740 | insn = insn->next; | |
3741 | } | |
3742 | } | |
3743 | ||
3744 | /* Look for a ready instruction whose model_classify_priority is zero | |
3745 | or negative, picking the highest-priority one. Adding such an | |
3746 | instruction to the schedule now should do no harm, and may actually | |
3747 | do some good. | |
3748 | ||
3749 | Failing that, see whether there is an instruction with the highest | |
3750 | extant model_priority that is not yet ready, but which would reduce | |
3751 | pressure if it became ready. This is designed to catch cases like: | |
3752 | ||
3753 | (set (mem (reg R1)) (reg R2)) | |
3754 | ||
3755 | where the instruction is the last remaining use of R1 and where the | |
3756 | value of R2 is not yet available (or vice versa). The death of R1 | |
3757 | means that this instruction already reduces pressure. It is of | |
3758 | course possible that the computation of R2 involves other registers | |
3759 | that are hard to kill, but such cases are rare enough for this | |
3760 | heuristic to be a win in general. | |
3761 | ||
3762 | Failing that, just pick the highest-priority instruction in the | |
3763 | worklist. */ | |
3764 | count = MAX_SCHED_READY_INSNS; | |
3765 | insn = model_worklist; | |
3766 | fallback = 0; | |
3767 | for (;;) | |
3768 | { | |
3769 | if (count == 0 || !insn) | |
3770 | { | |
3771 | insn = fallback ? fallback : model_worklist; | |
3772 | break; | |
3773 | } | |
3774 | if (insn->unscheduled_preds) | |
3775 | { | |
3776 | if (model_worklist->model_priority == insn->model_priority | |
3777 | && !fallback | |
3778 | && model_classify_pressure (insn) < 0) | |
3779 | fallback = insn; | |
3780 | } | |
3781 | else | |
3782 | { | |
3783 | if (model_classify_pressure (insn) <= 0) | |
3784 | break; | |
3785 | } | |
3786 | count--; | |
3787 | insn = insn->next; | |
3788 | } | |
3789 | ||
3790 | if (sched_verbose >= 7 && insn != model_worklist) | |
3791 | { | |
3792 | if (insn->unscheduled_preds) | |
3793 | fprintf (sched_dump, ";;\t+--- promoting insn %d, with dependencies\n", | |
3794 | INSN_UID (insn->insn)); | |
3795 | else | |
3796 | fprintf (sched_dump, ";;\t+--- promoting insn %d, which is ready\n", | |
3797 | INSN_UID (insn->insn)); | |
3798 | } | |
3799 | if (insn->unscheduled_preds) | |
3800 | /* INSN isn't yet ready to issue. Give all its predecessors the | |
3801 | highest priority. */ | |
3802 | model_promote_predecessors (insn); | |
3803 | else | |
3804 | { | |
3805 | /* INSN is ready. Add it to the end of model_schedule and | |
3806 | process its successors. */ | |
3807 | model_add_successors_to_worklist (insn); | |
3808 | model_remove_from_worklist (insn); | |
3809 | model_add_to_schedule (insn->insn); | |
3810 | model_record_pressures (insn); | |
3811 | update_register_pressure (insn->insn); | |
3812 | } | |
3813 | } | |
3814 | ||
3815 | /* Restore all QUEUE_INDEXs to the values that they had before | |
3816 | model_start_schedule was called. */ | |
3817 | ||
3818 | static void | |
3819 | model_reset_queue_indices (void) | |
3820 | { | |
3821 | unsigned int i; | |
f57aa6b0 | 3822 | rtx_insn *insn; |
7a1aca9c | 3823 | |
9771b263 | 3824 | FOR_EACH_VEC_ELT (model_schedule, i, insn) |
7a1aca9c RS |
3825 | QUEUE_INDEX (insn) = MODEL_INSN_INFO (insn)->old_queue; |
3826 | } | |
3827 | ||
3828 | /* We have calculated the model schedule and spill costs. Print a summary | |
3829 | to sched_dump. */ | |
3830 | ||
3831 | static void | |
3832 | model_dump_pressure_summary (void) | |
3833 | { | |
3834 | int pci, cl; | |
3835 | ||
3836 | fprintf (sched_dump, ";; Pressure summary:"); | |
3837 | for (pci = 0; pci < ira_pressure_classes_num; pci++) | |
3838 | { | |
3839 | cl = ira_pressure_classes[pci]; | |
3840 | fprintf (sched_dump, " %s:%d", reg_class_names[cl], | |
3841 | model_before_pressure.limits[pci].pressure); | |
3842 | } | |
3843 | fprintf (sched_dump, "\n\n"); | |
3844 | } | |
3845 | ||
3846 | /* Initialize the SCHED_PRESSURE_MODEL information for the current | |
3847 | scheduling region. */ | |
3848 | ||
3849 | static void | |
4dd9ac6c | 3850 | model_start_schedule (basic_block bb) |
7a1aca9c | 3851 | { |
7a1aca9c | 3852 | model_next_priority = 1; |
9771b263 | 3853 | model_schedule.create (sched_max_luid); |
7a1aca9c RS |
3854 | model_insns = XCNEWVEC (struct model_insn_info, sched_max_luid); |
3855 | ||
4dd9ac6c | 3856 | gcc_assert (bb == BLOCK_FOR_INSN (NEXT_INSN (current_sched_info->prev_head))); |
7a1aca9c RS |
3857 | initiate_reg_pressure_info (df_get_live_in (bb)); |
3858 | ||
3859 | model_analyze_insns (); | |
3860 | model_init_pressure_group (&model_before_pressure); | |
3861 | while (model_worklist) | |
3862 | model_choose_insn (); | |
9771b263 | 3863 | gcc_assert (model_num_insns == (int) model_schedule.length ()); |
7a1aca9c RS |
3864 | if (sched_verbose >= 2) |
3865 | fprintf (sched_dump, "\n"); | |
3866 | ||
3867 | model_record_final_pressures (&model_before_pressure); | |
3868 | model_reset_queue_indices (); | |
3869 | ||
3870 | XDELETEVEC (model_insns); | |
3871 | ||
3872 | model_curr_point = 0; | |
3873 | initiate_reg_pressure_info (df_get_live_in (bb)); | |
3874 | if (sched_verbose >= 1) | |
3875 | model_dump_pressure_summary (); | |
3876 | } | |
3877 | ||
3878 | /* Free the information associated with GROUP. */ | |
3879 | ||
3880 | static void | |
3881 | model_finalize_pressure_group (struct model_pressure_group *group) | |
3882 | { | |
3883 | XDELETEVEC (group->model); | |
3884 | } | |
3885 | ||
3886 | /* Free the information created by model_start_schedule. */ | |
3887 | ||
3888 | static void | |
3889 | model_end_schedule (void) | |
3890 | { | |
3891 | model_finalize_pressure_group (&model_before_pressure); | |
9771b263 | 3892 | model_schedule.release (); |
7a1aca9c | 3893 | } |
4dd9ac6c MK |
3894 | |
3895 | /* Prepare reg pressure scheduling for basic block BB. */ | |
3896 | static void | |
3897 | sched_pressure_start_bb (basic_block bb) | |
3898 | { | |
3899 | /* Set the number of available registers for each class taking into account | |
3900 | relative probability of current basic block versus function prologue and | |
3901 | epilogue. | |
3902 | * If the basic block executes much more often than the prologue/epilogue | |
3903 | (e.g., inside a hot loop), then cost of spill in the prologue is close to | |
3904 | nil, so the effective number of available registers is | |
6d5c37d0 | 3905 | (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] - 0). |
4dd9ac6c MK |
3906 | * If the basic block executes as often as the prologue/epilogue, |
3907 | then spill in the block is as costly as in the prologue, so the effective | |
3908 | number of available registers is | |
6d5c37d0 PH |
3909 | (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] |
3910 | - call_saved_regs_num[cl]). | |
4dd9ac6c MK |
3911 | Note that all-else-equal, we prefer to spill in the prologue, since that |
3912 | allows "extra" registers for other basic blocks of the function. | |
3913 | * If the basic block is on the cold path of the function and executes | |
3914 | rarely, then we should always prefer to spill in the block, rather than | |
3915 | in the prologue/epilogue. The effective number of available register is | |
6d5c37d0 PH |
3916 | (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] |
3917 | - call_saved_regs_num[cl]). */ | |
4dd9ac6c MK |
3918 | { |
3919 | int i; | |
e7a74006 JH |
3920 | int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun); |
3921 | int bb_freq = bb->count.to_frequency (cfun); | |
4dd9ac6c MK |
3922 | |
3923 | if (bb_freq == 0) | |
3924 | { | |
3925 | if (entry_freq == 0) | |
3926 | entry_freq = bb_freq = 1; | |
3927 | } | |
3928 | if (bb_freq < entry_freq) | |
3929 | bb_freq = entry_freq; | |
3930 | ||
3931 | for (i = 0; i < ira_pressure_classes_num; ++i) | |
3932 | { | |
3933 | enum reg_class cl = ira_pressure_classes[i]; | |
6d5c37d0 PH |
3934 | sched_class_regs_num[cl] = ira_class_hard_regs_num[cl] |
3935 | - fixed_regs_num[cl]; | |
4dd9ac6c | 3936 | sched_class_regs_num[cl] |
6d5c37d0 | 3937 | -= (call_saved_regs_num[cl] * entry_freq) / bb_freq; |
4dd9ac6c MK |
3938 | } |
3939 | } | |
3940 | ||
3941 | if (sched_pressure == SCHED_PRESSURE_MODEL) | |
3942 | model_start_schedule (bb); | |
3943 | } | |
7a1aca9c | 3944 | \f |
51564684 BS |
3945 | /* A structure that holds local state for the loop in schedule_block. */ |
3946 | struct sched_block_state | |
3947 | { | |
3948 | /* True if no real insns have been scheduled in the current cycle. */ | |
3949 | bool first_cycle_insn_p; | |
26965010 BS |
3950 | /* True if a shadow insn has been scheduled in the current cycle, which |
3951 | means that no more normal insns can be issued. */ | |
3952 | bool shadows_only_p; | |
06d7e8e7 BS |
3953 | /* True if we're winding down a modulo schedule, which means that we only |
3954 | issue insns with INSN_EXACT_TICK set. */ | |
3955 | bool modulo_epilogue; | |
51564684 BS |
3956 | /* Initialized with the machine's issue rate every cycle, and updated |
3957 | by calls to the variable_issue hook. */ | |
3958 | int can_issue_more; | |
3959 | }; | |
ce18efcb | 3960 | |
8c660648 | 3961 | /* INSN is the "currently executing insn". Launch each insn which was |
176f9a7b | 3962 | waiting on INSN. READY is the ready list which contains the insns |
58fb7809 VM |
3963 | that are ready to fire. CLOCK is the current cycle. The function |
3964 | returns necessary cycle advance after issuing the insn (it is not | |
3965 | zero for insns in a schedule group). */ | |
8c660648 | 3966 | |
58fb7809 | 3967 | static int |
ce1ce33a | 3968 | schedule_insn (rtx_insn *insn) |
8c660648 | 3969 | { |
e2f6ff94 MK |
3970 | sd_iterator_def sd_it; |
3971 | dep_t dep; | |
ce18efcb | 3972 | int i; |
58fb7809 | 3973 | int advance = 0; |
8c660648 | 3974 | |
fa0aee89 | 3975 | if (sched_verbose >= 1) |
8c660648 | 3976 | { |
ce18efcb | 3977 | struct reg_pressure_data *pressure_info; |
d5c67efd | 3978 | fprintf (sched_dump, ";;\t%3i--> %s %-40s:", |
7eba871a SB |
3979 | clock_var, (*current_sched_info->print_insn) (insn, 1), |
3980 | str_pattern_slim (PATTERN (insn))); | |
fae15c93 | 3981 | |
6d0de005 JH |
3982 | if (recog_memoized (insn) < 0) |
3983 | fprintf (sched_dump, "nothing"); | |
3984 | else | |
3985 | print_reservation (sched_dump, insn); | |
ce18efcb VM |
3986 | pressure_info = INSN_REG_PRESSURE (insn); |
3987 | if (pressure_info != NULL) | |
3988 | { | |
3989 | fputc (':', sched_dump); | |
1756cb66 | 3990 | for (i = 0; i < ira_pressure_classes_num; i++) |
f20f2613 VM |
3991 | fprintf (sched_dump, "%s%s%+d(%d)", |
3992 | scheduled_insns.length () > 1 | |
3993 | && INSN_LUID (insn) | |
3994 | < INSN_LUID (scheduled_insns[scheduled_insns.length () - 2]) ? "@" : "", | |
1756cb66 | 3995 | reg_class_names[ira_pressure_classes[i]], |
ce18efcb VM |
3996 | pressure_info[i].set_increase, pressure_info[i].change); |
3997 | } | |
7a1aca9c RS |
3998 | if (sched_pressure == SCHED_PRESSURE_MODEL |
3999 | && model_curr_point < model_num_insns | |
4000 | && model_index (insn) == model_curr_point) | |
4001 | fprintf (sched_dump, ":model %d", model_curr_point); | |
6d0de005 JH |
4002 | fputc ('\n', sched_dump); |
4003 | } | |
8c660648 | 4004 | |
60867e8c | 4005 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED && !DEBUG_INSN_P (insn)) |
ce18efcb VM |
4006 | update_reg_and_insn_max_reg_pressure (insn); |
4007 | ||
63f54b1a MK |
4008 | /* Scheduling instruction should have all its dependencies resolved and |
4009 | should have been removed from the ready list. */ | |
e2724e63 | 4010 | gcc_assert (sd_lists_empty_p (insn, SD_LIST_HARD_BACK)); |
569fa502 | 4011 | |
f49b295a AO |
4012 | /* Reset debug insns invalidated by moving this insn. */ |
4013 | if (MAY_HAVE_DEBUG_INSNS && !DEBUG_INSN_P (insn)) | |
4014 | for (sd_it = sd_iterator_start (insn, SD_LIST_BACK); | |
4015 | sd_iterator_cond (&sd_it, &dep);) | |
4016 | { | |
9b517712 | 4017 | rtx_insn *dbg = DEP_PRO (dep); |
1de12eab | 4018 | struct reg_use_data *use, *next; |
f49b295a | 4019 | |
e2724e63 BS |
4020 | if (DEP_STATUS (dep) & DEP_CANCELLED) |
4021 | { | |
4022 | sd_iterator_next (&sd_it); | |
4023 | continue; | |
4024 | } | |
4025 | ||
f49b295a AO |
4026 | gcc_assert (DEBUG_INSN_P (dbg)); |
4027 | ||
4028 | if (sched_verbose >= 6) | |
4029 | fprintf (sched_dump, ";;\t\tresetting: debug insn %d\n", | |
4030 | INSN_UID (dbg)); | |
4031 | ||
4032 | /* ??? Rather than resetting the debug insn, we might be able | |
4033 | to emit a debug temp before the just-scheduled insn, but | |
4034 | this would involve checking that the expression at the | |
4035 | point of the debug insn is equivalent to the expression | |
4036 | before the just-scheduled insn. They might not be: the | |
4037 | expression in the debug insn may depend on other insns not | |
4038 | yet scheduled that set MEMs, REGs or even other debug | |
4039 | insns. It's not clear that attempting to preserve debug | |
4040 | information in these cases is worth the effort, given how | |
4041 | uncommon these resets are and the likelihood that the debug | |
4042 | temps introduced won't survive the schedule change. */ | |
4043 | INSN_VAR_LOCATION_LOC (dbg) = gen_rtx_UNKNOWN_VAR_LOC (); | |
4044 | df_insn_rescan (dbg); | |
4045 | ||
1de12eab JJ |
4046 | /* Unknown location doesn't use any registers. */ |
4047 | for (use = INSN_REG_USE_LIST (dbg); use != NULL; use = next) | |
4048 | { | |
14db98d4 JJ |
4049 | struct reg_use_data *prev = use; |
4050 | ||
4051 | /* Remove use from the cyclic next_regno_use chain first. */ | |
4052 | while (prev->next_regno_use != use) | |
4053 | prev = prev->next_regno_use; | |
4054 | prev->next_regno_use = use->next_regno_use; | |
1de12eab JJ |
4055 | next = use->next_insn_use; |
4056 | free (use); | |
4057 | } | |
4058 | INSN_REG_USE_LIST (dbg) = NULL; | |
4059 | ||
f49b295a AO |
4060 | /* We delete rather than resolve these deps, otherwise we |
4061 | crash in sched_free_deps(), because forward deps are | |
4062 | expected to be released before backward deps. */ | |
4063 | sd_delete_dep (sd_it); | |
4064 | } | |
4065 | ||
b198261f | 4066 | gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE); |
63f54b1a | 4067 | QUEUE_INDEX (insn) = QUEUE_SCHEDULED; |
b198261f | 4068 | |
7a1aca9c RS |
4069 | if (sched_pressure == SCHED_PRESSURE_MODEL |
4070 | && model_curr_point < model_num_insns | |
4071 | && NONDEBUG_INSN_P (insn)) | |
4072 | { | |
4073 | if (model_index (insn) == model_curr_point) | |
4074 | do | |
4075 | model_curr_point++; | |
4076 | while (model_curr_point < model_num_insns | |
4077 | && (QUEUE_INDEX (MODEL_INSN (model_curr_point)) | |
4078 | == QUEUE_SCHEDULED)); | |
4079 | else | |
4080 | model_recompute (insn); | |
4081 | model_update_limit_points (); | |
4082 | update_register_pressure (insn); | |
4083 | if (sched_verbose >= 2) | |
4084 | print_curr_reg_pressure (); | |
4085 | } | |
4086 | ||
63f54b1a MK |
4087 | gcc_assert (INSN_TICK (insn) >= MIN_TICK); |
4088 | if (INSN_TICK (insn) > clock_var) | |
4089 | /* INSN has been prematurely moved from the queue to the ready list. | |
b16abbcb BC |
4090 | This is possible only if following flags are set. */ |
4091 | gcc_assert (flag_sched_stalled_insns || sched_fusion); | |
63f54b1a MK |
4092 | |
4093 | /* ??? Probably, if INSN is scheduled prematurely, we should leave | |
4094 | INSN_TICK untouched. This is a machine-dependent issue, actually. */ | |
4095 | INSN_TICK (insn) = clock_var; | |
4096 | ||
e2724e63 BS |
4097 | check_clobbered_conditions (insn); |
4098 | ||
1a83e602 BS |
4099 | /* Update dependent instructions. First, see if by scheduling this insn |
4100 | now we broke a dependence in a way that requires us to change another | |
4101 | insn. */ | |
4102 | for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); | |
4103 | sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it)) | |
4104 | { | |
4105 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
9b517712 | 4106 | rtx_insn *pro = DEP_PRO (dep); |
1a83e602 BS |
4107 | if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED |
4108 | && desc != NULL && desc->insn == pro) | |
4109 | apply_replacement (dep, false); | |
4110 | } | |
4111 | ||
4112 | /* Go through and resolve forward dependencies. */ | |
e2f6ff94 MK |
4113 | for (sd_it = sd_iterator_start (insn, SD_LIST_FORW); |
4114 | sd_iterator_cond (&sd_it, &dep);) | |
8c660648 | 4115 | { |
9b517712 | 4116 | rtx_insn *next = DEP_CON (dep); |
e2724e63 | 4117 | bool cancelled = (DEP_STATUS (dep) & DEP_CANCELLED) != 0; |
8c660648 | 4118 | |
e2f6ff94 MK |
4119 | /* Resolve the dependence between INSN and NEXT. |
4120 | sd_resolve_dep () moves current dep to another list thus | |
4121 | advancing the iterator. */ | |
4122 | sd_resolve_dep (sd_it); | |
8c660648 | 4123 | |
e2724e63 BS |
4124 | if (cancelled) |
4125 | { | |
1a83e602 BS |
4126 | if (must_restore_pattern_p (next, dep)) |
4127 | restore_pattern (dep, false); | |
e2724e63 BS |
4128 | continue; |
4129 | } | |
4130 | ||
f49b295a AO |
4131 | /* Don't bother trying to mark next as ready if insn is a debug |
4132 | insn. If insn is the last hard dependency, it will have | |
4133 | already been discounted. */ | |
4134 | if (DEBUG_INSN_P (insn) && !DEBUG_INSN_P (next)) | |
4135 | continue; | |
4136 | ||
d7bfd907 | 4137 | if (!IS_SPECULATION_BRANCHY_CHECK_P (insn)) |
496d7bb0 | 4138 | { |
b8698a0f L |
4139 | int effective_cost; |
4140 | ||
496d7bb0 | 4141 | effective_cost = try_ready (next); |
b8698a0f | 4142 | |
496d7bb0 MK |
4143 | if (effective_cost >= 0 |
4144 | && SCHED_GROUP_P (next) | |
4145 | && advance < effective_cost) | |
4146 | advance = effective_cost; | |
4147 | } | |
4148 | else | |
4149 | /* Check always has only one forward dependence (to the first insn in | |
4150 | the recovery block), therefore, this will be executed only once. */ | |
4151 | { | |
e2f6ff94 | 4152 | gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW)); |
496d7bb0 MK |
4153 | fix_recovery_deps (RECOVERY_BLOCK (insn)); |
4154 | } | |
8c660648 JL |
4155 | } |
4156 | ||
7a403706 | 4157 | /* Annotate the instruction with issue information -- TImode |
4bdc8810 RH |
4158 | indicates that the instruction is expected not to be able |
4159 | to issue on the same cycle as the previous insn. A machine | |
4160 | may use this information to decide how the instruction should | |
4161 | be aligned. */ | |
30028c85 | 4162 | if (issue_rate > 1 |
fae15c93 | 4163 | && GET_CODE (PATTERN (insn)) != USE |
b5b8b0ac AO |
4164 | && GET_CODE (PATTERN (insn)) != CLOBBER |
4165 | && !DEBUG_INSN_P (insn)) | |
4bdc8810 | 4166 | { |
30028c85 | 4167 | if (reload_completed) |
63f54b1a MK |
4168 | PUT_MODE (insn, clock_var > last_clock_var ? TImode : VOIDmode); |
4169 | last_clock_var = clock_var; | |
4bdc8810 | 4170 | } |
63f54b1a | 4171 | |
06bc63d1 MK |
4172 | if (nonscheduled_insns_begin != NULL_RTX) |
4173 | /* Indicate to debug counters that INSN is scheduled. */ | |
4174 | nonscheduled_insns_begin = insn; | |
4175 | ||
58fb7809 | 4176 | return advance; |
8c660648 JL |
4177 | } |
4178 | ||
63de6c74 | 4179 | /* Functions for handling of notes. */ |
8c660648 | 4180 | |
e855c69d AB |
4181 | /* Add note list that ends on FROM_END to the end of TO_ENDP. */ |
4182 | void | |
b311fd0f | 4183 | concat_note_lists (rtx_insn *from_end, rtx_insn **to_endp) |
e855c69d | 4184 | { |
b311fd0f | 4185 | rtx_insn *from_start; |
e855c69d | 4186 | |
e67271bd | 4187 | /* It's easy when have nothing to concat. */ |
e855c69d | 4188 | if (from_end == NULL) |
e855c69d AB |
4189 | return; |
4190 | ||
e67271bd | 4191 | /* It's also easy when destination is empty. */ |
e855c69d | 4192 | if (*to_endp == NULL) |
e855c69d AB |
4193 | { |
4194 | *to_endp = from_end; | |
4195 | return; | |
4196 | } | |
4197 | ||
4198 | from_start = from_end; | |
b8698a0f | 4199 | while (PREV_INSN (from_start) != NULL) |
e855c69d AB |
4200 | from_start = PREV_INSN (from_start); |
4201 | ||
0f82e5c9 DM |
4202 | SET_PREV_INSN (from_start) = *to_endp; |
4203 | SET_NEXT_INSN (*to_endp) = from_start; | |
e855c69d AB |
4204 | *to_endp = from_end; |
4205 | } | |
4206 | ||
e67271bd RH |
4207 | /* Delete notes between HEAD and TAIL and put them in the chain |
4208 | of notes ended by NOTE_LIST. */ | |
4209 | void | |
ce1ce33a | 4210 | remove_notes (rtx_insn *head, rtx_insn *tail) |
8c660648 | 4211 | { |
ce1ce33a | 4212 | rtx_insn *next_tail, *insn, *next; |
8c660648 | 4213 | |
e67271bd RH |
4214 | note_list = 0; |
4215 | if (head == tail && !INSN_P (head)) | |
4216 | return; | |
4217 | ||
4218 | next_tail = NEXT_INSN (tail); | |
e67271bd | 4219 | for (insn = head; insn != next_tail; insn = next) |
8c660648 | 4220 | { |
e67271bd RH |
4221 | next = NEXT_INSN (insn); |
4222 | if (!NOTE_P (insn)) | |
1124098b | 4223 | continue; |
f70b22c9 | 4224 | |
e67271bd RH |
4225 | switch (NOTE_KIND (insn)) |
4226 | { | |
4227 | case NOTE_INSN_BASIC_BLOCK: | |
e67271bd | 4228 | continue; |
8c660648 | 4229 | |
e67271bd RH |
4230 | case NOTE_INSN_EPILOGUE_BEG: |
4231 | if (insn != tail) | |
4232 | { | |
4233 | remove_insn (insn); | |
4234 | add_reg_note (next, REG_SAVE_NOTE, | |
4235 | GEN_INT (NOTE_INSN_EPILOGUE_BEG)); | |
4236 | break; | |
4237 | } | |
4238 | /* FALLTHRU */ | |
e855c69d | 4239 | |
e67271bd RH |
4240 | default: |
4241 | remove_insn (insn); | |
e855c69d | 4242 | |
e67271bd | 4243 | /* Add the note to list that ends at NOTE_LIST. */ |
0f82e5c9 DM |
4244 | SET_PREV_INSN (insn) = note_list; |
4245 | SET_NEXT_INSN (insn) = NULL_RTX; | |
e67271bd | 4246 | if (note_list) |
0f82e5c9 | 4247 | SET_NEXT_INSN (note_list) = insn; |
e67271bd RH |
4248 | note_list = insn; |
4249 | break; | |
4250 | } | |
4251 | ||
4252 | gcc_assert ((sel_sched_p () || insn != tail) && insn != head); | |
4253 | } | |
8c660648 JL |
4254 | } |
4255 | ||
26965010 BS |
4256 | /* A structure to record enough data to allow us to backtrack the scheduler to |
4257 | a previous state. */ | |
4258 | struct haifa_saved_data | |
4259 | { | |
4260 | /* Next entry on the list. */ | |
4261 | struct haifa_saved_data *next; | |
4262 | ||
4263 | /* Backtracking is associated with scheduling insns that have delay slots. | |
4264 | DELAY_PAIR points to the structure that contains the insns involved, and | |
4265 | the number of cycles between them. */ | |
4266 | struct delay_pair *delay_pair; | |
4267 | ||
4268 | /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */ | |
4269 | void *fe_saved_data; | |
4270 | /* Data used by the backend. */ | |
4271 | void *be_saved_data; | |
4272 | ||
4273 | /* Copies of global state. */ | |
4274 | int clock_var, last_clock_var; | |
4275 | struct ready_list ready; | |
4276 | state_t curr_state; | |
4277 | ||
ce1ce33a | 4278 | rtx_insn *last_scheduled_insn; |
90831096 | 4279 | rtx_insn *last_nondebug_scheduled_insn; |
ce1ce33a | 4280 | rtx_insn *nonscheduled_insns_begin; |
26965010 BS |
4281 | int cycle_issued_insns; |
4282 | ||
4283 | /* Copies of state used in the inner loop of schedule_block. */ | |
4284 | struct sched_block_state sched_block; | |
4285 | ||
4286 | /* We don't need to save q_ptr, as its value is arbitrary and we can set it | |
4287 | to 0 when restoring. */ | |
4288 | int q_size; | |
3dc99c19 | 4289 | rtx_insn_list **insn_queue; |
1a83e602 BS |
4290 | |
4291 | /* Describe pattern replacements that occurred since this backtrack point | |
4292 | was queued. */ | |
9771b263 DN |
4293 | vec<dep_t> replacement_deps; |
4294 | vec<int> replace_apply; | |
1a83e602 BS |
4295 | |
4296 | /* A copy of the next-cycle replacement vectors at the time of the backtrack | |
4297 | point. */ | |
9771b263 DN |
4298 | vec<dep_t> next_cycle_deps; |
4299 | vec<int> next_cycle_apply; | |
26965010 BS |
4300 | }; |
4301 | ||
4302 | /* A record, in reverse order, of all scheduled insns which have delay slots | |
4303 | and may require backtracking. */ | |
4304 | static struct haifa_saved_data *backtrack_queue; | |
4305 | ||
4306 | /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according | |
4307 | to SET_P. */ | |
4308 | static void | |
90831096 | 4309 | mark_backtrack_feeds (rtx_insn *insn, int set_p) |
26965010 BS |
4310 | { |
4311 | sd_iterator_def sd_it; | |
4312 | dep_t dep; | |
4313 | FOR_EACH_DEP (insn, SD_LIST_HARD_BACK, sd_it, dep) | |
4314 | { | |
4315 | FEEDS_BACKTRACK_INSN (DEP_PRO (dep)) = set_p; | |
4316 | } | |
4317 | } | |
4318 | ||
26965010 BS |
4319 | /* Save the current scheduler state so that we can backtrack to it |
4320 | later if necessary. PAIR gives the insns that make it necessary to | |
4321 | save this point. SCHED_BLOCK is the local state of schedule_block | |
4322 | that need to be saved. */ | |
4323 | static void | |
4324 | save_backtrack_point (struct delay_pair *pair, | |
4325 | struct sched_block_state sched_block) | |
4326 | { | |
4327 | int i; | |
4328 | struct haifa_saved_data *save = XNEW (struct haifa_saved_data); | |
4329 | ||
4330 | save->curr_state = xmalloc (dfa_state_size); | |
4331 | memcpy (save->curr_state, curr_state, dfa_state_size); | |
4332 | ||
4333 | save->ready.first = ready.first; | |
4334 | save->ready.n_ready = ready.n_ready; | |
4335 | save->ready.n_debug = ready.n_debug; | |
4336 | save->ready.veclen = ready.veclen; | |
ce1ce33a | 4337 | save->ready.vec = XNEWVEC (rtx_insn *, ready.veclen); |
26965010 BS |
4338 | memcpy (save->ready.vec, ready.vec, ready.veclen * sizeof (rtx)); |
4339 | ||
3dc99c19 | 4340 | save->insn_queue = XNEWVEC (rtx_insn_list *, max_insn_queue_index + 1); |
26965010 BS |
4341 | save->q_size = q_size; |
4342 | for (i = 0; i <= max_insn_queue_index; i++) | |
4343 | { | |
4344 | int q = NEXT_Q_AFTER (q_ptr, i); | |
e2724e63 | 4345 | save->insn_queue[i] = copy_INSN_LIST (insn_queue[q]); |
26965010 BS |
4346 | } |
4347 | ||
4348 | save->clock_var = clock_var; | |
4349 | save->last_clock_var = last_clock_var; | |
4350 | save->cycle_issued_insns = cycle_issued_insns; | |
4351 | save->last_scheduled_insn = last_scheduled_insn; | |
4352 | save->last_nondebug_scheduled_insn = last_nondebug_scheduled_insn; | |
06bc63d1 | 4353 | save->nonscheduled_insns_begin = nonscheduled_insns_begin; |
26965010 BS |
4354 | |
4355 | save->sched_block = sched_block; | |
4356 | ||
9771b263 DN |
4357 | save->replacement_deps.create (0); |
4358 | save->replace_apply.create (0); | |
4359 | save->next_cycle_deps = next_cycle_replace_deps.copy (); | |
4360 | save->next_cycle_apply = next_cycle_apply.copy (); | |
1a83e602 | 4361 | |
26965010 BS |
4362 | if (current_sched_info->save_state) |
4363 | save->fe_saved_data = (*current_sched_info->save_state) (); | |
4364 | ||
4365 | if (targetm.sched.alloc_sched_context) | |
4366 | { | |
4367 | save->be_saved_data = targetm.sched.alloc_sched_context (); | |
4368 | targetm.sched.init_sched_context (save->be_saved_data, false); | |
4369 | } | |
4370 | else | |
4371 | save->be_saved_data = NULL; | |
4372 | ||
4373 | save->delay_pair = pair; | |
4374 | ||
4375 | save->next = backtrack_queue; | |
4376 | backtrack_queue = save; | |
4377 | ||
4378 | while (pair) | |
4379 | { | |
4380 | mark_backtrack_feeds (pair->i2, 1); | |
4381 | INSN_TICK (pair->i2) = INVALID_TICK; | |
4382 | INSN_EXACT_TICK (pair->i2) = clock_var + pair_delay (pair); | |
06d7e8e7 | 4383 | SHADOW_P (pair->i2) = pair->stages == 0; |
26965010 BS |
4384 | pair = pair->next_same_i1; |
4385 | } | |
4386 | } | |
4387 | ||
e2724e63 BS |
4388 | /* Walk the ready list and all queues. If any insns have unresolved backwards |
4389 | dependencies, these must be cancelled deps, broken by predication. Set or | |
4390 | clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */ | |
4391 | ||
4392 | static void | |
4393 | toggle_cancelled_flags (bool set) | |
4394 | { | |
4395 | int i; | |
4396 | sd_iterator_def sd_it; | |
4397 | dep_t dep; | |
4398 | ||
4399 | if (ready.n_ready > 0) | |
4400 | { | |
ce1ce33a | 4401 | rtx_insn **first = ready_lastpos (&ready); |
e2724e63 BS |
4402 | for (i = 0; i < ready.n_ready; i++) |
4403 | FOR_EACH_DEP (first[i], SD_LIST_BACK, sd_it, dep) | |
4404 | if (!DEBUG_INSN_P (DEP_PRO (dep))) | |
4405 | { | |
4406 | if (set) | |
4407 | DEP_STATUS (dep) |= DEP_CANCELLED; | |
4408 | else | |
4409 | DEP_STATUS (dep) &= ~DEP_CANCELLED; | |
4410 | } | |
4411 | } | |
4412 | for (i = 0; i <= max_insn_queue_index; i++) | |
4413 | { | |
4414 | int q = NEXT_Q_AFTER (q_ptr, i); | |
96b70414 DM |
4415 | rtx_insn_list *link; |
4416 | for (link = insn_queue[q]; link; link = link->next ()) | |
e2724e63 | 4417 | { |
96b70414 | 4418 | rtx_insn *insn = link->insn (); |
e2724e63 BS |
4419 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) |
4420 | if (!DEBUG_INSN_P (DEP_PRO (dep))) | |
4421 | { | |
4422 | if (set) | |
4423 | DEP_STATUS (dep) |= DEP_CANCELLED; | |
4424 | else | |
4425 | DEP_STATUS (dep) &= ~DEP_CANCELLED; | |
4426 | } | |
4427 | } | |
4428 | } | |
4429 | } | |
4430 | ||
1a83e602 BS |
4431 | /* Undo the replacements that have occurred after backtrack point SAVE |
4432 | was placed. */ | |
4433 | static void | |
4434 | undo_replacements_for_backtrack (struct haifa_saved_data *save) | |
4435 | { | |
9771b263 | 4436 | while (!save->replacement_deps.is_empty ()) |
1a83e602 | 4437 | { |
9771b263 DN |
4438 | dep_t dep = save->replacement_deps.pop (); |
4439 | int apply_p = save->replace_apply.pop (); | |
1a83e602 BS |
4440 | |
4441 | if (apply_p) | |
4442 | restore_pattern (dep, true); | |
4443 | else | |
4444 | apply_replacement (dep, true); | |
4445 | } | |
9771b263 DN |
4446 | save->replacement_deps.release (); |
4447 | save->replace_apply.release (); | |
1a83e602 BS |
4448 | } |
4449 | ||
26965010 BS |
4450 | /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN. |
4451 | Restore their dependencies to an unresolved state, and mark them as | |
4452 | queued nowhere. */ | |
4453 | ||
4454 | static void | |
90831096 | 4455 | unschedule_insns_until (rtx_insn *insn) |
26965010 | 4456 | { |
f57aa6b0 | 4457 | auto_vec<rtx_insn *> recompute_vec; |
e2724e63 BS |
4458 | |
4459 | /* Make two passes over the insns to be unscheduled. First, we clear out | |
4460 | dependencies and other trivial bookkeeping. */ | |
26965010 BS |
4461 | for (;;) |
4462 | { | |
f57aa6b0 | 4463 | rtx_insn *last; |
26965010 BS |
4464 | sd_iterator_def sd_it; |
4465 | dep_t dep; | |
4466 | ||
9771b263 | 4467 | last = scheduled_insns.pop (); |
26965010 BS |
4468 | |
4469 | /* This will be changed by restore_backtrack_point if the insn is in | |
4470 | any queue. */ | |
4471 | QUEUE_INDEX (last) = QUEUE_NOWHERE; | |
4472 | if (last != insn) | |
4473 | INSN_TICK (last) = INVALID_TICK; | |
4474 | ||
06d7e8e7 BS |
4475 | if (modulo_ii > 0 && INSN_UID (last) < modulo_iter0_max_uid) |
4476 | modulo_insns_scheduled--; | |
4477 | ||
26965010 BS |
4478 | for (sd_it = sd_iterator_start (last, SD_LIST_RES_FORW); |
4479 | sd_iterator_cond (&sd_it, &dep);) | |
4480 | { | |
9b517712 | 4481 | rtx_insn *con = DEP_CON (dep); |
26965010 | 4482 | sd_unresolve_dep (sd_it); |
e2724e63 BS |
4483 | if (!MUST_RECOMPUTE_SPEC_P (con)) |
4484 | { | |
4485 | MUST_RECOMPUTE_SPEC_P (con) = 1; | |
9771b263 | 4486 | recompute_vec.safe_push (con); |
e2724e63 | 4487 | } |
26965010 BS |
4488 | } |
4489 | ||
4490 | if (last == insn) | |
4491 | break; | |
4492 | } | |
e2724e63 BS |
4493 | |
4494 | /* A second pass, to update ready and speculation status for insns | |
4495 | depending on the unscheduled ones. The first pass must have | |
4496 | popped the scheduled_insns vector up to the point where we | |
4497 | restart scheduling, as recompute_todo_spec requires it to be | |
4498 | up-to-date. */ | |
9771b263 | 4499 | while (!recompute_vec.is_empty ()) |
e2724e63 | 4500 | { |
f57aa6b0 | 4501 | rtx_insn *con; |
e2724e63 | 4502 | |
9771b263 | 4503 | con = recompute_vec.pop (); |
e2724e63 BS |
4504 | MUST_RECOMPUTE_SPEC_P (con) = 0; |
4505 | if (!sd_lists_empty_p (con, SD_LIST_HARD_BACK)) | |
4506 | { | |
4507 | TODO_SPEC (con) = HARD_DEP; | |
4508 | INSN_TICK (con) = INVALID_TICK; | |
4509 | if (PREDICATED_PAT (con) != NULL_RTX) | |
4510 | haifa_change_pattern (con, ORIG_PAT (con)); | |
4511 | } | |
4512 | else if (QUEUE_INDEX (con) != QUEUE_SCHEDULED) | |
1a83e602 | 4513 | TODO_SPEC (con) = recompute_todo_spec (con, true); |
e2724e63 | 4514 | } |
26965010 BS |
4515 | } |
4516 | ||
4517 | /* Restore scheduler state from the topmost entry on the backtracking queue. | |
4518 | PSCHED_BLOCK_P points to the local data of schedule_block that we must | |
4519 | overwrite with the saved data. | |
4520 | The caller must already have called unschedule_insns_until. */ | |
4521 | ||
4522 | static void | |
4523 | restore_last_backtrack_point (struct sched_block_state *psched_block) | |
26965010 | 4524 | { |
26965010 BS |
4525 | int i; |
4526 | struct haifa_saved_data *save = backtrack_queue; | |
4527 | ||
4528 | backtrack_queue = save->next; | |
4529 | ||
4530 | if (current_sched_info->restore_state) | |
4531 | (*current_sched_info->restore_state) (save->fe_saved_data); | |
4532 | ||
4533 | if (targetm.sched.alloc_sched_context) | |
4534 | { | |
4535 | targetm.sched.set_sched_context (save->be_saved_data); | |
4536 | targetm.sched.free_sched_context (save->be_saved_data); | |
4537 | } | |
4538 | ||
1a83e602 BS |
4539 | /* Do this first since it clobbers INSN_TICK of the involved |
4540 | instructions. */ | |
4541 | undo_replacements_for_backtrack (save); | |
4542 | ||
26965010 BS |
4543 | /* Clear the QUEUE_INDEX of everything in the ready list or one |
4544 | of the queues. */ | |
4545 | if (ready.n_ready > 0) | |
4546 | { | |
ce1ce33a | 4547 | rtx_insn **first = ready_lastpos (&ready); |
26965010 BS |
4548 | for (i = 0; i < ready.n_ready; i++) |
4549 | { | |
ce1ce33a | 4550 | rtx_insn *insn = first[i]; |
e2724e63 BS |
4551 | QUEUE_INDEX (insn) = QUEUE_NOWHERE; |
4552 | INSN_TICK (insn) = INVALID_TICK; | |
26965010 BS |
4553 | } |
4554 | } | |
4555 | for (i = 0; i <= max_insn_queue_index; i++) | |
4556 | { | |
4557 | int q = NEXT_Q_AFTER (q_ptr, i); | |
4558 | ||
96b70414 | 4559 | for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ()) |
26965010 | 4560 | { |
96b70414 | 4561 | rtx_insn *x = link->insn (); |
26965010 BS |
4562 | QUEUE_INDEX (x) = QUEUE_NOWHERE; |
4563 | INSN_TICK (x) = INVALID_TICK; | |
4564 | } | |
4565 | free_INSN_LIST_list (&insn_queue[q]); | |
4566 | } | |
4567 | ||
4568 | free (ready.vec); | |
4569 | ready = save->ready; | |
4570 | ||
4571 | if (ready.n_ready > 0) | |
4572 | { | |
ce1ce33a | 4573 | rtx_insn **first = ready_lastpos (&ready); |
26965010 BS |
4574 | for (i = 0; i < ready.n_ready; i++) |
4575 | { | |
ce1ce33a | 4576 | rtx_insn *insn = first[i]; |
e2724e63 | 4577 | QUEUE_INDEX (insn) = QUEUE_READY; |
1a83e602 | 4578 | TODO_SPEC (insn) = recompute_todo_spec (insn, true); |
e2724e63 | 4579 | INSN_TICK (insn) = save->clock_var; |
26965010 BS |
4580 | } |
4581 | } | |
4582 | ||
4583 | q_ptr = 0; | |
4584 | q_size = save->q_size; | |
4585 | for (i = 0; i <= max_insn_queue_index; i++) | |
4586 | { | |
4587 | int q = NEXT_Q_AFTER (q_ptr, i); | |
4588 | ||
4589 | insn_queue[q] = save->insn_queue[q]; | |
4590 | ||
96b70414 | 4591 | for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ()) |
26965010 | 4592 | { |
96b70414 | 4593 | rtx_insn *x = link->insn (); |
26965010 | 4594 | QUEUE_INDEX (x) = i; |
1a83e602 | 4595 | TODO_SPEC (x) = recompute_todo_spec (x, true); |
26965010 BS |
4596 | INSN_TICK (x) = save->clock_var + i; |
4597 | } | |
4598 | } | |
4599 | free (save->insn_queue); | |
4600 | ||
e2724e63 BS |
4601 | toggle_cancelled_flags (true); |
4602 | ||
26965010 BS |
4603 | clock_var = save->clock_var; |
4604 | last_clock_var = save->last_clock_var; | |
4605 | cycle_issued_insns = save->cycle_issued_insns; | |
4606 | last_scheduled_insn = save->last_scheduled_insn; | |
4607 | last_nondebug_scheduled_insn = save->last_nondebug_scheduled_insn; | |
06bc63d1 | 4608 | nonscheduled_insns_begin = save->nonscheduled_insns_begin; |
26965010 BS |
4609 | |
4610 | *psched_block = save->sched_block; | |
4611 | ||
4612 | memcpy (curr_state, save->curr_state, dfa_state_size); | |
4613 | free (save->curr_state); | |
4614 | ||
4615 | mark_backtrack_feeds (save->delay_pair->i2, 0); | |
4616 | ||
9771b263 DN |
4617 | gcc_assert (next_cycle_replace_deps.is_empty ()); |
4618 | next_cycle_replace_deps = save->next_cycle_deps.copy (); | |
4619 | next_cycle_apply = save->next_cycle_apply.copy (); | |
1a83e602 | 4620 | |
26965010 BS |
4621 | free (save); |
4622 | ||
4623 | for (save = backtrack_queue; save; save = save->next) | |
4624 | { | |
4625 | mark_backtrack_feeds (save->delay_pair->i2, 1); | |
4626 | } | |
4627 | } | |
4628 | ||
4629 | /* Discard all data associated with the topmost entry in the backtrack | |
4630 | queue. If RESET_TICK is false, we just want to free the data. If true, | |
4631 | we are doing this because we discovered a reason to backtrack. In the | |
4632 | latter case, also reset the INSN_TICK for the shadow insn. */ | |
4633 | static void | |
4634 | free_topmost_backtrack_point (bool reset_tick) | |
4635 | { | |
4636 | struct haifa_saved_data *save = backtrack_queue; | |
4637 | int i; | |
4638 | ||
4639 | backtrack_queue = save->next; | |
4640 | ||
4641 | if (reset_tick) | |
4642 | { | |
4643 | struct delay_pair *pair = save->delay_pair; | |
4644 | while (pair) | |
4645 | { | |
4646 | INSN_TICK (pair->i2) = INVALID_TICK; | |
4647 | INSN_EXACT_TICK (pair->i2) = INVALID_TICK; | |
4648 | pair = pair->next_same_i1; | |
4649 | } | |
1a83e602 BS |
4650 | undo_replacements_for_backtrack (save); |
4651 | } | |
4652 | else | |
4653 | { | |
9771b263 DN |
4654 | save->replacement_deps.release (); |
4655 | save->replace_apply.release (); | |
26965010 | 4656 | } |
1a83e602 | 4657 | |
26965010 BS |
4658 | if (targetm.sched.free_sched_context) |
4659 | targetm.sched.free_sched_context (save->be_saved_data); | |
4660 | if (current_sched_info->restore_state) | |
4661 | free (save->fe_saved_data); | |
4662 | for (i = 0; i <= max_insn_queue_index; i++) | |
4663 | free_INSN_LIST_list (&save->insn_queue[i]); | |
4664 | free (save->insn_queue); | |
4665 | free (save->curr_state); | |
4666 | free (save->ready.vec); | |
4667 | free (save); | |
4668 | } | |
4669 | ||
4670 | /* Free the entire backtrack queue. */ | |
4671 | static void | |
4672 | free_backtrack_queue (void) | |
4673 | { | |
4674 | while (backtrack_queue) | |
4675 | free_topmost_backtrack_point (false); | |
4676 | } | |
4677 | ||
1a83e602 BS |
4678 | /* Apply a replacement described by DESC. If IMMEDIATELY is false, we |
4679 | may have to postpone the replacement until the start of the next cycle, | |
4680 | at which point we will be called again with IMMEDIATELY true. This is | |
4681 | only done for machines which have instruction packets with explicit | |
4682 | parallelism however. */ | |
4683 | static void | |
4684 | apply_replacement (dep_t dep, bool immediately) | |
4685 | { | |
4686 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
4687 | if (!immediately && targetm.sched.exposed_pipeline && reload_completed) | |
4688 | { | |
9771b263 DN |
4689 | next_cycle_replace_deps.safe_push (dep); |
4690 | next_cycle_apply.safe_push (1); | |
1a83e602 BS |
4691 | } |
4692 | else | |
4693 | { | |
4694 | bool success; | |
4695 | ||
4696 | if (QUEUE_INDEX (desc->insn) == QUEUE_SCHEDULED) | |
4697 | return; | |
4698 | ||
4699 | if (sched_verbose >= 5) | |
4700 | fprintf (sched_dump, "applying replacement for insn %d\n", | |
4701 | INSN_UID (desc->insn)); | |
4702 | ||
4703 | success = validate_change (desc->insn, desc->loc, desc->newval, 0); | |
4704 | gcc_assert (success); | |
4705 | ||
4706 | update_insn_after_change (desc->insn); | |
4707 | if ((TODO_SPEC (desc->insn) & (HARD_DEP | DEP_POSTPONED)) == 0) | |
4708 | fix_tick_ready (desc->insn); | |
4709 | ||
4710 | if (backtrack_queue != NULL) | |
4711 | { | |
9771b263 DN |
4712 | backtrack_queue->replacement_deps.safe_push (dep); |
4713 | backtrack_queue->replace_apply.safe_push (1); | |
1a83e602 BS |
4714 | } |
4715 | } | |
4716 | } | |
4717 | ||
4718 | /* We have determined that a pattern involved in DEP must be restored. | |
4719 | If IMMEDIATELY is false, we may have to postpone the replacement | |
4720 | until the start of the next cycle, at which point we will be called | |
4721 | again with IMMEDIATELY true. */ | |
4722 | static void | |
4723 | restore_pattern (dep_t dep, bool immediately) | |
4724 | { | |
9b517712 | 4725 | rtx_insn *next = DEP_CON (dep); |
1a83e602 BS |
4726 | int tick = INSN_TICK (next); |
4727 | ||
4728 | /* If we already scheduled the insn, the modified version is | |
4729 | correct. */ | |
4730 | if (QUEUE_INDEX (next) == QUEUE_SCHEDULED) | |
4731 | return; | |
4732 | ||
4733 | if (!immediately && targetm.sched.exposed_pipeline && reload_completed) | |
4734 | { | |
9771b263 DN |
4735 | next_cycle_replace_deps.safe_push (dep); |
4736 | next_cycle_apply.safe_push (0); | |
1a83e602 BS |
4737 | return; |
4738 | } | |
4739 | ||
4740 | ||
4741 | if (DEP_TYPE (dep) == REG_DEP_CONTROL) | |
4742 | { | |
4743 | if (sched_verbose >= 5) | |
4744 | fprintf (sched_dump, "restoring pattern for insn %d\n", | |
4745 | INSN_UID (next)); | |
4746 | haifa_change_pattern (next, ORIG_PAT (next)); | |
4747 | } | |
4748 | else | |
4749 | { | |
4750 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
4751 | bool success; | |
4752 | ||
4753 | if (sched_verbose >= 5) | |
4754 | fprintf (sched_dump, "restoring pattern for insn %d\n", | |
4755 | INSN_UID (desc->insn)); | |
4756 | tick = INSN_TICK (desc->insn); | |
4757 | ||
4758 | success = validate_change (desc->insn, desc->loc, desc->orig, 0); | |
4759 | gcc_assert (success); | |
4760 | update_insn_after_change (desc->insn); | |
4761 | if (backtrack_queue != NULL) | |
4762 | { | |
9771b263 DN |
4763 | backtrack_queue->replacement_deps.safe_push (dep); |
4764 | backtrack_queue->replace_apply.safe_push (0); | |
1a83e602 BS |
4765 | } |
4766 | } | |
4767 | INSN_TICK (next) = tick; | |
4768 | if (TODO_SPEC (next) == DEP_POSTPONED) | |
4769 | return; | |
4770 | ||
4771 | if (sd_lists_empty_p (next, SD_LIST_BACK)) | |
4772 | TODO_SPEC (next) = 0; | |
4773 | else if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK)) | |
4774 | TODO_SPEC (next) = HARD_DEP; | |
4775 | } | |
4776 | ||
4777 | /* Perform pattern replacements that were queued up until the next | |
4778 | cycle. */ | |
4779 | static void | |
4780 | perform_replacements_new_cycle (void) | |
4781 | { | |
4782 | int i; | |
4783 | dep_t dep; | |
9771b263 | 4784 | FOR_EACH_VEC_ELT (next_cycle_replace_deps, i, dep) |
1a83e602 | 4785 | { |
9771b263 | 4786 | int apply_p = next_cycle_apply[i]; |
1a83e602 BS |
4787 | if (apply_p) |
4788 | apply_replacement (dep, true); | |
4789 | else | |
4790 | restore_pattern (dep, true); | |
4791 | } | |
9771b263 DN |
4792 | next_cycle_replace_deps.truncate (0); |
4793 | next_cycle_apply.truncate (0); | |
1a83e602 BS |
4794 | } |
4795 | ||
26965010 BS |
4796 | /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of |
4797 | instructions we've previously encountered, a set bit prevents | |
4798 | recursion. BUDGET is a limit on how far ahead we look, it is | |
4799 | reduced on recursive calls. Return true if we produced a good | |
4800 | estimate, or false if we exceeded the budget. */ | |
4801 | static bool | |
f57aa6b0 | 4802 | estimate_insn_tick (bitmap processed, rtx_insn *insn, int budget) |
26965010 BS |
4803 | { |
4804 | sd_iterator_def sd_it; | |
4805 | dep_t dep; | |
4806 | int earliest = INSN_TICK (insn); | |
4807 | ||
4808 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) | |
4809 | { | |
9b517712 | 4810 | rtx_insn *pro = DEP_PRO (dep); |
26965010 BS |
4811 | int t; |
4812 | ||
e2724e63 BS |
4813 | if (DEP_STATUS (dep) & DEP_CANCELLED) |
4814 | continue; | |
4815 | ||
26965010 BS |
4816 | if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED) |
4817 | gcc_assert (INSN_TICK (pro) + dep_cost (dep) <= INSN_TICK (insn)); | |
4818 | else | |
4819 | { | |
4820 | int cost = dep_cost (dep); | |
4821 | if (cost >= budget) | |
4822 | return false; | |
4823 | if (!bitmap_bit_p (processed, INSN_LUID (pro))) | |
4824 | { | |
4825 | if (!estimate_insn_tick (processed, pro, budget - cost)) | |
4826 | return false; | |
4827 | } | |
4828 | gcc_assert (INSN_TICK_ESTIMATE (pro) != INVALID_TICK); | |
4829 | t = INSN_TICK_ESTIMATE (pro) + cost; | |
4830 | if (earliest == INVALID_TICK || t > earliest) | |
4831 | earliest = t; | |
4832 | } | |
4833 | } | |
4834 | bitmap_set_bit (processed, INSN_LUID (insn)); | |
4835 | INSN_TICK_ESTIMATE (insn) = earliest; | |
4836 | return true; | |
4837 | } | |
4838 | ||
4839 | /* Examine the pair of insns in P, and estimate (optimistically, assuming | |
4840 | infinite resources) the cycle in which the delayed shadow can be issued. | |
4841 | Return the number of cycles that must pass before the real insn can be | |
4842 | issued in order to meet this constraint. */ | |
4843 | static int | |
4844 | estimate_shadow_tick (struct delay_pair *p) | |
4845 | { | |
8f9b31f7 | 4846 | auto_bitmap processed; |
26965010 BS |
4847 | int t; |
4848 | bool cutoff; | |
26965010 | 4849 | |
8f9b31f7 | 4850 | cutoff = !estimate_insn_tick (processed, p->i2, |
26965010 | 4851 | max_insn_queue_index + pair_delay (p)); |
26965010 BS |
4852 | if (cutoff) |
4853 | return max_insn_queue_index; | |
4854 | t = INSN_TICK_ESTIMATE (p->i2) - (clock_var + pair_delay (p) + 1); | |
4855 | if (t > 0) | |
4856 | return t; | |
4857 | return 0; | |
4858 | } | |
e67271bd | 4859 | |
06d7e8e7 BS |
4860 | /* If INSN has no unresolved backwards dependencies, add it to the schedule and |
4861 | recursively resolve all its forward dependencies. */ | |
4862 | static void | |
ce1ce33a | 4863 | resolve_dependencies (rtx_insn *insn) |
06d7e8e7 BS |
4864 | { |
4865 | sd_iterator_def sd_it; | |
4866 | dep_t dep; | |
4867 | ||
4868 | /* Don't use sd_lists_empty_p; it ignores debug insns. */ | |
4869 | if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn)) != NULL | |
4870 | || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn)) != NULL) | |
4871 | return; | |
4872 | ||
4873 | if (sched_verbose >= 4) | |
4874 | fprintf (sched_dump, ";;\tquickly resolving %d\n", INSN_UID (insn)); | |
4875 | ||
4876 | if (QUEUE_INDEX (insn) >= 0) | |
4877 | queue_remove (insn); | |
4878 | ||
9771b263 | 4879 | scheduled_insns.safe_push (insn); |
06d7e8e7 BS |
4880 | |
4881 | /* Update dependent instructions. */ | |
4882 | for (sd_it = sd_iterator_start (insn, SD_LIST_FORW); | |
4883 | sd_iterator_cond (&sd_it, &dep);) | |
4884 | { | |
9b517712 | 4885 | rtx_insn *next = DEP_CON (dep); |
06d7e8e7 BS |
4886 | |
4887 | if (sched_verbose >= 4) | |
4888 | fprintf (sched_dump, ";;\t\tdep %d against %d\n", INSN_UID (insn), | |
4889 | INSN_UID (next)); | |
4890 | ||
4891 | /* Resolve the dependence between INSN and NEXT. | |
4892 | sd_resolve_dep () moves current dep to another list thus | |
4893 | advancing the iterator. */ | |
4894 | sd_resolve_dep (sd_it); | |
4895 | ||
4896 | if (!IS_SPECULATION_BRANCHY_CHECK_P (insn)) | |
4897 | { | |
4898 | resolve_dependencies (next); | |
4899 | } | |
4900 | else | |
4901 | /* Check always has only one forward dependence (to the first insn in | |
4902 | the recovery block), therefore, this will be executed only once. */ | |
4903 | { | |
4904 | gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW)); | |
4905 | } | |
4906 | } | |
4907 | } | |
4908 | ||
4909 | ||
496d7bb0 MK |
4910 | /* Return the head and tail pointers of ebb starting at BEG and ending |
4911 | at END. */ | |
b4ead7d4 | 4912 | void |
52d251b5 DM |
4913 | get_ebb_head_tail (basic_block beg, basic_block end, |
4914 | rtx_insn **headp, rtx_insn **tailp) | |
496d7bb0 | 4915 | { |
52d251b5 DM |
4916 | rtx_insn *beg_head = BB_HEAD (beg); |
4917 | rtx_insn * beg_tail = BB_END (beg); | |
4918 | rtx_insn * end_head = BB_HEAD (end); | |
4919 | rtx_insn * end_tail = BB_END (end); | |
496d7bb0 MK |
4920 | |
4921 | /* Don't include any notes or labels at the beginning of the BEG | |
4922 | basic block, or notes at the end of the END basic blocks. */ | |
4923 | ||
4924 | if (LABEL_P (beg_head)) | |
4925 | beg_head = NEXT_INSN (beg_head); | |
4926 | ||
4927 | while (beg_head != beg_tail) | |
a59d15cf | 4928 | if (NOTE_P (beg_head)) |
496d7bb0 | 4929 | beg_head = NEXT_INSN (beg_head); |
a59d15cf AO |
4930 | else if (DEBUG_INSN_P (beg_head)) |
4931 | { | |
52d251b5 | 4932 | rtx_insn * note, *next; |
a59d15cf AO |
4933 | |
4934 | for (note = NEXT_INSN (beg_head); | |
4935 | note != beg_tail; | |
4936 | note = next) | |
4937 | { | |
4938 | next = NEXT_INSN (note); | |
4939 | if (NOTE_P (note)) | |
4940 | { | |
4941 | if (sched_verbose >= 9) | |
4942 | fprintf (sched_dump, "reorder %i\n", INSN_UID (note)); | |
4943 | ||
4944 | reorder_insns_nobb (note, note, PREV_INSN (beg_head)); | |
4945 | ||
4946 | if (BLOCK_FOR_INSN (note) != beg) | |
4947 | df_insn_change_bb (note, beg); | |
4948 | } | |
4949 | else if (!DEBUG_INSN_P (note)) | |
4950 | break; | |
4951 | } | |
4952 | ||
4953 | break; | |
4954 | } | |
496d7bb0 MK |
4955 | else |
4956 | break; | |
4957 | ||
4958 | *headp = beg_head; | |
4959 | ||
4960 | if (beg == end) | |
4961 | end_head = beg_head; | |
4962 | else if (LABEL_P (end_head)) | |
4963 | end_head = NEXT_INSN (end_head); | |
4964 | ||
4965 | while (end_head != end_tail) | |
a59d15cf | 4966 | if (NOTE_P (end_tail)) |
496d7bb0 | 4967 | end_tail = PREV_INSN (end_tail); |
a59d15cf AO |
4968 | else if (DEBUG_INSN_P (end_tail)) |
4969 | { | |
52d251b5 | 4970 | rtx_insn * note, *prev; |
a59d15cf AO |
4971 | |
4972 | for (note = PREV_INSN (end_tail); | |
4973 | note != end_head; | |
4974 | note = prev) | |
4975 | { | |
4976 | prev = PREV_INSN (note); | |
4977 | if (NOTE_P (note)) | |
4978 | { | |
4979 | if (sched_verbose >= 9) | |
4980 | fprintf (sched_dump, "reorder %i\n", INSN_UID (note)); | |
4981 | ||
4982 | reorder_insns_nobb (note, note, end_tail); | |
4983 | ||
4984 | if (end_tail == BB_END (end)) | |
1130d5e3 | 4985 | BB_END (end) = note; |
a59d15cf AO |
4986 | |
4987 | if (BLOCK_FOR_INSN (note) != end) | |
4988 | df_insn_change_bb (note, end); | |
4989 | } | |
4990 | else if (!DEBUG_INSN_P (note)) | |
4991 | break; | |
4992 | } | |
4993 | ||
4994 | break; | |
4995 | } | |
496d7bb0 MK |
4996 | else |
4997 | break; | |
8c660648 | 4998 | |
496d7bb0 | 4999 | *tailp = end_tail; |
8c660648 JL |
5000 | } |
5001 | ||
1708fd40 BS |
5002 | /* Return nonzero if there are no real insns in the range [ HEAD, TAIL ]. */ |
5003 | ||
b4ead7d4 | 5004 | int |
dc01c3d1 | 5005 | no_real_insns_p (const rtx_insn *head, const rtx_insn *tail) |
1708fd40 BS |
5006 | { |
5007 | while (head != NEXT_INSN (tail)) | |
5008 | { | |
a59d15cf | 5009 | if (!NOTE_P (head) && !LABEL_P (head)) |
1708fd40 BS |
5010 | return 0; |
5011 | head = NEXT_INSN (head); | |
5012 | } | |
5013 | return 1; | |
5014 | } | |
5015 | ||
e855c69d AB |
5016 | /* Restore-other-notes: NOTE_LIST is the end of a chain of notes |
5017 | previously found among the insns. Insert them just before HEAD. */ | |
ce1ce33a DM |
5018 | rtx_insn * |
5019 | restore_other_notes (rtx_insn *head, basic_block head_bb) | |
e855c69d AB |
5020 | { |
5021 | if (note_list != 0) | |
5022 | { | |
ce1ce33a | 5023 | rtx_insn *note_head = note_list; |
e855c69d AB |
5024 | |
5025 | if (head) | |
5026 | head_bb = BLOCK_FOR_INSN (head); | |
5027 | else | |
5028 | head = NEXT_INSN (bb_note (head_bb)); | |
5029 | ||
5030 | while (PREV_INSN (note_head)) | |
5031 | { | |
5032 | set_block_for_insn (note_head, head_bb); | |
5033 | note_head = PREV_INSN (note_head); | |
8c660648 | 5034 | } |
e855c69d AB |
5035 | /* In the above cycle we've missed this note. */ |
5036 | set_block_for_insn (note_head, head_bb); | |
5037 | ||
0f82e5c9 DM |
5038 | SET_PREV_INSN (note_head) = PREV_INSN (head); |
5039 | SET_NEXT_INSN (PREV_INSN (head)) = note_head; | |
5040 | SET_PREV_INSN (head) = note_list; | |
5041 | SET_NEXT_INSN (note_list) = head; | |
e855c69d AB |
5042 | |
5043 | if (BLOCK_FOR_INSN (head) != head_bb) | |
1130d5e3 | 5044 | BB_END (head_bb) = note_list; |
e855c69d AB |
5045 | |
5046 | head = note_head; | |
8c660648 | 5047 | } |
e855c69d AB |
5048 | |
5049 | return head; | |
8c660648 JL |
5050 | } |
5051 | ||
1a83e602 BS |
5052 | /* When we know we are going to discard the schedule due to a failed attempt |
5053 | at modulo scheduling, undo all replacements. */ | |
5054 | static void | |
5055 | undo_all_replacements (void) | |
5056 | { | |
ce1ce33a | 5057 | rtx_insn *insn; |
1a83e602 BS |
5058 | int i; |
5059 | ||
9771b263 | 5060 | FOR_EACH_VEC_ELT (scheduled_insns, i, insn) |
1a83e602 BS |
5061 | { |
5062 | sd_iterator_def sd_it; | |
5063 | dep_t dep; | |
5064 | ||
5065 | /* See if we must undo a replacement. */ | |
5066 | for (sd_it = sd_iterator_start (insn, SD_LIST_RES_FORW); | |
5067 | sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it)) | |
5068 | { | |
5069 | struct dep_replacement *desc = DEP_REPLACE (dep); | |
5070 | if (desc != NULL) | |
5071 | validate_change (desc->insn, desc->loc, desc->orig, 0); | |
5072 | } | |
5073 | } | |
5074 | } | |
5075 | ||
06bc63d1 MK |
5076 | /* Return first non-scheduled insn in the current scheduling block. |
5077 | This is mostly used for debug-counter purposes. */ | |
ce1ce33a | 5078 | static rtx_insn * |
06bc63d1 MK |
5079 | first_nonscheduled_insn (void) |
5080 | { | |
ce1ce33a DM |
5081 | rtx_insn *insn = (nonscheduled_insns_begin != NULL_RTX |
5082 | ? nonscheduled_insns_begin | |
5083 | : current_sched_info->prev_head); | |
06bc63d1 MK |
5084 | |
5085 | do | |
5086 | { | |
5087 | insn = next_nonnote_nondebug_insn (insn); | |
5088 | } | |
5089 | while (QUEUE_INDEX (insn) == QUEUE_SCHEDULED); | |
5090 | ||
5091 | return insn; | |
5092 | } | |
5093 | ||
8c660648 JL |
5094 | /* Move insns that became ready to fire from queue to ready list. */ |
5095 | ||
176f9a7b | 5096 | static void |
1d088dee | 5097 | queue_to_ready (struct ready_list *ready) |
8c660648 | 5098 | { |
ce1ce33a | 5099 | rtx_insn *insn; |
3dc99c19 | 5100 | rtx_insn_list *link; |
90831096 | 5101 | rtx_insn *skip_insn; |
8c660648 JL |
5102 | |
5103 | q_ptr = NEXT_Q (q_ptr); | |
5104 | ||
b631c5f7 | 5105 | if (dbg_cnt (sched_insn) == false) |
06bc63d1 MK |
5106 | /* If debug counter is activated do not requeue the first |
5107 | nonscheduled insn. */ | |
5108 | skip_insn = first_nonscheduled_insn (); | |
b631c5f7 | 5109 | else |
90831096 | 5110 | skip_insn = NULL; |
b631c5f7 | 5111 | |
8c660648 | 5112 | /* Add all pending insns that can be scheduled without stalls to the |
b4ead7d4 | 5113 | ready list. */ |
3dc99c19 | 5114 | for (link = insn_queue[q_ptr]; link; link = link->next ()) |
b4ead7d4 | 5115 | { |
3dc99c19 | 5116 | insn = link->insn (); |
b4ead7d4 | 5117 | q_size -= 1; |
1708fd40 | 5118 | |
b4ead7d4 BS |
5119 | if (sched_verbose >= 2) |
5120 | fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ", | |
5121 | (*current_sched_info->print_insn) (insn, 0)); | |
1708fd40 | 5122 | |
6f8dd94b EB |
5123 | /* If the ready list is full, delay the insn for 1 cycle. |
5124 | See the comment in schedule_block for the rationale. */ | |
5125 | if (!reload_completed | |
7a1aca9c RS |
5126 | && (ready->n_ready - ready->n_debug > MAX_SCHED_READY_INSNS |
5127 | || (sched_pressure == SCHED_PRESSURE_MODEL | |
5128 | /* Limit pressure recalculations to MAX_SCHED_READY_INSNS | |
5129 | instructions too. */ | |
5130 | && model_index (insn) > (model_curr_point | |
5131 | + MAX_SCHED_READY_INSNS))) | |
5132 | && !(sched_pressure == SCHED_PRESSURE_MODEL | |
5133 | && model_curr_point < model_num_insns | |
5134 | /* Always allow the next model instruction to issue. */ | |
5135 | && model_index (insn) == model_curr_point) | |
b631c5f7 SP |
5136 | && !SCHED_GROUP_P (insn) |
5137 | && insn != skip_insn) | |
d5c67efd MK |
5138 | { |
5139 | if (sched_verbose >= 2) | |
5140 | fprintf (sched_dump, "keeping in queue, ready full\n"); | |
5141 | queue_insn (insn, 1, "ready full"); | |
5142 | } | |
6f8dd94b EB |
5143 | else |
5144 | { | |
5145 | ready_add (ready, insn, false); | |
5146 | if (sched_verbose >= 2) | |
5147 | fprintf (sched_dump, "moving to ready without stalls\n"); | |
5148 | } | |
1708fd40 | 5149 | } |
63f54b1a | 5150 | free_INSN_LIST_list (&insn_queue[q_ptr]); |
b4ead7d4 BS |
5151 | |
5152 | /* If there are no ready insns, stall until one is ready and add all | |
5153 | of the pending insns at that point to the ready list. */ | |
5154 | if (ready->n_ready == 0) | |
1708fd40 | 5155 | { |
b3694847 | 5156 | int stalls; |
1708fd40 | 5157 | |
fa0aee89 | 5158 | for (stalls = 1; stalls <= max_insn_queue_index; stalls++) |
b4ead7d4 BS |
5159 | { |
5160 | if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)])) | |
5161 | { | |
3dc99c19 | 5162 | for (; link; link = link->next ()) |
b4ead7d4 | 5163 | { |
96b70414 | 5164 | insn = link->insn (); |
b4ead7d4 | 5165 | q_size -= 1; |
1708fd40 | 5166 | |
b4ead7d4 BS |
5167 | if (sched_verbose >= 2) |
5168 | fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ", | |
5169 | (*current_sched_info->print_insn) (insn, 0)); | |
1708fd40 | 5170 | |
63f54b1a | 5171 | ready_add (ready, insn, false); |
b4ead7d4 BS |
5172 | if (sched_verbose >= 2) |
5173 | fprintf (sched_dump, "moving to ready with %d stalls\n", stalls); | |
5174 | } | |
63f54b1a | 5175 | free_INSN_LIST_list (&insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]); |
1708fd40 | 5176 | |
fae15c93 VM |
5177 | advance_one_cycle (); |
5178 | ||
5179 | break; | |
b4ead7d4 | 5180 | } |
fae15c93 VM |
5181 | |
5182 | advance_one_cycle (); | |
b4ead7d4 | 5183 | } |
1708fd40 | 5184 | |
b4ead7d4 BS |
5185 | q_ptr = NEXT_Q_AFTER (q_ptr, stalls); |
5186 | clock_var += stalls; | |
d5c67efd MK |
5187 | if (sched_verbose >= 2) |
5188 | fprintf (sched_dump, ";;\tAdvancing clock by %d cycle[s] to %d\n", | |
5189 | stalls, clock_var); | |
1708fd40 | 5190 | } |
1708fd40 BS |
5191 | } |
5192 | ||
569fa502 | 5193 | /* Used by early_queue_to_ready. Determines whether it is "ok" to |
b8698a0f L |
5194 | prematurely move INSN from the queue to the ready list. Currently, |
5195 | if a target defines the hook 'is_costly_dependence', this function | |
569fa502 | 5196 | uses the hook to check whether there exist any dependences which are |
b8698a0f | 5197 | considered costly by the target, between INSN and other insns that |
569fa502 DN |
5198 | have already been scheduled. Dependences are checked up to Y cycles |
5199 | back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows | |
b8698a0f L |
5200 | controlling this value. |
5201 | (Other considerations could be taken into account instead (or in | |
569fa502 DN |
5202 | addition) depending on user flags and target hooks. */ |
5203 | ||
b8698a0f | 5204 | static bool |
90831096 | 5205 | ok_for_early_queue_removal (rtx_insn *insn) |
569fa502 | 5206 | { |
569fa502 DN |
5207 | if (targetm.sched.is_costly_dependence) |
5208 | { | |
9b69cf83 | 5209 | int n_cycles; |
9771b263 | 5210 | int i = scheduled_insns.length (); |
569fa502 DN |
5211 | for (n_cycles = flag_sched_stalled_insns_dep; n_cycles; n_cycles--) |
5212 | { | |
9b69cf83 | 5213 | while (i-- > 0) |
569fa502 | 5214 | { |
b198261f | 5215 | int cost; |
569fa502 | 5216 | |
21afc57d | 5217 | rtx_insn *prev_insn = scheduled_insns[i]; |
ec8628e8 | 5218 | |
4b4bf941 | 5219 | if (!NOTE_P (prev_insn)) |
569fa502 | 5220 | { |
e2f6ff94 | 5221 | dep_t dep; |
b198261f | 5222 | |
e2f6ff94 | 5223 | dep = sd_find_dep_between (prev_insn, insn, true); |
b198261f | 5224 | |
e2f6ff94 | 5225 | if (dep != NULL) |
569fa502 | 5226 | { |
b198261f MK |
5227 | cost = dep_cost (dep); |
5228 | ||
5229 | if (targetm.sched.is_costly_dependence (dep, cost, | |
569fa502 DN |
5230 | flag_sched_stalled_insns_dep - n_cycles)) |
5231 | return false; | |
5232 | } | |
5233 | } | |
5234 | ||
5235 | if (GET_MODE (prev_insn) == TImode) /* end of dispatch group */ | |
5236 | break; | |
5237 | } | |
5238 | ||
9b69cf83 | 5239 | if (i == 0) |
569fa502 | 5240 | break; |
569fa502 DN |
5241 | } |
5242 | } | |
5243 | ||
5244 | return true; | |
5245 | } | |
5246 | ||
5247 | ||
5248 | /* Remove insns from the queue, before they become "ready" with respect | |
6614fd40 | 5249 | to FU latency considerations. */ |
569fa502 | 5250 | |
b8698a0f | 5251 | static int |
569fa502 DN |
5252 | early_queue_to_ready (state_t state, struct ready_list *ready) |
5253 | { | |
ce1ce33a | 5254 | rtx_insn *insn; |
3dc99c19 DM |
5255 | rtx_insn_list *link; |
5256 | rtx_insn_list *next_link; | |
5257 | rtx_insn_list *prev_link; | |
569fa502 DN |
5258 | bool move_to_ready; |
5259 | int cost; | |
5260 | state_t temp_state = alloca (dfa_state_size); | |
5261 | int stalls; | |
5262 | int insns_removed = 0; | |
5263 | ||
5264 | /* | |
b8698a0f L |
5265 | Flag '-fsched-stalled-insns=X' determines the aggressiveness of this |
5266 | function: | |
569fa502 | 5267 | |
b8698a0f | 5268 | X == 0: There is no limit on how many queued insns can be removed |
569fa502 DN |
5269 | prematurely. (flag_sched_stalled_insns = -1). |
5270 | ||
b8698a0f | 5271 | X >= 1: Only X queued insns can be removed prematurely in each |
569fa502 DN |
5272 | invocation. (flag_sched_stalled_insns = X). |
5273 | ||
5274 | Otherwise: Early queue removal is disabled. | |
5275 | (flag_sched_stalled_insns = 0) | |
5276 | */ | |
5277 | ||
b8698a0f | 5278 | if (! flag_sched_stalled_insns) |
569fa502 DN |
5279 | return 0; |
5280 | ||
fa0aee89 | 5281 | for (stalls = 0; stalls <= max_insn_queue_index; stalls++) |
569fa502 DN |
5282 | { |
5283 | if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)])) | |
5284 | { | |
5285 | if (sched_verbose > 6) | |
5286 | fprintf (sched_dump, ";; look at index %d + %d\n", q_ptr, stalls); | |
5287 | ||
5288 | prev_link = 0; | |
5289 | while (link) | |
5290 | { | |
3dc99c19 DM |
5291 | next_link = link->next (); |
5292 | insn = link->insn (); | |
569fa502 DN |
5293 | if (insn && sched_verbose > 6) |
5294 | print_rtl_single (sched_dump, insn); | |
5295 | ||
5296 | memcpy (temp_state, state, dfa_state_size); | |
b8698a0f | 5297 | if (recog_memoized (insn) < 0) |
569fa502 DN |
5298 | /* non-negative to indicate that it's not ready |
5299 | to avoid infinite Q->R->Q->R... */ | |
5300 | cost = 0; | |
5301 | else | |
5302 | cost = state_transition (temp_state, insn); | |
5303 | ||
5304 | if (sched_verbose >= 6) | |
5305 | fprintf (sched_dump, "transition cost = %d\n", cost); | |
5306 | ||
5307 | move_to_ready = false; | |
b8698a0f | 5308 | if (cost < 0) |
569fa502 DN |
5309 | { |
5310 | move_to_ready = ok_for_early_queue_removal (insn); | |
5311 | if (move_to_ready == true) | |
5312 | { | |
5313 | /* move from Q to R */ | |
5314 | q_size -= 1; | |
63f54b1a | 5315 | ready_add (ready, insn, false); |
569fa502 | 5316 | |
b8698a0f | 5317 | if (prev_link) |
569fa502 DN |
5318 | XEXP (prev_link, 1) = next_link; |
5319 | else | |
5320 | insn_queue[NEXT_Q_AFTER (q_ptr, stalls)] = next_link; | |
5321 | ||
5322 | free_INSN_LIST_node (link); | |
5323 | ||
5324 | if (sched_verbose >= 2) | |
5325 | fprintf (sched_dump, ";;\t\tEarly Q-->Ready: insn %s\n", | |
5326 | (*current_sched_info->print_insn) (insn, 0)); | |
5327 | ||
5328 | insns_removed++; | |
5329 | if (insns_removed == flag_sched_stalled_insns) | |
63f54b1a MK |
5330 | /* Remove no more than flag_sched_stalled_insns insns |
5331 | from Q at a time. */ | |
569fa502 DN |
5332 | return insns_removed; |
5333 | } | |
5334 | } | |
5335 | ||
5336 | if (move_to_ready == false) | |
5337 | prev_link = link; | |
5338 | ||
5339 | link = next_link; | |
5340 | } /* while link */ | |
b8698a0f | 5341 | } /* if link */ |
569fa502 DN |
5342 | |
5343 | } /* for stalls.. */ | |
5344 | ||
b8698a0f | 5345 | return insns_removed; |
569fa502 DN |
5346 | } |
5347 | ||
5348 | ||
d5c67efd MK |
5349 | /* Print the ready list for debugging purposes. |
5350 | If READY_TRY is non-zero then only print insns that max_issue | |
5351 | will consider. */ | |
b4ead7d4 | 5352 | static void |
4960a0cb | 5353 | debug_ready_list_1 (struct ready_list *ready, signed char *ready_try) |
1708fd40 | 5354 | { |
ce1ce33a | 5355 | rtx_insn **p; |
b4ead7d4 | 5356 | int i; |
1708fd40 | 5357 | |
b4ead7d4 | 5358 | if (ready->n_ready == 0) |
fae15c93 VM |
5359 | { |
5360 | fprintf (sched_dump, "\n"); | |
5361 | return; | |
5362 | } | |
1708fd40 | 5363 | |
b4ead7d4 BS |
5364 | p = ready_lastpos (ready); |
5365 | for (i = 0; i < ready->n_ready; i++) | |
ce18efcb | 5366 | { |
d5c67efd MK |
5367 | if (ready_try != NULL && ready_try[ready->n_ready - i - 1]) |
5368 | continue; | |
5369 | ||
ce18efcb VM |
5370 | fprintf (sched_dump, " %s:%d", |
5371 | (*current_sched_info->print_insn) (p[i], 0), | |
5372 | INSN_LUID (p[i])); | |
7a1aca9c | 5373 | if (sched_pressure != SCHED_PRESSURE_NONE) |
ce18efcb VM |
5374 | fprintf (sched_dump, "(cost=%d", |
5375 | INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p[i])); | |
d5c67efd | 5376 | fprintf (sched_dump, ":prio=%d", INSN_PRIORITY (p[i])); |
ce18efcb VM |
5377 | if (INSN_TICK (p[i]) > clock_var) |
5378 | fprintf (sched_dump, ":delay=%d", INSN_TICK (p[i]) - clock_var); | |
b75ae7f4 MK |
5379 | if (sched_pressure == SCHED_PRESSURE_MODEL) |
5380 | fprintf (sched_dump, ":idx=%d", | |
5381 | model_index (p[i])); | |
7a1aca9c | 5382 | if (sched_pressure != SCHED_PRESSURE_NONE) |
ce18efcb VM |
5383 | fprintf (sched_dump, ")"); |
5384 | } | |
b4ead7d4 BS |
5385 | fprintf (sched_dump, "\n"); |
5386 | } | |
1708fd40 | 5387 | |
d5c67efd MK |
5388 | /* Print the ready list. Callable from debugger. */ |
5389 | static void | |
5390 | debug_ready_list (struct ready_list *ready) | |
5391 | { | |
5392 | debug_ready_list_1 (ready, NULL); | |
5393 | } | |
5394 | ||
e67271bd RH |
5395 | /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn |
5396 | NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb | |
5397 | replaces the epilogue note in the correct basic block. */ | |
e855c69d | 5398 | void |
b2908ba6 | 5399 | reemit_notes (rtx_insn *insn) |
8c660648 | 5400 | { |
b2908ba6 DM |
5401 | rtx note; |
5402 | rtx_insn *last = insn; | |
8c660648 | 5403 | |
8c660648 JL |
5404 | for (note = REG_NOTES (insn); note; note = XEXP (note, 1)) |
5405 | { | |
c46a37c4 | 5406 | if (REG_NOTE_KIND (note) == REG_SAVE_NOTE) |
8c660648 | 5407 | { |
81f40b79 | 5408 | enum insn_note note_type = (enum insn_note) INTVAL (XEXP (note, 0)); |
b3b42a4d | 5409 | |
63f4a88e JH |
5410 | last = emit_note_before (note_type, last); |
5411 | remove_note (insn, note); | |
8c660648 JL |
5412 | } |
5413 | } | |
8c660648 JL |
5414 | } |
5415 | ||
496d7bb0 MK |
5416 | /* Move INSN. Reemit notes if needed. Update CFG, if needed. */ |
5417 | static void | |
e6eda746 | 5418 | move_insn (rtx_insn *insn, rtx_insn *last, rtx nt) |
496d7bb0 | 5419 | { |
496d7bb0 MK |
5420 | if (PREV_INSN (insn) != last) |
5421 | { | |
5422 | basic_block bb; | |
ce1ce33a | 5423 | rtx_insn *note; |
496d7bb0 | 5424 | int jump_p = 0; |
8c660648 | 5425 | |
496d7bb0 | 5426 | bb = BLOCK_FOR_INSN (insn); |
b8698a0f | 5427 | |
496d7bb0 | 5428 | /* BB_HEAD is either LABEL or NOTE. */ |
b8698a0f | 5429 | gcc_assert (BB_HEAD (bb) != insn); |
496d7bb0 MK |
5430 | |
5431 | if (BB_END (bb) == insn) | |
5432 | /* If this is last instruction in BB, move end marker one | |
5433 | instruction up. */ | |
5434 | { | |
5435 | /* Jumps are always placed at the end of basic block. */ | |
5436 | jump_p = control_flow_insn_p (insn); | |
5437 | ||
5438 | gcc_assert (!jump_p | |
e855c69d | 5439 | || ((common_sched_info->sched_pass_id == SCHED_RGN_PASS) |
d7bfd907 | 5440 | && IS_SPECULATION_BRANCHY_CHECK_P (insn)) |
e855c69d AB |
5441 | || (common_sched_info->sched_pass_id |
5442 | == SCHED_EBB_PASS)); | |
b8698a0f | 5443 | |
496d7bb0 MK |
5444 | gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn)) == bb); |
5445 | ||
1130d5e3 | 5446 | BB_END (bb) = PREV_INSN (insn); |
496d7bb0 | 5447 | } |
8c660648 | 5448 | |
496d7bb0 | 5449 | gcc_assert (BB_END (bb) != last); |
c9e03727 | 5450 | |
496d7bb0 MK |
5451 | if (jump_p) |
5452 | /* We move the block note along with jump. */ | |
5453 | { | |
e855c69d | 5454 | gcc_assert (nt); |
496d7bb0 MK |
5455 | |
5456 | note = NEXT_INSN (insn); | |
5457 | while (NOTE_NOT_BB_P (note) && note != nt) | |
5458 | note = NEXT_INSN (note); | |
5459 | ||
5460 | if (note != nt | |
5461 | && (LABEL_P (note) | |
5462 | || BARRIER_P (note))) | |
5463 | note = NEXT_INSN (note); | |
b8698a0f | 5464 | |
496d7bb0 MK |
5465 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note)); |
5466 | } | |
5467 | else | |
5468 | note = insn; | |
5469 | ||
0f82e5c9 DM |
5470 | SET_NEXT_INSN (PREV_INSN (insn)) = NEXT_INSN (note); |
5471 | SET_PREV_INSN (NEXT_INSN (note)) = PREV_INSN (insn); | |
496d7bb0 | 5472 | |
0f82e5c9 DM |
5473 | SET_NEXT_INSN (note) = NEXT_INSN (last); |
5474 | SET_PREV_INSN (NEXT_INSN (last)) = note; | |
c9e03727 | 5475 | |
0f82e5c9 DM |
5476 | SET_NEXT_INSN (last) = insn; |
5477 | SET_PREV_INSN (insn) = last; | |
496d7bb0 MK |
5478 | |
5479 | bb = BLOCK_FOR_INSN (last); | |
5480 | ||
5481 | if (jump_p) | |
5482 | { | |
5483 | fix_jump_move (insn); | |
5484 | ||
5485 | if (BLOCK_FOR_INSN (insn) != bb) | |
5486 | move_block_after_check (insn); | |
5487 | ||
5488 | gcc_assert (BB_END (bb) == last); | |
5489 | } | |
5490 | ||
63642d5a | 5491 | df_insn_change_bb (insn, bb); |
b8698a0f | 5492 | |
496d7bb0 MK |
5493 | /* Update BB_END, if needed. */ |
5494 | if (BB_END (bb) == last) | |
1130d5e3 | 5495 | BB_END (bb) = insn; |
496d7bb0 | 5496 | } |
58fb7809 | 5497 | |
b8698a0f | 5498 | SCHED_GROUP_P (insn) = 0; |
8c660648 JL |
5499 | } |
5500 | ||
356c23b3 MK |
5501 | /* Return true if scheduling INSN will finish current clock cycle. */ |
5502 | static bool | |
ce1ce33a | 5503 | insn_finishes_cycle_p (rtx_insn *insn) |
356c23b3 MK |
5504 | { |
5505 | if (SCHED_GROUP_P (insn)) | |
5506 | /* After issuing INSN, rest of the sched_group will be forced to issue | |
5507 | in order. Don't make any plans for the rest of cycle. */ | |
5508 | return true; | |
5509 | ||
5510 | /* Finishing the block will, apparently, finish the cycle. */ | |
5511 | if (current_sched_info->insn_finishes_block_p | |
5512 | && current_sched_info->insn_finishes_block_p (insn)) | |
5513 | return true; | |
5514 | ||
5515 | return false; | |
5516 | } | |
5517 | ||
613743a2 KT |
5518 | /* Helper for autopref_multipass_init. Given a SET in PAT and whether |
5519 | we're expecting a memory WRITE or not, check that the insn is relevant to | |
5520 | the autoprefetcher modelling code. Return true iff that is the case. | |
5521 | If it is relevant, record the base register of the memory op in BASE and | |
5522 | the offset in OFFSET. */ | |
5523 | ||
5524 | static bool | |
5525 | analyze_set_insn_for_autopref (rtx pat, bool write, rtx *base, int *offset) | |
5526 | { | |
5527 | if (GET_CODE (pat) != SET) | |
5528 | return false; | |
5529 | ||
5530 | rtx mem = write ? SET_DEST (pat) : SET_SRC (pat); | |
5531 | if (!MEM_P (mem)) | |
5532 | return false; | |
5533 | ||
5534 | struct address_info info; | |
5535 | decompose_mem_address (&info, mem); | |
5536 | ||
5537 | /* TODO: Currently only (base+const) addressing is supported. */ | |
5538 | if (info.base == NULL || !REG_P (*info.base) | |
5539 | || (info.disp != NULL && !CONST_INT_P (*info.disp))) | |
5540 | return false; | |
5541 | ||
5542 | *base = *info.base; | |
5543 | *offset = info.disp ? INTVAL (*info.disp) : 0; | |
5544 | return true; | |
5545 | } | |
5546 | ||
340c7904 MK |
5547 | /* Functions to model cache auto-prefetcher. |
5548 | ||
5549 | Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate | |
5550 | memory prefetches if it sees instructions with consequitive memory accesses | |
5551 | in the instruction stream. Details of such hardware units are not published, | |
5552 | so we can only guess what exactly is going on there. | |
5553 | In the scheduler, we model abstract auto-prefetcher. If there are memory | |
5554 | insns in the ready list (or the queue) that have same memory base, but | |
5555 | different offsets, then we delay the insns with larger offsets until insns | |
5556 | with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH | |
5557 | is "1", then we look at the ready list; if it is N>1, then we also look | |
5558 | through N-1 queue entries. | |
5559 | If the param is N>=0, then rank_for_schedule will consider auto-prefetching | |
5560 | among its heuristics. | |
5561 | Param value of "-1" disables modelling of the auto-prefetcher. */ | |
5562 | ||
5563 | /* Initialize autoprefetcher model data for INSN. */ | |
5564 | static void | |
5565 | autopref_multipass_init (const rtx_insn *insn, int write) | |
5566 | { | |
5567 | autopref_multipass_data_t data = &INSN_AUTOPREF_MULTIPASS_DATA (insn)[write]; | |
5568 | ||
5569 | gcc_assert (data->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED); | |
5570 | data->base = NULL_RTX; | |
f9f43fb3 | 5571 | data->offset = 0; |
340c7904 MK |
5572 | /* Set insn entry initialized, but not relevant for auto-prefetcher. */ |
5573 | data->status = AUTOPREF_MULTIPASS_DATA_IRRELEVANT; | |
5574 | ||
613743a2 KT |
5575 | rtx pat = PATTERN (insn); |
5576 | ||
5577 | /* We have a multi-set insn like a load-multiple or store-multiple. | |
5578 | We care about these as long as all the memory ops inside the PARALLEL | |
5579 | have the same base register. We care about the minimum and maximum | |
5580 | offsets from that base but don't check for the order of those offsets | |
5581 | within the PARALLEL insn itself. */ | |
5582 | if (GET_CODE (pat) == PARALLEL) | |
5583 | { | |
5584 | int n_elems = XVECLEN (pat, 0); | |
5585 | ||
f9f43fb3 WD |
5586 | int i, offset; |
5587 | rtx base, prev_base = NULL_RTX; | |
5588 | int min_offset = INT_MAX; | |
613743a2 KT |
5589 | |
5590 | for (i = 0; i < n_elems; i++) | |
5591 | { | |
5592 | rtx set = XVECEXP (pat, 0, i); | |
5593 | if (GET_CODE (set) != SET) | |
5594 | return; | |
5595 | ||
613743a2 KT |
5596 | if (!analyze_set_insn_for_autopref (set, write, &base, &offset)) |
5597 | return; | |
5598 | ||
613743a2 KT |
5599 | /* Ensure that all memory operations in the PARALLEL use the same |
5600 | base register. */ | |
f9f43fb3 | 5601 | if (i > 0 && REGNO (base) != REGNO (prev_base)) |
613743a2 | 5602 | return; |
f9f43fb3 WD |
5603 | prev_base = base; |
5604 | min_offset = MIN (min_offset, offset); | |
613743a2 KT |
5605 | } |
5606 | ||
f9f43fb3 WD |
5607 | /* If we reached here then we have a valid PARALLEL of multiple memory ops |
5608 | with prev_base as the base and min_offset containing the offset. */ | |
613743a2 KT |
5609 | gcc_assert (prev_base); |
5610 | data->base = prev_base; | |
f9f43fb3 | 5611 | data->offset = min_offset; |
613743a2 | 5612 | data->status = AUTOPREF_MULTIPASS_DATA_NORMAL; |
613743a2 KT |
5613 | return; |
5614 | } | |
5615 | ||
5616 | /* Otherwise this is a single set memory operation. */ | |
340c7904 MK |
5617 | rtx set = single_set (insn); |
5618 | if (set == NULL_RTX) | |
5619 | return; | |
5620 | ||
613743a2 | 5621 | if (!analyze_set_insn_for_autopref (set, write, &data->base, |
f9f43fb3 | 5622 | &data->offset)) |
340c7904 MK |
5623 | return; |
5624 | ||
613743a2 KT |
5625 | /* This insn is relevant for the auto-prefetcher. |
5626 | The base and offset fields will have been filled in the | |
5627 | analyze_set_insn_for_autopref call above. */ | |
5628 | data->status = AUTOPREF_MULTIPASS_DATA_NORMAL; | |
5629 | } | |
340c7904 | 5630 | |
340c7904 MK |
5631 | /* Helper function for rank_for_schedule sorting. */ |
5632 | static int | |
5633 | autopref_rank_for_schedule (const rtx_insn *insn1, const rtx_insn *insn2) | |
5634 | { | |
ab90c27b AM |
5635 | int r = 0; |
5636 | for (int write = 0; write < 2 && !r; ++write) | |
340c7904 MK |
5637 | { |
5638 | autopref_multipass_data_t data1 | |
5639 | = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write]; | |
5640 | autopref_multipass_data_t data2 | |
5641 | = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write]; | |
5642 | ||
5643 | if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED) | |
5644 | autopref_multipass_init (insn1, write); | |
340c7904 MK |
5645 | |
5646 | if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED) | |
5647 | autopref_multipass_init (insn2, write); | |
340c7904 | 5648 | |
ab90c27b AM |
5649 | int irrel1 = data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT; |
5650 | int irrel2 = data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT; | |
340c7904 | 5651 | |
ab90c27b | 5652 | if (!irrel1 && !irrel2) |
f9f43fb3 | 5653 | r = data1->offset - data2->offset; |
ab90c27b AM |
5654 | else |
5655 | r = irrel2 - irrel1; | |
340c7904 MK |
5656 | } |
5657 | ||
ab90c27b | 5658 | return r; |
340c7904 MK |
5659 | } |
5660 | ||
5661 | /* True if header of debug dump was printed. */ | |
5662 | static bool autopref_multipass_dfa_lookahead_guard_started_dump_p; | |
5663 | ||
5664 | /* Helper for autopref_multipass_dfa_lookahead_guard. | |
5665 | Return "1" if INSN1 should be delayed in favor of INSN2. */ | |
5666 | static int | |
5667 | autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn *insn1, | |
5668 | const rtx_insn *insn2, int write) | |
5669 | { | |
5670 | autopref_multipass_data_t data1 | |
5671 | = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write]; | |
5672 | autopref_multipass_data_t data2 | |
5673 | = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write]; | |
5674 | ||
5675 | if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED) | |
5676 | autopref_multipass_init (insn2, write); | |
5677 | if (data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT) | |
5678 | return 0; | |
5679 | ||
5680 | if (rtx_equal_p (data1->base, data2->base) | |
f9f43fb3 | 5681 | && data1->offset > data2->offset) |
340c7904 MK |
5682 | { |
5683 | if (sched_verbose >= 2) | |
5684 | { | |
5685 | if (!autopref_multipass_dfa_lookahead_guard_started_dump_p) | |
5686 | { | |
5687 | fprintf (sched_dump, | |
5688 | ";;\t\tnot trying in max_issue due to autoprefetch " | |
5689 | "model: "); | |
5690 | autopref_multipass_dfa_lookahead_guard_started_dump_p = true; | |
5691 | } | |
5692 | ||
5693 | fprintf (sched_dump, " %d(%d)", INSN_UID (insn1), INSN_UID (insn2)); | |
5694 | } | |
5695 | ||
5696 | return 1; | |
5697 | } | |
5698 | ||
5699 | return 0; | |
5700 | } | |
5701 | ||
5702 | /* General note: | |
5703 | ||
5704 | We could have also hooked autoprefetcher model into | |
5705 | first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks | |
5706 | to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle | |
5707 | (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets | |
5708 | unblocked). We don't bother about this yet because target of interest | |
5709 | (ARM Cortex-A15) can issue only 1 memory operation per cycle. */ | |
5710 | ||
5711 | /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook. | |
5712 | Return "1" if INSN1 should not be considered in max_issue due to | |
5713 | auto-prefetcher considerations. */ | |
5714 | int | |
5715 | autopref_multipass_dfa_lookahead_guard (rtx_insn *insn1, int ready_index) | |
5716 | { | |
5717 | int r = 0; | |
5718 | ||
df554b0e KT |
5719 | /* Exit early if the param forbids this or if we're not entering here through |
5720 | normal haifa scheduling. This can happen if selective scheduling is | |
5721 | explicitly enabled. */ | |
5722 | if (!insn_queue || PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) <= 0) | |
340c7904 MK |
5723 | return 0; |
5724 | ||
5725 | if (sched_verbose >= 2 && ready_index == 0) | |
5726 | autopref_multipass_dfa_lookahead_guard_started_dump_p = false; | |
5727 | ||
5728 | for (int write = 0; write < 2; ++write) | |
5729 | { | |
5730 | autopref_multipass_data_t data1 | |
5731 | = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write]; | |
5732 | ||
5733 | if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED) | |
5734 | autopref_multipass_init (insn1, write); | |
5735 | if (data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT) | |
5736 | continue; | |
5737 | ||
5738 | if (ready_index == 0 | |
5739 | && data1->status == AUTOPREF_MULTIPASS_DATA_DONT_DELAY) | |
5740 | /* We allow only a single delay on priviledged instructions. | |
5741 | Doing otherwise would cause infinite loop. */ | |
5742 | { | |
5743 | if (sched_verbose >= 2) | |
5744 | { | |
5745 | if (!autopref_multipass_dfa_lookahead_guard_started_dump_p) | |
5746 | { | |
5747 | fprintf (sched_dump, | |
5748 | ";;\t\tnot trying in max_issue due to autoprefetch " | |
5749 | "model: "); | |
5750 | autopref_multipass_dfa_lookahead_guard_started_dump_p = true; | |
5751 | } | |
5752 | ||
5753 | fprintf (sched_dump, " *%d*", INSN_UID (insn1)); | |
5754 | } | |
5755 | continue; | |
5756 | } | |
5757 | ||
5758 | for (int i2 = 0; i2 < ready.n_ready; ++i2) | |
5759 | { | |
5760 | rtx_insn *insn2 = get_ready_element (i2); | |
5761 | if (insn1 == insn2) | |
5762 | continue; | |
5763 | r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2, write); | |
5764 | if (r) | |
5765 | { | |
5766 | if (ready_index == 0) | |
5767 | { | |
5768 | r = -1; | |
5769 | data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY; | |
5770 | } | |
5771 | goto finish; | |
5772 | } | |
5773 | } | |
5774 | ||
5775 | if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) == 1) | |
5776 | continue; | |
5777 | ||
5778 | /* Everything from the current queue slot should have been moved to | |
5779 | the ready list. */ | |
5780 | gcc_assert (insn_queue[NEXT_Q_AFTER (q_ptr, 0)] == NULL_RTX); | |
5781 | ||
5782 | int n_stalls = PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) - 1; | |
5783 | if (n_stalls > max_insn_queue_index) | |
5784 | n_stalls = max_insn_queue_index; | |
5785 | ||
5786 | for (int stalls = 1; stalls <= n_stalls; ++stalls) | |
5787 | { | |
5788 | for (rtx_insn_list *link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]; | |
5789 | link != NULL_RTX; | |
5790 | link = link->next ()) | |
5791 | { | |
5792 | rtx_insn *insn2 = link->insn (); | |
5793 | r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2, | |
5794 | write); | |
5795 | if (r) | |
5796 | { | |
5797 | /* Queue INSN1 until INSN2 can issue. */ | |
5798 | r = -stalls; | |
5799 | if (ready_index == 0) | |
5800 | data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY; | |
5801 | goto finish; | |
5802 | } | |
5803 | } | |
5804 | } | |
5805 | } | |
5806 | ||
5807 | finish: | |
5808 | if (sched_verbose >= 2 | |
5809 | && autopref_multipass_dfa_lookahead_guard_started_dump_p | |
5810 | && (ready_index == ready.n_ready - 1 || r < 0)) | |
5811 | /* This does not /always/ trigger. We don't output EOL if the last | |
5812 | insn is not recognized (INSN_CODE < 0) and lookahead_guard is not | |
5813 | called. We can live with this. */ | |
5814 | fprintf (sched_dump, "\n"); | |
5815 | ||
5816 | return r; | |
5817 | } | |
5818 | ||
894fd6f2 MK |
5819 | /* Define type for target data used in multipass scheduling. */ |
5820 | #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T | |
5821 | # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int | |
5822 | #endif | |
5823 | typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t; | |
5824 | ||
30028c85 VM |
5825 | /* The following structure describe an entry of the stack of choices. */ |
5826 | struct choice_entry | |
5827 | { | |
5828 | /* Ordinal number of the issued insn in the ready queue. */ | |
5829 | int index; | |
5830 | /* The number of the rest insns whose issues we should try. */ | |
5831 | int rest; | |
5832 | /* The number of issued essential insns. */ | |
5833 | int n; | |
5834 | /* State after issuing the insn. */ | |
5835 | state_t state; | |
894fd6f2 MK |
5836 | /* Target-specific data. */ |
5837 | first_cycle_multipass_data_t target_data; | |
30028c85 VM |
5838 | }; |
5839 | ||
5840 | /* The following array is used to implement a stack of choices used in | |
5841 | function max_issue. */ | |
5842 | static struct choice_entry *choice_stack; | |
5843 | ||
e855c69d AB |
5844 | /* This holds the value of the target dfa_lookahead hook. */ |
5845 | int dfa_lookahead; | |
30028c85 | 5846 | |
880efc46 VM |
5847 | /* The following variable value is maximal number of tries of issuing |
5848 | insns for the first cycle multipass insn scheduling. We define | |
5849 | this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not | |
5850 | need this constraint if all real insns (with non-negative codes) | |
5851 | had reservations because in this case the algorithm complexity is | |
5852 | O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions | |
5853 | might be incomplete and such insn might occur. For such | |
5854 | descriptions, the complexity of algorithm (without the constraint) | |
5855 | could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */ | |
5856 | static int max_lookahead_tries; | |
5857 | ||
fae15c93 VM |
5858 | /* The following function returns maximal (or close to maximal) number |
5859 | of insns which can be issued on the same cycle and one of which | |
30028c85 | 5860 | insns is insns with the best rank (the first insn in READY). To |
fae15c93 VM |
5861 | make this function tries different samples of ready insns. READY |
5862 | is current queue `ready'. Global array READY_TRY reflects what | |
84fabdde | 5863 | insns are already issued in this try. The function stops immediately, |
496d7bb0 MK |
5864 | if it reached the such a solution, that all instruction can be issued. |
5865 | INDEX will contain index of the best insn in READY. The following | |
e855c69d AB |
5866 | function is used only for first cycle multipass scheduling. |
5867 | ||
5868 | PRIVILEGED_N >= 0 | |
5869 | ||
5870 | This function expects recognized insns only. All USEs, | |
5871 | CLOBBERs, etc must be filtered elsewhere. */ | |
5872 | int | |
5873 | max_issue (struct ready_list *ready, int privileged_n, state_t state, | |
894fd6f2 | 5874 | bool first_cycle_insn_p, int *index) |
fae15c93 | 5875 | { |
84fabdde | 5876 | int n, i, all, n_ready, best, delay, tries_num; |
e855c69d | 5877 | int more_issue; |
30028c85 | 5878 | struct choice_entry *top; |
ce1ce33a | 5879 | rtx_insn *insn; |
fae15c93 | 5880 | |
b16abbcb BC |
5881 | if (sched_fusion) |
5882 | return 0; | |
5883 | ||
e855c69d AB |
5884 | n_ready = ready->n_ready; |
5885 | gcc_assert (dfa_lookahead >= 1 && privileged_n >= 0 | |
5886 | && privileged_n <= n_ready); | |
5887 | ||
5888 | /* Init MAX_LOOKAHEAD_TRIES. */ | |
8aec463b | 5889 | if (max_lookahead_tries == 0) |
e855c69d | 5890 | { |
e855c69d AB |
5891 | max_lookahead_tries = 100; |
5892 | for (i = 0; i < issue_rate; i++) | |
5893 | max_lookahead_tries *= dfa_lookahead; | |
5894 | } | |
5895 | ||
5896 | /* Init max_points. */ | |
e855c69d | 5897 | more_issue = issue_rate - cycle_issued_insns; |
646e6f41 | 5898 | gcc_assert (more_issue >= 0); |
e855c69d | 5899 | |
e855c69d | 5900 | /* The number of the issued insns in the best solution. */ |
fae15c93 | 5901 | best = 0; |
e855c69d | 5902 | |
30028c85 | 5903 | top = choice_stack; |
e855c69d AB |
5904 | |
5905 | /* Set initial state of the search. */ | |
5906 | memcpy (top->state, state, dfa_state_size); | |
5907 | top->rest = dfa_lookahead; | |
30028c85 | 5908 | top->n = 0; |
894fd6f2 MK |
5909 | if (targetm.sched.first_cycle_multipass_begin) |
5910 | targetm.sched.first_cycle_multipass_begin (&top->target_data, | |
5911 | ready_try, n_ready, | |
5912 | first_cycle_insn_p); | |
e855c69d AB |
5913 | |
5914 | /* Count the number of the insns to search among. */ | |
30028c85 | 5915 | for (all = i = 0; i < n_ready; i++) |
fae15c93 | 5916 | if (!ready_try [i]) |
30028c85 | 5917 | all++; |
e855c69d | 5918 | |
d5c67efd MK |
5919 | if (sched_verbose >= 2) |
5920 | { | |
5921 | fprintf (sched_dump, ";;\t\tmax_issue among %d insns:", all); | |
5922 | debug_ready_list_1 (ready, ready_try); | |
5923 | } | |
5924 | ||
e855c69d | 5925 | /* I is the index of the insn to try next. */ |
30028c85 | 5926 | i = 0; |
880efc46 | 5927 | tries_num = 0; |
30028c85 VM |
5928 | for (;;) |
5929 | { | |
e855c69d AB |
5930 | if (/* If we've reached a dead end or searched enough of what we have |
5931 | been asked... */ | |
5932 | top->rest == 0 | |
84fabdde JZ |
5933 | /* or have nothing else to try... */ |
5934 | || i >= n_ready | |
5935 | /* or should not issue more. */ | |
5936 | || top->n >= more_issue) | |
30028c85 | 5937 | { |
e855c69d AB |
5938 | /* ??? (... || i == n_ready). */ |
5939 | gcc_assert (i <= n_ready); | |
5940 | ||
84fabdde JZ |
5941 | /* We should not issue more than issue_rate instructions. */ |
5942 | gcc_assert (top->n <= more_issue); | |
5943 | ||
30028c85 VM |
5944 | if (top == choice_stack) |
5945 | break; | |
e855c69d AB |
5946 | |
5947 | if (best < top - choice_stack) | |
30028c85 | 5948 | { |
e855c69d AB |
5949 | if (privileged_n) |
5950 | { | |
5951 | n = privileged_n; | |
5952 | /* Try to find issued privileged insn. */ | |
e84a58ff EB |
5953 | while (n && !ready_try[--n]) |
5954 | ; | |
e855c69d AB |
5955 | } |
5956 | ||
5957 | if (/* If all insns are equally good... */ | |
5958 | privileged_n == 0 | |
5959 | /* Or a privileged insn will be issued. */ | |
5960 | || ready_try[n]) | |
5961 | /* Then we have a solution. */ | |
5962 | { | |
5963 | best = top - choice_stack; | |
5964 | /* This is the index of the insn issued first in this | |
5965 | solution. */ | |
5966 | *index = choice_stack [1].index; | |
84fabdde | 5967 | if (top->n == more_issue || best == all) |
e855c69d AB |
5968 | break; |
5969 | } | |
30028c85 | 5970 | } |
e855c69d AB |
5971 | |
5972 | /* Set ready-list index to point to the last insn | |
5973 | ('i++' below will advance it to the next insn). */ | |
30028c85 | 5974 | i = top->index; |
e855c69d AB |
5975 | |
5976 | /* Backtrack. */ | |
30028c85 | 5977 | ready_try [i] = 0; |
894fd6f2 MK |
5978 | |
5979 | if (targetm.sched.first_cycle_multipass_backtrack) | |
5980 | targetm.sched.first_cycle_multipass_backtrack (&top->target_data, | |
5981 | ready_try, n_ready); | |
5982 | ||
30028c85 | 5983 | top--; |
e855c69d | 5984 | memcpy (state, top->state, dfa_state_size); |
30028c85 VM |
5985 | } |
5986 | else if (!ready_try [i]) | |
5987 | { | |
880efc46 VM |
5988 | tries_num++; |
5989 | if (tries_num > max_lookahead_tries) | |
5990 | break; | |
30028c85 | 5991 | insn = ready_element (ready, i); |
e855c69d | 5992 | delay = state_transition (state, insn); |
30028c85 VM |
5993 | if (delay < 0) |
5994 | { | |
356c23b3 MK |
5995 | if (state_dead_lock_p (state) |
5996 | || insn_finishes_cycle_p (insn)) | |
9d701248 BS |
5997 | /* We won't issue any more instructions in the next |
5998 | choice_state. */ | |
30028c85 VM |
5999 | top->rest = 0; |
6000 | else | |
6001 | top->rest--; | |
e855c69d | 6002 | |
30028c85 | 6003 | n = top->n; |
e855c69d | 6004 | if (memcmp (top->state, state, dfa_state_size) != 0) |
84fabdde | 6005 | n++; |
e855c69d AB |
6006 | |
6007 | /* Advance to the next choice_entry. */ | |
30028c85 | 6008 | top++; |
e855c69d AB |
6009 | /* Initialize it. */ |
6010 | top->rest = dfa_lookahead; | |
30028c85 VM |
6011 | top->index = i; |
6012 | top->n = n; | |
e855c69d | 6013 | memcpy (top->state, state, dfa_state_size); |
30028c85 | 6014 | ready_try [i] = 1; |
894fd6f2 MK |
6015 | |
6016 | if (targetm.sched.first_cycle_multipass_issue) | |
6017 | targetm.sched.first_cycle_multipass_issue (&top->target_data, | |
6018 | ready_try, n_ready, | |
6019 | insn, | |
6020 | &((top - 1) | |
6021 | ->target_data)); | |
6022 | ||
30028c85 VM |
6023 | i = -1; |
6024 | } | |
6025 | } | |
e855c69d AB |
6026 | |
6027 | /* Increase ready-list index. */ | |
30028c85 VM |
6028 | i++; |
6029 | } | |
496d7bb0 | 6030 | |
894fd6f2 MK |
6031 | if (targetm.sched.first_cycle_multipass_end) |
6032 | targetm.sched.first_cycle_multipass_end (best != 0 | |
6033 | ? &choice_stack[1].target_data | |
6034 | : NULL); | |
6035 | ||
e855c69d | 6036 | /* Restore the original state of the DFA. */ |
b8698a0f | 6037 | memcpy (state, choice_stack->state, dfa_state_size); |
e855c69d | 6038 | |
fae15c93 VM |
6039 | return best; |
6040 | } | |
6041 | ||
6042 | /* The following function chooses insn from READY and modifies | |
e855c69d | 6043 | READY. The following function is used only for first |
b631c5f7 SP |
6044 | cycle multipass scheduling. |
6045 | Return: | |
6046 | -1 if cycle should be advanced, | |
6047 | 0 if INSN_PTR is set to point to the desirable insn, | |
6048 | 1 if choose_ready () should be restarted without advancing the cycle. */ | |
6049 | static int | |
894fd6f2 | 6050 | choose_ready (struct ready_list *ready, bool first_cycle_insn_p, |
ce1ce33a | 6051 | rtx_insn **insn_ptr) |
fae15c93 | 6052 | { |
b631c5f7 SP |
6053 | if (dbg_cnt (sched_insn) == false) |
6054 | { | |
06bc63d1 MK |
6055 | if (nonscheduled_insns_begin == NULL_RTX) |
6056 | nonscheduled_insns_begin = current_sched_info->prev_head; | |
6057 | ||
ce1ce33a | 6058 | rtx_insn *insn = first_nonscheduled_insn (); |
b631c5f7 SP |
6059 | |
6060 | if (QUEUE_INDEX (insn) == QUEUE_READY) | |
6061 | /* INSN is in the ready_list. */ | |
6062 | { | |
6063 | ready_remove_insn (insn); | |
6064 | *insn_ptr = insn; | |
6065 | return 0; | |
6066 | } | |
6067 | ||
6068 | /* INSN is in the queue. Advance cycle to move it to the ready list. */ | |
06bc63d1 | 6069 | gcc_assert (QUEUE_INDEX (insn) >= 0); |
b631c5f7 SP |
6070 | return -1; |
6071 | } | |
6072 | ||
8aec463b | 6073 | if (dfa_lookahead <= 0 || SCHED_GROUP_P (ready_element (ready, 0)) |
b5b8b0ac | 6074 | || DEBUG_INSN_P (ready_element (ready, 0))) |
b631c5f7 | 6075 | { |
ac44248e | 6076 | if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON)) |
7942e47e RY |
6077 | *insn_ptr = ready_remove_first_dispatch (ready); |
6078 | else | |
6079 | *insn_ptr = ready_remove_first (ready); | |
6080 | ||
b631c5f7 SP |
6081 | return 0; |
6082 | } | |
fae15c93 VM |
6083 | else |
6084 | { | |
4960a0cb | 6085 | /* Try to choose the best insn. */ |
16d83dd6 | 6086 | int index = 0, i; |
ce1ce33a | 6087 | rtx_insn *insn; |
b8698a0f | 6088 | |
30028c85 VM |
6089 | insn = ready_element (ready, 0); |
6090 | if (INSN_CODE (insn) < 0) | |
b631c5f7 SP |
6091 | { |
6092 | *insn_ptr = ready_remove_first (ready); | |
6093 | return 0; | |
6094 | } | |
496d7bb0 | 6095 | |
4960a0cb MK |
6096 | /* Filter the search space. */ |
6097 | for (i = 0; i < ready->n_ready; i++) | |
496d7bb0 | 6098 | { |
4960a0cb | 6099 | ready_try[i] = 0; |
496d7bb0 | 6100 | |
4960a0cb | 6101 | insn = ready_element (ready, i); |
496d7bb0 | 6102 | |
4960a0cb MK |
6103 | /* If this insn is recognizable we should have already |
6104 | recognized it earlier. | |
6105 | ??? Not very clear where this is supposed to be done. | |
6106 | See dep_cost_1. */ | |
6107 | gcc_checking_assert (INSN_CODE (insn) >= 0 | |
6108 | || recog_memoized (insn) < 0); | |
6109 | if (INSN_CODE (insn) < 0) | |
6110 | { | |
6111 | /* Non-recognized insns at position 0 are handled above. */ | |
6112 | gcc_assert (i > 0); | |
6113 | ready_try[i] = 1; | |
6114 | continue; | |
6115 | } | |
6116 | ||
6117 | if (targetm.sched.first_cycle_multipass_dfa_lookahead_guard) | |
6118 | { | |
6119 | ready_try[i] | |
6120 | = (targetm.sched.first_cycle_multipass_dfa_lookahead_guard | |
6121 | (insn, i)); | |
6122 | ||
6123 | if (ready_try[i] < 0) | |
6124 | /* Queue instruction for several cycles. | |
6125 | We need to restart choose_ready as we have changed | |
6126 | the ready list. */ | |
6127 | { | |
6128 | change_queue_index (insn, -ready_try[i]); | |
6129 | return 1; | |
6130 | } | |
6131 | ||
6132 | /* Make sure that we didn't end up with 0'th insn filtered out. | |
6133 | Don't be tempted to make life easier for backends and just | |
6134 | requeue 0'th insn if (ready_try[0] == 0) and restart | |
6135 | choose_ready. Backends should be very considerate about | |
6136 | requeueing instructions -- especially the highest priority | |
6137 | one at position 0. */ | |
6138 | gcc_assert (ready_try[i] == 0 || i > 0); | |
6139 | if (ready_try[i]) | |
6140 | continue; | |
6141 | } | |
6142 | ||
6143 | gcc_assert (ready_try[i] == 0); | |
6144 | /* INSN made it through the scrutiny of filters! */ | |
6145 | } | |
e855c69d | 6146 | |
894fd6f2 | 6147 | if (max_issue (ready, 1, curr_state, first_cycle_insn_p, &index) == 0) |
b631c5f7 SP |
6148 | { |
6149 | *insn_ptr = ready_remove_first (ready); | |
9c575182 | 6150 | if (sched_verbose >= 4) |
b8698a0f | 6151 | fprintf (sched_dump, ";;\t\tChosen insn (but can't issue) : %s \n", |
9c575182 | 6152 | (*current_sched_info->print_insn) (*insn_ptr, 0)); |
b631c5f7 SP |
6153 | return 0; |
6154 | } | |
fae15c93 | 6155 | else |
b631c5f7 | 6156 | { |
b8698a0f | 6157 | if (sched_verbose >= 4) |
e855c69d AB |
6158 | fprintf (sched_dump, ";;\t\tChosen insn : %s\n", |
6159 | (*current_sched_info->print_insn) | |
6160 | (ready_element (ready, index), 0)); | |
b8698a0f | 6161 | |
b631c5f7 SP |
6162 | *insn_ptr = ready_remove (ready, index); |
6163 | return 0; | |
6164 | } | |
fae15c93 VM |
6165 | } |
6166 | } | |
6167 | ||
86014d07 BS |
6168 | /* This function is called when we have successfully scheduled a |
6169 | block. It uses the schedule stored in the scheduled_insns vector | |
6170 | to rearrange the RTL. PREV_HEAD is used as the anchor to which we | |
6171 | append the scheduled insns; TAIL is the insn after the scheduled | |
6172 | block. TARGET_BB is the argument passed to schedule_block. */ | |
6173 | ||
6174 | static void | |
f57aa6b0 | 6175 | commit_schedule (rtx_insn *prev_head, rtx_insn *tail, basic_block *target_bb) |
86014d07 | 6176 | { |
9b69cf83 | 6177 | unsigned int i; |
ce1ce33a | 6178 | rtx_insn *insn; |
86014d07 BS |
6179 | |
6180 | last_scheduled_insn = prev_head; | |
9b69cf83 | 6181 | for (i = 0; |
9771b263 | 6182 | scheduled_insns.iterate (i, &insn); |
9b69cf83 | 6183 | i++) |
86014d07 | 6184 | { |
86014d07 BS |
6185 | if (control_flow_insn_p (last_scheduled_insn) |
6186 | || current_sched_info->advance_target_bb (*target_bb, insn)) | |
6187 | { | |
6188 | *target_bb = current_sched_info->advance_target_bb (*target_bb, 0); | |
6189 | ||
6190 | if (sched_verbose) | |
6191 | { | |
f57aa6b0 | 6192 | rtx_insn *x; |
86014d07 BS |
6193 | |
6194 | x = next_real_insn (last_scheduled_insn); | |
6195 | gcc_assert (x); | |
6196 | dump_new_block_header (1, *target_bb, x, tail); | |
6197 | } | |
6198 | ||
6199 | last_scheduled_insn = bb_note (*target_bb); | |
6200 | } | |
6201 | ||
6202 | if (current_sched_info->begin_move_insn) | |
6203 | (*current_sched_info->begin_move_insn) (insn, last_scheduled_insn); | |
6204 | move_insn (insn, last_scheduled_insn, | |
6205 | current_sched_info->next_tail); | |
6206 | if (!DEBUG_INSN_P (insn)) | |
6207 | reemit_notes (insn); | |
6208 | last_scheduled_insn = insn; | |
6209 | } | |
6210 | ||
9771b263 | 6211 | scheduled_insns.truncate (0); |
86014d07 BS |
6212 | } |
6213 | ||
9d701248 BS |
6214 | /* Examine all insns on the ready list and queue those which can't be |
6215 | issued in this cycle. TEMP_STATE is temporary scheduler state we | |
6216 | can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns | |
6217 | have been issued for the current cycle, which means it is valid to | |
26965010 BS |
6218 | issue an asm statement. |
6219 | ||
6220 | If SHADOWS_ONLY_P is true, we eliminate all real insns and only | |
06d7e8e7 BS |
6221 | leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true, |
6222 | we only leave insns which have an INSN_EXACT_TICK. */ | |
9d701248 BS |
6223 | |
6224 | static void | |
26965010 | 6225 | prune_ready_list (state_t temp_state, bool first_cycle_insn_p, |
06d7e8e7 | 6226 | bool shadows_only_p, bool modulo_epilogue_p) |
9d701248 | 6227 | { |
05641603 | 6228 | int i, pass; |
b75f962c | 6229 | bool sched_group_found = false; |
e9593da8 | 6230 | int min_cost_group = 0; |
9d701248 | 6231 | |
b16abbcb BC |
6232 | if (sched_fusion) |
6233 | return; | |
6234 | ||
9d701248 BS |
6235 | for (i = 0; i < ready.n_ready; i++) |
6236 | { | |
ce1ce33a | 6237 | rtx_insn *insn = ready_element (&ready, i); |
05641603 | 6238 | if (SCHED_GROUP_P (insn)) |
b75f962c BS |
6239 | { |
6240 | sched_group_found = true; | |
05641603 | 6241 | break; |
b75f962c | 6242 | } |
05641603 | 6243 | } |
b75f962c | 6244 | |
05641603 | 6245 | /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle |
e9593da8 JW |
6246 | such an insn first and note its cost. If at least one SCHED_GROUP_P insn |
6247 | gets queued, then all other insns get queued for one cycle later. */ | |
05641603 BS |
6248 | for (pass = sched_group_found ? 0 : 1; pass < 2; ) |
6249 | { | |
6250 | int n = ready.n_ready; | |
6251 | for (i = 0; i < n; i++) | |
9d701248 | 6252 | { |
ce1ce33a | 6253 | rtx_insn *insn = ready_element (&ready, i); |
05641603 BS |
6254 | int cost = 0; |
6255 | const char *reason = "resource conflict"; | |
26965010 | 6256 | |
05641603 BS |
6257 | if (DEBUG_INSN_P (insn)) |
6258 | continue; | |
6259 | ||
e9593da8 JW |
6260 | if (sched_group_found && !SCHED_GROUP_P (insn) |
6261 | && ((pass == 0) || (min_cost_group >= 1))) | |
05641603 BS |
6262 | { |
6263 | if (pass == 0) | |
6264 | continue; | |
6265 | cost = min_cost_group; | |
6266 | reason = "not in sched group"; | |
6267 | } | |
6268 | else if (modulo_epilogue_p | |
6269 | && INSN_EXACT_TICK (insn) == INVALID_TICK) | |
6270 | { | |
6271 | cost = max_insn_queue_index; | |
6272 | reason = "not an epilogue insn"; | |
6273 | } | |
6274 | else if (shadows_only_p && !SHADOW_P (insn)) | |
26965010 | 6275 | { |
05641603 BS |
6276 | cost = 1; |
6277 | reason = "not a shadow"; | |
6278 | } | |
6279 | else if (recog_memoized (insn) < 0) | |
6280 | { | |
6281 | if (!first_cycle_insn_p | |
6282 | && (GET_CODE (PATTERN (insn)) == ASM_INPUT | |
6283 | || asm_noperands (PATTERN (insn)) >= 0)) | |
6284 | cost = 1; | |
6285 | reason = "asm"; | |
6286 | } | |
7a1aca9c RS |
6287 | else if (sched_pressure != SCHED_PRESSURE_NONE) |
6288 | { | |
6289 | if (sched_pressure == SCHED_PRESSURE_MODEL | |
6290 | && INSN_TICK (insn) <= clock_var) | |
6291 | { | |
6292 | memcpy (temp_state, curr_state, dfa_state_size); | |
6293 | if (state_transition (temp_state, insn) >= 0) | |
6294 | INSN_TICK (insn) = clock_var + 1; | |
6295 | } | |
6296 | cost = 0; | |
6297 | } | |
05641603 BS |
6298 | else |
6299 | { | |
6300 | int delay_cost = 0; | |
6301 | ||
c203e8a7 | 6302 | if (delay_htab) |
26965010 | 6303 | { |
05641603 BS |
6304 | struct delay_pair *delay_entry; |
6305 | delay_entry | |
c203e8a7 TS |
6306 | = delay_htab->find_with_hash (insn, |
6307 | htab_hash_pointer (insn)); | |
05641603 BS |
6308 | while (delay_entry && delay_cost == 0) |
6309 | { | |
6310 | delay_cost = estimate_shadow_tick (delay_entry); | |
6311 | if (delay_cost > max_insn_queue_index) | |
6312 | delay_cost = max_insn_queue_index; | |
6313 | delay_entry = delay_entry->next_same_i1; | |
6314 | } | |
26965010 | 6315 | } |
26965010 | 6316 | |
05641603 BS |
6317 | memcpy (temp_state, curr_state, dfa_state_size); |
6318 | cost = state_transition (temp_state, insn); | |
6319 | if (cost < 0) | |
6320 | cost = 0; | |
6321 | else if (cost == 0) | |
6322 | cost = 1; | |
6323 | if (cost < delay_cost) | |
6324 | { | |
6325 | cost = delay_cost; | |
6326 | reason = "shadow tick"; | |
6327 | } | |
6328 | } | |
6329 | if (cost >= 1) | |
26965010 | 6330 | { |
05641603 BS |
6331 | if (SCHED_GROUP_P (insn) && cost > min_cost_group) |
6332 | min_cost_group = cost; | |
6333 | ready_remove (&ready, i); | |
3f9b5ffb JL |
6334 | /* Normally we'd want to queue INSN for COST cycles. However, |
6335 | if SCHED_GROUP_P is set, then we must ensure that nothing | |
6336 | else comes between INSN and its predecessor. If there is | |
6337 | some other insn ready to fire on the next cycle, then that | |
6338 | invariant would be broken. | |
6339 | ||
6340 | So when SCHED_GROUP_P is set, just queue this insn for a | |
6341 | single cycle. */ | |
6342 | queue_insn (insn, SCHED_GROUP_P (insn) ? 1 : cost, reason); | |
05641603 BS |
6343 | if (i + 1 < n) |
6344 | break; | |
26965010 | 6345 | } |
9d701248 | 6346 | } |
05641603 BS |
6347 | if (i == n) |
6348 | pass++; | |
9d701248 BS |
6349 | } |
6350 | } | |
6351 | ||
26965010 BS |
6352 | /* Called when we detect that the schedule is impossible. We examine the |
6353 | backtrack queue to find the earliest insn that caused this condition. */ | |
6354 | ||
6355 | static struct haifa_saved_data * | |
6356 | verify_shadows (void) | |
6357 | { | |
6358 | struct haifa_saved_data *save, *earliest_fail = NULL; | |
6359 | for (save = backtrack_queue; save; save = save->next) | |
6360 | { | |
6361 | int t; | |
6362 | struct delay_pair *pair = save->delay_pair; | |
f57aa6b0 | 6363 | rtx_insn *i1 = pair->i1; |
26965010 BS |
6364 | |
6365 | for (; pair; pair = pair->next_same_i1) | |
6366 | { | |
f57aa6b0 | 6367 | rtx_insn *i2 = pair->i2; |
26965010 BS |
6368 | |
6369 | if (QUEUE_INDEX (i2) == QUEUE_SCHEDULED) | |
6370 | continue; | |
6371 | ||
6372 | t = INSN_TICK (i1) + pair_delay (pair); | |
6373 | if (t < clock_var) | |
6374 | { | |
6375 | if (sched_verbose >= 2) | |
6376 | fprintf (sched_dump, | |
6377 | ";;\t\tfailed delay requirements for %d/%d (%d->%d)" | |
6378 | ", not ready\n", | |
6379 | INSN_UID (pair->i1), INSN_UID (pair->i2), | |
6380 | INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2)); | |
6381 | earliest_fail = save; | |
6382 | break; | |
6383 | } | |
6384 | if (QUEUE_INDEX (i2) >= 0) | |
6385 | { | |
6386 | int queued_for = INSN_TICK (i2); | |
6387 | ||
6388 | if (t < queued_for) | |
6389 | { | |
6390 | if (sched_verbose >= 2) | |
6391 | fprintf (sched_dump, | |
6392 | ";;\t\tfailed delay requirements for %d/%d" | |
6393 | " (%d->%d), queued too late\n", | |
6394 | INSN_UID (pair->i1), INSN_UID (pair->i2), | |
6395 | INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2)); | |
6396 | earliest_fail = save; | |
6397 | break; | |
6398 | } | |
6399 | } | |
6400 | } | |
6401 | } | |
6402 | ||
6403 | return earliest_fail; | |
6404 | } | |
6405 | ||
d5c67efd MK |
6406 | /* Print instructions together with useful scheduling information between |
6407 | HEAD and TAIL (inclusive). */ | |
6408 | static void | |
f57aa6b0 | 6409 | dump_insn_stream (rtx_insn *head, rtx_insn *tail) |
d5c67efd MK |
6410 | { |
6411 | fprintf (sched_dump, ";;\t| insn | prio |\n"); | |
6412 | ||
f57aa6b0 DM |
6413 | rtx_insn *next_tail = NEXT_INSN (tail); |
6414 | for (rtx_insn *insn = head; insn != next_tail; insn = NEXT_INSN (insn)) | |
d5c67efd MK |
6415 | { |
6416 | int priority = NOTE_P (insn) ? 0 : INSN_PRIORITY (insn); | |
6417 | const char *pattern = (NOTE_P (insn) | |
6418 | ? "note" | |
6419 | : str_pattern_slim (PATTERN (insn))); | |
6420 | ||
6421 | fprintf (sched_dump, ";;\t| %4d | %4d | %-30s ", | |
6422 | INSN_UID (insn), priority, pattern); | |
6423 | ||
6424 | if (sched_verbose >= 4) | |
6425 | { | |
81e63b63 | 6426 | if (NOTE_P (insn) || LABEL_P (insn) || recog_memoized (insn) < 0) |
d5c67efd MK |
6427 | fprintf (sched_dump, "nothing"); |
6428 | else | |
6429 | print_reservation (sched_dump, insn); | |
6430 | } | |
6431 | fprintf (sched_dump, "\n"); | |
6432 | } | |
6433 | } | |
6434 | ||
496d7bb0 MK |
6435 | /* Use forward list scheduling to rearrange insns of block pointed to by |
6436 | TARGET_BB, possibly bringing insns from subsequent blocks in the same | |
6437 | region. */ | |
8c660648 | 6438 | |
06d7e8e7 | 6439 | bool |
975ccf22 | 6440 | schedule_block (basic_block *target_bb, state_t init_state) |
8c660648 | 6441 | { |
894fd6f2 | 6442 | int i; |
06d7e8e7 | 6443 | bool success = modulo_ii == 0; |
51564684 | 6444 | struct sched_block_state ls; |
fae15c93 | 6445 | state_t temp_state = NULL; /* It is used for multipass scheduling. */ |
58fb7809 | 6446 | int sort_p, advance, start_clock_var; |
8c660648 | 6447 | |
63de6c74 | 6448 | /* Head/tail info for this block. */ |
ce1ce33a | 6449 | rtx_insn *prev_head = current_sched_info->prev_head; |
dc01c3d1 | 6450 | rtx_insn *next_tail = current_sched_info->next_tail; |
ce1ce33a DM |
6451 | rtx_insn *head = NEXT_INSN (prev_head); |
6452 | rtx_insn *tail = PREV_INSN (next_tail); | |
8c660648 | 6453 | |
4f66c9bc | 6454 | if ((current_sched_info->flags & DONT_BREAK_DEPENDENCIES) == 0 |
b16abbcb | 6455 | && sched_pressure != SCHED_PRESSURE_MODEL && !sched_fusion) |
1a83e602 BS |
6456 | find_modifiable_mems (head, tail); |
6457 | ||
484df988 JL |
6458 | /* We used to have code to avoid getting parameters moved from hard |
6459 | argument registers into pseudos. | |
8c660648 | 6460 | |
484df988 JL |
6461 | However, it was removed when it proved to be of marginal benefit |
6462 | and caused problems because schedule_block and compute_forward_dependences | |
6463 | had different notions of what the "head" insn was. */ | |
8c660648 | 6464 | |
535a42b1 | 6465 | gcc_assert (head != tail || INSN_P (head)); |
8c660648 | 6466 | |
e2f6ff94 | 6467 | haifa_recovery_bb_recently_added_p = false; |
496d7bb0 | 6468 | |
26965010 BS |
6469 | backtrack_queue = NULL; |
6470 | ||
63de6c74 | 6471 | /* Debug info. */ |
8c660648 | 6472 | if (sched_verbose) |
d5c67efd MK |
6473 | { |
6474 | dump_new_block_header (0, *target_bb, head, tail); | |
6475 | ||
6476 | if (sched_verbose >= 2) | |
4bd12f3d MK |
6477 | { |
6478 | dump_insn_stream (head, tail); | |
6479 | memset (&rank_for_schedule_stats, 0, | |
6480 | sizeof (rank_for_schedule_stats)); | |
6481 | } | |
d5c67efd | 6482 | } |
8c660648 | 6483 | |
975ccf22 BS |
6484 | if (init_state == NULL) |
6485 | state_reset (curr_state); | |
6486 | else | |
6487 | memcpy (curr_state, init_state, dfa_state_size); | |
8c660648 | 6488 | |
e855c69d | 6489 | /* Clear the ready list. */ |
176f9a7b | 6490 | ready.first = ready.veclen - 1; |
176f9a7b | 6491 | ready.n_ready = 0; |
b5b8b0ac | 6492 | ready.n_debug = 0; |
8c660648 | 6493 | |
fa0aee89 PB |
6494 | /* It is used for first cycle multipass scheduling. */ |
6495 | temp_state = alloca (dfa_state_size); | |
fae15c93 | 6496 | |
38f8b050 JR |
6497 | if (targetm.sched.init) |
6498 | targetm.sched.init (sched_dump, sched_verbose, ready.veclen); | |
e4da5f6d | 6499 | |
89076bb3 | 6500 | /* We start inserting insns after PREV_HEAD. */ |
06bc63d1 | 6501 | last_scheduled_insn = prev_head; |
90831096 | 6502 | last_nondebug_scheduled_insn = NULL; |
ce1ce33a | 6503 | nonscheduled_insns_begin = NULL; |
8c660648 | 6504 | |
b5b8b0ac | 6505 | gcc_assert ((NOTE_P (last_scheduled_insn) |
a59d15cf | 6506 | || DEBUG_INSN_P (last_scheduled_insn)) |
496d7bb0 MK |
6507 | && BLOCK_FOR_INSN (last_scheduled_insn) == *target_bb); |
6508 | ||
1708fd40 BS |
6509 | /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the |
6510 | queue. */ | |
8c660648 JL |
6511 | q_ptr = 0; |
6512 | q_size = 0; | |
fae15c93 | 6513 | |
3dc99c19 | 6514 | insn_queue = XALLOCAVEC (rtx_insn_list *, max_insn_queue_index + 1); |
fa0aee89 | 6515 | memset (insn_queue, 0, (max_insn_queue_index + 1) * sizeof (rtx)); |
8c660648 | 6516 | |
197043f5 RH |
6517 | /* Start just before the beginning of time. */ |
6518 | clock_var = -1; | |
63f54b1a | 6519 | |
b8698a0f | 6520 | /* We need queue and ready lists and clock_var be initialized |
63f54b1a MK |
6521 | in try_ready () (which is called through init_ready_list ()). */ |
6522 | (*current_sched_info->init_ready_list) (); | |
6523 | ||
4dd9ac6c MK |
6524 | if (sched_pressure) |
6525 | sched_pressure_start_bb (*target_bb); | |
7a1aca9c | 6526 | |
6f8dd94b EB |
6527 | /* The algorithm is O(n^2) in the number of ready insns at any given |
6528 | time in the worst case. Before reload we are more likely to have | |
6529 | big lists so truncate them to a reasonable size. */ | |
b5b8b0ac AO |
6530 | if (!reload_completed |
6531 | && ready.n_ready - ready.n_debug > MAX_SCHED_READY_INSNS) | |
6f8dd94b | 6532 | { |
696d5fa1 MK |
6533 | ready_sort_debug (&ready); |
6534 | ready_sort_real (&ready); | |
6f8dd94b | 6535 | |
b5b8b0ac AO |
6536 | /* Find first free-standing insn past MAX_SCHED_READY_INSNS. |
6537 | If there are debug insns, we know they're first. */ | |
6538 | for (i = MAX_SCHED_READY_INSNS + ready.n_debug; i < ready.n_ready; i++) | |
6f8dd94b EB |
6539 | if (!SCHED_GROUP_P (ready_element (&ready, i))) |
6540 | break; | |
6541 | ||
6542 | if (sched_verbose >= 2) | |
6543 | { | |
6544 | fprintf (sched_dump, | |
696d5fa1 MK |
6545 | ";;\t\tReady list on entry: %d insns: ", ready.n_ready); |
6546 | debug_ready_list (&ready); | |
6f8dd94b EB |
6547 | fprintf (sched_dump, |
6548 | ";;\t\t before reload => truncated to %d insns\n", i); | |
6549 | } | |
6550 | ||
b631c5f7 SP |
6551 | /* Delay all insns past it for 1 cycle. If debug counter is |
6552 | activated make an exception for the insn right after | |
9b69cf83 | 6553 | nonscheduled_insns_begin. */ |
b631c5f7 | 6554 | { |
ce1ce33a | 6555 | rtx_insn *skip_insn; |
b631c5f7 SP |
6556 | |
6557 | if (dbg_cnt (sched_insn) == false) | |
06bc63d1 | 6558 | skip_insn = first_nonscheduled_insn (); |
b631c5f7 | 6559 | else |
ce1ce33a | 6560 | skip_insn = NULL; |
b631c5f7 SP |
6561 | |
6562 | while (i < ready.n_ready) | |
6563 | { | |
ce1ce33a | 6564 | rtx_insn *insn; |
b631c5f7 SP |
6565 | |
6566 | insn = ready_remove (&ready, i); | |
6567 | ||
6568 | if (insn != skip_insn) | |
ec4efea9 | 6569 | queue_insn (insn, 1, "list truncated"); |
b631c5f7 | 6570 | } |
9b69cf83 BS |
6571 | if (skip_insn) |
6572 | ready_add (&ready, skip_insn, true); | |
b631c5f7 | 6573 | } |
6f8dd94b EB |
6574 | } |
6575 | ||
496d7bb0 MK |
6576 | /* Now we can restore basic block notes and maintain precise cfg. */ |
6577 | restore_bb_notes (*target_bb); | |
6578 | ||
63f54b1a MK |
6579 | last_clock_var = -1; |
6580 | ||
58fb7809 | 6581 | advance = 0; |
197043f5 | 6582 | |
9771b263 | 6583 | gcc_assert (scheduled_insns.length () == 0); |
30028c85 | 6584 | sort_p = TRUE; |
26965010 | 6585 | must_backtrack = false; |
06d7e8e7 BS |
6586 | modulo_insns_scheduled = 0; |
6587 | ||
6588 | ls.modulo_epilogue = false; | |
f50d71ec | 6589 | ls.first_cycle_insn_p = true; |
26965010 | 6590 | |
63de6c74 | 6591 | /* Loop until all the insns in BB are scheduled. */ |
1708fd40 | 6592 | while ((*current_sched_info->schedule_more_p) ()) |
8c660648 | 6593 | { |
1a83e602 | 6594 | perform_replacements_new_cycle (); |
58fb7809 | 6595 | do |
8c660648 | 6596 | { |
58fb7809 VM |
6597 | start_clock_var = clock_var; |
6598 | ||
6599 | clock_var++; | |
1d088dee | 6600 | |
58fb7809 | 6601 | advance_one_cycle (); |
1d088dee | 6602 | |
58fb7809 VM |
6603 | /* Add to the ready list all pending insns that can be issued now. |
6604 | If there are no ready insns, increment clock until one | |
6605 | is ready and add all pending insns at that point to the ready | |
6606 | list. */ | |
6607 | queue_to_ready (&ready); | |
1d088dee | 6608 | |
535a42b1 | 6609 | gcc_assert (ready.n_ready); |
1d088dee | 6610 | |
58fb7809 VM |
6611 | if (sched_verbose >= 2) |
6612 | { | |
d5c67efd | 6613 | fprintf (sched_dump, ";;\t\tReady list after queue_to_ready:"); |
58fb7809 VM |
6614 | debug_ready_list (&ready); |
6615 | } | |
6616 | advance -= clock_var - start_clock_var; | |
8c660648 | 6617 | } |
58fb7809 | 6618 | while (advance > 0); |
8c660648 | 6619 | |
06d7e8e7 BS |
6620 | if (ls.modulo_epilogue) |
6621 | { | |
6622 | int stage = clock_var / modulo_ii; | |
6623 | if (stage > modulo_last_stage * 2 + 2) | |
6624 | { | |
6625 | if (sched_verbose >= 2) | |
6626 | fprintf (sched_dump, | |
6627 | ";;\t\tmodulo scheduled succeeded at II %d\n", | |
6628 | modulo_ii); | |
6629 | success = true; | |
6630 | goto end_schedule; | |
6631 | } | |
6632 | } | |
6633 | else if (modulo_ii > 0) | |
6634 | { | |
6635 | int stage = clock_var / modulo_ii; | |
6636 | if (stage > modulo_max_stages) | |
6637 | { | |
6638 | if (sched_verbose >= 2) | |
6639 | fprintf (sched_dump, | |
6640 | ";;\t\tfailing schedule due to excessive stages\n"); | |
6641 | goto end_schedule; | |
6642 | } | |
6643 | if (modulo_n_insns == modulo_insns_scheduled | |
6644 | && stage > modulo_last_stage) | |
6645 | { | |
6646 | if (sched_verbose >= 2) | |
6647 | fprintf (sched_dump, | |
6648 | ";;\t\tfound kernel after %d stages, II %d\n", | |
6649 | stage, modulo_ii); | |
6650 | ls.modulo_epilogue = true; | |
6651 | } | |
6652 | } | |
6653 | ||
6654 | prune_ready_list (temp_state, true, false, ls.modulo_epilogue); | |
9d701248 | 6655 | if (ready.n_ready == 0) |
7b1ac803 | 6656 | continue; |
26965010 BS |
6657 | if (must_backtrack) |
6658 | goto do_backtrack; | |
9d701248 | 6659 | |
26965010 | 6660 | ls.shadows_only_p = false; |
7b1ac803 | 6661 | cycle_issued_insns = 0; |
51564684 | 6662 | ls.can_issue_more = issue_rate; |
7b1ac803 | 6663 | for (;;) |
30028c85 | 6664 | { |
ce1ce33a | 6665 | rtx_insn *insn; |
7b1ac803 | 6666 | int cost; |
26965010 | 6667 | bool asm_p; |
1d088dee | 6668 | |
7b1ac803 | 6669 | if (sort_p && ready.n_ready > 0) |
30028c85 | 6670 | { |
7b1ac803 BS |
6671 | /* Sort the ready list based on priority. This must be |
6672 | done every iteration through the loop, as schedule_insn | |
6673 | may have readied additional insns that will not be | |
6674 | sorted correctly. */ | |
6675 | ready_sort (&ready); | |
6676 | ||
6677 | if (sched_verbose >= 2) | |
6678 | { | |
d5c67efd MK |
6679 | fprintf (sched_dump, |
6680 | ";;\t\tReady list after ready_sort: "); | |
7b1ac803 BS |
6681 | debug_ready_list (&ready); |
6682 | } | |
30028c85 | 6683 | } |
197043f5 | 6684 | |
7b1ac803 BS |
6685 | /* We don't want md sched reorder to even see debug isns, so put |
6686 | them out right away. */ | |
6687 | if (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0)) | |
6688 | && (*current_sched_info->schedule_more_p) ()) | |
b5b8b0ac | 6689 | { |
7b1ac803 BS |
6690 | while (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0))) |
6691 | { | |
ce1ce33a | 6692 | rtx_insn *insn = ready_remove_first (&ready); |
7b1ac803 BS |
6693 | gcc_assert (DEBUG_INSN_P (insn)); |
6694 | (*current_sched_info->begin_schedule_ready) (insn); | |
9771b263 | 6695 | scheduled_insns.safe_push (insn); |
7b1ac803 BS |
6696 | last_scheduled_insn = insn; |
6697 | advance = schedule_insn (insn); | |
6698 | gcc_assert (advance == 0); | |
6699 | if (ready.n_ready > 0) | |
6700 | ready_sort (&ready); | |
6701 | } | |
b5b8b0ac AO |
6702 | } |
6703 | ||
51564684 | 6704 | if (ls.first_cycle_insn_p && !ready.n_ready) |
7b1ac803 | 6705 | break; |
e1306f49 | 6706 | |
26965010 | 6707 | resume_after_backtrack: |
7b1ac803 BS |
6708 | /* Allow the target to reorder the list, typically for |
6709 | better instruction bundling. */ | |
6710 | if (sort_p | |
6711 | && (ready.n_ready == 0 | |
6712 | || !SCHED_GROUP_P (ready_element (&ready, 0)))) | |
6713 | { | |
51564684 BS |
6714 | if (ls.first_cycle_insn_p && targetm.sched.reorder) |
6715 | ls.can_issue_more | |
7b1ac803 BS |
6716 | = targetm.sched.reorder (sched_dump, sched_verbose, |
6717 | ready_lastpos (&ready), | |
6718 | &ready.n_ready, clock_var); | |
51564684 BS |
6719 | else if (!ls.first_cycle_insn_p && targetm.sched.reorder2) |
6720 | ls.can_issue_more | |
7b1ac803 BS |
6721 | = targetm.sched.reorder2 (sched_dump, sched_verbose, |
6722 | ready.n_ready | |
6723 | ? ready_lastpos (&ready) : NULL, | |
6724 | &ready.n_ready, clock_var); | |
6725 | } | |
fae15c93 | 6726 | |
7b1ac803 | 6727 | restart_choose_ready: |
6d0de005 | 6728 | if (sched_verbose >= 2) |
fae15c93 | 6729 | { |
496d7bb0 | 6730 | fprintf (sched_dump, ";;\tReady list (t = %3d): ", |
fae15c93 VM |
6731 | clock_var); |
6732 | debug_ready_list (&ready); | |
60867e8c | 6733 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED) |
ce18efcb | 6734 | print_curr_reg_pressure (); |
fae15c93 VM |
6735 | } |
6736 | ||
b8698a0f | 6737 | if (ready.n_ready == 0 |
51564684 | 6738 | && ls.can_issue_more |
b8698a0f | 6739 | && reload_completed) |
fae15c93 | 6740 | { |
fa0aee89 PB |
6741 | /* Allow scheduling insns directly from the queue in case |
6742 | there's nothing better to do (ready list is empty) but | |
6743 | there are still vacant dispatch slots in the current cycle. */ | |
6744 | if (sched_verbose >= 6) | |
62e5bf5d | 6745 | fprintf (sched_dump,";;\t\tSecond chance\n"); |
fa0aee89 PB |
6746 | memcpy (temp_state, curr_state, dfa_state_size); |
6747 | if (early_queue_to_ready (temp_state, &ready)) | |
6748 | ready_sort (&ready); | |
fae15c93 | 6749 | } |
569fa502 | 6750 | |
b5b8b0ac | 6751 | if (ready.n_ready == 0 |
51564684 | 6752 | || !ls.can_issue_more |
fa0aee89 PB |
6753 | || state_dead_lock_p (curr_state) |
6754 | || !(*current_sched_info->schedule_more_p) ()) | |
6755 | break; | |
1d088dee | 6756 | |
fa0aee89 PB |
6757 | /* Select and remove the insn from the ready list. */ |
6758 | if (sort_p) | |
496d7bb0 | 6759 | { |
b631c5f7 SP |
6760 | int res; |
6761 | ||
ce1ce33a | 6762 | insn = NULL; |
51564684 | 6763 | res = choose_ready (&ready, ls.first_cycle_insn_p, &insn); |
b631c5f7 SP |
6764 | |
6765 | if (res < 0) | |
6766 | /* Finish cycle. */ | |
6767 | break; | |
6768 | if (res > 0) | |
7b1ac803 | 6769 | goto restart_choose_ready; |
b631c5f7 SP |
6770 | |
6771 | gcc_assert (insn != NULL_RTX); | |
496d7bb0 | 6772 | } |
fa0aee89 PB |
6773 | else |
6774 | insn = ready_remove_first (&ready); | |
1d088dee | 6775 | |
7a1aca9c | 6776 | if (sched_pressure != SCHED_PRESSURE_NONE |
60867e8c | 6777 | && INSN_TICK (insn) > clock_var) |
ce18efcb VM |
6778 | { |
6779 | ready_add (&ready, insn, true); | |
6780 | advance = 1; | |
6781 | break; | |
6782 | } | |
6783 | ||
fa0aee89 PB |
6784 | if (targetm.sched.dfa_new_cycle |
6785 | && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose, | |
6786 | insn, last_clock_var, | |
6787 | clock_var, &sort_p)) | |
63f54b1a MK |
6788 | /* SORT_P is used by the target to override sorting |
6789 | of the ready list. This is needed when the target | |
6790 | has modified its internal structures expecting that | |
6791 | the insn will be issued next. As we need the insn | |
6792 | to have the highest priority (so it will be returned by | |
6793 | the ready_remove_first call above), we invoke | |
6794 | ready_add (&ready, insn, true). | |
b8698a0f L |
6795 | But, still, there is one issue: INSN can be later |
6796 | discarded by scheduler's front end through | |
63f54b1a | 6797 | current_sched_info->can_schedule_ready_p, hence, won't |
b8698a0f | 6798 | be issued next. */ |
fa0aee89 | 6799 | { |
63f54b1a MK |
6800 | ready_add (&ready, insn, true); |
6801 | break; | |
fa0aee89 | 6802 | } |
1d088dee | 6803 | |
fa0aee89 | 6804 | sort_p = TRUE; |
e1306f49 | 6805 | |
496d7bb0 MK |
6806 | if (current_sched_info->can_schedule_ready_p |
6807 | && ! (*current_sched_info->can_schedule_ready_p) (insn)) | |
6808 | /* We normally get here only if we don't want to move | |
6809 | insn from the split block. */ | |
6810 | { | |
1a83e602 | 6811 | TODO_SPEC (insn) = DEP_POSTPONED; |
7b1ac803 | 6812 | goto restart_choose_ready; |
496d7bb0 MK |
6813 | } |
6814 | ||
c203e8a7 | 6815 | if (delay_htab) |
26965010 BS |
6816 | { |
6817 | /* If this insn is the first part of a delay-slot pair, record a | |
6818 | backtrack point. */ | |
6819 | struct delay_pair *delay_entry; | |
6820 | delay_entry | |
c203e8a7 | 6821 | = delay_htab->find_with_hash (insn, htab_hash_pointer (insn)); |
26965010 BS |
6822 | if (delay_entry) |
6823 | { | |
6824 | save_backtrack_point (delay_entry, ls); | |
6825 | if (sched_verbose >= 2) | |
6826 | fprintf (sched_dump, ";;\t\tsaving backtrack point\n"); | |
6827 | } | |
6828 | } | |
6829 | ||
b8698a0f L |
6830 | /* DECISION is made. */ |
6831 | ||
06d7e8e7 BS |
6832 | if (modulo_ii > 0 && INSN_UID (insn) < modulo_iter0_max_uid) |
6833 | { | |
6834 | modulo_insns_scheduled++; | |
6835 | modulo_last_stage = clock_var / modulo_ii; | |
6836 | } | |
496d7bb0 MK |
6837 | if (TODO_SPEC (insn) & SPECULATIVE) |
6838 | generate_recovery_code (insn); | |
6839 | ||
ac44248e | 6840 | if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON)) |
7942e47e RY |
6841 | targetm.sched.dispatch_do (insn, ADD_TO_DISPATCH_WINDOW); |
6842 | ||
86014d07 BS |
6843 | /* Update counters, etc in the scheduler's front end. */ |
6844 | (*current_sched_info->begin_schedule_ready) (insn); | |
9771b263 | 6845 | scheduled_insns.safe_push (insn); |
5c24671b AO |
6846 | gcc_assert (NONDEBUG_INSN_P (insn)); |
6847 | last_nondebug_scheduled_insn = last_scheduled_insn = insn; | |
b8698a0f | 6848 | |
9d701248 BS |
6849 | if (recog_memoized (insn) >= 0) |
6850 | { | |
fca96842 | 6851 | memcpy (temp_state, curr_state, dfa_state_size); |
9d701248 | 6852 | cost = state_transition (curr_state, insn); |
b16abbcb | 6853 | if (sched_pressure != SCHED_PRESSURE_WEIGHTED && !sched_fusion) |
9d701248 | 6854 | gcc_assert (cost < 0); |
fca96842 BS |
6855 | if (memcmp (temp_state, curr_state, dfa_state_size) != 0) |
6856 | cycle_issued_insns++; | |
9d701248 BS |
6857 | asm_p = false; |
6858 | } | |
6859 | else | |
6860 | asm_p = (GET_CODE (PATTERN (insn)) == ASM_INPUT | |
6861 | || asm_noperands (PATTERN (insn)) >= 0); | |
1d088dee | 6862 | |
c237e94a | 6863 | if (targetm.sched.variable_issue) |
51564684 | 6864 | ls.can_issue_more = |
5fd9b178 | 6865 | targetm.sched.variable_issue (sched_dump, sched_verbose, |
51564684 | 6866 | insn, ls.can_issue_more); |
85d69216 JL |
6867 | /* A naked CLOBBER or USE generates no instruction, so do |
6868 | not count them against the issue rate. */ | |
6869 | else if (GET_CODE (PATTERN (insn)) != USE | |
6870 | && GET_CODE (PATTERN (insn)) != CLOBBER) | |
51564684 | 6871 | ls.can_issue_more--; |
63f54b1a | 6872 | advance = schedule_insn (insn); |
d57f1617 | 6873 | |
26965010 BS |
6874 | if (SHADOW_P (insn)) |
6875 | ls.shadows_only_p = true; | |
6876 | ||
d57f1617 VM |
6877 | /* After issuing an asm insn we should start a new cycle. */ |
6878 | if (advance == 0 && asm_p) | |
6879 | advance = 1; | |
26965010 BS |
6880 | |
6881 | if (must_backtrack) | |
6882 | break; | |
6883 | ||
58fb7809 VM |
6884 | if (advance != 0) |
6885 | break; | |
8c660648 | 6886 | |
51564684 | 6887 | ls.first_cycle_insn_p = false; |
30028c85 | 6888 | if (ready.n_ready > 0) |
06d7e8e7 BS |
6889 | prune_ready_list (temp_state, false, ls.shadows_only_p, |
6890 | ls.modulo_epilogue); | |
b4ead7d4 | 6891 | } |
8c660648 | 6892 | |
26965010 BS |
6893 | do_backtrack: |
6894 | if (!must_backtrack) | |
6895 | for (i = 0; i < ready.n_ready; i++) | |
6896 | { | |
f57aa6b0 | 6897 | rtx_insn *insn = ready_element (&ready, i); |
26965010 BS |
6898 | if (INSN_EXACT_TICK (insn) == clock_var) |
6899 | { | |
6900 | must_backtrack = true; | |
6901 | clock_var++; | |
6902 | break; | |
6903 | } | |
6904 | } | |
06d7e8e7 BS |
6905 | if (must_backtrack && modulo_ii > 0) |
6906 | { | |
6907 | if (modulo_backtracks_left == 0) | |
6908 | goto end_schedule; | |
6909 | modulo_backtracks_left--; | |
6910 | } | |
26965010 BS |
6911 | while (must_backtrack) |
6912 | { | |
6913 | struct haifa_saved_data *failed; | |
ce1ce33a | 6914 | rtx_insn *failed_insn; |
26965010 BS |
6915 | |
6916 | must_backtrack = false; | |
6917 | failed = verify_shadows (); | |
6918 | gcc_assert (failed); | |
6919 | ||
6920 | failed_insn = failed->delay_pair->i1; | |
1a83e602 BS |
6921 | /* Clear these queues. */ |
6922 | perform_replacements_new_cycle (); | |
e2724e63 | 6923 | toggle_cancelled_flags (false); |
26965010 BS |
6924 | unschedule_insns_until (failed_insn); |
6925 | while (failed != backtrack_queue) | |
6926 | free_topmost_backtrack_point (true); | |
6927 | restore_last_backtrack_point (&ls); | |
6928 | if (sched_verbose >= 2) | |
6929 | fprintf (sched_dump, ";;\t\trewind to cycle %d\n", clock_var); | |
6930 | /* Delay by at least a cycle. This could cause additional | |
6931 | backtracking. */ | |
6932 | queue_insn (failed_insn, 1, "backtracked"); | |
6933 | advance = 0; | |
6934 | if (must_backtrack) | |
6935 | continue; | |
6936 | if (ready.n_ready > 0) | |
6937 | goto resume_after_backtrack; | |
6938 | else | |
6939 | { | |
6940 | if (clock_var == 0 && ls.first_cycle_insn_p) | |
6941 | goto end_schedule; | |
6942 | advance = 1; | |
6943 | break; | |
6944 | } | |
6945 | } | |
f50d71ec | 6946 | ls.first_cycle_insn_p = true; |
26965010 | 6947 | } |
06d7e8e7 BS |
6948 | if (ls.modulo_epilogue) |
6949 | success = true; | |
26965010 | 6950 | end_schedule: |
2fb5f0da | 6951 | if (!ls.first_cycle_insn_p || advance) |
f50d71ec | 6952 | advance_one_cycle (); |
1a83e602 | 6953 | perform_replacements_new_cycle (); |
06d7e8e7 BS |
6954 | if (modulo_ii > 0) |
6955 | { | |
6956 | /* Once again, debug insn suckiness: they can be on the ready list | |
6957 | even if they have unresolved dependencies. To make our view | |
6958 | of the world consistent, remove such "ready" insns. */ | |
6959 | restart_debug_insn_loop: | |
6960 | for (i = ready.n_ready - 1; i >= 0; i--) | |
6961 | { | |
f57aa6b0 | 6962 | rtx_insn *x; |
06d7e8e7 BS |
6963 | |
6964 | x = ready_element (&ready, i); | |
6965 | if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x)) != NULL | |
6966 | || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x)) != NULL) | |
6967 | { | |
6968 | ready_remove (&ready, i); | |
6969 | goto restart_debug_insn_loop; | |
6970 | } | |
6971 | } | |
6972 | for (i = ready.n_ready - 1; i >= 0; i--) | |
6973 | { | |
ce1ce33a | 6974 | rtx_insn *x; |
06d7e8e7 BS |
6975 | |
6976 | x = ready_element (&ready, i); | |
6977 | resolve_dependencies (x); | |
6978 | } | |
6979 | for (i = 0; i <= max_insn_queue_index; i++) | |
6980 | { | |
3dc99c19 | 6981 | rtx_insn_list *link; |
06d7e8e7 BS |
6982 | while ((link = insn_queue[i]) != NULL) |
6983 | { | |
3dc99c19 DM |
6984 | rtx_insn *x = link->insn (); |
6985 | insn_queue[i] = link->next (); | |
06d7e8e7 BS |
6986 | QUEUE_INDEX (x) = QUEUE_NOWHERE; |
6987 | free_INSN_LIST_node (link); | |
6988 | resolve_dependencies (x); | |
6989 | } | |
6990 | } | |
6991 | } | |
6992 | ||
1a83e602 BS |
6993 | if (!success) |
6994 | undo_all_replacements (); | |
6995 | ||
b4ead7d4 BS |
6996 | /* Debug info. */ |
6997 | if (sched_verbose) | |
6998 | { | |
6999 | fprintf (sched_dump, ";;\tReady list (final): "); | |
7000 | debug_ready_list (&ready); | |
b4ead7d4 | 7001 | } |
8c660648 | 7002 | |
06d7e8e7 | 7003 | if (modulo_ii == 0 && current_sched_info->queue_must_finish_empty) |
63f54b1a MK |
7004 | /* Sanity check -- queue must be empty now. Meaningless if region has |
7005 | multiple bbs. */ | |
b5b8b0ac | 7006 | gcc_assert (!q_size && !ready.n_ready && !ready.n_debug); |
06d7e8e7 | 7007 | else if (modulo_ii == 0) |
30028c85 | 7008 | { |
63f54b1a MK |
7009 | /* We must maintain QUEUE_INDEX between blocks in region. */ |
7010 | for (i = ready.n_ready - 1; i >= 0; i--) | |
496d7bb0 | 7011 | { |
f57aa6b0 | 7012 | rtx_insn *x; |
b8698a0f | 7013 | |
496d7bb0 MK |
7014 | x = ready_element (&ready, i); |
7015 | QUEUE_INDEX (x) = QUEUE_NOWHERE; | |
b953c2b8 | 7016 | TODO_SPEC (x) = HARD_DEP; |
496d7bb0 | 7017 | } |
63f54b1a | 7018 | |
b8698a0f | 7019 | if (q_size) |
63f54b1a MK |
7020 | for (i = 0; i <= max_insn_queue_index; i++) |
7021 | { | |
96b70414 DM |
7022 | rtx_insn_list *link; |
7023 | for (link = insn_queue[i]; link; link = link->next ()) | |
496d7bb0 | 7024 | { |
f57aa6b0 | 7025 | rtx_insn *x; |
496d7bb0 | 7026 | |
96b70414 | 7027 | x = link->insn (); |
496d7bb0 | 7028 | QUEUE_INDEX (x) = QUEUE_NOWHERE; |
b953c2b8 | 7029 | TODO_SPEC (x) = HARD_DEP; |
496d7bb0 | 7030 | } |
63f54b1a MK |
7031 | free_INSN_LIST_list (&insn_queue[i]); |
7032 | } | |
496d7bb0 | 7033 | } |
1d088dee | 7034 | |
7a1aca9c RS |
7035 | if (sched_pressure == SCHED_PRESSURE_MODEL) |
7036 | model_end_schedule (); | |
7037 | ||
06d7e8e7 BS |
7038 | if (success) |
7039 | { | |
7040 | commit_schedule (prev_head, tail, target_bb); | |
7041 | if (sched_verbose) | |
7042 | fprintf (sched_dump, ";; total time = %d\n", clock_var); | |
7043 | } | |
7044 | else | |
7045 | last_scheduled_insn = tail; | |
7046 | ||
9771b263 | 7047 | scheduled_insns.truncate (0); |
e855c69d | 7048 | |
496d7bb0 | 7049 | if (!current_sched_info->queue_must_finish_empty |
e2f6ff94 | 7050 | || haifa_recovery_bb_recently_added_p) |
496d7bb0 | 7051 | { |
30028c85 VM |
7052 | /* INSN_TICK (minimum clock tick at which the insn becomes |
7053 | ready) may be not correct for the insn in the subsequent | |
7054 | blocks of the region. We should use a correct value of | |
7055 | `clock_var' or modify INSN_TICK. It is better to keep | |
7056 | clock_var value equal to 0 at the start of a basic block. | |
7057 | Therefore we modify INSN_TICK here. */ | |
63f54b1a | 7058 | fix_inter_tick (NEXT_INSN (prev_head), last_scheduled_insn); |
30028c85 VM |
7059 | } |
7060 | ||
38f8b050 | 7061 | if (targetm.sched.finish) |
e2f6ff94 | 7062 | { |
38f8b050 | 7063 | targetm.sched.finish (sched_dump, sched_verbose); |
fa10beec | 7064 | /* Target might have added some instructions to the scheduled block |
e2f6ff94 MK |
7065 | in its md_finish () hook. These new insns don't have any data |
7066 | initialized and to identify them we extend h_i_d so that they'll | |
e855c69d | 7067 | get zero luids. */ |
a95b23b4 | 7068 | sched_extend_luids (); |
e2f6ff94 | 7069 | } |
63f54b1a MK |
7070 | |
7071 | /* Update head/tail boundaries. */ | |
7072 | head = NEXT_INSN (prev_head); | |
7073 | tail = last_scheduled_insn; | |
7074 | ||
d5c67efd MK |
7075 | if (sched_verbose) |
7076 | { | |
7077 | fprintf (sched_dump, ";; new head = %d\n;; new tail = %d\n", | |
7078 | INSN_UID (head), INSN_UID (tail)); | |
7079 | ||
7080 | if (sched_verbose >= 2) | |
4bd12f3d MK |
7081 | { |
7082 | dump_insn_stream (head, tail); | |
b75ae7f4 MK |
7083 | print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats, |
7084 | NULL); | |
4bd12f3d | 7085 | } |
d5c67efd MK |
7086 | |
7087 | fprintf (sched_dump, "\n"); | |
7088 | } | |
7089 | ||
e855c69d | 7090 | head = restore_other_notes (head, NULL); |
8c660648 | 7091 | |
b4ead7d4 BS |
7092 | current_sched_info->head = head; |
7093 | current_sched_info->tail = tail; | |
26965010 BS |
7094 | |
7095 | free_backtrack_queue (); | |
06d7e8e7 BS |
7096 | |
7097 | return success; | |
8c660648 | 7098 | } |
b4ead7d4 | 7099 | \f |
63de6c74 | 7100 | /* Set_priorities: compute priority of each insn in the block. */ |
8c660648 | 7101 | |
b4ead7d4 | 7102 | int |
0e0ce50d | 7103 | set_priorities (rtx_insn *head, rtx_insn *tail) |
8c660648 | 7104 | { |
0e0ce50d | 7105 | rtx_insn *insn; |
8c660648 | 7106 | int n_insn; |
b8698a0f | 7107 | int sched_max_insns_priority = |
79ae11c4 | 7108 | current_sched_info->sched_max_insns_priority; |
ce1ce33a | 7109 | rtx_insn *prev_head; |
8c660648 | 7110 | |
a59d15cf | 7111 | if (head == tail && ! INSN_P (head)) |
b5b8b0ac | 7112 | gcc_unreachable (); |
8c660648 JL |
7113 | |
7114 | n_insn = 0; | |
63f54b1a MK |
7115 | |
7116 | prev_head = PREV_INSN (head); | |
8c660648 JL |
7117 | for (insn = tail; insn != prev_head; insn = PREV_INSN (insn)) |
7118 | { | |
63f54b1a | 7119 | if (!INSN_P (insn)) |
8c660648 JL |
7120 | continue; |
7121 | ||
58fb7809 | 7122 | n_insn++; |
8c660648 | 7123 | (void) priority (insn); |
79ae11c4 | 7124 | |
916fa4f0 MK |
7125 | gcc_assert (INSN_PRIORITY_KNOWN (insn)); |
7126 | ||
7127 | sched_max_insns_priority = MAX (sched_max_insns_priority, | |
7128 | INSN_PRIORITY (insn)); | |
8c660648 | 7129 | } |
63f54b1a MK |
7130 | |
7131 | current_sched_info->sched_max_insns_priority = sched_max_insns_priority; | |
8c660648 JL |
7132 | |
7133 | return n_insn; | |
7134 | } | |
7135 | ||
4111f1c9 | 7136 | /* Set sched_dump and sched_verbose for the desired debugging output. */ |
e855c69d AB |
7137 | void |
7138 | setup_sched_dump (void) | |
7139 | { | |
7140 | sched_verbose = sched_verbose_param; | |
4111f1c9 NB |
7141 | sched_dump = dump_file; |
7142 | if (!dump_file) | |
7143 | sched_verbose = 0; | |
e855c69d | 7144 | } |
496d7bb0 | 7145 | |
c4cd7435 AB |
7146 | /* Allocate data for register pressure sensitive scheduling. */ |
7147 | static void | |
7148 | alloc_global_sched_pressure_data (void) | |
7149 | { | |
7150 | if (sched_pressure != SCHED_PRESSURE_NONE) | |
7151 | { | |
7152 | int i, max_regno = max_reg_num (); | |
7153 | ||
7154 | if (sched_dump != NULL) | |
7155 | /* We need info about pseudos for rtl dumps about pseudo | |
7156 | classes and costs. */ | |
7157 | regstat_init_n_sets_and_refs (); | |
7158 | ira_set_pseudo_classes (true, sched_verbose ? sched_dump : NULL); | |
7159 | sched_regno_pressure_class | |
7160 | = (enum reg_class *) xmalloc (max_regno * sizeof (enum reg_class)); | |
7161 | for (i = 0; i < max_regno; i++) | |
7162 | sched_regno_pressure_class[i] | |
7163 | = (i < FIRST_PSEUDO_REGISTER | |
7164 | ? ira_pressure_class_translate[REGNO_REG_CLASS (i)] | |
7165 | : ira_pressure_class_translate[reg_allocno_class (i)]); | |
7166 | curr_reg_live = BITMAP_ALLOC (NULL); | |
7167 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED) | |
7168 | { | |
7169 | saved_reg_live = BITMAP_ALLOC (NULL); | |
7170 | region_ref_regs = BITMAP_ALLOC (NULL); | |
7171 | } | |
f380f608 RS |
7172 | if (sched_pressure == SCHED_PRESSURE_MODEL) |
7173 | tmp_bitmap = BITMAP_ALLOC (NULL); | |
4dd9ac6c | 7174 | |
6d5c37d0 PH |
7175 | /* Calculate number of CALL_SAVED_REGS and FIXED_REGS in register classes |
7176 | that we calculate register pressure for. */ | |
4dd9ac6c MK |
7177 | for (int c = 0; c < ira_pressure_classes_num; ++c) |
7178 | { | |
7179 | enum reg_class cl = ira_pressure_classes[c]; | |
7180 | ||
6d5c37d0 PH |
7181 | call_saved_regs_num[cl] = 0; |
7182 | fixed_regs_num[cl] = 0; | |
4dd9ac6c MK |
7183 | |
7184 | for (int i = 0; i < ira_class_hard_regs_num[cl]; ++i) | |
6d5c37d0 PH |
7185 | if (!call_used_regs[ira_class_hard_regs[cl][i]]) |
7186 | ++call_saved_regs_num[cl]; | |
7187 | else if (fixed_regs[ira_class_hard_regs[cl][i]]) | |
7188 | ++fixed_regs_num[cl]; | |
4dd9ac6c | 7189 | } |
c4cd7435 AB |
7190 | } |
7191 | } | |
7192 | ||
7193 | /* Free data for register pressure sensitive scheduling. Also called | |
7194 | from schedule_region when stopping sched-pressure early. */ | |
7195 | void | |
7196 | free_global_sched_pressure_data (void) | |
7197 | { | |
7198 | if (sched_pressure != SCHED_PRESSURE_NONE) | |
7199 | { | |
7200 | if (regstat_n_sets_and_refs != NULL) | |
7201 | regstat_free_n_sets_and_refs (); | |
7202 | if (sched_pressure == SCHED_PRESSURE_WEIGHTED) | |
7203 | { | |
7204 | BITMAP_FREE (region_ref_regs); | |
7205 | BITMAP_FREE (saved_reg_live); | |
7206 | } | |
f380f608 RS |
7207 | if (sched_pressure == SCHED_PRESSURE_MODEL) |
7208 | BITMAP_FREE (tmp_bitmap); | |
c4cd7435 AB |
7209 | BITMAP_FREE (curr_reg_live); |
7210 | free (sched_regno_pressure_class); | |
7211 | } | |
7212 | } | |
7213 | ||
b8698a0f | 7214 | /* Initialize some global state for the scheduler. This function works |
e855c69d AB |
7215 | with the common data shared between all the schedulers. It is called |
7216 | from the scheduler specific initialization routine. */ | |
8c660648 | 7217 | |
b4ead7d4 | 7218 | void |
10d22567 | 7219 | sched_init (void) |
8c660648 | 7220 | { |
63de6c74 | 7221 | /* Disable speculative loads in their presence if cc0 defined. */ |
058eb3b0 | 7222 | if (HAVE_cc0) |
8c660648 | 7223 | flag_schedule_speculative_load = 0; |
8c660648 | 7224 | |
ac44248e DM |
7225 | if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON)) |
7226 | targetm.sched.dispatch_do (NULL, DISPATCH_INIT); | |
7942e47e | 7227 | |
f20f2613 VM |
7228 | if (live_range_shrinkage_p) |
7229 | sched_pressure = SCHED_PRESSURE_WEIGHTED; | |
7230 | else if (flag_sched_pressure | |
7231 | && !reload_completed | |
7232 | && common_sched_info->sched_pass_id == SCHED_RGN_PASS) | |
c881de02 RS |
7233 | sched_pressure = ((enum sched_pressure_algorithm) |
7234 | PARAM_VALUE (PARAM_SCHED_PRESSURE_ALGORITHM)); | |
60867e8c RS |
7235 | else |
7236 | sched_pressure = SCHED_PRESSURE_NONE; | |
7942e47e | 7237 | |
7a1aca9c | 7238 | if (sched_pressure != SCHED_PRESSURE_NONE) |
8d49e7ef | 7239 | ira_setup_eliminable_regset (); |
ce18efcb | 7240 | |
496d7bb0 MK |
7241 | /* Initialize SPEC_INFO. */ |
7242 | if (targetm.sched.set_sched_flags) | |
7243 | { | |
7244 | spec_info = &spec_info_var; | |
7245 | targetm.sched.set_sched_flags (spec_info); | |
e855c69d AB |
7246 | |
7247 | if (spec_info->mask != 0) | |
7248 | { | |
7249 | spec_info->data_weakness_cutoff = | |
7250 | (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF) * MAX_DEP_WEAK) / 100; | |
7251 | spec_info->control_weakness_cutoff = | |
7252 | (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF) | |
7253 | * REG_BR_PROB_BASE) / 100; | |
7254 | } | |
496d7bb0 | 7255 | else |
6fc0bb99 | 7256 | /* So we won't read anything accidentally. */ |
e855c69d AB |
7257 | spec_info = NULL; |
7258 | ||
496d7bb0 MK |
7259 | } |
7260 | else | |
6fc0bb99 | 7261 | /* So we won't read anything accidentally. */ |
496d7bb0 MK |
7262 | spec_info = 0; |
7263 | ||
63de6c74 | 7264 | /* Initialize issue_rate. */ |
c237e94a | 7265 | if (targetm.sched.issue_rate) |
5fd9b178 | 7266 | issue_rate = targetm.sched.issue_rate (); |
c237e94a ZW |
7267 | else |
7268 | issue_rate = 1; | |
8c660648 | 7269 | |
d205caad MK |
7270 | if (targetm.sched.first_cycle_multipass_dfa_lookahead |
7271 | /* Don't use max_issue with reg_pressure scheduling. Multipass | |
7272 | scheduling and reg_pressure scheduling undo each other's decisions. */ | |
7273 | && sched_pressure == SCHED_PRESSURE_NONE) | |
e855c69d AB |
7274 | dfa_lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead (); |
7275 | else | |
7276 | dfa_lookahead = 0; | |
fae15c93 | 7277 | |
8aec463b MK |
7278 | /* Set to "0" so that we recalculate. */ |
7279 | max_lookahead_tries = 0; | |
7280 | ||
fa0aee89 PB |
7281 | if (targetm.sched.init_dfa_pre_cycle_insn) |
7282 | targetm.sched.init_dfa_pre_cycle_insn (); | |
1d088dee | 7283 | |
fa0aee89 PB |
7284 | if (targetm.sched.init_dfa_post_cycle_insn) |
7285 | targetm.sched.init_dfa_post_cycle_insn (); | |
1d088dee | 7286 | |
fa0aee89 PB |
7287 | dfa_start (); |
7288 | dfa_state_size = state_size (); | |
fae15c93 | 7289 | |
e855c69d | 7290 | init_alias_analysis (); |
f77e39fc | 7291 | |
26965010 BS |
7292 | if (!sched_no_dce) |
7293 | df_set_flags (DF_LR_RUN_DCE); | |
e855c69d | 7294 | df_note_add_problem (); |
f77e39fc | 7295 | |
e855c69d AB |
7296 | /* More problems needed for interloop dep calculation in SMS. */ |
7297 | if (common_sched_info->sched_pass_id == SCHED_SMS_PASS) | |
7298 | { | |
7299 | df_rd_add_problem (); | |
7300 | df_chain_add_problem (DF_DU_CHAIN + DF_UD_CHAIN); | |
7301 | } | |
7a403706 | 7302 | |
e855c69d | 7303 | df_analyze (); |
b8698a0f L |
7304 | |
7305 | /* Do not run DCE after reload, as this can kill nops inserted | |
e855c69d AB |
7306 | by bundling. */ |
7307 | if (reload_completed) | |
7308 | df_clear_flags (DF_LR_RUN_DCE); | |
a88f02e7 | 7309 | |
e855c69d | 7310 | regstat_compute_calls_crossed (); |
a88f02e7 | 7311 | |
38f8b050 JR |
7312 | if (targetm.sched.init_global) |
7313 | targetm.sched.init_global (sched_dump, sched_verbose, get_max_uid () + 1); | |
a88f02e7 | 7314 | |
c4cd7435 | 7315 | alloc_global_sched_pressure_data (); |
b8698a0f | 7316 | |
e855c69d AB |
7317 | curr_state = xmalloc (dfa_state_size); |
7318 | } | |
58565a33 | 7319 | |
e855c69d | 7320 | static void haifa_init_only_bb (basic_block, basic_block); |
496d7bb0 | 7321 | |
e855c69d AB |
7322 | /* Initialize data structures specific to the Haifa scheduler. */ |
7323 | void | |
7324 | haifa_sched_init (void) | |
7325 | { | |
7326 | setup_sched_dump (); | |
7327 | sched_init (); | |
7328 | ||
9771b263 | 7329 | scheduled_insns.create (0); |
9b69cf83 | 7330 | |
e855c69d AB |
7331 | if (spec_info != NULL) |
7332 | { | |
7333 | sched_deps_info->use_deps_list = 1; | |
7334 | sched_deps_info->generate_spec_deps = 1; | |
7335 | } | |
7336 | ||
7337 | /* Initialize luids, dependency caches, target and h_i_d for the | |
7338 | whole function. */ | |
7339 | { | |
e855c69d AB |
7340 | sched_init_bbs (); |
7341 | ||
8c681247 TS |
7342 | auto_vec<basic_block> bbs (n_basic_blocks_for_fn (cfun)); |
7343 | basic_block bb; | |
11cd3bed | 7344 | FOR_EACH_BB_FN (bb, cfun) |
9771b263 | 7345 | bbs.quick_push (bb); |
a95b23b4 | 7346 | sched_init_luids (bbs); |
e855c69d AB |
7347 | sched_deps_init (true); |
7348 | sched_extend_target (); | |
a95b23b4 | 7349 | haifa_init_h_i_d (bbs); |
e855c69d AB |
7350 | } |
7351 | ||
7352 | sched_init_only_bb = haifa_init_only_bb; | |
7353 | sched_split_block = sched_split_block_1; | |
7354 | sched_create_empty_bb = sched_create_empty_bb_1; | |
e2f6ff94 MK |
7355 | haifa_recovery_bb_ever_added_p = false; |
7356 | ||
e855c69d AB |
7357 | nr_begin_data = nr_begin_control = nr_be_in_data = nr_be_in_control = 0; |
7358 | before_recovery = 0; | |
7359 | after_recovery = 0; | |
06d7e8e7 BS |
7360 | |
7361 | modulo_ii = 0; | |
e855c69d | 7362 | } |
8c660648 | 7363 | |
e855c69d | 7364 | /* Finish work with the data specific to the Haifa scheduler. */ |
a88f02e7 | 7365 | void |
e855c69d | 7366 | haifa_sched_finish (void) |
a88f02e7 | 7367 | { |
e855c69d AB |
7368 | sched_create_empty_bb = NULL; |
7369 | sched_split_block = NULL; | |
7370 | sched_init_only_bb = NULL; | |
58565a33 | 7371 | |
496d7bb0 MK |
7372 | if (spec_info && spec_info->dump) |
7373 | { | |
7374 | char c = reload_completed ? 'a' : 'b'; | |
7375 | ||
7376 | fprintf (spec_info->dump, | |
7377 | ";; %s:\n", current_function_name ()); | |
7378 | ||
7379 | fprintf (spec_info->dump, | |
7380 | ";; Procedure %cr-begin-data-spec motions == %d\n", | |
7381 | c, nr_begin_data); | |
7382 | fprintf (spec_info->dump, | |
7383 | ";; Procedure %cr-be-in-data-spec motions == %d\n", | |
7384 | c, nr_be_in_data); | |
7385 | fprintf (spec_info->dump, | |
7386 | ";; Procedure %cr-begin-control-spec motions == %d\n", | |
7387 | c, nr_begin_control); | |
7388 | fprintf (spec_info->dump, | |
7389 | ";; Procedure %cr-be-in-control-spec motions == %d\n", | |
7390 | c, nr_be_in_control); | |
7391 | } | |
7392 | ||
9771b263 | 7393 | scheduled_insns.release (); |
9b69cf83 | 7394 | |
e855c69d AB |
7395 | /* Finalize h_i_d, dependency caches, and luids for the whole |
7396 | function. Target will be finalized in md_global_finish (). */ | |
7397 | sched_deps_finish (); | |
7398 | sched_finish_luids (); | |
7399 | current_sched_info = NULL; | |
df554b0e | 7400 | insn_queue = NULL; |
e855c69d AB |
7401 | sched_finish (); |
7402 | } | |
7403 | ||
b8698a0f | 7404 | /* Free global data used during insn scheduling. This function works with |
e855c69d AB |
7405 | the common data shared between the schedulers. */ |
7406 | ||
7407 | void | |
7408 | sched_finish (void) | |
7409 | { | |
7410 | haifa_finish_h_i_d (); | |
c4cd7435 | 7411 | free_global_sched_pressure_data (); |
e855c69d AB |
7412 | free (curr_state); |
7413 | ||
38f8b050 JR |
7414 | if (targetm.sched.finish_global) |
7415 | targetm.sched.finish_global (sched_dump, sched_verbose); | |
e855c69d AB |
7416 | |
7417 | end_alias_analysis (); | |
7418 | ||
7419 | regstat_free_calls_crossed (); | |
7420 | ||
7421 | dfa_finish (); | |
8c660648 | 7422 | } |
63f54b1a | 7423 | |
26965010 BS |
7424 | /* Free all delay_pair structures that were recorded. */ |
7425 | void | |
7426 | free_delay_pairs (void) | |
7427 | { | |
c203e8a7 | 7428 | if (delay_htab) |
26965010 | 7429 | { |
c203e8a7 TS |
7430 | delay_htab->empty (); |
7431 | delay_htab_i2->empty (); | |
26965010 BS |
7432 | } |
7433 | } | |
7434 | ||
63f54b1a | 7435 | /* Fix INSN_TICKs of the instructions in the current block as well as |
917f1b7e | 7436 | INSN_TICKs of their dependents. |
63f54b1a MK |
7437 | HEAD and TAIL are the begin and the end of the current scheduled block. */ |
7438 | static void | |
f57aa6b0 | 7439 | fix_inter_tick (rtx_insn *head, rtx_insn *tail) |
63f54b1a MK |
7440 | { |
7441 | /* Set of instructions with corrected INSN_TICK. */ | |
8f9b31f7 | 7442 | auto_bitmap processed; |
e2f6ff94 MK |
7443 | /* ??? It is doubtful if we should assume that cycle advance happens on |
7444 | basic block boundaries. Basically insns that are unconditionally ready | |
7445 | on the start of the block are more preferable then those which have | |
7446 | a one cycle dependency over insn from the previous block. */ | |
63f54b1a MK |
7447 | int next_clock = clock_var + 1; |
7448 | ||
63f54b1a MK |
7449 | /* Iterates over scheduled instructions and fix their INSN_TICKs and |
7450 | INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent | |
7451 | across different blocks. */ | |
7452 | for (tail = NEXT_INSN (tail); head != tail; head = NEXT_INSN (head)) | |
7453 | { | |
7454 | if (INSN_P (head)) | |
7455 | { | |
7456 | int tick; | |
e2f6ff94 MK |
7457 | sd_iterator_def sd_it; |
7458 | dep_t dep; | |
b8698a0f | 7459 | |
63f54b1a MK |
7460 | tick = INSN_TICK (head); |
7461 | gcc_assert (tick >= MIN_TICK); | |
b8698a0f | 7462 | |
63f54b1a | 7463 | /* Fix INSN_TICK of instruction from just scheduled block. */ |
8f9b31f7 | 7464 | if (bitmap_set_bit (processed, INSN_LUID (head))) |
63f54b1a | 7465 | { |
63f54b1a | 7466 | tick -= next_clock; |
b8698a0f | 7467 | |
63f54b1a MK |
7468 | if (tick < MIN_TICK) |
7469 | tick = MIN_TICK; | |
b8698a0f L |
7470 | |
7471 | INSN_TICK (head) = tick; | |
63f54b1a | 7472 | } |
b8698a0f | 7473 | |
35d59da7 UB |
7474 | if (DEBUG_INSN_P (head)) |
7475 | continue; | |
7476 | ||
e2f6ff94 | 7477 | FOR_EACH_DEP (head, SD_LIST_RES_FORW, sd_it, dep) |
63f54b1a | 7478 | { |
9b517712 | 7479 | rtx_insn *next; |
b8698a0f | 7480 | |
e2f6ff94 | 7481 | next = DEP_CON (dep); |
63f54b1a MK |
7482 | tick = INSN_TICK (next); |
7483 | ||
7484 | if (tick != INVALID_TICK | |
7485 | /* If NEXT has its INSN_TICK calculated, fix it. | |
7486 | If not - it will be properly calculated from | |
7487 | scratch later in fix_tick_ready. */ | |
8f9b31f7 | 7488 | && bitmap_set_bit (processed, INSN_LUID (next))) |
63f54b1a | 7489 | { |
63f54b1a | 7490 | tick -= next_clock; |
b8698a0f | 7491 | |
63f54b1a MK |
7492 | if (tick < MIN_TICK) |
7493 | tick = MIN_TICK; | |
b8698a0f | 7494 | |
63f54b1a MK |
7495 | if (tick > INTER_TICK (next)) |
7496 | INTER_TICK (next) = tick; | |
7497 | else | |
7498 | tick = INTER_TICK (next); | |
e2f6ff94 | 7499 | |
63f54b1a MK |
7500 | INSN_TICK (next) = tick; |
7501 | } | |
7502 | } | |
7503 | } | |
7504 | } | |
63f54b1a | 7505 | } |
e855c69d | 7506 | |
63f54b1a MK |
7507 | /* Check if NEXT is ready to be added to the ready or queue list. |
7508 | If "yes", add it to the proper list. | |
7509 | Returns: | |
7510 | -1 - is not ready yet, | |
7511 | 0 - added to the ready list, | |
7512 | 0 < N - queued for N cycles. */ | |
7513 | int | |
ce1ce33a | 7514 | try_ready (rtx_insn *next) |
b8698a0f | 7515 | { |
b953c2b8 | 7516 | ds_t old_ts, new_ts; |
63f54b1a | 7517 | |
b953c2b8 | 7518 | old_ts = TODO_SPEC (next); |
63f54b1a | 7519 | |
1a83e602 BS |
7520 | gcc_assert (!(old_ts & ~(SPECULATIVE | HARD_DEP | DEP_CONTROL | DEP_POSTPONED)) |
7521 | && (old_ts == HARD_DEP | |
7522 | || old_ts == DEP_POSTPONED | |
e2724e63 | 7523 | || (old_ts & SPECULATIVE) |
1a83e602 | 7524 | || old_ts == DEP_CONTROL)); |
e2f6ff94 | 7525 | |
1a83e602 | 7526 | new_ts = recompute_todo_spec (next, false); |
b198261f | 7527 | |
1a83e602 | 7528 | if (new_ts & (HARD_DEP | DEP_POSTPONED)) |
e2724e63 | 7529 | gcc_assert (new_ts == old_ts |
496d7bb0 MK |
7530 | && QUEUE_INDEX (next) == QUEUE_NOWHERE); |
7531 | else if (current_sched_info->new_ready) | |
b953c2b8 | 7532 | new_ts = current_sched_info->new_ready (next, new_ts); |
496d7bb0 | 7533 | |
b198261f | 7534 | /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might |
496d7bb0 MK |
7535 | have its original pattern or changed (speculative) one. This is due |
7536 | to changing ebb in region scheduling. | |
7537 | * But if (old_ts & SPECULATIVE), then we are pretty sure that insn | |
7538 | has speculative pattern. | |
b198261f | 7539 | |
b953c2b8 | 7540 | We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because |
496d7bb0 MK |
7541 | control-speculative NEXT could have been discarded by sched-rgn.c |
7542 | (the same case as when discarded by can_schedule_ready_p ()). */ | |
b198261f | 7543 | |
b953c2b8 BS |
7544 | if ((new_ts & SPECULATIVE) |
7545 | /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't | |
496d7bb0 | 7546 | need to change anything. */ |
b953c2b8 | 7547 | && new_ts != old_ts) |
496d7bb0 MK |
7548 | { |
7549 | int res; | |
7550 | rtx new_pat; | |
b8698a0f | 7551 | |
e2724e63 | 7552 | gcc_assert ((new_ts & SPECULATIVE) && !(new_ts & ~SPECULATIVE)); |
b8698a0f | 7553 | |
b953c2b8 | 7554 | res = haifa_speculate_insn (next, new_ts, &new_pat); |
b8698a0f | 7555 | |
496d7bb0 MK |
7556 | switch (res) |
7557 | { | |
7558 | case -1: | |
7559 | /* It would be nice to change DEP_STATUS of all dependences, | |
b953c2b8 | 7560 | which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP, |
496d7bb0 | 7561 | so we won't reanalyze anything. */ |
b953c2b8 | 7562 | new_ts = HARD_DEP; |
496d7bb0 | 7563 | break; |
b8698a0f | 7564 | |
496d7bb0 MK |
7565 | case 0: |
7566 | /* We follow the rule, that every speculative insn | |
7567 | has non-null ORIG_PAT. */ | |
7568 | if (!ORIG_PAT (next)) | |
7569 | ORIG_PAT (next) = PATTERN (next); | |
7570 | break; | |
b8698a0f L |
7571 | |
7572 | case 1: | |
496d7bb0 MK |
7573 | if (!ORIG_PAT (next)) |
7574 | /* If we gonna to overwrite the original pattern of insn, | |
7575 | save it. */ | |
7576 | ORIG_PAT (next) = PATTERN (next); | |
b8698a0f | 7577 | |
e2724e63 BS |
7578 | res = haifa_change_pattern (next, new_pat); |
7579 | gcc_assert (res); | |
496d7bb0 | 7580 | break; |
b8698a0f | 7581 | |
496d7bb0 MK |
7582 | default: |
7583 | gcc_unreachable (); | |
7584 | } | |
7585 | } | |
b8698a0f | 7586 | |
b953c2b8 BS |
7587 | /* We need to restore pattern only if (new_ts == 0), because otherwise it is |
7588 | either correct (new_ts & SPECULATIVE), | |
7589 | or we simply don't care (new_ts & HARD_DEP). */ | |
b8698a0f | 7590 | |
496d7bb0 | 7591 | gcc_assert (!ORIG_PAT (next) |
d7bfd907 | 7592 | || !IS_SPECULATION_BRANCHY_CHECK_P (next)); |
b8698a0f | 7593 | |
b953c2b8 BS |
7594 | TODO_SPEC (next) = new_ts; |
7595 | ||
1a83e602 | 7596 | if (new_ts & (HARD_DEP | DEP_POSTPONED)) |
496d7bb0 MK |
7597 | { |
7598 | /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because | |
7599 | control-speculative NEXT could have been discarded by sched-rgn.c | |
7600 | (the same case as when discarded by can_schedule_ready_p ()). */ | |
7601 | /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/ | |
b8698a0f | 7602 | |
496d7bb0 | 7603 | change_queue_index (next, QUEUE_NOWHERE); |
e2724e63 | 7604 | |
496d7bb0 MK |
7605 | return -1; |
7606 | } | |
b953c2b8 | 7607 | else if (!(new_ts & BEGIN_SPEC) |
e2724e63 BS |
7608 | && ORIG_PAT (next) && PREDICATED_PAT (next) == NULL_RTX |
7609 | && !IS_SPECULATION_CHECK_P (next)) | |
b8698a0f | 7610 | /* We should change pattern of every previously speculative |
682b6a9e | 7611 | instruction - and we determine if NEXT was speculative by using |
d7bfd907 MK |
7612 | ORIG_PAT field. Except one case - speculation checks have ORIG_PAT |
7613 | pat too, so skip them. */ | |
682b6a9e | 7614 | { |
e2724e63 BS |
7615 | bool success = haifa_change_pattern (next, ORIG_PAT (next)); |
7616 | gcc_assert (success); | |
682b6a9e MK |
7617 | ORIG_PAT (next) = 0; |
7618 | } | |
496d7bb0 MK |
7619 | |
7620 | if (sched_verbose >= 2) | |
b8698a0f | 7621 | { |
496d7bb0 MK |
7622 | fprintf (sched_dump, ";;\t\tdependencies resolved: insn %s", |
7623 | (*current_sched_info->print_insn) (next, 0)); | |
b8698a0f | 7624 | |
496d7bb0 MK |
7625 | if (spec_info && spec_info->dump) |
7626 | { | |
b953c2b8 | 7627 | if (new_ts & BEGIN_DATA) |
496d7bb0 | 7628 | fprintf (spec_info->dump, "; data-spec;"); |
b953c2b8 | 7629 | if (new_ts & BEGIN_CONTROL) |
496d7bb0 | 7630 | fprintf (spec_info->dump, "; control-spec;"); |
b953c2b8 | 7631 | if (new_ts & BE_IN_CONTROL) |
496d7bb0 MK |
7632 | fprintf (spec_info->dump, "; in-control-spec;"); |
7633 | } | |
e2724e63 BS |
7634 | if (TODO_SPEC (next) & DEP_CONTROL) |
7635 | fprintf (sched_dump, " predicated"); | |
496d7bb0 | 7636 | fprintf (sched_dump, "\n"); |
b8698a0f L |
7637 | } |
7638 | ||
496d7bb0 | 7639 | adjust_priority (next); |
b8698a0f | 7640 | |
496d7bb0 MK |
7641 | return fix_tick_ready (next); |
7642 | } | |
7643 | ||
7644 | /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */ | |
7645 | static int | |
ce1ce33a | 7646 | fix_tick_ready (rtx_insn *next) |
63f54b1a | 7647 | { |
63f54b1a MK |
7648 | int tick, delay; |
7649 | ||
b2cddfc8 | 7650 | if (!DEBUG_INSN_P (next) && !sd_lists_empty_p (next, SD_LIST_RES_BACK)) |
63f54b1a MK |
7651 | { |
7652 | int full_p; | |
e2f6ff94 MK |
7653 | sd_iterator_def sd_it; |
7654 | dep_t dep; | |
63f54b1a MK |
7655 | |
7656 | tick = INSN_TICK (next); | |
496d7bb0 | 7657 | /* if tick is not equal to INVALID_TICK, then update |
63f54b1a | 7658 | INSN_TICK of NEXT with the most recent resolved dependence |
917f1b7e | 7659 | cost. Otherwise, recalculate from scratch. */ |
b198261f MK |
7660 | full_p = (tick == INVALID_TICK); |
7661 | ||
e2f6ff94 | 7662 | FOR_EACH_DEP (next, SD_LIST_RES_BACK, sd_it, dep) |
b8698a0f | 7663 | { |
9b517712 | 7664 | rtx_insn *pro = DEP_PRO (dep); |
63f54b1a | 7665 | int tick1; |
b8698a0f | 7666 | |
63f54b1a | 7667 | gcc_assert (INSN_TICK (pro) >= MIN_TICK); |
496d7bb0 | 7668 | |
b198261f | 7669 | tick1 = INSN_TICK (pro) + dep_cost (dep); |
63f54b1a MK |
7670 | if (tick1 > tick) |
7671 | tick = tick1; | |
b198261f MK |
7672 | |
7673 | if (!full_p) | |
7674 | break; | |
63f54b1a | 7675 | } |
63f54b1a MK |
7676 | } |
7677 | else | |
7678 | tick = -1; | |
7679 | ||
7680 | INSN_TICK (next) = tick; | |
7681 | ||
7682 | delay = tick - clock_var; | |
b16abbcb | 7683 | if (delay <= 0 || sched_pressure != SCHED_PRESSURE_NONE || sched_fusion) |
63f54b1a MK |
7684 | delay = QUEUE_READY; |
7685 | ||
7686 | change_queue_index (next, delay); | |
496d7bb0 | 7687 | |
63f54b1a MK |
7688 | return delay; |
7689 | } | |
7690 | ||
7691 | /* Move NEXT to the proper queue list with (DELAY >= 1), | |
7692 | or add it to the ready list (DELAY == QUEUE_READY), | |
7693 | or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */ | |
7694 | static void | |
ce1ce33a | 7695 | change_queue_index (rtx_insn *next, int delay) |
63f54b1a MK |
7696 | { |
7697 | int i = QUEUE_INDEX (next); | |
7698 | ||
b8698a0f | 7699 | gcc_assert (QUEUE_NOWHERE <= delay && delay <= max_insn_queue_index |
63f54b1a MK |
7700 | && delay != 0); |
7701 | gcc_assert (i != QUEUE_SCHEDULED); | |
b8698a0f | 7702 | |
63f54b1a MK |
7703 | if ((delay > 0 && NEXT_Q_AFTER (q_ptr, delay) == i) |
7704 | || (delay < 0 && delay == i)) | |
7705 | /* We have nothing to do. */ | |
7706 | return; | |
7707 | ||
917f1b7e | 7708 | /* Remove NEXT from wherever it is now. */ |
63f54b1a MK |
7709 | if (i == QUEUE_READY) |
7710 | ready_remove_insn (next); | |
7711 | else if (i >= 0) | |
7712 | queue_remove (next); | |
b8698a0f | 7713 | |
63f54b1a MK |
7714 | /* Add it to the proper place. */ |
7715 | if (delay == QUEUE_READY) | |
7716 | ready_add (readyp, next, false); | |
7717 | else if (delay >= 1) | |
ec4efea9 | 7718 | queue_insn (next, delay, "change queue index"); |
b8698a0f | 7719 | |
63f54b1a | 7720 | if (sched_verbose >= 2) |
b8698a0f | 7721 | { |
63f54b1a MK |
7722 | fprintf (sched_dump, ";;\t\ttick updated: insn %s", |
7723 | (*current_sched_info->print_insn) (next, 0)); | |
b8698a0f | 7724 | |
63f54b1a MK |
7725 | if (delay == QUEUE_READY) |
7726 | fprintf (sched_dump, " into ready\n"); | |
7727 | else if (delay >= 1) | |
7728 | fprintf (sched_dump, " into queue with cost=%d\n", delay); | |
7729 | else | |
7730 | fprintf (sched_dump, " removed from ready or queue lists\n"); | |
7731 | } | |
7732 | } | |
7733 | ||
e855c69d | 7734 | static int sched_ready_n_insns = -1; |
496d7bb0 | 7735 | |
e855c69d AB |
7736 | /* Initialize per region data structures. */ |
7737 | void | |
7738 | sched_extend_ready_list (int new_sched_ready_n_insns) | |
496d7bb0 MK |
7739 | { |
7740 | int i; | |
7741 | ||
e855c69d AB |
7742 | if (sched_ready_n_insns == -1) |
7743 | /* At the first call we need to initialize one more choice_stack | |
7744 | entry. */ | |
7745 | { | |
7746 | i = 0; | |
7747 | sched_ready_n_insns = 0; | |
9771b263 | 7748 | scheduled_insns.reserve (new_sched_ready_n_insns); |
e855c69d AB |
7749 | } |
7750 | else | |
7751 | i = sched_ready_n_insns + 1; | |
496d7bb0 | 7752 | |
e855c69d | 7753 | ready.veclen = new_sched_ready_n_insns + issue_rate; |
ce1ce33a | 7754 | ready.vec = XRESIZEVEC (rtx_insn *, ready.vec, ready.veclen); |
496d7bb0 | 7755 | |
e855c69d | 7756 | gcc_assert (new_sched_ready_n_insns >= sched_ready_n_insns); |
496d7bb0 | 7757 | |
4960a0cb MK |
7758 | ready_try = (signed char *) xrecalloc (ready_try, new_sched_ready_n_insns, |
7759 | sched_ready_n_insns, | |
7760 | sizeof (*ready_try)); | |
496d7bb0 | 7761 | |
e855c69d AB |
7762 | /* We allocate +1 element to save initial state in the choice_stack[0] |
7763 | entry. */ | |
7764 | choice_stack = XRESIZEVEC (struct choice_entry, choice_stack, | |
7765 | new_sched_ready_n_insns + 1); | |
e2f6ff94 | 7766 | |
e855c69d | 7767 | for (; i <= new_sched_ready_n_insns; i++) |
894fd6f2 MK |
7768 | { |
7769 | choice_stack[i].state = xmalloc (dfa_state_size); | |
7770 | ||
7771 | if (targetm.sched.first_cycle_multipass_init) | |
7772 | targetm.sched.first_cycle_multipass_init (&(choice_stack[i] | |
7773 | .target_data)); | |
7774 | } | |
496d7bb0 | 7775 | |
e855c69d | 7776 | sched_ready_n_insns = new_sched_ready_n_insns; |
496d7bb0 MK |
7777 | } |
7778 | ||
e855c69d AB |
7779 | /* Free per region data structures. */ |
7780 | void | |
7781 | sched_finish_ready_list (void) | |
a0d33ff8 | 7782 | { |
e855c69d | 7783 | int i; |
a0d33ff8 | 7784 | |
e855c69d AB |
7785 | free (ready.vec); |
7786 | ready.vec = NULL; | |
7787 | ready.veclen = 0; | |
a0d33ff8 | 7788 | |
e855c69d AB |
7789 | free (ready_try); |
7790 | ready_try = NULL; | |
496d7bb0 | 7791 | |
e855c69d | 7792 | for (i = 0; i <= sched_ready_n_insns; i++) |
894fd6f2 MK |
7793 | { |
7794 | if (targetm.sched.first_cycle_multipass_fini) | |
7795 | targetm.sched.first_cycle_multipass_fini (&(choice_stack[i] | |
7796 | .target_data)); | |
7797 | ||
7798 | free (choice_stack [i].state); | |
7799 | } | |
e855c69d AB |
7800 | free (choice_stack); |
7801 | choice_stack = NULL; | |
a0d33ff8 | 7802 | |
e855c69d | 7803 | sched_ready_n_insns = -1; |
496d7bb0 MK |
7804 | } |
7805 | ||
e855c69d AB |
7806 | static int |
7807 | haifa_luid_for_non_insn (rtx x) | |
496d7bb0 | 7808 | { |
e855c69d AB |
7809 | gcc_assert (NOTE_P (x) || LABEL_P (x)); |
7810 | ||
7811 | return 0; | |
496d7bb0 MK |
7812 | } |
7813 | ||
7814 | /* Generates recovery code for INSN. */ | |
7815 | static void | |
ce1ce33a | 7816 | generate_recovery_code (rtx_insn *insn) |
496d7bb0 MK |
7817 | { |
7818 | if (TODO_SPEC (insn) & BEGIN_SPEC) | |
7819 | begin_speculative_block (insn); | |
b8698a0f | 7820 | |
496d7bb0 MK |
7821 | /* Here we have insn with no dependencies to |
7822 | instructions other then CHECK_SPEC ones. */ | |
b8698a0f | 7823 | |
496d7bb0 MK |
7824 | if (TODO_SPEC (insn) & BE_IN_SPEC) |
7825 | add_to_speculative_block (insn); | |
7826 | } | |
7827 | ||
7828 | /* Helper function. | |
7829 | Tries to add speculative dependencies of type FS between instructions | |
b198261f | 7830 | in deps_list L and TWIN. */ |
496d7bb0 | 7831 | static void |
90831096 | 7832 | process_insn_forw_deps_be_in_spec (rtx_insn *insn, rtx_insn *twin, ds_t fs) |
496d7bb0 | 7833 | { |
e2f6ff94 MK |
7834 | sd_iterator_def sd_it; |
7835 | dep_t dep; | |
b198261f | 7836 | |
e2f6ff94 | 7837 | FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep) |
496d7bb0 MK |
7838 | { |
7839 | ds_t ds; | |
0e0ce50d | 7840 | rtx_insn *consumer; |
496d7bb0 | 7841 | |
e2f6ff94 | 7842 | consumer = DEP_CON (dep); |
496d7bb0 | 7843 | |
e2f6ff94 | 7844 | ds = DEP_STATUS (dep); |
496d7bb0 | 7845 | |
682b6a9e MK |
7846 | if (/* If we want to create speculative dep. */ |
7847 | fs | |
7848 | /* And we can do that because this is a true dep. */ | |
7849 | && (ds & DEP_TYPES) == DEP_TRUE) | |
7850 | { | |
7851 | gcc_assert (!(ds & BE_IN_SPEC)); | |
7852 | ||
917f1b7e | 7853 | if (/* If this dep can be overcome with 'begin speculation'. */ |
682b6a9e MK |
7854 | ds & BEGIN_SPEC) |
7855 | /* Then we have a choice: keep the dep 'begin speculative' | |
7856 | or transform it into 'be in speculative'. */ | |
7857 | { | |
7858 | if (/* In try_ready we assert that if insn once became ready | |
7859 | it can be removed from the ready (or queue) list only | |
7860 | due to backend decision. Hence we can't let the | |
7861 | probability of the speculative dep to decrease. */ | |
e855c69d | 7862 | ds_weak (ds) <= ds_weak (fs)) |
93b4b4cc MK |
7863 | { |
7864 | ds_t new_ds; | |
7865 | ||
7866 | new_ds = (ds & ~BEGIN_SPEC) | fs; | |
b8698a0f | 7867 | |
93b4b4cc MK |
7868 | if (/* consumer can 'be in speculative'. */ |
7869 | sched_insn_is_legitimate_for_speculation_p (consumer, | |
7870 | new_ds)) | |
7871 | /* Transform it to be in speculative. */ | |
7872 | ds = new_ds; | |
7873 | } | |
682b6a9e MK |
7874 | } |
7875 | else | |
7876 | /* Mark the dep as 'be in speculative'. */ | |
7877 | ds |= fs; | |
7878 | } | |
496d7bb0 | 7879 | |
e2f6ff94 MK |
7880 | { |
7881 | dep_def _new_dep, *new_dep = &_new_dep; | |
7882 | ||
7883 | init_dep_1 (new_dep, twin, consumer, DEP_TYPE (dep), ds); | |
7884 | sd_add_dep (new_dep, false); | |
7885 | } | |
496d7bb0 MK |
7886 | } |
7887 | } | |
7888 | ||
7889 | /* Generates recovery code for BEGIN speculative INSN. */ | |
7890 | static void | |
ce1ce33a | 7891 | begin_speculative_block (rtx_insn *insn) |
496d7bb0 MK |
7892 | { |
7893 | if (TODO_SPEC (insn) & BEGIN_DATA) | |
b8698a0f | 7894 | nr_begin_data++; |
496d7bb0 MK |
7895 | if (TODO_SPEC (insn) & BEGIN_CONTROL) |
7896 | nr_begin_control++; | |
7897 | ||
7898 | create_check_block_twin (insn, false); | |
7899 | ||
7900 | TODO_SPEC (insn) &= ~BEGIN_SPEC; | |
7901 | } | |
7902 | ||
f57aa6b0 | 7903 | static void haifa_init_insn (rtx_insn *); |
e855c69d | 7904 | |
496d7bb0 MK |
7905 | /* Generates recovery code for BE_IN speculative INSN. */ |
7906 | static void | |
6144a836 | 7907 | add_to_speculative_block (rtx_insn *insn) |
496d7bb0 MK |
7908 | { |
7909 | ds_t ts; | |
e2f6ff94 MK |
7910 | sd_iterator_def sd_it; |
7911 | dep_t dep; | |
30a7199a | 7912 | auto_vec<rtx_insn *, 10> twins; |
496d7bb0 MK |
7913 | |
7914 | ts = TODO_SPEC (insn); | |
7915 | gcc_assert (!(ts & ~BE_IN_SPEC)); | |
7916 | ||
7917 | if (ts & BE_IN_DATA) | |
7918 | nr_be_in_data++; | |
7919 | if (ts & BE_IN_CONTROL) | |
7920 | nr_be_in_control++; | |
7921 | ||
7922 | TODO_SPEC (insn) &= ~BE_IN_SPEC; | |
7923 | gcc_assert (!TODO_SPEC (insn)); | |
b8698a0f | 7924 | |
496d7bb0 MK |
7925 | DONE_SPEC (insn) |= ts; |
7926 | ||
7927 | /* First we convert all simple checks to branchy. */ | |
e2f6ff94 MK |
7928 | for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); |
7929 | sd_iterator_cond (&sd_it, &dep);) | |
496d7bb0 | 7930 | { |
9b517712 | 7931 | rtx_insn *check = DEP_PRO (dep); |
496d7bb0 | 7932 | |
d7bfd907 | 7933 | if (IS_SPECULATION_SIMPLE_CHECK_P (check)) |
496d7bb0 MK |
7934 | { |
7935 | create_check_block_twin (check, true); | |
b198261f MK |
7936 | |
7937 | /* Restart search. */ | |
e2f6ff94 | 7938 | sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); |
496d7bb0 MK |
7939 | } |
7940 | else | |
b198261f | 7941 | /* Continue search. */ |
e2f6ff94 | 7942 | sd_iterator_next (&sd_it); |
496d7bb0 MK |
7943 | } |
7944 | ||
8c681247 | 7945 | auto_vec<rtx_insn *> priorities_roots; |
916fa4f0 | 7946 | clear_priorities (insn, &priorities_roots); |
e2f6ff94 MK |
7947 | |
7948 | while (1) | |
496d7bb0 | 7949 | { |
9b517712 | 7950 | rtx_insn *check, *twin; |
496d7bb0 MK |
7951 | basic_block rec; |
7952 | ||
e2f6ff94 MK |
7953 | /* Get the first backward dependency of INSN. */ |
7954 | sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); | |
7955 | if (!sd_iterator_cond (&sd_it, &dep)) | |
7956 | /* INSN has no backward dependencies left. */ | |
7957 | break; | |
496d7bb0 | 7958 | |
e2f6ff94 MK |
7959 | gcc_assert ((DEP_STATUS (dep) & BEGIN_SPEC) == 0 |
7960 | && (DEP_STATUS (dep) & BE_IN_SPEC) != 0 | |
7961 | && (DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE); | |
b198261f | 7962 | |
e2f6ff94 | 7963 | check = DEP_PRO (dep); |
d7bfd907 MK |
7964 | |
7965 | gcc_assert (!IS_SPECULATION_CHECK_P (check) && !ORIG_PAT (check) | |
496d7bb0 | 7966 | && QUEUE_INDEX (check) == QUEUE_NOWHERE); |
e2f6ff94 | 7967 | |
496d7bb0 | 7968 | rec = BLOCK_FOR_INSN (check); |
e2f6ff94 | 7969 | |
05742530 | 7970 | twin = emit_insn_before (copy_insn (PATTERN (insn)), BB_END (rec)); |
e855c69d | 7971 | haifa_init_insn (twin); |
496d7bb0 | 7972 | |
e2f6ff94 | 7973 | sd_copy_back_deps (twin, insn, true); |
496d7bb0 MK |
7974 | |
7975 | if (sched_verbose && spec_info->dump) | |
7976 | /* INSN_BB (insn) isn't determined for twin insns yet. | |
7977 | So we can't use current_sched_info->print_insn. */ | |
7978 | fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n", | |
7979 | INSN_UID (twin), rec->index); | |
7980 | ||
30a7199a | 7981 | twins.safe_push (twin); |
496d7bb0 | 7982 | |
917f1b7e | 7983 | /* Add dependences between TWIN and all appropriate |
496d7bb0 | 7984 | instructions from REC. */ |
e2f6ff94 MK |
7985 | FOR_EACH_DEP (insn, SD_LIST_SPEC_BACK, sd_it, dep) |
7986 | { | |
9b517712 | 7987 | rtx_insn *pro = DEP_PRO (dep); |
b198261f | 7988 | |
e2f6ff94 MK |
7989 | gcc_assert (DEP_TYPE (dep) == REG_DEP_TRUE); |
7990 | ||
7991 | /* INSN might have dependencies from the instructions from | |
7992 | several recovery blocks. At this iteration we process those | |
7993 | producers that reside in REC. */ | |
7994 | if (BLOCK_FOR_INSN (pro) == rec) | |
7995 | { | |
7996 | dep_def _new_dep, *new_dep = &_new_dep; | |
7997 | ||
7998 | init_dep (new_dep, pro, twin, REG_DEP_TRUE); | |
7999 | sd_add_dep (new_dep, false); | |
496d7bb0 | 8000 | } |
496d7bb0 | 8001 | } |
496d7bb0 | 8002 | |
e2f6ff94 | 8003 | process_insn_forw_deps_be_in_spec (insn, twin, ts); |
496d7bb0 | 8004 | |
b198261f | 8005 | /* Remove all dependencies between INSN and insns in REC. */ |
e2f6ff94 MK |
8006 | for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); |
8007 | sd_iterator_cond (&sd_it, &dep);) | |
496d7bb0 | 8008 | { |
9b517712 | 8009 | rtx_insn *pro = DEP_PRO (dep); |
b198261f | 8010 | |
e2f6ff94 MK |
8011 | if (BLOCK_FOR_INSN (pro) == rec) |
8012 | sd_delete_dep (sd_it); | |
496d7bb0 | 8013 | else |
e2f6ff94 | 8014 | sd_iterator_next (&sd_it); |
496d7bb0 MK |
8015 | } |
8016 | } | |
496d7bb0 | 8017 | |
b198261f MK |
8018 | /* We couldn't have added the dependencies between INSN and TWINS earlier |
8019 | because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */ | |
30a7199a TS |
8020 | unsigned int i; |
8021 | rtx_insn *twin; | |
8022 | FOR_EACH_VEC_ELT_REVERSE (twins, i, twin) | |
496d7bb0 | 8023 | { |
30a7199a | 8024 | dep_def _new_dep, *new_dep = &_new_dep; |
496d7bb0 | 8025 | |
30a7199a TS |
8026 | init_dep (new_dep, insn, twin, REG_DEP_OUTPUT); |
8027 | sd_add_dep (new_dep, false); | |
496d7bb0 | 8028 | } |
916fa4f0 MK |
8029 | |
8030 | calc_priorities (priorities_roots); | |
496d7bb0 MK |
8031 | } |
8032 | ||
8033 | /* Extends and fills with zeros (only the new part) array pointed to by P. */ | |
8034 | void * | |
8035 | xrecalloc (void *p, size_t new_nmemb, size_t old_nmemb, size_t size) | |
8036 | { | |
8037 | gcc_assert (new_nmemb >= old_nmemb); | |
8038 | p = XRESIZEVAR (void, p, new_nmemb * size); | |
8039 | memset (((char *) p) + old_nmemb * size, 0, (new_nmemb - old_nmemb) * size); | |
8040 | return p; | |
8041 | } | |
8042 | ||
496d7bb0 MK |
8043 | /* Helper function. |
8044 | Find fallthru edge from PRED. */ | |
e855c69d | 8045 | edge |
0fd4b31d | 8046 | find_fallthru_edge_from (basic_block pred) |
496d7bb0 MK |
8047 | { |
8048 | edge e; | |
496d7bb0 MK |
8049 | basic_block succ; |
8050 | ||
8051 | succ = pred->next_bb; | |
8052 | gcc_assert (succ->prev_bb == pred); | |
8053 | ||
8054 | if (EDGE_COUNT (pred->succs) <= EDGE_COUNT (succ->preds)) | |
8055 | { | |
0fd4b31d NF |
8056 | e = find_fallthru_edge (pred->succs); |
8057 | ||
8058 | if (e) | |
8059 | { | |
8060 | gcc_assert (e->dest == succ); | |
8061 | return e; | |
8062 | } | |
496d7bb0 MK |
8063 | } |
8064 | else | |
8065 | { | |
0fd4b31d NF |
8066 | e = find_fallthru_edge (succ->preds); |
8067 | ||
8068 | if (e) | |
8069 | { | |
8070 | gcc_assert (e->src == pred); | |
8071 | return e; | |
8072 | } | |
496d7bb0 MK |
8073 | } |
8074 | ||
8075 | return NULL; | |
8076 | } | |
8077 | ||
e855c69d AB |
8078 | /* Extend per basic block data structures. */ |
8079 | static void | |
8080 | sched_extend_bb (void) | |
8081 | { | |
e855c69d | 8082 | /* The following is done to keep current_sched_info->next_tail non null. */ |
9b517712 DM |
8083 | rtx_insn *end = BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb); |
8084 | rtx_insn *insn = DEBUG_INSN_P (end) ? prev_nondebug_insn (end) : end; | |
cb5cb194 | 8085 | if (NEXT_INSN (end) == 0 |
e855c69d AB |
8086 | || (!NOTE_P (insn) |
8087 | && !LABEL_P (insn) | |
8088 | /* Don't emit a NOTE if it would end up before a BARRIER. */ | |
cb5cb194 | 8089 | && !BARRIER_P (NEXT_INSN (end)))) |
e855c69d | 8090 | { |
66e8df53 | 8091 | rtx_note *note = emit_note_after (NOTE_INSN_DELETED, end); |
cb5cb194 | 8092 | /* Make note appear outside BB. */ |
e855c69d | 8093 | set_block_for_insn (note, NULL); |
1130d5e3 | 8094 | BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb) = end; |
e855c69d AB |
8095 | } |
8096 | } | |
8097 | ||
8098 | /* Init per basic block data structures. */ | |
8099 | void | |
8100 | sched_init_bbs (void) | |
8101 | { | |
8102 | sched_extend_bb (); | |
8103 | } | |
8104 | ||
496d7bb0 MK |
8105 | /* Initialize BEFORE_RECOVERY variable. */ |
8106 | static void | |
e855c69d | 8107 | init_before_recovery (basic_block *before_recovery_ptr) |
496d7bb0 MK |
8108 | { |
8109 | basic_block last; | |
8110 | edge e; | |
8111 | ||
fefa31b5 | 8112 | last = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb; |
0fd4b31d | 8113 | e = find_fallthru_edge_from (last); |
496d7bb0 MK |
8114 | |
8115 | if (e) | |
8116 | { | |
b8698a0f | 8117 | /* We create two basic blocks: |
496d7bb0 | 8118 | 1. Single instruction block is inserted right after E->SRC |
b8698a0f | 8119 | and has jump to |
496d7bb0 MK |
8120 | 2. Empty block right before EXIT_BLOCK. |
8121 | Between these two blocks recovery blocks will be emitted. */ | |
8122 | ||
8123 | basic_block single, empty; | |
496d7bb0 | 8124 | |
b8698a0f | 8125 | /* If the fallthrough edge to exit we've found is from the block we've |
e855c69d AB |
8126 | created before, don't do anything more. */ |
8127 | if (last == after_recovery) | |
8128 | return; | |
496d7bb0 | 8129 | |
e855c69d AB |
8130 | adding_bb_to_current_region_p = false; |
8131 | ||
8132 | single = sched_create_empty_bb (last); | |
8133 | empty = sched_create_empty_bb (single); | |
8134 | ||
8135 | /* Add new blocks to the root loop. */ | |
8136 | if (current_loops != NULL) | |
8137 | { | |
9771b263 DN |
8138 | add_bb_to_loop (single, (*current_loops->larray)[0]); |
8139 | add_bb_to_loop (empty, (*current_loops->larray)[0]); | |
e855c69d AB |
8140 | } |
8141 | ||
8142 | single->count = last->count; | |
496d7bb0 | 8143 | empty->count = last->count; |
496d7bb0 MK |
8144 | BB_COPY_PARTITION (single, last); |
8145 | BB_COPY_PARTITION (empty, last); | |
8146 | ||
8147 | redirect_edge_succ (e, single); | |
8148 | make_single_succ_edge (single, empty, 0); | |
fefa31b5 DM |
8149 | make_single_succ_edge (empty, EXIT_BLOCK_PTR_FOR_FN (cfun), |
8150 | EDGE_FALLTHRU); | |
496d7bb0 | 8151 | |
e67d1102 | 8152 | rtx_code_label *label = block_label (empty); |
ec4a505f | 8153 | rtx_jump_insn *x = emit_jump_insn_after (targetm.gen_jump (label), |
e67d1102 | 8154 | BB_END (single)); |
496d7bb0 MK |
8155 | JUMP_LABEL (x) = label; |
8156 | LABEL_NUSES (label)++; | |
e855c69d | 8157 | haifa_init_insn (x); |
b8698a0f | 8158 | |
496d7bb0 MK |
8159 | emit_barrier_after (x); |
8160 | ||
e855c69d AB |
8161 | sched_init_only_bb (empty, NULL); |
8162 | sched_init_only_bb (single, NULL); | |
8163 | sched_extend_bb (); | |
496d7bb0 | 8164 | |
e855c69d | 8165 | adding_bb_to_current_region_p = true; |
496d7bb0 | 8166 | before_recovery = single; |
e855c69d AB |
8167 | after_recovery = empty; |
8168 | ||
8169 | if (before_recovery_ptr) | |
8170 | *before_recovery_ptr = before_recovery; | |
496d7bb0 MK |
8171 | |
8172 | if (sched_verbose >= 2 && spec_info->dump) | |
8173 | fprintf (spec_info->dump, | |
b8698a0f L |
8174 | ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n", |
8175 | last->index, single->index, empty->index); | |
496d7bb0 MK |
8176 | } |
8177 | else | |
8178 | before_recovery = last; | |
8179 | } | |
8180 | ||
8181 | /* Returns new recovery block. */ | |
e855c69d AB |
8182 | basic_block |
8183 | sched_create_recovery_block (basic_block *before_recovery_ptr) | |
496d7bb0 | 8184 | { |
9b517712 | 8185 | rtx_insn *barrier; |
496d7bb0 | 8186 | basic_block rec; |
b8698a0f | 8187 | |
e2f6ff94 MK |
8188 | haifa_recovery_bb_recently_added_p = true; |
8189 | haifa_recovery_bb_ever_added_p = true; | |
496d7bb0 | 8190 | |
e855c69d | 8191 | init_before_recovery (before_recovery_ptr); |
496d7bb0 | 8192 | |
96370780 MK |
8193 | barrier = get_last_bb_insn (before_recovery); |
8194 | gcc_assert (BARRIER_P (barrier)); | |
8195 | ||
e67d1102 | 8196 | rtx_insn *label = emit_label_after (gen_label_rtx (), barrier); |
96370780 MK |
8197 | |
8198 | rec = create_basic_block (label, label, before_recovery); | |
8199 | ||
e855c69d | 8200 | /* A recovery block always ends with an unconditional jump. */ |
496d7bb0 MK |
8201 | emit_barrier_after (BB_END (rec)); |
8202 | ||
8203 | if (BB_PARTITION (before_recovery) != BB_UNPARTITIONED) | |
8204 | BB_SET_PARTITION (rec, BB_COLD_PARTITION); | |
b8698a0f L |
8205 | |
8206 | if (sched_verbose && spec_info->dump) | |
496d7bb0 MK |
8207 | fprintf (spec_info->dump, ";;\t\tGenerated recovery block rec%d\n", |
8208 | rec->index); | |
8209 | ||
496d7bb0 MK |
8210 | return rec; |
8211 | } | |
8212 | ||
e855c69d AB |
8213 | /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB |
8214 | and emit necessary jumps. */ | |
8215 | void | |
8216 | sched_create_recovery_edges (basic_block first_bb, basic_block rec, | |
8217 | basic_block second_bb) | |
8218 | { | |
e855c69d AB |
8219 | int edge_flags; |
8220 | ||
8221 | /* This is fixing of incoming edge. */ | |
b8698a0f | 8222 | /* ??? Which other flags should be specified? */ |
e855c69d AB |
8223 | if (BB_PARTITION (first_bb) != BB_PARTITION (rec)) |
8224 | /* Partition type is the same, if it is "unpartitioned". */ | |
8225 | edge_flags = EDGE_CROSSING; | |
8226 | else | |
8227 | edge_flags = 0; | |
b8698a0f | 8228 | |
cea768b0 JH |
8229 | edge e2 = single_succ_edge (first_bb); |
8230 | edge e = make_edge (first_bb, rec, edge_flags); | |
8231 | ||
8232 | /* TODO: The actual probability can be determined and is computed as | |
8233 | 'todo_spec' variable in create_check_block_twin and | |
8234 | in sel-sched.c `check_ds' in create_speculation_check. */ | |
8235 | e->probability = profile_probability::very_unlikely (); | |
ef30ab83 | 8236 | rec->count = e->count (); |
cea768b0 | 8237 | e2->probability = e->probability.invert (); |
cea768b0 | 8238 | |
e67d1102 | 8239 | rtx_code_label *label = block_label (second_bb); |
ec4a505f RS |
8240 | rtx_jump_insn *jump = emit_jump_insn_after (targetm.gen_jump (label), |
8241 | BB_END (rec)); | |
e855c69d AB |
8242 | JUMP_LABEL (jump) = label; |
8243 | LABEL_NUSES (label)++; | |
8244 | ||
8245 | if (BB_PARTITION (second_bb) != BB_PARTITION (rec)) | |
8246 | /* Partition type is the same, if it is "unpartitioned". */ | |
8247 | { | |
8248 | /* Rewritten from cfgrtl.c. */ | |
0b6bc904 | 8249 | if (crtl->has_bb_partition && targetm_common.have_named_sections) |
e855c69d | 8250 | { |
efc0b2bd ILT |
8251 | /* We don't need the same note for the check because |
8252 | any_condjump_p (check) == true. */ | |
339ba33b | 8253 | CROSSING_JUMP_P (jump) = 1; |
e855c69d AB |
8254 | } |
8255 | edge_flags = EDGE_CROSSING; | |
8256 | } | |
8257 | else | |
b8698a0f | 8258 | edge_flags = 0; |
e855c69d | 8259 | |
b8698a0f | 8260 | make_single_succ_edge (rec, second_bb, edge_flags); |
00c4e97c AB |
8261 | if (dom_info_available_p (CDI_DOMINATORS)) |
8262 | set_immediate_dominator (CDI_DOMINATORS, rec, first_bb); | |
e855c69d AB |
8263 | } |
8264 | ||
496d7bb0 MK |
8265 | /* This function creates recovery code for INSN. If MUTATE_P is nonzero, |
8266 | INSN is a simple check, that should be converted to branchy one. */ | |
8267 | static void | |
ce1ce33a | 8268 | create_check_block_twin (rtx_insn *insn, bool mutate_p) |
496d7bb0 MK |
8269 | { |
8270 | basic_block rec; | |
ce1ce33a DM |
8271 | rtx_insn *label, *check, *twin; |
8272 | rtx check_pat; | |
496d7bb0 | 8273 | ds_t fs; |
e2f6ff94 MK |
8274 | sd_iterator_def sd_it; |
8275 | dep_t dep; | |
8276 | dep_def _new_dep, *new_dep = &_new_dep; | |
e855c69d | 8277 | ds_t todo_spec; |
496d7bb0 | 8278 | |
e855c69d AB |
8279 | gcc_assert (ORIG_PAT (insn) != NULL_RTX); |
8280 | ||
8281 | if (!mutate_p) | |
8282 | todo_spec = TODO_SPEC (insn); | |
8283 | else | |
8284 | { | |
8285 | gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn) | |
8286 | && (TODO_SPEC (insn) & SPECULATIVE) == 0); | |
8287 | ||
8288 | todo_spec = CHECK_SPEC (insn); | |
8289 | } | |
8290 | ||
8291 | todo_spec &= SPECULATIVE; | |
496d7bb0 MK |
8292 | |
8293 | /* Create recovery block. */ | |
388092d5 | 8294 | if (mutate_p || targetm.sched.needs_block_p (todo_spec)) |
496d7bb0 | 8295 | { |
e855c69d | 8296 | rec = sched_create_recovery_block (NULL); |
496d7bb0 MK |
8297 | label = BB_HEAD (rec); |
8298 | } | |
8299 | else | |
8300 | { | |
fefa31b5 | 8301 | rec = EXIT_BLOCK_PTR_FOR_FN (cfun); |
ce1ce33a | 8302 | label = NULL; |
496d7bb0 MK |
8303 | } |
8304 | ||
8305 | /* Emit CHECK. */ | |
ce1ce33a | 8306 | check_pat = targetm.sched.gen_spec_check (insn, label, todo_spec); |
496d7bb0 | 8307 | |
fefa31b5 | 8308 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 MK |
8309 | { |
8310 | /* To have mem_reg alive at the beginning of second_bb, | |
b8698a0f | 8311 | we emit check BEFORE insn, so insn after splitting |
496d7bb0 MK |
8312 | insn will be at the beginning of second_bb, which will |
8313 | provide us with the correct life information. */ | |
ce1ce33a | 8314 | check = emit_jump_insn_before (check_pat, insn); |
496d7bb0 MK |
8315 | JUMP_LABEL (check) = label; |
8316 | LABEL_NUSES (label)++; | |
8317 | } | |
8318 | else | |
ce1ce33a | 8319 | check = emit_insn_before (check_pat, insn); |
496d7bb0 MK |
8320 | |
8321 | /* Extend data structures. */ | |
e855c69d AB |
8322 | haifa_init_insn (check); |
8323 | ||
8324 | /* CHECK is being added to current region. Extend ready list. */ | |
8325 | gcc_assert (sched_ready_n_insns != -1); | |
8326 | sched_extend_ready_list (sched_ready_n_insns + 1); | |
8327 | ||
8328 | if (current_sched_info->add_remove_insn) | |
8329 | current_sched_info->add_remove_insn (insn, 0); | |
8330 | ||
496d7bb0 MK |
8331 | RECOVERY_BLOCK (check) = rec; |
8332 | ||
8333 | if (sched_verbose && spec_info->dump) | |
8334 | fprintf (spec_info->dump, ";;\t\tGenerated check insn : %s\n", | |
8335 | (*current_sched_info->print_insn) (check, 0)); | |
8336 | ||
8337 | gcc_assert (ORIG_PAT (insn)); | |
8338 | ||
917f1b7e | 8339 | /* Initialize TWIN (twin is a duplicate of original instruction |
496d7bb0 | 8340 | in the recovery block). */ |
fefa31b5 | 8341 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 | 8342 | { |
e2f6ff94 MK |
8343 | sd_iterator_def sd_it; |
8344 | dep_t dep; | |
8345 | ||
8346 | FOR_EACH_DEP (insn, SD_LIST_RES_BACK, sd_it, dep) | |
8347 | if ((DEP_STATUS (dep) & DEP_OUTPUT) != 0) | |
496d7bb0 | 8348 | { |
e2f6ff94 | 8349 | struct _dep _dep2, *dep2 = &_dep2; |
b198261f | 8350 | |
e2f6ff94 | 8351 | init_dep (dep2, DEP_PRO (dep), check, REG_DEP_TRUE); |
b198261f | 8352 | |
e2f6ff94 | 8353 | sd_add_dep (dep2, true); |
496d7bb0 MK |
8354 | } |
8355 | ||
8356 | twin = emit_insn_after (ORIG_PAT (insn), BB_END (rec)); | |
e855c69d | 8357 | haifa_init_insn (twin); |
496d7bb0 MK |
8358 | |
8359 | if (sched_verbose && spec_info->dump) | |
8360 | /* INSN_BB (insn) isn't determined for twin insns yet. | |
8361 | So we can't use current_sched_info->print_insn. */ | |
8362 | fprintf (spec_info->dump, ";;\t\tGenerated twin insn : %d/rec%d\n", | |
8363 | INSN_UID (twin), rec->index); | |
8364 | } | |
8365 | else | |
8366 | { | |
8367 | ORIG_PAT (check) = ORIG_PAT (insn); | |
8368 | HAS_INTERNAL_DEP (check) = 1; | |
8369 | twin = check; | |
8370 | /* ??? We probably should change all OUTPUT dependencies to | |
8371 | (TRUE | OUTPUT). */ | |
8372 | } | |
8373 | ||
e2f6ff94 MK |
8374 | /* Copy all resolved back dependencies of INSN to TWIN. This will |
8375 | provide correct value for INSN_TICK (TWIN). */ | |
8376 | sd_copy_back_deps (twin, insn, true); | |
496d7bb0 | 8377 | |
fefa31b5 | 8378 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 MK |
8379 | /* In case of branchy check, fix CFG. */ |
8380 | { | |
8381 | basic_block first_bb, second_bb; | |
f57aa6b0 | 8382 | rtx_insn *jump; |
496d7bb0 MK |
8383 | |
8384 | first_bb = BLOCK_FOR_INSN (check); | |
e855c69d | 8385 | second_bb = sched_split_block (first_bb, check); |
496d7bb0 | 8386 | |
e855c69d | 8387 | sched_create_recovery_edges (first_bb, rec, second_bb); |
496d7bb0 | 8388 | |
b8698a0f | 8389 | sched_init_only_bb (second_bb, first_bb); |
fefa31b5 | 8390 | sched_init_only_bb (rec, EXIT_BLOCK_PTR_FOR_FN (cfun)); |
e855c69d AB |
8391 | |
8392 | jump = BB_END (rec); | |
8393 | haifa_init_insn (jump); | |
496d7bb0 MK |
8394 | } |
8395 | ||
b8698a0f | 8396 | /* Move backward dependences from INSN to CHECK and |
496d7bb0 | 8397 | move forward dependences from INSN to TWIN. */ |
e2f6ff94 MK |
8398 | |
8399 | /* First, create dependencies between INSN's producers and CHECK & TWIN. */ | |
8400 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) | |
496d7bb0 | 8401 | { |
9b517712 | 8402 | rtx_insn *pro = DEP_PRO (dep); |
496d7bb0 MK |
8403 | ds_t ds; |
8404 | ||
8405 | /* If BEGIN_DATA: [insn ~~TRUE~~> producer]: | |
8406 | check --TRUE--> producer ??? or ANTI ??? | |
8407 | twin --TRUE--> producer | |
8408 | twin --ANTI--> check | |
b8698a0f | 8409 | |
496d7bb0 MK |
8410 | If BEGIN_CONTROL: [insn ~~ANTI~~> producer]: |
8411 | check --ANTI--> producer | |
8412 | twin --ANTI--> producer | |
8413 | twin --ANTI--> check | |
8414 | ||
8415 | If BE_IN_SPEC: [insn ~~TRUE~~> producer]: | |
8416 | check ~~TRUE~~> producer | |
8417 | twin ~~TRUE~~> producer | |
b8698a0f | 8418 | twin --ANTI--> check */ |
496d7bb0 | 8419 | |
e2f6ff94 | 8420 | ds = DEP_STATUS (dep); |
496d7bb0 MK |
8421 | |
8422 | if (ds & BEGIN_SPEC) | |
8423 | { | |
8424 | gcc_assert (!mutate_p); | |
8425 | ds &= ~BEGIN_SPEC; | |
8426 | } | |
8427 | ||
e2f6ff94 MK |
8428 | init_dep_1 (new_dep, pro, check, DEP_TYPE (dep), ds); |
8429 | sd_add_dep (new_dep, false); | |
8430 | ||
fefa31b5 | 8431 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 | 8432 | { |
0e0ce50d | 8433 | DEP_CON (new_dep) = twin; |
e2f6ff94 | 8434 | sd_add_dep (new_dep, false); |
b8698a0f | 8435 | } |
496d7bb0 MK |
8436 | } |
8437 | ||
e2f6ff94 MK |
8438 | /* Second, remove backward dependencies of INSN. */ |
8439 | for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK); | |
8440 | sd_iterator_cond (&sd_it, &dep);) | |
8441 | { | |
8442 | if ((DEP_STATUS (dep) & BEGIN_SPEC) | |
8443 | || mutate_p) | |
8444 | /* We can delete this dep because we overcome it with | |
8445 | BEGIN_SPECULATION. */ | |
8446 | sd_delete_dep (sd_it); | |
8447 | else | |
8448 | sd_iterator_next (&sd_it); | |
8449 | } | |
496d7bb0 | 8450 | |
e2f6ff94 | 8451 | /* Future Speculations. Determine what BE_IN speculations will be like. */ |
496d7bb0 MK |
8452 | fs = 0; |
8453 | ||
8454 | /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only | |
8455 | here. */ | |
b8698a0f | 8456 | |
496d7bb0 | 8457 | gcc_assert (!DONE_SPEC (insn)); |
b8698a0f | 8458 | |
496d7bb0 | 8459 | if (!mutate_p) |
b8698a0f | 8460 | { |
496d7bb0 MK |
8461 | ds_t ts = TODO_SPEC (insn); |
8462 | ||
8463 | DONE_SPEC (insn) = ts & BEGIN_SPEC; | |
8464 | CHECK_SPEC (check) = ts & BEGIN_SPEC; | |
8465 | ||
15dc95cb | 8466 | /* Luckiness of future speculations solely depends upon initial |
e2f6ff94 | 8467 | BEGIN speculation. */ |
496d7bb0 MK |
8468 | if (ts & BEGIN_DATA) |
8469 | fs = set_dep_weak (fs, BE_IN_DATA, get_dep_weak (ts, BEGIN_DATA)); | |
8470 | if (ts & BEGIN_CONTROL) | |
e2f6ff94 MK |
8471 | fs = set_dep_weak (fs, BE_IN_CONTROL, |
8472 | get_dep_weak (ts, BEGIN_CONTROL)); | |
496d7bb0 MK |
8473 | } |
8474 | else | |
8475 | CHECK_SPEC (check) = CHECK_SPEC (insn); | |
8476 | ||
8477 | /* Future speculations: call the helper. */ | |
e2f6ff94 | 8478 | process_insn_forw_deps_be_in_spec (insn, twin, fs); |
496d7bb0 | 8479 | |
fefa31b5 | 8480 | if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 MK |
8481 | { |
8482 | /* Which types of dependencies should we use here is, | |
8483 | generally, machine-dependent question... But, for now, | |
8484 | it is not. */ | |
8485 | ||
8486 | if (!mutate_p) | |
8487 | { | |
e2f6ff94 MK |
8488 | init_dep (new_dep, insn, check, REG_DEP_TRUE); |
8489 | sd_add_dep (new_dep, false); | |
8490 | ||
8491 | init_dep (new_dep, insn, twin, REG_DEP_OUTPUT); | |
8492 | sd_add_dep (new_dep, false); | |
496d7bb0 MK |
8493 | } |
8494 | else | |
8495 | { | |
b8698a0f | 8496 | if (spec_info->dump) |
496d7bb0 MK |
8497 | fprintf (spec_info->dump, ";;\t\tRemoved simple check : %s\n", |
8498 | (*current_sched_info->print_insn) (insn, 0)); | |
8499 | ||
e2f6ff94 MK |
8500 | /* Remove all dependencies of the INSN. */ |
8501 | { | |
8502 | sd_it = sd_iterator_start (insn, (SD_LIST_FORW | |
8503 | | SD_LIST_BACK | |
8504 | | SD_LIST_RES_BACK)); | |
8505 | while (sd_iterator_cond (&sd_it, &dep)) | |
8506 | sd_delete_dep (sd_it); | |
8507 | } | |
496d7bb0 | 8508 | |
e2f6ff94 MK |
8509 | /* If former check (INSN) already was moved to the ready (or queue) |
8510 | list, add new check (CHECK) there too. */ | |
496d7bb0 MK |
8511 | if (QUEUE_INDEX (insn) != QUEUE_NOWHERE) |
8512 | try_ready (check); | |
8513 | ||
e2f6ff94 MK |
8514 | /* Remove old check from instruction stream and free its |
8515 | data. */ | |
496d7bb0 MK |
8516 | sched_remove_insn (insn); |
8517 | } | |
8518 | ||
e2f6ff94 MK |
8519 | init_dep (new_dep, check, twin, REG_DEP_ANTI); |
8520 | sd_add_dep (new_dep, false); | |
496d7bb0 MK |
8521 | } |
8522 | else | |
e2f6ff94 MK |
8523 | { |
8524 | init_dep_1 (new_dep, insn, check, REG_DEP_TRUE, DEP_TRUE | DEP_OUTPUT); | |
8525 | sd_add_dep (new_dep, false); | |
8526 | } | |
496d7bb0 MK |
8527 | |
8528 | if (!mutate_p) | |
917f1b7e | 8529 | /* Fix priorities. If MUTATE_P is nonzero, this is not necessary, |
496d7bb0 MK |
8530 | because it'll be done later in add_to_speculative_block. */ |
8531 | { | |
8c681247 | 8532 | auto_vec<rtx_insn *> priorities_roots; |
916fa4f0 MK |
8533 | |
8534 | clear_priorities (twin, &priorities_roots); | |
8535 | calc_priorities (priorities_roots); | |
496d7bb0 MK |
8536 | } |
8537 | } | |
8538 | ||
8539 | /* Removes dependency between instructions in the recovery block REC | |
8540 | and usual region instructions. It keeps inner dependences so it | |
917f1b7e | 8541 | won't be necessary to recompute them. */ |
496d7bb0 MK |
8542 | static void |
8543 | fix_recovery_deps (basic_block rec) | |
8544 | { | |
9b517712 | 8545 | rtx_insn *note, *insn, *jump; |
9cad8a67 | 8546 | auto_vec<rtx_insn *, 10> ready_list; |
8f9b31f7 | 8547 | auto_bitmap in_ready; |
b8698a0f | 8548 | |
496d7bb0 MK |
8549 | /* NOTE - a basic block note. */ |
8550 | note = NEXT_INSN (BB_HEAD (rec)); | |
8551 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note)); | |
8552 | insn = BB_END (rec); | |
8553 | gcc_assert (JUMP_P (insn)); | |
8554 | insn = PREV_INSN (insn); | |
8555 | ||
8556 | do | |
e2f6ff94 MK |
8557 | { |
8558 | sd_iterator_def sd_it; | |
8559 | dep_t dep; | |
496d7bb0 | 8560 | |
e2f6ff94 MK |
8561 | for (sd_it = sd_iterator_start (insn, SD_LIST_FORW); |
8562 | sd_iterator_cond (&sd_it, &dep);) | |
8563 | { | |
9b517712 | 8564 | rtx_insn *consumer = DEP_CON (dep); |
496d7bb0 MK |
8565 | |
8566 | if (BLOCK_FOR_INSN (consumer) != rec) | |
8567 | { | |
e2f6ff94 | 8568 | sd_delete_dep (sd_it); |
496d7bb0 | 8569 | |
8f9b31f7 | 8570 | if (bitmap_set_bit (in_ready, INSN_LUID (consumer))) |
9cad8a67 | 8571 | ready_list.safe_push (consumer); |
496d7bb0 MK |
8572 | } |
8573 | else | |
8574 | { | |
e2f6ff94 | 8575 | gcc_assert ((DEP_STATUS (dep) & DEP_TYPES) == DEP_TRUE); |
496d7bb0 | 8576 | |
e2f6ff94 | 8577 | sd_iterator_next (&sd_it); |
496d7bb0 MK |
8578 | } |
8579 | } | |
b8698a0f | 8580 | |
496d7bb0 MK |
8581 | insn = PREV_INSN (insn); |
8582 | } | |
8583 | while (insn != note); | |
8584 | ||
496d7bb0 | 8585 | /* Try to add instructions to the ready or queue list. */ |
9cad8a67 TS |
8586 | unsigned int i; |
8587 | rtx_insn *temp; | |
8588 | FOR_EACH_VEC_ELT_REVERSE (ready_list, i, temp) | |
8589 | try_ready (temp); | |
496d7bb0 MK |
8590 | |
8591 | /* Fixing jump's dependences. */ | |
8592 | insn = BB_HEAD (rec); | |
8593 | jump = BB_END (rec); | |
b8698a0f | 8594 | |
496d7bb0 MK |
8595 | gcc_assert (LABEL_P (insn)); |
8596 | insn = NEXT_INSN (insn); | |
b8698a0f | 8597 | |
496d7bb0 MK |
8598 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn)); |
8599 | add_jump_dependencies (insn, jump); | |
8600 | } | |
8601 | ||
e2724e63 BS |
8602 | /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa |
8603 | instruction data. */ | |
8604 | static bool | |
f57aa6b0 | 8605 | haifa_change_pattern (rtx_insn *insn, rtx new_pat) |
496d7bb0 MK |
8606 | { |
8607 | int t; | |
8608 | ||
8609 | t = validate_change (insn, &PATTERN (insn), new_pat, 0); | |
e2724e63 BS |
8610 | if (!t) |
8611 | return false; | |
e855c69d | 8612 | |
1a83e602 | 8613 | update_insn_after_change (insn); |
e2724e63 | 8614 | return true; |
496d7bb0 MK |
8615 | } |
8616 | ||
496d7bb0 MK |
8617 | /* -1 - can't speculate, |
8618 | 0 - for speculation with REQUEST mode it is OK to use | |
8619 | current instruction pattern, | |
8620 | 1 - need to change pattern for *NEW_PAT to be speculative. */ | |
e855c69d | 8621 | int |
ac44248e | 8622 | sched_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat) |
496d7bb0 MK |
8623 | { |
8624 | gcc_assert (current_sched_info->flags & DO_SPECULATION | |
e2f6ff94 MK |
8625 | && (request & SPECULATIVE) |
8626 | && sched_insn_is_legitimate_for_speculation_p (insn, request)); | |
496d7bb0 | 8627 | |
e2f6ff94 | 8628 | if ((request & spec_info->mask) != request) |
496d7bb0 | 8629 | return -1; |
496d7bb0 | 8630 | |
e2f6ff94 MK |
8631 | if (request & BE_IN_SPEC |
8632 | && !(request & BEGIN_SPEC)) | |
8633 | return 0; | |
496d7bb0 | 8634 | |
e855c69d AB |
8635 | return targetm.sched.speculate_insn (insn, request, new_pat); |
8636 | } | |
8637 | ||
8638 | static int | |
f57aa6b0 | 8639 | haifa_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat) |
e855c69d AB |
8640 | { |
8641 | gcc_assert (sched_deps_info->generate_spec_deps | |
8642 | && !IS_SPECULATION_CHECK_P (insn)); | |
8643 | ||
8644 | if (HAS_INTERNAL_DEP (insn) | |
8645 | || SCHED_GROUP_P (insn)) | |
8646 | return -1; | |
8647 | ||
8648 | return sched_speculate_insn (insn, request, new_pat); | |
496d7bb0 MK |
8649 | } |
8650 | ||
8651 | /* Print some information about block BB, which starts with HEAD and | |
8652 | ends with TAIL, before scheduling it. | |
8653 | I is zero, if scheduler is about to start with the fresh ebb. */ | |
8654 | static void | |
f57aa6b0 | 8655 | dump_new_block_header (int i, basic_block bb, rtx_insn *head, rtx_insn *tail) |
496d7bb0 MK |
8656 | { |
8657 | if (!i) | |
8658 | fprintf (sched_dump, | |
8659 | ";; ======================================================\n"); | |
8660 | else | |
8661 | fprintf (sched_dump, | |
8662 | ";; =====================ADVANCING TO=====================\n"); | |
8663 | fprintf (sched_dump, | |
8664 | ";; -- basic block %d from %d to %d -- %s reload\n", | |
8665 | bb->index, INSN_UID (head), INSN_UID (tail), | |
8666 | (reload_completed ? "after" : "before")); | |
8667 | fprintf (sched_dump, | |
8668 | ";; ======================================================\n"); | |
8669 | fprintf (sched_dump, "\n"); | |
8670 | } | |
8671 | ||
8672 | /* Unlink basic block notes and labels and saves them, so they | |
8673 | can be easily restored. We unlink basic block notes in EBB to | |
917f1b7e | 8674 | provide back-compatibility with the previous code, as target backends |
496d7bb0 MK |
8675 | assume, that there'll be only instructions between |
8676 | current_sched_info->{head and tail}. We restore these notes as soon | |
8677 | as we can. | |
8678 | FIRST (LAST) is the first (last) basic block in the ebb. | |
8679 | NB: In usual case (FIRST == LAST) nothing is really done. */ | |
8680 | void | |
8681 | unlink_bb_notes (basic_block first, basic_block last) | |
8682 | { | |
8683 | /* We DON'T unlink basic block notes of the first block in the ebb. */ | |
8684 | if (first == last) | |
8685 | return; | |
8686 | ||
9b517712 | 8687 | bb_header = XNEWVEC (rtx_insn *, last_basic_block_for_fn (cfun)); |
496d7bb0 MK |
8688 | |
8689 | /* Make a sentinel. */ | |
fefa31b5 | 8690 | if (last->next_bb != EXIT_BLOCK_PTR_FOR_FN (cfun)) |
496d7bb0 MK |
8691 | bb_header[last->next_bb->index] = 0; |
8692 | ||
8693 | first = first->next_bb; | |
8694 | do | |
8695 | { | |
9b517712 | 8696 | rtx_insn *prev, *label, *note, *next; |
496d7bb0 MK |
8697 | |
8698 | label = BB_HEAD (last); | |
8699 | if (LABEL_P (label)) | |
8700 | note = NEXT_INSN (label); | |
8701 | else | |
b8698a0f | 8702 | note = label; |
496d7bb0 MK |
8703 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note)); |
8704 | ||
8705 | prev = PREV_INSN (label); | |
8706 | next = NEXT_INSN (note); | |
8707 | gcc_assert (prev && next); | |
8708 | ||
0f82e5c9 DM |
8709 | SET_NEXT_INSN (prev) = next; |
8710 | SET_PREV_INSN (next) = prev; | |
496d7bb0 MK |
8711 | |
8712 | bb_header[last->index] = label; | |
8713 | ||
8714 | if (last == first) | |
8715 | break; | |
b8698a0f | 8716 | |
496d7bb0 MK |
8717 | last = last->prev_bb; |
8718 | } | |
8719 | while (1); | |
8720 | } | |
8721 | ||
8722 | /* Restore basic block notes. | |
8723 | FIRST is the first basic block in the ebb. */ | |
8724 | static void | |
8725 | restore_bb_notes (basic_block first) | |
8726 | { | |
8727 | if (!bb_header) | |
8728 | return; | |
8729 | ||
8730 | /* We DON'T unlink basic block notes of the first block in the ebb. */ | |
b8698a0f | 8731 | first = first->next_bb; |
496d7bb0 MK |
8732 | /* Remember: FIRST is actually a second basic block in the ebb. */ |
8733 | ||
fefa31b5 | 8734 | while (first != EXIT_BLOCK_PTR_FOR_FN (cfun) |
496d7bb0 MK |
8735 | && bb_header[first->index]) |
8736 | { | |
9b517712 | 8737 | rtx_insn *prev, *label, *note, *next; |
b8698a0f | 8738 | |
496d7bb0 MK |
8739 | label = bb_header[first->index]; |
8740 | prev = PREV_INSN (label); | |
8741 | next = NEXT_INSN (prev); | |
8742 | ||
8743 | if (LABEL_P (label)) | |
8744 | note = NEXT_INSN (label); | |
8745 | else | |
b8698a0f | 8746 | note = label; |
496d7bb0 MK |
8747 | gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note)); |
8748 | ||
8749 | bb_header[first->index] = 0; | |
8750 | ||
0f82e5c9 DM |
8751 | SET_NEXT_INSN (prev) = label; |
8752 | SET_NEXT_INSN (note) = next; | |
8753 | SET_PREV_INSN (next) = note; | |
b8698a0f | 8754 | |
496d7bb0 MK |
8755 | first = first->next_bb; |
8756 | } | |
8757 | ||
8758 | free (bb_header); | |
8759 | bb_header = 0; | |
8760 | } | |
8761 | ||
496d7bb0 MK |
8762 | /* Helper function. |
8763 | Fix CFG after both in- and inter-block movement of | |
8764 | control_flow_insn_p JUMP. */ | |
8765 | static void | |
f57aa6b0 | 8766 | fix_jump_move (rtx_insn *jump) |
496d7bb0 MK |
8767 | { |
8768 | basic_block bb, jump_bb, jump_bb_next; | |
8769 | ||
8770 | bb = BLOCK_FOR_INSN (PREV_INSN (jump)); | |
8771 | jump_bb = BLOCK_FOR_INSN (jump); | |
8772 | jump_bb_next = jump_bb->next_bb; | |
8773 | ||
e855c69d | 8774 | gcc_assert (common_sched_info->sched_pass_id == SCHED_EBB_PASS |
d7bfd907 | 8775 | || IS_SPECULATION_BRANCHY_CHECK_P (jump)); |
b8698a0f | 8776 | |
496d7bb0 MK |
8777 | if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next))) |
8778 | /* if jump_bb_next is not empty. */ | |
1130d5e3 | 8779 | BB_END (jump_bb) = BB_END (jump_bb_next); |
496d7bb0 MK |
8780 | |
8781 | if (BB_END (bb) != PREV_INSN (jump)) | |
8782 | /* Then there are instruction after jump that should be placed | |
8783 | to jump_bb_next. */ | |
1130d5e3 | 8784 | BB_END (jump_bb_next) = BB_END (bb); |
496d7bb0 MK |
8785 | else |
8786 | /* Otherwise jump_bb_next is empty. */ | |
1130d5e3 | 8787 | BB_END (jump_bb_next) = NEXT_INSN (BB_HEAD (jump_bb_next)); |
496d7bb0 MK |
8788 | |
8789 | /* To make assertion in move_insn happy. */ | |
1130d5e3 | 8790 | BB_END (bb) = PREV_INSN (jump); |
496d7bb0 MK |
8791 | |
8792 | update_bb_for_insn (jump_bb_next); | |
8793 | } | |
8794 | ||
8795 | /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */ | |
8796 | static void | |
f57aa6b0 | 8797 | move_block_after_check (rtx_insn *jump) |
496d7bb0 MK |
8798 | { |
8799 | basic_block bb, jump_bb, jump_bb_next; | |
9771b263 | 8800 | vec<edge, va_gc> *t; |
496d7bb0 MK |
8801 | |
8802 | bb = BLOCK_FOR_INSN (PREV_INSN (jump)); | |
8803 | jump_bb = BLOCK_FOR_INSN (jump); | |
8804 | jump_bb_next = jump_bb->next_bb; | |
b8698a0f | 8805 | |
496d7bb0 | 8806 | update_bb_for_insn (jump_bb); |
b8698a0f | 8807 | |
d7bfd907 MK |
8808 | gcc_assert (IS_SPECULATION_CHECK_P (jump) |
8809 | || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next))); | |
496d7bb0 MK |
8810 | |
8811 | unlink_block (jump_bb_next); | |
8812 | link_block (jump_bb_next, bb); | |
8813 | ||
8814 | t = bb->succs; | |
8815 | bb->succs = 0; | |
8816 | move_succs (&(jump_bb->succs), bb); | |
8817 | move_succs (&(jump_bb_next->succs), jump_bb); | |
8818 | move_succs (&t, jump_bb_next); | |
6fb5fa3c DB |
8819 | |
8820 | df_mark_solutions_dirty (); | |
b8698a0f | 8821 | |
e855c69d AB |
8822 | common_sched_info->fix_recovery_cfg |
8823 | (bb->index, jump_bb->index, jump_bb_next->index); | |
496d7bb0 MK |
8824 | } |
8825 | ||
8826 | /* Helper function for move_block_after_check. | |
8827 | This functions attaches edge vector pointed to by SUCCSP to | |
8828 | block TO. */ | |
8829 | static void | |
9771b263 | 8830 | move_succs (vec<edge, va_gc> **succsp, basic_block to) |
496d7bb0 MK |
8831 | { |
8832 | edge e; | |
8833 | edge_iterator ei; | |
8834 | ||
8835 | gcc_assert (to->succs == 0); | |
8836 | ||
8837 | to->succs = *succsp; | |
8838 | ||
8839 | FOR_EACH_EDGE (e, ei, to->succs) | |
8840 | e->src = to; | |
8841 | ||
8842 | *succsp = 0; | |
8843 | } | |
8844 | ||
496d7bb0 MK |
8845 | /* Remove INSN from the instruction stream. |
8846 | INSN should have any dependencies. */ | |
8847 | static void | |
ce1ce33a | 8848 | sched_remove_insn (rtx_insn *insn) |
496d7bb0 | 8849 | { |
e2f6ff94 MK |
8850 | sd_finish_insn (insn); |
8851 | ||
496d7bb0 MK |
8852 | change_queue_index (insn, QUEUE_NOWHERE); |
8853 | current_sched_info->add_remove_insn (insn, 1); | |
1f397f45 | 8854 | delete_insn (insn); |
496d7bb0 MK |
8855 | } |
8856 | ||
916fa4f0 MK |
8857 | /* Clear priorities of all instructions, that are forward dependent on INSN. |
8858 | Store in vector pointed to by ROOTS_PTR insns on which priority () should | |
8859 | be invoked to initialize all cleared priorities. */ | |
496d7bb0 | 8860 | static void |
6144a836 | 8861 | clear_priorities (rtx_insn *insn, rtx_vec_t *roots_ptr) |
496d7bb0 | 8862 | { |
e2f6ff94 MK |
8863 | sd_iterator_def sd_it; |
8864 | dep_t dep; | |
916fa4f0 MK |
8865 | bool insn_is_root_p = true; |
8866 | ||
8867 | gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED); | |
496d7bb0 | 8868 | |
e2f6ff94 | 8869 | FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep) |
496d7bb0 | 8870 | { |
9b517712 | 8871 | rtx_insn *pro = DEP_PRO (dep); |
496d7bb0 | 8872 | |
916fa4f0 MK |
8873 | if (INSN_PRIORITY_STATUS (pro) >= 0 |
8874 | && QUEUE_INDEX (insn) != QUEUE_SCHEDULED) | |
496d7bb0 | 8875 | { |
916fa4f0 MK |
8876 | /* If DEP doesn't contribute to priority then INSN itself should |
8877 | be added to priority roots. */ | |
8878 | if (contributes_to_priority_p (dep)) | |
8879 | insn_is_root_p = false; | |
8880 | ||
8881 | INSN_PRIORITY_STATUS (pro) = -1; | |
8882 | clear_priorities (pro, roots_ptr); | |
496d7bb0 MK |
8883 | } |
8884 | } | |
916fa4f0 MK |
8885 | |
8886 | if (insn_is_root_p) | |
9771b263 | 8887 | roots_ptr->safe_push (insn); |
496d7bb0 MK |
8888 | } |
8889 | ||
8890 | /* Recompute priorities of instructions, whose priorities might have been | |
916fa4f0 MK |
8891 | changed. ROOTS is a vector of instructions whose priority computation will |
8892 | trigger initialization of all cleared priorities. */ | |
496d7bb0 | 8893 | static void |
916fa4f0 | 8894 | calc_priorities (rtx_vec_t roots) |
496d7bb0 | 8895 | { |
916fa4f0 | 8896 | int i; |
6144a836 | 8897 | rtx_insn *insn; |
496d7bb0 | 8898 | |
9771b263 | 8899 | FOR_EACH_VEC_ELT (roots, i, insn) |
916fa4f0 | 8900 | priority (insn); |
496d7bb0 MK |
8901 | } |
8902 | ||
8903 | ||
8904 | /* Add dependences between JUMP and other instructions in the recovery | |
8905 | block. INSN is the first insn the recovery block. */ | |
8906 | static void | |
0e0ce50d | 8907 | add_jump_dependencies (rtx_insn *insn, rtx_insn *jump) |
496d7bb0 MK |
8908 | { |
8909 | do | |
8910 | { | |
8911 | insn = NEXT_INSN (insn); | |
8912 | if (insn == jump) | |
8913 | break; | |
b8698a0f | 8914 | |
7a1aca9c | 8915 | if (dep_list_size (insn, SD_LIST_FORW) == 0) |
e2f6ff94 MK |
8916 | { |
8917 | dep_def _new_dep, *new_dep = &_new_dep; | |
8918 | ||
8919 | init_dep (new_dep, insn, jump, REG_DEP_ANTI); | |
8920 | sd_add_dep (new_dep, false); | |
8921 | } | |
496d7bb0 MK |
8922 | } |
8923 | while (1); | |
b198261f | 8924 | |
e2f6ff94 | 8925 | gcc_assert (!sd_lists_empty_p (jump, SD_LIST_BACK)); |
496d7bb0 MK |
8926 | } |
8927 | ||
e855c69d | 8928 | /* Extend data structures for logical insn UID. */ |
a95b23b4 BS |
8929 | void |
8930 | sched_extend_luids (void) | |
e855c69d AB |
8931 | { |
8932 | int new_luids_max_uid = get_max_uid () + 1; | |
8933 | ||
9771b263 | 8934 | sched_luids.safe_grow_cleared (new_luids_max_uid); |
e855c69d AB |
8935 | } |
8936 | ||
8937 | /* Initialize LUID for INSN. */ | |
a95b23b4 | 8938 | void |
f57aa6b0 | 8939 | sched_init_insn_luid (rtx_insn *insn) |
e855c69d AB |
8940 | { |
8941 | int i = INSN_P (insn) ? 1 : common_sched_info->luid_for_non_insn (insn); | |
8942 | int luid; | |
8943 | ||
8944 | if (i >= 0) | |
8945 | { | |
8946 | luid = sched_max_luid; | |
8947 | sched_max_luid += i; | |
8948 | } | |
8949 | else | |
8950 | luid = -1; | |
8951 | ||
8952 | SET_INSN_LUID (insn, luid); | |
8953 | } | |
8954 | ||
a95b23b4 | 8955 | /* Initialize luids for BBS. |
e855c69d AB |
8956 | The hook common_sched_info->luid_for_non_insn () is used to determine |
8957 | if notes, labels, etc. need luids. */ | |
8958 | void | |
a95b23b4 | 8959 | sched_init_luids (bb_vec_t bbs) |
e855c69d | 8960 | { |
a95b23b4 BS |
8961 | int i; |
8962 | basic_block bb; | |
8963 | ||
8964 | sched_extend_luids (); | |
9771b263 | 8965 | FOR_EACH_VEC_ELT (bbs, i, bb) |
e855c69d | 8966 | { |
f57aa6b0 | 8967 | rtx_insn *insn; |
e855c69d | 8968 | |
a95b23b4 BS |
8969 | FOR_BB_INSNS (bb, insn) |
8970 | sched_init_insn_luid (insn); | |
8971 | } | |
e855c69d AB |
8972 | } |
8973 | ||
8974 | /* Free LUIDs. */ | |
8975 | void | |
8976 | sched_finish_luids (void) | |
8977 | { | |
9771b263 | 8978 | sched_luids.release (); |
e855c69d AB |
8979 | sched_max_luid = 1; |
8980 | } | |
8981 | ||
8982 | /* Return logical uid of INSN. Helpful while debugging. */ | |
8983 | int | |
f57aa6b0 | 8984 | insn_luid (rtx_insn *insn) |
e855c69d AB |
8985 | { |
8986 | return INSN_LUID (insn); | |
8987 | } | |
8988 | ||
8989 | /* Extend per insn data in the target. */ | |
8990 | void | |
8991 | sched_extend_target (void) | |
8992 | { | |
8993 | if (targetm.sched.h_i_d_extended) | |
8994 | targetm.sched.h_i_d_extended (); | |
8995 | } | |
8996 | ||
8997 | /* Extend global scheduler structures (those, that live across calls to | |
8998 | schedule_block) to include information about just emitted INSN. */ | |
8999 | static void | |
9000 | extend_h_i_d (void) | |
9001 | { | |
9771b263 | 9002 | int reserve = (get_max_uid () + 1 - h_i_d.length ()); |
b8698a0f | 9003 | if (reserve > 0 |
9771b263 | 9004 | && ! h_i_d.space (reserve)) |
e855c69d | 9005 | { |
9771b263 | 9006 | h_i_d.safe_grow_cleared (3 * get_max_uid () / 2); |
e855c69d AB |
9007 | sched_extend_target (); |
9008 | } | |
9009 | } | |
9010 | ||
9011 | /* Initialize h_i_d entry of the INSN with default values. | |
9012 | Values, that are not explicitly initialized here, hold zero. */ | |
9013 | static void | |
f57aa6b0 | 9014 | init_h_i_d (rtx_insn *insn) |
e855c69d AB |
9015 | { |
9016 | if (INSN_LUID (insn) > 0) | |
9017 | { | |
9018 | INSN_COST (insn) = -1; | |
e855c69d AB |
9019 | QUEUE_INDEX (insn) = QUEUE_NOWHERE; |
9020 | INSN_TICK (insn) = INVALID_TICK; | |
26965010 | 9021 | INSN_EXACT_TICK (insn) = INVALID_TICK; |
e855c69d AB |
9022 | INTER_TICK (insn) = INVALID_TICK; |
9023 | TODO_SPEC (insn) = HARD_DEP; | |
340c7904 MK |
9024 | INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status |
9025 | = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED; | |
9026 | INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status | |
9027 | = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED; | |
e855c69d AB |
9028 | } |
9029 | } | |
9030 | ||
a95b23b4 | 9031 | /* Initialize haifa_insn_data for BBS. */ |
e855c69d | 9032 | void |
a95b23b4 | 9033 | haifa_init_h_i_d (bb_vec_t bbs) |
e855c69d | 9034 | { |
a95b23b4 BS |
9035 | int i; |
9036 | basic_block bb; | |
9037 | ||
9038 | extend_h_i_d (); | |
9771b263 | 9039 | FOR_EACH_VEC_ELT (bbs, i, bb) |
e855c69d | 9040 | { |
f57aa6b0 | 9041 | rtx_insn *insn; |
e855c69d | 9042 | |
a95b23b4 BS |
9043 | FOR_BB_INSNS (bb, insn) |
9044 | init_h_i_d (insn); | |
9045 | } | |
e855c69d AB |
9046 | } |
9047 | ||
9048 | /* Finalize haifa_insn_data. */ | |
9049 | void | |
9050 | haifa_finish_h_i_d (void) | |
9051 | { | |
ce18efcb VM |
9052 | int i; |
9053 | haifa_insn_data_t data; | |
36fb33de ML |
9054 | reg_use_data *use, *next_use; |
9055 | reg_set_data *set, *next_set; | |
ce18efcb | 9056 | |
9771b263 | 9057 | FOR_EACH_VEC_ELT (h_i_d, i, data) |
ce18efcb | 9058 | { |
7a1aca9c | 9059 | free (data->max_reg_pressure); |
04695783 | 9060 | free (data->reg_pressure); |
36fb33de | 9061 | for (use = data->reg_use_list; use != NULL; use = next_use) |
ce18efcb | 9062 | { |
36fb33de | 9063 | next_use = use->next_insn_use; |
ce18efcb VM |
9064 | free (use); |
9065 | } | |
36fb33de ML |
9066 | for (set = data->reg_set_list; set != NULL; set = next_set) |
9067 | { | |
9068 | next_set = set->next_insn_set; | |
9069 | free (set); | |
9070 | } | |
9071 | ||
ce18efcb | 9072 | } |
9771b263 | 9073 | h_i_d.release (); |
e855c69d AB |
9074 | } |
9075 | ||
9076 | /* Init data for the new insn INSN. */ | |
9077 | static void | |
f57aa6b0 | 9078 | haifa_init_insn (rtx_insn *insn) |
e855c69d AB |
9079 | { |
9080 | gcc_assert (insn != NULL); | |
9081 | ||
a95b23b4 BS |
9082 | sched_extend_luids (); |
9083 | sched_init_insn_luid (insn); | |
e855c69d AB |
9084 | sched_extend_target (); |
9085 | sched_deps_init (false); | |
a95b23b4 BS |
9086 | extend_h_i_d (); |
9087 | init_h_i_d (insn); | |
e855c69d AB |
9088 | |
9089 | if (adding_bb_to_current_region_p) | |
9090 | { | |
9091 | sd_init_insn (insn); | |
9092 | ||
9093 | /* Extend dependency caches by one element. */ | |
9094 | extend_dependency_caches (1, false); | |
9095 | } | |
7a1aca9c | 9096 | if (sched_pressure != SCHED_PRESSURE_NONE) |
19ac7892 | 9097 | init_insn_reg_pressure_info (insn); |
e855c69d AB |
9098 | } |
9099 | ||
9100 | /* Init data for the new basic block BB which comes after AFTER. */ | |
9101 | static void | |
9102 | haifa_init_only_bb (basic_block bb, basic_block after) | |
9103 | { | |
9104 | gcc_assert (bb != NULL); | |
9105 | ||
9106 | sched_init_bbs (); | |
9107 | ||
9108 | if (common_sched_info->add_block) | |
9109 | /* This changes only data structures of the front-end. */ | |
9110 | common_sched_info->add_block (bb, after); | |
9111 | } | |
9112 | ||
9113 | /* A generic version of sched_split_block (). */ | |
9114 | basic_block | |
9115 | sched_split_block_1 (basic_block first_bb, rtx after) | |
9116 | { | |
9117 | edge e; | |
9118 | ||
9119 | e = split_block (first_bb, after); | |
9120 | gcc_assert (e->src == first_bb); | |
9121 | ||
b8698a0f | 9122 | /* sched_split_block emits note if *check == BB_END. Probably it |
e855c69d AB |
9123 | is better to rip that note off. */ |
9124 | ||
9125 | return e->dest; | |
9126 | } | |
9127 | ||
9128 | /* A generic version of sched_create_empty_bb (). */ | |
9129 | basic_block | |
9130 | sched_create_empty_bb_1 (basic_block after) | |
9131 | { | |
9132 | return create_empty_bb (after); | |
9133 | } | |
9134 | ||
9dcc2e87 TS |
9135 | /* Insert PAT as an INSN into the schedule and update the necessary data |
9136 | structures to account for it. */ | |
ce1ce33a | 9137 | rtx_insn * |
9dcc2e87 TS |
9138 | sched_emit_insn (rtx pat) |
9139 | { | |
ce1ce33a | 9140 | rtx_insn *insn = emit_insn_before (pat, first_nonscheduled_insn ()); |
9dcc2e87 | 9141 | haifa_init_insn (insn); |
2dfdcb4b UW |
9142 | |
9143 | if (current_sched_info->add_remove_insn) | |
9144 | current_sched_info->add_remove_insn (insn, 0); | |
9145 | ||
9146 | (*current_sched_info->begin_schedule_ready) (insn); | |
9771b263 | 9147 | scheduled_insns.safe_push (insn); |
2dfdcb4b UW |
9148 | |
9149 | last_scheduled_insn = insn; | |
9dcc2e87 TS |
9150 | return insn; |
9151 | } | |
9152 | ||
7942e47e RY |
9153 | /* This function returns a candidate satisfying dispatch constraints from |
9154 | the ready list. */ | |
9155 | ||
ce1ce33a | 9156 | static rtx_insn * |
7942e47e RY |
9157 | ready_remove_first_dispatch (struct ready_list *ready) |
9158 | { | |
9159 | int i; | |
ce1ce33a | 9160 | rtx_insn *insn = ready_element (ready, 0); |
7942e47e RY |
9161 | |
9162 | if (ready->n_ready == 1 | |
7942e47e | 9163 | || !INSN_P (insn) |
d7b6661b | 9164 | || INSN_CODE (insn) < 0 |
7942e47e RY |
9165 | || !active_insn_p (insn) |
9166 | || targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW)) | |
9167 | return ready_remove_first (ready); | |
9168 | ||
9169 | for (i = 1; i < ready->n_ready; i++) | |
9170 | { | |
9171 | insn = ready_element (ready, i); | |
9172 | ||
d7b6661b SB |
9173 | if (!INSN_P (insn) |
9174 | || INSN_CODE (insn) < 0 | |
7942e47e RY |
9175 | || !active_insn_p (insn)) |
9176 | continue; | |
9177 | ||
9178 | if (targetm.sched.dispatch (insn, FITS_DISPATCH_WINDOW)) | |
9179 | { | |
9180 | /* Return ith element of ready. */ | |
9181 | insn = ready_remove (ready, i); | |
9182 | return insn; | |
9183 | } | |
9184 | } | |
9185 | ||
ac44248e | 9186 | if (targetm.sched.dispatch (NULL, DISPATCH_VIOLATION)) |
7942e47e RY |
9187 | return ready_remove_first (ready); |
9188 | ||
9189 | for (i = 1; i < ready->n_ready; i++) | |
9190 | { | |
9191 | insn = ready_element (ready, i); | |
9192 | ||
98eaa66f | 9193 | if (!INSN_P (insn) |
d7b6661b | 9194 | || INSN_CODE (insn) < 0 |
7942e47e RY |
9195 | || !active_insn_p (insn)) |
9196 | continue; | |
9197 | ||
9198 | /* Return i-th element of ready. */ | |
9199 | if (targetm.sched.dispatch (insn, IS_CMP)) | |
9200 | return ready_remove (ready, i); | |
9201 | } | |
9202 | ||
9203 | return ready_remove_first (ready); | |
9204 | } | |
9205 | ||
9206 | /* Get number of ready insn in the ready list. */ | |
9207 | ||
9208 | int | |
9209 | number_in_ready (void) | |
9210 | { | |
9211 | return ready.n_ready; | |
9212 | } | |
9213 | ||
9214 | /* Get number of ready's in the ready list. */ | |
9215 | ||
6a263466 | 9216 | rtx_insn * |
7942e47e RY |
9217 | get_ready_element (int i) |
9218 | { | |
9219 | return ready_element (&ready, i); | |
9220 | } | |
9221 | ||
8c660648 | 9222 | #endif /* INSN_SCHEDULING */ |