]>
Commit | Line | Data |
---|---|---|
fb85abff | 1 | /* Vectorizer |
2 | Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free | |
3 | Software Foundation, Inc. | |
c91e8223 | 4 | Contributed by Dorit Naishlos <dorit@il.ibm.com> |
5 | ||
6 | This file is part of GCC. | |
7 | ||
8 | GCC is free software; you can redistribute it and/or modify it under | |
9 | the terms of the GNU General Public License as published by the Free | |
8c4c00c1 | 10 | Software Foundation; either version 3, or (at your option) any later |
c91e8223 | 11 | version. |
12 | ||
13 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
14 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
15 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
16 | for more details. | |
17 | ||
18 | You should have received a copy of the GNU General Public License | |
8c4c00c1 | 19 | along with GCC; see the file COPYING3. If not see |
20 | <http://www.gnu.org/licenses/>. */ | |
c91e8223 | 21 | |
22 | #ifndef GCC_TREE_VECTORIZER_H | |
23 | #define GCC_TREE_VECTORIZER_H | |
24 | ||
fb85abff | 25 | #include "tree-data-ref.h" |
26 | ||
9c85a98a | 27 | typedef source_location LOC; |
28 | #define UNKNOWN_LOC UNKNOWN_LOCATION | |
29 | #define EXPR_LOC(e) EXPR_LOCATION(e) | |
30 | #define LOC_FILE(l) LOCATION_FILE (l) | |
31 | #define LOC_LINE(l) LOCATION_LINE (l) | |
90242140 | 32 | |
c91e8223 | 33 | /* Used for naming of new temporaries. */ |
34 | enum vect_var_kind { | |
35 | vect_simple_var, | |
ea8f3370 | 36 | vect_pointer_var, |
37 | vect_scalar_var | |
c91e8223 | 38 | }; |
39 | ||
4a61a337 | 40 | /* Defines type of operation. */ |
c91e8223 | 41 | enum operation_type { |
42 | unary_op = 1, | |
4a61a337 | 43 | binary_op, |
44 | ternary_op | |
c91e8223 | 45 | }; |
46 | ||
1a9b4618 | 47 | /* Define type of available alignment support. */ |
48 | enum dr_alignment_support { | |
49 | dr_unaligned_unsupported, | |
50 | dr_unaligned_supported, | |
b0eb8c66 | 51 | dr_explicit_realign, |
52 | dr_explicit_realign_optimized, | |
1a9b4618 | 53 | dr_aligned |
54 | }; | |
55 | ||
ce10738f | 56 | /* Define type of def-use cross-iteration cycle. */ |
e12906b9 | 57 | enum vect_def_type { |
bc620c5c | 58 | vect_uninitialized_def = 0, |
59 | vect_constant_def, | |
e12906b9 | 60 | vect_invariant_def, |
61 | vect_loop_def, | |
62 | vect_induction_def, | |
63 | vect_reduction_def, | |
64 | vect_unknown_def_type | |
65 | }; | |
66 | ||
459e8543 | 67 | /* Define verbosity levels. */ |
68 | enum verbosity_levels { | |
69 | REPORT_NONE, | |
70 | REPORT_VECTORIZED_LOOPS, | |
71 | REPORT_UNVECTORIZED_LOOPS, | |
1737b19b | 72 | REPORT_COST, |
459e8543 | 73 | REPORT_ALIGNMENT, |
516849c7 | 74 | REPORT_DR_DETAILS, |
459e8543 | 75 | REPORT_BAD_FORM_LOOPS, |
76 | REPORT_OUTER_LOOPS, | |
c6895939 | 77 | REPORT_SLP, |
459e8543 | 78 | REPORT_DETAILS, |
79 | /* New verbosity levels should be added before this one. */ | |
80 | MAX_VERBOSITY_LEVEL | |
81 | }; | |
82 | ||
c6895939 | 83 | /************************************************************************ |
84 | SLP | |
85 | ************************************************************************/ | |
86 | ||
87 | /* A computation tree of an SLP instance. Each node corresponds to a group of | |
88 | stmts to be packed in a SIMD stmt. */ | |
89 | typedef struct _slp_tree { | |
90 | /* Only binary and unary operations are supported. LEFT child corresponds to | |
91 | the first operand and RIGHT child to the second if the operation is | |
92 | binary. */ | |
93 | struct _slp_tree *left; | |
94 | struct _slp_tree *right; | |
95 | /* A group of scalar stmts to be vectorized together. */ | |
75a70cf9 | 96 | VEC (gimple, heap) *stmts; |
c6895939 | 97 | /* Vectorized stmt/s. */ |
75a70cf9 | 98 | VEC (gimple, heap) *vec_stmts; |
c6895939 | 99 | /* Number of vector stmts that are created to replace the group of scalar |
100 | stmts. It is calculated during the transformation phase as the number of | |
101 | scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF | |
102 | divided by vector size. */ | |
103 | unsigned int vec_stmts_size; | |
104 | /* Vectorization costs associated with SLP node. */ | |
105 | struct | |
106 | { | |
107 | int outside_of_loop; /* Statements generated outside loop. */ | |
108 | int inside_of_loop; /* Statements generated inside loop. */ | |
109 | } cost; | |
110 | } *slp_tree; | |
111 | ||
a0515226 | 112 | DEF_VEC_P(slp_tree); |
113 | DEF_VEC_ALLOC_P(slp_tree, heap); | |
c6895939 | 114 | |
115 | /* SLP instance is a sequence of stmts in a loop that can be packed into | |
116 | SIMD stmts. */ | |
117 | typedef struct _slp_instance { | |
118 | /* The root of SLP tree. */ | |
119 | slp_tree root; | |
120 | ||
121 | /* Size of groups of scalar stmts that will be replaced by SIMD stmt/s. */ | |
122 | unsigned int group_size; | |
123 | ||
124 | /* The unrolling factor required to vectorized this SLP instance. */ | |
125 | unsigned int unrolling_factor; | |
126 | ||
127 | /* Vectorization costs associated with SLP instance. */ | |
128 | struct | |
129 | { | |
130 | int outside_of_loop; /* Statements generated outside loop. */ | |
131 | int inside_of_loop; /* Statements generated inside loop. */ | |
132 | } cost; | |
a0515226 | 133 | |
134 | /* Loads permutation relatively to the stores, NULL if there is no | |
135 | permutation. */ | |
136 | VEC (int, heap) *load_permutation; | |
137 | ||
138 | /* The group of nodes that contain loads of this SLP instance. */ | |
139 | VEC (slp_tree, heap) *loads; | |
bdc89b8f | 140 | |
141 | /* The first scalar load of the instance. The created vector loads will be | |
142 | inserted before this statement. */ | |
143 | gimple first_load; | |
c6895939 | 144 | } *slp_instance; |
145 | ||
146 | DEF_VEC_P(slp_instance); | |
147 | DEF_VEC_ALLOC_P(slp_instance, heap); | |
148 | ||
149 | /* Access Functions. */ | |
150 | #define SLP_INSTANCE_TREE(S) (S)->root | |
151 | #define SLP_INSTANCE_GROUP_SIZE(S) (S)->group_size | |
152 | #define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor | |
153 | #define SLP_INSTANCE_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop | |
154 | #define SLP_INSTANCE_INSIDE_OF_LOOP_COST(S) (S)->cost.inside_of_loop | |
a0515226 | 155 | #define SLP_INSTANCE_LOAD_PERMUTATION(S) (S)->load_permutation |
156 | #define SLP_INSTANCE_LOADS(S) (S)->loads | |
bdc89b8f | 157 | #define SLP_INSTANCE_FIRST_LOAD_STMT(S) (S)->first_load |
c6895939 | 158 | |
159 | #define SLP_TREE_LEFT(S) (S)->left | |
160 | #define SLP_TREE_RIGHT(S) (S)->right | |
161 | #define SLP_TREE_SCALAR_STMTS(S) (S)->stmts | |
162 | #define SLP_TREE_VEC_STMTS(S) (S)->vec_stmts | |
163 | #define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size | |
164 | #define SLP_TREE_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop | |
165 | #define SLP_TREE_INSIDE_OF_LOOP_COST(S) (S)->cost.inside_of_loop | |
166 | ||
4e58562d | 167 | /*-----------------------------------------------------------------*/ |
168 | /* Info on vectorized loops. */ | |
169 | /*-----------------------------------------------------------------*/ | |
170 | typedef struct _loop_vec_info { | |
171 | ||
172 | /* The loop to which this info struct refers to. */ | |
173 | struct loop *loop; | |
174 | ||
175 | /* The loop basic blocks. */ | |
176 | basic_block *bbs; | |
177 | ||
4e58562d | 178 | /* Number of iterations. */ |
179 | tree num_iters; | |
be53c6d4 | 180 | tree num_iters_unchanged; |
4e58562d | 181 | |
62612fe9 | 182 | /* Minimum number of iterations below which vectorization is expected to |
183 | not be profitable (as estimated by the cost model). | |
184 | -1 indicates that vectorization will not be profitable. | |
185 | FORNOW: This field is an int. Will be a tree in the future, to represent | |
186 | values unknown at compile time. */ | |
187 | int min_profitable_iters; | |
188 | ||
4e58562d | 189 | /* Is the loop vectorizable? */ |
190 | bool vectorizable; | |
191 | ||
192 | /* Unrolling factor */ | |
193 | int vectorization_factor; | |
194 | ||
195 | /* Unknown DRs according to which loop was peeled. */ | |
196 | struct data_reference *unaligned_dr; | |
197 | ||
39b8f742 | 198 | /* peeling_for_alignment indicates whether peeling for alignment will take |
199 | place, and what the peeling factor should be: | |
200 | peeling_for_alignment = X means: | |
201 | If X=0: Peeling for alignment will not be applied. | |
202 | If X>0: Peel first X iterations. | |
203 | If X=-1: Generate a runtime test to calculate the number of iterations | |
204 | to be peeled, using the dataref recorded in the field | |
205 | unaligned_dr. */ | |
206 | int peeling_for_alignment; | |
4e58562d | 207 | |
25e3c2e8 | 208 | /* The mask used to check the alignment of pointers or arrays. */ |
209 | int ptr_mask; | |
210 | ||
516849c7 | 211 | /* All data references in the loop. */ |
41c7a324 | 212 | VEC (data_reference_p, heap) *datarefs; |
4e58562d | 213 | |
516849c7 | 214 | /* All data dependences in the loop. */ |
41c7a324 | 215 | VEC (ddr_p, heap) *ddrs; |
4e58562d | 216 | |
45b13dc3 | 217 | /* Data Dependence Relations defining address ranges that are candidates |
218 | for a run-time aliasing check. */ | |
219 | VEC (ddr_p, heap) *may_alias_ddrs; | |
220 | ||
25e3c2e8 | 221 | /* Statements in the loop that have data references that are candidates for a |
222 | runtime (loop versioning) misalignment check. */ | |
75a70cf9 | 223 | VEC(gimple,heap) *may_misalign_stmts; |
25e3c2e8 | 224 | |
516849c7 | 225 | /* The loop location in the source. */ |
226 | LOC loop_line_number; | |
c6895939 | 227 | |
228 | /* All interleaving chains of stores in the loop, represented by the first | |
229 | stmt in the chain. */ | |
75a70cf9 | 230 | VEC(gimple, heap) *strided_stores; |
c6895939 | 231 | |
232 | /* All SLP instances in the loop. This is a subset of the set of STRIDED_STORES | |
233 | of the loop. */ | |
234 | VEC(slp_instance, heap) *slp_instances; | |
235 | ||
236 | /* The unrolling factor needed to SLP the loop. In case of that pure SLP is | |
237 | applied to the loop, i.e., no unrolling is needed, this is 1. */ | |
238 | unsigned slp_unrolling_factor; | |
4e58562d | 239 | } *loop_vec_info; |
240 | ||
25e3c2e8 | 241 | /* Access Functions. */ |
516849c7 | 242 | #define LOOP_VINFO_LOOP(L) (L)->loop |
243 | #define LOOP_VINFO_BBS(L) (L)->bbs | |
516849c7 | 244 | #define LOOP_VINFO_NITERS(L) (L)->num_iters |
82ba6513 | 245 | /* Since LOOP_VINFO_NITERS can change after prologue peeling |
246 | retain total unchanged scalar loop iterations for cost model. */ | |
be53c6d4 | 247 | #define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged |
62612fe9 | 248 | #define LOOP_VINFO_COST_MODEL_MIN_ITERS(L) (L)->min_profitable_iters |
516849c7 | 249 | #define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable |
250 | #define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor | |
25e3c2e8 | 251 | #define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask |
516849c7 | 252 | #define LOOP_VINFO_DATAREFS(L) (L)->datarefs |
253 | #define LOOP_VINFO_DDRS(L) (L)->ddrs | |
254 | #define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters)) | |
39b8f742 | 255 | #define LOOP_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment |
516849c7 | 256 | #define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr |
25e3c2e8 | 257 | #define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts |
516849c7 | 258 | #define LOOP_VINFO_LOC(L) (L)->loop_line_number |
45b13dc3 | 259 | #define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs |
c6895939 | 260 | #define LOOP_VINFO_STRIDED_STORES(L) (L)->strided_stores |
261 | #define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances | |
262 | #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor | |
4e58562d | 263 | |
33bbe730 | 264 | #define NITERS_KNOWN_P(n) \ |
265 | (host_integerp ((n),0) \ | |
266 | && TREE_INT_CST_LOW ((n)) > 0) | |
267 | ||
4e58562d | 268 | #define LOOP_VINFO_NITERS_KNOWN_P(L) \ |
33bbe730 | 269 | NITERS_KNOWN_P((L)->num_iters) |
4e58562d | 270 | |
221e9a92 | 271 | static inline loop_vec_info |
272 | loop_vec_info_for_loop (struct loop *loop) | |
273 | { | |
274 | return (loop_vec_info) loop->aux; | |
275 | } | |
276 | ||
277 | static inline bool | |
75a70cf9 | 278 | nested_in_vect_loop_p (struct loop *loop, gimple stmt) |
221e9a92 | 279 | { |
280 | return (loop->inner | |
75a70cf9 | 281 | && (loop->inner == (gimple_bb (stmt))->loop_father)); |
221e9a92 | 282 | } |
283 | ||
c91e8223 | 284 | /*-----------------------------------------------------------------*/ |
285 | /* Info on vectorized defs. */ | |
286 | /*-----------------------------------------------------------------*/ | |
287 | enum stmt_vec_info_type { | |
288 | undef_vec_info_type = 0, | |
289 | load_vec_info_type, | |
290 | store_vec_info_type, | |
291 | op_vec_info_type, | |
22c2f6bd | 292 | call_vec_info_type, |
e9705e7f | 293 | assignment_vec_info_type, |
ea8f3370 | 294 | condition_vec_info_type, |
c6c91d61 | 295 | reduc_vec_info_type, |
6fada017 | 296 | induc_vec_info_type, |
c6c91d61 | 297 | type_promotion_vec_info_type, |
9d8bf4aa | 298 | type_demotion_vec_info_type, |
221e9a92 | 299 | type_conversion_vec_info_type, |
300 | loop_exit_ctrl_vec_info_type | |
c6c91d61 | 301 | }; |
302 | ||
303 | /* Indicates whether/how a variable is used in the loop. */ | |
304 | enum vect_relevant { | |
305 | vect_unused_in_loop = 0, | |
221e9a92 | 306 | vect_used_in_outer_by_reduction, |
307 | vect_used_in_outer, | |
bfe8bfe9 | 308 | |
309 | /* defs that feed computations that end up (only) in a reduction. These | |
310 | defs may be used by non-reduction stmts, but eventually, any | |
311 | computations/values that are affected by these defs are used to compute | |
312 | a reduction (i.e. don't get stored to memory, for example). We use this | |
313 | to identify computations that we can change the order in which they are | |
314 | computed. */ | |
c6c91d61 | 315 | vect_used_by_reduction, |
bfe8bfe9 | 316 | |
c6c91d61 | 317 | vect_used_in_loop |
c91e8223 | 318 | }; |
319 | ||
c6895939 | 320 | /* The type of vectorization that can be applied to the stmt: regular loop-based |
321 | vectorization; pure SLP - the stmt is a part of SLP instances and does not | |
322 | have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is | |
323 | a part of SLP instance and also must be loop-based vectorized, since it has | |
324 | uses outside SLP sequences. | |
325 | ||
326 | In the loop context the meanings of pure and hybrid SLP are slightly | |
327 | different. By saying that pure SLP is applied to the loop, we mean that we | |
328 | exploit only intra-iteration parallelism in the loop; i.e., the loop can be | |
329 | vectorized without doing any conceptual unrolling, cause we don't pack | |
330 | together stmts from different iterations, only within a single iteration. | |
331 | Loop hybrid SLP means that we exploit both intra-iteration and | |
332 | inter-iteration parallelism (e.g., number of elements in the vector is 4 | |
333 | and the slp-group-size is 2, in which case we don't have enough parallelism | |
334 | within an iteration, so we obtain the rest of the parallelism from subsequent | |
335 | iterations by unrolling the loop by 2). */ | |
336 | enum slp_vect_type { | |
337 | loop_vect = 0, | |
338 | pure_slp, | |
339 | hybrid | |
340 | }; | |
341 | ||
342 | ||
f1168a33 | 343 | typedef struct data_reference *dr_p; |
344 | DEF_VEC_P(dr_p); | |
345 | DEF_VEC_ALLOC_P(dr_p,heap); | |
346 | ||
c91e8223 | 347 | typedef struct _stmt_vec_info { |
348 | ||
349 | enum stmt_vec_info_type type; | |
350 | ||
351 | /* The stmt to which this info struct refers to. */ | |
75a70cf9 | 352 | gimple stmt; |
c91e8223 | 353 | |
4e58562d | 354 | /* The loop_vec_info with respect to which STMT is vectorized. */ |
355 | loop_vec_info loop_vinfo; | |
c91e8223 | 356 | |
c6c91d61 | 357 | /* Not all stmts in the loop need to be vectorized. e.g, the increment |
c91e8223 | 358 | of the loop induction variable and computation of array indexes. relevant |
359 | indicates whether the stmt needs to be vectorized. */ | |
c6c91d61 | 360 | enum vect_relevant relevant; |
c91e8223 | 361 | |
e12906b9 | 362 | /* Indicates whether this stmts is part of a computation whose result is |
363 | used outside the loop. */ | |
364 | bool live; | |
365 | ||
c91e8223 | 366 | /* The vector type to be used. */ |
367 | tree vectype; | |
368 | ||
369 | /* The vectorized version of the stmt. */ | |
75a70cf9 | 370 | gimple vectorized_stmt; |
c91e8223 | 371 | |
372 | ||
373 | /** The following is relevant only for stmts that contain a non-scalar | |
374 | data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have | |
375 | at most one such data-ref. **/ | |
376 | ||
b0eb8c66 | 377 | /* Information about the data-ref (access function, etc), |
378 | relative to the inner-most containing loop. */ | |
c91e8223 | 379 | struct data_reference *data_ref_info; |
380 | ||
b0eb8c66 | 381 | /* Information about the data-ref relative to this loop |
382 | nest (the loop that is being considered for vectorization). */ | |
383 | tree dr_base_address; | |
384 | tree dr_init; | |
385 | tree dr_offset; | |
386 | tree dr_step; | |
387 | tree dr_aligned_to; | |
388 | ||
4a61a337 | 389 | /* Stmt is part of some pattern (computation idiom) */ |
390 | bool in_pattern_p; | |
391 | ||
334ec2d8 | 392 | /* Used for various bookkeeping purposes, generally holding a pointer to |
4a61a337 | 393 | some other stmt S that is in some way "related" to this stmt. |
394 | Current use of this field is: | |
395 | If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is | |
396 | true): S is the "pattern stmt" that represents (and replaces) the | |
397 | sequence of stmts that constitutes the pattern. Similarly, the | |
398 | related_stmt of the "pattern stmt" points back to this stmt (which is | |
399 | the last stmt in the original sequence of stmts that constitutes the | |
400 | pattern). */ | |
75a70cf9 | 401 | gimple related_stmt; |
4a61a337 | 402 | |
f1168a33 | 403 | /* List of datarefs that are known to have the same alignment as the dataref |
404 | of this stmt. */ | |
405 | VEC(dr_p,heap) *same_align_refs; | |
406 | ||
e12906b9 | 407 | /* Classify the def of this stmt. */ |
408 | enum vect_def_type def_type; | |
409 | ||
6b8dbb53 | 410 | /* Interleaving info. */ |
411 | /* First data-ref in the interleaving group. */ | |
75a70cf9 | 412 | gimple first_dr; |
6b8dbb53 | 413 | /* Pointer to the next data-ref in the group. */ |
75a70cf9 | 414 | gimple next_dr; |
6b8dbb53 | 415 | /* The size of the interleaving group. */ |
416 | unsigned int size; | |
417 | /* For stores, number of stores from this group seen. We vectorize the last | |
418 | one. */ | |
419 | unsigned int store_count; | |
420 | /* For loads only, the gap from the previous load. For consecutive loads, GAP | |
421 | is 1. */ | |
422 | unsigned int gap; | |
423 | /* In case that two or more stmts share data-ref, this is the pointer to the | |
424 | previously detected stmt with the same dr. */ | |
75a70cf9 | 425 | gimple same_dr_stmt; |
e51f5136 | 426 | /* For loads only, if there is a store with the same location, this field is |
427 | TRUE. */ | |
428 | bool read_write_dep; | |
867c03eb | 429 | |
430 | /* Vectorization costs associated with statement. */ | |
431 | struct | |
432 | { | |
433 | int outside_of_loop; /* Statements generated outside loop. */ | |
434 | int inside_of_loop; /* Statements generated inside loop. */ | |
435 | } cost; | |
c6895939 | 436 | |
437 | /* Whether the stmt is SLPed, loop-based vectorized, or both. */ | |
438 | enum slp_vect_type slp_type; | |
c91e8223 | 439 | } *stmt_vec_info; |
440 | ||
441 | /* Access Functions. */ | |
6b8dbb53 | 442 | #define STMT_VINFO_TYPE(S) (S)->type |
443 | #define STMT_VINFO_STMT(S) (S)->stmt | |
444 | #define STMT_VINFO_LOOP_VINFO(S) (S)->loop_vinfo | |
445 | #define STMT_VINFO_RELEVANT(S) (S)->relevant | |
446 | #define STMT_VINFO_LIVE_P(S) (S)->live | |
447 | #define STMT_VINFO_VECTYPE(S) (S)->vectype | |
448 | #define STMT_VINFO_VEC_STMT(S) (S)->vectorized_stmt | |
449 | #define STMT_VINFO_DATA_REF(S) (S)->data_ref_info | |
b0eb8c66 | 450 | |
451 | #define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_base_address | |
452 | #define STMT_VINFO_DR_INIT(S) (S)->dr_init | |
453 | #define STMT_VINFO_DR_OFFSET(S) (S)->dr_offset | |
454 | #define STMT_VINFO_DR_STEP(S) (S)->dr_step | |
455 | #define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_aligned_to | |
456 | ||
6b8dbb53 | 457 | #define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p |
458 | #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt | |
459 | #define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs | |
460 | #define STMT_VINFO_DEF_TYPE(S) (S)->def_type | |
461 | #define STMT_VINFO_DR_GROUP_FIRST_DR(S) (S)->first_dr | |
462 | #define STMT_VINFO_DR_GROUP_NEXT_DR(S) (S)->next_dr | |
463 | #define STMT_VINFO_DR_GROUP_SIZE(S) (S)->size | |
464 | #define STMT_VINFO_DR_GROUP_STORE_COUNT(S) (S)->store_count | |
465 | #define STMT_VINFO_DR_GROUP_GAP(S) (S)->gap | |
466 | #define STMT_VINFO_DR_GROUP_SAME_DR_STMT(S)(S)->same_dr_stmt | |
e51f5136 | 467 | #define STMT_VINFO_DR_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep |
c6895939 | 468 | #define STMT_VINFO_STRIDED_ACCESS(S) ((S)->first_dr != NULL) |
6b8dbb53 | 469 | |
470 | #define DR_GROUP_FIRST_DR(S) (S)->first_dr | |
471 | #define DR_GROUP_NEXT_DR(S) (S)->next_dr | |
472 | #define DR_GROUP_SIZE(S) (S)->size | |
473 | #define DR_GROUP_STORE_COUNT(S) (S)->store_count | |
474 | #define DR_GROUP_GAP(S) (S)->gap | |
475 | #define DR_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt | |
e51f5136 | 476 | #define DR_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep |
c91e8223 | 477 | |
c6c91d61 | 478 | #define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_loop) |
867c03eb | 479 | #define STMT_VINFO_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop |
480 | #define STMT_VINFO_INSIDE_OF_LOOP_COST(S) (S)->cost.inside_of_loop | |
481 | ||
c6895939 | 482 | #define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid) |
483 | #define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp) | |
484 | #define STMT_SLP_TYPE(S) (S)->slp_type | |
485 | ||
867c03eb | 486 | /* These are some defines for the initial implementation of the vectorizer's |
487 | cost model. These will later be target specific hooks. */ | |
488 | ||
6202d4db | 489 | /* Cost of conditional taken branch. */ |
490 | #ifndef TARG_COND_TAKEN_BRANCH_COST | |
491 | #define TARG_COND_TAKEN_BRANCH_COST 3 | |
492 | #endif | |
493 | ||
494 | /* Cost of conditional not taken branch. */ | |
495 | #ifndef TARG_COND_NOT_TAKEN_BRANCH_COST | |
496 | #define TARG_COND_NOT_TAKEN_BRANCH_COST 1 | |
867c03eb | 497 | #endif |
498 | ||
a28df51d | 499 | /* Cost of any scalar operation, excluding load and store. */ |
500 | #ifndef TARG_SCALAR_STMT_COST | |
501 | #define TARG_SCALAR_STMT_COST 1 | |
502 | #endif | |
503 | ||
504 | /* Cost of scalar load. */ | |
505 | #ifndef TARG_SCALAR_LOAD_COST | |
506 | #define TARG_SCALAR_LOAD_COST 1 | |
507 | #endif | |
508 | ||
509 | /* Cost of scalar store. */ | |
510 | #ifndef TARG_SCALAR_STORE_COST | |
511 | #define TARG_SCALAR_STORE_COST 1 | |
512 | #endif | |
513 | ||
867c03eb | 514 | /* Cost of any vector operation, excluding load, store or vector to scalar |
515 | operation. */ | |
516 | #ifndef TARG_VEC_STMT_COST | |
517 | #define TARG_VEC_STMT_COST 1 | |
518 | #endif | |
519 | ||
520 | /* Cost of vector to scalar operation. */ | |
521 | #ifndef TARG_VEC_TO_SCALAR_COST | |
522 | #define TARG_VEC_TO_SCALAR_COST 1 | |
523 | #endif | |
524 | ||
62612fe9 | 525 | /* Cost of scalar to vector operation. */ |
526 | #ifndef TARG_SCALAR_TO_VEC_COST | |
527 | #define TARG_SCALAR_TO_VEC_COST 1 | |
528 | #endif | |
529 | ||
867c03eb | 530 | /* Cost of aligned vector load. */ |
531 | #ifndef TARG_VEC_LOAD_COST | |
532 | #define TARG_VEC_LOAD_COST 1 | |
533 | #endif | |
534 | ||
535 | /* Cost of misaligned vector load. */ | |
536 | #ifndef TARG_VEC_UNALIGNED_LOAD_COST | |
537 | #define TARG_VEC_UNALIGNED_LOAD_COST 2 | |
538 | #endif | |
539 | ||
540 | /* Cost of vector store. */ | |
541 | #ifndef TARG_VEC_STORE_COST | |
542 | #define TARG_VEC_STORE_COST 1 | |
543 | #endif | |
c6c91d61 | 544 | |
a0515226 | 545 | /* Cost of vector permutation. */ |
546 | #ifndef TARG_VEC_PERMUTE_COST | |
547 | #define TARG_VEC_PERMUTE_COST 1 | |
548 | #endif | |
549 | ||
862bb3cd | 550 | /* The maximum number of intermediate steps required in multi-step type |
551 | conversion. */ | |
552 | #define MAX_INTERM_CVT_STEPS 3 | |
553 | ||
75a70cf9 | 554 | /* Avoid GTY(()) on stmt_vec_info. */ |
555 | typedef void *vec_void_p; | |
556 | DEF_VEC_P (vec_void_p); | |
557 | DEF_VEC_ALLOC_P (vec_void_p, heap); | |
c91e8223 | 558 | |
75a70cf9 | 559 | extern VEC(vec_void_p,heap) *stmt_vec_info_vec; |
560 | ||
561 | void init_stmt_vec_info_vec (void); | |
562 | void free_stmt_vec_info_vec (void); | |
563 | ||
564 | static inline stmt_vec_info | |
565 | vinfo_for_stmt (gimple stmt) | |
c91e8223 | 566 | { |
75a70cf9 | 567 | unsigned int uid = gimple_uid (stmt); |
568 | if (uid == 0) | |
569 | return NULL; | |
570 | ||
571 | gcc_assert (uid <= VEC_length (vec_void_p, stmt_vec_info_vec)); | |
572 | return (stmt_vec_info) VEC_index (vec_void_p, stmt_vec_info_vec, uid - 1); | |
c91e8223 | 573 | } |
574 | ||
75a70cf9 | 575 | static inline void |
576 | set_vinfo_for_stmt (gimple stmt, stmt_vec_info info) | |
c91e8223 | 577 | { |
75a70cf9 | 578 | unsigned int uid = gimple_uid (stmt); |
579 | if (uid == 0) | |
580 | { | |
581 | gcc_assert (info); | |
582 | uid = VEC_length (vec_void_p, stmt_vec_info_vec) + 1; | |
583 | gimple_set_uid (stmt, uid); | |
584 | VEC_safe_push (vec_void_p, heap, stmt_vec_info_vec, (vec_void_p) info); | |
585 | } | |
586 | else | |
587 | VEC_replace (vec_void_p, stmt_vec_info_vec, uid - 1, (vec_void_p) info); | |
c91e8223 | 588 | } |
589 | ||
bdc89b8f | 590 | static inline gimple |
591 | get_earlier_stmt (gimple stmt1, gimple stmt2) | |
592 | { | |
593 | unsigned int uid1, uid2; | |
594 | ||
595 | if (stmt1 == NULL) | |
596 | return stmt2; | |
597 | ||
598 | if (stmt2 == NULL) | |
599 | return stmt1; | |
600 | ||
601 | uid1 = gimple_uid (stmt1); | |
602 | uid2 = gimple_uid (stmt2); | |
603 | ||
604 | if (uid1 == 0 || uid2 == 0) | |
605 | return NULL; | |
606 | ||
607 | gcc_assert (uid1 <= VEC_length (vec_void_p, stmt_vec_info_vec)); | |
608 | gcc_assert (uid2 <= VEC_length (vec_void_p, stmt_vec_info_vec)); | |
609 | ||
610 | if (uid1 < uid2) | |
611 | return stmt1; | |
612 | else | |
613 | return stmt2; | |
614 | } | |
615 | ||
213448e9 | 616 | static inline bool |
617 | is_pattern_stmt_p (stmt_vec_info stmt_info) | |
618 | { | |
75a70cf9 | 619 | gimple related_stmt; |
213448e9 | 620 | stmt_vec_info related_stmt_info; |
621 | ||
622 | related_stmt = STMT_VINFO_RELATED_STMT (stmt_info); | |
623 | if (related_stmt | |
624 | && (related_stmt_info = vinfo_for_stmt (related_stmt)) | |
625 | && STMT_VINFO_IN_PATTERN_P (related_stmt_info)) | |
626 | return true; | |
627 | ||
628 | return false; | |
629 | } | |
630 | ||
221e9a92 | 631 | static inline bool |
632 | is_loop_header_bb_p (basic_block bb) | |
633 | { | |
634 | if (bb == (bb->loop_father)->header) | |
635 | return true; | |
636 | gcc_assert (EDGE_COUNT (bb->preds) == 1); | |
637 | return false; | |
638 | } | |
639 | ||
84a15e8f | 640 | static inline void |
641 | stmt_vinfo_set_inside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node, | |
642 | int cost) | |
643 | { | |
644 | if (slp_node) | |
645 | SLP_TREE_INSIDE_OF_LOOP_COST (slp_node) = cost; | |
646 | else | |
647 | STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) = cost; | |
648 | } | |
649 | ||
650 | static inline void | |
651 | stmt_vinfo_set_outside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node, | |
652 | int cost) | |
653 | { | |
654 | if (slp_node) | |
655 | SLP_TREE_OUTSIDE_OF_LOOP_COST (slp_node) = cost; | |
656 | else | |
657 | STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = cost; | |
658 | } | |
659 | ||
862bb3cd | 660 | static inline int |
661 | vect_pow2 (int x) | |
662 | { | |
663 | int i, res = 1; | |
664 | ||
665 | for (i = 0; i < x; i++) | |
666 | res *= 2; | |
667 | ||
668 | return res; | |
669 | } | |
84a15e8f | 670 | |
c91e8223 | 671 | /*-----------------------------------------------------------------*/ |
672 | /* Info on data references alignment. */ | |
673 | /*-----------------------------------------------------------------*/ | |
674 | ||
39b8f742 | 675 | /* Reflects actual alignment of first access in the vectorized loop, |
676 | taking into account peeling/versioning if applied. */ | |
5c205353 | 677 | #define DR_MISALIGNMENT(DR) ((int) (size_t) (DR)->aux) |
678 | #define SET_DR_MISALIGNMENT(DR, VAL) ((DR)->aux = (void *) (size_t) (VAL)) | |
c91e8223 | 679 | |
680 | static inline bool | |
681 | aligned_access_p (struct data_reference *data_ref_info) | |
682 | { | |
683 | return (DR_MISALIGNMENT (data_ref_info) == 0); | |
684 | } | |
685 | ||
686 | static inline bool | |
39b8f742 | 687 | known_alignment_for_access_p (struct data_reference *data_ref_info) |
c91e8223 | 688 | { |
39b8f742 | 689 | return (DR_MISALIGNMENT (data_ref_info) != -1); |
c91e8223 | 690 | } |
691 | ||
f2983e95 | 692 | /* vect_dump will be set to stderr or dump_file if exist. */ |
693 | extern FILE *vect_dump; | |
fb85abff | 694 | extern LOC vect_loop_location; |
695 | ||
f2983e95 | 696 | extern enum verbosity_levels vect_verbosity_level; |
c91e8223 | 697 | |
1b1e5627 | 698 | /* Bitmap of virtual variables to be renamed. */ |
4fb5e5ca | 699 | extern bitmap vect_memsyms_to_rename; |
1b1e5627 | 700 | |
fb85abff | 701 | |
c91e8223 | 702 | /*-----------------------------------------------------------------*/ |
703 | /* Function prototypes. */ | |
704 | /*-----------------------------------------------------------------*/ | |
705 | ||
fb85abff | 706 | /* Simple loop peeling and versioning utilities for vectorizer's purposes - |
707 | in tree-vect-loop-manip.c. */ | |
f2983e95 | 708 | extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree); |
1f1872fd | 709 | extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge); |
23a3430d | 710 | extern void vect_loop_versioning (loop_vec_info, bool, tree *, gimple_seq *); |
711 | extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree *, | |
712 | tree, gimple_seq); | |
fb85abff | 713 | extern void vect_do_peeling_for_alignment (loop_vec_info); |
714 | extern LOC find_loop_location (struct loop *); | |
715 | extern bool vect_can_advance_ivs_p (loop_vec_info); | |
c91e8223 | 716 | |
fb85abff | 717 | /* In tree-vect-stmts.c. */ |
f2983e95 | 718 | extern tree get_vectype_for_scalar_type (tree); |
75a70cf9 | 719 | extern bool vect_is_simple_use (tree, loop_vec_info, gimple *, tree *, |
e12906b9 | 720 | enum vect_def_type *); |
75a70cf9 | 721 | extern bool supportable_widening_operation (enum tree_code, gimple, tree, |
fb85abff | 722 | tree *, tree *, enum tree_code *, |
723 | enum tree_code *, int *, | |
724 | VEC (tree, heap) **); | |
75a70cf9 | 725 | extern bool supportable_narrowing_operation (enum tree_code, const_gimple, |
fb85abff | 726 | tree, enum tree_code *, int *, |
727 | VEC (tree, heap) **); | |
75a70cf9 | 728 | extern stmt_vec_info new_stmt_vec_info (gimple stmt, loop_vec_info); |
729 | extern void free_stmt_vec_info (gimple stmt); | |
fb85abff | 730 | extern tree vectorizable_function (gimple, tree, tree); |
731 | extern void vect_model_simple_cost (stmt_vec_info, int, enum vect_def_type *, | |
732 | slp_tree); | |
733 | extern void vect_model_store_cost (stmt_vec_info, int, enum vect_def_type, | |
734 | slp_tree); | |
735 | extern void vect_model_load_cost (stmt_vec_info, int, slp_tree); | |
736 | extern void vect_finish_stmt_generation (gimple, gimple, | |
737 | gimple_stmt_iterator *); | |
738 | extern bool vect_mark_stmts_to_be_vectorized (loop_vec_info); | |
739 | extern int cost_for_stmt (gimple); | |
740 | extern tree vect_get_vec_def_for_operand (tree, gimple, tree *); | |
741 | extern tree vect_init_vector (gimple, tree, tree, | |
742 | gimple_stmt_iterator *); | |
743 | extern tree vect_get_vec_def_for_stmt_copy (enum vect_def_type, tree); | |
744 | extern bool vect_transform_stmt (gimple, gimple_stmt_iterator *, | |
745 | bool *, slp_tree, slp_instance); | |
746 | extern void vect_remove_stores (gimple); | |
747 | extern bool vect_analyze_operations (loop_vec_info); | |
748 | ||
749 | /* In tree-vect-data-refs.c. */ | |
750 | extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int); | |
751 | extern enum dr_alignment_support vect_supportable_dr_alignment | |
752 | (struct data_reference *); | |
753 | extern tree vect_get_smallest_scalar_type (gimple, HOST_WIDE_INT *, | |
754 | HOST_WIDE_INT *); | |
755 | extern bool vect_analyze_data_ref_dependences (loop_vec_info); | |
756 | extern bool vect_enhance_data_refs_alignment (loop_vec_info); | |
757 | extern bool vect_analyze_data_refs_alignment (loop_vec_info); | |
758 | extern bool vect_analyze_data_ref_accesses (loop_vec_info); | |
759 | extern bool vect_prune_runtime_alias_test_list (loop_vec_info); | |
760 | extern bool vect_analyze_data_refs (loop_vec_info); | |
761 | extern tree vect_create_data_ref_ptr (gimple, struct loop *, tree, tree *, | |
dd277d48 | 762 | gimple *, bool, bool *); |
fb85abff | 763 | extern tree bump_vector_ptr (tree, gimple, gimple_stmt_iterator *, gimple, tree); |
764 | extern tree vect_create_destination_var (tree, tree); | |
765 | extern bool vect_strided_store_supported (tree); | |
766 | extern bool vect_strided_load_supported (tree); | |
767 | extern bool vect_permute_store_chain (VEC(tree,heap) *,unsigned int, gimple, | |
768 | gimple_stmt_iterator *, VEC(tree,heap) **); | |
769 | extern tree vect_setup_realignment (gimple, gimple_stmt_iterator *, tree *, | |
770 | enum dr_alignment_support, tree, | |
771 | struct loop **); | |
772 | extern bool vect_permute_load_chain (VEC(tree,heap) *,unsigned int, gimple, | |
773 | gimple_stmt_iterator *, VEC(tree,heap) **); | |
774 | extern bool vect_transform_strided_load (gimple, VEC(tree,heap) *, int, | |
775 | gimple_stmt_iterator *); | |
776 | extern int vect_get_place_in_interleaving_chain (gimple, gimple); | |
777 | extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *); | |
778 | extern tree vect_create_addr_base_for_vector_ref (gimple, gimple_seq *, | |
779 | tree, struct loop *); | |
780 | ||
781 | /* In tree-vect-loop.c. */ | |
782 | /* FORNOW: Used in tree-parloops.c. */ | |
783 | extern void destroy_loop_vec_info (loop_vec_info, bool); | |
784 | extern gimple vect_is_simple_reduction (loop_vec_info, gimple); | |
785 | /* Drive for loop analysis stage. */ | |
f2983e95 | 786 | extern loop_vec_info vect_analyze_loop (struct loop *); |
fb85abff | 787 | /* Drive for loop transformation stage. */ |
788 | extern void vect_transform_loop (loop_vec_info); | |
cb7f680b | 789 | extern loop_vec_info vect_analyze_loop_form (struct loop *); |
fb85abff | 790 | extern bool vectorizable_live_operation (gimple, gimple_stmt_iterator *, |
791 | gimple *); | |
792 | extern bool vectorizable_reduction (gimple, gimple_stmt_iterator *, gimple *); | |
793 | extern bool vectorizable_induction (gimple, gimple_stmt_iterator *, gimple *); | |
794 | extern int vect_estimate_min_profitable_iters (loop_vec_info); | |
795 | extern tree get_initial_def_for_reduction (gimple, tree, tree *); | |
796 | extern int vect_min_worthwhile_factor (enum tree_code); | |
797 | ||
4a61a337 | 798 | |
fb85abff | 799 | /* In tree-vect-slp.c. */ |
800 | extern void vect_free_slp_instance (slp_instance); | |
801 | extern bool vect_transform_slp_perm_load (gimple, VEC (tree, heap) *, | |
802 | gimple_stmt_iterator *, int, | |
803 | slp_instance, bool); | |
804 | extern bool vect_schedule_slp (loop_vec_info); | |
805 | extern void vect_update_slp_costs_according_to_vf (loop_vec_info); | |
806 | extern bool vect_analyze_slp (loop_vec_info); | |
807 | extern void vect_make_slp_decision (loop_vec_info); | |
808 | extern void vect_detect_hybrid_slp (loop_vec_info); | |
809 | extern void vect_get_slp_defs (slp_tree, VEC (tree,heap) **, | |
810 | VEC (tree,heap) **); | |
811 | ||
812 | /* In tree-vect-patterns.c. */ | |
4a61a337 | 813 | /* Pattern recognition functions. |
814 | Additional pattern recognition functions can (and will) be added | |
815 | in the future. */ | |
75a70cf9 | 816 | typedef gimple (* vect_recog_func_ptr) (gimple, tree *, tree *); |
c37bea13 | 817 | #define NUM_PATTERNS 4 |
4a61a337 | 818 | void vect_pattern_recog (loop_vec_info); |
819 | ||
fb85abff | 820 | /* Vectorization debug information - in tree-vectorizer.c. */ |
e6e4c039 | 821 | extern bool vect_print_dump_info (enum verbosity_levels); |
f2983e95 | 822 | extern void vect_set_verbosity_level (const char *); |
c91e8223 | 823 | |
824 | #endif /* GCC_TREE_VECTORIZER_H */ |