]>
Commit | Line | Data |
---|---|---|
fb85abff | 1 | /* Vectorizer |
8e8f6434 | 2 | Copyright (C) 2003-2018 Free Software Foundation, Inc. |
c91e8223 | 3 | Contributed by Dorit Naishlos <dorit@il.ibm.com> |
4 | ||
5 | This file is part of GCC. | |
6 | ||
7 | GCC is free software; you can redistribute it and/or modify it under | |
8 | the terms of the GNU General Public License as published by the Free | |
8c4c00c1 | 9 | Software Foundation; either version 3, or (at your option) any later |
c91e8223 | 10 | version. |
11 | ||
12 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
13 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
14 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
15 | for more details. | |
16 | ||
17 | You should have received a copy of the GNU General Public License | |
8c4c00c1 | 18 | along with GCC; see the file COPYING3. If not see |
19 | <http://www.gnu.org/licenses/>. */ | |
c91e8223 | 20 | |
21 | #ifndef GCC_TREE_VECTORIZER_H | |
22 | #define GCC_TREE_VECTORIZER_H | |
23 | ||
fb85abff | 24 | #include "tree-data-ref.h" |
4f372c2c | 25 | #include "tree-hash-traits.h" |
f4ac3f3e | 26 | #include "target.h" |
fb85abff | 27 | |
c91e8223 | 28 | /* Used for naming of new temporaries. */ |
29 | enum vect_var_kind { | |
30 | vect_simple_var, | |
ea8f3370 | 31 | vect_pointer_var, |
dab48979 | 32 | vect_scalar_var, |
33 | vect_mask_var | |
c91e8223 | 34 | }; |
35 | ||
4a61a337 | 36 | /* Defines type of operation. */ |
c91e8223 | 37 | enum operation_type { |
38 | unary_op = 1, | |
4a61a337 | 39 | binary_op, |
40 | ternary_op | |
c91e8223 | 41 | }; |
42 | ||
1a9b4618 | 43 | /* Define type of available alignment support. */ |
44 | enum dr_alignment_support { | |
45 | dr_unaligned_unsupported, | |
46 | dr_unaligned_supported, | |
b0eb8c66 | 47 | dr_explicit_realign, |
48 | dr_explicit_realign_optimized, | |
1a9b4618 | 49 | dr_aligned |
50 | }; | |
51 | ||
ce10738f | 52 | /* Define type of def-use cross-iteration cycle. */ |
e12906b9 | 53 | enum vect_def_type { |
bc620c5c | 54 | vect_uninitialized_def = 0, |
f083cd24 | 55 | vect_constant_def = 1, |
56 | vect_external_def, | |
57 | vect_internal_def, | |
e12906b9 | 58 | vect_induction_def, |
59 | vect_reduction_def, | |
7aa0d350 | 60 | vect_double_reduction_def, |
ade2ac53 | 61 | vect_nested_cycle, |
e12906b9 | 62 | vect_unknown_def_type |
63 | }; | |
64 | ||
d09d8733 | 65 | /* Define type of reduction. */ |
66 | enum vect_reduction_type { | |
67 | TREE_CODE_REDUCTION, | |
b4552064 | 68 | COND_REDUCTION, |
56fb8e9d | 69 | INTEGER_INDUC_COND_REDUCTION, |
70 | CONST_COND_REDUCTION | |
d09d8733 | 71 | }; |
72 | ||
07be02da | 73 | #define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \ |
74 | || ((D) == vect_double_reduction_def) \ | |
75 | || ((D) == vect_nested_cycle)) | |
76 | ||
4db2b577 | 77 | /* Structure to encapsulate information about a group of like |
78 | instructions to be presented to the target cost model. */ | |
6dc50383 | 79 | struct stmt_info_for_cost { |
4db2b577 | 80 | int count; |
81 | enum vect_cost_for_stmt kind; | |
42acab1c | 82 | gimple *stmt; |
4db2b577 | 83 | int misalign; |
6dc50383 | 84 | }; |
4db2b577 | 85 | |
f1f41a6c | 86 | typedef vec<stmt_info_for_cost> stmt_vector_for_cost; |
4db2b577 | 87 | |
4f372c2c | 88 | /* Maps base addresses to an innermost_loop_behavior that gives the maximum |
89 | known alignment for that base. */ | |
90 | typedef hash_map<tree_operand_hash, | |
91 | innermost_loop_behavior *> vec_base_alignments; | |
92 | ||
c6895939 | 93 | /************************************************************************ |
94 | SLP | |
95 | ************************************************************************/ | |
40bcc7c2 | 96 | typedef struct _slp_tree *slp_tree; |
c6895939 | 97 | |
b0f64919 | 98 | /* A computation tree of an SLP instance. Each node corresponds to a group of |
c6895939 | 99 | stmts to be packed in a SIMD stmt. */ |
40bcc7c2 | 100 | struct _slp_tree { |
b0f64919 | 101 | /* Nodes that contain def-stmts of this node statements operands. */ |
40bcc7c2 | 102 | vec<slp_tree> children; |
c6895939 | 103 | /* A group of scalar stmts to be vectorized together. */ |
42acab1c | 104 | vec<gimple *> stmts; |
678e3d6e | 105 | /* Load permutation relative to the stores, NULL if there is no |
106 | permutation. */ | |
107 | vec<unsigned> load_permutation; | |
c6895939 | 108 | /* Vectorized stmt/s. */ |
42acab1c | 109 | vec<gimple *> vec_stmts; |
48e1416a | 110 | /* Number of vector stmts that are created to replace the group of scalar |
111 | stmts. It is calculated during the transformation phase as the number of | |
112 | scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF | |
c6895939 | 113 | divided by vector size. */ |
114 | unsigned int vec_stmts_size; | |
66e30248 | 115 | /* Whether the scalar computations use two different operators. */ |
116 | bool two_operators; | |
6d37c111 | 117 | /* The DEF type of this node. */ |
118 | enum vect_def_type def_type; | |
40bcc7c2 | 119 | }; |
c6895939 | 120 | |
121 | ||
122 | /* SLP instance is a sequence of stmts in a loop that can be packed into | |
123 | SIMD stmts. */ | |
124 | typedef struct _slp_instance { | |
125 | /* The root of SLP tree. */ | |
126 | slp_tree root; | |
127 | ||
128 | /* Size of groups of scalar stmts that will be replaced by SIMD stmt/s. */ | |
129 | unsigned int group_size; | |
130 | ||
131 | /* The unrolling factor required to vectorized this SLP instance. */ | |
d75596cd | 132 | poly_uint64 unrolling_factor; |
c6895939 | 133 | |
a0515226 | 134 | /* The group of nodes that contain loads of this SLP instance. */ |
f1f41a6c | 135 | vec<slp_tree> loads; |
6154acba | 136 | |
137 | /* The SLP node containing the reduction PHIs. */ | |
138 | slp_tree reduc_phis; | |
c6895939 | 139 | } *slp_instance; |
140 | ||
c6895939 | 141 | |
142 | /* Access Functions. */ | |
143 | #define SLP_INSTANCE_TREE(S) (S)->root | |
144 | #define SLP_INSTANCE_GROUP_SIZE(S) (S)->group_size | |
145 | #define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor | |
a0515226 | 146 | #define SLP_INSTANCE_LOADS(S) (S)->loads |
c6895939 | 147 | |
b0f64919 | 148 | #define SLP_TREE_CHILDREN(S) (S)->children |
c6895939 | 149 | #define SLP_TREE_SCALAR_STMTS(S) (S)->stmts |
150 | #define SLP_TREE_VEC_STMTS(S) (S)->vec_stmts | |
151 | #define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size | |
678e3d6e | 152 | #define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation |
66e30248 | 153 | #define SLP_TREE_TWO_OPERATORS(S) (S)->two_operators |
6d37c111 | 154 | #define SLP_TREE_DEF_TYPE(S) (S)->def_type |
b0f64919 | 155 | |
b0f64919 | 156 | |
0822b158 | 157 | |
f68a7726 | 158 | /* Describes two objects whose addresses must be unequal for the vectorized |
159 | loop to be valid. */ | |
160 | typedef std::pair<tree, tree> vec_object_pair; | |
161 | ||
e2c5c678 | 162 | /* Vectorizer state common between loop and basic-block vectorization. */ |
163 | struct vec_info { | |
e15e8a2a | 164 | enum vec_kind { bb, loop }; |
165 | ||
166 | vec_info (vec_kind, void *); | |
167 | ~vec_info (); | |
168 | ||
169 | /* The type of vectorization. */ | |
170 | vec_kind kind; | |
e2c5c678 | 171 | |
172 | /* All SLP instances. */ | |
e15e8a2a | 173 | auto_vec<slp_instance> slp_instances; |
e2c5c678 | 174 | |
e15e8a2a | 175 | /* All data references. Freed by free_data_refs, so not an auto_vec. */ |
e2c5c678 | 176 | vec<data_reference_p> datarefs; |
177 | ||
4f372c2c | 178 | /* Maps base addresses to an innermost_loop_behavior that gives the maximum |
179 | known alignment for that base. */ | |
180 | vec_base_alignments base_alignments; | |
181 | ||
e15e8a2a | 182 | /* All data dependences. Freed by free_dependence_relations, so not |
183 | an auto_vec. */ | |
e2c5c678 | 184 | vec<ddr_p> ddrs; |
185 | ||
186 | /* All interleaving chains of stores, represented by the first | |
187 | stmt in the chain. */ | |
e15e8a2a | 188 | auto_vec<gimple *> grouped_stores; |
e2c5c678 | 189 | |
190 | /* Cost data used by the target cost model. */ | |
191 | void *target_cost_data; | |
192 | }; | |
193 | ||
194 | struct _loop_vec_info; | |
195 | struct _bb_vec_info; | |
196 | ||
197 | template<> | |
198 | template<> | |
199 | inline bool | |
200 | is_a_helper <_loop_vec_info *>::test (vec_info *i) | |
201 | { | |
202 | return i->kind == vec_info::loop; | |
203 | } | |
204 | ||
205 | template<> | |
206 | template<> | |
207 | inline bool | |
208 | is_a_helper <_bb_vec_info *>::test (vec_info *i) | |
209 | { | |
210 | return i->kind == vec_info::bb; | |
211 | } | |
212 | ||
3e871d4d | 213 | |
4e58562d | 214 | /*-----------------------------------------------------------------*/ |
215 | /* Info on vectorized loops. */ | |
216 | /*-----------------------------------------------------------------*/ | |
e2c5c678 | 217 | typedef struct _loop_vec_info : public vec_info { |
e15e8a2a | 218 | _loop_vec_info (struct loop *); |
219 | ~_loop_vec_info (); | |
4e58562d | 220 | |
221 | /* The loop to which this info struct refers to. */ | |
222 | struct loop *loop; | |
223 | ||
224 | /* The loop basic blocks. */ | |
225 | basic_block *bbs; | |
226 | ||
796f6cba | 227 | /* Number of latch executions. */ |
228 | tree num_itersm1; | |
4e58562d | 229 | /* Number of iterations. */ |
230 | tree num_iters; | |
796f6cba | 231 | /* Number of iterations of the original loop. */ |
be53c6d4 | 232 | tree num_iters_unchanged; |
d5e80d93 | 233 | /* Condition under which this loop is analyzed and versioned. */ |
234 | tree num_iters_assumptions; | |
4e58562d | 235 | |
004a94a5 | 236 | /* Threshold of number of iterations below which vectorzation will not be |
237 | performed. It is calculated from MIN_PROFITABLE_ITERS and | |
238 | PARAM_MIN_VECT_LOOP_BOUND. */ | |
239 | unsigned int th; | |
240 | ||
7456a7ea | 241 | /* When applying loop versioning, the vector form should only be used |
242 | if the number of scalar iterations is >= this value, on top of all | |
243 | the other requirements. Ignored when loop versioning is not being | |
244 | used. */ | |
245 | poly_uint64 versioning_threshold; | |
246 | ||
4e58562d | 247 | /* Unrolling factor */ |
d75596cd | 248 | poly_uint64 vectorization_factor; |
4e58562d | 249 | |
4a85c0b1 | 250 | /* Maximum runtime vectorization factor, or MAX_VECTORIZATION_FACTOR |
251 | if there is no particular limit. */ | |
252 | unsigned HOST_WIDE_INT max_vectorization_factor; | |
253 | ||
4e58562d | 254 | /* Unknown DRs according to which loop was peeled. */ |
255 | struct data_reference *unaligned_dr; | |
256 | ||
39b8f742 | 257 | /* peeling_for_alignment indicates whether peeling for alignment will take |
258 | place, and what the peeling factor should be: | |
259 | peeling_for_alignment = X means: | |
260 | If X=0: Peeling for alignment will not be applied. | |
261 | If X>0: Peel first X iterations. | |
262 | If X=-1: Generate a runtime test to calculate the number of iterations | |
263 | to be peeled, using the dataref recorded in the field | |
264 | unaligned_dr. */ | |
265 | int peeling_for_alignment; | |
4e58562d | 266 | |
25e3c2e8 | 267 | /* The mask used to check the alignment of pointers or arrays. */ |
268 | int ptr_mask; | |
269 | ||
a8af2e86 | 270 | /* The loop nest in which the data dependences are computed. */ |
e15e8a2a | 271 | auto_vec<loop_p> loop_nest; |
a8af2e86 | 272 | |
45b13dc3 | 273 | /* Data Dependence Relations defining address ranges that are candidates |
274 | for a run-time aliasing check. */ | |
e15e8a2a | 275 | auto_vec<ddr_p> may_alias_ddrs; |
45b13dc3 | 276 | |
8a7b0f48 | 277 | /* Data Dependence Relations defining address ranges together with segment |
278 | lengths from which the run-time aliasing check is built. */ | |
e15e8a2a | 279 | auto_vec<dr_with_seg_len_pair_t> comp_alias_ddrs; |
8a7b0f48 | 280 | |
f68a7726 | 281 | /* Check that the addresses of each pair of objects is unequal. */ |
e15e8a2a | 282 | auto_vec<vec_object_pair> check_unequal_addrs; |
f68a7726 | 283 | |
25e3c2e8 | 284 | /* Statements in the loop that have data references that are candidates for a |
285 | runtime (loop versioning) misalignment check. */ | |
e15e8a2a | 286 | auto_vec<gimple *> may_misalign_stmts; |
25e3c2e8 | 287 | |
eefa05c8 | 288 | /* Reduction cycles detected in the loop. Used in loop-aware SLP. */ |
e15e8a2a | 289 | auto_vec<gimple *> reductions; |
0822b158 | 290 | |
39a5d6b1 | 291 | /* All reduction chains in the loop, represented by the first |
292 | stmt in the chain. */ | |
e15e8a2a | 293 | auto_vec<gimple *> reduction_chains; |
39a5d6b1 | 294 | |
2a9a3444 | 295 | /* Cost vector for a single scalar iteration. */ |
e15e8a2a | 296 | auto_vec<stmt_info_for_cost> scalar_cost_vec; |
2a9a3444 | 297 | |
487798e2 | 298 | /* The unrolling factor needed to SLP the loop. In case of that pure SLP is |
299 | applied to the loop, i.e., no unrolling is needed, this is 1. */ | |
d75596cd | 300 | poly_uint64 slp_unrolling_factor; |
487798e2 | 301 | |
2a9a3444 | 302 | /* Cost of a single scalar iteration. */ |
303 | int single_scalar_iteration_cost; | |
304 | ||
487798e2 | 305 | /* Is the loop vectorizable? */ |
306 | bool vectorizable; | |
307 | ||
ee612634 | 308 | /* When we have grouped data accesses with gaps, we may introduce invalid |
a4ee7fac | 309 | memory accesses. We peel the last iteration of the loop to prevent |
310 | this. */ | |
311 | bool peeling_for_gaps; | |
312 | ||
36f39b2e | 313 | /* When the number of iterations is not a multiple of the vector size |
314 | we need to peel off iterations at the end to form an epilogue loop. */ | |
315 | bool peeling_for_niter; | |
316 | ||
ba69439f | 317 | /* Reductions are canonicalized so that the last operand is the reduction |
318 | operand. If this places a constant into RHS1, this decanonicalizes | |
319 | GIMPLE for other phases, so we must track when this has occurred and | |
320 | fix it up. */ | |
321 | bool operands_swapped; | |
322 | ||
c7a8722c | 323 | /* True if there are no loop carried data dependencies in the loop. |
324 | If loop->safelen <= 1, then this is always true, either the loop | |
325 | didn't have any loop carried data dependencies, or the loop is being | |
326 | vectorized guarded with some runtime alias checks, or couldn't | |
327 | be vectorized at all, but then this field shouldn't be used. | |
328 | For loop->safelen >= 2, the user has asserted that there are no | |
329 | backward dependencies, but there still could be loop carried forward | |
330 | dependencies in such loops. This flag will be false if normal | |
331 | vectorizer data dependency analysis would fail or require versioning | |
332 | for alias, but because of loop->safelen >= 2 it has been vectorized | |
333 | even without versioning for alias. E.g. in: | |
334 | #pragma omp simd | |
335 | for (int i = 0; i < m; i++) | |
336 | a[i] = a[i + k] * c; | |
337 | (or #pragma simd or #pragma ivdep) we can vectorize this and it will | |
338 | DTRT even for k > 0 && k < m, but without safelen we would not | |
339 | vectorize this, so this field would be false. */ | |
340 | bool no_data_dependencies; | |
341 | ||
487798e2 | 342 | /* Mark loops having masked stores. */ |
343 | bool has_mask_store; | |
344 | ||
c71d3c24 | 345 | /* If if-conversion versioned this loop before conversion, this is the |
346 | loop version without if-conversion. */ | |
347 | struct loop *scalar_loop; | |
348 | ||
5b631e09 | 349 | /* For loops being epilogues of already vectorized loops |
350 | this points to the original vectorized loop. Otherwise NULL. */ | |
351 | _loop_vec_info *orig_loop_info; | |
352 | ||
4e58562d | 353 | } *loop_vec_info; |
354 | ||
25e3c2e8 | 355 | /* Access Functions. */ |
10095225 | 356 | #define LOOP_VINFO_LOOP(L) (L)->loop |
357 | #define LOOP_VINFO_BBS(L) (L)->bbs | |
796f6cba | 358 | #define LOOP_VINFO_NITERSM1(L) (L)->num_itersm1 |
10095225 | 359 | #define LOOP_VINFO_NITERS(L) (L)->num_iters |
796f6cba | 360 | /* Since LOOP_VINFO_NITERS and LOOP_VINFO_NITERSM1 can change after |
361 | prologue peeling retain total unchanged scalar loop iterations for | |
362 | cost model. */ | |
10095225 | 363 | #define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged |
d5e80d93 | 364 | #define LOOP_VINFO_NITERS_ASSUMPTIONS(L) (L)->num_iters_assumptions |
004a94a5 | 365 | #define LOOP_VINFO_COST_MODEL_THRESHOLD(L) (L)->th |
7456a7ea | 366 | #define LOOP_VINFO_VERSIONING_THRESHOLD(L) (L)->versioning_threshold |
10095225 | 367 | #define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable |
368 | #define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor | |
4a85c0b1 | 369 | #define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor |
10095225 | 370 | #define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask |
a8af2e86 | 371 | #define LOOP_VINFO_LOOP_NEST(L) (L)->loop_nest |
10095225 | 372 | #define LOOP_VINFO_DATAREFS(L) (L)->datarefs |
373 | #define LOOP_VINFO_DDRS(L) (L)->ddrs | |
374 | #define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters)) | |
313a5120 | 375 | #define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment |
10095225 | 376 | #define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr |
377 | #define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts | |
10095225 | 378 | #define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs |
8a7b0f48 | 379 | #define LOOP_VINFO_COMP_ALIAS_DDRS(L) (L)->comp_alias_ddrs |
f68a7726 | 380 | #define LOOP_VINFO_CHECK_UNEQUAL_ADDRS(L) (L)->check_unequal_addrs |
ee612634 | 381 | #define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores |
10095225 | 382 | #define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances |
c6895939 | 383 | #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor |
eefa05c8 | 384 | #define LOOP_VINFO_REDUCTIONS(L) (L)->reductions |
39a5d6b1 | 385 | #define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains |
4db2b577 | 386 | #define LOOP_VINFO_TARGET_COST_DATA(L) (L)->target_cost_data |
a4ee7fac | 387 | #define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps |
ba69439f | 388 | #define LOOP_VINFO_OPERANDS_SWAPPED(L) (L)->operands_swapped |
313a5120 | 389 | #define LOOP_VINFO_PEELING_FOR_NITER(L) (L)->peeling_for_niter |
c7a8722c | 390 | #define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies |
c71d3c24 | 391 | #define LOOP_VINFO_SCALAR_LOOP(L) (L)->scalar_loop |
cfd9ca84 | 392 | #define LOOP_VINFO_HAS_MASK_STORE(L) (L)->has_mask_store |
2a9a3444 | 393 | #define LOOP_VINFO_SCALAR_ITERATION_COST(L) (L)->scalar_cost_vec |
394 | #define LOOP_VINFO_SINGLE_SCALAR_ITERATION_COST(L) (L)->single_scalar_iteration_cost | |
5b631e09 | 395 | #define LOOP_VINFO_ORIG_LOOP_INFO(L) (L)->orig_loop_info |
4e58562d | 396 | |
d5e80d93 | 397 | #define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \ |
72ffab3c | 398 | ((L)->may_misalign_stmts.length () > 0) |
d5e80d93 | 399 | #define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \ |
f68a7726 | 400 | ((L)->comp_alias_ddrs.length () > 0 \ |
401 | || (L)->check_unequal_addrs.length () > 0) | |
d5e80d93 | 402 | #define LOOP_REQUIRES_VERSIONING_FOR_NITERS(L) \ |
403 | (LOOP_VINFO_NITERS_ASSUMPTIONS (L)) | |
404 | #define LOOP_REQUIRES_VERSIONING(L) \ | |
405 | (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (L) \ | |
406 | || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (L) \ | |
407 | || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L)) | |
33bbe730 | 408 | |
10095225 | 409 | #define LOOP_VINFO_NITERS_KNOWN_P(L) \ |
313a5120 | 410 | (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0) |
4e58562d | 411 | |
5b631e09 | 412 | #define LOOP_VINFO_EPILOGUE_P(L) \ |
413 | (LOOP_VINFO_ORIG_LOOP_INFO (L) != NULL) | |
414 | ||
4a85c0b1 | 415 | #define LOOP_VINFO_ORIG_MAX_VECT_FACTOR(L) \ |
416 | (LOOP_VINFO_MAX_VECT_FACTOR (LOOP_VINFO_ORIG_LOOP_INFO (L))) | |
5b631e09 | 417 | |
221e9a92 | 418 | static inline loop_vec_info |
419 | loop_vec_info_for_loop (struct loop *loop) | |
420 | { | |
421 | return (loop_vec_info) loop->aux; | |
422 | } | |
423 | ||
424 | static inline bool | |
42acab1c | 425 | nested_in_vect_loop_p (struct loop *loop, gimple *stmt) |
221e9a92 | 426 | { |
48e1416a | 427 | return (loop->inner |
75a70cf9 | 428 | && (loop->inner == (gimple_bb (stmt))->loop_father)); |
221e9a92 | 429 | } |
430 | ||
e2c5c678 | 431 | typedef struct _bb_vec_info : public vec_info |
432 | { | |
e15e8a2a | 433 | _bb_vec_info (gimple_stmt_iterator, gimple_stmt_iterator); |
434 | ~_bb_vec_info (); | |
435 | ||
37545e54 | 436 | basic_block bb; |
4c7587f5 | 437 | gimple_stmt_iterator region_begin; |
438 | gimple_stmt_iterator region_end; | |
37545e54 | 439 | } *bb_vec_info; |
440 | ||
4db2b577 | 441 | #define BB_VINFO_BB(B) (B)->bb |
442 | #define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores | |
443 | #define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances | |
444 | #define BB_VINFO_DATAREFS(B) (B)->datarefs | |
445 | #define BB_VINFO_DDRS(B) (B)->ddrs | |
446 | #define BB_VINFO_TARGET_COST_DATA(B) (B)->target_cost_data | |
37545e54 | 447 | |
448 | static inline bb_vec_info | |
449 | vec_info_for_bb (basic_block bb) | |
450 | { | |
451 | return (bb_vec_info) bb->aux; | |
452 | } | |
453 | ||
c91e8223 | 454 | /*-----------------------------------------------------------------*/ |
455 | /* Info on vectorized defs. */ | |
456 | /*-----------------------------------------------------------------*/ | |
457 | enum stmt_vec_info_type { | |
458 | undef_vec_info_type = 0, | |
459 | load_vec_info_type, | |
460 | store_vec_info_type, | |
09e31a48 | 461 | shift_vec_info_type, |
c91e8223 | 462 | op_vec_info_type, |
22c2f6bd | 463 | call_vec_info_type, |
d09768a4 | 464 | call_simd_clone_vec_info_type, |
e9705e7f | 465 | assignment_vec_info_type, |
ea8f3370 | 466 | condition_vec_info_type, |
dab48979 | 467 | comparison_vec_info_type, |
c6c91d61 | 468 | reduc_vec_info_type, |
6fada017 | 469 | induc_vec_info_type, |
c6c91d61 | 470 | type_promotion_vec_info_type, |
9d8bf4aa | 471 | type_demotion_vec_info_type, |
221e9a92 | 472 | type_conversion_vec_info_type, |
473 | loop_exit_ctrl_vec_info_type | |
c6c91d61 | 474 | }; |
475 | ||
48e1416a | 476 | /* Indicates whether/how a variable is used in the scope of loop/basic |
f083cd24 | 477 | block. */ |
c6c91d61 | 478 | enum vect_relevant { |
f083cd24 | 479 | vect_unused_in_scope = 0, |
75aae5b4 | 480 | |
481 | /* The def is only used outside the loop. */ | |
482 | vect_used_only_live, | |
ade2ac53 | 483 | /* The def is in the inner loop, and the use is in the outer loop, and the |
484 | use is a reduction stmt. */ | |
221e9a92 | 485 | vect_used_in_outer_by_reduction, |
ade2ac53 | 486 | /* The def is in the inner loop, and the use is in the outer loop (and is |
487 | not part of reduction). */ | |
221e9a92 | 488 | vect_used_in_outer, |
bfe8bfe9 | 489 | |
490 | /* defs that feed computations that end up (only) in a reduction. These | |
48e1416a | 491 | defs may be used by non-reduction stmts, but eventually, any |
492 | computations/values that are affected by these defs are used to compute | |
493 | a reduction (i.e. don't get stored to memory, for example). We use this | |
494 | to identify computations that we can change the order in which they are | |
bfe8bfe9 | 495 | computed. */ |
c6c91d61 | 496 | vect_used_by_reduction, |
bfe8bfe9 | 497 | |
48e1416a | 498 | vect_used_in_scope |
c91e8223 | 499 | }; |
500 | ||
c6895939 | 501 | /* The type of vectorization that can be applied to the stmt: regular loop-based |
502 | vectorization; pure SLP - the stmt is a part of SLP instances and does not | |
503 | have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is | |
504 | a part of SLP instance and also must be loop-based vectorized, since it has | |
48e1416a | 505 | uses outside SLP sequences. |
506 | ||
507 | In the loop context the meanings of pure and hybrid SLP are slightly | |
508 | different. By saying that pure SLP is applied to the loop, we mean that we | |
509 | exploit only intra-iteration parallelism in the loop; i.e., the loop can be | |
510 | vectorized without doing any conceptual unrolling, cause we don't pack | |
511 | together stmts from different iterations, only within a single iteration. | |
512 | Loop hybrid SLP means that we exploit both intra-iteration and | |
c6895939 | 513 | inter-iteration parallelism (e.g., number of elements in the vector is 4 |
48e1416a | 514 | and the slp-group-size is 2, in which case we don't have enough parallelism |
515 | within an iteration, so we obtain the rest of the parallelism from subsequent | |
c6895939 | 516 | iterations by unrolling the loop by 2). */ |
48e1416a | 517 | enum slp_vect_type { |
c6895939 | 518 | loop_vect = 0, |
519 | pure_slp, | |
520 | hybrid | |
521 | }; | |
522 | ||
0f54e40f | 523 | /* Says whether a statement is a load, a store of a vectorized statement |
524 | result, or a store of an invariant value. */ | |
525 | enum vec_load_store_type { | |
526 | VLS_LOAD, | |
527 | VLS_STORE, | |
528 | VLS_STORE_INVARIANT | |
529 | }; | |
530 | ||
85b53a1f | 531 | /* Describes how we're going to vectorize an individual load or store, |
532 | or a group of loads or stores. */ | |
533 | enum vect_memory_access_type { | |
989ceec3 | 534 | /* An access to an invariant address. This is used only for loads. */ |
535 | VMAT_INVARIANT, | |
536 | ||
85b53a1f | 537 | /* A simple contiguous access. */ |
538 | VMAT_CONTIGUOUS, | |
539 | ||
989ceec3 | 540 | /* A contiguous access that goes down in memory rather than up, |
541 | with no additional permutation. This is used only for stores | |
542 | of invariants. */ | |
543 | VMAT_CONTIGUOUS_DOWN, | |
544 | ||
85b53a1f | 545 | /* A simple contiguous access in which the elements need to be permuted |
546 | after loading or before storing. Only used for loop vectorization; | |
547 | SLP uses separate permutes. */ | |
548 | VMAT_CONTIGUOUS_PERMUTE, | |
549 | ||
989ceec3 | 550 | /* A simple contiguous access in which the elements need to be reversed |
551 | after loading or before storing. */ | |
552 | VMAT_CONTIGUOUS_REVERSE, | |
553 | ||
85b53a1f | 554 | /* An access that uses IFN_LOAD_LANES or IFN_STORE_LANES. */ |
555 | VMAT_LOAD_STORE_LANES, | |
556 | ||
557 | /* An access in which each scalar element is loaded or stored | |
558 | individually. */ | |
559 | VMAT_ELEMENTWISE, | |
560 | ||
561 | /* A hybrid of VMAT_CONTIGUOUS and VMAT_ELEMENTWISE, used for grouped | |
562 | SLP accesses. Each unrolled iteration uses a contiguous load | |
563 | or store for the whole group, but the groups from separate iterations | |
564 | are combined in the same way as for VMAT_ELEMENTWISE. */ | |
565 | VMAT_STRIDED_SLP, | |
566 | ||
567 | /* The access uses gather loads or scatter stores. */ | |
568 | VMAT_GATHER_SCATTER | |
569 | }; | |
c6895939 | 570 | |
f1168a33 | 571 | typedef struct data_reference *dr_p; |
f1168a33 | 572 | |
c91e8223 | 573 | typedef struct _stmt_vec_info { |
574 | ||
575 | enum stmt_vec_info_type type; | |
576 | ||
609c710b | 577 | /* Indicates whether this stmts is part of a computation whose result is |
578 | used outside the loop. */ | |
579 | bool live; | |
580 | ||
581 | /* Stmt is part of some pattern (computation idiom) */ | |
582 | bool in_pattern_p; | |
583 | ||
487798e2 | 584 | /* Is this statement vectorizable or should it be skipped in (partial) |
585 | vectorization. */ | |
586 | bool vectorizable; | |
587 | ||
c91e8223 | 588 | /* The stmt to which this info struct refers to. */ |
42acab1c | 589 | gimple *stmt; |
c91e8223 | 590 | |
e2c5c678 | 591 | /* The vec_info with respect to which STMT is vectorized. */ |
592 | vec_info *vinfo; | |
c91e8223 | 593 | |
b334cbba | 594 | /* The vector type to be used for the LHS of this statement. */ |
c91e8223 | 595 | tree vectype; |
596 | ||
597 | /* The vectorized version of the stmt. */ | |
42acab1c | 598 | gimple *vectorized_stmt; |
c91e8223 | 599 | |
600 | ||
16ed3c2c | 601 | /* The following is relevant only for stmts that contain a non-scalar |
48e1416a | 602 | data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have |
16ed3c2c | 603 | at most one such data-ref. */ |
c91e8223 | 604 | |
b0eb8c66 | 605 | /* Information about the data-ref (access function, etc), |
606 | relative to the inner-most containing loop. */ | |
c91e8223 | 607 | struct data_reference *data_ref_info; |
608 | ||
b0eb8c66 | 609 | /* Information about the data-ref relative to this loop |
610 | nest (the loop that is being considered for vectorization). */ | |
9e879814 | 611 | innermost_loop_behavior dr_wrt_vec_loop; |
b0eb8c66 | 612 | |
559260b3 | 613 | /* For loop PHI nodes, the base and evolution part of it. This makes sure |
86faead7 | 614 | this information is still available in vect_update_ivs_after_vectorizer |
615 | where we may not be able to re-analyze the PHI nodes evolution as | |
616 | peeling for the prologue loop can make it unanalyzable. The evolution | |
559260b3 | 617 | part is still correct after peeling, but the base may have changed from |
618 | the version here. */ | |
619 | tree loop_phi_evolution_base_unchanged; | |
86faead7 | 620 | tree loop_phi_evolution_part; |
621 | ||
48e1416a | 622 | /* Used for various bookkeeping purposes, generally holding a pointer to |
623 | some other stmt S that is in some way "related" to this stmt. | |
4a61a337 | 624 | Current use of this field is: |
48e1416a | 625 | If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is |
626 | true): S is the "pattern stmt" that represents (and replaces) the | |
627 | sequence of stmts that constitutes the pattern. Similarly, the | |
628 | related_stmt of the "pattern stmt" points back to this stmt (which is | |
629 | the last stmt in the original sequence of stmts that constitutes the | |
4a61a337 | 630 | pattern). */ |
42acab1c | 631 | gimple *related_stmt; |
4a61a337 | 632 | |
18937389 | 633 | /* Used to keep a sequence of def stmts of a pattern stmt if such exists. */ |
634 | gimple_seq pattern_def_seq; | |
45eea33f | 635 | |
f1168a33 | 636 | /* List of datarefs that are known to have the same alignment as the dataref |
637 | of this stmt. */ | |
f1f41a6c | 638 | vec<dr_p> same_align_refs; |
f1168a33 | 639 | |
295327ab | 640 | /* Selected SIMD clone's function info. First vector element |
641 | is SIMD clone's function decl, followed by a pair of trees (base + step) | |
642 | for linear arguments (pair of NULLs for other arguments). */ | |
643 | vec<tree> simd_clone_info; | |
d09768a4 | 644 | |
e12906b9 | 645 | /* Classify the def of this stmt. */ |
646 | enum vect_def_type def_type; | |
647 | ||
609c710b | 648 | /* Whether the stmt is SLPed, loop-based vectorized, or both. */ |
649 | enum slp_vect_type slp_type; | |
650 | ||
21009880 | 651 | /* Interleaving and reduction chains info. */ |
652 | /* First element in the group. */ | |
42acab1c | 653 | gimple *first_element; |
21009880 | 654 | /* Pointer to the next element in the group. */ |
42acab1c | 655 | gimple *next_element; |
21009880 | 656 | /* For data-refs, in case that two or more stmts share data-ref, this is the |
657 | pointer to the previously detected stmt with the same dr. */ | |
42acab1c | 658 | gimple *same_dr_stmt; |
21009880 | 659 | /* The size of the group. */ |
6b8dbb53 | 660 | unsigned int size; |
661 | /* For stores, number of stores from this group seen. We vectorize the last | |
662 | one. */ | |
663 | unsigned int store_count; | |
664 | /* For loads only, the gap from the previous load. For consecutive loads, GAP | |
665 | is 1. */ | |
666 | unsigned int gap; | |
609c710b | 667 | |
a8cf7702 | 668 | /* The minimum negative dependence distance this stmt participates in |
669 | or zero if none. */ | |
670 | unsigned int min_neg_dist; | |
671 | ||
609c710b | 672 | /* Not all stmts in the loop need to be vectorized. e.g, the increment |
673 | of the loop induction variable and computation of array indexes. relevant | |
674 | indicates whether the stmt needs to be vectorized. */ | |
675 | enum vect_relevant relevant; | |
867c03eb | 676 | |
0bd6d857 | 677 | /* For loads if this is a gather, for stores if this is a scatter. */ |
678 | bool gather_scatter_p; | |
e1c75243 | 679 | |
680 | /* True if this is an access with loop-invariant stride. */ | |
681 | bool strided_p; | |
3d483a94 | 682 | |
487798e2 | 683 | /* For both loads and stores. */ |
684 | bool simd_lane_access_p; | |
685 | ||
85b53a1f | 686 | /* Classifies how the load or store is going to be implemented |
687 | for loop vectorization. */ | |
688 | vect_memory_access_type memory_access_type; | |
689 | ||
d09d8733 | 690 | /* For reduction loops, this is the type of reduction. */ |
691 | enum vect_reduction_type v_reduc_type; | |
692 | ||
834a2c29 | 693 | /* For CONST_COND_REDUCTION, record the reduc code. */ |
694 | enum tree_code const_cond_reduc_code; | |
695 | ||
119a8852 | 696 | /* On a reduction PHI the reduction type as detected by |
697 | vect_force_simple_reduction. */ | |
698 | enum vect_reduction_type reduc_type; | |
699 | ||
44b24fa0 | 700 | /* On a reduction PHI the def returned by vect_force_simple_reduction. |
701 | On the def returned by vect_force_simple_reduction the | |
702 | corresponding PHI. */ | |
119a8852 | 703 | gimple *reduc_def; |
704 | ||
0d85be19 | 705 | /* The number of scalar stmt references from active SLP instances. */ |
706 | unsigned int num_slp_uses; | |
c91e8223 | 707 | } *stmt_vec_info; |
708 | ||
cf60da07 | 709 | /* Information about a gather/scatter call. */ |
710 | struct gather_scatter_info { | |
711 | /* The FUNCTION_DECL for the built-in gather/scatter function. */ | |
712 | tree decl; | |
713 | ||
714 | /* The loop-invariant base value. */ | |
715 | tree base; | |
716 | ||
717 | /* The original scalar offset, which is a non-loop-invariant SSA_NAME. */ | |
718 | tree offset; | |
719 | ||
720 | /* Each offset element should be multiplied by this amount before | |
721 | being added to the base. */ | |
722 | int scale; | |
723 | ||
724 | /* The definition type for the vectorized offset. */ | |
725 | enum vect_def_type offset_dt; | |
726 | ||
727 | /* The type of the vectorized offset. */ | |
728 | tree offset_vectype; | |
729 | }; | |
730 | ||
c91e8223 | 731 | /* Access Functions. */ |
6b8dbb53 | 732 | #define STMT_VINFO_TYPE(S) (S)->type |
733 | #define STMT_VINFO_STMT(S) (S)->stmt | |
e2c5c678 | 734 | inline loop_vec_info |
735 | STMT_VINFO_LOOP_VINFO (stmt_vec_info stmt_vinfo) | |
736 | { | |
737 | if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (stmt_vinfo->vinfo)) | |
738 | return loop_vinfo; | |
739 | return NULL; | |
740 | } | |
741 | inline bb_vec_info | |
742 | STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo) | |
743 | { | |
744 | if (bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (stmt_vinfo->vinfo)) | |
745 | return bb_vinfo; | |
746 | return NULL; | |
747 | } | |
6b8dbb53 | 748 | #define STMT_VINFO_RELEVANT(S) (S)->relevant |
749 | #define STMT_VINFO_LIVE_P(S) (S)->live | |
750 | #define STMT_VINFO_VECTYPE(S) (S)->vectype | |
751 | #define STMT_VINFO_VEC_STMT(S) (S)->vectorized_stmt | |
6ea6a380 | 752 | #define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable |
6b8dbb53 | 753 | #define STMT_VINFO_DATA_REF(S) (S)->data_ref_info |
0bd6d857 | 754 | #define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p |
e1c75243 | 755 | #define STMT_VINFO_STRIDED_P(S) (S)->strided_p |
85b53a1f | 756 | #define STMT_VINFO_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type |
3d483a94 | 757 | #define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p |
d09d8733 | 758 | #define STMT_VINFO_VEC_REDUCTION_TYPE(S) (S)->v_reduc_type |
834a2c29 | 759 | #define STMT_VINFO_VEC_CONST_COND_REDUC_CODE(S) (S)->const_cond_reduc_code |
b0eb8c66 | 760 | |
9e879814 | 761 | #define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop |
762 | #define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address | |
763 | #define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init | |
764 | #define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset | |
765 | #define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step | |
a5456a6d | 766 | #define STMT_VINFO_DR_BASE_ALIGNMENT(S) (S)->dr_wrt_vec_loop.base_alignment |
767 | #define STMT_VINFO_DR_BASE_MISALIGNMENT(S) \ | |
768 | (S)->dr_wrt_vec_loop.base_misalignment | |
a7e05ef2 | 769 | #define STMT_VINFO_DR_OFFSET_ALIGNMENT(S) \ |
770 | (S)->dr_wrt_vec_loop.offset_alignment | |
668dd7dc | 771 | #define STMT_VINFO_DR_STEP_ALIGNMENT(S) \ |
772 | (S)->dr_wrt_vec_loop.step_alignment | |
b0eb8c66 | 773 | |
6b8dbb53 | 774 | #define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p |
775 | #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt | |
18937389 | 776 | #define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq |
6b8dbb53 | 777 | #define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs |
295327ab | 778 | #define STMT_VINFO_SIMD_CLONE_INFO(S) (S)->simd_clone_info |
6b8dbb53 | 779 | #define STMT_VINFO_DEF_TYPE(S) (S)->def_type |
21009880 | 780 | #define STMT_VINFO_GROUP_FIRST_ELEMENT(S) (S)->first_element |
781 | #define STMT_VINFO_GROUP_NEXT_ELEMENT(S) (S)->next_element | |
782 | #define STMT_VINFO_GROUP_SIZE(S) (S)->size | |
783 | #define STMT_VINFO_GROUP_STORE_COUNT(S) (S)->store_count | |
784 | #define STMT_VINFO_GROUP_GAP(S) (S)->gap | |
785 | #define STMT_VINFO_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt | |
ee612634 | 786 | #define STMT_VINFO_GROUPED_ACCESS(S) ((S)->first_element != NULL && (S)->data_ref_info) |
559260b3 | 787 | #define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged |
86faead7 | 788 | #define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part |
a8cf7702 | 789 | #define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist |
0d85be19 | 790 | #define STMT_VINFO_NUM_SLP_USES(S) (S)->num_slp_uses |
119a8852 | 791 | #define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type |
792 | #define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def | |
21009880 | 793 | |
794 | #define GROUP_FIRST_ELEMENT(S) (S)->first_element | |
795 | #define GROUP_NEXT_ELEMENT(S) (S)->next_element | |
796 | #define GROUP_SIZE(S) (S)->size | |
797 | #define GROUP_STORE_COUNT(S) (S)->store_count | |
798 | #define GROUP_GAP(S) (S)->gap | |
799 | #define GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt | |
c91e8223 | 800 | |
f083cd24 | 801 | #define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope) |
867c03eb | 802 | |
c6895939 | 803 | #define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid) |
804 | #define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp) | |
805 | #define STMT_SLP_TYPE(S) (S)->slp_type | |
806 | ||
23e1875f | 807 | struct dataref_aux { |
aec313e5 | 808 | /* The misalignment in bytes of the reference, or -1 if not known. */ |
23e1875f | 809 | int misalignment; |
aec313e5 | 810 | /* The byte alignment that we'd ideally like the reference to have, |
811 | and the value that misalignment is measured against. */ | |
812 | int target_alignment; | |
9dd88d41 | 813 | /* If true the alignment of base_decl needs to be increased. */ |
814 | bool base_misaligned; | |
9dd88d41 | 815 | tree base_decl; |
23e1875f | 816 | }; |
817 | ||
9dd88d41 | 818 | #define DR_VECT_AUX(dr) ((dataref_aux *)(dr)->aux) |
819 | ||
0822b158 | 820 | #define VECT_MAX_COST 1000 |
821 | ||
862bb3cd | 822 | /* The maximum number of intermediate steps required in multi-step type |
823 | conversion. */ | |
824 | #define MAX_INTERM_CVT_STEPS 3 | |
825 | ||
d75596cd | 826 | #define MAX_VECTORIZATION_FACTOR INT_MAX |
91a74fc6 | 827 | |
69fcaae3 | 828 | /* Nonzero if TYPE represents a (scalar) boolean type or type |
829 | in the middle-end compatible with it (unsigned precision 1 integral | |
830 | types). Used to determine which types should be vectorized as | |
831 | VECTOR_BOOLEAN_TYPE_P. */ | |
832 | ||
833 | #define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE) \ | |
834 | (TREE_CODE (TYPE) == BOOLEAN_TYPE \ | |
835 | || ((TREE_CODE (TYPE) == INTEGER_TYPE \ | |
836 | || TREE_CODE (TYPE) == ENUMERAL_TYPE) \ | |
837 | && TYPE_PRECISION (TYPE) == 1 \ | |
838 | && TYPE_UNSIGNED (TYPE))) | |
839 | ||
3702cf13 | 840 | extern vec<stmt_vec_info> stmt_vec_info_vec; |
75a70cf9 | 841 | |
842 | void init_stmt_vec_info_vec (void); | |
843 | void free_stmt_vec_info_vec (void); | |
844 | ||
282bf14c | 845 | /* Return a stmt_vec_info corresponding to STMT. */ |
846 | ||
75a70cf9 | 847 | static inline stmt_vec_info |
42acab1c | 848 | vinfo_for_stmt (gimple *stmt) |
c91e8223 | 849 | { |
e15e8a2a | 850 | int uid = gimple_uid (stmt); |
851 | if (uid <= 0) | |
75a70cf9 | 852 | return NULL; |
853 | ||
3702cf13 | 854 | return stmt_vec_info_vec[uid - 1]; |
c91e8223 | 855 | } |
856 | ||
282bf14c | 857 | /* Set vectorizer information INFO for STMT. */ |
858 | ||
75a70cf9 | 859 | static inline void |
42acab1c | 860 | set_vinfo_for_stmt (gimple *stmt, stmt_vec_info info) |
c91e8223 | 861 | { |
75a70cf9 | 862 | unsigned int uid = gimple_uid (stmt); |
863 | if (uid == 0) | |
864 | { | |
e95895ef | 865 | gcc_checking_assert (info); |
f1f41a6c | 866 | uid = stmt_vec_info_vec.length () + 1; |
75a70cf9 | 867 | gimple_set_uid (stmt, uid); |
3702cf13 | 868 | stmt_vec_info_vec.safe_push (info); |
75a70cf9 | 869 | } |
870 | else | |
fc5f8be0 | 871 | { |
872 | gcc_checking_assert (info == NULL); | |
873 | stmt_vec_info_vec[uid - 1] = info; | |
874 | } | |
c91e8223 | 875 | } |
876 | ||
282bf14c | 877 | /* Return the earlier statement between STMT1 and STMT2. */ |
878 | ||
42acab1c | 879 | static inline gimple * |
880 | get_earlier_stmt (gimple *stmt1, gimple *stmt2) | |
bdc89b8f | 881 | { |
882 | unsigned int uid1, uid2; | |
883 | ||
884 | if (stmt1 == NULL) | |
885 | return stmt2; | |
886 | ||
887 | if (stmt2 == NULL) | |
888 | return stmt1; | |
889 | ||
890 | uid1 = gimple_uid (stmt1); | |
891 | uid2 = gimple_uid (stmt2); | |
892 | ||
893 | if (uid1 == 0 || uid2 == 0) | |
894 | return NULL; | |
895 | ||
f1f41a6c | 896 | gcc_checking_assert (uid1 <= stmt_vec_info_vec.length () |
897 | && uid2 <= stmt_vec_info_vec.length ()); | |
bdc89b8f | 898 | |
899 | if (uid1 < uid2) | |
900 | return stmt1; | |
901 | else | |
902 | return stmt2; | |
903 | } | |
904 | ||
282bf14c | 905 | /* Return the later statement between STMT1 and STMT2. */ |
906 | ||
42acab1c | 907 | static inline gimple * |
908 | get_later_stmt (gimple *stmt1, gimple *stmt2) | |
d4b21757 | 909 | { |
910 | unsigned int uid1, uid2; | |
911 | ||
912 | if (stmt1 == NULL) | |
913 | return stmt2; | |
914 | ||
915 | if (stmt2 == NULL) | |
916 | return stmt1; | |
917 | ||
918 | uid1 = gimple_uid (stmt1); | |
919 | uid2 = gimple_uid (stmt2); | |
920 | ||
921 | if (uid1 == 0 || uid2 == 0) | |
922 | return NULL; | |
923 | ||
f1f41a6c | 924 | gcc_assert (uid1 <= stmt_vec_info_vec.length ()); |
925 | gcc_assert (uid2 <= stmt_vec_info_vec.length ()); | |
d4b21757 | 926 | |
927 | if (uid1 > uid2) | |
928 | return stmt1; | |
929 | else | |
930 | return stmt2; | |
931 | } | |
932 | ||
282bf14c | 933 | /* Return TRUE if a statement represented by STMT_INFO is a part of a |
934 | pattern. */ | |
935 | ||
213448e9 | 936 | static inline bool |
937 | is_pattern_stmt_p (stmt_vec_info stmt_info) | |
938 | { | |
42acab1c | 939 | gimple *related_stmt; |
213448e9 | 940 | stmt_vec_info related_stmt_info; |
941 | ||
942 | related_stmt = STMT_VINFO_RELATED_STMT (stmt_info); | |
943 | if (related_stmt | |
944 | && (related_stmt_info = vinfo_for_stmt (related_stmt)) | |
945 | && STMT_VINFO_IN_PATTERN_P (related_stmt_info)) | |
946 | return true; | |
947 | ||
948 | return false; | |
949 | } | |
950 | ||
282bf14c | 951 | /* Return true if BB is a loop header. */ |
952 | ||
221e9a92 | 953 | static inline bool |
954 | is_loop_header_bb_p (basic_block bb) | |
955 | { | |
956 | if (bb == (bb->loop_father)->header) | |
957 | return true; | |
e95895ef | 958 | gcc_checking_assert (EDGE_COUNT (bb->preds) == 1); |
221e9a92 | 959 | return false; |
960 | } | |
961 | ||
282bf14c | 962 | /* Return pow2 (X). */ |
963 | ||
862bb3cd | 964 | static inline int |
965 | vect_pow2 (int x) | |
966 | { | |
967 | int i, res = 1; | |
968 | ||
969 | for (i = 0; i < x; i++) | |
970 | res *= 2; | |
971 | ||
972 | return res; | |
973 | } | |
84a15e8f | 974 | |
f97dec81 | 975 | /* Alias targetm.vectorize.builtin_vectorization_cost. */ |
976 | ||
977 | static inline int | |
978 | builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost, | |
979 | tree vectype, int misalign) | |
980 | { | |
981 | return targetm.vectorize.builtin_vectorization_cost (type_of_cost, | |
982 | vectype, misalign); | |
983 | } | |
984 | ||
f4ac3f3e | 985 | /* Get cost by calling cost target builtin. */ |
986 | ||
987 | static inline | |
988 | int vect_get_stmt_cost (enum vect_cost_for_stmt type_of_cost) | |
989 | { | |
f97dec81 | 990 | return builtin_vectorization_cost (type_of_cost, NULL, 0); |
f4ac3f3e | 991 | } |
992 | ||
4db2b577 | 993 | /* Alias targetm.vectorize.init_cost. */ |
994 | ||
995 | static inline void * | |
996 | init_cost (struct loop *loop_info) | |
997 | { | |
998 | return targetm.vectorize.init_cost (loop_info); | |
999 | } | |
1000 | ||
1001 | /* Alias targetm.vectorize.add_stmt_cost. */ | |
1002 | ||
1003 | static inline unsigned | |
1004 | add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind, | |
f97dec81 | 1005 | stmt_vec_info stmt_info, int misalign, |
1006 | enum vect_cost_model_location where) | |
4db2b577 | 1007 | { |
1008 | return targetm.vectorize.add_stmt_cost (data, count, kind, | |
f97dec81 | 1009 | stmt_info, misalign, where); |
4db2b577 | 1010 | } |
1011 | ||
1012 | /* Alias targetm.vectorize.finish_cost. */ | |
1013 | ||
f97dec81 | 1014 | static inline void |
1015 | finish_cost (void *data, unsigned *prologue_cost, | |
1016 | unsigned *body_cost, unsigned *epilogue_cost) | |
4db2b577 | 1017 | { |
f97dec81 | 1018 | targetm.vectorize.finish_cost (data, prologue_cost, body_cost, epilogue_cost); |
4db2b577 | 1019 | } |
1020 | ||
1021 | /* Alias targetm.vectorize.destroy_cost_data. */ | |
1022 | ||
1023 | static inline void | |
1024 | destroy_cost_data (void *data) | |
1025 | { | |
1026 | targetm.vectorize.destroy_cost_data (data); | |
1027 | } | |
1028 | ||
c91e8223 | 1029 | /*-----------------------------------------------------------------*/ |
1030 | /* Info on data references alignment. */ | |
1031 | /*-----------------------------------------------------------------*/ | |
23e1875f | 1032 | inline void |
1033 | set_dr_misalignment (struct data_reference *dr, int val) | |
1034 | { | |
9dd88d41 | 1035 | dataref_aux *data_aux = DR_VECT_AUX (dr); |
23e1875f | 1036 | |
1037 | if (!data_aux) | |
1038 | { | |
1039 | data_aux = XCNEW (dataref_aux); | |
1040 | dr->aux = data_aux; | |
1041 | } | |
1042 | ||
1043 | data_aux->misalignment = val; | |
1044 | } | |
1045 | ||
1046 | inline int | |
1047 | dr_misalignment (struct data_reference *dr) | |
1048 | { | |
9dd88d41 | 1049 | return DR_VECT_AUX (dr)->misalignment; |
23e1875f | 1050 | } |
c91e8223 | 1051 | |
39b8f742 | 1052 | /* Reflects actual alignment of first access in the vectorized loop, |
1053 | taking into account peeling/versioning if applied. */ | |
23e1875f | 1054 | #define DR_MISALIGNMENT(DR) dr_misalignment (DR) |
1055 | #define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL) | |
df8e9f7a | 1056 | #define DR_MISALIGNMENT_UNKNOWN (-1) |
c91e8223 | 1057 | |
aec313e5 | 1058 | /* Only defined once DR_MISALIGNMENT is defined. */ |
1059 | #define DR_TARGET_ALIGNMENT(DR) DR_VECT_AUX (DR)->target_alignment | |
1060 | ||
1061 | /* Return true if data access DR is aligned to its target alignment | |
1062 | (which may be less than a full vector). */ | |
282bf14c | 1063 | |
c91e8223 | 1064 | static inline bool |
1065 | aligned_access_p (struct data_reference *data_ref_info) | |
1066 | { | |
1067 | return (DR_MISALIGNMENT (data_ref_info) == 0); | |
1068 | } | |
1069 | ||
282bf14c | 1070 | /* Return TRUE if the alignment of the data access is known, and FALSE |
1071 | otherwise. */ | |
1072 | ||
c91e8223 | 1073 | static inline bool |
39b8f742 | 1074 | known_alignment_for_access_p (struct data_reference *data_ref_info) |
c91e8223 | 1075 | { |
df8e9f7a | 1076 | return (DR_MISALIGNMENT (data_ref_info) != DR_MISALIGNMENT_UNKNOWN); |
c91e8223 | 1077 | } |
1078 | ||
aec313e5 | 1079 | /* Return the minimum alignment in bytes that the vectorized version |
1080 | of DR is guaranteed to have. */ | |
1081 | ||
1082 | static inline unsigned int | |
1083 | vect_known_alignment_in_bytes (struct data_reference *dr) | |
1084 | { | |
1085 | if (DR_MISALIGNMENT (dr) == DR_MISALIGNMENT_UNKNOWN) | |
1086 | return TYPE_ALIGN_UNIT (TREE_TYPE (DR_REF (dr))); | |
1087 | if (DR_MISALIGNMENT (dr) == 0) | |
1088 | return DR_TARGET_ALIGNMENT (dr); | |
1089 | return DR_MISALIGNMENT (dr) & -DR_MISALIGNMENT (dr); | |
1090 | } | |
1091 | ||
9e879814 | 1092 | /* Return the behavior of DR with respect to the vectorization context |
1093 | (which for outer loop vectorization might not be the behavior recorded | |
1094 | in DR itself). */ | |
1095 | ||
1096 | static inline innermost_loop_behavior * | |
1097 | vect_dr_behavior (data_reference *dr) | |
1098 | { | |
1099 | gimple *stmt = DR_STMT (dr); | |
1100 | stmt_vec_info stmt_info = vinfo_for_stmt (stmt); | |
1101 | loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); | |
1102 | if (loop_vinfo == NULL | |
1103 | || !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt)) | |
1104 | return &DR_INNERMOST (dr); | |
1105 | else | |
1106 | return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info); | |
1107 | } | |
1dbf9bd1 | 1108 | |
1109 | /* Return true if the vect cost model is unlimited. */ | |
1110 | static inline bool | |
3e398f5b | 1111 | unlimited_cost_model (loop_p loop) |
1dbf9bd1 | 1112 | { |
4c73695b | 1113 | if (loop != NULL && loop->force_vectorize |
3e398f5b | 1114 | && flag_simd_cost_model != VECT_COST_MODEL_DEFAULT) |
1115 | return flag_simd_cost_model == VECT_COST_MODEL_UNLIMITED; | |
1116 | return (flag_vect_cost_model == VECT_COST_MODEL_UNLIMITED); | |
1dbf9bd1 | 1117 | } |
1118 | ||
d75596cd | 1119 | /* Return the number of vectors of type VECTYPE that are needed to get |
1120 | NUNITS elements. NUNITS should be based on the vectorization factor, | |
1121 | so it is always a known multiple of the number of elements in VECTYPE. */ | |
1122 | ||
1123 | static inline unsigned int | |
1124 | vect_get_num_vectors (poly_uint64 nunits, tree vectype) | |
1125 | { | |
1126 | return exact_div (nunits, TYPE_VECTOR_SUBPARTS (vectype)).to_constant (); | |
1127 | } | |
1128 | ||
4eb17cb6 | 1129 | /* Return the number of copies needed for loop vectorization when |
1130 | a statement operates on vectors of type VECTYPE. This is the | |
1131 | vectorization factor divided by the number of elements in | |
1132 | VECTYPE and is always known at compile time. */ | |
1133 | ||
1134 | static inline unsigned int | |
1135 | vect_get_num_copies (loop_vec_info loop_vinfo, tree vectype) | |
1136 | { | |
d75596cd | 1137 | return vect_get_num_vectors (LOOP_VINFO_VECT_FACTOR (loop_vinfo), vectype); |
1138 | } | |
1139 | ||
1140 | /* Update maximum unit count *MAX_NUNITS so that it accounts for | |
1141 | the number of units in vector type VECTYPE. *MAX_NUNITS can be 1 | |
1142 | if we haven't yet recorded any vector types. */ | |
1143 | ||
1144 | static inline void | |
1145 | vect_update_max_nunits (poly_uint64 *max_nunits, tree vectype) | |
1146 | { | |
1147 | /* All unit counts have the form current_vector_size * X for some | |
1148 | rational X, so two unit sizes must have a common multiple. | |
1149 | Everything is a multiple of the initial value of 1. */ | |
1150 | poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); | |
1151 | *max_nunits = force_common_multiple (*max_nunits, nunits); | |
1152 | } | |
1153 | ||
1154 | /* Return the vectorization factor that should be used for costing | |
1155 | purposes while vectorizing the loop described by LOOP_VINFO. | |
1156 | Pick a reasonable estimate if the vectorization factor isn't | |
1157 | known at compile time. */ | |
1158 | ||
1159 | static inline unsigned int | |
1160 | vect_vf_for_cost (loop_vec_info loop_vinfo) | |
1161 | { | |
1162 | return estimated_poly_value (LOOP_VINFO_VECT_FACTOR (loop_vinfo)); | |
4eb17cb6 | 1163 | } |
1164 | ||
09de8b78 | 1165 | /* Estimate the number of elements in VEC_TYPE for costing purposes. |
1166 | Pick a reasonable estimate if the exact number isn't known at | |
1167 | compile time. */ | |
1168 | ||
1169 | static inline unsigned int | |
1170 | vect_nunits_for_cost (tree vec_type) | |
1171 | { | |
1172 | return estimated_poly_value (TYPE_VECTOR_SUBPARTS (vec_type)); | |
1173 | } | |
1174 | ||
33482edf | 1175 | /* Return the size of the value accessed by unvectorized data reference DR. |
1176 | This is only valid once STMT_VINFO_VECTYPE has been calculated for the | |
1177 | associated gimple statement, since that guarantees that DR accesses | |
1178 | either a scalar or a scalar equivalent. ("Scalar equivalent" here | |
1179 | includes things like V1SI, which can be vectorized in the same way | |
1180 | as a plain SI.) */ | |
1181 | ||
1182 | inline unsigned int | |
1183 | vect_get_scalar_dr_size (struct data_reference *dr) | |
1184 | { | |
1185 | return tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr)))); | |
1186 | } | |
1187 | ||
7bd765d4 | 1188 | /* Source location */ |
36f39b2e | 1189 | extern source_location vect_location; |
fb85abff | 1190 | |
c91e8223 | 1191 | /*-----------------------------------------------------------------*/ |
1192 | /* Function prototypes. */ | |
1193 | /*-----------------------------------------------------------------*/ | |
1194 | ||
48e1416a | 1195 | /* Simple loop peeling and versioning utilities for vectorizer's purposes - |
fb85abff | 1196 | in tree-vect-loop-manip.c. */ |
cde959e7 | 1197 | extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree, tree, |
1198 | tree, bool); | |
1f1872fd | 1199 | extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge); |
c71d3c24 | 1200 | struct loop *slpeel_tree_duplicate_loop_to_edge_cfg (struct loop *, |
1201 | struct loop *, edge); | |
7456a7ea | 1202 | extern void vect_loop_versioning (loop_vec_info, unsigned int, bool, |
1203 | poly_uint64); | |
5b631e09 | 1204 | extern struct loop *vect_do_peeling (loop_vec_info, tree, tree, |
cde959e7 | 1205 | tree *, tree *, tree *, int, bool, bool); |
36f39b2e | 1206 | extern source_location find_loop_location (struct loop *); |
fb85abff | 1207 | extern bool vect_can_advance_ivs_p (loop_vec_info); |
c91e8223 | 1208 | |
fb85abff | 1209 | /* In tree-vect-stmts.c. */ |
3106770a | 1210 | extern poly_uint64 current_vector_size; |
f2983e95 | 1211 | extern tree get_vectype_for_scalar_type (tree); |
dab48979 | 1212 | extern tree get_mask_type_for_scalar_type (tree); |
b334cbba | 1213 | extern tree get_same_sized_vectype (tree, tree); |
5cc2ea45 | 1214 | extern bool vect_is_simple_use (tree, vec_info *, gimple **, |
1215 | enum vect_def_type *); | |
1216 | extern bool vect_is_simple_use (tree, vec_info *, gimple **, | |
1217 | enum vect_def_type *, tree *); | |
42acab1c | 1218 | extern bool supportable_widening_operation (enum tree_code, gimple *, tree, |
1219 | tree, enum tree_code *, | |
1220 | enum tree_code *, int *, | |
1221 | vec<tree> *); | |
b334cbba | 1222 | extern bool supportable_narrowing_operation (enum tree_code, tree, tree, |
1223 | enum tree_code *, | |
f1f41a6c | 1224 | int *, vec<tree> *); |
e2c5c678 | 1225 | extern stmt_vec_info new_stmt_vec_info (gimple *stmt, vec_info *); |
42acab1c | 1226 | extern void free_stmt_vec_info (gimple *stmt); |
fb85abff | 1227 | extern void vect_model_simple_cost (stmt_vec_info, int, enum vect_def_type *, |
9bf6e01f | 1228 | int, stmt_vector_for_cost *, |
f97dec81 | 1229 | stmt_vector_for_cost *); |
85b53a1f | 1230 | extern void vect_model_store_cost (stmt_vec_info, int, vect_memory_access_type, |
0f54e40f | 1231 | vec_load_store_type, slp_tree, |
f97dec81 | 1232 | stmt_vector_for_cost *, |
4db2b577 | 1233 | stmt_vector_for_cost *); |
85b53a1f | 1234 | extern void vect_model_load_cost (stmt_vec_info, int, vect_memory_access_type, |
1235 | slp_tree, stmt_vector_for_cost *, | |
4db2b577 | 1236 | stmt_vector_for_cost *); |
1237 | extern unsigned record_stmt_cost (stmt_vector_for_cost *, int, | |
f97dec81 | 1238 | enum vect_cost_for_stmt, stmt_vec_info, |
1239 | int, enum vect_cost_model_location); | |
42acab1c | 1240 | extern void vect_finish_stmt_generation (gimple *, gimple *, |
fb85abff | 1241 | gimple_stmt_iterator *); |
1242 | extern bool vect_mark_stmts_to_be_vectorized (loop_vec_info); | |
707ba526 | 1243 | extern tree vect_get_vec_def_for_operand_1 (gimple *, enum vect_def_type); |
dab48979 | 1244 | extern tree vect_get_vec_def_for_operand (tree, gimple *, tree = NULL); |
44b24fa0 | 1245 | extern void vect_get_vec_defs (tree, tree, gimple *, vec<tree> *, |
1246 | vec<tree> *, slp_tree); | |
1247 | extern void vect_get_vec_defs_for_stmt_copy (enum vect_def_type *, | |
1248 | vec<tree> *, vec<tree> *); | |
42acab1c | 1249 | extern tree vect_init_vector (gimple *, tree, tree, |
fb85abff | 1250 | gimple_stmt_iterator *); |
1251 | extern tree vect_get_vec_def_for_stmt_copy (enum vect_def_type, tree); | |
42acab1c | 1252 | extern bool vect_transform_stmt (gimple *, gimple_stmt_iterator *, |
fb85abff | 1253 | bool *, slp_tree, slp_instance); |
42acab1c | 1254 | extern void vect_remove_stores (gimple *); |
6154acba | 1255 | extern bool vect_analyze_stmt (gimple *, bool *, slp_tree, slp_instance); |
42acab1c | 1256 | extern bool vectorizable_condition (gimple *, gimple_stmt_iterator *, |
1257 | gimple **, tree, int, slp_tree); | |
0822b158 | 1258 | extern void vect_get_load_cost (struct data_reference *, int, bool, |
4db2b577 | 1259 | unsigned int *, unsigned int *, |
f97dec81 | 1260 | stmt_vector_for_cost *, |
1261 | stmt_vector_for_cost *, bool); | |
4db2b577 | 1262 | extern void vect_get_store_cost (struct data_reference *, int, |
1263 | unsigned int *, stmt_vector_for_cost *); | |
45eea33f | 1264 | extern bool vect_supportable_shift (enum tree_code, tree); |
25eb7c31 | 1265 | extern tree vect_gen_perm_mask_any (tree, const vec_perm_indices &); |
1266 | extern tree vect_gen_perm_mask_checked (tree, const vec_perm_indices &); | |
cfd9ca84 | 1267 | extern void optimize_mask_stores (struct loop*); |
48e1416a | 1268 | |
fb85abff | 1269 | /* In tree-vect-data-refs.c. */ |
1270 | extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int); | |
1271 | extern enum dr_alignment_support vect_supportable_dr_alignment | |
0822b158 | 1272 | (struct data_reference *, bool); |
42acab1c | 1273 | extern tree vect_get_smallest_scalar_type (gimple *, HOST_WIDE_INT *, |
fb85abff | 1274 | HOST_WIDE_INT *); |
d75596cd | 1275 | extern bool vect_analyze_data_ref_dependences (loop_vec_info, unsigned int *); |
c256513d | 1276 | extern bool vect_slp_analyze_instance_dependence (slp_instance); |
fb85abff | 1277 | extern bool vect_enhance_data_refs_alignment (loop_vec_info); |
2f6fec15 | 1278 | extern bool vect_analyze_data_refs_alignment (loop_vec_info); |
1279 | extern bool vect_verify_datarefs_alignment (loop_vec_info); | |
1280 | extern bool vect_slp_analyze_and_verify_instance_alignment (slp_instance); | |
e2c5c678 | 1281 | extern bool vect_analyze_data_ref_accesses (vec_info *); |
fb85abff | 1282 | extern bool vect_prune_runtime_alias_test_list (loop_vec_info); |
cf60da07 | 1283 | extern bool vect_check_gather_scatter (gimple *, loop_vec_info, |
1284 | gather_scatter_info *); | |
d75596cd | 1285 | extern bool vect_analyze_data_refs (vec_info *, poly_uint64 *); |
4f372c2c | 1286 | extern void vect_record_base_alignments (vec_info *); |
42acab1c | 1287 | extern tree vect_create_data_ref_ptr (gimple *, tree, struct loop *, tree, |
bd5ba09f | 1288 | tree *, gimple_stmt_iterator *, |
42acab1c | 1289 | gimple **, bool, bool *, |
1ec61bbd | 1290 | tree = NULL_TREE); |
42acab1c | 1291 | extern tree bump_vector_ptr (tree, gimple *, gimple_stmt_iterator *, gimple *, |
1292 | tree); | |
fb85abff | 1293 | extern tree vect_create_destination_var (tree, tree); |
ee612634 | 1294 | extern bool vect_grouped_store_supported (tree, unsigned HOST_WIDE_INT); |
94b7b4dd | 1295 | extern bool vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT); |
bc691ae4 | 1296 | extern bool vect_grouped_load_supported (tree, bool, unsigned HOST_WIDE_INT); |
94b7b4dd | 1297 | extern bool vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT); |
42acab1c | 1298 | extern void vect_permute_store_chain (vec<tree> ,unsigned int, gimple *, |
f1f41a6c | 1299 | gimple_stmt_iterator *, vec<tree> *); |
42acab1c | 1300 | extern tree vect_setup_realignment (gimple *, gimple_stmt_iterator *, tree *, |
48e1416a | 1301 | enum dr_alignment_support, tree, |
fb85abff | 1302 | struct loop **); |
42acab1c | 1303 | extern void vect_transform_grouped_load (gimple *, vec<tree> , int, |
fb85abff | 1304 | gimple_stmt_iterator *); |
42acab1c | 1305 | extern void vect_record_grouped_load_vectors (gimple *, vec<tree> ); |
fb85abff | 1306 | extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *); |
23ffec42 | 1307 | extern tree vect_get_new_ssa_name (tree, enum vect_var_kind, |
1308 | const char * = NULL); | |
42acab1c | 1309 | extern tree vect_create_addr_base_for_vector_ref (gimple *, gimple_seq *, |
9e879814 | 1310 | tree, tree = NULL_TREE); |
fb85abff | 1311 | |
1312 | /* In tree-vect-loop.c. */ | |
1313 | /* FORNOW: Used in tree-parloops.c. */ | |
119a8852 | 1314 | extern gimple *vect_force_simple_reduction (loop_vec_info, gimple *, |
42acab1c | 1315 | bool *, bool); |
5051abaf | 1316 | /* Used in gimple-loop-interchange.c. */ |
1317 | extern bool check_reduction_path (location_t, loop_p, gphi *, tree, | |
1318 | enum tree_code); | |
fb85abff | 1319 | /* Drive for loop analysis stage. */ |
5b631e09 | 1320 | extern loop_vec_info vect_analyze_loop (struct loop *, loop_vec_info); |
3a815241 | 1321 | extern tree vect_build_loop_niters (loop_vec_info, bool * = NULL); |
cde959e7 | 1322 | extern void vect_gen_vector_loop_niters (loop_vec_info, tree, tree *, |
1323 | tree *, bool); | |
fb85abff | 1324 | /* Drive for loop transformation stage. */ |
5b631e09 | 1325 | extern struct loop *vect_transform_loop (loop_vec_info); |
cb7f680b | 1326 | extern loop_vec_info vect_analyze_loop_form (struct loop *); |
42acab1c | 1327 | extern bool vectorizable_live_operation (gimple *, gimple_stmt_iterator *, |
75aae5b4 | 1328 | slp_tree, int, gimple **); |
42acab1c | 1329 | extern bool vectorizable_reduction (gimple *, gimple_stmt_iterator *, |
6154acba | 1330 | gimple **, slp_tree, slp_instance); |
5cc7beaa | 1331 | extern bool vectorizable_induction (gimple *, gimple_stmt_iterator *, |
1332 | gimple **, slp_tree); | |
42acab1c | 1333 | extern tree get_initial_def_for_reduction (gimple *, tree, tree *); |
fec8b6d0 | 1334 | extern bool vect_worthwhile_without_simd_p (vec_info *, tree_code); |
7a66d0cf | 1335 | extern int vect_get_known_peeling_cost (loop_vec_info, int, int *, |
1336 | stmt_vector_for_cost *, | |
f97dec81 | 1337 | stmt_vector_for_cost *, |
1338 | stmt_vector_for_cost *); | |
4a61a337 | 1339 | |
fb85abff | 1340 | /* In tree-vect-slp.c. */ |
1341 | extern void vect_free_slp_instance (slp_instance); | |
678e3d6e | 1342 | extern bool vect_transform_slp_perm_load (slp_tree, vec<tree> , |
d75596cd | 1343 | gimple_stmt_iterator *, poly_uint64, |
1344 | slp_instance, bool, unsigned *); | |
1c57101b | 1345 | extern bool vect_slp_analyze_operations (vec_info *); |
e2c5c678 | 1346 | extern bool vect_schedule_slp (vec_info *); |
1347 | extern bool vect_analyze_slp (vec_info *, unsigned); | |
bc937a44 | 1348 | extern bool vect_make_slp_decision (loop_vec_info); |
fb85abff | 1349 | extern void vect_detect_hybrid_slp (loop_vec_info); |
4f0d4cce | 1350 | extern void vect_get_slp_defs (vec<tree> , slp_tree, vec<vec<tree> > *); |
0a08c1bc | 1351 | extern bool vect_slp_bb (basic_block); |
77d241ed | 1352 | extern gimple *vect_find_last_scalar_stmt_in_slp (slp_tree); |
75aae5b4 | 1353 | extern bool is_simple_and_all_uses_invariant (gimple *, loop_vec_info); |
fb85abff | 1354 | |
1355 | /* In tree-vect-patterns.c. */ | |
4a61a337 | 1356 | /* Pattern recognition functions. |
1357 | Additional pattern recognition functions can (and will) be added | |
1358 | in the future. */ | |
42acab1c | 1359 | typedef gimple *(* vect_recog_func_ptr) (vec<gimple *> *, tree *, tree *); |
959c4b00 | 1360 | #define NUM_PATTERNS 14 |
e2c5c678 | 1361 | void vect_pattern_recog (vec_info *); |
4a61a337 | 1362 | |
10230637 | 1363 | /* In tree-vectorizer.c. */ |
1364 | unsigned vectorize_loops (void); | |
4c7587f5 | 1365 | bool vect_stmt_in_region_p (vec_info *, gimple *); |
d5e80d93 | 1366 | void vect_free_loop_info_assumptions (struct loop *); |
c91e8223 | 1367 | |
1368 | #endif /* GCC_TREE_VECTORIZER_H */ |