]>
Commit | Line | Data |
---|---|---|
ebfd146a | 1 | /* Vectorizer |
5624e564 | 2 | Copyright (C) 2003-2015 Free Software Foundation, Inc. |
79fe1b3b DN |
3 | Contributed by Dorit Naishlos <dorit@il.ibm.com> |
4 | ||
5 | This file is part of GCC. | |
6 | ||
7 | GCC is free software; you can redistribute it and/or modify it under | |
8 | the terms of the GNU General Public License as published by the Free | |
9dcd6f09 | 9 | Software Foundation; either version 3, or (at your option) any later |
79fe1b3b DN |
10 | version. |
11 | ||
12 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
13 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
14 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
15 | for more details. | |
16 | ||
17 | You should have received a copy of the GNU General Public License | |
9dcd6f09 NC |
18 | along with GCC; see the file COPYING3. If not see |
19 | <http://www.gnu.org/licenses/>. */ | |
79fe1b3b DN |
20 | |
21 | #ifndef GCC_TREE_VECTORIZER_H | |
22 | #define GCC_TREE_VECTORIZER_H | |
23 | ||
ebfd146a | 24 | #include "tree-data-ref.h" |
b6cef4e0 | 25 | #include "target.h" |
bf190e8d | 26 | #include "hash-table.h" |
ebfd146a | 27 | |
79fe1b3b DN |
28 | /* Used for naming of new temporaries. */ |
29 | enum vect_var_kind { | |
30 | vect_simple_var, | |
61d3cdbb DN |
31 | vect_pointer_var, |
32 | vect_scalar_var | |
79fe1b3b DN |
33 | }; |
34 | ||
20f06221 | 35 | /* Defines type of operation. */ |
79fe1b3b DN |
36 | enum operation_type { |
37 | unary_op = 1, | |
20f06221 DN |
38 | binary_op, |
39 | ternary_op | |
79fe1b3b DN |
40 | }; |
41 | ||
0dc0a70b DN |
42 | /* Define type of available alignment support. */ |
43 | enum dr_alignment_support { | |
44 | dr_unaligned_unsupported, | |
45 | dr_unaligned_supported, | |
468c2ac0 DN |
46 | dr_explicit_realign, |
47 | dr_explicit_realign_optimized, | |
0dc0a70b DN |
48 | dr_aligned |
49 | }; | |
50 | ||
607fb860 | 51 | /* Define type of def-use cross-iteration cycle. */ |
88088c03 | 52 | enum vect_def_type { |
81f40b79 | 53 | vect_uninitialized_def = 0, |
8644a673 IR |
54 | vect_constant_def = 1, |
55 | vect_external_def, | |
56 | vect_internal_def, | |
88088c03 DN |
57 | vect_induction_def, |
58 | vect_reduction_def, | |
06066f92 | 59 | vect_double_reduction_def, |
7c5222ff | 60 | vect_nested_cycle, |
88088c03 DN |
61 | vect_unknown_def_type |
62 | }; | |
63 | ||
99f51320 IR |
64 | #define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \ |
65 | || ((D) == vect_double_reduction_def) \ | |
66 | || ((D) == vect_nested_cycle)) | |
67 | ||
c3e7ee41 BS |
68 | /* Structure to encapsulate information about a group of like |
69 | instructions to be presented to the target cost model. */ | |
70 | typedef struct _stmt_info_for_cost { | |
71 | int count; | |
72 | enum vect_cost_for_stmt kind; | |
73 | gimple stmt; | |
74 | int misalign; | |
75 | } stmt_info_for_cost; | |
76 | ||
c3e7ee41 | 77 | |
9771b263 | 78 | typedef vec<stmt_info_for_cost> stmt_vector_for_cost; |
c3e7ee41 BS |
79 | |
80 | static inline void | |
81 | add_stmt_info_to_vec (stmt_vector_for_cost *stmt_cost_vec, int count, | |
82 | enum vect_cost_for_stmt kind, gimple stmt, int misalign) | |
83 | { | |
84 | stmt_info_for_cost si; | |
85 | si.count = count; | |
86 | si.kind = kind; | |
87 | si.stmt = stmt; | |
88 | si.misalign = misalign; | |
9771b263 | 89 | stmt_cost_vec->safe_push (si); |
c3e7ee41 BS |
90 | } |
91 | ||
805e2059 IR |
92 | /************************************************************************ |
93 | SLP | |
94 | ************************************************************************/ | |
d755c7ef | 95 | typedef struct _slp_tree *slp_tree; |
805e2059 | 96 | |
d092494c | 97 | /* A computation tree of an SLP instance. Each node corresponds to a group of |
805e2059 | 98 | stmts to be packed in a SIMD stmt. */ |
d755c7ef | 99 | struct _slp_tree { |
d092494c | 100 | /* Nodes that contain def-stmts of this node statements operands. */ |
d755c7ef | 101 | vec<slp_tree> children; |
805e2059 | 102 | /* A group of scalar stmts to be vectorized together. */ |
9771b263 | 103 | vec<gimple> stmts; |
01d8bf07 RB |
104 | /* Load permutation relative to the stores, NULL if there is no |
105 | permutation. */ | |
106 | vec<unsigned> load_permutation; | |
805e2059 | 107 | /* Vectorized stmt/s. */ |
9771b263 | 108 | vec<gimple> vec_stmts; |
b8698a0f L |
109 | /* Number of vector stmts that are created to replace the group of scalar |
110 | stmts. It is calculated during the transformation phase as the number of | |
111 | scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF | |
805e2059 IR |
112 | divided by vector size. */ |
113 | unsigned int vec_stmts_size; | |
d755c7ef | 114 | }; |
805e2059 IR |
115 | |
116 | ||
117 | /* SLP instance is a sequence of stmts in a loop that can be packed into | |
118 | SIMD stmts. */ | |
119 | typedef struct _slp_instance { | |
120 | /* The root of SLP tree. */ | |
121 | slp_tree root; | |
122 | ||
123 | /* Size of groups of scalar stmts that will be replaced by SIMD stmt/s. */ | |
124 | unsigned int group_size; | |
125 | ||
126 | /* The unrolling factor required to vectorized this SLP instance. */ | |
127 | unsigned int unrolling_factor; | |
128 | ||
129 | /* Vectorization costs associated with SLP instance. */ | |
92345349 | 130 | stmt_vector_for_cost body_cost_vec; |
c3e7ee41 | 131 | |
0fca40f5 | 132 | /* The group of nodes that contain loads of this SLP instance. */ |
9771b263 | 133 | vec<slp_tree> loads; |
5e038cad IR |
134 | |
135 | /* The first scalar load of the instance. The created vector loads will be | |
136 | inserted before this statement. */ | |
137 | gimple first_load; | |
805e2059 IR |
138 | } *slp_instance; |
139 | ||
805e2059 IR |
140 | |
141 | /* Access Functions. */ | |
142 | #define SLP_INSTANCE_TREE(S) (S)->root | |
143 | #define SLP_INSTANCE_GROUP_SIZE(S) (S)->group_size | |
144 | #define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor | |
92345349 | 145 | #define SLP_INSTANCE_BODY_COST_VEC(S) (S)->body_cost_vec |
0fca40f5 | 146 | #define SLP_INSTANCE_LOADS(S) (S)->loads |
5e038cad | 147 | #define SLP_INSTANCE_FIRST_LOAD_STMT(S) (S)->first_load |
805e2059 | 148 | |
d092494c | 149 | #define SLP_TREE_CHILDREN(S) (S)->children |
805e2059 IR |
150 | #define SLP_TREE_SCALAR_STMTS(S) (S)->stmts |
151 | #define SLP_TREE_VEC_STMTS(S) (S)->vec_stmts | |
152 | #define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size | |
01d8bf07 | 153 | #define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation |
805e2059 | 154 | |
d092494c IR |
155 | /* This structure is used in creation of an SLP tree. Each instance |
156 | corresponds to the same operand in a group of scalar stmts in an SLP | |
157 | node. */ | |
158 | typedef struct _slp_oprnd_info | |
159 | { | |
160 | /* Def-stmts for the operands. */ | |
9771b263 | 161 | vec<gimple> def_stmts; |
d092494c IR |
162 | /* Information about the first statement, its vector def-type, type, the |
163 | operand itself in case it's constant, and an indication if it's a pattern | |
164 | stmt. */ | |
165 | enum vect_def_type first_dt; | |
793d9a16 | 166 | tree first_op_type; |
d092494c IR |
167 | bool first_pattern; |
168 | } *slp_oprnd_info; | |
169 | ||
d092494c | 170 | |
720f5239 | 171 | |
a05a89fa | 172 | /* This struct is used to store the information of a data reference, |
93bdc3ed CH |
173 | including the data ref itself, the access offset (calculated by summing its |
174 | offset and init) and the segment length for aliasing checks. | |
175 | This is used to merge alias checks. */ | |
a05a89fa | 176 | |
93bdc3ed | 177 | struct dr_with_seg_len |
a05a89fa | 178 | { |
93bdc3ed CH |
179 | dr_with_seg_len (data_reference_p d, tree len) |
180 | : dr (d), | |
181 | offset (size_binop (PLUS_EXPR, DR_OFFSET (d), DR_INIT (d))), | |
182 | seg_len (len) {} | |
a05a89fa | 183 | |
93bdc3ed | 184 | data_reference_p dr; |
a05a89fa CH |
185 | tree offset; |
186 | tree seg_len; | |
187 | }; | |
188 | ||
93bdc3ed | 189 | /* This struct contains two dr_with_seg_len objects with aliasing data |
a05a89fa CH |
190 | refs. Two comparisons are generated from them. */ |
191 | ||
93bdc3ed | 192 | struct dr_with_seg_len_pair_t |
a05a89fa | 193 | { |
93bdc3ed CH |
194 | dr_with_seg_len_pair_t (const dr_with_seg_len& d1, |
195 | const dr_with_seg_len& d2) | |
a05a89fa CH |
196 | : first (d1), second (d2) {} |
197 | ||
93bdc3ed CH |
198 | dr_with_seg_len first; |
199 | dr_with_seg_len second; | |
a05a89fa CH |
200 | }; |
201 | ||
202 | ||
720f5239 IR |
203 | typedef struct _vect_peel_info |
204 | { | |
205 | int npeel; | |
206 | struct data_reference *dr; | |
207 | unsigned int count; | |
208 | } *vect_peel_info; | |
209 | ||
210 | typedef struct _vect_peel_extended_info | |
211 | { | |
212 | struct _vect_peel_info peel_info; | |
213 | unsigned int inside_cost; | |
214 | unsigned int outside_cost; | |
92345349 | 215 | stmt_vector_for_cost body_cost_vec; |
720f5239 IR |
216 | } *vect_peel_extended_info; |
217 | ||
bf190e8d LC |
218 | |
219 | /* Peeling hashtable helpers. */ | |
220 | ||
221 | struct peel_info_hasher : typed_free_remove <_vect_peel_info> | |
222 | { | |
67f58944 TS |
223 | typedef _vect_peel_info *value_type; |
224 | typedef _vect_peel_info *compare_type; | |
225 | static inline hashval_t hash (const _vect_peel_info *); | |
226 | static inline bool equal (const _vect_peel_info *, const _vect_peel_info *); | |
bf190e8d LC |
227 | }; |
228 | ||
229 | inline hashval_t | |
67f58944 | 230 | peel_info_hasher::hash (const _vect_peel_info *peel_info) |
bf190e8d LC |
231 | { |
232 | return (hashval_t) peel_info->npeel; | |
233 | } | |
234 | ||
235 | inline bool | |
67f58944 | 236 | peel_info_hasher::equal (const _vect_peel_info *a, const _vect_peel_info *b) |
bf190e8d LC |
237 | { |
238 | return (a->npeel == b->npeel); | |
239 | } | |
240 | ||
241 | ||
ef302293 LB |
242 | /*-----------------------------------------------------------------*/ |
243 | /* Info on vectorized loops. */ | |
244 | /*-----------------------------------------------------------------*/ | |
245 | typedef struct _loop_vec_info { | |
246 | ||
247 | /* The loop to which this info struct refers to. */ | |
248 | struct loop *loop; | |
249 | ||
250 | /* The loop basic blocks. */ | |
251 | basic_block *bbs; | |
252 | ||
95b3eff3 RB |
253 | /* Number of latch executions. */ |
254 | tree num_itersm1; | |
ef302293 LB |
255 | /* Number of iterations. */ |
256 | tree num_iters; | |
95b3eff3 | 257 | /* Number of iterations of the original loop. */ |
4f1f33aa | 258 | tree num_iters_unchanged; |
ef302293 | 259 | |
3a70f3ef | 260 | /* Minimum number of iterations below which vectorization is expected to |
b8698a0f | 261 | not be profitable (as estimated by the cost model). |
3a70f3ef DN |
262 | -1 indicates that vectorization will not be profitable. |
263 | FORNOW: This field is an int. Will be a tree in the future, to represent | |
b8698a0f L |
264 | values unknown at compile time. */ |
265 | int min_profitable_iters; | |
266 | ||
090cd8dc CH |
267 | /* Threshold of number of iterations below which vectorzation will not be |
268 | performed. It is calculated from MIN_PROFITABLE_ITERS and | |
269 | PARAM_MIN_VECT_LOOP_BOUND. */ | |
270 | unsigned int th; | |
271 | ||
ef302293 LB |
272 | /* Is the loop vectorizable? */ |
273 | bool vectorizable; | |
274 | ||
275 | /* Unrolling factor */ | |
276 | int vectorization_factor; | |
277 | ||
278 | /* Unknown DRs according to which loop was peeled. */ | |
279 | struct data_reference *unaligned_dr; | |
280 | ||
5f55a1ba DN |
281 | /* peeling_for_alignment indicates whether peeling for alignment will take |
282 | place, and what the peeling factor should be: | |
283 | peeling_for_alignment = X means: | |
284 | If X=0: Peeling for alignment will not be applied. | |
285 | If X>0: Peel first X iterations. | |
286 | If X=-1: Generate a runtime test to calculate the number of iterations | |
287 | to be peeled, using the dataref recorded in the field | |
288 | unaligned_dr. */ | |
289 | int peeling_for_alignment; | |
ef302293 | 290 | |
c12cc930 KB |
291 | /* The mask used to check the alignment of pointers or arrays. */ |
292 | int ptr_mask; | |
293 | ||
01be8516 | 294 | /* The loop nest in which the data dependences are computed. */ |
9771b263 | 295 | vec<loop_p> loop_nest; |
01be8516 | 296 | |
86a07404 | 297 | /* All data references in the loop. */ |
9771b263 | 298 | vec<data_reference_p> datarefs; |
ef302293 | 299 | |
86a07404 | 300 | /* All data dependences in the loop. */ |
9771b263 | 301 | vec<ddr_p> ddrs; |
ef302293 | 302 | |
bc1edb77 VK |
303 | /* Data Dependence Relations defining address ranges that are candidates |
304 | for a run-time aliasing check. */ | |
9771b263 | 305 | vec<ddr_p> may_alias_ddrs; |
bc1edb77 | 306 | |
a05a89fa CH |
307 | /* Data Dependence Relations defining address ranges together with segment |
308 | lengths from which the run-time aliasing check is built. */ | |
93bdc3ed | 309 | vec<dr_with_seg_len_pair_t> comp_alias_ddrs; |
a05a89fa | 310 | |
c12cc930 KB |
311 | /* Statements in the loop that have data references that are candidates for a |
312 | runtime (loop versioning) misalignment check. */ | |
9771b263 | 313 | vec<gimple> may_misalign_stmts; |
c12cc930 | 314 | |
805e2059 IR |
315 | /* All interleaving chains of stores in the loop, represented by the first |
316 | stmt in the chain. */ | |
9771b263 | 317 | vec<gimple> grouped_stores; |
805e2059 | 318 | |
0d0293ac | 319 | /* All SLP instances in the loop. This is a subset of the set of GROUP_STORES |
805e2059 | 320 | of the loop. */ |
9771b263 | 321 | vec<slp_instance> slp_instances; |
805e2059 | 322 | |
b8698a0f | 323 | /* The unrolling factor needed to SLP the loop. In case of that pure SLP is |
805e2059 IR |
324 | applied to the loop, i.e., no unrolling is needed, this is 1. */ |
325 | unsigned slp_unrolling_factor; | |
b5aeb3bb IR |
326 | |
327 | /* Reduction cycles detected in the loop. Used in loop-aware SLP. */ | |
9771b263 | 328 | vec<gimple> reductions; |
720f5239 | 329 | |
b010117a IR |
330 | /* All reduction chains in the loop, represented by the first |
331 | stmt in the chain. */ | |
9771b263 | 332 | vec<gimple> reduction_chains; |
b010117a | 333 | |
720f5239 | 334 | /* Hash table used to choose the best peeling option. */ |
c203e8a7 | 335 | hash_table<peel_info_hasher> *peeling_htab; |
720f5239 | 336 | |
c3e7ee41 BS |
337 | /* Cost data used by the target cost model. */ |
338 | void *target_cost_data; | |
339 | ||
0d0293ac | 340 | /* When we have grouped data accesses with gaps, we may introduce invalid |
48df3fa6 IR |
341 | memory accesses. We peel the last iteration of the loop to prevent |
342 | this. */ | |
343 | bool peeling_for_gaps; | |
344 | ||
b05e0233 RB |
345 | /* When the number of iterations is not a multiple of the vector size |
346 | we need to peel off iterations at the end to form an epilogue loop. */ | |
347 | bool peeling_for_niter; | |
348 | ||
3393a711 BS |
349 | /* Reductions are canonicalized so that the last operand is the reduction |
350 | operand. If this places a constant into RHS1, this decanonicalizes | |
351 | GIMPLE for other phases, so we must track when this has occurred and | |
352 | fix it up. */ | |
353 | bool operands_swapped; | |
354 | ||
d1417442 JJ |
355 | /* True if there are no loop carried data dependencies in the loop. |
356 | If loop->safelen <= 1, then this is always true, either the loop | |
357 | didn't have any loop carried data dependencies, or the loop is being | |
358 | vectorized guarded with some runtime alias checks, or couldn't | |
359 | be vectorized at all, but then this field shouldn't be used. | |
360 | For loop->safelen >= 2, the user has asserted that there are no | |
361 | backward dependencies, but there still could be loop carried forward | |
362 | dependencies in such loops. This flag will be false if normal | |
363 | vectorizer data dependency analysis would fail or require versioning | |
364 | for alias, but because of loop->safelen >= 2 it has been vectorized | |
365 | even without versioning for alias. E.g. in: | |
366 | #pragma omp simd | |
367 | for (int i = 0; i < m; i++) | |
368 | a[i] = a[i + k] * c; | |
369 | (or #pragma simd or #pragma ivdep) we can vectorize this and it will | |
370 | DTRT even for k > 0 && k < m, but without safelen we would not | |
371 | vectorize this, so this field would be false. */ | |
372 | bool no_data_dependencies; | |
373 | ||
5ce9450f JJ |
374 | /* If if-conversion versioned this loop before conversion, this is the |
375 | loop version without if-conversion. */ | |
376 | struct loop *scalar_loop; | |
377 | ||
ef302293 LB |
378 | } *loop_vec_info; |
379 | ||
c12cc930 | 380 | /* Access Functions. */ |
e9dbe7bb IR |
381 | #define LOOP_VINFO_LOOP(L) (L)->loop |
382 | #define LOOP_VINFO_BBS(L) (L)->bbs | |
95b3eff3 | 383 | #define LOOP_VINFO_NITERSM1(L) (L)->num_itersm1 |
e9dbe7bb | 384 | #define LOOP_VINFO_NITERS(L) (L)->num_iters |
95b3eff3 RB |
385 | /* Since LOOP_VINFO_NITERS and LOOP_VINFO_NITERSM1 can change after |
386 | prologue peeling retain total unchanged scalar loop iterations for | |
387 | cost model. */ | |
e9dbe7bb IR |
388 | #define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged |
389 | #define LOOP_VINFO_COST_MODEL_MIN_ITERS(L) (L)->min_profitable_iters | |
090cd8dc | 390 | #define LOOP_VINFO_COST_MODEL_THRESHOLD(L) (L)->th |
e9dbe7bb IR |
391 | #define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable |
392 | #define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor | |
393 | #define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask | |
01be8516 | 394 | #define LOOP_VINFO_LOOP_NEST(L) (L)->loop_nest |
e9dbe7bb IR |
395 | #define LOOP_VINFO_DATAREFS(L) (L)->datarefs |
396 | #define LOOP_VINFO_DDRS(L) (L)->ddrs | |
397 | #define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters)) | |
15e693cc | 398 | #define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment |
e9dbe7bb IR |
399 | #define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr |
400 | #define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts | |
e9dbe7bb | 401 | #define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs |
a05a89fa | 402 | #define LOOP_VINFO_COMP_ALIAS_DDRS(L) (L)->comp_alias_ddrs |
0d0293ac | 403 | #define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores |
e9dbe7bb | 404 | #define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances |
805e2059 | 405 | #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor |
b5aeb3bb | 406 | #define LOOP_VINFO_REDUCTIONS(L) (L)->reductions |
b010117a | 407 | #define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains |
720f5239 | 408 | #define LOOP_VINFO_PEELING_HTAB(L) (L)->peeling_htab |
c3e7ee41 | 409 | #define LOOP_VINFO_TARGET_COST_DATA(L) (L)->target_cost_data |
48df3fa6 | 410 | #define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps |
3393a711 | 411 | #define LOOP_VINFO_OPERANDS_SWAPPED(L) (L)->operands_swapped |
15e693cc | 412 | #define LOOP_VINFO_PEELING_FOR_NITER(L) (L)->peeling_for_niter |
d1417442 | 413 | #define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies |
5ce9450f | 414 | #define LOOP_VINFO_SCALAR_LOOP(L) (L)->scalar_loop |
ef302293 | 415 | |
e9dbe7bb | 416 | #define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \ |
ccc089dc | 417 | ((L)->may_misalign_stmts.length () > 0) |
e9dbe7bb | 418 | #define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \ |
ccc089dc | 419 | ((L)->may_alias_ddrs.length () > 0) |
28e44f4f | 420 | |
e9dbe7bb | 421 | #define LOOP_VINFO_NITERS_KNOWN_P(L) \ |
15e693cc | 422 | (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0) |
ef302293 | 423 | |
d29de1bf DN |
424 | static inline loop_vec_info |
425 | loop_vec_info_for_loop (struct loop *loop) | |
426 | { | |
427 | return (loop_vec_info) loop->aux; | |
428 | } | |
429 | ||
430 | static inline bool | |
726a989a | 431 | nested_in_vect_loop_p (struct loop *loop, gimple stmt) |
d29de1bf | 432 | { |
b8698a0f | 433 | return (loop->inner |
726a989a | 434 | && (loop->inner == (gimple_bb (stmt))->loop_father)); |
d29de1bf DN |
435 | } |
436 | ||
a70d6342 IR |
437 | typedef struct _bb_vec_info { |
438 | ||
439 | basic_block bb; | |
b8698a0f | 440 | /* All interleaving chains of stores in the basic block, represented by the |
a70d6342 | 441 | first stmt in the chain. */ |
9771b263 | 442 | vec<gimple> grouped_stores; |
a70d6342 | 443 | |
b8698a0f | 444 | /* All SLP instances in the basic block. This is a subset of the set of |
0d0293ac | 445 | GROUP_STORES of the basic block. */ |
9771b263 | 446 | vec<slp_instance> slp_instances; |
a70d6342 IR |
447 | |
448 | /* All data references in the basic block. */ | |
9771b263 | 449 | vec<data_reference_p> datarefs; |
a70d6342 IR |
450 | |
451 | /* All data dependences in the basic block. */ | |
9771b263 | 452 | vec<ddr_p> ddrs; |
c3e7ee41 BS |
453 | |
454 | /* Cost data used by the target cost model. */ | |
455 | void *target_cost_data; | |
456 | ||
a70d6342 IR |
457 | } *bb_vec_info; |
458 | ||
c3e7ee41 BS |
459 | #define BB_VINFO_BB(B) (B)->bb |
460 | #define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores | |
461 | #define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances | |
462 | #define BB_VINFO_DATAREFS(B) (B)->datarefs | |
463 | #define BB_VINFO_DDRS(B) (B)->ddrs | |
464 | #define BB_VINFO_TARGET_COST_DATA(B) (B)->target_cost_data | |
a70d6342 IR |
465 | |
466 | static inline bb_vec_info | |
467 | vec_info_for_bb (basic_block bb) | |
468 | { | |
469 | return (bb_vec_info) bb->aux; | |
470 | } | |
471 | ||
79fe1b3b DN |
472 | /*-----------------------------------------------------------------*/ |
473 | /* Info on vectorized defs. */ | |
474 | /*-----------------------------------------------------------------*/ | |
475 | enum stmt_vec_info_type { | |
476 | undef_vec_info_type = 0, | |
477 | load_vec_info_type, | |
478 | store_vec_info_type, | |
9dc3f7de | 479 | shift_vec_info_type, |
79fe1b3b | 480 | op_vec_info_type, |
2505a3f2 | 481 | call_vec_info_type, |
0136f8f0 | 482 | call_simd_clone_vec_info_type, |
b52485c6 | 483 | assignment_vec_info_type, |
61d3cdbb | 484 | condition_vec_info_type, |
89d67cca | 485 | reduc_vec_info_type, |
cd38ca7f | 486 | induc_vec_info_type, |
89d67cca | 487 | type_promotion_vec_info_type, |
f57d17f1 | 488 | type_demotion_vec_info_type, |
d29de1bf DN |
489 | type_conversion_vec_info_type, |
490 | loop_exit_ctrl_vec_info_type | |
89d67cca DN |
491 | }; |
492 | ||
b8698a0f | 493 | /* Indicates whether/how a variable is used in the scope of loop/basic |
8644a673 | 494 | block. */ |
89d67cca | 495 | enum vect_relevant { |
8644a673 | 496 | vect_unused_in_scope = 0, |
7c5222ff IR |
497 | /* The def is in the inner loop, and the use is in the outer loop, and the |
498 | use is a reduction stmt. */ | |
d29de1bf | 499 | vect_used_in_outer_by_reduction, |
7c5222ff IR |
500 | /* The def is in the inner loop, and the use is in the outer loop (and is |
501 | not part of reduction). */ | |
d29de1bf | 502 | vect_used_in_outer, |
b3832a9f DN |
503 | |
504 | /* defs that feed computations that end up (only) in a reduction. These | |
b8698a0f L |
505 | defs may be used by non-reduction stmts, but eventually, any |
506 | computations/values that are affected by these defs are used to compute | |
507 | a reduction (i.e. don't get stored to memory, for example). We use this | |
508 | to identify computations that we can change the order in which they are | |
b3832a9f | 509 | computed. */ |
89d67cca | 510 | vect_used_by_reduction, |
b3832a9f | 511 | |
b8698a0f | 512 | vect_used_in_scope |
79fe1b3b DN |
513 | }; |
514 | ||
805e2059 IR |
515 | /* The type of vectorization that can be applied to the stmt: regular loop-based |
516 | vectorization; pure SLP - the stmt is a part of SLP instances and does not | |
517 | have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is | |
518 | a part of SLP instance and also must be loop-based vectorized, since it has | |
b8698a0f L |
519 | uses outside SLP sequences. |
520 | ||
521 | In the loop context the meanings of pure and hybrid SLP are slightly | |
522 | different. By saying that pure SLP is applied to the loop, we mean that we | |
523 | exploit only intra-iteration parallelism in the loop; i.e., the loop can be | |
524 | vectorized without doing any conceptual unrolling, cause we don't pack | |
525 | together stmts from different iterations, only within a single iteration. | |
526 | Loop hybrid SLP means that we exploit both intra-iteration and | |
805e2059 | 527 | inter-iteration parallelism (e.g., number of elements in the vector is 4 |
b8698a0f L |
528 | and the slp-group-size is 2, in which case we don't have enough parallelism |
529 | within an iteration, so we obtain the rest of the parallelism from subsequent | |
805e2059 | 530 | iterations by unrolling the loop by 2). */ |
b8698a0f | 531 | enum slp_vect_type { |
805e2059 IR |
532 | loop_vect = 0, |
533 | pure_slp, | |
534 | hybrid | |
535 | }; | |
536 | ||
537 | ||
bb748329 | 538 | typedef struct data_reference *dr_p; |
bb748329 | 539 | |
79fe1b3b DN |
540 | typedef struct _stmt_vec_info { |
541 | ||
542 | enum stmt_vec_info_type type; | |
543 | ||
448f65db NF |
544 | /* Indicates whether this stmts is part of a computation whose result is |
545 | used outside the loop. */ | |
546 | bool live; | |
547 | ||
548 | /* Stmt is part of some pattern (computation idiom) */ | |
549 | bool in_pattern_p; | |
550 | ||
79fe1b3b | 551 | /* The stmt to which this info struct refers to. */ |
726a989a | 552 | gimple stmt; |
79fe1b3b | 553 | |
ef302293 LB |
554 | /* The loop_vec_info with respect to which STMT is vectorized. */ |
555 | loop_vec_info loop_vinfo; | |
79fe1b3b | 556 | |
b690cc0f | 557 | /* The vector type to be used for the LHS of this statement. */ |
79fe1b3b DN |
558 | tree vectype; |
559 | ||
560 | /* The vectorized version of the stmt. */ | |
726a989a | 561 | gimple vectorized_stmt; |
79fe1b3b DN |
562 | |
563 | ||
564 | /** The following is relevant only for stmts that contain a non-scalar | |
b8698a0f | 565 | data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have |
79fe1b3b DN |
566 | at most one such data-ref. **/ |
567 | ||
468c2ac0 DN |
568 | /* Information about the data-ref (access function, etc), |
569 | relative to the inner-most containing loop. */ | |
79fe1b3b DN |
570 | struct data_reference *data_ref_info; |
571 | ||
468c2ac0 DN |
572 | /* Information about the data-ref relative to this loop |
573 | nest (the loop that is being considered for vectorization). */ | |
574 | tree dr_base_address; | |
575 | tree dr_init; | |
576 | tree dr_offset; | |
577 | tree dr_step; | |
578 | tree dr_aligned_to; | |
579 | ||
0ac168a1 RG |
580 | /* For loop PHI nodes, the evolution part of it. This makes sure |
581 | this information is still available in vect_update_ivs_after_vectorizer | |
582 | where we may not be able to re-analyze the PHI nodes evolution as | |
583 | peeling for the prologue loop can make it unanalyzable. The evolution | |
584 | part is still correct though. */ | |
585 | tree loop_phi_evolution_part; | |
586 | ||
b8698a0f L |
587 | /* Used for various bookkeeping purposes, generally holding a pointer to |
588 | some other stmt S that is in some way "related" to this stmt. | |
20f06221 | 589 | Current use of this field is: |
b8698a0f L |
590 | If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is |
591 | true): S is the "pattern stmt" that represents (and replaces) the | |
592 | sequence of stmts that constitutes the pattern. Similarly, the | |
593 | related_stmt of the "pattern stmt" points back to this stmt (which is | |
594 | the last stmt in the original sequence of stmts that constitutes the | |
20f06221 | 595 | pattern). */ |
726a989a | 596 | gimple related_stmt; |
20f06221 | 597 | |
363477c0 JJ |
598 | /* Used to keep a sequence of def stmts of a pattern stmt if such exists. */ |
599 | gimple_seq pattern_def_seq; | |
1107f3ae | 600 | |
bb748329 DN |
601 | /* List of datarefs that are known to have the same alignment as the dataref |
602 | of this stmt. */ | |
9771b263 | 603 | vec<dr_p> same_align_refs; |
bb748329 | 604 | |
6c9e85fb JJ |
605 | /* Selected SIMD clone's function info. First vector element |
606 | is SIMD clone's function decl, followed by a pair of trees (base + step) | |
607 | for linear arguments (pair of NULLs for other arguments). */ | |
608 | vec<tree> simd_clone_info; | |
0136f8f0 | 609 | |
88088c03 DN |
610 | /* Classify the def of this stmt. */ |
611 | enum vect_def_type def_type; | |
612 | ||
448f65db NF |
613 | /* Whether the stmt is SLPed, loop-based vectorized, or both. */ |
614 | enum slp_vect_type slp_type; | |
615 | ||
e14c1050 IR |
616 | /* Interleaving and reduction chains info. */ |
617 | /* First element in the group. */ | |
618 | gimple first_element; | |
619 | /* Pointer to the next element in the group. */ | |
620 | gimple next_element; | |
621 | /* For data-refs, in case that two or more stmts share data-ref, this is the | |
622 | pointer to the previously detected stmt with the same dr. */ | |
448f65db | 623 | gimple same_dr_stmt; |
e14c1050 | 624 | /* The size of the group. */ |
98b44b0e IR |
625 | unsigned int size; |
626 | /* For stores, number of stores from this group seen. We vectorize the last | |
627 | one. */ | |
628 | unsigned int store_count; | |
629 | /* For loads only, the gap from the previous load. For consecutive loads, GAP | |
630 | is 1. */ | |
631 | unsigned int gap; | |
448f65db | 632 | |
f2556b68 RB |
633 | /* The minimum negative dependence distance this stmt participates in |
634 | or zero if none. */ | |
635 | unsigned int min_neg_dist; | |
636 | ||
448f65db NF |
637 | /* Not all stmts in the loop need to be vectorized. e.g, the increment |
638 | of the loop induction variable and computation of array indexes. relevant | |
639 | indicates whether the stmt needs to be vectorized. */ | |
640 | enum vect_relevant relevant; | |
792ed98b | 641 | |
a70d6342 IR |
642 | /* The bb_vec_info with respect to which STMT is vectorized. */ |
643 | bb_vec_info bb_vinfo; | |
4b5caab7 IR |
644 | |
645 | /* Is this statement vectorizable or should it be skipped in (partial) | |
646 | vectorization. */ | |
647 | bool vectorizable; | |
aec7ae7d JJ |
648 | |
649 | /* For loads only, true if this is a gather load. */ | |
650 | bool gather_p; | |
7d75abc8 | 651 | bool stride_load_p; |
74bf76ed JJ |
652 | |
653 | /* For both loads and stores. */ | |
654 | bool simd_lane_access_p; | |
79fe1b3b DN |
655 | } *stmt_vec_info; |
656 | ||
657 | /* Access Functions. */ | |
98b44b0e IR |
658 | #define STMT_VINFO_TYPE(S) (S)->type |
659 | #define STMT_VINFO_STMT(S) (S)->stmt | |
660 | #define STMT_VINFO_LOOP_VINFO(S) (S)->loop_vinfo | |
a70d6342 | 661 | #define STMT_VINFO_BB_VINFO(S) (S)->bb_vinfo |
98b44b0e IR |
662 | #define STMT_VINFO_RELEVANT(S) (S)->relevant |
663 | #define STMT_VINFO_LIVE_P(S) (S)->live | |
664 | #define STMT_VINFO_VECTYPE(S) (S)->vectype | |
665 | #define STMT_VINFO_VEC_STMT(S) (S)->vectorized_stmt | |
4b5caab7 | 666 | #define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable |
98b44b0e | 667 | #define STMT_VINFO_DATA_REF(S) (S)->data_ref_info |
aec7ae7d | 668 | #define STMT_VINFO_GATHER_P(S) (S)->gather_p |
7d75abc8 | 669 | #define STMT_VINFO_STRIDE_LOAD_P(S) (S)->stride_load_p |
74bf76ed | 670 | #define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p |
468c2ac0 DN |
671 | |
672 | #define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_base_address | |
673 | #define STMT_VINFO_DR_INIT(S) (S)->dr_init | |
674 | #define STMT_VINFO_DR_OFFSET(S) (S)->dr_offset | |
675 | #define STMT_VINFO_DR_STEP(S) (S)->dr_step | |
676 | #define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_aligned_to | |
677 | ||
98b44b0e IR |
678 | #define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p |
679 | #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt | |
363477c0 | 680 | #define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq |
98b44b0e | 681 | #define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs |
6c9e85fb | 682 | #define STMT_VINFO_SIMD_CLONE_INFO(S) (S)->simd_clone_info |
98b44b0e | 683 | #define STMT_VINFO_DEF_TYPE(S) (S)->def_type |
e14c1050 IR |
684 | #define STMT_VINFO_GROUP_FIRST_ELEMENT(S) (S)->first_element |
685 | #define STMT_VINFO_GROUP_NEXT_ELEMENT(S) (S)->next_element | |
686 | #define STMT_VINFO_GROUP_SIZE(S) (S)->size | |
687 | #define STMT_VINFO_GROUP_STORE_COUNT(S) (S)->store_count | |
688 | #define STMT_VINFO_GROUP_GAP(S) (S)->gap | |
689 | #define STMT_VINFO_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt | |
0d0293ac | 690 | #define STMT_VINFO_GROUPED_ACCESS(S) ((S)->first_element != NULL && (S)->data_ref_info) |
0ac168a1 | 691 | #define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part |
f2556b68 | 692 | #define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist |
e14c1050 IR |
693 | |
694 | #define GROUP_FIRST_ELEMENT(S) (S)->first_element | |
695 | #define GROUP_NEXT_ELEMENT(S) (S)->next_element | |
696 | #define GROUP_SIZE(S) (S)->size | |
697 | #define GROUP_STORE_COUNT(S) (S)->store_count | |
698 | #define GROUP_GAP(S) (S)->gap | |
699 | #define GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt | |
79fe1b3b | 700 | |
8644a673 | 701 | #define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope) |
792ed98b | 702 | |
805e2059 IR |
703 | #define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid) |
704 | #define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp) | |
705 | #define STMT_SLP_TYPE(S) (S)->slp_type | |
706 | ||
c716e67f XDL |
707 | struct dataref_aux { |
708 | tree base_decl; | |
709 | bool base_misaligned; | |
710 | int misalignment; | |
711 | }; | |
712 | ||
720f5239 IR |
713 | #define VECT_MAX_COST 1000 |
714 | ||
5d593372 IR |
715 | /* The maximum number of intermediate steps required in multi-step type |
716 | conversion. */ | |
717 | #define MAX_INTERM_CVT_STEPS 3 | |
718 | ||
d3c2fee0 AI |
719 | /* The maximum vectorization factor supported by any target (V64QI). */ |
720 | #define MAX_VECTORIZATION_FACTOR 64 | |
777e1f09 | 721 | |
726a989a RB |
722 | /* Avoid GTY(()) on stmt_vec_info. */ |
723 | typedef void *vec_void_p; | |
79fe1b3b | 724 | |
9771b263 | 725 | extern vec<vec_void_p> stmt_vec_info_vec; |
726a989a RB |
726 | |
727 | void init_stmt_vec_info_vec (void); | |
728 | void free_stmt_vec_info_vec (void); | |
729 | ||
ff802fa1 IR |
730 | /* Return a stmt_vec_info corresponding to STMT. */ |
731 | ||
726a989a RB |
732 | static inline stmt_vec_info |
733 | vinfo_for_stmt (gimple stmt) | |
79fe1b3b | 734 | { |
726a989a RB |
735 | unsigned int uid = gimple_uid (stmt); |
736 | if (uid == 0) | |
737 | return NULL; | |
738 | ||
9771b263 | 739 | return (stmt_vec_info) stmt_vec_info_vec[uid - 1]; |
79fe1b3b DN |
740 | } |
741 | ||
ff802fa1 IR |
742 | /* Set vectorizer information INFO for STMT. */ |
743 | ||
726a989a RB |
744 | static inline void |
745 | set_vinfo_for_stmt (gimple stmt, stmt_vec_info info) | |
79fe1b3b | 746 | { |
726a989a RB |
747 | unsigned int uid = gimple_uid (stmt); |
748 | if (uid == 0) | |
749 | { | |
06795261 | 750 | gcc_checking_assert (info); |
9771b263 | 751 | uid = stmt_vec_info_vec.length () + 1; |
726a989a | 752 | gimple_set_uid (stmt, uid); |
9771b263 | 753 | stmt_vec_info_vec.safe_push ((vec_void_p) info); |
726a989a RB |
754 | } |
755 | else | |
9771b263 | 756 | stmt_vec_info_vec[uid - 1] = (vec_void_p) info; |
79fe1b3b DN |
757 | } |
758 | ||
ff802fa1 IR |
759 | /* Return the earlier statement between STMT1 and STMT2. */ |
760 | ||
5e038cad IR |
761 | static inline gimple |
762 | get_earlier_stmt (gimple stmt1, gimple stmt2) | |
763 | { | |
764 | unsigned int uid1, uid2; | |
765 | ||
766 | if (stmt1 == NULL) | |
767 | return stmt2; | |
768 | ||
769 | if (stmt2 == NULL) | |
770 | return stmt1; | |
771 | ||
772 | uid1 = gimple_uid (stmt1); | |
773 | uid2 = gimple_uid (stmt2); | |
774 | ||
775 | if (uid1 == 0 || uid2 == 0) | |
776 | return NULL; | |
777 | ||
9771b263 DN |
778 | gcc_checking_assert (uid1 <= stmt_vec_info_vec.length () |
779 | && uid2 <= stmt_vec_info_vec.length ()); | |
5e038cad IR |
780 | |
781 | if (uid1 < uid2) | |
782 | return stmt1; | |
783 | else | |
784 | return stmt2; | |
785 | } | |
786 | ||
ff802fa1 IR |
787 | /* Return the later statement between STMT1 and STMT2. */ |
788 | ||
e4a707c4 IR |
789 | static inline gimple |
790 | get_later_stmt (gimple stmt1, gimple stmt2) | |
791 | { | |
792 | unsigned int uid1, uid2; | |
793 | ||
794 | if (stmt1 == NULL) | |
795 | return stmt2; | |
796 | ||
797 | if (stmt2 == NULL) | |
798 | return stmt1; | |
799 | ||
800 | uid1 = gimple_uid (stmt1); | |
801 | uid2 = gimple_uid (stmt2); | |
802 | ||
803 | if (uid1 == 0 || uid2 == 0) | |
804 | return NULL; | |
805 | ||
9771b263 DN |
806 | gcc_assert (uid1 <= stmt_vec_info_vec.length ()); |
807 | gcc_assert (uid2 <= stmt_vec_info_vec.length ()); | |
e4a707c4 IR |
808 | |
809 | if (uid1 > uid2) | |
810 | return stmt1; | |
811 | else | |
812 | return stmt2; | |
813 | } | |
814 | ||
ff802fa1 IR |
815 | /* Return TRUE if a statement represented by STMT_INFO is a part of a |
816 | pattern. */ | |
817 | ||
878aa817 DN |
818 | static inline bool |
819 | is_pattern_stmt_p (stmt_vec_info stmt_info) | |
820 | { | |
726a989a | 821 | gimple related_stmt; |
878aa817 DN |
822 | stmt_vec_info related_stmt_info; |
823 | ||
824 | related_stmt = STMT_VINFO_RELATED_STMT (stmt_info); | |
825 | if (related_stmt | |
826 | && (related_stmt_info = vinfo_for_stmt (related_stmt)) | |
827 | && STMT_VINFO_IN_PATTERN_P (related_stmt_info)) | |
828 | return true; | |
829 | ||
830 | return false; | |
831 | } | |
832 | ||
ff802fa1 IR |
833 | /* Return true if BB is a loop header. */ |
834 | ||
d29de1bf DN |
835 | static inline bool |
836 | is_loop_header_bb_p (basic_block bb) | |
837 | { | |
838 | if (bb == (bb->loop_father)->header) | |
839 | return true; | |
06795261 | 840 | gcc_checking_assert (EDGE_COUNT (bb->preds) == 1); |
d29de1bf DN |
841 | return false; |
842 | } | |
843 | ||
ff802fa1 IR |
844 | /* Return pow2 (X). */ |
845 | ||
5d593372 IR |
846 | static inline int |
847 | vect_pow2 (int x) | |
848 | { | |
849 | int i, res = 1; | |
850 | ||
851 | for (i = 0; i < x; i++) | |
852 | res *= 2; | |
853 | ||
854 | return res; | |
855 | } | |
89675e73 | 856 | |
92345349 BS |
857 | /* Alias targetm.vectorize.builtin_vectorization_cost. */ |
858 | ||
859 | static inline int | |
860 | builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost, | |
861 | tree vectype, int misalign) | |
862 | { | |
863 | return targetm.vectorize.builtin_vectorization_cost (type_of_cost, | |
864 | vectype, misalign); | |
865 | } | |
866 | ||
b6cef4e0 BS |
867 | /* Get cost by calling cost target builtin. */ |
868 | ||
869 | static inline | |
870 | int vect_get_stmt_cost (enum vect_cost_for_stmt type_of_cost) | |
871 | { | |
92345349 | 872 | return builtin_vectorization_cost (type_of_cost, NULL, 0); |
b6cef4e0 BS |
873 | } |
874 | ||
c3e7ee41 BS |
875 | /* Alias targetm.vectorize.init_cost. */ |
876 | ||
877 | static inline void * | |
878 | init_cost (struct loop *loop_info) | |
879 | { | |
880 | return targetm.vectorize.init_cost (loop_info); | |
881 | } | |
882 | ||
883 | /* Alias targetm.vectorize.add_stmt_cost. */ | |
884 | ||
885 | static inline unsigned | |
886 | add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind, | |
92345349 BS |
887 | stmt_vec_info stmt_info, int misalign, |
888 | enum vect_cost_model_location where) | |
c3e7ee41 BS |
889 | { |
890 | return targetm.vectorize.add_stmt_cost (data, count, kind, | |
92345349 | 891 | stmt_info, misalign, where); |
c3e7ee41 BS |
892 | } |
893 | ||
894 | /* Alias targetm.vectorize.finish_cost. */ | |
895 | ||
92345349 BS |
896 | static inline void |
897 | finish_cost (void *data, unsigned *prologue_cost, | |
898 | unsigned *body_cost, unsigned *epilogue_cost) | |
c3e7ee41 | 899 | { |
92345349 | 900 | targetm.vectorize.finish_cost (data, prologue_cost, body_cost, epilogue_cost); |
c3e7ee41 BS |
901 | } |
902 | ||
903 | /* Alias targetm.vectorize.destroy_cost_data. */ | |
904 | ||
905 | static inline void | |
906 | destroy_cost_data (void *data) | |
907 | { | |
908 | targetm.vectorize.destroy_cost_data (data); | |
909 | } | |
910 | ||
911 | ||
79fe1b3b DN |
912 | /*-----------------------------------------------------------------*/ |
913 | /* Info on data references alignment. */ | |
914 | /*-----------------------------------------------------------------*/ | |
c716e67f XDL |
915 | inline void |
916 | set_dr_misalignment (struct data_reference *dr, int val) | |
917 | { | |
918 | dataref_aux *data_aux = (dataref_aux *) dr->aux; | |
919 | ||
920 | if (!data_aux) | |
921 | { | |
922 | data_aux = XCNEW (dataref_aux); | |
923 | dr->aux = data_aux; | |
924 | } | |
925 | ||
926 | data_aux->misalignment = val; | |
927 | } | |
928 | ||
929 | inline int | |
930 | dr_misalignment (struct data_reference *dr) | |
931 | { | |
932 | gcc_assert (dr->aux); | |
933 | return ((dataref_aux *) dr->aux)->misalignment; | |
934 | } | |
79fe1b3b | 935 | |
5f55a1ba DN |
936 | /* Reflects actual alignment of first access in the vectorized loop, |
937 | taking into account peeling/versioning if applied. */ | |
c716e67f XDL |
938 | #define DR_MISALIGNMENT(DR) dr_misalignment (DR) |
939 | #define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL) | |
79fe1b3b | 940 | |
ff802fa1 IR |
941 | /* Return TRUE if the data access is aligned, and FALSE otherwise. */ |
942 | ||
79fe1b3b DN |
943 | static inline bool |
944 | aligned_access_p (struct data_reference *data_ref_info) | |
945 | { | |
946 | return (DR_MISALIGNMENT (data_ref_info) == 0); | |
947 | } | |
948 | ||
ff802fa1 IR |
949 | /* Return TRUE if the alignment of the data access is known, and FALSE |
950 | otherwise. */ | |
951 | ||
79fe1b3b | 952 | static inline bool |
5f55a1ba | 953 | known_alignment_for_access_p (struct data_reference *data_ref_info) |
79fe1b3b | 954 | { |
5f55a1ba | 955 | return (DR_MISALIGNMENT (data_ref_info) != -1); |
79fe1b3b DN |
956 | } |
957 | ||
d6d11272 XDL |
958 | |
959 | /* Return true if the vect cost model is unlimited. */ | |
960 | static inline bool | |
8b5e1202 | 961 | unlimited_cost_model (loop_p loop) |
d6d11272 | 962 | { |
b15b5979 | 963 | if (loop != NULL && loop->force_vectorize |
8b5e1202 SO |
964 | && flag_simd_cost_model != VECT_COST_MODEL_DEFAULT) |
965 | return flag_simd_cost_model == VECT_COST_MODEL_UNLIMITED; | |
966 | return (flag_vect_cost_model == VECT_COST_MODEL_UNLIMITED); | |
d6d11272 XDL |
967 | } |
968 | ||
78c60e3d | 969 | /* Source location */ |
b05e0233 | 970 | extern source_location vect_location; |
ebfd146a | 971 | |
79fe1b3b DN |
972 | /*-----------------------------------------------------------------*/ |
973 | /* Function prototypes. */ | |
974 | /*-----------------------------------------------------------------*/ | |
975 | ||
b8698a0f | 976 | /* Simple loop peeling and versioning utilities for vectorizer's purposes - |
ebfd146a | 977 | in tree-vect-loop-manip.c. */ |
f7064d11 | 978 | extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree); |
22ea9ec0 | 979 | extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge); |
5ce9450f JJ |
980 | struct loop *slpeel_tree_duplicate_loop_to_edge_cfg (struct loop *, |
981 | struct loop *, edge); | |
368117e8 | 982 | extern void vect_loop_versioning (loop_vec_info, unsigned int, bool); |
f3c92486 | 983 | extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree, tree, |
368117e8 | 984 | unsigned int, bool); |
f3c92486 RB |
985 | extern void vect_do_peeling_for_alignment (loop_vec_info, tree, |
986 | unsigned int, bool); | |
b05e0233 | 987 | extern source_location find_loop_location (struct loop *); |
ebfd146a | 988 | extern bool vect_can_advance_ivs_p (loop_vec_info); |
79fe1b3b | 989 | |
ebfd146a | 990 | /* In tree-vect-stmts.c. */ |
bb67d9c7 | 991 | extern unsigned int current_vector_size; |
f7064d11 | 992 | extern tree get_vectype_for_scalar_type (tree); |
b690cc0f | 993 | extern tree get_same_sized_vectype (tree, tree); |
24ee1384 IR |
994 | extern bool vect_is_simple_use (tree, gimple, loop_vec_info, |
995 | bb_vec_info, gimple *, | |
a70d6342 | 996 | tree *, enum vect_def_type *); |
24ee1384 IR |
997 | extern bool vect_is_simple_use_1 (tree, gimple, loop_vec_info, |
998 | bb_vec_info, gimple *, | |
b690cc0f RG |
999 | tree *, enum vect_def_type *, tree *); |
1000 | extern bool supportable_widening_operation (enum tree_code, gimple, tree, tree, | |
a86ec597 | 1001 | enum tree_code *, enum tree_code *, |
9771b263 | 1002 | int *, vec<tree> *); |
b690cc0f RG |
1003 | extern bool supportable_narrowing_operation (enum tree_code, tree, tree, |
1004 | enum tree_code *, | |
9771b263 | 1005 | int *, vec<tree> *); |
b8698a0f | 1006 | extern stmt_vec_info new_stmt_vec_info (gimple stmt, loop_vec_info, |
a70d6342 | 1007 | bb_vec_info); |
726a989a | 1008 | extern void free_stmt_vec_info (gimple stmt); |
538dd0b7 | 1009 | extern tree vectorizable_function (gcall *, tree, tree); |
ebfd146a | 1010 | extern void vect_model_simple_cost (stmt_vec_info, int, enum vect_def_type *, |
92345349 BS |
1011 | stmt_vector_for_cost *, |
1012 | stmt_vector_for_cost *); | |
272c6793 | 1013 | extern void vect_model_store_cost (stmt_vec_info, int, bool, |
c3e7ee41 | 1014 | enum vect_def_type, slp_tree, |
92345349 | 1015 | stmt_vector_for_cost *, |
c3e7ee41 BS |
1016 | stmt_vector_for_cost *); |
1017 | extern void vect_model_load_cost (stmt_vec_info, int, bool, slp_tree, | |
92345349 | 1018 | stmt_vector_for_cost *, |
c3e7ee41 BS |
1019 | stmt_vector_for_cost *); |
1020 | extern unsigned record_stmt_cost (stmt_vector_for_cost *, int, | |
92345349 BS |
1021 | enum vect_cost_for_stmt, stmt_vec_info, |
1022 | int, enum vect_cost_model_location); | |
ebfd146a IR |
1023 | extern void vect_finish_stmt_generation (gimple, gimple, |
1024 | gimple_stmt_iterator *); | |
1025 | extern bool vect_mark_stmts_to_be_vectorized (loop_vec_info); | |
ebfd146a IR |
1026 | extern tree vect_get_vec_def_for_operand (tree, gimple, tree *); |
1027 | extern tree vect_init_vector (gimple, tree, tree, | |
1028 | gimple_stmt_iterator *); | |
1029 | extern tree vect_get_vec_def_for_stmt_copy (enum vect_def_type, tree); | |
1030 | extern bool vect_transform_stmt (gimple, gimple_stmt_iterator *, | |
1031 | bool *, slp_tree, slp_instance); | |
1032 | extern void vect_remove_stores (gimple); | |
a70d6342 | 1033 | extern bool vect_analyze_stmt (gimple, bool *, slp_tree); |
b8698a0f | 1034 | extern bool vectorizable_condition (gimple, gimple_stmt_iterator *, gimple *, |
f7e531cf | 1035 | tree, int, slp_tree); |
720f5239 | 1036 | extern void vect_get_load_cost (struct data_reference *, int, bool, |
c3e7ee41 | 1037 | unsigned int *, unsigned int *, |
92345349 BS |
1038 | stmt_vector_for_cost *, |
1039 | stmt_vector_for_cost *, bool); | |
c3e7ee41 BS |
1040 | extern void vect_get_store_cost (struct data_reference *, int, |
1041 | unsigned int *, stmt_vector_for_cost *); | |
1107f3ae | 1042 | extern bool vect_supportable_shift (enum tree_code, tree); |
9771b263 DN |
1043 | extern void vect_get_vec_defs (tree, tree, gimple, vec<tree> *, |
1044 | vec<tree> *, slp_tree, int); | |
557be5a8 AL |
1045 | extern tree vect_gen_perm_mask_any (tree, const unsigned char *); |
1046 | extern tree vect_gen_perm_mask_checked (tree, const unsigned char *); | |
b8698a0f | 1047 | |
ebfd146a IR |
1048 | /* In tree-vect-data-refs.c. */ |
1049 | extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int); | |
1050 | extern enum dr_alignment_support vect_supportable_dr_alignment | |
720f5239 | 1051 | (struct data_reference *, bool); |
ebfd146a IR |
1052 | extern tree vect_get_smallest_scalar_type (gimple, HOST_WIDE_INT *, |
1053 | HOST_WIDE_INT *); | |
5abe1e05 RB |
1054 | extern bool vect_analyze_data_ref_dependences (loop_vec_info, int *); |
1055 | extern bool vect_slp_analyze_data_ref_dependences (bb_vec_info); | |
ebfd146a | 1056 | extern bool vect_enhance_data_refs_alignment (loop_vec_info); |
a70d6342 IR |
1057 | extern bool vect_analyze_data_refs_alignment (loop_vec_info, bb_vec_info); |
1058 | extern bool vect_verify_datarefs_alignment (loop_vec_info, bb_vec_info); | |
1059 | extern bool vect_analyze_data_ref_accesses (loop_vec_info, bb_vec_info); | |
ebfd146a | 1060 | extern bool vect_prune_runtime_alias_test_list (loop_vec_info); |
aec7ae7d JJ |
1061 | extern tree vect_check_gather (gimple, loop_vec_info, tree *, tree *, |
1062 | int *); | |
1428105c RB |
1063 | extern bool vect_analyze_data_refs (loop_vec_info, bb_vec_info, int *, |
1064 | unsigned *); | |
920e8172 RS |
1065 | extern tree vect_create_data_ref_ptr (gimple, tree, struct loop *, tree, |
1066 | tree *, gimple_stmt_iterator *, | |
356bbc4c JJ |
1067 | gimple *, bool, bool *, |
1068 | tree = NULL_TREE); | |
ebfd146a IR |
1069 | extern tree bump_vector_ptr (tree, gimple, gimple_stmt_iterator *, gimple, tree); |
1070 | extern tree vect_create_destination_var (tree, tree); | |
0d0293ac | 1071 | extern bool vect_grouped_store_supported (tree, unsigned HOST_WIDE_INT); |
272c6793 | 1072 | extern bool vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT); |
0d0293ac | 1073 | extern bool vect_grouped_load_supported (tree, unsigned HOST_WIDE_INT); |
272c6793 | 1074 | extern bool vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT); |
9771b263 DN |
1075 | extern void vect_permute_store_chain (vec<tree> ,unsigned int, gimple, |
1076 | gimple_stmt_iterator *, vec<tree> *); | |
ebfd146a | 1077 | extern tree vect_setup_realignment (gimple, gimple_stmt_iterator *, tree *, |
b8698a0f | 1078 | enum dr_alignment_support, tree, |
ebfd146a | 1079 | struct loop **); |
9771b263 | 1080 | extern void vect_transform_grouped_load (gimple, vec<tree> , int, |
ebfd146a | 1081 | gimple_stmt_iterator *); |
9771b263 | 1082 | extern void vect_record_grouped_load_vectors (gimple, vec<tree> ); |
ebfd146a IR |
1083 | extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *); |
1084 | extern tree vect_create_addr_base_for_vector_ref (gimple, gimple_seq *, | |
356bbc4c JJ |
1085 | tree, struct loop *, |
1086 | tree = NULL_TREE); | |
ebfd146a IR |
1087 | |
1088 | /* In tree-vect-loop.c. */ | |
1089 | /* FORNOW: Used in tree-parloops.c. */ | |
1090 | extern void destroy_loop_vec_info (loop_vec_info, bool); | |
8a9ecffd | 1091 | extern gimple vect_force_simple_reduction (loop_vec_info, gimple, bool, bool *); |
ebfd146a | 1092 | /* Drive for loop analysis stage. */ |
f7064d11 | 1093 | extern loop_vec_info vect_analyze_loop (struct loop *); |
ebfd146a IR |
1094 | /* Drive for loop transformation stage. */ |
1095 | extern void vect_transform_loop (loop_vec_info); | |
a509ebb5 | 1096 | extern loop_vec_info vect_analyze_loop_form (struct loop *); |
ebfd146a IR |
1097 | extern bool vectorizable_live_operation (gimple, gimple_stmt_iterator *, |
1098 | gimple *); | |
b5aeb3bb IR |
1099 | extern bool vectorizable_reduction (gimple, gimple_stmt_iterator *, gimple *, |
1100 | slp_tree); | |
ebfd146a | 1101 | extern bool vectorizable_induction (gimple, gimple_stmt_iterator *, gimple *); |
ebfd146a IR |
1102 | extern tree get_initial_def_for_reduction (gimple, tree, tree *); |
1103 | extern int vect_min_worthwhile_factor (enum tree_code); | |
30c5a937 RB |
1104 | extern int vect_get_known_peeling_cost (loop_vec_info, int, int *, |
1105 | stmt_vector_for_cost *, | |
92345349 BS |
1106 | stmt_vector_for_cost *, |
1107 | stmt_vector_for_cost *); | |
30c5a937 RB |
1108 | extern int vect_get_single_scalar_iteration_cost (loop_vec_info, |
1109 | stmt_vector_for_cost *); | |
20f06221 | 1110 | |
ebfd146a IR |
1111 | /* In tree-vect-slp.c. */ |
1112 | extern void vect_free_slp_instance (slp_instance); | |
01d8bf07 | 1113 | extern bool vect_transform_slp_perm_load (slp_tree, vec<tree> , |
b8698a0f | 1114 | gimple_stmt_iterator *, int, |
ebfd146a | 1115 | slp_instance, bool); |
a70d6342 | 1116 | extern bool vect_schedule_slp (loop_vec_info, bb_vec_info); |
ebfd146a | 1117 | extern void vect_update_slp_costs_according_to_vf (loop_vec_info); |
1428105c | 1118 | extern bool vect_analyze_slp (loop_vec_info, bb_vec_info, unsigned); |
437f4a00 | 1119 | extern bool vect_make_slp_decision (loop_vec_info); |
ebfd146a | 1120 | extern void vect_detect_hybrid_slp (loop_vec_info); |
9771b263 | 1121 | extern void vect_get_slp_defs (vec<tree> , slp_tree, |
37b5ec8f | 1122 | vec<vec<tree> > *, int); |
d092494c | 1123 | |
b05e0233 | 1124 | extern source_location find_bb_location (basic_block); |
a70d6342 IR |
1125 | extern bb_vec_info vect_slp_analyze_bb (basic_block); |
1126 | extern void vect_slp_transform_bb (basic_block); | |
ebfd146a IR |
1127 | |
1128 | /* In tree-vect-patterns.c. */ | |
20f06221 DN |
1129 | /* Pattern recognition functions. |
1130 | Additional pattern recognition functions can (and will) be added | |
1131 | in the future. */ | |
9771b263 | 1132 | typedef gimple (* vect_recog_func_ptr) (vec<gimple> *, tree *, tree *); |
79d652a5 | 1133 | #define NUM_PATTERNS 12 |
f5709183 | 1134 | void vect_pattern_recog (loop_vec_info, bb_vec_info); |
20f06221 | 1135 | |
8ed77e22 SZ |
1136 | /* In tree-vectorizer.c. */ |
1137 | unsigned vectorize_loops (void); | |
c716e67f | 1138 | void vect_destroy_datarefs (loop_vec_info, bb_vec_info); |
79fe1b3b DN |
1139 | |
1140 | #endif /* GCC_TREE_VECTORIZER_H */ |