]>
Commit | Line | Data |
---|---|---|
ebfd146a | 1 | /* Vectorizer |
d1e082c2 | 2 | Copyright (C) 2003-2013 Free Software Foundation, Inc. |
b8698a0f | 3 | Contributed by Dorit Naishlos <dorit@il.ibm.com> |
79fe1b3b DN |
4 | |
5 | This file is part of GCC. | |
6 | ||
7 | GCC is free software; you can redistribute it and/or modify it under | |
8 | the terms of the GNU General Public License as published by the Free | |
9dcd6f09 | 9 | Software Foundation; either version 3, or (at your option) any later |
79fe1b3b DN |
10 | version. |
11 | ||
12 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
13 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
14 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
15 | for more details. | |
16 | ||
17 | You should have received a copy of the GNU General Public License | |
9dcd6f09 NC |
18 | along with GCC; see the file COPYING3. If not see |
19 | <http://www.gnu.org/licenses/>. */ | |
79fe1b3b | 20 | |
ebfd146a | 21 | /* Loop and basic block vectorizer. |
7ccf35ed | 22 | |
b8698a0f L |
23 | This file contains drivers for the three vectorizers: |
24 | (1) loop vectorizer (inter-iteration parallelism), | |
ebfd146a IR |
25 | (2) loop-aware SLP (intra-iteration parallelism) (invoked by the loop |
26 | vectorizer) | |
27 | (3) BB vectorizer (out-of-loops), aka SLP | |
b8698a0f | 28 | |
ebfd146a | 29 | The rest of the vectorizer's code is organized as follows: |
b8698a0f L |
30 | - tree-vect-loop.c - loop specific parts such as reductions, etc. These are |
31 | used by drivers (1) and (2). | |
32 | - tree-vect-loop-manip.c - vectorizer's loop control-flow utilities, used by | |
33 | drivers (1) and (2). | |
34 | - tree-vect-slp.c - BB vectorization specific analysis and transformation, | |
ebfd146a IR |
35 | used by drivers (2) and (3). |
36 | - tree-vect-stmts.c - statements analysis and transformation (used by all). | |
b8698a0f | 37 | - tree-vect-data-refs.c - vectorizer specific data-refs analysis and |
ebfd146a IR |
38 | manipulations (used by all). |
39 | - tree-vect-patterns.c - vectorizable code patterns detector (used by all) | |
40 | ||
41 | Here's a poor attempt at illustrating that: | |
42 | ||
43 | tree-vectorizer.c: | |
44 | loop_vect() loop_aware_slp() slp_vect() | |
45 | | / \ / | |
46 | | / \ / | |
47 | tree-vect-loop.c tree-vect-slp.c | |
48 | | \ \ / / | | |
49 | | \ \/ / | | |
50 | | \ /\ / | | |
51 | | \ / \ / | | |
52 | tree-vect-stmts.c tree-vect-data-refs.c | |
53 | \ / | |
54 | tree-vect-patterns.c | |
55 | */ | |
89d67cca | 56 | |
ebfd146a IR |
57 | #include "config.h" |
58 | #include "system.h" | |
59 | #include "coretypes.h" | |
78c60e3d | 60 | #include "dumpfile.h" |
ebfd146a IR |
61 | #include "tm.h" |
62 | #include "ggc.h" | |
63 | #include "tree.h" | |
cf835838 | 64 | #include "tree-pretty-print.h" |
442b4905 AM |
65 | #include "gimple.h" |
66 | #include "gimple-ssa.h" | |
67 | #include "cgraph.h" | |
68 | #include "tree-phinodes.h" | |
69 | #include "ssa-iterators.h" | |
70 | #include "tree-ssa-loop.h" | |
ebfd146a | 71 | #include "cfgloop.h" |
ebfd146a IR |
72 | #include "tree-vectorizer.h" |
73 | #include "tree-pass.h" | |
74bf76ed JJ |
74 | #include "hash-table.h" |
75 | #include "tree-ssa-propagate.h" | |
c716e67f | 76 | #include "dbgcnt.h" |
89d67cca | 77 | |
a70d6342 | 78 | /* Loop or bb location. */ |
8644a673 | 79 | LOC vect_location; |
ad2dd72a | 80 | |
ebfd146a | 81 | /* Vector mapping GIMPLE stmt to stmt_vec_info. */ |
9771b263 | 82 | vec<vec_void_p> stmt_vec_info_vec; |
74bf76ed JJ |
83 | \f |
84 | /* For mapping simduid to vectorization factor. */ | |
85 | ||
86 | struct simduid_to_vf : typed_free_remove<simduid_to_vf> | |
87 | { | |
88 | unsigned int simduid; | |
89 | int vf; | |
90 | ||
91 | /* hash_table support. */ | |
92 | typedef simduid_to_vf value_type; | |
93 | typedef simduid_to_vf compare_type; | |
94 | static inline hashval_t hash (const value_type *); | |
95 | static inline int equal (const value_type *, const compare_type *); | |
96 | }; | |
97 | ||
98 | inline hashval_t | |
99 | simduid_to_vf::hash (const value_type *p) | |
100 | { | |
101 | return p->simduid; | |
102 | } | |
103 | ||
104 | inline int | |
105 | simduid_to_vf::equal (const value_type *p1, const value_type *p2) | |
106 | { | |
107 | return p1->simduid == p2->simduid; | |
108 | } | |
109 | ||
110 | /* This hash maps the OMP simd array to the corresponding simduid used | |
111 | to index into it. Like thus, | |
112 | ||
113 | _7 = GOMP_SIMD_LANE (simduid.0) | |
114 | ... | |
115 | ... | |
116 | D.1737[_7] = stuff; | |
117 | ||
118 | ||
acf0174b JJ |
119 | This hash maps from the OMP simd array (D.1737[]) to DECL_UID of |
120 | simduid.0. */ | |
74bf76ed JJ |
121 | |
122 | struct simd_array_to_simduid : typed_free_remove<simd_array_to_simduid> | |
123 | { | |
124 | tree decl; | |
125 | unsigned int simduid; | |
126 | ||
127 | /* hash_table support. */ | |
128 | typedef simd_array_to_simduid value_type; | |
129 | typedef simd_array_to_simduid compare_type; | |
130 | static inline hashval_t hash (const value_type *); | |
131 | static inline int equal (const value_type *, const compare_type *); | |
132 | }; | |
133 | ||
134 | inline hashval_t | |
135 | simd_array_to_simduid::hash (const value_type *p) | |
136 | { | |
137 | return DECL_UID (p->decl); | |
138 | } | |
139 | ||
140 | inline int | |
141 | simd_array_to_simduid::equal (const value_type *p1, const value_type *p2) | |
142 | { | |
143 | return p1->decl == p2->decl; | |
144 | } | |
145 | ||
146 | /* Fold IFN_GOMP_SIMD_LANE, IFN_GOMP_SIMD_VF and IFN_GOMP_SIMD_LAST_LANE | |
147 | into their corresponding constants. */ | |
148 | ||
149 | static void | |
150 | adjust_simduid_builtins (hash_table <simduid_to_vf> &htab) | |
151 | { | |
152 | basic_block bb; | |
153 | ||
154 | FOR_EACH_BB (bb) | |
155 | { | |
156 | gimple_stmt_iterator i; | |
157 | ||
158 | for (i = gsi_start_bb (bb); !gsi_end_p (i); gsi_next (&i)) | |
159 | { | |
160 | unsigned int vf = 1; | |
161 | enum internal_fn ifn; | |
162 | gimple stmt = gsi_stmt (i); | |
163 | tree t; | |
164 | if (!is_gimple_call (stmt) | |
165 | || !gimple_call_internal_p (stmt)) | |
166 | continue; | |
167 | ifn = gimple_call_internal_fn (stmt); | |
168 | switch (ifn) | |
169 | { | |
170 | case IFN_GOMP_SIMD_LANE: | |
171 | case IFN_GOMP_SIMD_VF: | |
172 | case IFN_GOMP_SIMD_LAST_LANE: | |
173 | break; | |
174 | default: | |
175 | continue; | |
176 | } | |
177 | tree arg = gimple_call_arg (stmt, 0); | |
178 | gcc_assert (arg != NULL_TREE); | |
179 | gcc_assert (TREE_CODE (arg) == SSA_NAME); | |
180 | simduid_to_vf *p = NULL, data; | |
181 | data.simduid = DECL_UID (SSA_NAME_VAR (arg)); | |
182 | if (htab.is_created ()) | |
183 | p = htab.find (&data); | |
184 | if (p) | |
185 | vf = p->vf; | |
186 | switch (ifn) | |
187 | { | |
188 | case IFN_GOMP_SIMD_VF: | |
189 | t = build_int_cst (unsigned_type_node, vf); | |
190 | break; | |
191 | case IFN_GOMP_SIMD_LANE: | |
192 | t = build_int_cst (unsigned_type_node, 0); | |
193 | break; | |
194 | case IFN_GOMP_SIMD_LAST_LANE: | |
195 | t = gimple_call_arg (stmt, 1); | |
196 | break; | |
197 | default: | |
198 | gcc_unreachable (); | |
199 | } | |
200 | update_call_from_tree (&i, t); | |
201 | } | |
202 | } | |
203 | } | |
89d67cca | 204 | |
74bf76ed JJ |
205 | /* Helper structure for note_simd_array_uses. */ |
206 | ||
207 | struct note_simd_array_uses_struct | |
208 | { | |
209 | hash_table <simd_array_to_simduid> *htab; | |
210 | unsigned int simduid; | |
211 | }; | |
212 | ||
213 | /* Callback for note_simd_array_uses, called through walk_gimple_op. */ | |
214 | ||
215 | static tree | |
216 | note_simd_array_uses_cb (tree *tp, int *walk_subtrees, void *data) | |
217 | { | |
218 | struct walk_stmt_info *wi = (struct walk_stmt_info *) data; | |
219 | struct note_simd_array_uses_struct *ns | |
220 | = (struct note_simd_array_uses_struct *) wi->info; | |
221 | ||
222 | if (TYPE_P (*tp)) | |
223 | *walk_subtrees = 0; | |
224 | else if (VAR_P (*tp) | |
225 | && lookup_attribute ("omp simd array", DECL_ATTRIBUTES (*tp)) | |
226 | && DECL_CONTEXT (*tp) == current_function_decl) | |
227 | { | |
228 | simd_array_to_simduid data; | |
229 | if (!ns->htab->is_created ()) | |
230 | ns->htab->create (15); | |
231 | data.decl = *tp; | |
232 | data.simduid = ns->simduid; | |
233 | simd_array_to_simduid **slot = ns->htab->find_slot (&data, INSERT); | |
234 | if (*slot == NULL) | |
235 | { | |
236 | simd_array_to_simduid *p = XNEW (simd_array_to_simduid); | |
237 | *p = data; | |
238 | *slot = p; | |
239 | } | |
240 | else if ((*slot)->simduid != ns->simduid) | |
241 | (*slot)->simduid = -1U; | |
242 | *walk_subtrees = 0; | |
243 | } | |
244 | return NULL_TREE; | |
245 | } | |
246 | ||
247 | /* Find "omp simd array" temporaries and map them to corresponding | |
248 | simduid. */ | |
249 | ||
250 | static void | |
251 | note_simd_array_uses (hash_table <simd_array_to_simduid> *htab) | |
252 | { | |
253 | basic_block bb; | |
254 | gimple_stmt_iterator gsi; | |
255 | struct walk_stmt_info wi; | |
256 | struct note_simd_array_uses_struct ns; | |
257 | ||
258 | memset (&wi, 0, sizeof (wi)); | |
259 | wi.info = &ns; | |
260 | ns.htab = htab; | |
261 | ||
262 | FOR_EACH_BB (bb) | |
263 | for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) | |
264 | { | |
265 | gimple stmt = gsi_stmt (gsi); | |
266 | if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt)) | |
267 | continue; | |
268 | switch (gimple_call_internal_fn (stmt)) | |
269 | { | |
270 | case IFN_GOMP_SIMD_LANE: | |
271 | case IFN_GOMP_SIMD_VF: | |
272 | case IFN_GOMP_SIMD_LAST_LANE: | |
273 | break; | |
274 | default: | |
275 | continue; | |
276 | } | |
277 | tree lhs = gimple_call_lhs (stmt); | |
278 | if (lhs == NULL_TREE) | |
279 | continue; | |
280 | imm_use_iterator use_iter; | |
281 | gimple use_stmt; | |
282 | ns.simduid = DECL_UID (SSA_NAME_VAR (gimple_call_arg (stmt, 0))); | |
283 | FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, lhs) | |
284 | if (!is_gimple_debug (use_stmt)) | |
285 | walk_gimple_op (use_stmt, note_simd_array_uses_cb, &wi); | |
286 | } | |
287 | } | |
ebfd146a | 288 | \f |
c716e67f XDL |
289 | /* A helper function to free data refs. */ |
290 | ||
291 | void | |
292 | vect_destroy_datarefs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo) | |
293 | { | |
294 | vec<data_reference_p> datarefs; | |
295 | struct data_reference *dr; | |
296 | unsigned int i; | |
297 | ||
298 | if (loop_vinfo) | |
299 | datarefs = LOOP_VINFO_DATAREFS (loop_vinfo); | |
300 | else | |
301 | datarefs = BB_VINFO_DATAREFS (bb_vinfo); | |
302 | ||
303 | FOR_EACH_VEC_ELT (datarefs, i, dr) | |
304 | if (dr->aux) | |
305 | { | |
306 | free (dr->aux); | |
307 | dr->aux = NULL; | |
308 | } | |
309 | ||
310 | free_data_refs (datarefs); | |
311 | } | |
312 | ||
313 | ||
79fe1b3b | 314 | /* Function vectorize_loops. |
b8698a0f | 315 | |
8644a673 | 316 | Entry point to loop vectorization phase. */ |
79fe1b3b | 317 | |
4d2280f6 | 318 | unsigned |
d73be268 | 319 | vectorize_loops (void) |
79fe1b3b | 320 | { |
b52485c6 | 321 | unsigned int i; |
79fe1b3b | 322 | unsigned int num_vectorized_loops = 0; |
42fd6772 ZD |
323 | unsigned int vect_loops_num; |
324 | loop_iterator li; | |
325 | struct loop *loop; | |
74bf76ed JJ |
326 | hash_table <simduid_to_vf> simduid_to_vf_htab; |
327 | hash_table <simd_array_to_simduid> simd_array_to_simduid_htab; | |
79fe1b3b | 328 | |
0fc822d0 | 329 | vect_loops_num = number_of_loops (cfun); |
f9be04cd RG |
330 | |
331 | /* Bail out if there are no loops. */ | |
332 | if (vect_loops_num <= 1) | |
74bf76ed JJ |
333 | { |
334 | if (cfun->has_simduid_loops) | |
335 | adjust_simduid_builtins (simduid_to_vf_htab); | |
336 | return 0; | |
337 | } | |
338 | ||
339 | if (cfun->has_simduid_loops) | |
340 | note_simd_array_uses (&simd_array_to_simduid_htab); | |
f9be04cd | 341 | |
726a989a RB |
342 | init_stmt_vec_info_vec (); |
343 | ||
79fe1b3b DN |
344 | /* ----------- Analyze loops. ----------- */ |
345 | ||
b8698a0f | 346 | /* If some loop was duplicated, it gets bigger number |
ff802fa1 | 347 | than all previously defined loops. This fact allows us to run |
79fe1b3b | 348 | only over initial loops skipping newly generated ones. */ |
677cc14d | 349 | FOR_EACH_LOOP (li, loop, 0) |
ea0f3e87 | 350 | if ((flag_tree_loop_vectorize && optimize_loop_nest_for_speed_p (loop)) |
74bf76ed | 351 | || loop->force_vect) |
8bcf15f6 JH |
352 | { |
353 | loop_vec_info loop_vinfo; | |
8644a673 | 354 | vect_location = find_loop_location (loop); |
84df911b | 355 | if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOC |
73fbfcad | 356 | && dump_enabled_p ()) |
ccb3ad87 | 357 | dump_printf (MSG_NOTE, "\nAnalyzing loop at %s:%d\n", |
78c60e3d | 358 | LOC_FILE (vect_location), LOC_LINE (vect_location)); |
7cd3603b | 359 | |
8bcf15f6 JH |
360 | loop_vinfo = vect_analyze_loop (loop); |
361 | loop->aux = loop_vinfo; | |
79fe1b3b | 362 | |
8bcf15f6 JH |
363 | if (!loop_vinfo || !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo)) |
364 | continue; | |
79fe1b3b | 365 | |
c716e67f XDL |
366 | if (!dbg_cnt (vect_loop)) |
367 | break; | |
368 | ||
84df911b | 369 | if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOC |
73fbfcad | 370 | && dump_enabled_p ()) |
5d318fd4 | 371 | dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location, |
9cc1fb4b | 372 | "loop vectorized\n"); |
8bcf15f6 JH |
373 | vect_transform_loop (loop_vinfo); |
374 | num_vectorized_loops++; | |
74bf76ed JJ |
375 | /* Now that the loop has been vectorized, allow it to be unrolled |
376 | etc. */ | |
377 | loop->force_vect = false; | |
378 | ||
379 | if (loop->simduid) | |
380 | { | |
381 | simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf); | |
382 | if (!simduid_to_vf_htab.is_created ()) | |
383 | simduid_to_vf_htab.create (15); | |
384 | simduid_to_vf_data->simduid = DECL_UID (loop->simduid); | |
385 | simduid_to_vf_data->vf = loop_vinfo->vectorization_factor; | |
386 | *simduid_to_vf_htab.find_slot (simduid_to_vf_data, INSERT) | |
387 | = simduid_to_vf_data; | |
388 | } | |
8bcf15f6 | 389 | } |
8644a673 IR |
390 | |
391 | vect_location = UNKNOWN_LOC; | |
79fe1b3b | 392 | |
01902653 | 393 | statistics_counter_event (cfun, "Vectorized loops", num_vectorized_loops); |
73fbfcad SS |
394 | if (dump_enabled_p () |
395 | || (num_vectorized_loops > 0 && dump_enabled_p ())) | |
ccb3ad87 | 396 | dump_printf_loc (MSG_NOTE, vect_location, |
78c60e3d SS |
397 | "vectorized %u loops in function.\n", |
398 | num_vectorized_loops); | |
79fe1b3b DN |
399 | |
400 | /* ----------- Finalize. ----------- */ | |
401 | ||
b52485c6 | 402 | for (i = 1; i < vect_loops_num; i++) |
79fe1b3b | 403 | { |
6775f1f3 IR |
404 | loop_vec_info loop_vinfo; |
405 | ||
0fc822d0 | 406 | loop = get_loop (cfun, i); |
79fe1b3b | 407 | if (!loop) |
6775f1f3 | 408 | continue; |
3d9a9f94 | 409 | loop_vinfo = (loop_vec_info) loop->aux; |
d29de1bf | 410 | destroy_loop_vec_info (loop_vinfo, true); |
79fe1b3b DN |
411 | loop->aux = NULL; |
412 | } | |
4d2280f6 | 413 | |
726a989a RB |
414 | free_stmt_vec_info_vec (); |
415 | ||
74bf76ed JJ |
416 | /* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE} builtins. */ |
417 | if (cfun->has_simduid_loops) | |
418 | adjust_simduid_builtins (simduid_to_vf_htab); | |
419 | ||
420 | /* Shrink any "omp array simd" temporary arrays to the | |
421 | actual vectorization factors. */ | |
422 | if (simd_array_to_simduid_htab.is_created ()) | |
423 | { | |
424 | for (hash_table <simd_array_to_simduid>::iterator iter | |
425 | = simd_array_to_simduid_htab.begin (); | |
426 | iter != simd_array_to_simduid_htab.end (); ++iter) | |
427 | if ((*iter).simduid != -1U) | |
428 | { | |
429 | tree decl = (*iter).decl; | |
430 | int vf = 1; | |
431 | if (simduid_to_vf_htab.is_created ()) | |
432 | { | |
433 | simduid_to_vf *p = NULL, data; | |
434 | data.simduid = (*iter).simduid; | |
435 | p = simduid_to_vf_htab.find (&data); | |
436 | if (p) | |
437 | vf = p->vf; | |
438 | } | |
439 | tree atype | |
440 | = build_array_type_nelts (TREE_TYPE (TREE_TYPE (decl)), vf); | |
441 | TREE_TYPE (decl) = atype; | |
442 | relayout_decl (decl); | |
443 | } | |
444 | ||
445 | simd_array_to_simduid_htab.dispose (); | |
446 | } | |
447 | if (simduid_to_vf_htab.is_created ()) | |
448 | simduid_to_vf_htab.dispose (); | |
449 | ||
789c34e3 RB |
450 | if (num_vectorized_loops > 0) |
451 | { | |
452 | /* If we vectorized any loop only virtual SSA form needs to be updated. | |
453 | ??? Also while we try hard to update loop-closed SSA form we fail | |
454 | to properly do this in some corner-cases (see PR56286). */ | |
455 | rewrite_into_loop_closed_ssa (NULL, TODO_update_ssa_only_virtuals); | |
456 | return TODO_cleanup_cfg; | |
457 | } | |
458 | ||
459 | return 0; | |
79fe1b3b | 460 | } |
b8698a0f | 461 | |
f4b3ca72 | 462 | |
a70d6342 IR |
463 | /* Entry point to basic block SLP phase. */ |
464 | ||
465 | static unsigned int | |
466 | execute_vect_slp (void) | |
467 | { | |
468 | basic_block bb; | |
469 | ||
a70d6342 IR |
470 | init_stmt_vec_info_vec (); |
471 | ||
472 | FOR_EACH_BB (bb) | |
473 | { | |
474 | vect_location = find_bb_location (bb); | |
475 | ||
476 | if (vect_slp_analyze_bb (bb)) | |
477 | { | |
c716e67f XDL |
478 | if (!dbg_cnt (vect_slp)) |
479 | break; | |
480 | ||
a70d6342 | 481 | vect_slp_transform_bb (bb); |
73fbfcad | 482 | if (dump_enabled_p ()) |
5d318fd4 | 483 | dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location, |
9cc1fb4b | 484 | "basic block vectorized\n"); |
a70d6342 IR |
485 | } |
486 | } | |
487 | ||
488 | free_stmt_vec_info_vec (); | |
489 | return 0; | |
490 | } | |
491 | ||
492 | static bool | |
493 | gate_vect_slp (void) | |
494 | { | |
ea0f3e87 | 495 | return flag_tree_slp_vectorize != 0; |
a70d6342 IR |
496 | } |
497 | ||
27a4cd48 DM |
498 | namespace { |
499 | ||
500 | const pass_data pass_data_slp_vectorize = | |
a70d6342 | 501 | { |
27a4cd48 DM |
502 | GIMPLE_PASS, /* type */ |
503 | "slp", /* name */ | |
504 | OPTGROUP_LOOP | OPTGROUP_VEC, /* optinfo_flags */ | |
505 | true, /* has_gate */ | |
506 | true, /* has_execute */ | |
507 | TV_TREE_SLP_VECTORIZATION, /* tv_id */ | |
508 | ( PROP_ssa | PROP_cfg ), /* properties_required */ | |
509 | 0, /* properties_provided */ | |
510 | 0, /* properties_destroyed */ | |
511 | 0, /* todo_flags_start */ | |
512 | ( TODO_verify_ssa | TODO_update_ssa | |
513 | | TODO_verify_stmts ), /* todo_flags_finish */ | |
a70d6342 IR |
514 | }; |
515 | ||
27a4cd48 DM |
516 | class pass_slp_vectorize : public gimple_opt_pass |
517 | { | |
518 | public: | |
c3284718 RS |
519 | pass_slp_vectorize (gcc::context *ctxt) |
520 | : gimple_opt_pass (pass_data_slp_vectorize, ctxt) | |
27a4cd48 DM |
521 | {} |
522 | ||
523 | /* opt_pass methods: */ | |
524 | bool gate () { return gate_vect_slp (); } | |
525 | unsigned int execute () { return execute_vect_slp (); } | |
526 | ||
527 | }; // class pass_slp_vectorize | |
528 | ||
529 | } // anon namespace | |
530 | ||
531 | gimple_opt_pass * | |
532 | make_pass_slp_vectorize (gcc::context *ctxt) | |
533 | { | |
534 | return new pass_slp_vectorize (ctxt); | |
535 | } | |
536 | ||
a70d6342 | 537 | |
f4b3ca72 JH |
538 | /* Increase alignment of global arrays to improve vectorization potential. |
539 | TODO: | |
540 | - Consider also structs that have an array field. | |
541 | - Use ipa analysis to prune arrays that can't be vectorized? | |
542 | This should involve global alignment analysis and in the future also | |
543 | array padding. */ | |
544 | ||
545 | static unsigned int | |
546 | increase_alignment (void) | |
547 | { | |
548 | struct varpool_node *vnode; | |
549 | ||
a3d7af04 SS |
550 | vect_location = UNKNOWN_LOC; |
551 | ||
f4b3ca72 | 552 | /* Increase the alignment of all global arrays for vectorization. */ |
65c70e6b | 553 | FOR_EACH_DEFINED_VARIABLE (vnode) |
f4b3ca72 | 554 | { |
960bfb69 | 555 | tree vectype, decl = vnode->symbol.decl; |
cba146eb | 556 | tree t; |
f4b3ca72 JH |
557 | unsigned int alignment; |
558 | ||
c3284718 | 559 | t = TREE_TYPE (decl); |
cba146eb | 560 | if (TREE_CODE (t) != ARRAY_TYPE) |
ebfd146a | 561 | continue; |
cba146eb | 562 | vectype = get_vectype_for_scalar_type (strip_array_types (t)); |
f4b3ca72 | 563 | if (!vectype) |
ebfd146a | 564 | continue; |
f4b3ca72 JH |
565 | alignment = TYPE_ALIGN (vectype); |
566 | if (DECL_ALIGN (decl) >= alignment) | |
ebfd146a | 567 | continue; |
f4b3ca72 JH |
568 | |
569 | if (vect_can_force_dr_alignment_p (decl, alignment)) | |
ebfd146a IR |
570 | { |
571 | DECL_ALIGN (decl) = TYPE_ALIGN (vectype); | |
572 | DECL_USER_ALIGN (decl) = 1; | |
78c60e3d SS |
573 | dump_printf (MSG_NOTE, "Increasing alignment of decl: "); |
574 | dump_generic_expr (MSG_NOTE, TDF_SLIM, decl); | |
575 | dump_printf (MSG_NOTE, "\n"); | |
ebfd146a | 576 | } |
f4b3ca72 JH |
577 | } |
578 | return 0; | |
579 | } | |
580 | ||
ebfd146a | 581 | |
fe1c7546 | 582 | static bool |
f4b3ca72 JH |
583 | gate_increase_alignment (void) |
584 | { | |
ea0f3e87 | 585 | return flag_section_anchors && flag_tree_loop_vectorize; |
f4b3ca72 JH |
586 | } |
587 | ||
ebfd146a | 588 | |
27a4cd48 DM |
589 | namespace { |
590 | ||
591 | const pass_data pass_data_ipa_increase_alignment = | |
f4b3ca72 | 592 | { |
27a4cd48 DM |
593 | SIMPLE_IPA_PASS, /* type */ |
594 | "increase_alignment", /* name */ | |
595 | OPTGROUP_LOOP | OPTGROUP_VEC, /* optinfo_flags */ | |
596 | true, /* has_gate */ | |
597 | true, /* has_execute */ | |
598 | TV_IPA_OPT, /* tv_id */ | |
599 | 0, /* properties_required */ | |
600 | 0, /* properties_provided */ | |
601 | 0, /* properties_destroyed */ | |
602 | 0, /* todo_flags_start */ | |
603 | 0, /* todo_flags_finish */ | |
f4b3ca72 | 604 | }; |
27a4cd48 DM |
605 | |
606 | class pass_ipa_increase_alignment : public simple_ipa_opt_pass | |
607 | { | |
608 | public: | |
c3284718 RS |
609 | pass_ipa_increase_alignment (gcc::context *ctxt) |
610 | : simple_ipa_opt_pass (pass_data_ipa_increase_alignment, ctxt) | |
27a4cd48 DM |
611 | {} |
612 | ||
613 | /* opt_pass methods: */ | |
614 | bool gate () { return gate_increase_alignment (); } | |
615 | unsigned int execute () { return increase_alignment (); } | |
616 | ||
617 | }; // class pass_ipa_increase_alignment | |
618 | ||
619 | } // anon namespace | |
620 | ||
621 | simple_ipa_opt_pass * | |
622 | make_pass_ipa_increase_alignment (gcc::context *ctxt) | |
623 | { | |
624 | return new pass_ipa_increase_alignment (ctxt); | |
625 | } |