2 Copyright (C) 2003-2021 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Loop and basic block vectorizer.
23 This file contains drivers for the three vectorizers:
24 (1) loop vectorizer (inter-iteration parallelism),
25 (2) loop-aware SLP (intra-iteration parallelism) (invoked by the loop
27 (3) BB vectorizer (out-of-loops), aka SLP
29 The rest of the vectorizer's code is organized as follows:
30 - tree-vect-loop.c - loop specific parts such as reductions, etc. These are
31 used by drivers (1) and (2).
32 - tree-vect-loop-manip.c - vectorizer's loop control-flow utilities, used by
34 - tree-vect-slp.c - BB vectorization specific analysis and transformation,
35 used by drivers (2) and (3).
36 - tree-vect-stmts.c - statements analysis and transformation (used by all).
37 - tree-vect-data-refs.c - vectorizer specific data-refs analysis and
38 manipulations (used by all).
39 - tree-vect-patterns.c - vectorizable code patterns detector (used by all)
41 Here's a poor attempt at illustrating that:
44 loop_vect() loop_aware_slp() slp_vect()
47 tree-vect-loop.c tree-vect-slp.c
52 tree-vect-stmts.c tree-vect-data-refs.c
59 #include "coretypes.h"
64 #include "tree-pass.h"
67 #include "fold-const.h"
68 #include "stor-layout.h"
69 #include "gimple-iterator.h"
70 #include "gimple-walk.h"
71 #include "tree-ssa-loop-manip.h"
72 #include "tree-ssa-loop-niter.h"
75 #include "tree-vectorizer.h"
76 #include "tree-ssa-propagate.h"
78 #include "tree-scalar-evolution.h"
79 #include "stringpool.h"
81 #include "gimple-pretty-print.h"
82 #include "opt-problem.h"
83 #include "internal-fn.h"
86 /* Loop or bb location, with hotness information. */
87 dump_user_location_t vect_location
;
89 /* auto_purge_vect_location's dtor: reset the vect_location
90 global, to avoid stale location_t values that could reference
93 auto_purge_vect_location::~auto_purge_vect_location ()
95 vect_location
= dump_user_location_t ();
98 /* Dump a cost entry according to args to F. */
101 dump_stmt_cost (FILE *f
, void *data
, int count
, enum vect_cost_for_stmt kind
,
102 stmt_vec_info stmt_info
, tree
, int misalign
, unsigned cost
,
103 enum vect_cost_model_location where
)
105 fprintf (f
, "%p ", data
);
108 print_gimple_expr (f
, STMT_VINFO_STMT (stmt_info
), 0, TDF_SLIM
);
112 fprintf (f
, "<unknown> ");
113 fprintf (f
, "%d times ", count
);
114 const char *ks
= "unknown";
132 case vector_gather_load
:
133 ks
= "vector_gather_load";
136 ks
= "unaligned_load";
138 case unaligned_store
:
139 ks
= "unaligned_store";
144 case vector_scatter_store
:
145 ks
= "vector_scatter_store";
148 ks
= "vec_to_scalar";
151 ks
= "scalar_to_vec";
153 case cond_branch_not_taken
:
154 ks
= "cond_branch_not_taken";
156 case cond_branch_taken
:
157 ks
= "cond_branch_taken";
162 case vec_promote_demote
:
163 ks
= "vec_promote_demote";
166 ks
= "vec_construct";
169 fprintf (f
, "%s ", ks
);
170 if (kind
== unaligned_load
|| kind
== unaligned_store
)
171 fprintf (f
, "(misalign %d) ", misalign
);
172 fprintf (f
, "costs %u ", cost
);
173 const char *ws
= "unknown";
186 fprintf (f
, "in %s\n", ws
);
189 /* For mapping simduid to vectorization factor. */
191 class simduid_to_vf
: public free_ptr_hash
<simduid_to_vf
>
194 unsigned int simduid
;
197 /* hash_table support. */
198 static inline hashval_t
hash (const simduid_to_vf
*);
199 static inline int equal (const simduid_to_vf
*, const simduid_to_vf
*);
203 simduid_to_vf::hash (const simduid_to_vf
*p
)
209 simduid_to_vf::equal (const simduid_to_vf
*p1
, const simduid_to_vf
*p2
)
211 return p1
->simduid
== p2
->simduid
;
214 /* This hash maps the OMP simd array to the corresponding simduid used
215 to index into it. Like thus,
217 _7 = GOMP_SIMD_LANE (simduid.0)
223 This hash maps from the OMP simd array (D.1737[]) to DECL_UID of
226 struct simd_array_to_simduid
: free_ptr_hash
<simd_array_to_simduid
>
229 unsigned int simduid
;
231 /* hash_table support. */
232 static inline hashval_t
hash (const simd_array_to_simduid
*);
233 static inline int equal (const simd_array_to_simduid
*,
234 const simd_array_to_simduid
*);
238 simd_array_to_simduid::hash (const simd_array_to_simduid
*p
)
240 return DECL_UID (p
->decl
);
244 simd_array_to_simduid::equal (const simd_array_to_simduid
*p1
,
245 const simd_array_to_simduid
*p2
)
247 return p1
->decl
== p2
->decl
;
250 /* Fold IFN_GOMP_SIMD_LANE, IFN_GOMP_SIMD_VF, IFN_GOMP_SIMD_LAST_LANE,
251 into their corresponding constants and remove
252 IFN_GOMP_SIMD_ORDERED_{START,END}. */
255 adjust_simduid_builtins (hash_table
<simduid_to_vf
> *htab
)
259 FOR_EACH_BB_FN (bb
, cfun
)
261 gimple_stmt_iterator i
;
263 for (i
= gsi_start_bb (bb
); !gsi_end_p (i
); )
266 enum internal_fn ifn
;
267 gimple
*stmt
= gsi_stmt (i
);
269 if (!is_gimple_call (stmt
)
270 || !gimple_call_internal_p (stmt
))
275 ifn
= gimple_call_internal_fn (stmt
);
278 case IFN_GOMP_SIMD_LANE
:
279 case IFN_GOMP_SIMD_VF
:
280 case IFN_GOMP_SIMD_LAST_LANE
:
282 case IFN_GOMP_SIMD_ORDERED_START
:
283 case IFN_GOMP_SIMD_ORDERED_END
:
284 if (integer_onep (gimple_call_arg (stmt
, 0)))
286 enum built_in_function bcode
287 = (ifn
== IFN_GOMP_SIMD_ORDERED_START
288 ? BUILT_IN_GOMP_ORDERED_START
289 : BUILT_IN_GOMP_ORDERED_END
);
291 = gimple_build_call (builtin_decl_explicit (bcode
), 0);
292 gimple_move_vops (g
, stmt
);
293 gsi_replace (&i
, g
, true);
296 gsi_remove (&i
, true);
297 unlink_stmt_vdef (stmt
);
303 tree arg
= gimple_call_arg (stmt
, 0);
304 gcc_assert (arg
!= NULL_TREE
);
305 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
306 simduid_to_vf
*p
= NULL
, data
;
307 data
.simduid
= DECL_UID (SSA_NAME_VAR (arg
));
308 /* Need to nullify loop safelen field since it's value is not
309 valid after transformation. */
310 if (bb
->loop_father
&& bb
->loop_father
->safelen
> 0)
311 bb
->loop_father
->safelen
= 0;
314 p
= htab
->find (&data
);
320 case IFN_GOMP_SIMD_VF
:
321 t
= build_int_cst (unsigned_type_node
, vf
);
323 case IFN_GOMP_SIMD_LANE
:
324 t
= build_int_cst (unsigned_type_node
, 0);
326 case IFN_GOMP_SIMD_LAST_LANE
:
327 t
= gimple_call_arg (stmt
, 1);
332 tree lhs
= gimple_call_lhs (stmt
);
334 replace_uses_by (lhs
, t
);
336 gsi_remove (&i
, true);
341 /* Helper structure for note_simd_array_uses. */
343 struct note_simd_array_uses_struct
345 hash_table
<simd_array_to_simduid
> **htab
;
346 unsigned int simduid
;
349 /* Callback for note_simd_array_uses, called through walk_gimple_op. */
352 note_simd_array_uses_cb (tree
*tp
, int *walk_subtrees
, void *data
)
354 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
355 struct note_simd_array_uses_struct
*ns
356 = (struct note_simd_array_uses_struct
*) wi
->info
;
361 && lookup_attribute ("omp simd array", DECL_ATTRIBUTES (*tp
))
362 && DECL_CONTEXT (*tp
) == current_function_decl
)
364 simd_array_to_simduid data
;
366 *ns
->htab
= new hash_table
<simd_array_to_simduid
> (15);
368 data
.simduid
= ns
->simduid
;
369 simd_array_to_simduid
**slot
= (*ns
->htab
)->find_slot (&data
, INSERT
);
372 simd_array_to_simduid
*p
= XNEW (simd_array_to_simduid
);
376 else if ((*slot
)->simduid
!= ns
->simduid
)
377 (*slot
)->simduid
= -1U;
383 /* Find "omp simd array" temporaries and map them to corresponding
387 note_simd_array_uses (hash_table
<simd_array_to_simduid
> **htab
)
390 gimple_stmt_iterator gsi
;
391 struct walk_stmt_info wi
;
392 struct note_simd_array_uses_struct ns
;
394 memset (&wi
, 0, sizeof (wi
));
398 FOR_EACH_BB_FN (bb
, cfun
)
399 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
401 gimple
*stmt
= gsi_stmt (gsi
);
402 if (!is_gimple_call (stmt
) || !gimple_call_internal_p (stmt
))
404 switch (gimple_call_internal_fn (stmt
))
406 case IFN_GOMP_SIMD_LANE
:
407 case IFN_GOMP_SIMD_VF
:
408 case IFN_GOMP_SIMD_LAST_LANE
:
413 tree lhs
= gimple_call_lhs (stmt
);
414 if (lhs
== NULL_TREE
)
416 imm_use_iterator use_iter
;
418 ns
.simduid
= DECL_UID (SSA_NAME_VAR (gimple_call_arg (stmt
, 0)));
419 FOR_EACH_IMM_USE_STMT (use_stmt
, use_iter
, lhs
)
420 if (!is_gimple_debug (use_stmt
))
421 walk_gimple_op (use_stmt
, note_simd_array_uses_cb
, &wi
);
425 /* Shrink arrays with "omp simd array" attribute to the corresponding
426 vectorization factor. */
430 (hash_table
<simd_array_to_simduid
> *simd_array_to_simduid_htab
,
431 hash_table
<simduid_to_vf
> *simduid_to_vf_htab
)
433 for (hash_table
<simd_array_to_simduid
>::iterator iter
434 = simd_array_to_simduid_htab
->begin ();
435 iter
!= simd_array_to_simduid_htab
->end (); ++iter
)
436 if ((*iter
)->simduid
!= -1U)
438 tree decl
= (*iter
)->decl
;
440 if (simduid_to_vf_htab
)
442 simduid_to_vf
*p
= NULL
, data
;
443 data
.simduid
= (*iter
)->simduid
;
444 p
= simduid_to_vf_htab
->find (&data
);
449 = build_array_type_nelts (TREE_TYPE (TREE_TYPE (decl
)), vf
);
450 TREE_TYPE (decl
) = atype
;
451 relayout_decl (decl
);
454 delete simd_array_to_simduid_htab
;
457 /* Initialize the vec_info with kind KIND_IN and target cost data
458 TARGET_COST_DATA_IN. */
460 vec_info::vec_info (vec_info::vec_kind kind_in
, void *target_cost_data_in
,
461 vec_info_shared
*shared_
)
464 stmt_vec_info_ro (false),
465 target_cost_data (target_cost_data_in
)
467 stmt_vec_infos
.create (50);
470 vec_info::~vec_info ()
472 for (slp_instance
&instance
: slp_instances
)
473 vect_free_slp_instance (instance
);
475 destroy_cost_data (target_cost_data
);
476 free_stmt_vec_infos ();
479 vec_info_shared::vec_info_shared ()
481 datarefs_copy (vNULL
),
486 vec_info_shared::~vec_info_shared ()
488 free_data_refs (datarefs
);
489 free_dependence_relations (ddrs
);
490 datarefs_copy
.release ();
494 vec_info_shared::save_datarefs ()
498 datarefs_copy
.reserve_exact (datarefs
.length ());
499 for (unsigned i
= 0; i
< datarefs
.length (); ++i
)
500 datarefs_copy
.quick_push (*datarefs
[i
]);
504 vec_info_shared::check_datarefs ()
508 gcc_assert (datarefs
.length () == datarefs_copy
.length ());
509 for (unsigned i
= 0; i
< datarefs
.length (); ++i
)
510 if (memcmp (&datarefs_copy
[i
], datarefs
[i
], sizeof (data_reference
)) != 0)
514 /* Record that STMT belongs to the vectorizable region. Create and return
515 an associated stmt_vec_info. */
518 vec_info::add_stmt (gimple
*stmt
)
520 stmt_vec_info res
= new_stmt_vec_info (stmt
);
521 set_vinfo_for_stmt (stmt
, res
);
525 /* Record that STMT belongs to the vectorizable region. Create a new
526 stmt_vec_info and mark VECINFO as being related and return the new
530 vec_info::add_pattern_stmt (gimple
*stmt
, stmt_vec_info stmt_info
)
532 stmt_vec_info res
= new_stmt_vec_info (stmt
);
533 set_vinfo_for_stmt (stmt
, res
, false);
534 STMT_VINFO_RELATED_STMT (res
) = stmt_info
;
538 /* If STMT has an associated stmt_vec_info, return that vec_info, otherwise
539 return null. It is safe to call this function on any statement, even if
540 it might not be part of the vectorizable region. */
543 vec_info::lookup_stmt (gimple
*stmt
)
545 unsigned int uid
= gimple_uid (stmt
);
546 if (uid
> 0 && uid
- 1 < stmt_vec_infos
.length ())
548 stmt_vec_info res
= stmt_vec_infos
[uid
- 1];
549 if (res
&& res
->stmt
== stmt
)
555 /* If NAME is an SSA_NAME and its definition has an associated stmt_vec_info,
556 return that stmt_vec_info, otherwise return null. It is safe to call
557 this on arbitrary operands. */
560 vec_info::lookup_def (tree name
)
562 if (TREE_CODE (name
) == SSA_NAME
563 && !SSA_NAME_IS_DEFAULT_DEF (name
))
564 return lookup_stmt (SSA_NAME_DEF_STMT (name
));
568 /* See whether there is a single non-debug statement that uses LHS and
569 whether that statement has an associated stmt_vec_info. Return the
570 stmt_vec_info if so, otherwise return null. */
573 vec_info::lookup_single_use (tree lhs
)
577 if (single_imm_use (lhs
, &dummy
, &use_stmt
))
578 return lookup_stmt (use_stmt
);
582 /* Return vectorization information about DR. */
585 vec_info::lookup_dr (data_reference
*dr
)
587 stmt_vec_info stmt_info
= lookup_stmt (DR_STMT (dr
));
588 /* DR_STMT should never refer to a stmt in a pattern replacement. */
589 gcc_checking_assert (!is_pattern_stmt_p (stmt_info
));
590 return STMT_VINFO_DR_INFO (stmt_info
->dr_aux
.stmt
);
593 /* Record that NEW_STMT_INFO now implements the same data reference
597 vec_info::move_dr (stmt_vec_info new_stmt_info
, stmt_vec_info old_stmt_info
)
599 gcc_assert (!is_pattern_stmt_p (old_stmt_info
));
600 STMT_VINFO_DR_INFO (old_stmt_info
)->stmt
= new_stmt_info
;
601 new_stmt_info
->dr_aux
= old_stmt_info
->dr_aux
;
602 STMT_VINFO_DR_WRT_VEC_LOOP (new_stmt_info
)
603 = STMT_VINFO_DR_WRT_VEC_LOOP (old_stmt_info
);
604 STMT_VINFO_GATHER_SCATTER_P (new_stmt_info
)
605 = STMT_VINFO_GATHER_SCATTER_P (old_stmt_info
);
608 /* Permanently remove the statement described by STMT_INFO from the
612 vec_info::remove_stmt (stmt_vec_info stmt_info
)
614 gcc_assert (!stmt_info
->pattern_stmt_p
);
615 set_vinfo_for_stmt (stmt_info
->stmt
, NULL
);
616 unlink_stmt_vdef (stmt_info
->stmt
);
617 gimple_stmt_iterator si
= gsi_for_stmt (stmt_info
->stmt
);
618 gsi_remove (&si
, true);
619 release_defs (stmt_info
->stmt
);
620 free_stmt_vec_info (stmt_info
);
623 /* Replace the statement at GSI by NEW_STMT, both the vectorization
624 information and the function itself. STMT_INFO describes the statement
628 vec_info::replace_stmt (gimple_stmt_iterator
*gsi
, stmt_vec_info stmt_info
,
631 gimple
*old_stmt
= stmt_info
->stmt
;
632 gcc_assert (!stmt_info
->pattern_stmt_p
&& old_stmt
== gsi_stmt (*gsi
));
633 gimple_set_uid (new_stmt
, gimple_uid (old_stmt
));
634 stmt_info
->stmt
= new_stmt
;
635 gsi_replace (gsi
, new_stmt
, true);
638 /* Insert stmts in SEQ on the VEC_INFO region entry. If CONTEXT is
639 not NULL it specifies whether to use the sub-region entry
640 determined by it, currently used for loop vectorization to insert
641 on the inner loop entry vs. the outer loop entry. */
644 vec_info::insert_seq_on_entry (stmt_vec_info context
, gimple_seq seq
)
646 if (loop_vec_info loop_vinfo
= dyn_cast
<loop_vec_info
> (this))
648 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
652 if (context
&& nested_in_vect_loop_p (loop
, context
))
655 pe
= loop_preheader_edge (loop
);
656 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
657 gcc_assert (!new_bb
);
661 bb_vec_info bb_vinfo
= as_a
<bb_vec_info
> (this);
662 gimple_stmt_iterator gsi_region_begin
663 = gsi_after_labels (bb_vinfo
->bbs
[0]);
664 gsi_insert_seq_before (&gsi_region_begin
, seq
, GSI_SAME_STMT
);
668 /* Like insert_seq_on_entry but just inserts the single stmt NEW_STMT. */
671 vec_info::insert_on_entry (stmt_vec_info context
, gimple
*new_stmt
)
673 gimple_seq seq
= NULL
;
674 gimple_stmt_iterator gsi
= gsi_start (seq
);
675 gsi_insert_before_without_update (&gsi
, new_stmt
, GSI_SAME_STMT
);
676 insert_seq_on_entry (context
, seq
);
679 /* Create and initialize a new stmt_vec_info struct for STMT. */
682 vec_info::new_stmt_vec_info (gimple
*stmt
)
684 stmt_vec_info res
= XCNEW (class _stmt_vec_info
);
687 STMT_VINFO_TYPE (res
) = undef_vec_info_type
;
688 STMT_VINFO_RELEVANT (res
) = vect_unused_in_scope
;
689 STMT_VINFO_VECTORIZABLE (res
) = true;
690 STMT_VINFO_REDUC_TYPE (res
) = TREE_CODE_REDUCTION
;
691 STMT_VINFO_REDUC_CODE (res
) = ERROR_MARK
;
692 STMT_VINFO_REDUC_FN (res
) = IFN_LAST
;
693 STMT_VINFO_REDUC_IDX (res
) = -1;
694 STMT_VINFO_SLP_VECT_ONLY (res
) = false;
695 STMT_VINFO_SLP_VECT_ONLY_PATTERN (res
) = false;
696 STMT_VINFO_VEC_STMTS (res
) = vNULL
;
697 res
->reduc_initial_values
= vNULL
;
698 res
->reduc_scalar_results
= vNULL
;
700 if (is_a
<loop_vec_info
> (this)
701 && gimple_code (stmt
) == GIMPLE_PHI
702 && is_loop_header_bb_p (gimple_bb (stmt
)))
703 STMT_VINFO_DEF_TYPE (res
) = vect_unknown_def_type
;
705 STMT_VINFO_DEF_TYPE (res
) = vect_internal_def
;
707 STMT_SLP_TYPE (res
) = loop_vect
;
709 /* This is really "uninitialized" until vect_compute_data_ref_alignment. */
710 res
->dr_aux
.misalignment
= DR_MISALIGNMENT_UNINITIALIZED
;
715 /* Associate STMT with INFO. */
718 vec_info::set_vinfo_for_stmt (gimple
*stmt
, stmt_vec_info info
, bool check_ro
)
720 unsigned int uid
= gimple_uid (stmt
);
723 gcc_assert (!check_ro
|| !stmt_vec_info_ro
);
724 gcc_checking_assert (info
);
725 uid
= stmt_vec_infos
.length () + 1;
726 gimple_set_uid (stmt
, uid
);
727 stmt_vec_infos
.safe_push (info
);
731 gcc_checking_assert (info
== NULL
);
732 stmt_vec_infos
[uid
- 1] = info
;
736 /* Free the contents of stmt_vec_infos. */
739 vec_info::free_stmt_vec_infos (void)
741 for (stmt_vec_info
&info
: stmt_vec_infos
)
743 free_stmt_vec_info (info
);
744 stmt_vec_infos
.release ();
747 /* Free STMT_INFO. */
750 vec_info::free_stmt_vec_info (stmt_vec_info stmt_info
)
752 if (stmt_info
->pattern_stmt_p
)
754 gimple_set_bb (stmt_info
->stmt
, NULL
);
755 tree lhs
= gimple_get_lhs (stmt_info
->stmt
);
756 if (lhs
&& TREE_CODE (lhs
) == SSA_NAME
)
757 release_ssa_name (lhs
);
760 stmt_info
->reduc_initial_values
.release ();
761 stmt_info
->reduc_scalar_results
.release ();
762 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).release ();
763 STMT_VINFO_VEC_STMTS (stmt_info
).release ();
767 /* Returns true if S1 dominates S2. */
770 vect_stmt_dominates_stmt_p (gimple
*s1
, gimple
*s2
)
772 basic_block bb1
= gimple_bb (s1
), bb2
= gimple_bb (s2
);
774 /* If bb1 is NULL, it should be a GIMPLE_NOP def stmt of an (D)
775 SSA_NAME. Assume it lives at the beginning of function and
776 thus dominates everything. */
777 if (!bb1
|| s1
== s2
)
780 /* If bb2 is NULL, it doesn't dominate any stmt with a bb. */
785 return dominated_by_p (CDI_DOMINATORS
, bb2
, bb1
);
787 /* PHIs in the same basic block are assumed to be
788 executed all in parallel, if only one stmt is a PHI,
789 it dominates the other stmt in the same basic block. */
790 if (gimple_code (s1
) == GIMPLE_PHI
)
793 if (gimple_code (s2
) == GIMPLE_PHI
)
796 /* Inserted vectorized stmts all have UID 0 while the original stmts
797 in the IL have UID increasing within a BB. Walk from both sides
798 until we find the other stmt or a stmt with UID != 0. */
799 gimple_stmt_iterator gsi1
= gsi_for_stmt (s1
);
800 while (gimple_uid (gsi_stmt (gsi1
)) == 0)
803 if (gsi_end_p (gsi1
))
805 if (gsi_stmt (gsi1
) == s2
)
808 if (gimple_uid (gsi_stmt (gsi1
)) == -1u)
811 gimple_stmt_iterator gsi2
= gsi_for_stmt (s2
);
812 while (gimple_uid (gsi_stmt (gsi2
)) == 0)
815 if (gsi_end_p (gsi2
))
817 if (gsi_stmt (gsi2
) == s1
)
820 if (gimple_uid (gsi_stmt (gsi2
)) == -1u)
823 if (gimple_uid (gsi_stmt (gsi1
)) <= gimple_uid (gsi_stmt (gsi2
)))
828 /* A helper function to free scev and LOOP niter information, as well as
829 clear loop constraint LOOP_C_FINITE. */
832 vect_free_loop_info_assumptions (class loop
*loop
)
835 /* We need to explicitly reset upper bound information since they are
836 used even after free_numbers_of_iterations_estimates. */
837 loop
->any_upper_bound
= false;
838 loop
->any_likely_upper_bound
= false;
839 free_numbers_of_iterations_estimates (loop
);
840 loop_constraint_clear (loop
, LOOP_C_FINITE
);
843 /* If LOOP has been versioned during ifcvt, return the internal call
847 vect_loop_vectorized_call (class loop
*loop
, gcond
**cond
)
849 basic_block bb
= loop_preheader_edge (loop
)->src
;
856 if (!single_pred_p (bb
))
858 bb
= single_pred (bb
);
861 if (g
&& gimple_code (g
) == GIMPLE_COND
)
864 *cond
= as_a
<gcond
*> (g
);
865 gimple_stmt_iterator gsi
= gsi_for_stmt (g
);
867 if (!gsi_end_p (gsi
))
870 if (gimple_call_internal_p (g
, IFN_LOOP_VECTORIZED
)
871 && (tree_to_shwi (gimple_call_arg (g
, 0)) == loop
->num
872 || tree_to_shwi (gimple_call_arg (g
, 1)) == loop
->num
))
879 /* If LOOP has been versioned during loop distribution, return the gurading
883 vect_loop_dist_alias_call (class loop
*loop
)
887 class loop
*outer
, *orig
;
888 gimple_stmt_iterator gsi
;
891 if (loop
->orig_loop_num
== 0)
894 orig
= get_loop (cfun
, loop
->orig_loop_num
);
897 /* The original loop is somehow destroyed. Clear the information. */
898 loop
->orig_loop_num
= 0;
903 bb
= nearest_common_dominator (CDI_DOMINATORS
, loop
->header
, orig
->header
);
905 bb
= loop_preheader_edge (loop
)->src
;
907 outer
= bb
->loop_father
;
908 entry
= ENTRY_BLOCK_PTR_FOR_FN (cfun
);
910 /* Look upward in dominance tree. */
911 for (; bb
!= entry
&& flow_bb_inside_loop_p (outer
, bb
);
912 bb
= get_immediate_dominator (CDI_DOMINATORS
, bb
))
915 if (g
== NULL
|| gimple_code (g
) != GIMPLE_COND
)
918 gsi
= gsi_for_stmt (g
);
924 /* The guarding internal function call must have the same distribution
926 if (gimple_call_internal_p (g
, IFN_LOOP_DIST_ALIAS
)
927 && (tree_to_shwi (gimple_call_arg (g
, 0)) == loop
->orig_loop_num
))
933 /* Set the uids of all the statements in basic blocks inside loop
934 represented by LOOP_VINFO. LOOP_VECTORIZED_CALL is the internal
935 call guarding the loop which has been if converted. */
937 set_uid_loop_bbs (loop_vec_info loop_vinfo
, gimple
*loop_vectorized_call
)
939 tree arg
= gimple_call_arg (loop_vectorized_call
, 1);
942 class loop
*scalar_loop
= get_loop (cfun
, tree_to_shwi (arg
));
944 LOOP_VINFO_SCALAR_LOOP (loop_vinfo
) = scalar_loop
;
945 gcc_checking_assert (vect_loop_vectorized_call (scalar_loop
)
946 == loop_vectorized_call
);
947 /* If we are going to vectorize outer loop, prevent vectorization
948 of the inner loop in the scalar loop - either the scalar loop is
949 thrown away, so it is a wasted work, or is used only for
951 if (scalar_loop
->inner
)
953 gimple
*g
= vect_loop_vectorized_call (scalar_loop
->inner
);
956 arg
= gimple_call_arg (g
, 0);
957 get_loop (cfun
, tree_to_shwi (arg
))->dont_vectorize
= true;
958 fold_loop_internal_call (g
, boolean_false_node
);
961 bbs
= get_loop_body (scalar_loop
);
962 for (i
= 0; i
< scalar_loop
->num_nodes
; i
++)
964 basic_block bb
= bbs
[i
];
965 gimple_stmt_iterator gsi
;
966 for (gsi
= gsi_start_phis (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
968 gimple
*phi
= gsi_stmt (gsi
);
969 gimple_set_uid (phi
, 0);
971 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
973 gimple
*stmt
= gsi_stmt (gsi
);
974 gimple_set_uid (stmt
, 0);
980 /* Try to vectorize LOOP. */
983 try_vectorize_loop_1 (hash_table
<simduid_to_vf
> *&simduid_to_vf_htab
,
984 unsigned *num_vectorized_loops
, loop_p loop
,
985 gimple
*loop_vectorized_call
,
986 gimple
*loop_dist_alias_call
)
989 vec_info_shared shared
;
990 auto_purge_vect_location sentinel
;
991 vect_location
= find_loop_location (loop
);
993 if (LOCATION_LOCUS (vect_location
.get_location_t ()) != UNKNOWN_LOCATION
994 && dump_enabled_p ())
995 dump_printf (MSG_NOTE
| MSG_PRIORITY_INTERNALS
,
996 "\nAnalyzing loop at %s:%d\n",
997 LOCATION_FILE (vect_location
.get_location_t ()),
998 LOCATION_LINE (vect_location
.get_location_t ()));
1000 opt_loop_vec_info loop_vinfo
= opt_loop_vec_info::success (NULL
);
1001 /* In the case of epilogue vectorization the loop already has its
1002 loop_vec_info set, we do not require to analyze the loop in this case. */
1003 if (loop_vec_info vinfo
= loop_vec_info_for_loop (loop
))
1004 loop_vinfo
= opt_loop_vec_info::success (vinfo
);
1007 /* Try to analyze the loop, retaining an opt_problem if dump_enabled_p. */
1008 loop_vinfo
= vect_analyze_loop (loop
, &shared
);
1009 loop
->aux
= loop_vinfo
;
1013 if (dump_enabled_p ())
1014 if (opt_problem
*problem
= loop_vinfo
.get_problem ())
1016 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1017 "couldn't vectorize loop\n");
1018 problem
->emit_and_clear ();
1021 if (!loop_vinfo
|| !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo
))
1023 /* Free existing information if loop is analyzed with some
1025 if (loop_constraint_set_p (loop
, LOOP_C_FINITE
))
1026 vect_free_loop_info_assumptions (loop
);
1028 /* If we applied if-conversion then try to vectorize the
1029 BB of innermost loops.
1030 ??? Ideally BB vectorization would learn to vectorize
1031 control flow by applying if-conversion on-the-fly, the
1032 following retains the if-converted loop body even when
1033 only non-if-converted parts took part in BB vectorization. */
1034 if (flag_tree_slp_vectorize
!= 0
1035 && loop_vectorized_call
1038 basic_block bb
= loop
->header
;
1039 bool require_loop_vectorize
= false;
1040 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
);
1041 !gsi_end_p (gsi
); gsi_next (&gsi
))
1043 gimple
*stmt
= gsi_stmt (gsi
);
1044 gcall
*call
= dyn_cast
<gcall
*> (stmt
);
1045 if (call
&& gimple_call_internal_p (call
))
1047 internal_fn ifn
= gimple_call_internal_fn (call
);
1048 if (ifn
== IFN_MASK_LOAD
|| ifn
== IFN_MASK_STORE
1049 /* Don't keep the if-converted parts when the ifn with
1050 specifc type is not supported by the backend. */
1051 || (direct_internal_fn_p (ifn
)
1052 && !direct_internal_fn_supported_p
1053 (call
, OPTIMIZE_FOR_SPEED
)))
1055 require_loop_vectorize
= true;
1059 gimple_set_uid (stmt
, -1);
1060 gimple_set_visited (stmt
, false);
1062 if (!require_loop_vectorize
)
1064 tree arg
= gimple_call_arg (loop_vectorized_call
, 1);
1065 class loop
*scalar_loop
= get_loop (cfun
, tree_to_shwi (arg
));
1066 if (vect_slp_if_converted_bb (bb
, scalar_loop
))
1068 fold_loop_internal_call (loop_vectorized_call
,
1070 loop_vectorized_call
= NULL
;
1071 ret
|= TODO_cleanup_cfg
| TODO_update_ssa_only_virtuals
;
1075 /* If outer loop vectorization fails for LOOP_VECTORIZED guarded
1076 loop, don't vectorize its inner loop; we'll attempt to
1077 vectorize LOOP_VECTORIZED guarded inner loop of the scalar
1079 if (loop_vectorized_call
&& loop
->inner
)
1080 loop
->inner
->dont_vectorize
= true;
1084 /* Only count the original scalar loops. */
1085 if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo
) && !dbg_cnt (vect_loop
))
1087 /* Free existing information if loop is analyzed with some
1089 if (loop_constraint_set_p (loop
, LOOP_C_FINITE
))
1090 vect_free_loop_info_assumptions (loop
);
1094 if (loop_vectorized_call
)
1095 set_uid_loop_bbs (loop_vinfo
, loop_vectorized_call
);
1097 unsigned HOST_WIDE_INT bytes
;
1098 if (dump_enabled_p ())
1100 if (GET_MODE_SIZE (loop_vinfo
->vector_mode
).is_constant (&bytes
))
1101 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, vect_location
,
1102 "loop vectorized using %wu byte vectors\n", bytes
);
1104 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, vect_location
,
1105 "loop vectorized using variable length vectors\n");
1108 loop_p new_loop
= vect_transform_loop (loop_vinfo
,
1109 loop_vectorized_call
);
1110 (*num_vectorized_loops
)++;
1111 /* Now that the loop has been vectorized, allow it to be unrolled
1113 loop
->force_vectorize
= false;
1117 simduid_to_vf
*simduid_to_vf_data
= XNEW (simduid_to_vf
);
1118 if (!simduid_to_vf_htab
)
1119 simduid_to_vf_htab
= new hash_table
<simduid_to_vf
> (15);
1120 simduid_to_vf_data
->simduid
= DECL_UID (loop
->simduid
);
1121 simduid_to_vf_data
->vf
= loop_vinfo
->vectorization_factor
;
1122 *simduid_to_vf_htab
->find_slot (simduid_to_vf_data
, INSERT
)
1123 = simduid_to_vf_data
;
1126 if (loop_vectorized_call
)
1128 fold_loop_internal_call (loop_vectorized_call
, boolean_true_node
);
1129 loop_vectorized_call
= NULL
;
1130 ret
|= TODO_cleanup_cfg
;
1132 if (loop_dist_alias_call
)
1134 tree value
= gimple_call_arg (loop_dist_alias_call
, 1);
1135 fold_loop_internal_call (loop_dist_alias_call
, value
);
1136 loop_dist_alias_call
= NULL
;
1137 ret
|= TODO_cleanup_cfg
;
1140 /* Epilogue of vectorized loop must be vectorized too. */
1143 /* Don't include vectorized epilogues in the "vectorized loops" count.
1145 unsigned dont_count
= *num_vectorized_loops
;
1146 ret
|= try_vectorize_loop_1 (simduid_to_vf_htab
, &dont_count
,
1147 new_loop
, NULL
, NULL
);
1153 /* Try to vectorize LOOP. */
1156 try_vectorize_loop (hash_table
<simduid_to_vf
> *&simduid_to_vf_htab
,
1157 unsigned *num_vectorized_loops
, loop_p loop
)
1159 if (!((flag_tree_loop_vectorize
1160 && optimize_loop_nest_for_speed_p (loop
))
1161 || loop
->force_vectorize
))
1164 return try_vectorize_loop_1 (simduid_to_vf_htab
, num_vectorized_loops
, loop
,
1165 vect_loop_vectorized_call (loop
),
1166 vect_loop_dist_alias_call (loop
));
1170 /* Function vectorize_loops.
1172 Entry point to loop vectorization phase. */
1175 vectorize_loops (void)
1178 unsigned int num_vectorized_loops
= 0;
1179 unsigned int vect_loops_num
;
1181 hash_table
<simduid_to_vf
> *simduid_to_vf_htab
= NULL
;
1182 hash_table
<simd_array_to_simduid
> *simd_array_to_simduid_htab
= NULL
;
1183 bool any_ifcvt_loops
= false;
1186 vect_loops_num
= number_of_loops (cfun
);
1188 /* Bail out if there are no loops. */
1189 if (vect_loops_num
<= 1)
1194 if (cfun
->has_simduid_loops
)
1195 note_simd_array_uses (&simd_array_to_simduid_htab
);
1197 /* ----------- Analyze loops. ----------- */
1199 /* If some loop was duplicated, it gets bigger number
1200 than all previously defined loops. This fact allows us to run
1201 only over initial loops skipping newly generated ones. */
1202 for (auto loop
: loops_list (cfun
, 0))
1203 if (loop
->dont_vectorize
)
1205 any_ifcvt_loops
= true;
1206 /* If-conversion sometimes versions both the outer loop
1207 (for the case when outer loop vectorization might be
1208 desirable) as well as the inner loop in the scalar version
1209 of the loop. So we have:
1210 if (LOOP_VECTORIZED (1, 3))
1216 loop3 (copy of loop1)
1217 if (LOOP_VECTORIZED (4, 5))
1218 loop4 (copy of loop2)
1220 loop5 (copy of loop4)
1221 If loops' iteration gives us loop3 first (which has
1222 dont_vectorize set), make sure to process loop1 before loop4;
1223 so that we can prevent vectorization of loop4 if loop1
1224 is successfully vectorized. */
1227 gimple
*loop_vectorized_call
1228 = vect_loop_vectorized_call (loop
);
1229 if (loop_vectorized_call
1230 && vect_loop_vectorized_call (loop
->inner
))
1232 tree arg
= gimple_call_arg (loop_vectorized_call
, 0);
1233 class loop
*vector_loop
1234 = get_loop (cfun
, tree_to_shwi (arg
));
1235 if (vector_loop
&& vector_loop
!= loop
)
1237 /* Make sure we don't vectorize it twice. */
1238 vector_loop
->dont_vectorize
= true;
1239 ret
|= try_vectorize_loop (simduid_to_vf_htab
,
1240 &num_vectorized_loops
,
1247 ret
|= try_vectorize_loop (simduid_to_vf_htab
, &num_vectorized_loops
,
1250 vect_location
= dump_user_location_t ();
1252 statistics_counter_event (cfun
, "Vectorized loops", num_vectorized_loops
);
1253 if (dump_enabled_p ()
1254 || (num_vectorized_loops
> 0 && dump_enabled_p ()))
1255 dump_printf_loc (MSG_NOTE
, vect_location
,
1256 "vectorized %u loops in function.\n",
1257 num_vectorized_loops
);
1259 /* ----------- Finalize. ----------- */
1261 if (any_ifcvt_loops
)
1262 for (i
= 1; i
< number_of_loops (cfun
); i
++)
1264 loop
= get_loop (cfun
, i
);
1265 if (loop
&& loop
->dont_vectorize
)
1267 gimple
*g
= vect_loop_vectorized_call (loop
);
1270 fold_loop_internal_call (g
, boolean_false_node
);
1271 ret
|= TODO_cleanup_cfg
;
1275 g
= vect_loop_dist_alias_call (loop
);
1279 fold_loop_internal_call (g
, boolean_false_node
);
1280 ret
|= TODO_cleanup_cfg
;
1285 for (i
= 1; i
< number_of_loops (cfun
); i
++)
1287 loop_vec_info loop_vinfo
;
1288 bool has_mask_store
;
1290 loop
= get_loop (cfun
, i
);
1291 if (!loop
|| !loop
->aux
)
1293 loop_vinfo
= (loop_vec_info
) loop
->aux
;
1294 has_mask_store
= LOOP_VINFO_HAS_MASK_STORE (loop_vinfo
);
1297 && targetm
.vectorize
.empty_mask_is_expensive (IFN_MASK_STORE
))
1298 optimize_mask_stores (loop
);
1302 /* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE,ORDERED_{START,END}} builtins. */
1303 if (cfun
->has_simduid_loops
)
1305 adjust_simduid_builtins (simduid_to_vf_htab
);
1306 /* Avoid stale SCEV cache entries for the SIMD_LANE defs. */
1310 /* Shrink any "omp array simd" temporary arrays to the
1311 actual vectorization factors. */
1312 if (simd_array_to_simduid_htab
)
1313 shrink_simd_arrays (simd_array_to_simduid_htab
, simduid_to_vf_htab
);
1314 delete simduid_to_vf_htab
;
1315 cfun
->has_simduid_loops
= false;
1318 if (num_vectorized_loops
> 0)
1320 /* If we vectorized any loop only virtual SSA form needs to be updated.
1321 ??? Also while we try hard to update loop-closed SSA form we fail
1322 to properly do this in some corner-cases (see PR56286). */
1323 rewrite_into_loop_closed_ssa (NULL
, TODO_update_ssa_only_virtuals
);
1324 return TODO_cleanup_cfg
;
1331 /* Entry point to the simduid cleanup pass. */
1335 const pass_data pass_data_simduid_cleanup
=
1337 GIMPLE_PASS
, /* type */
1338 "simduid", /* name */
1339 OPTGROUP_NONE
, /* optinfo_flags */
1340 TV_NONE
, /* tv_id */
1341 ( PROP_ssa
| PROP_cfg
), /* properties_required */
1342 0, /* properties_provided */
1343 0, /* properties_destroyed */
1344 0, /* todo_flags_start */
1345 0, /* todo_flags_finish */
1348 class pass_simduid_cleanup
: public gimple_opt_pass
1351 pass_simduid_cleanup (gcc::context
*ctxt
)
1352 : gimple_opt_pass (pass_data_simduid_cleanup
, ctxt
)
1355 /* opt_pass methods: */
1356 opt_pass
* clone () { return new pass_simduid_cleanup (m_ctxt
); }
1357 virtual bool gate (function
*fun
) { return fun
->has_simduid_loops
; }
1358 virtual unsigned int execute (function
*);
1360 }; // class pass_simduid_cleanup
1363 pass_simduid_cleanup::execute (function
*fun
)
1365 hash_table
<simd_array_to_simduid
> *simd_array_to_simduid_htab
= NULL
;
1367 note_simd_array_uses (&simd_array_to_simduid_htab
);
1369 /* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE,ORDERED_{START,END}} builtins. */
1370 adjust_simduid_builtins (NULL
);
1372 /* Shrink any "omp array simd" temporary arrays to the
1373 actual vectorization factors. */
1374 if (simd_array_to_simduid_htab
)
1375 shrink_simd_arrays (simd_array_to_simduid_htab
, NULL
);
1376 fun
->has_simduid_loops
= false;
1383 make_pass_simduid_cleanup (gcc::context
*ctxt
)
1385 return new pass_simduid_cleanup (ctxt
);
1389 /* Entry point to basic block SLP phase. */
1393 const pass_data pass_data_slp_vectorize
=
1395 GIMPLE_PASS
, /* type */
1397 OPTGROUP_LOOP
| OPTGROUP_VEC
, /* optinfo_flags */
1398 TV_TREE_SLP_VECTORIZATION
, /* tv_id */
1399 ( PROP_ssa
| PROP_cfg
), /* properties_required */
1400 0, /* properties_provided */
1401 0, /* properties_destroyed */
1402 0, /* todo_flags_start */
1403 TODO_update_ssa
, /* todo_flags_finish */
1406 class pass_slp_vectorize
: public gimple_opt_pass
1409 pass_slp_vectorize (gcc::context
*ctxt
)
1410 : gimple_opt_pass (pass_data_slp_vectorize
, ctxt
)
1413 /* opt_pass methods: */
1414 opt_pass
* clone () { return new pass_slp_vectorize (m_ctxt
); }
1415 virtual bool gate (function
*) { return flag_tree_slp_vectorize
!= 0; }
1416 virtual unsigned int execute (function
*);
1418 }; // class pass_slp_vectorize
1421 pass_slp_vectorize::execute (function
*fun
)
1423 auto_purge_vect_location sentinel
;
1426 bool in_loop_pipeline
= scev_initialized_p ();
1427 if (!in_loop_pipeline
)
1429 loop_optimizer_init (LOOPS_NORMAL
);
1433 /* Mark all stmts as not belonging to the current region and unvisited. */
1434 FOR_EACH_BB_FN (bb
, fun
)
1436 for (gphi_iterator gsi
= gsi_start_phis (bb
); !gsi_end_p (gsi
);
1439 gphi
*stmt
= gsi
.phi ();
1440 gimple_set_uid (stmt
, -1);
1441 gimple_set_visited (stmt
, false);
1443 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
1446 gimple
*stmt
= gsi_stmt (gsi
);
1447 gimple_set_uid (stmt
, -1);
1448 gimple_set_visited (stmt
, false);
1454 vect_slp_function (fun
);
1458 if (!in_loop_pipeline
)
1461 loop_optimizer_finalize ();
1470 make_pass_slp_vectorize (gcc::context
*ctxt
)
1472 return new pass_slp_vectorize (ctxt
);
1476 /* Increase alignment of global arrays to improve vectorization potential.
1478 - Consider also structs that have an array field.
1479 - Use ipa analysis to prune arrays that can't be vectorized?
1480 This should involve global alignment analysis and in the future also
1483 static unsigned get_vec_alignment_for_type (tree
);
1484 static hash_map
<tree
, unsigned> *type_align_map
;
1486 /* Return alignment of array's vector type corresponding to scalar type.
1487 0 if no vector type exists. */
1489 get_vec_alignment_for_array_type (tree type
)
1491 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1492 poly_uint64 array_size
, vector_size
;
1494 tree scalar_type
= strip_array_types (type
);
1495 tree vectype
= get_related_vectype_for_scalar_type (VOIDmode
, scalar_type
);
1497 || !poly_int_tree_p (TYPE_SIZE (type
), &array_size
)
1498 || !poly_int_tree_p (TYPE_SIZE (vectype
), &vector_size
)
1499 || maybe_lt (array_size
, vector_size
))
1502 return TYPE_ALIGN (vectype
);
1505 /* Return alignment of field having maximum alignment of vector type
1506 corresponding to it's scalar type. For now, we only consider fields whose
1507 offset is a multiple of it's vector alignment.
1508 0 if no suitable field is found. */
1510 get_vec_alignment_for_record_type (tree type
)
1512 gcc_assert (TREE_CODE (type
) == RECORD_TYPE
);
1514 unsigned max_align
= 0, alignment
;
1515 HOST_WIDE_INT offset
;
1518 if (TYPE_PACKED (type
))
1521 unsigned *slot
= type_align_map
->get (type
);
1525 for (tree field
= first_field (type
);
1527 field
= DECL_CHAIN (field
))
1529 /* Skip if not FIELD_DECL or if alignment is set by user. */
1530 if (TREE_CODE (field
) != FIELD_DECL
1531 || DECL_USER_ALIGN (field
)
1532 || DECL_ARTIFICIAL (field
))
1535 /* We don't need to process the type further if offset is variable,
1536 since the offsets of remaining members will also be variable. */
1537 if (TREE_CODE (DECL_FIELD_OFFSET (field
)) != INTEGER_CST
1538 || TREE_CODE (DECL_FIELD_BIT_OFFSET (field
)) != INTEGER_CST
)
1541 /* Similarly stop processing the type if offset_tree
1542 does not fit in unsigned HOST_WIDE_INT. */
1543 offset_tree
= bit_position (field
);
1544 if (!tree_fits_uhwi_p (offset_tree
))
1547 offset
= tree_to_uhwi (offset_tree
);
1548 alignment
= get_vec_alignment_for_type (TREE_TYPE (field
));
1550 /* Get maximum alignment of vectorized field/array among those members
1551 whose offset is multiple of the vector alignment. */
1553 && (offset
% alignment
== 0)
1554 && (alignment
> max_align
))
1555 max_align
= alignment
;
1558 type_align_map
->put (type
, max_align
);
1562 /* Return alignment of vector type corresponding to decl's scalar type
1563 or 0 if it doesn't exist or the vector alignment is lesser than
1564 decl's alignment. */
1566 get_vec_alignment_for_type (tree type
)
1568 if (type
== NULL_TREE
)
1571 gcc_assert (TYPE_P (type
));
1573 static unsigned alignment
= 0;
1574 switch (TREE_CODE (type
))
1577 alignment
= get_vec_alignment_for_array_type (type
);
1580 alignment
= get_vec_alignment_for_record_type (type
);
1587 return (alignment
> TYPE_ALIGN (type
)) ? alignment
: 0;
1590 /* Entry point to increase_alignment pass. */
1592 increase_alignment (void)
1594 varpool_node
*vnode
;
1596 vect_location
= dump_user_location_t ();
1597 type_align_map
= new hash_map
<tree
, unsigned>;
1599 /* Increase the alignment of all global arrays for vectorization. */
1600 FOR_EACH_DEFINED_VARIABLE (vnode
)
1602 tree decl
= vnode
->decl
;
1603 unsigned int alignment
;
1605 if ((decl_in_symtab_p (decl
)
1606 && !symtab_node::get (decl
)->can_increase_alignment_p ())
1607 || DECL_USER_ALIGN (decl
) || DECL_ARTIFICIAL (decl
))
1610 alignment
= get_vec_alignment_for_type (TREE_TYPE (decl
));
1611 if (alignment
&& vect_can_force_dr_alignment_p (decl
, alignment
))
1613 vnode
->increase_alignment (alignment
);
1614 if (dump_enabled_p ())
1615 dump_printf (MSG_NOTE
, "Increasing alignment of decl: %T\n", decl
);
1619 delete type_align_map
;
1626 const pass_data pass_data_ipa_increase_alignment
=
1628 SIMPLE_IPA_PASS
, /* type */
1629 "increase_alignment", /* name */
1630 OPTGROUP_LOOP
| OPTGROUP_VEC
, /* optinfo_flags */
1631 TV_IPA_OPT
, /* tv_id */
1632 0, /* properties_required */
1633 0, /* properties_provided */
1634 0, /* properties_destroyed */
1635 0, /* todo_flags_start */
1636 0, /* todo_flags_finish */
1639 class pass_ipa_increase_alignment
: public simple_ipa_opt_pass
1642 pass_ipa_increase_alignment (gcc::context
*ctxt
)
1643 : simple_ipa_opt_pass (pass_data_ipa_increase_alignment
, ctxt
)
1646 /* opt_pass methods: */
1647 virtual bool gate (function
*)
1649 return flag_section_anchors
&& flag_tree_loop_vectorize
;
1652 virtual unsigned int execute (function
*) { return increase_alignment (); }
1654 }; // class pass_ipa_increase_alignment
1658 simple_ipa_opt_pass
*
1659 make_pass_ipa_increase_alignment (gcc::context
*ctxt
)
1661 return new pass_ipa_increase_alignment (ctxt
);
1664 /* If the condition represented by T is a comparison or the SSA name
1665 result of a comparison, extract the comparison's operands. Represent
1666 T as NE_EXPR <T, 0> otherwise. */
1669 scalar_cond_masked_key::get_cond_ops_from_tree (tree t
)
1671 if (TREE_CODE_CLASS (TREE_CODE (t
)) == tcc_comparison
)
1673 this->code
= TREE_CODE (t
);
1674 this->op0
= TREE_OPERAND (t
, 0);
1675 this->op1
= TREE_OPERAND (t
, 1);
1679 if (TREE_CODE (t
) == SSA_NAME
)
1680 if (gassign
*stmt
= dyn_cast
<gassign
*> (SSA_NAME_DEF_STMT (t
)))
1682 tree_code code
= gimple_assign_rhs_code (stmt
);
1683 if (TREE_CODE_CLASS (code
) == tcc_comparison
)
1686 this->op0
= gimple_assign_rhs1 (stmt
);
1687 this->op1
= gimple_assign_rhs2 (stmt
);
1692 this->code
= NE_EXPR
;
1694 this->op1
= build_zero_cst (TREE_TYPE (t
));