1 /* Analysis Utilities for Loop Vectorization.
2 Copyright (C) 2006-2016 Free Software Foundation, Inc.
3 Contributed by Dorit Nuzman <dorit@il.ibm.com>
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
30 #include "optabs-tree.h"
31 #include "insn-config.h"
32 #include "recog.h" /* FIXME: for insn_data */
33 #include "fold-const.h"
34 #include "stor-layout.h"
37 #include "gimple-iterator.h"
39 #include "tree-vectorizer.h"
42 #include "internal-fn.h"
43 #include "case-cfn-macros.h"
45 /* Pattern recognition functions */
46 static gimple
*vect_recog_widen_sum_pattern (vec
<gimple
*> *, tree
*,
48 static gimple
*vect_recog_widen_mult_pattern (vec
<gimple
*> *, tree
*,
50 static gimple
*vect_recog_dot_prod_pattern (vec
<gimple
*> *, tree
*,
52 static gimple
*vect_recog_sad_pattern (vec
<gimple
*> *, tree
*,
54 static gimple
*vect_recog_pow_pattern (vec
<gimple
*> *, tree
*, tree
*);
55 static gimple
*vect_recog_over_widening_pattern (vec
<gimple
*> *, tree
*,
57 static gimple
*vect_recog_widen_shift_pattern (vec
<gimple
*> *,
59 static gimple
*vect_recog_rotate_pattern (vec
<gimple
*> *, tree
*, tree
*);
60 static gimple
*vect_recog_vector_vector_shift_pattern (vec
<gimple
*> *,
62 static gimple
*vect_recog_divmod_pattern (vec
<gimple
*> *,
65 static gimple
*vect_recog_mult_pattern (vec
<gimple
*> *,
68 static gimple
*vect_recog_mixed_size_cond_pattern (vec
<gimple
*> *,
70 static gimple
*vect_recog_bool_pattern (vec
<gimple
*> *, tree
*, tree
*);
71 static gimple
*vect_recog_mask_conversion_pattern (vec
<gimple
*> *, tree
*, tree
*);
73 struct vect_recog_func
75 vect_recog_func_ptr fn
;
78 static vect_recog_func vect_vect_recog_func_ptrs
[NUM_PATTERNS
] = {
79 { vect_recog_widen_mult_pattern
, "widen_mult" },
80 { vect_recog_widen_sum_pattern
, "widen_sum" },
81 { vect_recog_dot_prod_pattern
, "dot_prod" },
82 { vect_recog_sad_pattern
, "sad" },
83 { vect_recog_pow_pattern
, "pow" },
84 { vect_recog_widen_shift_pattern
, "widen_shift" },
85 { vect_recog_over_widening_pattern
, "over_widening" },
86 { vect_recog_rotate_pattern
, "rotate" },
87 { vect_recog_vector_vector_shift_pattern
, "vector_vector_shift" },
88 { vect_recog_divmod_pattern
, "divmod" },
89 { vect_recog_mult_pattern
, "mult" },
90 { vect_recog_mixed_size_cond_pattern
, "mixed_size_cond" },
91 { vect_recog_bool_pattern
, "bool" },
92 { vect_recog_mask_conversion_pattern
, "mask_conversion" }
96 append_pattern_def_seq (stmt_vec_info stmt_info
, gimple
*stmt
)
98 gimple_seq_add_stmt_without_update (&STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
),
103 new_pattern_def_seq (stmt_vec_info stmt_info
, gimple
*stmt
)
105 STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
) = NULL
;
106 append_pattern_def_seq (stmt_info
, stmt
);
109 /* Check whether STMT2 is in the same loop or basic block as STMT1.
110 Which of the two applies depends on whether we're currently doing
111 loop-based or basic-block-based vectorization, as determined by
112 the vinfo_for_stmt for STMT1 (which must be defined).
114 If this returns true, vinfo_for_stmt for STMT2 is guaranteed
115 to be defined as well. */
118 vect_same_loop_or_bb_p (gimple
*stmt1
, gimple
*stmt2
)
120 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt1
);
121 return vect_stmt_in_region_p (stmt_vinfo
->vinfo
, stmt2
);
124 /* If the LHS of DEF_STMT has a single use, and that statement is
125 in the same loop or basic block, return it. */
128 vect_single_imm_use (gimple
*def_stmt
)
130 tree lhs
= gimple_assign_lhs (def_stmt
);
134 if (!single_imm_use (lhs
, &use_p
, &use_stmt
))
137 if (!vect_same_loop_or_bb_p (def_stmt
, use_stmt
))
143 /* Check whether NAME, an ssa-name used in USE_STMT,
144 is a result of a type promotion, such that:
145 DEF_STMT: NAME = NOP (name0)
146 If CHECK_SIGN is TRUE, check that either both types are signed or both are
150 type_conversion_p (tree name
, gimple
*use_stmt
, bool check_sign
,
151 tree
*orig_type
, gimple
**def_stmt
, bool *promotion
)
153 gimple
*dummy_gimple
;
154 stmt_vec_info stmt_vinfo
;
155 tree type
= TREE_TYPE (name
);
157 enum vect_def_type dt
;
159 stmt_vinfo
= vinfo_for_stmt (use_stmt
);
160 if (!vect_is_simple_use (name
, stmt_vinfo
->vinfo
, def_stmt
, &dt
))
163 if (dt
!= vect_internal_def
164 && dt
!= vect_external_def
&& dt
!= vect_constant_def
)
170 if (!is_gimple_assign (*def_stmt
))
173 if (!CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (*def_stmt
)))
176 oprnd0
= gimple_assign_rhs1 (*def_stmt
);
178 *orig_type
= TREE_TYPE (oprnd0
);
179 if (!INTEGRAL_TYPE_P (type
) || !INTEGRAL_TYPE_P (*orig_type
)
180 || ((TYPE_UNSIGNED (type
) != TYPE_UNSIGNED (*orig_type
)) && check_sign
))
183 if (TYPE_PRECISION (type
) >= (TYPE_PRECISION (*orig_type
) * 2))
188 if (!vect_is_simple_use (oprnd0
, stmt_vinfo
->vinfo
, &dummy_gimple
, &dt
))
194 /* Helper to return a new temporary for pattern of TYPE for STMT. If STMT
195 is NULL, the caller must set SSA_NAME_DEF_STMT for the returned SSA var. */
198 vect_recog_temp_ssa_var (tree type
, gimple
*stmt
)
200 return make_temp_ssa_name (type
, stmt
, "patt");
203 /* Function vect_recog_dot_prod_pattern
205 Try to find the following pattern:
211 sum_0 = phi <init, sum_1>
214 S3 x_T = (TYPE1) x_t;
215 S4 y_T = (TYPE1) y_t;
217 [S6 prod = (TYPE2) prod; #optional]
218 S7 sum_1 = prod + sum_0;
220 where 'TYPE1' is exactly double the size of type 'type', and 'TYPE2' is the
221 same size of 'TYPE1' or bigger. This is a special case of a reduction
226 * STMTS: Contains a stmt from which the pattern search begins. In the
227 example, when this function is called with S7, the pattern {S3,S4,S5,S6,S7}
232 * TYPE_IN: The type of the input arguments to the pattern.
234 * TYPE_OUT: The type of the output of this pattern.
236 * Return value: A new stmt that will be used to replace the sequence of
237 stmts that constitute the pattern. In this case it will be:
238 WIDEN_DOT_PRODUCT <x_t, y_t, sum_0>
240 Note: The dot-prod idiom is a widening reduction pattern that is
241 vectorized without preserving all the intermediate results. It
242 produces only N/2 (widened) results (by summing up pairs of
243 intermediate results) rather than all N results. Therefore, we
244 cannot allow this pattern when we want to get all the results and in
245 the correct order (as is the case when this computation is in an
246 inner-loop nested in an outer-loop that us being vectorized). */
249 vect_recog_dot_prod_pattern (vec
<gimple
*> *stmts
, tree
*type_in
,
252 gimple
*stmt
, *last_stmt
= (*stmts
)[0];
254 tree oprnd00
, oprnd01
;
255 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (last_stmt
);
256 tree type
, half_type
;
257 gimple
*pattern_stmt
;
259 loop_vec_info loop_info
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
267 loop
= LOOP_VINFO_LOOP (loop_info
);
269 /* We don't allow changing the order of the computation in the inner-loop
270 when doing outer-loop vectorization. */
271 if (loop
&& nested_in_vect_loop_p (loop
, last_stmt
))
274 if (!is_gimple_assign (last_stmt
))
277 type
= gimple_expr_type (last_stmt
);
279 /* Look for the following pattern
283 DDPROD = (TYPE2) DPROD;
284 sum_1 = DDPROD + sum_0;
286 - DX is double the size of X
287 - DY is double the size of Y
288 - DX, DY, DPROD all have the same type
289 - sum is the same size of DPROD or bigger
290 - sum has been recognized as a reduction variable.
292 This is equivalent to:
293 DPROD = X w* Y; #widen mult
294 sum_1 = DPROD w+ sum_0; #widen summation
296 DPROD = X w* Y; #widen mult
297 sum_1 = DPROD + sum_0; #summation
300 /* Starting from LAST_STMT, follow the defs of its uses in search
301 of the above pattern. */
303 if (gimple_assign_rhs_code (last_stmt
) != PLUS_EXPR
)
306 if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo
))
308 /* Has been detected as widening-summation? */
310 stmt
= STMT_VINFO_RELATED_STMT (stmt_vinfo
);
311 type
= gimple_expr_type (stmt
);
312 if (gimple_assign_rhs_code (stmt
) != WIDEN_SUM_EXPR
)
314 oprnd0
= gimple_assign_rhs1 (stmt
);
315 oprnd1
= gimple_assign_rhs2 (stmt
);
316 half_type
= TREE_TYPE (oprnd0
);
322 if (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
323 && ! STMT_VINFO_GROUP_FIRST_ELEMENT (stmt_vinfo
))
325 oprnd0
= gimple_assign_rhs1 (last_stmt
);
326 oprnd1
= gimple_assign_rhs2 (last_stmt
);
327 if (!types_compatible_p (TREE_TYPE (oprnd0
), type
)
328 || !types_compatible_p (TREE_TYPE (oprnd1
), type
))
332 if (type_conversion_p (oprnd0
, stmt
, true, &half_type
, &def_stmt
,
337 oprnd0
= gimple_assign_rhs1 (stmt
);
343 /* So far so good. Since last_stmt was detected as a (summation) reduction,
344 we know that oprnd1 is the reduction variable (defined by a loop-header
345 phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
346 Left to check that oprnd0 is defined by a (widen_)mult_expr */
347 if (TREE_CODE (oprnd0
) != SSA_NAME
)
350 prod_type
= half_type
;
351 stmt
= SSA_NAME_DEF_STMT (oprnd0
);
353 /* It could not be the dot_prod pattern if the stmt is outside the loop. */
354 if (!gimple_bb (stmt
) || !flow_bb_inside_loop_p (loop
, gimple_bb (stmt
)))
357 /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
358 inside the loop (in case we are analyzing an outer-loop). */
359 if (!is_gimple_assign (stmt
))
361 stmt_vinfo
= vinfo_for_stmt (stmt
);
362 gcc_assert (stmt_vinfo
);
363 if (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_internal_def
)
365 if (gimple_assign_rhs_code (stmt
) != MULT_EXPR
)
367 if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo
))
369 /* Has been detected as a widening multiplication? */
371 stmt
= STMT_VINFO_RELATED_STMT (stmt_vinfo
);
372 if (gimple_assign_rhs_code (stmt
) != WIDEN_MULT_EXPR
)
374 stmt_vinfo
= vinfo_for_stmt (stmt
);
375 gcc_assert (stmt_vinfo
);
376 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_internal_def
);
377 oprnd00
= gimple_assign_rhs1 (stmt
);
378 oprnd01
= gimple_assign_rhs2 (stmt
);
379 STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (last_stmt
))
380 = STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo
);
384 tree half_type0
, half_type1
;
388 oprnd0
= gimple_assign_rhs1 (stmt
);
389 oprnd1
= gimple_assign_rhs2 (stmt
);
390 if (!types_compatible_p (TREE_TYPE (oprnd0
), prod_type
)
391 || !types_compatible_p (TREE_TYPE (oprnd1
), prod_type
))
393 if (!type_conversion_p (oprnd0
, stmt
, true, &half_type0
, &def_stmt
,
397 oprnd00
= gimple_assign_rhs1 (def_stmt
);
398 if (!type_conversion_p (oprnd1
, stmt
, true, &half_type1
, &def_stmt
,
402 oprnd01
= gimple_assign_rhs1 (def_stmt
);
403 if (!types_compatible_p (half_type0
, half_type1
))
405 if (TYPE_PRECISION (prod_type
) != TYPE_PRECISION (half_type0
) * 2)
409 half_type
= TREE_TYPE (oprnd00
);
410 *type_in
= half_type
;
413 /* Pattern detected. Create a stmt to be used to replace the pattern: */
414 var
= vect_recog_temp_ssa_var (type
, NULL
);
415 pattern_stmt
= gimple_build_assign (var
, DOT_PROD_EXPR
,
416 oprnd00
, oprnd01
, oprnd1
);
418 if (dump_enabled_p ())
420 dump_printf_loc (MSG_NOTE
, vect_location
,
421 "vect_recog_dot_prod_pattern: detected: ");
422 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_stmt
, 0);
423 dump_printf (MSG_NOTE
, "\n");
430 /* Function vect_recog_sad_pattern
432 Try to find the following Sum of Absolute Difference (SAD) pattern:
435 signed TYPE1 diff, abs_diff;
438 sum_0 = phi <init, sum_1>
441 S3 x_T = (TYPE1) x_t;
442 S4 y_T = (TYPE1) y_t;
444 S6 abs_diff = ABS_EXPR <diff>;
445 [S7 abs_diff = (TYPE2) abs_diff; #optional]
446 S8 sum_1 = abs_diff + sum_0;
448 where 'TYPE1' is at least double the size of type 'type', and 'TYPE2' is the
449 same size of 'TYPE1' or bigger. This is a special case of a reduction
454 * STMTS: Contains a stmt from which the pattern search begins. In the
455 example, when this function is called with S8, the pattern
456 {S3,S4,S5,S6,S7,S8} will be detected.
460 * TYPE_IN: The type of the input arguments to the pattern.
462 * TYPE_OUT: The type of the output of this pattern.
464 * Return value: A new stmt that will be used to replace the sequence of
465 stmts that constitute the pattern. In this case it will be:
466 SAD_EXPR <x_t, y_t, sum_0>
470 vect_recog_sad_pattern (vec
<gimple
*> *stmts
, tree
*type_in
,
473 gimple
*last_stmt
= (*stmts
)[0];
474 tree sad_oprnd0
, sad_oprnd1
;
475 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (last_stmt
);
477 loop_vec_info loop_info
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
484 loop
= LOOP_VINFO_LOOP (loop_info
);
486 /* We don't allow changing the order of the computation in the inner-loop
487 when doing outer-loop vectorization. */
488 if (loop
&& nested_in_vect_loop_p (loop
, last_stmt
))
491 if (!is_gimple_assign (last_stmt
))
494 tree sum_type
= gimple_expr_type (last_stmt
);
496 /* Look for the following pattern
500 DAD = ABS_EXPR <DDIFF>;
501 DDPROD = (TYPE2) DPROD;
504 - DX is at least double the size of X
505 - DY is at least double the size of Y
506 - DX, DY, DDIFF, DAD all have the same type
507 - sum is the same size of DAD or bigger
508 - sum has been recognized as a reduction variable.
510 This is equivalent to:
511 DDIFF = X w- Y; #widen sub
512 DAD = ABS_EXPR <DDIFF>;
513 sum_1 = DAD w+ sum_0; #widen summation
515 DDIFF = X w- Y; #widen sub
516 DAD = ABS_EXPR <DDIFF>;
517 sum_1 = DAD + sum_0; #summation
520 /* Starting from LAST_STMT, follow the defs of its uses in search
521 of the above pattern. */
523 if (gimple_assign_rhs_code (last_stmt
) != PLUS_EXPR
)
526 tree plus_oprnd0
, plus_oprnd1
;
528 if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo
))
530 /* Has been detected as widening-summation? */
532 gimple
*stmt
= STMT_VINFO_RELATED_STMT (stmt_vinfo
);
533 sum_type
= gimple_expr_type (stmt
);
534 if (gimple_assign_rhs_code (stmt
) != WIDEN_SUM_EXPR
)
536 plus_oprnd0
= gimple_assign_rhs1 (stmt
);
537 plus_oprnd1
= gimple_assign_rhs2 (stmt
);
538 half_type
= TREE_TYPE (plus_oprnd0
);
544 if (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
545 && ! STMT_VINFO_GROUP_FIRST_ELEMENT (stmt_vinfo
))
547 plus_oprnd0
= gimple_assign_rhs1 (last_stmt
);
548 plus_oprnd1
= gimple_assign_rhs2 (last_stmt
);
549 if (!types_compatible_p (TREE_TYPE (plus_oprnd0
), sum_type
)
550 || !types_compatible_p (TREE_TYPE (plus_oprnd1
), sum_type
))
553 /* The type conversion could be promotion, demotion,
554 or just signed -> unsigned. */
555 if (type_conversion_p (plus_oprnd0
, last_stmt
, false,
556 &half_type
, &def_stmt
, &promotion
))
557 plus_oprnd0
= gimple_assign_rhs1 (def_stmt
);
559 half_type
= sum_type
;
562 /* So far so good. Since last_stmt was detected as a (summation) reduction,
563 we know that plus_oprnd1 is the reduction variable (defined by a loop-header
564 phi), and plus_oprnd0 is an ssa-name defined by a stmt in the loop body.
565 Then check that plus_oprnd0 is defined by an abs_expr. */
567 if (TREE_CODE (plus_oprnd0
) != SSA_NAME
)
570 tree abs_type
= half_type
;
571 gimple
*abs_stmt
= SSA_NAME_DEF_STMT (plus_oprnd0
);
573 /* It could not be the sad pattern if the abs_stmt is outside the loop. */
574 if (!gimple_bb (abs_stmt
) || !flow_bb_inside_loop_p (loop
, gimple_bb (abs_stmt
)))
577 /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
578 inside the loop (in case we are analyzing an outer-loop). */
579 if (!is_gimple_assign (abs_stmt
))
582 stmt_vec_info abs_stmt_vinfo
= vinfo_for_stmt (abs_stmt
);
583 gcc_assert (abs_stmt_vinfo
);
584 if (STMT_VINFO_DEF_TYPE (abs_stmt_vinfo
) != vect_internal_def
)
586 if (gimple_assign_rhs_code (abs_stmt
) != ABS_EXPR
)
589 tree abs_oprnd
= gimple_assign_rhs1 (abs_stmt
);
590 if (!types_compatible_p (TREE_TYPE (abs_oprnd
), abs_type
))
592 if (TYPE_UNSIGNED (abs_type
))
595 /* We then detect if the operand of abs_expr is defined by a minus_expr. */
597 if (TREE_CODE (abs_oprnd
) != SSA_NAME
)
600 gimple
*diff_stmt
= SSA_NAME_DEF_STMT (abs_oprnd
);
602 /* It could not be the sad pattern if the diff_stmt is outside the loop. */
603 if (!gimple_bb (diff_stmt
)
604 || !flow_bb_inside_loop_p (loop
, gimple_bb (diff_stmt
)))
607 /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
608 inside the loop (in case we are analyzing an outer-loop). */
609 if (!is_gimple_assign (diff_stmt
))
612 stmt_vec_info diff_stmt_vinfo
= vinfo_for_stmt (diff_stmt
);
613 gcc_assert (diff_stmt_vinfo
);
614 if (STMT_VINFO_DEF_TYPE (diff_stmt_vinfo
) != vect_internal_def
)
616 if (gimple_assign_rhs_code (diff_stmt
) != MINUS_EXPR
)
619 tree half_type0
, half_type1
;
622 tree minus_oprnd0
= gimple_assign_rhs1 (diff_stmt
);
623 tree minus_oprnd1
= gimple_assign_rhs2 (diff_stmt
);
625 if (!types_compatible_p (TREE_TYPE (minus_oprnd0
), abs_type
)
626 || !types_compatible_p (TREE_TYPE (minus_oprnd1
), abs_type
))
628 if (!type_conversion_p (minus_oprnd0
, diff_stmt
, false,
629 &half_type0
, &def_stmt
, &promotion
)
632 sad_oprnd0
= gimple_assign_rhs1 (def_stmt
);
634 if (!type_conversion_p (minus_oprnd1
, diff_stmt
, false,
635 &half_type1
, &def_stmt
, &promotion
)
638 sad_oprnd1
= gimple_assign_rhs1 (def_stmt
);
640 if (!types_compatible_p (half_type0
, half_type1
))
642 if (TYPE_PRECISION (abs_type
) < TYPE_PRECISION (half_type0
) * 2
643 || TYPE_PRECISION (sum_type
) < TYPE_PRECISION (half_type0
) * 2)
646 *type_in
= TREE_TYPE (sad_oprnd0
);
647 *type_out
= sum_type
;
649 /* Pattern detected. Create a stmt to be used to replace the pattern: */
650 tree var
= vect_recog_temp_ssa_var (sum_type
, NULL
);
651 gimple
*pattern_stmt
= gimple_build_assign (var
, SAD_EXPR
, sad_oprnd0
,
652 sad_oprnd1
, plus_oprnd1
);
654 if (dump_enabled_p ())
656 dump_printf_loc (MSG_NOTE
, vect_location
,
657 "vect_recog_sad_pattern: detected: ");
658 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_stmt
, 0);
659 dump_printf (MSG_NOTE
, "\n");
666 /* Handle widening operation by a constant. At the moment we support MULT_EXPR
669 For MULT_EXPR we check that CONST_OPRND fits HALF_TYPE, and for LSHIFT_EXPR
670 we check that CONST_OPRND is less or equal to the size of HALF_TYPE.
672 Otherwise, if the type of the result (TYPE) is at least 4 times bigger than
673 HALF_TYPE, and there is an intermediate type (2 times smaller than TYPE)
674 that satisfies the above restrictions, we can perform a widening opeartion
675 from the intermediate type to TYPE and replace a_T = (TYPE) a_t;
676 with a_it = (interm_type) a_t; Store such operation in *WSTMT. */
679 vect_handle_widen_op_by_const (gimple
*stmt
, enum tree_code code
,
680 tree const_oprnd
, tree
*oprnd
,
681 gimple
**wstmt
, tree type
,
682 tree
*half_type
, gimple
*def_stmt
)
684 tree new_type
, new_oprnd
;
686 if (code
!= MULT_EXPR
&& code
!= LSHIFT_EXPR
)
689 if (((code
== MULT_EXPR
&& int_fits_type_p (const_oprnd
, *half_type
))
690 || (code
== LSHIFT_EXPR
691 && compare_tree_int (const_oprnd
, TYPE_PRECISION (*half_type
))
693 && TYPE_PRECISION (type
) == (TYPE_PRECISION (*half_type
) * 2))
695 /* CONST_OPRND is a constant of HALF_TYPE. */
696 *oprnd
= gimple_assign_rhs1 (def_stmt
);
700 if (TYPE_PRECISION (type
) < (TYPE_PRECISION (*half_type
) * 4))
703 if (!vect_same_loop_or_bb_p (stmt
, def_stmt
))
706 /* TYPE is 4 times bigger than HALF_TYPE, try widening operation for
707 a type 2 times bigger than HALF_TYPE. */
708 new_type
= build_nonstandard_integer_type (TYPE_PRECISION (type
) / 2,
709 TYPE_UNSIGNED (type
));
710 if ((code
== MULT_EXPR
&& !int_fits_type_p (const_oprnd
, new_type
))
711 || (code
== LSHIFT_EXPR
712 && compare_tree_int (const_oprnd
, TYPE_PRECISION (new_type
)) == 1))
715 /* Use NEW_TYPE for widening operation and create a_T = (NEW_TYPE) a_t; */
716 *oprnd
= gimple_assign_rhs1 (def_stmt
);
717 new_oprnd
= make_ssa_name (new_type
);
718 *wstmt
= gimple_build_assign (new_oprnd
, NOP_EXPR
, *oprnd
);
721 *half_type
= new_type
;
726 /* Function vect_recog_widen_mult_pattern
728 Try to find the following pattern:
732 TYPE a_T, b_T, prod_T;
738 S5 prod_T = a_T * b_T;
740 where type 'TYPE' is at least double the size of type 'type1' and 'type2'.
742 Also detect unsigned cases:
746 unsigned TYPE u_prod_T;
747 TYPE a_T, b_T, prod_T;
753 S5 prod_T = a_T * b_T;
754 S6 u_prod_T = (unsigned TYPE) prod_T;
756 and multiplication by constants:
763 S5 prod_T = a_T * CONST;
765 A special case of multiplication by constants is when 'TYPE' is 4 times
766 bigger than 'type', but CONST fits an intermediate type 2 times smaller
767 than 'TYPE'. In that case we create an additional pattern stmt for S3
768 to create a variable of the intermediate type, and perform widen-mult
769 on the intermediate type as well:
773 TYPE a_T, prod_T, prod_T';
777 '--> a_it = (interm_type) a_t;
778 S5 prod_T = a_T * CONST;
779 '--> prod_T' = a_it w* CONST;
783 * STMTS: Contains a stmt from which the pattern search begins. In the
784 example, when this function is called with S5, the pattern {S3,S4,S5,(S6)}
785 is detected. In case of unsigned widen-mult, the original stmt (S5) is
786 replaced with S6 in STMTS. In case of multiplication by a constant
787 of an intermediate type (the last case above), STMTS also contains S3
788 (inserted before S5).
792 * TYPE_IN: The type of the input arguments to the pattern.
794 * TYPE_OUT: The type of the output of this pattern.
796 * Return value: A new stmt that will be used to replace the sequence of
797 stmts that constitute the pattern. In this case it will be:
798 WIDEN_MULT <a_t, b_t>
799 If the result of WIDEN_MULT needs to be converted to a larger type, the
800 returned stmt will be this type conversion stmt.
804 vect_recog_widen_mult_pattern (vec
<gimple
*> *stmts
,
805 tree
*type_in
, tree
*type_out
)
807 gimple
*last_stmt
= stmts
->pop ();
808 gimple
*def_stmt0
, *def_stmt1
;
810 tree type
, half_type0
, half_type1
;
811 gimple
*new_stmt
= NULL
, *pattern_stmt
= NULL
;
812 tree vectype
, vecitype
;
814 enum tree_code dummy_code
;
820 if (!is_gimple_assign (last_stmt
))
823 type
= gimple_expr_type (last_stmt
);
825 /* Starting from LAST_STMT, follow the defs of its uses in search
826 of the above pattern. */
828 if (gimple_assign_rhs_code (last_stmt
) != MULT_EXPR
)
831 oprnd0
= gimple_assign_rhs1 (last_stmt
);
832 oprnd1
= gimple_assign_rhs2 (last_stmt
);
833 if (!types_compatible_p (TREE_TYPE (oprnd0
), type
)
834 || !types_compatible_p (TREE_TYPE (oprnd1
), type
))
837 /* Check argument 0. */
838 if (!type_conversion_p (oprnd0
, last_stmt
, false, &half_type0
, &def_stmt0
,
842 /* Check argument 1. */
843 op1_ok
= type_conversion_p (oprnd1
, last_stmt
, false, &half_type1
,
844 &def_stmt1
, &promotion
);
846 if (op1_ok
&& promotion
)
848 oprnd0
= gimple_assign_rhs1 (def_stmt0
);
849 oprnd1
= gimple_assign_rhs1 (def_stmt1
);
853 if (TREE_CODE (oprnd1
) == INTEGER_CST
854 && TREE_CODE (half_type0
) == INTEGER_TYPE
855 && vect_handle_widen_op_by_const (last_stmt
, MULT_EXPR
, oprnd1
,
856 &oprnd0
, &new_stmt
, type
,
857 &half_type0
, def_stmt0
))
859 half_type1
= half_type0
;
860 oprnd1
= fold_convert (half_type1
, oprnd1
);
866 /* If the two arguments have different sizes, convert the one with
867 the smaller type into the larger type. */
868 if (TYPE_PRECISION (half_type0
) != TYPE_PRECISION (half_type1
))
870 /* If we already used up the single-stmt slot give up. */
875 gimple
*def_stmt
= NULL
;
877 if (TYPE_PRECISION (half_type0
) < TYPE_PRECISION (half_type1
))
879 def_stmt
= def_stmt0
;
880 half_type0
= half_type1
;
885 def_stmt
= def_stmt1
;
886 half_type1
= half_type0
;
890 tree old_oprnd
= gimple_assign_rhs1 (def_stmt
);
891 tree new_oprnd
= make_ssa_name (half_type0
);
892 new_stmt
= gimple_build_assign (new_oprnd
, NOP_EXPR
, old_oprnd
);
896 /* Handle unsigned case. Look for
897 S6 u_prod_T = (unsigned TYPE) prod_T;
898 Use unsigned TYPE as the type for WIDEN_MULT_EXPR. */
899 if (TYPE_UNSIGNED (type
) != TYPE_UNSIGNED (half_type0
))
905 if (TYPE_UNSIGNED (type
) == TYPE_UNSIGNED (half_type1
))
908 use_stmt
= vect_single_imm_use (last_stmt
);
909 if (!use_stmt
|| !is_gimple_assign (use_stmt
)
910 || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (use_stmt
)))
913 use_lhs
= gimple_assign_lhs (use_stmt
);
914 use_type
= TREE_TYPE (use_lhs
);
915 if (!INTEGRAL_TYPE_P (use_type
)
916 || (TYPE_UNSIGNED (type
) == TYPE_UNSIGNED (use_type
))
917 || (TYPE_PRECISION (type
) != TYPE_PRECISION (use_type
)))
921 last_stmt
= use_stmt
;
924 if (!types_compatible_p (half_type0
, half_type1
))
927 /* If TYPE is more than twice larger than HALF_TYPE, we use WIDEN_MULT
928 to get an intermediate result of type ITYPE. In this case we need
929 to build a statement to convert this intermediate result to type TYPE. */
931 if (TYPE_PRECISION (type
) > TYPE_PRECISION (half_type0
) * 2)
932 itype
= build_nonstandard_integer_type
933 (GET_MODE_BITSIZE (TYPE_MODE (half_type0
)) * 2,
934 TYPE_UNSIGNED (type
));
936 /* Pattern detected. */
937 if (dump_enabled_p ())
938 dump_printf_loc (MSG_NOTE
, vect_location
,
939 "vect_recog_widen_mult_pattern: detected:\n");
941 /* Check target support */
942 vectype
= get_vectype_for_scalar_type (half_type0
);
943 vecitype
= get_vectype_for_scalar_type (itype
);
946 || !supportable_widening_operation (WIDEN_MULT_EXPR
, last_stmt
,
948 &dummy_code
, &dummy_code
,
949 &dummy_int
, &dummy_vec
))
953 *type_out
= get_vectype_for_scalar_type (type
);
955 /* Pattern supported. Create a stmt to be used to replace the pattern: */
956 var
= vect_recog_temp_ssa_var (itype
, NULL
);
957 pattern_stmt
= gimple_build_assign (var
, WIDEN_MULT_EXPR
, oprnd0
, oprnd1
);
959 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (last_stmt
);
960 STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo
) = NULL
;
962 /* If the original two operands have different sizes, we may need to convert
963 the smaller one into the larget type. If this is the case, at this point
964 the new stmt is already built. */
967 append_pattern_def_seq (stmt_vinfo
, new_stmt
);
968 stmt_vec_info new_stmt_info
969 = new_stmt_vec_info (new_stmt
, stmt_vinfo
->vinfo
);
970 set_vinfo_for_stmt (new_stmt
, new_stmt_info
);
971 STMT_VINFO_VECTYPE (new_stmt_info
) = vectype
;
974 /* If ITYPE is not TYPE, we need to build a type convertion stmt to convert
975 the result of the widen-mult operation into type TYPE. */
978 append_pattern_def_seq (stmt_vinfo
, pattern_stmt
);
979 stmt_vec_info pattern_stmt_info
980 = new_stmt_vec_info (pattern_stmt
, stmt_vinfo
->vinfo
);
981 set_vinfo_for_stmt (pattern_stmt
, pattern_stmt_info
);
982 STMT_VINFO_VECTYPE (pattern_stmt_info
) = vecitype
;
983 pattern_stmt
= gimple_build_assign (vect_recog_temp_ssa_var (type
, NULL
),
985 gimple_assign_lhs (pattern_stmt
));
988 if (dump_enabled_p ())
989 dump_gimple_stmt_loc (MSG_NOTE
, vect_location
, TDF_SLIM
, pattern_stmt
, 0);
991 stmts
->safe_push (last_stmt
);
996 /* Function vect_recog_pow_pattern
998 Try to find the following pattern:
1002 with POW being one of pow, powf, powi, powif and N being
1007 * LAST_STMT: A stmt from which the pattern search begins.
1011 * TYPE_IN: The type of the input arguments to the pattern.
1013 * TYPE_OUT: The type of the output of this pattern.
1015 * Return value: A new stmt that will be used to replace the sequence of
1016 stmts that constitute the pattern. In this case it will be:
1023 vect_recog_pow_pattern (vec
<gimple
*> *stmts
, tree
*type_in
,
1026 gimple
*last_stmt
= (*stmts
)[0];
1027 tree base
, exp
= NULL
;
1031 if (!is_gimple_call (last_stmt
) || gimple_call_lhs (last_stmt
) == NULL
)
1034 switch (gimple_call_combined_fn (last_stmt
))
1038 base
= gimple_call_arg (last_stmt
, 0);
1039 exp
= gimple_call_arg (last_stmt
, 1);
1040 if (TREE_CODE (exp
) != REAL_CST
1041 && TREE_CODE (exp
) != INTEGER_CST
)
1049 /* We now have a pow or powi builtin function call with a constant
1052 *type_out
= NULL_TREE
;
1054 /* Catch squaring. */
1055 if ((tree_fits_shwi_p (exp
)
1056 && tree_to_shwi (exp
) == 2)
1057 || (TREE_CODE (exp
) == REAL_CST
1058 && real_equal (&TREE_REAL_CST (exp
), &dconst2
)))
1060 *type_in
= TREE_TYPE (base
);
1062 var
= vect_recog_temp_ssa_var (TREE_TYPE (base
), NULL
);
1063 stmt
= gimple_build_assign (var
, MULT_EXPR
, base
, base
);
1067 /* Catch square root. */
1068 if (TREE_CODE (exp
) == REAL_CST
1069 && real_equal (&TREE_REAL_CST (exp
), &dconsthalf
))
1071 *type_in
= get_vectype_for_scalar_type (TREE_TYPE (base
));
1073 && direct_internal_fn_supported_p (IFN_SQRT
, *type_in
,
1074 OPTIMIZE_FOR_SPEED
))
1076 gcall
*stmt
= gimple_build_call_internal (IFN_SQRT
, 1, base
);
1077 var
= vect_recog_temp_ssa_var (TREE_TYPE (base
), stmt
);
1078 gimple_call_set_lhs (stmt
, var
);
1087 /* Function vect_recog_widen_sum_pattern
1089 Try to find the following pattern:
1092 TYPE x_T, sum = init;
1094 sum_0 = phi <init, sum_1>
1096 S2 x_T = (TYPE) x_t;
1097 S3 sum_1 = x_T + sum_0;
1099 where type 'TYPE' is at least double the size of type 'type', i.e - we're
1100 summing elements of type 'type' into an accumulator of type 'TYPE'. This is
1101 a special case of a reduction computation.
1105 * LAST_STMT: A stmt from which the pattern search begins. In the example,
1106 when this function is called with S3, the pattern {S2,S3} will be detected.
1110 * TYPE_IN: The type of the input arguments to the pattern.
1112 * TYPE_OUT: The type of the output of this pattern.
1114 * Return value: A new stmt that will be used to replace the sequence of
1115 stmts that constitute the pattern. In this case it will be:
1116 WIDEN_SUM <x_t, sum_0>
1118 Note: The widening-sum idiom is a widening reduction pattern that is
1119 vectorized without preserving all the intermediate results. It
1120 produces only N/2 (widened) results (by summing up pairs of
1121 intermediate results) rather than all N results. Therefore, we
1122 cannot allow this pattern when we want to get all the results and in
1123 the correct order (as is the case when this computation is in an
1124 inner-loop nested in an outer-loop that us being vectorized). */
1127 vect_recog_widen_sum_pattern (vec
<gimple
*> *stmts
, tree
*type_in
,
1130 gimple
*stmt
, *last_stmt
= (*stmts
)[0];
1131 tree oprnd0
, oprnd1
;
1132 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (last_stmt
);
1133 tree type
, half_type
;
1134 gimple
*pattern_stmt
;
1135 loop_vec_info loop_info
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1143 loop
= LOOP_VINFO_LOOP (loop_info
);
1145 /* We don't allow changing the order of the computation in the inner-loop
1146 when doing outer-loop vectorization. */
1147 if (loop
&& nested_in_vect_loop_p (loop
, last_stmt
))
1150 if (!is_gimple_assign (last_stmt
))
1153 type
= gimple_expr_type (last_stmt
);
1155 /* Look for the following pattern
1158 In which DX is at least double the size of X, and sum_1 has been
1159 recognized as a reduction variable.
1162 /* Starting from LAST_STMT, follow the defs of its uses in search
1163 of the above pattern. */
1165 if (gimple_assign_rhs_code (last_stmt
) != PLUS_EXPR
)
1168 if (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
1169 && ! STMT_VINFO_GROUP_FIRST_ELEMENT (stmt_vinfo
))
1172 oprnd0
= gimple_assign_rhs1 (last_stmt
);
1173 oprnd1
= gimple_assign_rhs2 (last_stmt
);
1174 if (!types_compatible_p (TREE_TYPE (oprnd0
), type
)
1175 || !types_compatible_p (TREE_TYPE (oprnd1
), type
))
1178 /* So far so good. Since last_stmt was detected as a (summation) reduction,
1179 we know that oprnd1 is the reduction variable (defined by a loop-header
1180 phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
1181 Left to check that oprnd0 is defined by a cast from type 'type' to type
1184 if (!type_conversion_p (oprnd0
, last_stmt
, true, &half_type
, &stmt
,
1189 oprnd0
= gimple_assign_rhs1 (stmt
);
1190 *type_in
= half_type
;
1193 /* Pattern detected. Create a stmt to be used to replace the pattern: */
1194 var
= vect_recog_temp_ssa_var (type
, NULL
);
1195 pattern_stmt
= gimple_build_assign (var
, WIDEN_SUM_EXPR
, oprnd0
, oprnd1
);
1197 if (dump_enabled_p ())
1199 dump_printf_loc (MSG_NOTE
, vect_location
,
1200 "vect_recog_widen_sum_pattern: detected: ");
1201 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_stmt
, 0);
1202 dump_printf (MSG_NOTE
, "\n");
1205 return pattern_stmt
;
1209 /* Return TRUE if the operation in STMT can be performed on a smaller type.
1212 STMT - a statement to check.
1213 DEF - we support operations with two operands, one of which is constant.
1214 The other operand can be defined by a demotion operation, or by a
1215 previous statement in a sequence of over-promoted operations. In the
1216 later case DEF is used to replace that operand. (It is defined by a
1217 pattern statement we created for the previous statement in the
1221 NEW_TYPE - Output: a smaller type that we are trying to use. Input: if not
1222 NULL, it's the type of DEF.
1223 STMTS - additional pattern statements. If a pattern statement (type
1224 conversion) is created in this function, its original statement is
1228 OP0, OP1 - if the operation fits a smaller type, OP0 and OP1 are the new
1229 operands to use in the new pattern statement for STMT (will be created
1230 in vect_recog_over_widening_pattern ()).
1231 NEW_DEF_STMT - in case DEF has to be promoted, we create two pattern
1232 statements for STMT: the first one is a type promotion and the second
1233 one is the operation itself. We return the type promotion statement
1234 in NEW_DEF_STMT and further store it in STMT_VINFO_PATTERN_DEF_SEQ of
1235 the second pattern statement. */
1238 vect_operation_fits_smaller_type (gimple
*stmt
, tree def
, tree
*new_type
,
1239 tree
*op0
, tree
*op1
, gimple
**new_def_stmt
,
1240 vec
<gimple
*> *stmts
)
1242 enum tree_code code
;
1243 tree const_oprnd
, oprnd
;
1244 tree interm_type
= NULL_TREE
, half_type
, new_oprnd
, type
;
1245 gimple
*def_stmt
, *new_stmt
;
1251 *new_def_stmt
= NULL
;
1253 if (!is_gimple_assign (stmt
))
1256 code
= gimple_assign_rhs_code (stmt
);
1257 if (code
!= LSHIFT_EXPR
&& code
!= RSHIFT_EXPR
1258 && code
!= BIT_IOR_EXPR
&& code
!= BIT_XOR_EXPR
&& code
!= BIT_AND_EXPR
)
1261 oprnd
= gimple_assign_rhs1 (stmt
);
1262 const_oprnd
= gimple_assign_rhs2 (stmt
);
1263 type
= gimple_expr_type (stmt
);
1265 if (TREE_CODE (oprnd
) != SSA_NAME
1266 || TREE_CODE (const_oprnd
) != INTEGER_CST
)
1269 /* If oprnd has other uses besides that in stmt we cannot mark it
1270 as being part of a pattern only. */
1271 if (!has_single_use (oprnd
))
1274 /* If we are in the middle of a sequence, we use DEF from a previous
1275 statement. Otherwise, OPRND has to be a result of type promotion. */
1278 half_type
= *new_type
;
1284 if (!type_conversion_p (oprnd
, stmt
, false, &half_type
, &def_stmt
,
1287 || !vect_same_loop_or_bb_p (stmt
, def_stmt
))
1291 /* Can we perform the operation on a smaller type? */
1297 if (!int_fits_type_p (const_oprnd
, half_type
))
1299 /* HALF_TYPE is not enough. Try a bigger type if possible. */
1300 if (TYPE_PRECISION (type
) < (TYPE_PRECISION (half_type
) * 4))
1303 interm_type
= build_nonstandard_integer_type (
1304 TYPE_PRECISION (half_type
) * 2, TYPE_UNSIGNED (type
));
1305 if (!int_fits_type_p (const_oprnd
, interm_type
))
1312 /* Try intermediate type - HALF_TYPE is not enough for sure. */
1313 if (TYPE_PRECISION (type
) < (TYPE_PRECISION (half_type
) * 4))
1316 /* Check that HALF_TYPE size + shift amount <= INTERM_TYPE size.
1317 (e.g., if the original value was char, the shift amount is at most 8
1318 if we want to use short). */
1319 if (compare_tree_int (const_oprnd
, TYPE_PRECISION (half_type
)) == 1)
1322 interm_type
= build_nonstandard_integer_type (
1323 TYPE_PRECISION (half_type
) * 2, TYPE_UNSIGNED (type
));
1325 if (!vect_supportable_shift (code
, interm_type
))
1331 if (vect_supportable_shift (code
, half_type
))
1334 /* Try intermediate type - HALF_TYPE is not supported. */
1335 if (TYPE_PRECISION (type
) < (TYPE_PRECISION (half_type
) * 4))
1338 interm_type
= build_nonstandard_integer_type (
1339 TYPE_PRECISION (half_type
) * 2, TYPE_UNSIGNED (type
));
1341 if (!vect_supportable_shift (code
, interm_type
))
1350 /* There are four possible cases:
1351 1. OPRND is defined by a type promotion (in that case FIRST is TRUE, it's
1352 the first statement in the sequence)
1353 a. The original, HALF_TYPE, is not enough - we replace the promotion
1354 from HALF_TYPE to TYPE with a promotion to INTERM_TYPE.
1355 b. HALF_TYPE is sufficient, OPRND is set as the RHS of the original
1357 2. OPRND is defined by a pattern statement we created.
1358 a. Its type is not sufficient for the operation, we create a new stmt:
1359 a type conversion for OPRND from HALF_TYPE to INTERM_TYPE. We store
1360 this statement in NEW_DEF_STMT, and it is later put in
1361 STMT_VINFO_PATTERN_DEF_SEQ of the pattern statement for STMT.
1362 b. OPRND is good to use in the new statement. */
1367 /* Replace the original type conversion HALF_TYPE->TYPE with
1368 HALF_TYPE->INTERM_TYPE. */
1369 if (STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt
)))
1371 new_stmt
= STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt
));
1372 /* Check if the already created pattern stmt is what we need. */
1373 if (!is_gimple_assign (new_stmt
)
1374 || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (new_stmt
))
1375 || TREE_TYPE (gimple_assign_lhs (new_stmt
)) != interm_type
)
1378 stmts
->safe_push (def_stmt
);
1379 oprnd
= gimple_assign_lhs (new_stmt
);
1383 /* Create NEW_OPRND = (INTERM_TYPE) OPRND. */
1384 oprnd
= gimple_assign_rhs1 (def_stmt
);
1385 new_oprnd
= make_ssa_name (interm_type
);
1386 new_stmt
= gimple_build_assign (new_oprnd
, NOP_EXPR
, oprnd
);
1387 STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt
)) = new_stmt
;
1388 stmts
->safe_push (def_stmt
);
1394 /* Retrieve the operand before the type promotion. */
1395 oprnd
= gimple_assign_rhs1 (def_stmt
);
1402 /* Create a type conversion HALF_TYPE->INTERM_TYPE. */
1403 new_oprnd
= make_ssa_name (interm_type
);
1404 new_stmt
= gimple_build_assign (new_oprnd
, NOP_EXPR
, oprnd
);
1406 *new_def_stmt
= new_stmt
;
1409 /* Otherwise, OPRND is already set. */
1413 *new_type
= interm_type
;
1415 *new_type
= half_type
;
1418 *op1
= fold_convert (*new_type
, const_oprnd
);
1424 /* Try to find a statement or a sequence of statements that can be performed
1428 TYPE x_T, res0_T, res1_T;
1431 S2 x_T = (TYPE) x_t;
1432 S3 res0_T = op (x_T, C0);
1433 S4 res1_T = op (res0_T, C1);
1434 S5 ... = () res1_T; - type demotion
1436 where type 'TYPE' is at least double the size of type 'type', C0 and C1 are
1438 Check if S3 and S4 can be done on a smaller type than 'TYPE', it can either
1439 be 'type' or some intermediate type. For now, we expect S5 to be a type
1440 demotion operation. We also check that S3 and S4 have only one use. */
1443 vect_recog_over_widening_pattern (vec
<gimple
*> *stmts
,
1444 tree
*type_in
, tree
*type_out
)
1446 gimple
*stmt
= stmts
->pop ();
1447 gimple
*pattern_stmt
= NULL
, *new_def_stmt
, *prev_stmt
= NULL
,
1449 tree op0
, op1
, vectype
= NULL_TREE
, use_lhs
, use_type
;
1450 tree var
= NULL_TREE
, new_type
= NULL_TREE
, new_oprnd
;
1457 if (!vinfo_for_stmt (stmt
)
1458 || STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (stmt
)))
1461 new_def_stmt
= NULL
;
1462 if (!vect_operation_fits_smaller_type (stmt
, var
, &new_type
,
1463 &op0
, &op1
, &new_def_stmt
,
1472 /* STMT can be performed on a smaller type. Check its uses. */
1473 use_stmt
= vect_single_imm_use (stmt
);
1474 if (!use_stmt
|| !is_gimple_assign (use_stmt
))
1477 /* Create pattern statement for STMT. */
1478 vectype
= get_vectype_for_scalar_type (new_type
);
1482 /* We want to collect all the statements for which we create pattern
1483 statetments, except for the case when the last statement in the
1484 sequence doesn't have a corresponding pattern statement. In such
1485 case we associate the last pattern statement with the last statement
1486 in the sequence. Therefore, we only add the original statement to
1487 the list if we know that it is not the last. */
1489 stmts
->safe_push (prev_stmt
);
1491 var
= vect_recog_temp_ssa_var (new_type
, NULL
);
1493 = gimple_build_assign (var
, gimple_assign_rhs_code (stmt
), op0
, op1
);
1494 STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt
)) = pattern_stmt
;
1495 new_pattern_def_seq (vinfo_for_stmt (stmt
), new_def_stmt
);
1497 if (dump_enabled_p ())
1499 dump_printf_loc (MSG_NOTE
, vect_location
,
1500 "created pattern stmt: ");
1501 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_stmt
, 0);
1502 dump_printf (MSG_NOTE
, "\n");
1505 type
= gimple_expr_type (stmt
);
1512 /* We got a sequence. We expect it to end with a type demotion operation.
1513 Otherwise, we quit (for now). There are three possible cases: the
1514 conversion is to NEW_TYPE (we don't do anything), the conversion is to
1515 a type bigger than NEW_TYPE and/or the signedness of USE_TYPE and
1516 NEW_TYPE differs (we create a new conversion statement). */
1517 if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (use_stmt
)))
1519 use_lhs
= gimple_assign_lhs (use_stmt
);
1520 use_type
= TREE_TYPE (use_lhs
);
1521 /* Support only type demotion or signedess change. */
1522 if (!INTEGRAL_TYPE_P (use_type
)
1523 || TYPE_PRECISION (type
) <= TYPE_PRECISION (use_type
))
1526 /* Check that NEW_TYPE is not bigger than the conversion result. */
1527 if (TYPE_PRECISION (new_type
) > TYPE_PRECISION (use_type
))
1530 if (TYPE_UNSIGNED (new_type
) != TYPE_UNSIGNED (use_type
)
1531 || TYPE_PRECISION (new_type
) != TYPE_PRECISION (use_type
))
1533 /* Create NEW_TYPE->USE_TYPE conversion. */
1534 new_oprnd
= make_ssa_name (use_type
);
1535 pattern_stmt
= gimple_build_assign (new_oprnd
, NOP_EXPR
, var
);
1536 STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use_stmt
)) = pattern_stmt
;
1538 *type_in
= get_vectype_for_scalar_type (new_type
);
1539 *type_out
= get_vectype_for_scalar_type (use_type
);
1541 /* We created a pattern statement for the last statement in the
1542 sequence, so we don't need to associate it with the pattern
1543 statement created for PREV_STMT. Therefore, we add PREV_STMT
1544 to the list in order to mark it later in vect_pattern_recog_1. */
1546 stmts
->safe_push (prev_stmt
);
1551 STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (use_stmt
))
1552 = STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (prev_stmt
));
1555 *type_out
= NULL_TREE
;
1558 stmts
->safe_push (use_stmt
);
1561 /* TODO: support general case, create a conversion to the correct type. */
1564 /* Pattern detected. */
1565 if (dump_enabled_p ())
1567 dump_printf_loc (MSG_NOTE
, vect_location
,
1568 "vect_recog_over_widening_pattern: detected: ");
1569 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_stmt
, 0);
1570 dump_printf (MSG_NOTE
, "\n");
1573 return pattern_stmt
;
1576 /* Detect widening shift pattern:
1582 S2 a_T = (TYPE) a_t;
1583 S3 res_T = a_T << CONST;
1585 where type 'TYPE' is at least double the size of type 'type'.
1587 Also detect cases where the shift result is immediately converted
1588 to another type 'result_type' that is no larger in size than 'TYPE'.
1589 In those cases we perform a widen-shift that directly results in
1590 'result_type', to avoid a possible over-widening situation:
1594 result_type res_result;
1597 S2 a_T = (TYPE) a_t;
1598 S3 res_T = a_T << CONST;
1599 S4 res_result = (result_type) res_T;
1600 '--> res_result' = a_t w<< CONST;
1602 And a case when 'TYPE' is 4 times bigger than 'type'. In that case we
1603 create an additional pattern stmt for S2 to create a variable of an
1604 intermediate type, and perform widen-shift on the intermediate type:
1608 TYPE a_T, res_T, res_T';
1611 S2 a_T = (TYPE) a_t;
1612 '--> a_it = (interm_type) a_t;
1613 S3 res_T = a_T << CONST;
1614 '--> res_T' = a_it <<* CONST;
1618 * STMTS: Contains a stmt from which the pattern search begins.
1619 In case of unsigned widen-shift, the original stmt (S3) is replaced with S4
1620 in STMTS. When an intermediate type is used and a pattern statement is
1621 created for S2, we also put S2 here (before S3).
1625 * TYPE_IN: The type of the input arguments to the pattern.
1627 * TYPE_OUT: The type of the output of this pattern.
1629 * Return value: A new stmt that will be used to replace the sequence of
1630 stmts that constitute the pattern. In this case it will be:
1631 WIDEN_LSHIFT_EXPR <a_t, CONST>. */
1634 vect_recog_widen_shift_pattern (vec
<gimple
*> *stmts
,
1635 tree
*type_in
, tree
*type_out
)
1637 gimple
*last_stmt
= stmts
->pop ();
1639 tree oprnd0
, oprnd1
;
1640 tree type
, half_type0
;
1641 gimple
*pattern_stmt
;
1642 tree vectype
, vectype_out
= NULL_TREE
;
1644 enum tree_code dummy_code
;
1646 vec
<tree
> dummy_vec
;
1650 if (!is_gimple_assign (last_stmt
) || !vinfo_for_stmt (last_stmt
))
1653 if (STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (last_stmt
)))
1656 if (gimple_assign_rhs_code (last_stmt
) != LSHIFT_EXPR
)
1659 oprnd0
= gimple_assign_rhs1 (last_stmt
);
1660 oprnd1
= gimple_assign_rhs2 (last_stmt
);
1661 if (TREE_CODE (oprnd0
) != SSA_NAME
|| TREE_CODE (oprnd1
) != INTEGER_CST
)
1664 /* Check operand 0: it has to be defined by a type promotion. */
1665 if (!type_conversion_p (oprnd0
, last_stmt
, false, &half_type0
, &def_stmt0
,
1670 /* Check operand 1: has to be positive. We check that it fits the type
1671 in vect_handle_widen_op_by_const (). */
1672 if (tree_int_cst_compare (oprnd1
, size_zero_node
) <= 0)
1675 oprnd0
= gimple_assign_rhs1 (def_stmt0
);
1676 type
= gimple_expr_type (last_stmt
);
1678 /* Check for subsequent conversion to another type. */
1679 use_stmt
= vect_single_imm_use (last_stmt
);
1680 if (use_stmt
&& is_gimple_assign (use_stmt
)
1681 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (use_stmt
))
1682 && !STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (use_stmt
)))
1684 tree use_lhs
= gimple_assign_lhs (use_stmt
);
1685 tree use_type
= TREE_TYPE (use_lhs
);
1687 if (INTEGRAL_TYPE_P (use_type
)
1688 && TYPE_PRECISION (use_type
) <= TYPE_PRECISION (type
))
1690 last_stmt
= use_stmt
;
1695 /* Check if this a widening operation. */
1696 gimple
*wstmt
= NULL
;
1697 if (!vect_handle_widen_op_by_const (last_stmt
, LSHIFT_EXPR
, oprnd1
,
1699 type
, &half_type0
, def_stmt0
))
1702 /* Pattern detected. */
1703 if (dump_enabled_p ())
1704 dump_printf_loc (MSG_NOTE
, vect_location
,
1705 "vect_recog_widen_shift_pattern: detected:\n");
1707 /* Check target support. */
1708 vectype
= get_vectype_for_scalar_type (half_type0
);
1709 vectype_out
= get_vectype_for_scalar_type (type
);
1713 || !supportable_widening_operation (WIDEN_LSHIFT_EXPR
, last_stmt
,
1714 vectype_out
, vectype
,
1715 &dummy_code
, &dummy_code
,
1716 &dummy_int
, &dummy_vec
))
1720 *type_out
= vectype_out
;
1722 /* Pattern supported. Create a stmt to be used to replace the pattern. */
1723 var
= vect_recog_temp_ssa_var (type
, NULL
);
1725 gimple_build_assign (var
, WIDEN_LSHIFT_EXPR
, oprnd0
, oprnd1
);
1728 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (last_stmt
);
1729 new_pattern_def_seq (stmt_vinfo
, wstmt
);
1730 stmt_vec_info new_stmt_info
1731 = new_stmt_vec_info (wstmt
, stmt_vinfo
->vinfo
);
1732 set_vinfo_for_stmt (wstmt
, new_stmt_info
);
1733 STMT_VINFO_VECTYPE (new_stmt_info
) = vectype
;
1736 if (dump_enabled_p ())
1737 dump_gimple_stmt_loc (MSG_NOTE
, vect_location
, TDF_SLIM
, pattern_stmt
, 0);
1739 stmts
->safe_push (last_stmt
);
1740 return pattern_stmt
;
1743 /* Detect a rotate pattern wouldn't be otherwise vectorized:
1747 S0 a_t = b_t r<< c_t;
1751 * STMTS: Contains a stmt from which the pattern search begins,
1752 i.e. the shift/rotate stmt. The original stmt (S0) is replaced
1756 S2 e_t = d_t & (B - 1);
1757 S3 f_t = b_t << c_t;
1758 S4 g_t = b_t >> e_t;
1761 where B is element bitsize of type.
1765 * TYPE_IN: The type of the input arguments to the pattern.
1767 * TYPE_OUT: The type of the output of this pattern.
1769 * Return value: A new stmt that will be used to replace the rotate
1773 vect_recog_rotate_pattern (vec
<gimple
*> *stmts
, tree
*type_in
, tree
*type_out
)
1775 gimple
*last_stmt
= stmts
->pop ();
1776 tree oprnd0
, oprnd1
, lhs
, var
, var1
, var2
, vectype
, type
, stype
, def
, def2
;
1777 gimple
*pattern_stmt
, *def_stmt
;
1778 enum tree_code rhs_code
;
1779 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (last_stmt
);
1780 vec_info
*vinfo
= stmt_vinfo
->vinfo
;
1781 enum vect_def_type dt
;
1782 optab optab1
, optab2
;
1783 edge ext_def
= NULL
;
1785 if (!is_gimple_assign (last_stmt
))
1788 rhs_code
= gimple_assign_rhs_code (last_stmt
);
1798 if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo
))
1801 lhs
= gimple_assign_lhs (last_stmt
);
1802 oprnd0
= gimple_assign_rhs1 (last_stmt
);
1803 type
= TREE_TYPE (oprnd0
);
1804 oprnd1
= gimple_assign_rhs2 (last_stmt
);
1805 if (TREE_CODE (oprnd0
) != SSA_NAME
1806 || TYPE_PRECISION (TREE_TYPE (lhs
)) != TYPE_PRECISION (type
)
1807 || !INTEGRAL_TYPE_P (type
)
1808 || !TYPE_UNSIGNED (type
))
1811 if (!vect_is_simple_use (oprnd1
, vinfo
, &def_stmt
, &dt
))
1814 if (dt
!= vect_internal_def
1815 && dt
!= vect_constant_def
1816 && dt
!= vect_external_def
)
1819 vectype
= get_vectype_for_scalar_type (type
);
1820 if (vectype
== NULL_TREE
)
1823 /* If vector/vector or vector/scalar rotate is supported by the target,
1824 don't do anything here. */
1825 optab1
= optab_for_tree_code (rhs_code
, vectype
, optab_vector
);
1827 && optab_handler (optab1
, TYPE_MODE (vectype
)) != CODE_FOR_nothing
)
1830 if (is_a
<bb_vec_info
> (vinfo
) || dt
!= vect_internal_def
)
1832 optab2
= optab_for_tree_code (rhs_code
, vectype
, optab_scalar
);
1834 && optab_handler (optab2
, TYPE_MODE (vectype
)) != CODE_FOR_nothing
)
1838 /* If vector/vector or vector/scalar shifts aren't supported by the target,
1839 don't do anything here either. */
1840 optab1
= optab_for_tree_code (LSHIFT_EXPR
, vectype
, optab_vector
);
1841 optab2
= optab_for_tree_code (RSHIFT_EXPR
, vectype
, optab_vector
);
1843 || optab_handler (optab1
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
1845 || optab_handler (optab2
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
)
1847 if (! is_a
<bb_vec_info
> (vinfo
) && dt
== vect_internal_def
)
1849 optab1
= optab_for_tree_code (LSHIFT_EXPR
, vectype
, optab_scalar
);
1850 optab2
= optab_for_tree_code (RSHIFT_EXPR
, vectype
, optab_scalar
);
1852 || optab_handler (optab1
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
1854 || optab_handler (optab2
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
)
1859 *type_out
= vectype
;
1860 if (*type_in
== NULL_TREE
)
1863 if (dt
== vect_external_def
1864 && TREE_CODE (oprnd1
) == SSA_NAME
1865 && is_a
<loop_vec_info
> (vinfo
))
1867 struct loop
*loop
= as_a
<loop_vec_info
> (vinfo
)->loop
;
1868 ext_def
= loop_preheader_edge (loop
);
1869 if (!SSA_NAME_IS_DEFAULT_DEF (oprnd1
))
1871 basic_block bb
= gimple_bb (SSA_NAME_DEF_STMT (oprnd1
));
1873 || !dominated_by_p (CDI_DOMINATORS
, ext_def
->dest
, bb
))
1879 if (TREE_CODE (oprnd1
) == INTEGER_CST
1880 || TYPE_MODE (TREE_TYPE (oprnd1
)) == TYPE_MODE (type
))
1882 else if (def_stmt
&& gimple_assign_cast_p (def_stmt
))
1884 tree rhs1
= gimple_assign_rhs1 (def_stmt
);
1885 if (TYPE_MODE (TREE_TYPE (rhs1
)) == TYPE_MODE (type
)
1886 && TYPE_PRECISION (TREE_TYPE (rhs1
))
1887 == TYPE_PRECISION (type
))
1891 STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo
) = NULL
;
1892 if (def
== NULL_TREE
)
1894 def
= vect_recog_temp_ssa_var (type
, NULL
);
1895 def_stmt
= gimple_build_assign (def
, NOP_EXPR
, oprnd1
);
1899 = gsi_insert_on_edge_immediate (ext_def
, def_stmt
);
1900 gcc_assert (!new_bb
);
1903 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
1905 stype
= TREE_TYPE (def
);
1907 if (TREE_CODE (def
) == INTEGER_CST
)
1909 if (!tree_fits_uhwi_p (def
)
1910 || tree_to_uhwi (def
) >= GET_MODE_PRECISION (TYPE_MODE (type
))
1911 || integer_zerop (def
))
1913 def2
= build_int_cst (stype
,
1914 GET_MODE_PRECISION (TYPE_MODE (type
))
1915 - tree_to_uhwi (def
));
1919 tree vecstype
= get_vectype_for_scalar_type (stype
);
1920 stmt_vec_info def_stmt_vinfo
;
1922 if (vecstype
== NULL_TREE
)
1924 def2
= vect_recog_temp_ssa_var (stype
, NULL
);
1925 def_stmt
= gimple_build_assign (def2
, NEGATE_EXPR
, def
);
1929 = gsi_insert_on_edge_immediate (ext_def
, def_stmt
);
1930 gcc_assert (!new_bb
);
1934 def_stmt_vinfo
= new_stmt_vec_info (def_stmt
, vinfo
);
1935 set_vinfo_for_stmt (def_stmt
, def_stmt_vinfo
);
1936 STMT_VINFO_VECTYPE (def_stmt_vinfo
) = vecstype
;
1937 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
1940 def2
= vect_recog_temp_ssa_var (stype
, NULL
);
1942 = build_int_cst (stype
, GET_MODE_PRECISION (TYPE_MODE (stype
)) - 1);
1943 def_stmt
= gimple_build_assign (def2
, BIT_AND_EXPR
,
1944 gimple_assign_lhs (def_stmt
), mask
);
1948 = gsi_insert_on_edge_immediate (ext_def
, def_stmt
);
1949 gcc_assert (!new_bb
);
1953 def_stmt_vinfo
= new_stmt_vec_info (def_stmt
, vinfo
);
1954 set_vinfo_for_stmt (def_stmt
, def_stmt_vinfo
);
1955 STMT_VINFO_VECTYPE (def_stmt_vinfo
) = vecstype
;
1956 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
1960 var1
= vect_recog_temp_ssa_var (type
, NULL
);
1961 def_stmt
= gimple_build_assign (var1
, rhs_code
== LROTATE_EXPR
1962 ? LSHIFT_EXPR
: RSHIFT_EXPR
,
1964 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
1966 var2
= vect_recog_temp_ssa_var (type
, NULL
);
1967 def_stmt
= gimple_build_assign (var2
, rhs_code
== LROTATE_EXPR
1968 ? RSHIFT_EXPR
: LSHIFT_EXPR
,
1970 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
1972 /* Pattern detected. */
1973 if (dump_enabled_p ())
1974 dump_printf_loc (MSG_NOTE
, vect_location
,
1975 "vect_recog_rotate_pattern: detected:\n");
1977 /* Pattern supported. Create a stmt to be used to replace the pattern. */
1978 var
= vect_recog_temp_ssa_var (type
, NULL
);
1979 pattern_stmt
= gimple_build_assign (var
, BIT_IOR_EXPR
, var1
, var2
);
1981 if (dump_enabled_p ())
1982 dump_gimple_stmt_loc (MSG_NOTE
, vect_location
, TDF_SLIM
, pattern_stmt
, 0);
1984 stmts
->safe_push (last_stmt
);
1985 return pattern_stmt
;
1988 /* Detect a vector by vector shift pattern that wouldn't be otherwise
1996 S3 res_T = b_T op a_t;
1998 where type 'TYPE' is a type with different size than 'type',
1999 and op is <<, >> or rotate.
2004 TYPE b_T, c_T, res_T;
2007 S1 a_t = (type) c_T;
2009 S3 res_T = b_T op a_t;
2013 * STMTS: Contains a stmt from which the pattern search begins,
2014 i.e. the shift/rotate stmt. The original stmt (S3) is replaced
2015 with a shift/rotate which has same type on both operands, in the
2016 second case just b_T op c_T, in the first case with added cast
2017 from a_t to c_T in STMT_VINFO_PATTERN_DEF_SEQ.
2021 * TYPE_IN: The type of the input arguments to the pattern.
2023 * TYPE_OUT: The type of the output of this pattern.
2025 * Return value: A new stmt that will be used to replace the shift/rotate
2029 vect_recog_vector_vector_shift_pattern (vec
<gimple
*> *stmts
,
2030 tree
*type_in
, tree
*type_out
)
2032 gimple
*last_stmt
= stmts
->pop ();
2033 tree oprnd0
, oprnd1
, lhs
, var
;
2034 gimple
*pattern_stmt
, *def_stmt
;
2035 enum tree_code rhs_code
;
2036 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (last_stmt
);
2037 vec_info
*vinfo
= stmt_vinfo
->vinfo
;
2038 enum vect_def_type dt
;
2040 if (!is_gimple_assign (last_stmt
))
2043 rhs_code
= gimple_assign_rhs_code (last_stmt
);
2055 if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo
))
2058 lhs
= gimple_assign_lhs (last_stmt
);
2059 oprnd0
= gimple_assign_rhs1 (last_stmt
);
2060 oprnd1
= gimple_assign_rhs2 (last_stmt
);
2061 if (TREE_CODE (oprnd0
) != SSA_NAME
2062 || TREE_CODE (oprnd1
) != SSA_NAME
2063 || TYPE_MODE (TREE_TYPE (oprnd0
)) == TYPE_MODE (TREE_TYPE (oprnd1
))
2064 || TYPE_PRECISION (TREE_TYPE (oprnd1
))
2065 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (oprnd1
)))
2066 || TYPE_PRECISION (TREE_TYPE (lhs
))
2067 != TYPE_PRECISION (TREE_TYPE (oprnd0
)))
2070 if (!vect_is_simple_use (oprnd1
, vinfo
, &def_stmt
, &dt
))
2073 if (dt
!= vect_internal_def
)
2076 *type_in
= get_vectype_for_scalar_type (TREE_TYPE (oprnd0
));
2077 *type_out
= *type_in
;
2078 if (*type_in
== NULL_TREE
)
2081 tree def
= NULL_TREE
;
2082 if (gimple_assign_cast_p (def_stmt
))
2084 tree rhs1
= gimple_assign_rhs1 (def_stmt
);
2085 if (TYPE_MODE (TREE_TYPE (rhs1
)) == TYPE_MODE (TREE_TYPE (oprnd0
))
2086 && TYPE_PRECISION (TREE_TYPE (rhs1
))
2087 == TYPE_PRECISION (TREE_TYPE (oprnd0
)))
2091 if (def
== NULL_TREE
)
2093 def
= vect_recog_temp_ssa_var (TREE_TYPE (oprnd0
), NULL
);
2094 def_stmt
= gimple_build_assign (def
, NOP_EXPR
, oprnd1
);
2095 new_pattern_def_seq (stmt_vinfo
, def_stmt
);
2098 /* Pattern detected. */
2099 if (dump_enabled_p ())
2100 dump_printf_loc (MSG_NOTE
, vect_location
,
2101 "vect_recog_vector_vector_shift_pattern: detected:\n");
2103 /* Pattern supported. Create a stmt to be used to replace the pattern. */
2104 var
= vect_recog_temp_ssa_var (TREE_TYPE (oprnd0
), NULL
);
2105 pattern_stmt
= gimple_build_assign (var
, rhs_code
, oprnd0
, def
);
2107 if (dump_enabled_p ())
2108 dump_gimple_stmt_loc (MSG_NOTE
, vect_location
, TDF_SLIM
, pattern_stmt
, 0);
2110 stmts
->safe_push (last_stmt
);
2111 return pattern_stmt
;
2114 /* Detect multiplication by constant which are postive or negatives of power 2,
2115 and convert them to shift patterns.
2117 Mult with constants that are postive power of two.
2124 Mult with constants that are negative power of two.
2129 STMTS: Contains a stmt from which the pattern search begins,
2130 i.e. the mult stmt. Convert the mult operation to LSHIFT if
2131 constant operand is a power of 2.
2133 S1': b_t = a_t << log2 (n)
2135 Convert the mult operation to LSHIFT and followed by a NEGATE
2136 if constant operand is a negative power of 2.
2137 type a_t, b_t, res_T;
2138 S2': b_t = a_t << log2 (n)
2139 S3': res_T = - (b_t)
2143 * TYPE_IN: The type of the input arguments to the pattern.
2145 * TYPE_OUT: The type of the output of this pattern.
2147 * Return value: A new stmt that will be used to replace the multiplication
2151 vect_recog_mult_pattern (vec
<gimple
*> *stmts
,
2152 tree
*type_in
, tree
*type_out
)
2154 gimple
*last_stmt
= stmts
->pop ();
2155 tree oprnd0
, oprnd1
, vectype
, itype
;
2156 gimple
*pattern_stmt
, *def_stmt
;
2158 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (last_stmt
);
2159 int power2_val
, power2_neg_val
;
2162 if (!is_gimple_assign (last_stmt
))
2165 if (gimple_assign_rhs_code (last_stmt
) != MULT_EXPR
)
2168 oprnd0
= gimple_assign_rhs1 (last_stmt
);
2169 oprnd1
= gimple_assign_rhs2 (last_stmt
);
2170 itype
= TREE_TYPE (oprnd0
);
2172 if (TREE_CODE (oprnd0
) != SSA_NAME
2173 || TREE_CODE (oprnd1
) != INTEGER_CST
2174 || !INTEGRAL_TYPE_P (itype
)
2175 || TYPE_PRECISION (itype
) != GET_MODE_PRECISION (TYPE_MODE (itype
)))
2178 vectype
= get_vectype_for_scalar_type (itype
);
2179 if (vectype
== NULL_TREE
)
2182 /* If the target can handle vectorized multiplication natively,
2183 don't attempt to optimize this. */
2184 optab
= optab_for_tree_code (MULT_EXPR
, vectype
, optab_default
);
2185 if (optab
!= unknown_optab
)
2187 machine_mode vec_mode
= TYPE_MODE (vectype
);
2188 int icode
= (int) optab_handler (optab
, vec_mode
);
2189 if (icode
!= CODE_FOR_nothing
)
2193 /* If target cannot handle vector left shift then we cannot
2194 optimize and bail out. */
2195 optab
= optab_for_tree_code (LSHIFT_EXPR
, vectype
, optab_vector
);
2197 || optab_handler (optab
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
)
2200 power2_val
= wi::exact_log2 (oprnd1
);
2201 power2_neg_val
= wi::exact_log2 (wi::neg (oprnd1
));
2203 /* Handle constant operands that are postive or negative powers of 2. */
2204 if (power2_val
!= -1)
2206 shift
= build_int_cst (itype
, power2_val
);
2208 = gimple_build_assign (vect_recog_temp_ssa_var (itype
, NULL
),
2209 LSHIFT_EXPR
, oprnd0
, shift
);
2211 else if (power2_neg_val
!= -1)
2213 /* If the target cannot handle vector NEGATE then we cannot
2214 do the optimization. */
2215 optab
= optab_for_tree_code (NEGATE_EXPR
, vectype
, optab_vector
);
2217 || optab_handler (optab
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
)
2220 shift
= build_int_cst (itype
, power2_neg_val
);
2222 = gimple_build_assign (vect_recog_temp_ssa_var (itype
, NULL
),
2223 LSHIFT_EXPR
, oprnd0
, shift
);
2224 new_pattern_def_seq (stmt_vinfo
, def_stmt
);
2226 = gimple_build_assign (vect_recog_temp_ssa_var (itype
, NULL
),
2227 NEGATE_EXPR
, gimple_assign_lhs (def_stmt
));
2232 /* Pattern detected. */
2233 if (dump_enabled_p ())
2234 dump_printf_loc (MSG_NOTE
, vect_location
,
2235 "vect_recog_mult_pattern: detected:\n");
2237 if (dump_enabled_p ())
2238 dump_gimple_stmt_loc (MSG_NOTE
, vect_location
, TDF_SLIM
,
2241 stmts
->safe_push (last_stmt
);
2243 *type_out
= vectype
;
2245 return pattern_stmt
;
2248 /* Detect a signed division by a constant that wouldn't be
2249 otherwise vectorized:
2255 where type 'type' is an integral type and N is a constant.
2257 Similarly handle modulo by a constant:
2263 * STMTS: Contains a stmt from which the pattern search begins,
2264 i.e. the division stmt. S1 is replaced by if N is a power
2265 of two constant and type is signed:
2266 S3 y_t = b_t < 0 ? N - 1 : 0;
2268 S1' a_t = x_t >> log2 (N);
2270 S4 is replaced if N is a power of two constant and
2271 type is signed by (where *_T temporaries have unsigned type):
2272 S9 y_T = b_t < 0 ? -1U : 0U;
2273 S8 z_T = y_T >> (sizeof (type_t) * CHAR_BIT - log2 (N));
2274 S7 z_t = (type) z_T;
2276 S5 x_t = w_t & (N - 1);
2277 S4' a_t = x_t - z_t;
2281 * TYPE_IN: The type of the input arguments to the pattern.
2283 * TYPE_OUT: The type of the output of this pattern.
2285 * Return value: A new stmt that will be used to replace the division
2286 S1 or modulo S4 stmt. */
2289 vect_recog_divmod_pattern (vec
<gimple
*> *stmts
,
2290 tree
*type_in
, tree
*type_out
)
2292 gimple
*last_stmt
= stmts
->pop ();
2293 tree oprnd0
, oprnd1
, vectype
, itype
, cond
;
2294 gimple
*pattern_stmt
, *def_stmt
;
2295 enum tree_code rhs_code
;
2296 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (last_stmt
);
2297 vec_info
*vinfo
= stmt_vinfo
->vinfo
;
2300 int dummy_int
, prec
;
2301 stmt_vec_info def_stmt_vinfo
;
2303 if (!is_gimple_assign (last_stmt
))
2306 rhs_code
= gimple_assign_rhs_code (last_stmt
);
2309 case TRUNC_DIV_EXPR
:
2310 case TRUNC_MOD_EXPR
:
2316 if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo
))
2319 oprnd0
= gimple_assign_rhs1 (last_stmt
);
2320 oprnd1
= gimple_assign_rhs2 (last_stmt
);
2321 itype
= TREE_TYPE (oprnd0
);
2322 if (TREE_CODE (oprnd0
) != SSA_NAME
2323 || TREE_CODE (oprnd1
) != INTEGER_CST
2324 || TREE_CODE (itype
) != INTEGER_TYPE
2325 || TYPE_PRECISION (itype
) != GET_MODE_PRECISION (TYPE_MODE (itype
)))
2328 vectype
= get_vectype_for_scalar_type (itype
);
2329 if (vectype
== NULL_TREE
)
2332 /* If the target can handle vectorized division or modulo natively,
2333 don't attempt to optimize this. */
2334 optab
= optab_for_tree_code (rhs_code
, vectype
, optab_default
);
2335 if (optab
!= unknown_optab
)
2337 machine_mode vec_mode
= TYPE_MODE (vectype
);
2338 int icode
= (int) optab_handler (optab
, vec_mode
);
2339 if (icode
!= CODE_FOR_nothing
)
2343 prec
= TYPE_PRECISION (itype
);
2344 if (integer_pow2p (oprnd1
))
2346 if (TYPE_UNSIGNED (itype
) || tree_int_cst_sgn (oprnd1
) != 1)
2349 /* Pattern detected. */
2350 if (dump_enabled_p ())
2351 dump_printf_loc (MSG_NOTE
, vect_location
,
2352 "vect_recog_divmod_pattern: detected:\n");
2354 cond
= build2 (LT_EXPR
, boolean_type_node
, oprnd0
,
2355 build_int_cst (itype
, 0));
2356 if (rhs_code
== TRUNC_DIV_EXPR
)
2358 tree var
= vect_recog_temp_ssa_var (itype
, NULL
);
2361 = gimple_build_assign (var
, COND_EXPR
, cond
,
2362 fold_build2 (MINUS_EXPR
, itype
, oprnd1
,
2363 build_int_cst (itype
, 1)),
2364 build_int_cst (itype
, 0));
2365 new_pattern_def_seq (stmt_vinfo
, def_stmt
);
2366 var
= vect_recog_temp_ssa_var (itype
, NULL
);
2368 = gimple_build_assign (var
, PLUS_EXPR
, oprnd0
,
2369 gimple_assign_lhs (def_stmt
));
2370 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2372 shift
= build_int_cst (itype
, tree_log2 (oprnd1
));
2374 = gimple_build_assign (vect_recog_temp_ssa_var (itype
, NULL
),
2375 RSHIFT_EXPR
, var
, shift
);
2380 STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo
) = NULL
;
2381 if (compare_tree_int (oprnd1
, 2) == 0)
2383 signmask
= vect_recog_temp_ssa_var (itype
, NULL
);
2384 def_stmt
= gimple_build_assign (signmask
, COND_EXPR
, cond
,
2385 build_int_cst (itype
, 1),
2386 build_int_cst (itype
, 0));
2387 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2392 = build_nonstandard_integer_type (prec
, 1);
2393 tree vecutype
= get_vectype_for_scalar_type (utype
);
2395 = build_int_cst (utype
, GET_MODE_BITSIZE (TYPE_MODE (itype
))
2396 - tree_log2 (oprnd1
));
2397 tree var
= vect_recog_temp_ssa_var (utype
, NULL
);
2399 def_stmt
= gimple_build_assign (var
, COND_EXPR
, cond
,
2400 build_int_cst (utype
, -1),
2401 build_int_cst (utype
, 0));
2402 def_stmt_vinfo
= new_stmt_vec_info (def_stmt
, vinfo
);
2403 set_vinfo_for_stmt (def_stmt
, def_stmt_vinfo
);
2404 STMT_VINFO_VECTYPE (def_stmt_vinfo
) = vecutype
;
2405 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2406 var
= vect_recog_temp_ssa_var (utype
, NULL
);
2407 def_stmt
= gimple_build_assign (var
, RSHIFT_EXPR
,
2408 gimple_assign_lhs (def_stmt
),
2410 def_stmt_vinfo
= new_stmt_vec_info (def_stmt
, vinfo
);
2411 set_vinfo_for_stmt (def_stmt
, def_stmt_vinfo
);
2412 STMT_VINFO_VECTYPE (def_stmt_vinfo
) = vecutype
;
2413 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2414 signmask
= vect_recog_temp_ssa_var (itype
, NULL
);
2416 = gimple_build_assign (signmask
, NOP_EXPR
, var
);
2417 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2420 = gimple_build_assign (vect_recog_temp_ssa_var (itype
, NULL
),
2421 PLUS_EXPR
, oprnd0
, signmask
);
2422 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2424 = gimple_build_assign (vect_recog_temp_ssa_var (itype
, NULL
),
2425 BIT_AND_EXPR
, gimple_assign_lhs (def_stmt
),
2426 fold_build2 (MINUS_EXPR
, itype
, oprnd1
,
2427 build_int_cst (itype
, 1)));
2428 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2431 = gimple_build_assign (vect_recog_temp_ssa_var (itype
, NULL
),
2432 MINUS_EXPR
, gimple_assign_lhs (def_stmt
),
2436 if (dump_enabled_p ())
2437 dump_gimple_stmt_loc (MSG_NOTE
, vect_location
, TDF_SLIM
, pattern_stmt
,
2440 stmts
->safe_push (last_stmt
);
2443 *type_out
= vectype
;
2444 return pattern_stmt
;
2447 if (prec
> HOST_BITS_PER_WIDE_INT
2448 || integer_zerop (oprnd1
))
2451 if (!can_mult_highpart_p (TYPE_MODE (vectype
), TYPE_UNSIGNED (itype
)))
2454 STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo
) = NULL
;
2456 if (TYPE_UNSIGNED (itype
))
2458 unsigned HOST_WIDE_INT mh
, ml
;
2459 int pre_shift
, post_shift
;
2460 unsigned HOST_WIDE_INT d
= (TREE_INT_CST_LOW (oprnd1
)
2461 & GET_MODE_MASK (TYPE_MODE (itype
)));
2462 tree t1
, t2
, t3
, t4
;
2464 if (d
>= ((unsigned HOST_WIDE_INT
) 1 << (prec
- 1)))
2465 /* FIXME: Can transform this into oprnd0 >= oprnd1 ? 1 : 0. */
2468 /* Find a suitable multiplier and right shift count
2469 instead of multiplying with D. */
2470 mh
= choose_multiplier (d
, prec
, prec
, &ml
, &post_shift
, &dummy_int
);
2472 /* If the suggested multiplier is more than SIZE bits, we can do better
2473 for even divisors, using an initial right shift. */
2474 if (mh
!= 0 && (d
& 1) == 0)
2476 pre_shift
= floor_log2 (d
& -d
);
2477 mh
= choose_multiplier (d
>> pre_shift
, prec
, prec
- pre_shift
,
2478 &ml
, &post_shift
, &dummy_int
);
2486 if (post_shift
- 1 >= prec
)
2489 /* t1 = oprnd0 h* ml;
2493 q = t4 >> (post_shift - 1); */
2494 t1
= vect_recog_temp_ssa_var (itype
, NULL
);
2495 def_stmt
= gimple_build_assign (t1
, MULT_HIGHPART_EXPR
, oprnd0
,
2496 build_int_cst (itype
, ml
));
2497 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2499 t2
= vect_recog_temp_ssa_var (itype
, NULL
);
2501 = gimple_build_assign (t2
, MINUS_EXPR
, oprnd0
, t1
);
2502 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2504 t3
= vect_recog_temp_ssa_var (itype
, NULL
);
2506 = gimple_build_assign (t3
, RSHIFT_EXPR
, t2
, integer_one_node
);
2507 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2509 t4
= vect_recog_temp_ssa_var (itype
, NULL
);
2511 = gimple_build_assign (t4
, PLUS_EXPR
, t1
, t3
);
2513 if (post_shift
!= 1)
2515 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2517 q
= vect_recog_temp_ssa_var (itype
, NULL
);
2519 = gimple_build_assign (q
, RSHIFT_EXPR
, t4
,
2520 build_int_cst (itype
, post_shift
- 1));
2525 pattern_stmt
= def_stmt
;
2530 if (pre_shift
>= prec
|| post_shift
>= prec
)
2533 /* t1 = oprnd0 >> pre_shift;
2535 q = t2 >> post_shift; */
2538 t1
= vect_recog_temp_ssa_var (itype
, NULL
);
2540 = gimple_build_assign (t1
, RSHIFT_EXPR
, oprnd0
,
2541 build_int_cst (NULL
, pre_shift
));
2542 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2547 t2
= vect_recog_temp_ssa_var (itype
, NULL
);
2548 def_stmt
= gimple_build_assign (t2
, MULT_HIGHPART_EXPR
, t1
,
2549 build_int_cst (itype
, ml
));
2553 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2555 q
= vect_recog_temp_ssa_var (itype
, NULL
);
2557 = gimple_build_assign (q
, RSHIFT_EXPR
, t2
,
2558 build_int_cst (itype
, post_shift
));
2563 pattern_stmt
= def_stmt
;
2568 unsigned HOST_WIDE_INT ml
;
2570 HOST_WIDE_INT d
= TREE_INT_CST_LOW (oprnd1
);
2571 unsigned HOST_WIDE_INT abs_d
;
2573 tree t1
, t2
, t3
, t4
;
2575 /* Give up for -1. */
2579 /* Since d might be INT_MIN, we have to cast to
2580 unsigned HOST_WIDE_INT before negating to avoid
2581 undefined signed overflow. */
2583 ? (unsigned HOST_WIDE_INT
) d
2584 : - (unsigned HOST_WIDE_INT
) d
);
2586 /* n rem d = n rem -d */
2587 if (rhs_code
== TRUNC_MOD_EXPR
&& d
< 0)
2590 oprnd1
= build_int_cst (itype
, abs_d
);
2592 else if (HOST_BITS_PER_WIDE_INT
>= prec
2593 && abs_d
== (unsigned HOST_WIDE_INT
) 1 << (prec
- 1))
2594 /* This case is not handled correctly below. */
2597 choose_multiplier (abs_d
, prec
, prec
- 1, &ml
, &post_shift
, &dummy_int
);
2598 if (ml
>= (unsigned HOST_WIDE_INT
) 1 << (prec
- 1))
2601 ml
|= (~(unsigned HOST_WIDE_INT
) 0) << (prec
- 1);
2603 if (post_shift
>= prec
)
2606 /* t1 = oprnd0 h* ml; */
2607 t1
= vect_recog_temp_ssa_var (itype
, NULL
);
2608 def_stmt
= gimple_build_assign (t1
, MULT_HIGHPART_EXPR
, oprnd0
,
2609 build_int_cst (itype
, ml
));
2613 /* t2 = t1 + oprnd0; */
2614 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2615 t2
= vect_recog_temp_ssa_var (itype
, NULL
);
2616 def_stmt
= gimple_build_assign (t2
, PLUS_EXPR
, t1
, oprnd0
);
2623 /* t3 = t2 >> post_shift; */
2624 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2625 t3
= vect_recog_temp_ssa_var (itype
, NULL
);
2626 def_stmt
= gimple_build_assign (t3
, RSHIFT_EXPR
, t2
,
2627 build_int_cst (itype
, post_shift
));
2632 wide_int oprnd0_min
, oprnd0_max
;
2634 if (get_range_info (oprnd0
, &oprnd0_min
, &oprnd0_max
) == VR_RANGE
)
2636 if (!wi::neg_p (oprnd0_min
, TYPE_SIGN (itype
)))
2638 else if (wi::neg_p (oprnd0_max
, TYPE_SIGN (itype
)))
2642 if (msb
== 0 && d
>= 0)
2646 pattern_stmt
= def_stmt
;
2650 /* t4 = oprnd0 >> (prec - 1);
2651 or if we know from VRP that oprnd0 >= 0
2653 or if we know from VRP that oprnd0 < 0
2655 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2656 t4
= vect_recog_temp_ssa_var (itype
, NULL
);
2658 def_stmt
= gimple_build_assign (t4
, INTEGER_CST
,
2659 build_int_cst (itype
, msb
));
2661 def_stmt
= gimple_build_assign (t4
, RSHIFT_EXPR
, oprnd0
,
2662 build_int_cst (itype
, prec
- 1));
2663 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2665 /* q = t3 - t4; or q = t4 - t3; */
2666 q
= vect_recog_temp_ssa_var (itype
, NULL
);
2667 pattern_stmt
= gimple_build_assign (q
, MINUS_EXPR
, d
< 0 ? t4
: t3
,
2672 if (rhs_code
== TRUNC_MOD_EXPR
)
2676 /* We divided. Now finish by:
2679 append_pattern_def_seq (stmt_vinfo
, pattern_stmt
);
2681 t1
= vect_recog_temp_ssa_var (itype
, NULL
);
2682 def_stmt
= gimple_build_assign (t1
, MULT_EXPR
, q
, oprnd1
);
2683 append_pattern_def_seq (stmt_vinfo
, def_stmt
);
2685 r
= vect_recog_temp_ssa_var (itype
, NULL
);
2686 pattern_stmt
= gimple_build_assign (r
, MINUS_EXPR
, oprnd0
, t1
);
2689 /* Pattern detected. */
2690 if (dump_enabled_p ())
2692 dump_printf_loc (MSG_NOTE
, vect_location
,
2693 "vect_recog_divmod_pattern: detected: ");
2694 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_stmt
, 0);
2695 dump_printf (MSG_NOTE
, "\n");
2698 stmts
->safe_push (last_stmt
);
2701 *type_out
= vectype
;
2702 return pattern_stmt
;
2705 /* Function vect_recog_mixed_size_cond_pattern
2707 Try to find the following pattern:
2712 S1 a_T = x_t CMP y_t ? b_T : c_T;
2714 where type 'TYPE' is an integral type which has different size
2715 from 'type'. b_T and c_T are either constants (and if 'TYPE' is wider
2716 than 'type', the constants need to fit into an integer type
2717 with the same width as 'type') or results of conversion from 'type'.
2721 * LAST_STMT: A stmt from which the pattern search begins.
2725 * TYPE_IN: The type of the input arguments to the pattern.
2727 * TYPE_OUT: The type of the output of this pattern.
2729 * Return value: A new stmt that will be used to replace the pattern.
2730 Additionally a def_stmt is added.
2732 a_it = x_t CMP y_t ? b_it : c_it;
2733 a_T = (TYPE) a_it; */
2736 vect_recog_mixed_size_cond_pattern (vec
<gimple
*> *stmts
, tree
*type_in
,
2739 gimple
*last_stmt
= (*stmts
)[0];
2740 tree cond_expr
, then_clause
, else_clause
;
2741 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (last_stmt
), def_stmt_info
;
2742 tree type
, vectype
, comp_vectype
, itype
= NULL_TREE
, vecitype
;
2743 gimple
*pattern_stmt
, *def_stmt
;
2744 vec_info
*vinfo
= stmt_vinfo
->vinfo
;
2745 tree orig_type0
= NULL_TREE
, orig_type1
= NULL_TREE
;
2746 gimple
*def_stmt0
= NULL
, *def_stmt1
= NULL
;
2748 tree comp_scalar_type
;
2750 if (!is_gimple_assign (last_stmt
)
2751 || gimple_assign_rhs_code (last_stmt
) != COND_EXPR
2752 || STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_internal_def
)
2755 cond_expr
= gimple_assign_rhs1 (last_stmt
);
2756 then_clause
= gimple_assign_rhs2 (last_stmt
);
2757 else_clause
= gimple_assign_rhs3 (last_stmt
);
2759 if (!COMPARISON_CLASS_P (cond_expr
))
2762 comp_scalar_type
= TREE_TYPE (TREE_OPERAND (cond_expr
, 0));
2763 comp_vectype
= get_vectype_for_scalar_type (comp_scalar_type
);
2764 if (comp_vectype
== NULL_TREE
)
2767 type
= gimple_expr_type (last_stmt
);
2768 if (types_compatible_p (type
, comp_scalar_type
)
2769 || ((TREE_CODE (then_clause
) != INTEGER_CST
2770 || TREE_CODE (else_clause
) != INTEGER_CST
)
2771 && !INTEGRAL_TYPE_P (comp_scalar_type
))
2772 || !INTEGRAL_TYPE_P (type
))
2775 if ((TREE_CODE (then_clause
) != INTEGER_CST
2776 && !type_conversion_p (then_clause
, last_stmt
, false, &orig_type0
,
2777 &def_stmt0
, &promotion
))
2778 || (TREE_CODE (else_clause
) != INTEGER_CST
2779 && !type_conversion_p (else_clause
, last_stmt
, false, &orig_type1
,
2780 &def_stmt1
, &promotion
)))
2783 if (orig_type0
&& orig_type1
2784 && !types_compatible_p (orig_type0
, orig_type1
))
2789 if (!types_compatible_p (orig_type0
, comp_scalar_type
))
2791 then_clause
= gimple_assign_rhs1 (def_stmt0
);
2797 if (!types_compatible_p (orig_type1
, comp_scalar_type
))
2799 else_clause
= gimple_assign_rhs1 (def_stmt1
);
2804 HOST_WIDE_INT cmp_mode_size
2805 = GET_MODE_UNIT_BITSIZE (TYPE_MODE (comp_vectype
));
2807 if (GET_MODE_BITSIZE (TYPE_MODE (type
)) == cmp_mode_size
)
2810 vectype
= get_vectype_for_scalar_type (type
);
2811 if (vectype
== NULL_TREE
)
2814 if (expand_vec_cond_expr_p (vectype
, comp_vectype
))
2817 if (itype
== NULL_TREE
)
2818 itype
= build_nonstandard_integer_type (cmp_mode_size
,
2819 TYPE_UNSIGNED (type
));
2821 if (itype
== NULL_TREE
2822 || GET_MODE_BITSIZE (TYPE_MODE (itype
)) != cmp_mode_size
)
2825 vecitype
= get_vectype_for_scalar_type (itype
);
2826 if (vecitype
== NULL_TREE
)
2829 if (!expand_vec_cond_expr_p (vecitype
, comp_vectype
))
2832 if (GET_MODE_BITSIZE (TYPE_MODE (type
)) > cmp_mode_size
)
2834 if ((TREE_CODE (then_clause
) == INTEGER_CST
2835 && !int_fits_type_p (then_clause
, itype
))
2836 || (TREE_CODE (else_clause
) == INTEGER_CST
2837 && !int_fits_type_p (else_clause
, itype
)))
2841 def_stmt
= gimple_build_assign (vect_recog_temp_ssa_var (itype
, NULL
),
2842 COND_EXPR
, unshare_expr (cond_expr
),
2843 fold_convert (itype
, then_clause
),
2844 fold_convert (itype
, else_clause
));
2845 pattern_stmt
= gimple_build_assign (vect_recog_temp_ssa_var (type
, NULL
),
2846 NOP_EXPR
, gimple_assign_lhs (def_stmt
));
2848 new_pattern_def_seq (stmt_vinfo
, def_stmt
);
2849 def_stmt_info
= new_stmt_vec_info (def_stmt
, vinfo
);
2850 set_vinfo_for_stmt (def_stmt
, def_stmt_info
);
2851 STMT_VINFO_VECTYPE (def_stmt_info
) = vecitype
;
2852 *type_in
= vecitype
;
2853 *type_out
= vectype
;
2855 if (dump_enabled_p ())
2856 dump_printf_loc (MSG_NOTE
, vect_location
,
2857 "vect_recog_mixed_size_cond_pattern: detected:\n");
2859 return pattern_stmt
;
2863 /* Helper function of vect_recog_bool_pattern. Called recursively, return
2864 true if bool VAR can and should be optimized that way. Assume it shouldn't
2865 in case it's a result of a comparison which can be directly vectorized into
2866 a vector comparison. */
2869 check_bool_pattern (tree var
, vec_info
*vinfo
)
2872 enum vect_def_type dt
;
2874 enum tree_code rhs_code
;
2876 if (!vect_is_simple_use (var
, vinfo
, &def_stmt
, &dt
))
2879 if (dt
!= vect_internal_def
)
2882 if (!is_gimple_assign (def_stmt
))
2885 if (!has_single_use (var
))
2888 rhs1
= gimple_assign_rhs1 (def_stmt
);
2889 rhs_code
= gimple_assign_rhs_code (def_stmt
);
2893 return check_bool_pattern (rhs1
, vinfo
);
2896 if ((TYPE_PRECISION (TREE_TYPE (rhs1
)) != 1
2897 || !TYPE_UNSIGNED (TREE_TYPE (rhs1
)))
2898 && TREE_CODE (TREE_TYPE (rhs1
)) != BOOLEAN_TYPE
)
2900 return check_bool_pattern (rhs1
, vinfo
);
2903 return check_bool_pattern (rhs1
, vinfo
);
2908 if (!check_bool_pattern (rhs1
, vinfo
))
2910 return check_bool_pattern (gimple_assign_rhs2 (def_stmt
), vinfo
);
2913 if (TREE_CODE_CLASS (rhs_code
) == tcc_comparison
)
2915 tree vecitype
, comp_vectype
, mask_type
;
2917 /* If the comparison can throw, then is_gimple_condexpr will be
2918 false and we can't make a COND_EXPR/VEC_COND_EXPR out of it. */
2919 if (stmt_could_throw_p (def_stmt
))
2922 comp_vectype
= get_vectype_for_scalar_type (TREE_TYPE (rhs1
));
2923 if (comp_vectype
== NULL_TREE
)
2926 mask_type
= get_mask_type_for_scalar_type (TREE_TYPE (rhs1
));
2928 && expand_vec_cmp_expr_p (comp_vectype
, mask_type
))
2931 if (TREE_CODE (TREE_TYPE (rhs1
)) != INTEGER_TYPE
)
2933 machine_mode mode
= TYPE_MODE (TREE_TYPE (rhs1
));
2935 = build_nonstandard_integer_type (GET_MODE_BITSIZE (mode
), 1);
2936 vecitype
= get_vectype_for_scalar_type (itype
);
2937 if (vecitype
== NULL_TREE
)
2941 vecitype
= comp_vectype
;
2942 return expand_vec_cond_expr_p (vecitype
, comp_vectype
);
2949 /* Helper function of adjust_bool_pattern. Add a cast to TYPE to a previous
2950 stmt (SSA_NAME_DEF_STMT of VAR) by moving the COND_EXPR from RELATED_STMT
2951 to PATTERN_DEF_SEQ and adding a cast as RELATED_STMT. */
2954 adjust_bool_pattern_cast (tree type
, tree var
)
2956 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (SSA_NAME_DEF_STMT (var
));
2957 gimple
*cast_stmt
, *pattern_stmt
;
2959 gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo
));
2960 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_vinfo
);
2961 new_pattern_def_seq (stmt_vinfo
, pattern_stmt
);
2962 cast_stmt
= gimple_build_assign (vect_recog_temp_ssa_var (type
, NULL
),
2963 NOP_EXPR
, gimple_assign_lhs (pattern_stmt
));
2964 STMT_VINFO_RELATED_STMT (stmt_vinfo
) = cast_stmt
;
2965 return gimple_assign_lhs (cast_stmt
);
2969 /* Helper function of vect_recog_bool_pattern. Do the actual transformations,
2970 recursively. VAR is an SSA_NAME that should be transformed from bool
2971 to a wider integer type, OUT_TYPE is the desired final integer type of
2972 the whole pattern, TRUEVAL should be NULL unless optimizing
2973 BIT_AND_EXPR into a COND_EXPR with one integer from one of the operands
2974 in the then_clause, STMTS is where statements with added pattern stmts
2975 should be pushed to. */
2978 adjust_bool_pattern (tree var
, tree out_type
, tree trueval
,
2979 vec
<gimple
*> *stmts
)
2981 gimple
*stmt
= SSA_NAME_DEF_STMT (var
);
2982 enum tree_code rhs_code
, def_rhs_code
;
2983 tree itype
, cond_expr
, rhs1
, rhs2
, irhs1
, irhs2
;
2985 gimple
*pattern_stmt
, *def_stmt
;
2987 rhs1
= gimple_assign_rhs1 (stmt
);
2988 rhs2
= gimple_assign_rhs2 (stmt
);
2989 rhs_code
= gimple_assign_rhs_code (stmt
);
2990 loc
= gimple_location (stmt
);
2995 irhs1
= adjust_bool_pattern (rhs1
, out_type
, NULL_TREE
, stmts
);
2996 itype
= TREE_TYPE (irhs1
);
2998 = gimple_build_assign (vect_recog_temp_ssa_var (itype
, NULL
),
3003 irhs1
= adjust_bool_pattern (rhs1
, out_type
, NULL_TREE
, stmts
);
3004 itype
= TREE_TYPE (irhs1
);
3006 = gimple_build_assign (vect_recog_temp_ssa_var (itype
, NULL
),
3007 BIT_XOR_EXPR
, irhs1
, build_int_cst (itype
, 1));
3011 /* Try to optimize x = y & (a < b ? 1 : 0); into
3012 x = (a < b ? y : 0);
3018 S1 a_b = x1 CMP1 y1;
3019 S2 b_b = x2 CMP2 y2;
3021 S4 d_T = (TYPE) c_b;
3023 we would normally emit:
3025 S1' a_T = x1 CMP1 y1 ? 1 : 0;
3026 S2' b_T = x2 CMP2 y2 ? 1 : 0;
3027 S3' c_T = a_T & b_T;
3030 but we can save one stmt by using the
3031 result of one of the COND_EXPRs in the other COND_EXPR and leave
3032 BIT_AND_EXPR stmt out:
3034 S1' a_T = x1 CMP1 y1 ? 1 : 0;
3035 S3' c_T = x2 CMP2 y2 ? a_T : 0;
3038 At least when VEC_COND_EXPR is implemented using masks
3039 cond ? 1 : 0 is as expensive as cond ? var : 0, in both cases it
3040 computes the comparison masks and ands it, in one case with
3041 all ones vector, in the other case with a vector register.
3042 Don't do this for BIT_IOR_EXPR, because cond ? 1 : var; is
3043 often more expensive. */
3044 def_stmt
= SSA_NAME_DEF_STMT (rhs2
);
3045 def_rhs_code
= gimple_assign_rhs_code (def_stmt
);
3046 if (TREE_CODE_CLASS (def_rhs_code
) == tcc_comparison
)
3048 tree def_rhs1
= gimple_assign_rhs1 (def_stmt
);
3049 irhs1
= adjust_bool_pattern (rhs1
, out_type
, NULL_TREE
, stmts
);
3050 if (TYPE_PRECISION (TREE_TYPE (irhs1
))
3051 == GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (def_rhs1
))))
3054 stmt_vec_info stmt_def_vinfo
= vinfo_for_stmt (def_stmt
);
3055 irhs2
= adjust_bool_pattern (rhs2
, out_type
, irhs1
, stmts
);
3056 tstmt
= stmts
->pop ();
3057 gcc_assert (tstmt
== def_stmt
);
3058 stmts
->quick_push (stmt
);
3059 STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt
))
3060 = STMT_VINFO_RELATED_STMT (stmt_def_vinfo
);
3061 gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_def_vinfo
));
3062 STMT_VINFO_RELATED_STMT (stmt_def_vinfo
) = NULL
;
3066 irhs2
= adjust_bool_pattern (rhs2
, out_type
, NULL_TREE
, stmts
);
3069 def_stmt
= SSA_NAME_DEF_STMT (rhs1
);
3070 def_rhs_code
= gimple_assign_rhs_code (def_stmt
);
3071 if (TREE_CODE_CLASS (def_rhs_code
) == tcc_comparison
)
3073 tree def_rhs1
= gimple_assign_rhs1 (def_stmt
);
3074 irhs2
= adjust_bool_pattern (rhs2
, out_type
, NULL_TREE
, stmts
);
3075 if (TYPE_PRECISION (TREE_TYPE (irhs2
))
3076 == GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (def_rhs1
))))
3079 stmt_vec_info stmt_def_vinfo
= vinfo_for_stmt (def_stmt
);
3080 irhs1
= adjust_bool_pattern (rhs1
, out_type
, irhs2
, stmts
);
3081 tstmt
= stmts
->pop ();
3082 gcc_assert (tstmt
== def_stmt
);
3083 stmts
->quick_push (stmt
);
3084 STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt
))
3085 = STMT_VINFO_RELATED_STMT (stmt_def_vinfo
);
3086 gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_def_vinfo
));
3087 STMT_VINFO_RELATED_STMT (stmt_def_vinfo
) = NULL
;
3091 irhs1
= adjust_bool_pattern (rhs1
, out_type
, NULL_TREE
, stmts
);
3097 irhs1
= adjust_bool_pattern (rhs1
, out_type
, NULL_TREE
, stmts
);
3098 irhs2
= adjust_bool_pattern (rhs2
, out_type
, NULL_TREE
, stmts
);
3100 if (TYPE_PRECISION (TREE_TYPE (irhs1
))
3101 != TYPE_PRECISION (TREE_TYPE (irhs2
)))
3103 int prec1
= TYPE_PRECISION (TREE_TYPE (irhs1
));
3104 int prec2
= TYPE_PRECISION (TREE_TYPE (irhs2
));
3105 int out_prec
= TYPE_PRECISION (out_type
);
3106 if (absu_hwi (out_prec
- prec1
) < absu_hwi (out_prec
- prec2
))
3107 irhs2
= adjust_bool_pattern_cast (TREE_TYPE (irhs1
), rhs2
);
3108 else if (absu_hwi (out_prec
- prec1
) > absu_hwi (out_prec
- prec2
))
3109 irhs1
= adjust_bool_pattern_cast (TREE_TYPE (irhs2
), rhs1
);
3112 irhs1
= adjust_bool_pattern_cast (out_type
, rhs1
);
3113 irhs2
= adjust_bool_pattern_cast (out_type
, rhs2
);
3116 itype
= TREE_TYPE (irhs1
);
3118 = gimple_build_assign (vect_recog_temp_ssa_var (itype
, NULL
),
3119 rhs_code
, irhs1
, irhs2
);
3123 gcc_assert (TREE_CODE_CLASS (rhs_code
) == tcc_comparison
);
3124 if (TREE_CODE (TREE_TYPE (rhs1
)) != INTEGER_TYPE
3125 || !TYPE_UNSIGNED (TREE_TYPE (rhs1
))
3126 || (TYPE_PRECISION (TREE_TYPE (rhs1
))
3127 != GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (rhs1
)))))
3129 machine_mode mode
= TYPE_MODE (TREE_TYPE (rhs1
));
3131 = build_nonstandard_integer_type (GET_MODE_BITSIZE (mode
), 1);
3134 itype
= TREE_TYPE (rhs1
);
3135 cond_expr
= build2_loc (loc
, rhs_code
, itype
, rhs1
, rhs2
);
3136 if (trueval
== NULL_TREE
)
3137 trueval
= build_int_cst (itype
, 1);
3139 gcc_checking_assert (useless_type_conversion_p (itype
,
3140 TREE_TYPE (trueval
)));
3142 = gimple_build_assign (vect_recog_temp_ssa_var (itype
, NULL
),
3143 COND_EXPR
, cond_expr
, trueval
,
3144 build_int_cst (itype
, 0));
3148 stmts
->safe_push (stmt
);
3149 gimple_set_location (pattern_stmt
, loc
);
3150 STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt
)) = pattern_stmt
;
3151 return gimple_assign_lhs (pattern_stmt
);
3155 /* Return the proper type for converting bool VAR into
3156 an integer value or NULL_TREE if no such type exists.
3157 The type is chosen so that converted value has the
3158 same number of elements as VAR's vector type. */
3161 search_type_for_mask (tree var
, vec_info
*vinfo
)
3164 enum vect_def_type dt
;
3166 enum tree_code rhs_code
;
3167 tree res
= NULL_TREE
, res2
;
3169 if (TREE_CODE (var
) != SSA_NAME
)
3172 if ((TYPE_PRECISION (TREE_TYPE (var
)) != 1
3173 || !TYPE_UNSIGNED (TREE_TYPE (var
)))
3174 && TREE_CODE (TREE_TYPE (var
)) != BOOLEAN_TYPE
)
3177 if (!vect_is_simple_use (var
, vinfo
, &def_stmt
, &dt
))
3180 if (dt
!= vect_internal_def
)
3183 if (!is_gimple_assign (def_stmt
))
3186 rhs_code
= gimple_assign_rhs_code (def_stmt
);
3187 rhs1
= gimple_assign_rhs1 (def_stmt
);
3194 res
= search_type_for_mask (rhs1
, vinfo
);
3200 res
= search_type_for_mask (rhs1
, vinfo
);
3201 res2
= search_type_for_mask (gimple_assign_rhs2 (def_stmt
), vinfo
);
3202 if (!res
|| (res2
&& TYPE_PRECISION (res
) > TYPE_PRECISION (res2
)))
3207 if (TREE_CODE_CLASS (rhs_code
) == tcc_comparison
)
3209 tree comp_vectype
, mask_type
;
3211 comp_vectype
= get_vectype_for_scalar_type (TREE_TYPE (rhs1
));
3212 if (comp_vectype
== NULL_TREE
)
3215 mask_type
= get_mask_type_for_scalar_type (TREE_TYPE (rhs1
));
3217 || !expand_vec_cmp_expr_p (comp_vectype
, mask_type
))
3220 if (TREE_CODE (TREE_TYPE (rhs1
)) != INTEGER_TYPE
3221 || !TYPE_UNSIGNED (TREE_TYPE (rhs1
)))
3223 machine_mode mode
= TYPE_MODE (TREE_TYPE (rhs1
));
3224 res
= build_nonstandard_integer_type (GET_MODE_BITSIZE (mode
), 1);
3227 res
= TREE_TYPE (rhs1
);
3235 /* Function vect_recog_bool_pattern
3237 Try to find pattern like following:
3239 bool a_b, b_b, c_b, d_b, e_b;
3242 S1 a_b = x1 CMP1 y1;
3243 S2 b_b = x2 CMP2 y2;
3245 S4 d_b = x3 CMP3 y3;
3247 S6 f_T = (TYPE) e_b;
3249 where type 'TYPE' is an integral type. Or a similar pattern
3252 S6 f_Y = e_b ? r_Y : s_Y;
3254 as results from if-conversion of a complex condition.
3258 * LAST_STMT: A stmt at the end from which the pattern
3259 search begins, i.e. cast of a bool to
3264 * TYPE_IN: The type of the input arguments to the pattern.
3266 * TYPE_OUT: The type of the output of this pattern.
3268 * Return value: A new stmt that will be used to replace the pattern.
3270 Assuming size of TYPE is the same as size of all comparisons
3271 (otherwise some casts would be added where needed), the above
3272 sequence we create related pattern stmts:
3273 S1' a_T = x1 CMP1 y1 ? 1 : 0;
3274 S3' c_T = x2 CMP2 y2 ? a_T : 0;
3275 S4' d_T = x3 CMP3 y3 ? 1 : 0;
3276 S5' e_T = c_T | d_T;
3279 Instead of the above S3' we could emit:
3280 S2' b_T = x2 CMP2 y2 ? 1 : 0;
3281 S3' c_T = a_T | b_T;
3282 but the above is more efficient. */
3285 vect_recog_bool_pattern (vec
<gimple
*> *stmts
, tree
*type_in
,
3288 gimple
*last_stmt
= stmts
->pop ();
3289 enum tree_code rhs_code
;
3290 tree var
, lhs
, rhs
, vectype
;
3291 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (last_stmt
);
3292 stmt_vec_info new_stmt_info
;
3293 vec_info
*vinfo
= stmt_vinfo
->vinfo
;
3294 gimple
*pattern_stmt
;
3296 if (!is_gimple_assign (last_stmt
))
3299 var
= gimple_assign_rhs1 (last_stmt
);
3300 lhs
= gimple_assign_lhs (last_stmt
);
3302 if ((TYPE_PRECISION (TREE_TYPE (var
)) != 1
3303 || !TYPE_UNSIGNED (TREE_TYPE (var
)))
3304 && TREE_CODE (TREE_TYPE (var
)) != BOOLEAN_TYPE
)
3307 rhs_code
= gimple_assign_rhs_code (last_stmt
);
3308 if (CONVERT_EXPR_CODE_P (rhs_code
))
3310 if (TREE_CODE (TREE_TYPE (lhs
)) != INTEGER_TYPE
3311 || TYPE_PRECISION (TREE_TYPE (lhs
)) == 1)
3313 vectype
= get_vectype_for_scalar_type (TREE_TYPE (lhs
));
3314 if (vectype
== NULL_TREE
)
3317 if (check_bool_pattern (var
, vinfo
))
3319 rhs
= adjust_bool_pattern (var
, TREE_TYPE (lhs
), NULL_TREE
, stmts
);
3320 lhs
= vect_recog_temp_ssa_var (TREE_TYPE (lhs
), NULL
);
3321 if (useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (rhs
)))
3322 pattern_stmt
= gimple_build_assign (lhs
, SSA_NAME
, rhs
);
3325 = gimple_build_assign (lhs
, NOP_EXPR
, rhs
);
3329 tree type
= search_type_for_mask (var
, vinfo
);
3330 tree cst0
, cst1
, tmp
;
3335 /* We may directly use cond with narrowed type to avoid
3336 multiple cond exprs with following result packing and
3337 perform single cond with packed mask instead. In case
3338 of widening we better make cond first and then extract
3340 if (TYPE_MODE (type
) == TYPE_MODE (TREE_TYPE (lhs
)))
3341 type
= TREE_TYPE (lhs
);
3343 cst0
= build_int_cst (type
, 0);
3344 cst1
= build_int_cst (type
, 1);
3345 tmp
= vect_recog_temp_ssa_var (type
, NULL
);
3346 pattern_stmt
= gimple_build_assign (tmp
, COND_EXPR
, var
, cst1
, cst0
);
3348 if (!useless_type_conversion_p (type
, TREE_TYPE (lhs
)))
3350 tree new_vectype
= get_vectype_for_scalar_type (type
);
3351 new_stmt_info
= new_stmt_vec_info (pattern_stmt
, vinfo
);
3352 set_vinfo_for_stmt (pattern_stmt
, new_stmt_info
);
3353 STMT_VINFO_VECTYPE (new_stmt_info
) = new_vectype
;
3354 new_pattern_def_seq (stmt_vinfo
, pattern_stmt
);
3356 lhs
= vect_recog_temp_ssa_var (TREE_TYPE (lhs
), NULL
);
3357 pattern_stmt
= gimple_build_assign (lhs
, CONVERT_EXPR
, tmp
);
3361 *type_out
= vectype
;
3363 stmts
->safe_push (last_stmt
);
3364 if (dump_enabled_p ())
3365 dump_printf_loc (MSG_NOTE
, vect_location
,
3366 "vect_recog_bool_pattern: detected:\n");
3368 return pattern_stmt
;
3370 else if (rhs_code
== COND_EXPR
3371 && TREE_CODE (var
) == SSA_NAME
)
3373 vectype
= get_vectype_for_scalar_type (TREE_TYPE (lhs
));
3374 if (vectype
== NULL_TREE
)
3377 /* Build a scalar type for the boolean result that when
3378 vectorized matches the vector type of the result in
3379 size and number of elements. */
3381 = wi::udiv_trunc (TYPE_SIZE (vectype
),
3382 TYPE_VECTOR_SUBPARTS (vectype
)).to_uhwi ();
3384 = build_nonstandard_integer_type (prec
,
3385 TYPE_UNSIGNED (TREE_TYPE (var
)));
3386 if (get_vectype_for_scalar_type (type
) == NULL_TREE
)
3389 if (!check_bool_pattern (var
, vinfo
))
3392 rhs
= adjust_bool_pattern (var
, type
, NULL_TREE
, stmts
);
3394 lhs
= vect_recog_temp_ssa_var (TREE_TYPE (lhs
), NULL
);
3396 = gimple_build_assign (lhs
, COND_EXPR
,
3397 build2 (NE_EXPR
, boolean_type_node
,
3398 rhs
, build_int_cst (type
, 0)),
3399 gimple_assign_rhs2 (last_stmt
),
3400 gimple_assign_rhs3 (last_stmt
));
3401 *type_out
= vectype
;
3403 stmts
->safe_push (last_stmt
);
3404 if (dump_enabled_p ())
3405 dump_printf_loc (MSG_NOTE
, vect_location
,
3406 "vect_recog_bool_pattern: detected:\n");
3408 return pattern_stmt
;
3410 else if (rhs_code
== SSA_NAME
3411 && STMT_VINFO_DATA_REF (stmt_vinfo
))
3413 stmt_vec_info pattern_stmt_info
;
3414 vectype
= STMT_VINFO_VECTYPE (stmt_vinfo
);
3415 gcc_assert (vectype
!= NULL_TREE
);
3416 if (!VECTOR_MODE_P (TYPE_MODE (vectype
)))
3419 if (check_bool_pattern (var
, vinfo
))
3420 rhs
= adjust_bool_pattern (var
, TREE_TYPE (vectype
),
3424 tree type
= search_type_for_mask (var
, vinfo
);
3425 tree cst0
, cst1
, new_vectype
;
3430 if (TYPE_MODE (type
) == TYPE_MODE (TREE_TYPE (vectype
)))
3431 type
= TREE_TYPE (vectype
);
3433 cst0
= build_int_cst (type
, 0);
3434 cst1
= build_int_cst (type
, 1);
3435 new_vectype
= get_vectype_for_scalar_type (type
);
3437 rhs
= vect_recog_temp_ssa_var (type
, NULL
);
3438 pattern_stmt
= gimple_build_assign (rhs
, COND_EXPR
, var
, cst1
, cst0
);
3440 pattern_stmt_info
= new_stmt_vec_info (pattern_stmt
, vinfo
);
3441 set_vinfo_for_stmt (pattern_stmt
, pattern_stmt_info
);
3442 STMT_VINFO_VECTYPE (pattern_stmt_info
) = new_vectype
;
3443 append_pattern_def_seq (stmt_vinfo
, pattern_stmt
);
3446 lhs
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (vectype
), lhs
);
3447 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (rhs
)))
3449 tree rhs2
= vect_recog_temp_ssa_var (TREE_TYPE (lhs
), NULL
);
3450 gimple
*cast_stmt
= gimple_build_assign (rhs2
, NOP_EXPR
, rhs
);
3451 append_pattern_def_seq (stmt_vinfo
, cast_stmt
);
3454 pattern_stmt
= gimple_build_assign (lhs
, SSA_NAME
, rhs
);
3455 pattern_stmt_info
= new_stmt_vec_info (pattern_stmt
, vinfo
);
3456 set_vinfo_for_stmt (pattern_stmt
, pattern_stmt_info
);
3457 STMT_VINFO_DATA_REF (pattern_stmt_info
)
3458 = STMT_VINFO_DATA_REF (stmt_vinfo
);
3459 STMT_VINFO_DR_BASE_ADDRESS (pattern_stmt_info
)
3460 = STMT_VINFO_DR_BASE_ADDRESS (stmt_vinfo
);
3461 STMT_VINFO_DR_INIT (pattern_stmt_info
) = STMT_VINFO_DR_INIT (stmt_vinfo
);
3462 STMT_VINFO_DR_OFFSET (pattern_stmt_info
)
3463 = STMT_VINFO_DR_OFFSET (stmt_vinfo
);
3464 STMT_VINFO_DR_STEP (pattern_stmt_info
) = STMT_VINFO_DR_STEP (stmt_vinfo
);
3465 STMT_VINFO_DR_ALIGNED_TO (pattern_stmt_info
)
3466 = STMT_VINFO_DR_ALIGNED_TO (stmt_vinfo
);
3467 DR_STMT (STMT_VINFO_DATA_REF (stmt_vinfo
)) = pattern_stmt
;
3468 *type_out
= vectype
;
3470 stmts
->safe_push (last_stmt
);
3471 if (dump_enabled_p ())
3472 dump_printf_loc (MSG_NOTE
, vect_location
,
3473 "vect_recog_bool_pattern: detected:\n");
3474 return pattern_stmt
;
3481 /* A helper for vect_recog_mask_conversion_pattern. Build
3482 conversion of MASK to a type suitable for masking VECTYPE.
3483 Built statement gets required vectype and is appended to
3484 a pattern sequence of STMT_VINFO.
3486 Return converted mask. */
3489 build_mask_conversion (tree mask
, tree vectype
, stmt_vec_info stmt_vinfo
,
3494 stmt_vec_info new_stmt_info
;
3496 masktype
= build_same_sized_truth_vector_type (vectype
);
3497 tmp
= vect_recog_temp_ssa_var (TREE_TYPE (masktype
), NULL
);
3498 stmt
= gimple_build_assign (tmp
, CONVERT_EXPR
, mask
);
3499 new_stmt_info
= new_stmt_vec_info (stmt
, vinfo
);
3500 set_vinfo_for_stmt (stmt
, new_stmt_info
);
3501 STMT_VINFO_VECTYPE (new_stmt_info
) = masktype
;
3502 append_pattern_def_seq (stmt_vinfo
, stmt
);
3508 /* Function vect_recog_mask_conversion_pattern
3510 Try to find statements which require boolean type
3511 converison. Additional conversion statements are
3512 added to handle such cases. For example:
3522 S4 c_1 = m_3 ? c_2 : c_3;
3524 Will be transformed into:
3528 S3'' m_2' = (_Bool[bitsize=32])m_2
3529 S3' m_3' = m_1 & m_2';
3530 S4'' m_3'' = (_Bool[bitsize=8])m_3'
3531 S4' c_1' = m_3'' ? c_2 : c_3; */
3534 vect_recog_mask_conversion_pattern (vec
<gimple
*> *stmts
, tree
*type_in
,
3537 gimple
*last_stmt
= stmts
->pop ();
3538 enum tree_code rhs_code
;
3539 tree lhs
, rhs1
, rhs2
, tmp
, rhs1_type
, rhs2_type
, vectype1
, vectype2
;
3540 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (last_stmt
);
3541 stmt_vec_info pattern_stmt_info
;
3542 vec_info
*vinfo
= stmt_vinfo
->vinfo
;
3543 gimple
*pattern_stmt
;
3545 /* Check for MASK_LOAD ans MASK_STORE calls requiring mask conversion. */
3546 if (is_gimple_call (last_stmt
)
3547 && gimple_call_internal_p (last_stmt
)
3548 && (gimple_call_internal_fn (last_stmt
) == IFN_MASK_STORE
3549 || gimple_call_internal_fn (last_stmt
) == IFN_MASK_LOAD
))
3551 bool load
= (gimple_call_internal_fn (last_stmt
) == IFN_MASK_LOAD
);
3555 lhs
= gimple_call_lhs (last_stmt
);
3556 vectype1
= get_vectype_for_scalar_type (TREE_TYPE (lhs
));
3560 rhs2
= gimple_call_arg (last_stmt
, 3);
3561 vectype1
= get_vectype_for_scalar_type (TREE_TYPE (rhs2
));
3564 rhs1
= gimple_call_arg (last_stmt
, 2);
3565 rhs1_type
= search_type_for_mask (rhs1
, vinfo
);
3568 vectype2
= get_mask_type_for_scalar_type (rhs1_type
);
3570 if (!vectype1
|| !vectype2
3571 || TYPE_VECTOR_SUBPARTS (vectype1
) == TYPE_VECTOR_SUBPARTS (vectype2
))
3574 tmp
= build_mask_conversion (rhs1
, vectype1
, stmt_vinfo
, vinfo
);
3578 lhs
= vect_recog_temp_ssa_var (TREE_TYPE (lhs
), NULL
);
3580 = gimple_build_call_internal (IFN_MASK_LOAD
, 3,
3581 gimple_call_arg (last_stmt
, 0),
3582 gimple_call_arg (last_stmt
, 1),
3584 gimple_call_set_lhs (pattern_stmt
, lhs
);
3588 = gimple_build_call_internal (IFN_MASK_STORE
, 4,
3589 gimple_call_arg (last_stmt
, 0),
3590 gimple_call_arg (last_stmt
, 1),
3592 gimple_call_arg (last_stmt
, 3));
3595 pattern_stmt_info
= new_stmt_vec_info (pattern_stmt
, vinfo
);
3596 set_vinfo_for_stmt (pattern_stmt
, pattern_stmt_info
);
3597 STMT_VINFO_DATA_REF (pattern_stmt_info
)
3598 = STMT_VINFO_DATA_REF (stmt_vinfo
);
3599 STMT_VINFO_DR_BASE_ADDRESS (pattern_stmt_info
)
3600 = STMT_VINFO_DR_BASE_ADDRESS (stmt_vinfo
);
3601 STMT_VINFO_DR_INIT (pattern_stmt_info
) = STMT_VINFO_DR_INIT (stmt_vinfo
);
3602 STMT_VINFO_DR_OFFSET (pattern_stmt_info
)
3603 = STMT_VINFO_DR_OFFSET (stmt_vinfo
);
3604 STMT_VINFO_DR_STEP (pattern_stmt_info
) = STMT_VINFO_DR_STEP (stmt_vinfo
);
3605 STMT_VINFO_DR_ALIGNED_TO (pattern_stmt_info
)
3606 = STMT_VINFO_DR_ALIGNED_TO (stmt_vinfo
);
3607 DR_STMT (STMT_VINFO_DATA_REF (stmt_vinfo
)) = pattern_stmt
;
3609 *type_out
= vectype1
;
3610 *type_in
= vectype1
;
3611 stmts
->safe_push (last_stmt
);
3612 if (dump_enabled_p ())
3613 dump_printf_loc (MSG_NOTE
, vect_location
,
3614 "vect_recog_mask_conversion_pattern: detected:\n");
3616 return pattern_stmt
;
3619 if (!is_gimple_assign (last_stmt
))
3622 lhs
= gimple_assign_lhs (last_stmt
);
3623 rhs1
= gimple_assign_rhs1 (last_stmt
);
3624 rhs_code
= gimple_assign_rhs_code (last_stmt
);
3626 /* Check for cond expression requiring mask conversion. */
3627 if (rhs_code
== COND_EXPR
)
3629 /* vect_recog_mixed_size_cond_pattern could apply.
3631 if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo
))
3634 vectype1
= get_vectype_for_scalar_type (TREE_TYPE (lhs
));
3636 if (TREE_CODE (rhs1
) == SSA_NAME
)
3638 rhs1_type
= search_type_for_mask (rhs1
, vinfo
);
3643 rhs1_type
= TREE_TYPE (TREE_OPERAND (rhs1
, 0));
3645 vectype2
= get_mask_type_for_scalar_type (rhs1_type
);
3647 if (!vectype1
|| !vectype2
3648 || TYPE_VECTOR_SUBPARTS (vectype1
) == TYPE_VECTOR_SUBPARTS (vectype2
))
3651 /* If rhs1 is a comparison we need to move it into a
3652 separate statement. */
3653 if (TREE_CODE (rhs1
) != SSA_NAME
)
3655 tmp
= vect_recog_temp_ssa_var (TREE_TYPE (rhs1
), NULL
);
3656 pattern_stmt
= gimple_build_assign (tmp
, rhs1
);
3659 pattern_stmt_info
= new_stmt_vec_info (pattern_stmt
, vinfo
);
3660 set_vinfo_for_stmt (pattern_stmt
, pattern_stmt_info
);
3661 STMT_VINFO_VECTYPE (pattern_stmt_info
) = vectype2
;
3662 append_pattern_def_seq (stmt_vinfo
, pattern_stmt
);
3665 tmp
= build_mask_conversion (rhs1
, vectype1
, stmt_vinfo
, vinfo
);
3667 lhs
= vect_recog_temp_ssa_var (TREE_TYPE (lhs
), NULL
);
3668 pattern_stmt
= gimple_build_assign (lhs
, COND_EXPR
, tmp
,
3669 gimple_assign_rhs2 (last_stmt
),
3670 gimple_assign_rhs3 (last_stmt
));
3672 *type_out
= vectype1
;
3673 *type_in
= vectype1
;
3674 stmts
->safe_push (last_stmt
);
3675 if (dump_enabled_p ())
3676 dump_printf_loc (MSG_NOTE
, vect_location
,
3677 "vect_recog_mask_conversion_pattern: detected:\n");
3679 return pattern_stmt
;
3682 /* Now check for binary boolean operations requiring conversion for
3684 if (TREE_CODE (TREE_TYPE (lhs
)) != BOOLEAN_TYPE
)
3687 if (rhs_code
!= BIT_IOR_EXPR
3688 && rhs_code
!= BIT_XOR_EXPR
3689 && rhs_code
!= BIT_AND_EXPR
)
3692 rhs2
= gimple_assign_rhs2 (last_stmt
);
3694 rhs1_type
= search_type_for_mask (rhs1
, vinfo
);
3695 rhs2_type
= search_type_for_mask (rhs2
, vinfo
);
3697 if (!rhs1_type
|| !rhs2_type
3698 || TYPE_PRECISION (rhs1_type
) == TYPE_PRECISION (rhs2_type
))
3701 if (TYPE_PRECISION (rhs1_type
) < TYPE_PRECISION (rhs2_type
))
3703 vectype1
= get_mask_type_for_scalar_type (rhs1_type
);
3706 rhs2
= build_mask_conversion (rhs2
, vectype1
, stmt_vinfo
, vinfo
);
3710 vectype1
= get_mask_type_for_scalar_type (rhs2_type
);
3713 rhs1
= build_mask_conversion (rhs1
, vectype1
, stmt_vinfo
, vinfo
);
3716 lhs
= vect_recog_temp_ssa_var (TREE_TYPE (lhs
), NULL
);
3717 pattern_stmt
= gimple_build_assign (lhs
, rhs_code
, rhs1
, rhs2
);
3719 *type_out
= vectype1
;
3720 *type_in
= vectype1
;
3721 stmts
->safe_push (last_stmt
);
3722 if (dump_enabled_p ())
3723 dump_printf_loc (MSG_NOTE
, vect_location
,
3724 "vect_recog_mask_conversion_pattern: detected:\n");
3726 return pattern_stmt
;
3730 /* Mark statements that are involved in a pattern. */
3733 vect_mark_pattern_stmts (gimple
*orig_stmt
, gimple
*pattern_stmt
,
3734 tree pattern_vectype
)
3736 stmt_vec_info pattern_stmt_info
, def_stmt_info
;
3737 stmt_vec_info orig_stmt_info
= vinfo_for_stmt (orig_stmt
);
3738 vec_info
*vinfo
= orig_stmt_info
->vinfo
;
3741 pattern_stmt_info
= vinfo_for_stmt (pattern_stmt
);
3742 if (pattern_stmt_info
== NULL
)
3744 pattern_stmt_info
= new_stmt_vec_info (pattern_stmt
, vinfo
);
3745 set_vinfo_for_stmt (pattern_stmt
, pattern_stmt_info
);
3747 gimple_set_bb (pattern_stmt
, gimple_bb (orig_stmt
));
3749 STMT_VINFO_RELATED_STMT (pattern_stmt_info
) = orig_stmt
;
3750 STMT_VINFO_DEF_TYPE (pattern_stmt_info
)
3751 = STMT_VINFO_DEF_TYPE (orig_stmt_info
);
3752 STMT_VINFO_VECTYPE (pattern_stmt_info
) = pattern_vectype
;
3753 STMT_VINFO_IN_PATTERN_P (orig_stmt_info
) = true;
3754 STMT_VINFO_RELATED_STMT (orig_stmt_info
) = pattern_stmt
;
3755 STMT_VINFO_PATTERN_DEF_SEQ (pattern_stmt_info
)
3756 = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info
);
3757 if (STMT_VINFO_PATTERN_DEF_SEQ (pattern_stmt_info
))
3759 gimple_stmt_iterator si
;
3760 for (si
= gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (pattern_stmt_info
));
3761 !gsi_end_p (si
); gsi_next (&si
))
3763 def_stmt
= gsi_stmt (si
);
3764 def_stmt_info
= vinfo_for_stmt (def_stmt
);
3765 if (def_stmt_info
== NULL
)
3767 def_stmt_info
= new_stmt_vec_info (def_stmt
, vinfo
);
3768 set_vinfo_for_stmt (def_stmt
, def_stmt_info
);
3770 gimple_set_bb (def_stmt
, gimple_bb (orig_stmt
));
3771 STMT_VINFO_RELATED_STMT (def_stmt_info
) = orig_stmt
;
3772 STMT_VINFO_DEF_TYPE (def_stmt_info
) = vect_internal_def
;
3773 if (STMT_VINFO_VECTYPE (def_stmt_info
) == NULL_TREE
)
3774 STMT_VINFO_VECTYPE (def_stmt_info
) = pattern_vectype
;
3779 /* Function vect_pattern_recog_1
3782 PATTERN_RECOG_FUNC: A pointer to a function that detects a certain
3783 computation pattern.
3784 STMT: A stmt from which the pattern search should start.
3786 If PATTERN_RECOG_FUNC successfully detected the pattern, it creates an
3787 expression that computes the same functionality and can be used to
3788 replace the sequence of stmts that are involved in the pattern.
3791 This function checks if the expression returned by PATTERN_RECOG_FUNC is
3792 supported in vector form by the target. We use 'TYPE_IN' to obtain the
3793 relevant vector type. If 'TYPE_IN' is already a vector type, then this
3794 indicates that target support had already been checked by PATTERN_RECOG_FUNC.
3795 If 'TYPE_OUT' is also returned by PATTERN_RECOG_FUNC, we check that it fits
3796 to the available target pattern.
3798 This function also does some bookkeeping, as explained in the documentation
3799 for vect_recog_pattern. */
3802 vect_pattern_recog_1 (vect_recog_func
*recog_func
,
3803 gimple_stmt_iterator si
,
3804 vec
<gimple
*> *stmts_to_replace
)
3806 gimple
*stmt
= gsi_stmt (si
), *pattern_stmt
;
3807 stmt_vec_info stmt_info
;
3808 loop_vec_info loop_vinfo
;
3809 tree pattern_vectype
;
3810 tree type_in
, type_out
;
3811 enum tree_code code
;
3815 stmts_to_replace
->truncate (0);
3816 stmts_to_replace
->quick_push (stmt
);
3817 pattern_stmt
= recog_func
->fn (stmts_to_replace
, &type_in
, &type_out
);
3821 stmt
= stmts_to_replace
->last ();
3822 stmt_info
= vinfo_for_stmt (stmt
);
3823 loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3825 if (VECTOR_BOOLEAN_TYPE_P (type_in
)
3826 || VECTOR_MODE_P (TYPE_MODE (type_in
)))
3828 /* No need to check target support (already checked by the pattern
3829 recognition function). */
3830 pattern_vectype
= type_out
? type_out
: type_in
;
3834 machine_mode vec_mode
;
3835 enum insn_code icode
;
3838 /* Check target support */
3839 type_in
= get_vectype_for_scalar_type (type_in
);
3843 type_out
= get_vectype_for_scalar_type (type_out
);
3848 pattern_vectype
= type_out
;
3850 if (is_gimple_assign (pattern_stmt
))
3851 code
= gimple_assign_rhs_code (pattern_stmt
);
3854 gcc_assert (is_gimple_call (pattern_stmt
));
3858 optab
= optab_for_tree_code (code
, type_in
, optab_default
);
3859 vec_mode
= TYPE_MODE (type_in
);
3861 || (icode
= optab_handler (optab
, vec_mode
)) == CODE_FOR_nothing
3862 || (insn_data
[icode
].operand
[0].mode
!= TYPE_MODE (type_out
)))
3866 /* Found a vectorizable pattern. */
3867 if (dump_enabled_p ())
3869 dump_printf_loc (MSG_NOTE
, vect_location
,
3870 "%s pattern recognized: ", recog_func
->name
);
3871 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_stmt
, 0);
3874 /* Mark the stmts that are involved in the pattern. */
3875 vect_mark_pattern_stmts (stmt
, pattern_stmt
, pattern_vectype
);
3877 /* Patterns cannot be vectorized using SLP, because they change the order of
3880 FOR_EACH_VEC_ELT (LOOP_VINFO_REDUCTIONS (loop_vinfo
), i
, next
)
3882 LOOP_VINFO_REDUCTIONS (loop_vinfo
).ordered_remove (i
);
3884 /* It is possible that additional pattern stmts are created and inserted in
3885 STMTS_TO_REPLACE. We create a stmt_info for each of them, and mark the
3886 relevant statements. */
3887 for (i
= 0; stmts_to_replace
->iterate (i
, &stmt
)
3888 && (unsigned) i
< (stmts_to_replace
->length () - 1);
3891 stmt_info
= vinfo_for_stmt (stmt
);
3892 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
3893 if (dump_enabled_p ())
3895 dump_printf_loc (MSG_NOTE
, vect_location
,
3896 "additional pattern stmt: ");
3897 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_stmt
, 0);
3900 vect_mark_pattern_stmts (stmt
, pattern_stmt
, NULL_TREE
);
3907 /* Function vect_pattern_recog
3910 LOOP_VINFO - a struct_loop_info of a loop in which we want to look for
3913 Output - for each computation idiom that is detected we create a new stmt
3914 that provides the same functionality and that can be vectorized. We
3915 also record some information in the struct_stmt_info of the relevant
3916 stmts, as explained below:
3918 At the entry to this function we have the following stmts, with the
3919 following initial value in the STMT_VINFO fields:
3921 stmt in_pattern_p related_stmt vec_stmt
3922 S1: a_i = .... - - -
3923 S2: a_2 = ..use(a_i).. - - -
3924 S3: a_1 = ..use(a_2).. - - -
3925 S4: a_0 = ..use(a_1).. - - -
3926 S5: ... = ..use(a_0).. - - -
3928 Say the sequence {S1,S2,S3,S4} was detected as a pattern that can be
3929 represented by a single stmt. We then:
3930 - create a new stmt S6 equivalent to the pattern (the stmt is not
3931 inserted into the code)
3932 - fill in the STMT_VINFO fields as follows:
3934 in_pattern_p related_stmt vec_stmt
3935 S1: a_i = .... - - -
3936 S2: a_2 = ..use(a_i).. - - -
3937 S3: a_1 = ..use(a_2).. - - -
3938 S4: a_0 = ..use(a_1).. true S6 -
3939 '---> S6: a_new = .... - S4 -
3940 S5: ... = ..use(a_0).. - - -
3942 (the last stmt in the pattern (S4) and the new pattern stmt (S6) point
3943 to each other through the RELATED_STMT field).
3945 S6 will be marked as relevant in vect_mark_stmts_to_be_vectorized instead
3946 of S4 because it will replace all its uses. Stmts {S1,S2,S3} will
3947 remain irrelevant unless used by stmts other than S4.
3949 If vectorization succeeds, vect_transform_stmt will skip over {S1,S2,S3}
3950 (because they are marked as irrelevant). It will vectorize S6, and record
3951 a pointer to the new vector stmt VS6 from S6 (as usual).
3952 S4 will be skipped, and S5 will be vectorized as usual:
3954 in_pattern_p related_stmt vec_stmt
3955 S1: a_i = .... - - -
3956 S2: a_2 = ..use(a_i).. - - -
3957 S3: a_1 = ..use(a_2).. - - -
3958 > VS6: va_new = .... - - -
3959 S4: a_0 = ..use(a_1).. true S6 VS6
3960 '---> S6: a_new = .... - S4 VS6
3961 > VS5: ... = ..vuse(va_new).. - - -
3962 S5: ... = ..use(a_0).. - - -
3964 DCE could then get rid of {S1,S2,S3,S4,S5} (if their defs are not used
3965 elsewhere), and we'll end up with:
3968 VS5: ... = ..vuse(va_new)..
3970 In case of more than one pattern statements, e.g., widen-mult with
3974 S2 a_T = (TYPE) a_t;
3975 '--> S3: a_it = (interm_type) a_t;
3976 S4 prod_T = a_T * CONST;
3977 '--> S5: prod_T' = a_it w* CONST;
3979 there may be other users of a_T outside the pattern. In that case S2 will
3980 be marked as relevant (as well as S3), and both S2 and S3 will be analyzed
3981 and vectorized. The vector stmt VS2 will be recorded in S2, and VS3 will
3982 be recorded in S3. */
3985 vect_pattern_recog (vec_info
*vinfo
)
3990 gimple_stmt_iterator si
;
3992 auto_vec
<gimple
*, 1> stmts_to_replace
;
3995 if (dump_enabled_p ())
3996 dump_printf_loc (MSG_NOTE
, vect_location
,
3997 "=== vect_pattern_recog ===\n");
3999 if (loop_vec_info loop_vinfo
= dyn_cast
<loop_vec_info
> (vinfo
))
4001 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
4002 bbs
= LOOP_VINFO_BBS (loop_vinfo
);
4003 nbbs
= loop
->num_nodes
;
4005 /* Scan through the loop stmts, applying the pattern recognition
4006 functions starting at each stmt visited: */
4007 for (i
= 0; i
< nbbs
; i
++)
4009 basic_block bb
= bbs
[i
];
4010 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
4012 /* Scan over all generic vect_recog_xxx_pattern functions. */
4013 for (j
= 0; j
< NUM_PATTERNS
; j
++)
4014 if (vect_pattern_recog_1 (&vect_vect_recog_func_ptrs
[j
], si
,
4022 bb_vec_info bb_vinfo
= as_a
<bb_vec_info
> (vinfo
);
4023 for (si
= bb_vinfo
->region_begin
;
4024 gsi_stmt (si
) != gsi_stmt (bb_vinfo
->region_end
); gsi_next (&si
))
4026 if ((stmt
= gsi_stmt (si
))
4027 && vinfo_for_stmt (stmt
)
4028 && !STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt
)))
4031 /* Scan over all generic vect_recog_xxx_pattern functions. */
4032 for (j
= 0; j
< NUM_PATTERNS
; j
++)
4033 if (vect_pattern_recog_1 (&vect_vect_recog_func_ptrs
[j
], si
,