"vect_recog_widen_minus_pattern");
}
+/* Function vect_recog_ctz_ffs_pattern
+
+ Try to find the following pattern:
+
+ TYPE1 A;
+ TYPE1 B;
+
+ B = __builtin_ctz{,l,ll} (A);
+
+ or
+
+ B = __builtin_ffs{,l,ll} (A);
+
+ Input:
+
+ * STMT_VINFO: The stmt from which the pattern search begins.
+ here it starts with B = __builtin_* (A);
+
+ Output:
+
+ * TYPE_OUT: The vector type of the output of this pattern.
+
+ * Return value: A new stmt that will be used to replace the sequence of
+ stmts that constitute the pattern, using clz or popcount builtins. */
+
+static gimple *
+vect_recog_ctz_ffs_pattern (vec_info *vinfo, stmt_vec_info stmt_vinfo,
+ tree *type_out)
+{
+ gimple *call_stmt = stmt_vinfo->stmt;
+ gimple *pattern_stmt;
+ tree rhs_oprnd, rhs_type, lhs_oprnd, lhs_type, vec_type, vec_rhs_type;
+ tree new_var;
+ internal_fn ifn = IFN_LAST, ifnnew = IFN_LAST;
+ bool defined_at_zero = true, defined_at_zero_new = false;
+ int val = 0, val_new = 0;
+ int prec;
+ int sub = 0, add = 0;
+ location_t loc;
+
+ if (!is_gimple_call (call_stmt))
+ return NULL;
+
+ if (gimple_call_num_args (call_stmt) != 1)
+ return NULL;
+
+ rhs_oprnd = gimple_call_arg (call_stmt, 0);
+ rhs_type = TREE_TYPE (rhs_oprnd);
+ lhs_oprnd = gimple_call_lhs (call_stmt);
+ if (!lhs_oprnd)
+ return NULL;
+ lhs_type = TREE_TYPE (lhs_oprnd);
+ if (!INTEGRAL_TYPE_P (lhs_type)
+ || !INTEGRAL_TYPE_P (rhs_type)
+ || !type_has_mode_precision_p (rhs_type)
+ || TREE_CODE (rhs_oprnd) != SSA_NAME)
+ return NULL;
+
+ switch (gimple_call_combined_fn (call_stmt))
+ {
+ CASE_CFN_CTZ:
+ ifn = IFN_CTZ;
+ if (!gimple_call_internal_p (call_stmt)
+ || CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type),
+ val) != 2)
+ defined_at_zero = false;
+ break;
+ CASE_CFN_FFS:
+ ifn = IFN_FFS;
+ break;
+ default:
+ return NULL;
+ }
+
+ prec = TYPE_PRECISION (rhs_type);
+ loc = gimple_location (call_stmt);
+
+ vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
+ if (!vec_type)
+ return NULL;
+
+ vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type);
+ if (!vec_rhs_type)
+ return NULL;
+
+ /* Do it only if the backend doesn't have ctz<vector_mode>2 or
+ ffs<vector_mode>2 pattern but does have clz<vector_mode>2 or
+ popcount<vector_mode>2. */
+ if (!vec_type
+ || direct_internal_fn_supported_p (ifn, vec_rhs_type,
+ OPTIMIZE_FOR_SPEED))
+ return NULL;
+
+ if (ifn == IFN_FFS
+ && direct_internal_fn_supported_p (IFN_CTZ, vec_rhs_type,
+ OPTIMIZE_FOR_SPEED))
+ {
+ ifnnew = IFN_CTZ;
+ defined_at_zero_new
+ = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type),
+ val_new) == 2;
+ }
+ else if (direct_internal_fn_supported_p (IFN_CLZ, vec_rhs_type,
+ OPTIMIZE_FOR_SPEED))
+ {
+ ifnnew = IFN_CLZ;
+ defined_at_zero_new
+ = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type),
+ val_new) == 2;
+ }
+ if ((ifnnew == IFN_LAST
+ || (defined_at_zero && !defined_at_zero_new))
+ && direct_internal_fn_supported_p (IFN_POPCOUNT, vec_rhs_type,
+ OPTIMIZE_FOR_SPEED))
+ {
+ ifnnew = IFN_POPCOUNT;
+ defined_at_zero_new = true;
+ val_new = prec;
+ }
+ if (ifnnew == IFN_LAST)
+ return NULL;
+
+ vect_pattern_detected ("vec_recog_ctz_ffs_pattern", call_stmt);
+
+ if ((ifnnew == IFN_CLZ
+ && defined_at_zero
+ && defined_at_zero_new
+ && val == prec
+ && val_new == prec)
+ || (ifnnew == IFN_POPCOUNT && ifn == IFN_CLZ))
+ {
+ /* .CTZ (X) = PREC - .CLZ ((X - 1) & ~X)
+ .CTZ (X) = .POPCOUNT ((X - 1) & ~X). */
+ if (ifnnew == IFN_CLZ)
+ sub = prec;
+ val_new = prec;
+
+ if (!TYPE_UNSIGNED (rhs_type))
+ {
+ rhs_type = unsigned_type_for (rhs_type);
+ vec_rhs_type = get_vectype_for_scalar_type (vinfo, rhs_type);
+ new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
+ pattern_stmt = gimple_build_assign (new_var, NOP_EXPR, rhs_oprnd);
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt,
+ vec_rhs_type);
+ rhs_oprnd = new_var;
+ }
+
+ tree m1 = vect_recog_temp_ssa_var (rhs_type, NULL);
+ pattern_stmt = gimple_build_assign (m1, PLUS_EXPR, rhs_oprnd,
+ build_int_cst (rhs_type, -1));
+ gimple_set_location (pattern_stmt, loc);
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
+
+ new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
+ pattern_stmt = gimple_build_assign (new_var, BIT_NOT_EXPR, rhs_oprnd);
+ gimple_set_location (pattern_stmt, loc);
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
+ rhs_oprnd = new_var;
+
+ new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
+ pattern_stmt = gimple_build_assign (new_var, BIT_AND_EXPR,
+ m1, rhs_oprnd);
+ gimple_set_location (pattern_stmt, loc);
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
+ rhs_oprnd = new_var;
+ }
+ else if (ifnnew == IFN_CLZ)
+ {
+ /* .CTZ (X) = (PREC - 1) - .CLZ (X & -X)
+ .FFS (X) = PREC - .CLZ (X & -X). */
+ sub = prec - (ifn == IFN_CTZ);
+ val_new = sub - val_new;
+
+ tree neg = vect_recog_temp_ssa_var (rhs_type, NULL);
+ pattern_stmt = gimple_build_assign (neg, NEGATE_EXPR, rhs_oprnd);
+ gimple_set_location (pattern_stmt, loc);
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
+
+ new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
+ pattern_stmt = gimple_build_assign (new_var, BIT_AND_EXPR,
+ rhs_oprnd, neg);
+ gimple_set_location (pattern_stmt, loc);
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
+ rhs_oprnd = new_var;
+ }
+ else if (ifnnew == IFN_POPCOUNT)
+ {
+ /* .CTZ (X) = PREC - .POPCOUNT (X | -X)
+ .FFS (X) = (PREC + 1) - .POPCOUNT (X | -X). */
+ sub = prec + (ifn == IFN_FFS);
+ val_new = sub;
+
+ tree neg = vect_recog_temp_ssa_var (rhs_type, NULL);
+ pattern_stmt = gimple_build_assign (neg, NEGATE_EXPR, rhs_oprnd);
+ gimple_set_location (pattern_stmt, loc);
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
+
+ new_var = vect_recog_temp_ssa_var (rhs_type, NULL);
+ pattern_stmt = gimple_build_assign (new_var, BIT_IOR_EXPR,
+ rhs_oprnd, neg);
+ gimple_set_location (pattern_stmt, loc);
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_rhs_type);
+ rhs_oprnd = new_var;
+ }
+ else if (ifnnew == IFN_CTZ)
+ {
+ /* .FFS (X) = .CTZ (X) + 1. */
+ add = 1;
+ val_new++;
+ }
+
+ /* Create B = .IFNNEW (A). */
+ new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
+ pattern_stmt = gimple_build_call_internal (ifnnew, 1, rhs_oprnd);
+ gimple_call_set_lhs (pattern_stmt, new_var);
+ gimple_set_location (pattern_stmt, loc);
+ *type_out = vec_type;
+
+ if (sub)
+ {
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
+ tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
+ pattern_stmt = gimple_build_assign (ret_var, MINUS_EXPR,
+ build_int_cst (lhs_type, sub),
+ new_var);
+ gimple_set_location (pattern_stmt, loc);
+ new_var = ret_var;
+ }
+ else if (add)
+ {
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
+ tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
+ pattern_stmt = gimple_build_assign (ret_var, PLUS_EXPR, new_var,
+ build_int_cst (lhs_type, add));
+ gimple_set_location (pattern_stmt, loc);
+ new_var = ret_var;
+ }
+
+ if (defined_at_zero
+ && (!defined_at_zero_new || val != val_new))
+ {
+ append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
+ tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
+ rhs_oprnd = gimple_call_arg (call_stmt, 0);
+ rhs_type = TREE_TYPE (rhs_oprnd);
+ tree cmp = build2_loc (loc, NE_EXPR, boolean_type_node,
+ rhs_oprnd, build_zero_cst (rhs_type));
+ pattern_stmt = gimple_build_assign (ret_var, COND_EXPR, cmp,
+ new_var,
+ build_int_cst (lhs_type, val));
+ }
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "created pattern stmt: %G", pattern_stmt);
+
+ return pattern_stmt;
+}
+
/* Function vect_recog_popcount_clz_ctz_ffs_pattern
Try to find the following pattern:
gcc_unreachable ();
}
- vect_pattern_detected ("vec_recog_popcount_clz_ctz_ffs_pattern",
- call_stmt);
vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
/* Do it only if the backend has popcount<vector_mode>2 etc. pattern. */
- if (!vec_type
- || !direct_internal_fn_supported_p (ifn, vec_type,
- OPTIMIZE_FOR_SPEED))
+ if (!vec_type)
return NULL;
+ bool supported
+ = direct_internal_fn_supported_p (ifn, vec_type, OPTIMIZE_FOR_SPEED);
+ if (!supported)
+ switch (ifn)
+ {
+ case IFN_POPCOUNT:
+ case IFN_CLZ:
+ return NULL;
+ case IFN_FFS:
+ /* vect_recog_ctz_ffs_pattern can implement ffs using ctz. */
+ if (direct_internal_fn_supported_p (IFN_CTZ, vec_type,
+ OPTIMIZE_FOR_SPEED))
+ break;
+ /* FALLTHRU */
+ case IFN_CTZ:
+ /* vect_recog_ctz_ffs_pattern can implement ffs or ctz using
+ clz or popcount. */
+ if (direct_internal_fn_supported_p (IFN_CLZ, vec_type,
+ OPTIMIZE_FOR_SPEED))
+ break;
+ if (direct_internal_fn_supported_p (IFN_POPCOUNT, vec_type,
+ OPTIMIZE_FOR_SPEED))
+ break;
+ return NULL;
+ default:
+ gcc_unreachable ();
+ }
+
+ vect_pattern_detected ("vec_recog_popcount_clz_ctz_ffs_pattern",
+ call_stmt);
+
/* Create B = .POPCOUNT (A). */
new_var = vect_recog_temp_ssa_var (lhs_type, NULL);
pattern_stmt = gimple_build_call_internal (ifn, 1, unprom_diff.op);
if (addend)
{
+ gcc_assert (supported);
append_pattern_def_seq (vinfo, stmt_vinfo, pattern_stmt, vec_type);
tree ret_var = vect_recog_temp_ssa_var (lhs_type, NULL);
pattern_stmt = gimple_build_assign (ret_var, PLUS_EXPR, new_var,
build_int_cst (lhs_type, addend));
}
+ else if (!supported)
+ {
+ stmt_vec_info new_stmt_info = vinfo->add_stmt (pattern_stmt);
+ STMT_VINFO_VECTYPE (new_stmt_info) = vec_type;
+ pattern_stmt
+ = vect_recog_ctz_ffs_pattern (vinfo, new_stmt_info, type_out);
+ if (pattern_stmt == NULL)
+ return NULL;
+ if (gimple_seq seq = STMT_VINFO_PATTERN_DEF_SEQ (new_stmt_info))
+ {
+ gimple_seq *pseq = &STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo);
+ gimple_seq_add_seq_without_update (pseq, seq);
+ }
+ }
return pattern_stmt;
}
{ vect_recog_widen_sum_pattern, "widen_sum" },
{ vect_recog_pow_pattern, "pow" },
{ vect_recog_popcount_clz_ctz_ffs_pattern, "popcount_clz_ctz_ffs" },
+ { vect_recog_ctz_ffs_pattern, "ctz_ffs" },
{ vect_recog_widen_shift_pattern, "widen_shift" },
{ vect_recog_rotate_pattern, "rotate" },
{ vect_recog_vector_vector_shift_pattern, "vector_vector_shift" },
projects / thirdparty / gcc.git / commitdiff
