/* Induction variable optimizations.
- Copyright (C) 2003-2020 Free Software Foundation, Inc.
+ Copyright (C) 2003-2021 Free Software Foundation, Inc.
This file is part of GCC.
#include "tree-vectorizer.h"
#include "dbgcnt.h"
+/* For lang_hooks.types.type_for_mode. */
+#include "langhooks.h"
+
/* FIXME: Expressions are expanded to RTL in this pass to determine the
cost of different addressing modes. This should be moved to a TBD
interface between the GIMPLE and RTL worlds. */
{
case IFN_MASK_LOAD:
case IFN_MASK_LOAD_LANES:
+ case IFN_LEN_LOAD:
if (op_p == gimple_call_arg_ptr (call, 0))
return TREE_TYPE (gimple_call_lhs (call));
return NULL_TREE;
case IFN_MASK_STORE:
case IFN_MASK_STORE_LANES:
+ case IFN_LEN_STORE:
if (op_p == gimple_call_arg_ptr (call, 0))
return TREE_TYPE (gimple_call_arg (call, 3));
return NULL_TREE;
list_index = (unsigned) as * MAX_MACHINE_MODE + (unsigned) mem_mode;
if (list_index >= vec_safe_length (addr_list))
- vec_safe_grow_cleared (addr_list, list_index + MAX_MACHINE_MODE);
+ vec_safe_grow_cleared (addr_list, list_index + MAX_MACHINE_MODE, true);
addr = (*addr_list)[list_index];
if (!addr)
{
poly_uint64 offset;
tree base;
- tree basetype;
struct iv *iv = use->iv;
+ tree basetype = TREE_TYPE (iv->base);
+
+ /* Don't add candidate for iv_use with non integer, pointer or non-mode
+ precision types, instead, add candidate for the corresponding scev in
+ unsigned type with the same precision. See PR93674 for more info. */
+ if ((TREE_CODE (basetype) != INTEGER_TYPE && !POINTER_TYPE_P (basetype))
+ || !type_has_mode_precision_p (basetype))
+ {
+ basetype = lang_hooks.types.type_for_mode (TYPE_MODE (basetype),
+ TYPE_UNSIGNED (basetype));
+ add_candidate (data, fold_convert (basetype, iv->base),
+ fold_convert (basetype, iv->step), false, NULL);
+ return;
+ }
add_candidate (data, iv->base, iv->step, false, use);
unsigned nsize = ((unsigned) as + 1) *MAX_MACHINE_MODE;
gcc_assert (nsize > idx);
- ainc_cost_data_list.safe_grow_cleared (nsize);
+ ainc_cost_data_list.safe_grow_cleared (nsize, true);
}
ainc_cost_data *data = ainc_cost_data_list[idx];
}
}
+/* If PREFERRED_MODE is suitable and profitable, use the preferred
+ PREFERRED_MODE to compute doloop iv base from niter: base = niter + 1. */
+
+static tree
+compute_doloop_base_on_mode (machine_mode preferred_mode, tree niter,
+ const widest_int &iterations_max)
+{
+ tree ntype = TREE_TYPE (niter);
+ tree pref_type = lang_hooks.types.type_for_mode (preferred_mode, 1);
+ if (!pref_type)
+ return fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
+ build_int_cst (ntype, 1));
+
+ gcc_assert (TREE_CODE (pref_type) == INTEGER_TYPE);
+
+ int prec = TYPE_PRECISION (ntype);
+ int pref_prec = TYPE_PRECISION (pref_type);
+
+ tree base;
+
+ /* Check if the PREFERRED_MODED is able to present niter. */
+ if (pref_prec > prec
+ || wi::ltu_p (iterations_max,
+ widest_int::from (wi::max_value (pref_prec, UNSIGNED),
+ UNSIGNED)))
+ {
+ /* No wrap, it is safe to use preferred type after niter + 1. */
+ if (wi::ltu_p (iterations_max,
+ widest_int::from (wi::max_value (prec, UNSIGNED),
+ UNSIGNED)))
+ {
+ /* This could help to optimize "-1 +1" pair when niter looks
+ like "n-1": n is in original mode. "base = (n - 1) + 1"
+ in PREFERRED_MODED: it could be base = (PREFERRED_TYPE)n. */
+ base = fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
+ build_int_cst (ntype, 1));
+ base = fold_convert (pref_type, base);
+ }
+
+ /* To avoid wrap, convert niter to preferred type before plus 1. */
+ else
+ {
+ niter = fold_convert (pref_type, niter);
+ base = fold_build2 (PLUS_EXPR, pref_type, unshare_expr (niter),
+ build_int_cst (pref_type, 1));
+ }
+ }
+ else
+ base = fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
+ build_int_cst (ntype, 1));
+ return base;
+}
+
/* Add one doloop dedicated IV candidate:
- Base is (may_be_zero ? 1 : (niter + 1)).
- Step is -1. */
return;
}
- tree base = fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
- build_int_cst (ntype, 1));
+ machine_mode mode = TYPE_MODE (ntype);
+ machine_mode pref_mode = targetm.preferred_doloop_mode (mode);
+
+ tree base;
+ if (mode != pref_mode)
+ {
+ base = compute_doloop_base_on_mode (pref_mode, niter, niter_desc->max);
+ ntype = TREE_TYPE (base);
+ }
+ else
+ base = fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
+ build_int_cst (ntype, 1));
+
+
add_candidate (data, base, build_int_cst (ntype, -1), true, NULL, NULL, true);
}
}
comp = fold_convert (type, comp);
- if (!valid_gimple_rhs_p (comp)
- || (gimple_code (use->stmt) != GIMPLE_PHI
- /* We can't allow re-allocating the stmt as it might be pointed
- to still. */
- && (get_gimple_rhs_num_ops (TREE_CODE (comp))
- >= gimple_num_ops (gsi_stmt (bsi)))))
+ comp = force_gimple_operand (comp, &seq, false, NULL);
+ gimple_seq_add_seq (&stmt_list, seq);
+ if (gimple_code (use->stmt) != GIMPLE_PHI
+ /* We can't allow re-allocating the stmt as it might be pointed
+ to still. */
+ && (get_gimple_rhs_num_ops (TREE_CODE (comp))
+ >= gimple_num_ops (gsi_stmt (bsi))))
{
comp = force_gimple_operand (comp, &seq, true, NULL);
gimple_seq_add_seq (&stmt_list, seq);
case IFN_MASK_STORE:
case IFN_MASK_LOAD_LANES:
case IFN_MASK_STORE_LANES:
+ case IFN_LEN_LOAD:
+ case IFN_LEN_STORE:
/* The second argument contains the correct alias type. */
gcc_assert (use->op_p = gimple_call_arg_ptr (call, 0));
return TREE_TYPE (gimple_call_arg (call, 1));
count++;
if (count > 1)
- BREAK_FROM_IMM_USE_STMT (imm_iter);
+ break;
}
if (!count)
if (!flag_branch_on_count_reg)
return;
+ if (data->current_loop->unroll == USHRT_MAX)
+ return;
+
if (!generic_predict_doloop_p (data))
return;
void
tree_ssa_iv_optimize (void)
{
- class loop *loop;
struct ivopts_data data;
auto_bitmap toremove;
tree_ssa_iv_optimize_init (&data);
/* Optimize the loops starting with the innermost ones. */
- FOR_EACH_LOOP (loop, LI_FROM_INNERMOST)
+ for (auto loop : loops_list (cfun, LI_FROM_INNERMOST))
{
if (!dbg_cnt (ivopts_loop))
continue;
projects / thirdparty / gcc.git / blobdiff