#define mask_load_lanes_direct { -1, -1, false }
#define gather_load_direct { 3, 1, false }
#define len_load_direct { -1, -1, false }
-#define len_maskload_direct { -1, 3, false }
+#define len_maskload_direct { -1, 4, false }
#define mask_store_direct { 3, 2, false }
#define store_lanes_direct { 0, 0, false }
#define mask_store_lanes_direct { 0, 0, false }
#define vec_cond_direct { 2, 0, false }
#define scatter_store_direct { 3, 1, false }
#define len_store_direct { 3, 3, false }
-#define len_maskstore_direct { 4, 3, false }
+#define len_maskstore_direct { 4, 5, false }
#define vec_set_direct { 3, 3, false }
#define unary_direct { 0, 0, true }
#define unary_convert_direct { -1, 0, true }
return convert_optab_handler (optab, imode, vmode);
}
+/* Add len and mask arguments according to the STMT. */
+
+static unsigned int
+add_len_and_mask_args (expand_operand *ops, unsigned int opno, gcall *stmt)
+{
+ internal_fn ifn = gimple_call_internal_fn (stmt);
+ int len_index = internal_fn_len_index (ifn);
+ /* BIAS is always consecutive next of LEN. */
+ int bias_index = len_index + 1;
+ int mask_index = internal_fn_mask_index (ifn);
+ /* The order of arguments are always {len,bias,mask}. */
+ if (len_index >= 0)
+ {
+ tree len = gimple_call_arg (stmt, len_index);
+ rtx len_rtx = expand_normal (len);
+ create_convert_operand_from (&ops[opno++], len_rtx,
+ TYPE_MODE (TREE_TYPE (len)),
+ TYPE_UNSIGNED (TREE_TYPE (len)));
+ tree biast = gimple_call_arg (stmt, bias_index);
+ rtx bias = expand_normal (biast);
+ create_input_operand (&ops[opno++], bias, QImode);
+ }
+ if (mask_index >= 0)
+ {
+ tree mask = gimple_call_arg (stmt, mask_index);
+ rtx mask_rtx = expand_normal (mask);
+ create_input_operand (&ops[opno++], mask_rtx,
+ TYPE_MODE (TREE_TYPE (mask)));
+ }
+ return opno;
+}
+
/* Expand LOAD_LANES call STMT using optab OPTAB. */
static void
* OPTAB. */
static void
-expand_partial_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
+expand_partial_load_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
{
+ int i = 0;
class expand_operand ops[5];
- tree type, lhs, rhs, maskt, biast;
- rtx mem, target, mask, bias;
+ tree type, lhs, rhs, maskt;
+ rtx mem, target;
insn_code icode;
- maskt = gimple_call_arg (stmt, 2);
+ maskt = gimple_call_arg (stmt, internal_fn_mask_index (ifn));
lhs = gimple_call_lhs (stmt);
if (lhs == NULL_TREE)
return;
mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
gcc_assert (MEM_P (mem));
- mask = expand_normal (maskt);
target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
- create_output_operand (&ops[0], target, TYPE_MODE (type));
- create_fixed_operand (&ops[1], mem);
- if (optab == len_load_optab)
- {
- create_convert_operand_from (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)),
- TYPE_UNSIGNED (TREE_TYPE (maskt)));
- biast = gimple_call_arg (stmt, 3);
- bias = expand_normal (biast);
- create_input_operand (&ops[3], bias, QImode);
- expand_insn (icode, 4, ops);
- }
- else if (optab == len_maskload_optab)
- {
- create_convert_operand_from (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)),
- TYPE_UNSIGNED (TREE_TYPE (maskt)));
- maskt = gimple_call_arg (stmt, 3);
- mask = expand_normal (maskt);
- create_input_operand (&ops[3], mask, TYPE_MODE (TREE_TYPE (maskt)));
- icode = convert_optab_handler (optab, TYPE_MODE (type),
- TYPE_MODE (TREE_TYPE (maskt)));
- biast = gimple_call_arg (stmt, 4);
- bias = expand_normal (biast);
- create_input_operand (&ops[4], bias, QImode);
- expand_insn (icode, 5, ops);
- }
- else
- {
- create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
- expand_insn (icode, 3, ops);
- }
+ create_output_operand (&ops[i++], target, TYPE_MODE (type));
+ create_fixed_operand (&ops[i++], mem);
+ i = add_len_and_mask_args (ops, i, stmt);
+ expand_insn (icode, i, ops);
if (!rtx_equal_p (target, ops[0].value))
emit_move_insn (target, ops[0].value);
static void
expand_partial_store_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
{
+ int i = 0;
class expand_operand ops[5];
- tree type, lhs, rhs, maskt, biast;
- rtx mem, reg, mask, bias;
+ tree type, lhs, rhs, maskt;
+ rtx mem, reg;
insn_code icode;
- maskt = gimple_call_arg (stmt, 2);
+ maskt = gimple_call_arg (stmt, internal_fn_mask_index (ifn));
rhs = gimple_call_arg (stmt, internal_fn_stored_value_index (ifn));
type = TREE_TYPE (rhs);
lhs = expand_call_mem_ref (type, stmt, 0);
mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
gcc_assert (MEM_P (mem));
- mask = expand_normal (maskt);
reg = expand_normal (rhs);
- create_fixed_operand (&ops[0], mem);
- create_input_operand (&ops[1], reg, TYPE_MODE (type));
- if (optab == len_store_optab)
- {
- create_convert_operand_from (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)),
- TYPE_UNSIGNED (TREE_TYPE (maskt)));
- biast = gimple_call_arg (stmt, 4);
- bias = expand_normal (biast);
- create_input_operand (&ops[3], bias, QImode);
- expand_insn (icode, 4, ops);
- }
- else if (optab == len_maskstore_optab)
- {
- create_convert_operand_from (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)),
- TYPE_UNSIGNED (TREE_TYPE (maskt)));
- maskt = gimple_call_arg (stmt, 3);
- mask = expand_normal (maskt);
- create_input_operand (&ops[3], mask, TYPE_MODE (TREE_TYPE (maskt)));
- biast = gimple_call_arg (stmt, 5);
- bias = expand_normal (biast);
- create_input_operand (&ops[4], bias, QImode);
- icode = convert_optab_handler (optab, TYPE_MODE (type), GET_MODE (mask));
- expand_insn (icode, 5, ops);
- }
- else
- {
- create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
- expand_insn (icode, 3, ops);
- }
+ create_fixed_operand (&ops[i++], mem);
+ create_input_operand (&ops[i++], reg, TYPE_MODE (type));
+ i = add_len_and_mask_args (ops, i, stmt);
+ expand_insn (icode, i, ops);
}
#define expand_mask_store_optab_fn expand_partial_store_optab_fn
}
}
+/* If FN takes a vector len argument, return the index of that argument,
+ otherwise return -1. */
+
+int
+internal_fn_len_index (internal_fn fn)
+{
+ switch (fn)
+ {
+ case IFN_LEN_LOAD:
+ case IFN_LEN_STORE:
+ case IFN_LEN_MASK_LOAD:
+ case IFN_LEN_MASK_STORE:
+ return 2;
+
+ default:
+ return -1;
+ }
+}
+
/* If FN takes a vector mask argument, return the index of that argument,
otherwise return -1. */
case IFN_MASK_GATHER_LOAD:
case IFN_MASK_SCATTER_STORE:
- return 4;
-
case IFN_LEN_MASK_LOAD:
case IFN_LEN_MASK_STORE:
- return 3;
+ return 4;
default:
return (conditional_internal_fn_code (fn) != ERROR_MARK
case IFN_MASK_STORE_LANES:
case IFN_SCATTER_STORE:
case IFN_MASK_SCATTER_STORE:
- case IFN_LEN_STORE:
return 3;
- case IFN_LEN_MASK_STORE:
+ case IFN_LEN_STORE:
return 4;
+ case IFN_LEN_MASK_STORE:
+ return 5;
+
default:
return -1;
}
{
optab optab = direct_internal_fn_optab (ifn);
insn_code icode = direct_optab_handler (optab, mode);
- int bias_opno = 3;
if (icode == CODE_FOR_nothing)
{
optab = direct_internal_fn_optab (IFN_LEN_MASK_STORE);
}
icode = convert_optab_handler (optab, mode, mask_mode);
- bias_opno = 4;
}
if (icode != CODE_FOR_nothing)
{
/* For now we only support biases of 0 or -1. Try both of them. */
- if (insn_operand_matches (icode, bias_opno, GEN_INT (0)))
+ if (insn_operand_matches (icode, 3, GEN_INT (0)))
return 0;
- if (insn_operand_matches (icode, bias_opno, GEN_INT (-1)))
+ if (insn_operand_matches (icode, 3, GEN_INT (-1)))
return -1;
}
projects / thirdparty / gcc.git / commitdiff
