[thirdparty/gcc.git] / gcc / gimple-isel.cc

/* Schedule GIMPLE vector statements.
   Copyright (C) 2020-2022 Free Software Foundation, Inc.

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 3, or (at your option) any
later version.

GCC is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.  */

#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "rtl.h"
#include "tree.h"
#include "gimple.h"
#include "tree-pass.h"
#include "ssa.h"
#include "expmed.h"
#include "optabs-tree.h"
#include "tree-eh.h"
#include "gimple-iterator.h"
#include "gimplify-me.h"
#include "gimplify.h"
#include "tree-cfg.h"
#include "bitmap.h"
#include "tree-ssa-dce.h"
#include "memmodel.h"
#include "optabs.h"
#include "gimple-fold.h"
#include "internal-fn.h"

/* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to
   internal function based on vector type of selected expansion.
   i.e.:
     VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D);
   =>
     _7 = u;
     _8 = .VEC_SET (_7, i_4(D), _1);
     u = _8;  */

static bool
gimple_expand_vec_set_expr (struct function *fun, gimple_stmt_iterator *gsi)
{
  enum tree_code code;
  gcall *new_stmt = NULL;
  gassign *ass_stmt = NULL;
  bool cfg_changed = false;

  /* Only consider code == GIMPLE_ASSIGN.  */
  gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
  if (!stmt)
    return false;

  tree lhs = gimple_assign_lhs (stmt);
  code = TREE_CODE (lhs);
  if (code != ARRAY_REF)
    return false;

  tree val = gimple_assign_rhs1 (stmt);
  tree op0 = TREE_OPERAND (lhs, 0);
  if (TREE_CODE (op0) == VIEW_CONVERT_EXPR && DECL_P (TREE_OPERAND (op0, 0))
      && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0)))
      && TYPE_MODE (TREE_TYPE (lhs))
	   == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0)))))
    {
      tree pos = TREE_OPERAND (lhs, 1);
      tree view_op0 = TREE_OPERAND (op0, 0);
      machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0));
      if (auto_var_in_fn_p (view_op0, fun->decl)
	  && !TREE_ADDRESSABLE (view_op0) && can_vec_set_var_idx_p (outermode))
	{
	  location_t loc = gimple_location (stmt);
	  tree var_src = make_ssa_name (TREE_TYPE (view_op0));
	  tree var_dst = make_ssa_name (TREE_TYPE (view_op0));

	  ass_stmt = gimple_build_assign (var_src, view_op0);
	  gimple_set_vuse (ass_stmt, gimple_vuse (stmt));
	  gimple_set_location (ass_stmt, loc);
	  gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);

	  new_stmt
	    = gimple_build_call_internal (IFN_VEC_SET, 3, var_src, val, pos);
	  gimple_call_set_lhs (new_stmt, var_dst);
	  gimple_set_location (new_stmt, loc);
	  gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);

	  ass_stmt = gimple_build_assign (view_op0, var_dst);
	  gimple_set_location (ass_stmt, loc);
	  gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);

	  basic_block bb = gimple_bb (stmt);
	  gimple_move_vops (ass_stmt, stmt);
	  if (gsi_remove (gsi, true)
	      && gimple_purge_dead_eh_edges (bb))
	    cfg_changed = true;
	  *gsi = gsi_for_stmt (ass_stmt);
	}
    }

  return cfg_changed;
}

/* Expand all VEC_COND_EXPR gimple assignments into calls to internal
   function based on type of selected expansion.  */

static gimple *
gimple_expand_vec_cond_expr (struct function *fun, gimple_stmt_iterator *gsi,
			     hash_map<tree, unsigned int> *vec_cond_ssa_name_uses)
{
  tree lhs, op0a = NULL_TREE, op0b = NULL_TREE;
  enum tree_code code;
  enum tree_code tcode;
  machine_mode cmp_op_mode;
  bool unsignedp;
  enum insn_code icode;
  imm_use_iterator imm_iter;

  /* Only consider code == GIMPLE_ASSIGN.  */
  gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
  if (!stmt)
    return NULL;

  code = gimple_assign_rhs_code (stmt);
  if (code != VEC_COND_EXPR)
    return NULL;

  tree op0 = gimple_assign_rhs1 (stmt);
  tree op1 = gimple_assign_rhs2 (stmt);
  tree op2 = gimple_assign_rhs3 (stmt);
  lhs = gimple_assign_lhs (stmt);
  machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));

  /* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations.
     Those can end up generated by folding and at least for integer mode masks
     we cannot expect vcond expanders to exist.  We lower a ? b : c
     to (b & a) | (c & ~a).  */
  if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs))
      && !VECTOR_MODE_P (mode))
    {
      gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1)));
      gimple_seq stmts = NULL;
      tree type = TREE_TYPE (lhs);
      location_t loc = gimple_location (stmt);
      tree tem0 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op1, op0);
      tree tem1 = gimple_build (&stmts, loc, BIT_NOT_EXPR, type, op0);
      tree tem2 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op2, tem1);
      tree tem3 = gimple_build (&stmts, loc, BIT_IOR_EXPR, type, tem0, tem2);
      gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
      return gimple_build_assign (lhs, tem3);
    }

  bool can_compute_op0 = true;
  gcc_assert (!COMPARISON_CLASS_P (op0));
  if (TREE_CODE (op0) == SSA_NAME)
    {
      unsigned int used_vec_cond_exprs = 0;
      unsigned int *slot = vec_cond_ssa_name_uses->get (op0);
      if (slot)
	used_vec_cond_exprs = *slot;
      else
	{
	  gimple *use_stmt;
	  FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, op0)
	    {
	      gassign *assign = dyn_cast<gassign *> (use_stmt);
	      if (assign != NULL
		  && gimple_assign_rhs_code (assign) == VEC_COND_EXPR
		  && gimple_assign_rhs1 (assign) == op0)
		used_vec_cond_exprs++;
	    }
	  vec_cond_ssa_name_uses->put (op0, used_vec_cond_exprs);
	}

      gassign *def_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op0));
      if (def_stmt)
	{
	  tcode = gimple_assign_rhs_code (def_stmt);
	  op0a = gimple_assign_rhs1 (def_stmt);
	  op0b = gimple_assign_rhs2 (def_stmt);

	  tree op0_type = TREE_TYPE (op0);
	  tree op0a_type = TREE_TYPE (op0a);
	  if (TREE_CODE_CLASS (tcode) == tcc_comparison)
	    can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type,
						     tcode);

	  /* Try to fold x CMP y ? -1 : 0 to x CMP y.  */
	  if (can_compute_op0
	      && integer_minus_onep (op1)
	      && integer_zerop (op2)
	      && TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0)))
	    {
	      tree conv_op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), op0);
	      gassign *new_stmt = gimple_build_assign (lhs, conv_op);
	      gsi_replace (gsi, new_stmt, true);
	      return new_stmt;
	    }

	  /* When the compare has EH we do not want to forward it when
	     it has multiple uses and in general because of the complication
	     with EH redirection.  */
	  if (stmt_can_throw_internal (fun, def_stmt))
	    tcode = TREE_CODE (op0);

	  /* If we can compute op0 and have multiple uses, keep the SSA
	     name and use vcond_mask.  */
	  else if (can_compute_op0
		   && used_vec_cond_exprs >= 2
		   && (get_vcond_mask_icode (mode, TYPE_MODE (op0_type))
		       != CODE_FOR_nothing))
	    tcode = TREE_CODE (op0);
	}
      else
	tcode = TREE_CODE (op0);
    }
  else
    tcode = TREE_CODE (op0);

  if (TREE_CODE_CLASS (tcode) != tcc_comparison)
    {
      gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
      if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
	  != CODE_FOR_nothing)
	return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
      /* Fake op0 < 0.  */
      else
	{
	  gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
		      == MODE_VECTOR_INT);
	  op0a = op0;
	  op0b = build_zero_cst (TREE_TYPE (op0));
	  tcode = LT_EXPR;
	}
    }
  cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
  unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));

  gcc_assert (known_eq (GET_MODE_NUNITS (mode),
			GET_MODE_NUNITS (cmp_op_mode)));

  icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
  /* Some targets do not have vcondeq and only vcond with NE/EQ
     but not vcondu, so make sure to also try vcond here as
     vcond_icode_p would canonicalize the optab query to.  */
  if (icode == CODE_FOR_nothing
      && (tcode == NE_EXPR || tcode == EQ_EXPR)
      && ((icode = get_vcond_icode (mode, cmp_op_mode, !unsignedp))
	  != CODE_FOR_nothing))
    unsignedp = !unsignedp;
  if (icode == CODE_FOR_nothing)
    {
      if (tcode == LT_EXPR
	  && op0a == op0)
	{
	  /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
	     into a constant when only get_vcond_eq_icode is supported.
	     Try changing it to NE_EXPR.  */
	  tcode = NE_EXPR;
	}
      if ((tcode == EQ_EXPR || tcode == NE_EXPR)
	  && direct_internal_fn_supported_p (IFN_VCONDEQ, TREE_TYPE (lhs),
					     TREE_TYPE (op0a),
					     OPTIMIZE_FOR_BOTH))
	{
	  tree tcode_tree = build_int_cst (integer_type_node, tcode);
	  return gimple_build_call_internal (IFN_VCONDEQ, 5, op0a, op0b, op1,
					     op2, tcode_tree);
	}

      gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))
		  && can_compute_op0
		  && (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
		      != CODE_FOR_nothing));
      return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
    }

  tree tcode_tree = build_int_cst (integer_type_node, tcode);
  return gimple_build_call_internal (unsignedp ? IFN_VCONDU : IFN_VCOND,
				     5, op0a, op0b, op1, op2, tcode_tree);
}


/* Iterate all gimple statements and try to expand
   VEC_COND_EXPR assignments.  */

static unsigned int
gimple_expand_vec_exprs (struct function *fun)
{
  gimple_stmt_iterator gsi;
  basic_block bb;
  hash_map<tree, unsigned int> vec_cond_ssa_name_uses;
  auto_bitmap dce_ssa_names;
  bool cfg_changed = false;

  FOR_EACH_BB_FN (bb, fun)
    {
      for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
	{
	  gimple *g = gimple_expand_vec_cond_expr (fun, &gsi,
						   &vec_cond_ssa_name_uses);
	  if (g != NULL)
	    {
	      tree lhs = gimple_assign_lhs (gsi_stmt (gsi));
	      gimple_set_lhs (g, lhs);
	      gsi_replace (&gsi, g, false);
	    }

	  cfg_changed |= gimple_expand_vec_set_expr (fun, &gsi);
	  if (gsi_end_p (gsi))
	    break;
	}
    }

  for (hash_map<tree, unsigned int>::iterator it = vec_cond_ssa_name_uses.begin ();
       it != vec_cond_ssa_name_uses.end (); ++it)
    bitmap_set_bit (dce_ssa_names, SSA_NAME_VERSION ((*it).first));

  simple_dce_from_worklist (dce_ssa_names);

  return cfg_changed ? TODO_cleanup_cfg : 0;
}

namespace {

const pass_data pass_data_gimple_isel =
{
  GIMPLE_PASS, /* type */
  "isel", /* name */
  OPTGROUP_VEC, /* optinfo_flags */
  TV_NONE, /* tv_id */
  PROP_cfg, /* properties_required */
  0, /* properties_provided */
  0, /* properties_destroyed */
  0, /* todo_flags_start */
  TODO_update_ssa, /* todo_flags_finish */
};

class pass_gimple_isel : public gimple_opt_pass
{
public:
  pass_gimple_isel (gcc::context *ctxt)
    : gimple_opt_pass (pass_data_gimple_isel, ctxt)
  {}

  /* opt_pass methods: */
  bool gate (function *) final override
    {
      return true;
    }

  unsigned int execute (function *fun) final override
    {
      return gimple_expand_vec_exprs (fun);
    }

}; // class pass_gimple_isel

} // anon namespace

gimple_opt_pass *
make_pass_gimple_isel (gcc::context *ctxt)
{
  return new pass_gimple_isel (ctxt);
}
Commit	Line	Data
502d63b6	1	/* Schedule GIMPLE vector statements.
7adcbafe	2	Copyright (C) 2020-2022 Free Software Foundation, Inc.
502d63b6 ML	3
	4	This file is part of GCC.
	5
	6	GCC is free software; you can redistribute it and/or modify it
	7	under the terms of the GNU General Public License as published by the
	8	Free Software Foundation; either version 3, or (at your option) any
	9	later version.
	10
	11	GCC is distributed in the hope that it will be useful, but WITHOUT
	12	ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	13	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	14	for more details.
	15
	16	You should have received a copy of the GNU General Public License
	17	along with GCC; see the file COPYING3. If not see
	18	<http://www.gnu.org/licenses/>. */
	19
	20	#include "config.h"
	21	#include "system.h"
	22	#include "coretypes.h"
	23	#include "backend.h"
	24	#include "rtl.h"
	25	#include "tree.h"
	26	#include "gimple.h"
	27	#include "tree-pass.h"
	28	#include "ssa.h"
	29	#include "expmed.h"
	30	#include "optabs-tree.h"
	31	#include "tree-eh.h"
	32	#include "gimple-iterator.h"
	33	#include "gimplify-me.h"
	34	#include "gimplify.h"
	35	#include "tree-cfg.h"
a8d8caca ML	36	#include "bitmap.h"
a8d8caca ML	37	#include "tree-ssa-dce.h"
683e55fa XL	38	#include "memmodel.h"
683e55fa XL	39	#include "optabs.h"
fddc7f00	40	#include "gimple-fold.h"
298e76e6	41	#include "internal-fn.h"
683e55fa XL	42
	43	/* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to
	44	internal function based on vector type of selected expansion.
	45	i.e.:
ddd46293	46	VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D);
683e55fa XL	47	=>
	48	_7 = u;
	49	_8 = .VEC_SET (_7, i_4(D), _1);
	50	u = _8; */
	51
e1a41143	52	static bool
871afdc5	53	gimple_expand_vec_set_expr (struct function fun, gimple_stmt_iterator gsi)
683e55fa XL	54	{
	55	enum tree_code code;
	56	gcall *new_stmt = NULL;
	57	gassign *ass_stmt = NULL;
e1a41143	58	bool cfg_changed = false;
683e55fa XL	59
	60	/* Only consider code == GIMPLE_ASSIGN. */
	61	gassign stmt = dyn_cast<gassign > (gsi_stmt (*gsi));
	62	if (!stmt)
e1a41143	63	return false;
683e55fa XL	64
	65	tree lhs = gimple_assign_lhs (stmt);
	66	code = TREE_CODE (lhs);
	67	if (code != ARRAY_REF)
e1a41143	68	return false;
683e55fa XL	69
	70	tree val = gimple_assign_rhs1 (stmt);
	71	tree op0 = TREE_OPERAND (lhs, 0);
	72	if (TREE_CODE (op0) == VIEW_CONVERT_EXPR && DECL_P (TREE_OPERAND (op0, 0))
	73	&& VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0)))
	74	&& TYPE_MODE (TREE_TYPE (lhs))
	75	== TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0)))))
	76	{
	77	tree pos = TREE_OPERAND (lhs, 1);
	78	tree view_op0 = TREE_OPERAND (op0, 0);
	79	machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0));
871afdc5	80	if (auto_var_in_fn_p (view_op0, fun->decl)
683e55fa XL	81	&& !TREE_ADDRESSABLE (view_op0) && can_vec_set_var_idx_p (outermode))
	82	{
	83	location_t loc = gimple_location (stmt);
	84	tree var_src = make_ssa_name (TREE_TYPE (view_op0));
	85	tree var_dst = make_ssa_name (TREE_TYPE (view_op0));
	86
	87	ass_stmt = gimple_build_assign (var_src, view_op0);
	88	gimple_set_vuse (ass_stmt, gimple_vuse (stmt));
	89	gimple_set_location (ass_stmt, loc);
	90	gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
	91
	92	new_stmt
	93	= gimple_build_call_internal (IFN_VEC_SET, 3, var_src, val, pos);
	94	gimple_call_set_lhs (new_stmt, var_dst);
	95	gimple_set_location (new_stmt, loc);
	96	gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
	97
	98	ass_stmt = gimple_build_assign (view_op0, var_dst);
	99	gimple_set_location (ass_stmt, loc);
	100	gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
	101
e1a41143	102	basic_block bb = gimple_bb (stmt);
683e55fa	103	gimple_move_vops (ass_stmt, stmt);
e1a41143 RB	104	if (gsi_remove (gsi, true)
	105	&& gimple_purge_dead_eh_edges (bb))
	106	cfg_changed = true;
ddd46293	107	*gsi = gsi_for_stmt (ass_stmt);
683e55fa XL	108	}
	109	}
	110
e1a41143	111	return cfg_changed;
683e55fa	112	}
502d63b6 ML	113
	114	/* Expand all VEC_COND_EXPR gimple assignments into calls to internal
	115	function based on type of selected expansion. */
	116
	117	static gimple *
871afdc5	118	gimple_expand_vec_cond_expr (struct function fun, gimple_stmt_iterator gsi,
502d63b6 ML	119	hash_map<tree, unsigned int> *vec_cond_ssa_name_uses)
	120	{
	121	tree lhs, op0a = NULL_TREE, op0b = NULL_TREE;
	122	enum tree_code code;
	123	enum tree_code tcode;
	124	machine_mode cmp_op_mode;
	125	bool unsignedp;
	126	enum insn_code icode;
	127	imm_use_iterator imm_iter;
	128
	129	/* Only consider code == GIMPLE_ASSIGN. */
	130	gassign stmt = dyn_cast<gassign > (gsi_stmt (*gsi));
	131	if (!stmt)
	132	return NULL;
	133
	134	code = gimple_assign_rhs_code (stmt);
	135	if (code != VEC_COND_EXPR)
	136	return NULL;
	137
	138	tree op0 = gimple_assign_rhs1 (stmt);
	139	tree op1 = gimple_assign_rhs2 (stmt);
	140	tree op2 = gimple_assign_rhs3 (stmt);
	141	lhs = gimple_assign_lhs (stmt);
	142	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
	143
fddc7f00 RB	144	/* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations.
	145	Those can end up generated by folding and at least for integer mode masks
	146	we cannot expect vcond expanders to exist. We lower a ? b : c
	147	to (b & a) \| (c & ~a). */
5c197b83 RB	148	if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs))
5c197b83 RB	149	&& !VECTOR_MODE_P (mode))
fddc7f00	150	{
5c197b83	151	gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1)));
fddc7f00 RB	152	gimple_seq stmts = NULL;
	153	tree type = TREE_TYPE (lhs);
	154	location_t loc = gimple_location (stmt);
	155	tree tem0 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op1, op0);
	156	tree tem1 = gimple_build (&stmts, loc, BIT_NOT_EXPR, type, op0);
	157	tree tem2 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op2, tem1);
	158	tree tem3 = gimple_build (&stmts, loc, BIT_IOR_EXPR, type, tem0, tem2);
	159	gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
	160	return gimple_build_assign (lhs, tem3);
	161	}
	162
78595e91	163	bool can_compute_op0 = true;
502d63b6 ML	164	gcc_assert (!COMPARISON_CLASS_P (op0));
	165	if (TREE_CODE (op0) == SSA_NAME)
	166	{
	167	unsigned int used_vec_cond_exprs = 0;
	168	unsigned int *slot = vec_cond_ssa_name_uses->get (op0);
	169	if (slot)
	170	used_vec_cond_exprs = *slot;
	171	else
	172	{
	173	gimple *use_stmt;
	174	FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, op0)
	175	{
	176	gassign assign = dyn_cast<gassign > (use_stmt);
	177	if (assign != NULL
	178	&& gimple_assign_rhs_code (assign) == VEC_COND_EXPR
	179	&& gimple_assign_rhs1 (assign) == op0)
	180	used_vec_cond_exprs++;
	181	}
	182	vec_cond_ssa_name_uses->put (op0, used_vec_cond_exprs);
	183	}
	184
	185	gassign def_stmt = dyn_cast<gassign > (SSA_NAME_DEF_STMT (op0));
8f8f8c68	186	if (def_stmt)
502d63b6 ML	187	{
	188	tcode = gimple_assign_rhs_code (def_stmt);
	189	op0a = gimple_assign_rhs1 (def_stmt);
	190	op0b = gimple_assign_rhs2 (def_stmt);
	191
d0d8a165	192	tree op0_type = TREE_TYPE (op0);
502d63b6	193	tree op0a_type = TREE_TYPE (op0a);
78595e91 RS	194	if (TREE_CODE_CLASS (tcode) == tcc_comparison)
	195	can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type,
	196	tcode);
3a6e3ad3 PK	197
3a6e3ad3 PK	198	/* Try to fold x CMP y ? -1 : 0 to x CMP y. */
78595e91 RS	199	if (can_compute_op0
78595e91 RS	200	&& integer_minus_onep (op1)
3a6e3ad3	201	&& integer_zerop (op2)
78595e91	202	&& TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0)))
3a6e3ad3 PK	203	{
	204	tree conv_op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), op0);
	205	gassign *new_stmt = gimple_build_assign (lhs, conv_op);
	206	gsi_replace (gsi, new_stmt, true);
	207	return new_stmt;
	208	}
	209
8f8f8c68 RB	210	/* When the compare has EH we do not want to forward it when
	211	it has multiple uses and in general because of the complication
	212	with EH redirection. */
	213	if (stmt_can_throw_internal (fun, def_stmt))
	214	tcode = TREE_CODE (op0);
	215
	216	/* If we can compute op0 and have multiple uses, keep the SSA
	217	name and use vcond_mask. */
	218	else if (can_compute_op0
	219	&& used_vec_cond_exprs >= 2
	220	&& (get_vcond_mask_icode (mode, TYPE_MODE (op0_type))
	221	!= CODE_FOR_nothing))
	222	tcode = TREE_CODE (op0);
502d63b6 ML	223	}
	224	else
	225	tcode = TREE_CODE (op0);
	226	}
	227	else
	228	tcode = TREE_CODE (op0);
	229
	230	if (TREE_CODE_CLASS (tcode) != tcc_comparison)
	231	{
	232	gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
	233	if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
	234	!= CODE_FOR_nothing)
	235	return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
	236	/* Fake op0 < 0. */
	237	else
	238	{
	239	gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
	240	== MODE_VECTOR_INT);
	241	op0a = op0;
	242	op0b = build_zero_cst (TREE_TYPE (op0));
	243	tcode = LT_EXPR;
	244	}
	245	}
	246	cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
	247	unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
	248
e29dd0eb RS	249	gcc_assert (known_eq (GET_MODE_NUNITS (mode),
e29dd0eb RS	250	GET_MODE_NUNITS (cmp_op_mode)));
502d63b6 ML	251
502d63b6 ML	252	icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
36f1de95 RB	253	/* Some targets do not have vcondeq and only vcond with NE/EQ
	254	but not vcondu, so make sure to also try vcond here as
	255	vcond_icode_p would canonicalize the optab query to. */
	256	if (icode == CODE_FOR_nothing
	257	&& (tcode == NE_EXPR \|\| tcode == EQ_EXPR)
	258	&& ((icode = get_vcond_icode (mode, cmp_op_mode, !unsignedp))
	259	!= CODE_FOR_nothing))
	260	unsignedp = !unsignedp;
502d63b6 ML	261	if (icode == CODE_FOR_nothing)
	262	{
	263	if (tcode == LT_EXPR
3457dae5	264	&& op0a == op0)
502d63b6 ML	265	{
	266	/* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
	267	into a constant when only get_vcond_eq_icode is supported.
3457dae5	268	Try changing it to NE_EXPR. */
502d63b6 ML	269	tcode = NE_EXPR;
502d63b6 ML	270	}
298e76e6 RS	271	if ((tcode == EQ_EXPR \|\| tcode == NE_EXPR)
	272	&& direct_internal_fn_supported_p (IFN_VCONDEQ, TREE_TYPE (lhs),
	273	TREE_TYPE (op0a),
	274	OPTIMIZE_FOR_BOTH))
502d63b6 ML	275	{
	276	tree tcode_tree = build_int_cst (integer_type_node, tcode);
	277	return gimple_build_call_internal (IFN_VCONDEQ, 5, op0a, op0b, op1,
	278	op2, tcode_tree);
	279	}
502d63b6	280
78595e91 RS	281	gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))
	282	&& can_compute_op0
	283	&& (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
	284	!= CODE_FOR_nothing));
	285	return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
	286	}
	287
502d63b6 ML	288	tree tcode_tree = build_int_cst (integer_type_node, tcode);
	289	return gimple_build_call_internal (unsignedp ? IFN_VCONDU : IFN_VCOND,
	290	5, op0a, op0b, op1, op2, tcode_tree);
	291	}
	292
	293
	294
	295	/* Iterate all gimple statements and try to expand
	296	VEC_COND_EXPR assignments. */
	297
	298	static unsigned int
871afdc5	299	gimple_expand_vec_exprs (struct function *fun)
502d63b6 ML	300	{
	301	gimple_stmt_iterator gsi;
	302	basic_block bb;
502d63b6	303	hash_map<tree, unsigned int> vec_cond_ssa_name_uses;
a8d8caca	304	auto_bitmap dce_ssa_names;
e1a41143	305	bool cfg_changed = false;
502d63b6	306
871afdc5	307	FOR_EACH_BB_FN (bb, fun)
502d63b6 ML	308	{
	309	for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
	310	{
871afdc5	311	gimple *g = gimple_expand_vec_cond_expr (fun, &gsi,
502d63b6 ML	312	&vec_cond_ssa_name_uses);
	313	if (g != NULL)
	314	{
	315	tree lhs = gimple_assign_lhs (gsi_stmt (gsi));
	316	gimple_set_lhs (g, lhs);
	317	gsi_replace (&gsi, g, false);
	318	}
683e55fa	319
e1a41143	320	cfg_changed \|= gimple_expand_vec_set_expr (fun, &gsi);
5877c544 ML	321	if (gsi_end_p (gsi))
5877c544 ML	322	break;
502d63b6 ML	323	}
	324	}
	325
a8d8caca ML	326	for (hash_map<tree, unsigned int>::iterator it = vec_cond_ssa_name_uses.begin ();
	327	it != vec_cond_ssa_name_uses.end (); ++it)
	328	bitmap_set_bit (dce_ssa_names, SSA_NAME_VERSION ((*it).first));
	329
	330	simple_dce_from_worklist (dce_ssa_names);
	331
e1a41143	332	return cfg_changed ? TODO_cleanup_cfg : 0;
502d63b6 ML	333	}
	334
	335	namespace {
	336
	337	const pass_data pass_data_gimple_isel =
	338	{
	339	GIMPLE_PASS, /* type */
	340	"isel", /* name */
	341	OPTGROUP_VEC, /* optinfo_flags */
	342	TV_NONE, /* tv_id */
	343	PROP_cfg, /* properties_required */
	344	0, /* properties_provided */
	345	0, /* properties_destroyed */
	346	0, /* todo_flags_start */
	347	TODO_update_ssa, /* todo_flags_finish */
	348	};
	349
	350	class pass_gimple_isel : public gimple_opt_pass
	351	{
	352	public:
	353	pass_gimple_isel (gcc::context *ctxt)
	354	: gimple_opt_pass (pass_data_gimple_isel, ctxt)
	355	{}
	356
	357	/* opt_pass methods: */
725793af	358	bool gate (function *) final override
502d63b6 ML	359	{
	360	return true;
	361	}
	362
725793af	363	unsigned int execute (function *fun) final override
502d63b6	364	{
871afdc5	365	return gimple_expand_vec_exprs (fun);
502d63b6 ML	366	}
	367
	368	}; // class pass_gimple_isel
	369
	370	} // anon namespace
	371
	372	gimple_opt_pass *
	373	make_pass_gimple_isel (gcc::context *ctxt)
	374	{
	375	return new pass_gimple_isel (ctxt);
	376	}
	377