[thirdparty/gcc.git] / gcc / gimple-isel.cc

/* Schedule GIMPLE vector statements.
   Copyright (C) 2020-2024 Free Software Foundation, Inc.

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 3, or (at your option) any
later version.

GCC is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.  */

#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "rtl.h"
#include "tree.h"
#include "gimple.h"
#include "tree-pass.h"
#include "ssa.h"
#include "expmed.h"
#include "optabs-tree.h"
#include "tree-eh.h"
#include "gimple-iterator.h"
#include "gimplify-me.h"
#include "gimplify.h"
#include "tree-cfg.h"
#include "bitmap.h"
#include "tree-ssa-dce.h"
#include "memmodel.h"
#include "optabs.h"
#include "gimple-fold.h"
#include "internal-fn.h"

/* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to
   internal function based on vector type of selected expansion.

   For vec_set:

     VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D);
   =>
     _7 = u;
     _8 = .VEC_SET (_7, i_4(D), _1);
     u = _8;

   For vec_extract:

      _3 = VIEW_CONVERT_EXPR<intD.1[4]>(vD.2208)[idx_2(D)];
   =>
      _4 = vD.2208;
      _3 = .VEC_EXTRACT (_4, idx_2(D));  */

static bool
gimple_expand_vec_set_extract_expr (struct function *fun,
				    gimple_stmt_iterator *gsi)
{
  gcall *new_stmt = NULL;
  gassign *ass_stmt = NULL;
  bool cfg_changed = false;

  /* Only consider code == GIMPLE_ASSIGN.  */
  gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
  if (!stmt)
    return false;

  bool is_extract = false;

  tree lhs = gimple_assign_lhs (stmt);
  tree rhs = gimple_assign_rhs1 (stmt);
  tree val, ref;
  if (TREE_CODE (lhs) == ARRAY_REF)
    {
      /* Assume it is a vec_set.  */
      val = rhs;
      ref = lhs;
    }
  else if (TREE_CODE (rhs) == ARRAY_REF)
    {
      /* vec_extract.  */
      is_extract = true;
      val = lhs;
      ref = rhs;
    }
  else
    return false;

  tree op0 = TREE_OPERAND (ref, 0);
  if (TREE_CODE (op0) == VIEW_CONVERT_EXPR && DECL_P (TREE_OPERAND (op0, 0))
      && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0)))
      && TYPE_MODE (TREE_TYPE (ref))
	   == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0)))))
    {
      tree pos = TREE_OPERAND (ref, 1);

      tree view_op0 = TREE_OPERAND (op0, 0);
      machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0));
      machine_mode extract_mode = TYPE_MODE (TREE_TYPE (ref));

      if ((auto_var_in_fn_p (view_op0, fun->decl)
	   || (VAR_P (view_op0) && DECL_HARD_REGISTER (view_op0)))
	  && !TREE_ADDRESSABLE (view_op0)
	  && ((!is_extract && can_vec_set_var_idx_p (outermode))
	      || (is_extract
		  && can_vec_extract_var_idx_p (outermode, extract_mode))))
	{
	  location_t loc = gimple_location (stmt);
	  tree var_src = make_ssa_name (TREE_TYPE (view_op0));

	  ass_stmt = gimple_build_assign (var_src, view_op0);
	  gimple_set_vuse (ass_stmt, gimple_vuse (stmt));
	  gimple_set_location (ass_stmt, loc);
	  gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);

	  if (!is_extract)
	    {
	      tree var_dst = make_ssa_name (TREE_TYPE (view_op0));

	      new_stmt = gimple_build_call_internal (IFN_VEC_SET, 3, var_src,
						     val, pos);

	      gimple_call_set_lhs (new_stmt, var_dst);
	      gimple_set_location (new_stmt, loc);
	      gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);

	      ass_stmt = gimple_build_assign (view_op0, var_dst);
	      gimple_set_location (ass_stmt, loc);
	      gimple_move_vops (ass_stmt, stmt);
	      gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);

	      basic_block bb = gimple_bb (stmt);
	      if (gsi_remove (gsi, true)
		  && gimple_purge_dead_eh_edges (bb))
		cfg_changed = true;
	      *gsi = gsi_for_stmt (ass_stmt);
	    }
	  else
	    {
	      new_stmt
		= gimple_build_call_internal (IFN_VEC_EXTRACT, 2, var_src, pos);
	      gimple_call_set_lhs (new_stmt, lhs);

	      gsi_replace (gsi, new_stmt, true);
	      cfg_changed = true;
	    }
	}
    }

  return cfg_changed;
}

/* Expand all VEC_COND_EXPR gimple assignments into calls to internal
   function based on type of selected expansion.  */

static gimple *
gimple_expand_vec_cond_expr (struct function *fun, gimple_stmt_iterator *gsi,
			     hash_map<tree, unsigned int> *vec_cond_ssa_name_uses)
{
  tree lhs, op0a = NULL_TREE, op0b = NULL_TREE;
  enum tree_code code;
  enum tree_code tcode;
  machine_mode cmp_op_mode;
  bool unsignedp;
  enum insn_code icode;
  imm_use_iterator imm_iter;

  /* Only consider code == GIMPLE_ASSIGN.  */
  gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
  if (!stmt)
    return NULL;

  code = gimple_assign_rhs_code (stmt);
  if (code != VEC_COND_EXPR)
    return NULL;

  tree op0 = gimple_assign_rhs1 (stmt);
  tree op1 = gimple_assign_rhs2 (stmt);
  tree op2 = gimple_assign_rhs3 (stmt);
  lhs = gimple_assign_lhs (stmt);
  machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));

  /* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations.
     Those can end up generated by folding and at least for integer mode masks
     we cannot expect vcond expanders to exist.  We lower a ? b : c
     to (b & a) | (c & ~a).  */
  if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs))
      && !VECTOR_MODE_P (mode))
    {
      gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1)));
      gimple_seq stmts = NULL;
      tree type = TREE_TYPE (lhs);
      location_t loc = gimple_location (stmt);
      tree tem0 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op1, op0);
      tree tem1 = gimple_build (&stmts, loc, BIT_NOT_EXPR, type, op0);
      tree tem2 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op2, tem1);
      tree tem3 = gimple_build (&stmts, loc, BIT_IOR_EXPR, type, tem0, tem2);
      gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
      return gimple_build_assign (lhs, tem3);
    }

  bool can_compute_op0 = true;
  gcc_assert (!COMPARISON_CLASS_P (op0));
  if (TREE_CODE (op0) == SSA_NAME)
    {
      unsigned int used_vec_cond_exprs = 0;
      unsigned int *slot = vec_cond_ssa_name_uses->get (op0);
      if (slot)
	used_vec_cond_exprs = *slot;
      else
	{
	  gimple *use_stmt;
	  FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, op0)
	    {
	      gassign *assign = dyn_cast<gassign *> (use_stmt);
	      if (assign != NULL
		  && gimple_assign_rhs_code (assign) == VEC_COND_EXPR
		  && gimple_assign_rhs1 (assign) == op0)
		used_vec_cond_exprs++;
	    }
	  vec_cond_ssa_name_uses->put (op0, used_vec_cond_exprs);
	}

      gassign *def_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op0));
      if (def_stmt)
	{
	  tcode = gimple_assign_rhs_code (def_stmt);
	  op0a = gimple_assign_rhs1 (def_stmt);
	  op0b = gimple_assign_rhs2 (def_stmt);

	  tree op0_type = TREE_TYPE (op0);
	  tree op0a_type = TREE_TYPE (op0a);
	  if (TREE_CODE_CLASS (tcode) == tcc_comparison)
	    can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type,
						     tcode);

	  /* Try to fold x CMP y ? -1 : 0 to x CMP y.  */
	  if (can_compute_op0
	      && integer_minus_onep (op1)
	      && integer_zerop (op2)
	      && TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0)))
	    {
	      tree conv_op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), op0);
	      gassign *new_stmt = gimple_build_assign (lhs, conv_op);
	      gsi_replace (gsi, new_stmt, true);
	      return new_stmt;
	    }

	  /* When the compare has EH we do not want to forward it when
	     it has multiple uses and in general because of the complication
	     with EH redirection.  */
	  if (stmt_can_throw_internal (fun, def_stmt))
	    tcode = TREE_CODE (op0);

	  /* If we can compute op0 and have multiple uses, keep the SSA
	     name and use vcond_mask.  */
	  else if (can_compute_op0
		   && used_vec_cond_exprs >= 2
		   && (get_vcond_mask_icode (mode, TYPE_MODE (op0_type))
		       != CODE_FOR_nothing))
	    tcode = TREE_CODE (op0);
	}
      else
	tcode = TREE_CODE (op0);
    }
  else
    tcode = TREE_CODE (op0);

  if (TREE_CODE_CLASS (tcode) != tcc_comparison)
    {
      gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
      if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
	  != CODE_FOR_nothing)
	return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
      /* Fake op0 < 0.  */
      else
	{
	  gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
		      == MODE_VECTOR_INT);
	  op0a = op0;
	  op0b = build_zero_cst (TREE_TYPE (op0));
	  tcode = LT_EXPR;
	}
    }
  cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
  unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));

  gcc_assert (known_eq (GET_MODE_NUNITS (mode),
			GET_MODE_NUNITS (cmp_op_mode)));

  icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
  /* Some targets do not have vcondeq and only vcond with NE/EQ
     but not vcondu, so make sure to also try vcond here as
     vcond_icode_p would canonicalize the optab query to.  */
  if (icode == CODE_FOR_nothing
      && (tcode == NE_EXPR || tcode == EQ_EXPR)
      && ((icode = get_vcond_icode (mode, cmp_op_mode, !unsignedp))
	  != CODE_FOR_nothing))
    unsignedp = !unsignedp;
  if (icode == CODE_FOR_nothing)
    {
      if (tcode == LT_EXPR
	  && op0a == op0)
	{
	  /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
	     into a constant when only get_vcond_eq_icode is supported.
	     Try changing it to NE_EXPR.  */
	  tcode = NE_EXPR;
	}
      if ((tcode == EQ_EXPR || tcode == NE_EXPR)
	  && direct_internal_fn_supported_p (IFN_VCONDEQ, TREE_TYPE (lhs),
					     TREE_TYPE (op0a),
					     OPTIMIZE_FOR_BOTH))
	{
	  tree tcode_tree = build_int_cst (integer_type_node, tcode);
	  return gimple_build_call_internal (IFN_VCONDEQ, 5, op0a, op0b, op1,
					     op2, tcode_tree);
	}

      gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))
		  && can_compute_op0
		  && (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
		      != CODE_FOR_nothing));
      return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
    }

  tree tcode_tree = build_int_cst (integer_type_node, tcode);
  return gimple_build_call_internal (unsignedp ? IFN_VCONDU : IFN_VCOND,
				     5, op0a, op0b, op1, op2, tcode_tree);
}


namespace {

const pass_data pass_data_gimple_isel =
{
  GIMPLE_PASS, /* type */
  "isel", /* name */
  OPTGROUP_VEC, /* optinfo_flags */
  TV_NONE, /* tv_id */
  PROP_cfg, /* properties_required */
  0, /* properties_provided */
  0, /* properties_destroyed */
  0, /* todo_flags_start */
  TODO_update_ssa, /* todo_flags_finish */
};

class pass_gimple_isel : public gimple_opt_pass
{
public:
  pass_gimple_isel (gcc::context *ctxt)
    : gimple_opt_pass (pass_data_gimple_isel, ctxt)
  {}

  /* opt_pass methods: */
  bool gate (function *) final override
    {
      return true;
    }

  unsigned int execute (function *fun) final override;
}; // class pass_gimple_isel


/* Iterate all gimple statements and perform pre RTL expansion
   GIMPLE massaging to improve instruction selection.  */

unsigned int
pass_gimple_isel::execute (struct function *fun)
{
  gimple_stmt_iterator gsi;
  basic_block bb;
  hash_map<tree, unsigned int> vec_cond_ssa_name_uses;
  auto_bitmap dce_ssa_names;
  bool cfg_changed = false;

  FOR_EACH_BB_FN (bb, fun)
    {
      for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
	{
	  /* Pre-expand VEC_COND_EXPRs to .VCOND* internal function
	     calls mapping to supported optabs.  */
	  gimple *g = gimple_expand_vec_cond_expr (fun, &gsi,
						   &vec_cond_ssa_name_uses);
	  if (g != NULL)
	    {
	      tree lhs = gimple_assign_lhs (gsi_stmt (gsi));
	      gimple_set_lhs (g, lhs);
	      gsi_replace (&gsi, g, false);
	    }

	  /* Recognize .VEC_SET and .VEC_EXTRACT patterns.  */
	  cfg_changed |= gimple_expand_vec_set_extract_expr (fun, &gsi);
	  if (gsi_end_p (gsi))
	    break;

	  gassign *stmt = dyn_cast <gassign *> (*gsi);
	  if (!stmt)
	    continue;

	  tree_code code = gimple_assign_rhs_code (stmt);
	  tree lhs = gimple_assign_lhs (stmt);
	  if (TREE_CODE_CLASS (code) == tcc_comparison
	      && !has_single_use (lhs))
	    {
	      /* Duplicate COND_EXPR condition defs when they are
		 comparisons so RTL expansion with the help of TER
		 can perform better if conversion.  */
	      imm_use_iterator imm_iter;
	      use_operand_p use_p;
	      auto_vec<gassign *, 4> cond_exprs;
	      unsigned cnt = 0;
	      FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
		{
		  if (is_gimple_debug (USE_STMT (use_p)))
		    continue;
		  cnt++;
		  if (gimple_bb (USE_STMT (use_p)) == bb
		      && is_gimple_assign (USE_STMT (use_p))
		      && gimple_assign_rhs1_ptr (USE_STMT (use_p)) == use_p->use
		      && gimple_assign_rhs_code (USE_STMT (use_p)) == COND_EXPR)
		    cond_exprs.safe_push (as_a <gassign *> (USE_STMT (use_p)));
		}
	      for (unsigned i = cond_exprs.length () == cnt ? 1 : 0;
		   i < cond_exprs.length (); ++i)
		{
		  gassign *copy = as_a <gassign *> (gimple_copy (stmt));
		  tree new_def = duplicate_ssa_name (lhs, copy);
		  gimple_assign_set_lhs (copy, new_def);
		  auto gsi2 = gsi_for_stmt (cond_exprs[i]);
		  gsi_insert_before (&gsi2, copy, GSI_SAME_STMT);
		  gimple_assign_set_rhs1 (cond_exprs[i], new_def);
		  update_stmt (cond_exprs[i]);
		}
	    }
	}
    }

  for (auto it = vec_cond_ssa_name_uses.begin ();
       it != vec_cond_ssa_name_uses.end (); ++it)
    bitmap_set_bit (dce_ssa_names, SSA_NAME_VERSION ((*it).first));

  simple_dce_from_worklist (dce_ssa_names);

  return cfg_changed ? TODO_cleanup_cfg : 0;
}

} // anon namespace

gimple_opt_pass *
make_pass_gimple_isel (gcc::context *ctxt)
{
  return new pass_gimple_isel (ctxt);
}
Commit	Line	Data
502d63b6	1	/* Schedule GIMPLE vector statements.
a945c346	2	Copyright (C) 2020-2024 Free Software Foundation, Inc.
502d63b6 ML	3
	4	This file is part of GCC.
	5
	6	GCC is free software; you can redistribute it and/or modify it
	7	under the terms of the GNU General Public License as published by the
	8	Free Software Foundation; either version 3, or (at your option) any
	9	later version.
	10
	11	GCC is distributed in the hope that it will be useful, but WITHOUT
	12	ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	13	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	14	for more details.
	15
	16	You should have received a copy of the GNU General Public License
	17	along with GCC; see the file COPYING3. If not see
	18	<http://www.gnu.org/licenses/>. */
	19
	20	#include "config.h"
	21	#include "system.h"
	22	#include "coretypes.h"
	23	#include "backend.h"
	24	#include "rtl.h"
	25	#include "tree.h"
	26	#include "gimple.h"
	27	#include "tree-pass.h"
	28	#include "ssa.h"
	29	#include "expmed.h"
	30	#include "optabs-tree.h"
	31	#include "tree-eh.h"
	32	#include "gimple-iterator.h"
	33	#include "gimplify-me.h"
	34	#include "gimplify.h"
	35	#include "tree-cfg.h"
a8d8caca ML	36	#include "bitmap.h"
a8d8caca ML	37	#include "tree-ssa-dce.h"
683e55fa XL	38	#include "memmodel.h"
683e55fa XL	39	#include "optabs.h"
fddc7f00	40	#include "gimple-fold.h"
298e76e6	41	#include "internal-fn.h"
683e55fa XL	42
	43	/* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to
	44	internal function based on vector type of selected expansion.
c30efd8c RD	45
	46	For vec_set:
	47
ddd46293	48	VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D);
683e55fa XL	49	=>
	50	_7 = u;
	51	_8 = .VEC_SET (_7, i_4(D), _1);
c30efd8c RD	52	u = _8;
	53
	54	For vec_extract:
	55
	56	_3 = VIEW_CONVERT_EXPR<intD.1[4]>(vD.2208)[idx_2(D)];
	57	=>
	58	_4 = vD.2208;
	59	_3 = .VEC_EXTRACT (_4, idx_2(D)); */
683e55fa	60
e1a41143	61	static bool
c30efd8c RD	62	gimple_expand_vec_set_extract_expr (struct function *fun,
c30efd8c RD	63	gimple_stmt_iterator *gsi)
683e55fa	64	{
683e55fa XL	65	gcall *new_stmt = NULL;
683e55fa XL	66	gassign *ass_stmt = NULL;
e1a41143	67	bool cfg_changed = false;
683e55fa XL	68
	69	/* Only consider code == GIMPLE_ASSIGN. */
	70	gassign stmt = dyn_cast<gassign > (gsi_stmt (*gsi));
	71	if (!stmt)
e1a41143	72	return false;
683e55fa	73
c30efd8c RD	74	bool is_extract = false;
c30efd8c RD	75
683e55fa	76	tree lhs = gimple_assign_lhs (stmt);
c30efd8c RD	77	tree rhs = gimple_assign_rhs1 (stmt);
	78	tree val, ref;
	79	if (TREE_CODE (lhs) == ARRAY_REF)
	80	{
	81	/* Assume it is a vec_set. */
	82	val = rhs;
	83	ref = lhs;
	84	}
	85	else if (TREE_CODE (rhs) == ARRAY_REF)
	86	{
	87	/* vec_extract. */
	88	is_extract = true;
	89	val = lhs;
	90	ref = rhs;
	91	}
	92	else
e1a41143	93	return false;
683e55fa	94
c30efd8c	95	tree op0 = TREE_OPERAND (ref, 0);
683e55fa XL	96	if (TREE_CODE (op0) == VIEW_CONVERT_EXPR && DECL_P (TREE_OPERAND (op0, 0))
683e55fa XL	97	&& VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0)))
c30efd8c	98	&& TYPE_MODE (TREE_TYPE (ref))
683e55fa XL	99	== TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0)))))
683e55fa XL	100	{
c30efd8c RD	101	tree pos = TREE_OPERAND (ref, 1);
c30efd8c RD	102
683e55fa XL	103	tree view_op0 = TREE_OPERAND (op0, 0);
683e55fa XL	104	machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0));
c30efd8c RD	105	machine_mode extract_mode = TYPE_MODE (TREE_TYPE (ref));
c30efd8c RD	106
96bc048d RB	107	if ((auto_var_in_fn_p (view_op0, fun->decl)
96bc048d RB	108	\|\| (VAR_P (view_op0) && DECL_HARD_REGISTER (view_op0)))
c30efd8c RD	109	&& !TREE_ADDRESSABLE (view_op0)
	110	&& ((!is_extract && can_vec_set_var_idx_p (outermode))
	111	\|\| (is_extract
	112	&& can_vec_extract_var_idx_p (outermode, extract_mode))))
683e55fa XL	113	{
	114	location_t loc = gimple_location (stmt);
	115	tree var_src = make_ssa_name (TREE_TYPE (view_op0));
683e55fa XL	116
	117	ass_stmt = gimple_build_assign (var_src, view_op0);
	118	gimple_set_vuse (ass_stmt, gimple_vuse (stmt));
	119	gimple_set_location (ass_stmt, loc);
	120	gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
	121
c30efd8c RD	122	if (!is_extract)
	123	{
	124	tree var_dst = make_ssa_name (TREE_TYPE (view_op0));
683e55fa	125
c30efd8c RD	126	new_stmt = gimple_build_call_internal (IFN_VEC_SET, 3, var_src,
	127	val, pos);
	128
	129	gimple_call_set_lhs (new_stmt, var_dst);
	130	gimple_set_location (new_stmt, loc);
	131	gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
	132
	133	ass_stmt = gimple_build_assign (view_op0, var_dst);
	134	gimple_set_location (ass_stmt, loc);
	135	gimple_move_vops (ass_stmt, stmt);
	136	gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
683e55fa	137
c30efd8c RD	138	basic_block bb = gimple_bb (stmt);
	139	if (gsi_remove (gsi, true)
	140	&& gimple_purge_dead_eh_edges (bb))
	141	cfg_changed = true;
	142	*gsi = gsi_for_stmt (ass_stmt);
	143	}
	144	else
	145	{
	146	new_stmt
	147	= gimple_build_call_internal (IFN_VEC_EXTRACT, 2, var_src, pos);
	148	gimple_call_set_lhs (new_stmt, lhs);
	149
	150	gsi_replace (gsi, new_stmt, true);
	151	cfg_changed = true;
	152	}
683e55fa XL	153	}
	154	}
	155
e1a41143	156	return cfg_changed;
683e55fa	157	}
502d63b6 ML	158
	159	/* Expand all VEC_COND_EXPR gimple assignments into calls to internal
	160	function based on type of selected expansion. */
	161
	162	static gimple *
871afdc5	163	gimple_expand_vec_cond_expr (struct function fun, gimple_stmt_iterator gsi,
502d63b6 ML	164	hash_map<tree, unsigned int> *vec_cond_ssa_name_uses)
	165	{
	166	tree lhs, op0a = NULL_TREE, op0b = NULL_TREE;
	167	enum tree_code code;
	168	enum tree_code tcode;
	169	machine_mode cmp_op_mode;
	170	bool unsignedp;
	171	enum insn_code icode;
	172	imm_use_iterator imm_iter;
	173
	174	/* Only consider code == GIMPLE_ASSIGN. */
	175	gassign stmt = dyn_cast<gassign > (gsi_stmt (*gsi));
	176	if (!stmt)
	177	return NULL;
	178
	179	code = gimple_assign_rhs_code (stmt);
	180	if (code != VEC_COND_EXPR)
	181	return NULL;
	182
	183	tree op0 = gimple_assign_rhs1 (stmt);
	184	tree op1 = gimple_assign_rhs2 (stmt);
	185	tree op2 = gimple_assign_rhs3 (stmt);
	186	lhs = gimple_assign_lhs (stmt);
	187	machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
	188
fddc7f00 RB	189	/* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations.
	190	Those can end up generated by folding and at least for integer mode masks
	191	we cannot expect vcond expanders to exist. We lower a ? b : c
	192	to (b & a) \| (c & ~a). */
5c197b83 RB	193	if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs))
5c197b83 RB	194	&& !VECTOR_MODE_P (mode))
fddc7f00	195	{
5c197b83	196	gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1)));
fddc7f00 RB	197	gimple_seq stmts = NULL;
	198	tree type = TREE_TYPE (lhs);
	199	location_t loc = gimple_location (stmt);
	200	tree tem0 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op1, op0);
	201	tree tem1 = gimple_build (&stmts, loc, BIT_NOT_EXPR, type, op0);
	202	tree tem2 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op2, tem1);
	203	tree tem3 = gimple_build (&stmts, loc, BIT_IOR_EXPR, type, tem0, tem2);
	204	gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
	205	return gimple_build_assign (lhs, tem3);
	206	}
	207
78595e91	208	bool can_compute_op0 = true;
502d63b6 ML	209	gcc_assert (!COMPARISON_CLASS_P (op0));
	210	if (TREE_CODE (op0) == SSA_NAME)
	211	{
	212	unsigned int used_vec_cond_exprs = 0;
	213	unsigned int *slot = vec_cond_ssa_name_uses->get (op0);
	214	if (slot)
	215	used_vec_cond_exprs = *slot;
	216	else
	217	{
	218	gimple *use_stmt;
	219	FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, op0)
	220	{
	221	gassign assign = dyn_cast<gassign > (use_stmt);
	222	if (assign != NULL
	223	&& gimple_assign_rhs_code (assign) == VEC_COND_EXPR
	224	&& gimple_assign_rhs1 (assign) == op0)
	225	used_vec_cond_exprs++;
	226	}
	227	vec_cond_ssa_name_uses->put (op0, used_vec_cond_exprs);
	228	}
	229
	230	gassign def_stmt = dyn_cast<gassign > (SSA_NAME_DEF_STMT (op0));
8f8f8c68	231	if (def_stmt)
502d63b6 ML	232	{
	233	tcode = gimple_assign_rhs_code (def_stmt);
	234	op0a = gimple_assign_rhs1 (def_stmt);
	235	op0b = gimple_assign_rhs2 (def_stmt);
	236
d0d8a165	237	tree op0_type = TREE_TYPE (op0);
502d63b6	238	tree op0a_type = TREE_TYPE (op0a);
78595e91 RS	239	if (TREE_CODE_CLASS (tcode) == tcc_comparison)
	240	can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type,
	241	tcode);
3a6e3ad3 PK	242
3a6e3ad3 PK	243	/* Try to fold x CMP y ? -1 : 0 to x CMP y. */
78595e91 RS	244	if (can_compute_op0
78595e91 RS	245	&& integer_minus_onep (op1)
3a6e3ad3	246	&& integer_zerop (op2)
78595e91	247	&& TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0)))
3a6e3ad3 PK	248	{
	249	tree conv_op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), op0);
	250	gassign *new_stmt = gimple_build_assign (lhs, conv_op);
	251	gsi_replace (gsi, new_stmt, true);
	252	return new_stmt;
	253	}
	254
8f8f8c68 RB	255	/* When the compare has EH we do not want to forward it when
	256	it has multiple uses and in general because of the complication
	257	with EH redirection. */
	258	if (stmt_can_throw_internal (fun, def_stmt))
	259	tcode = TREE_CODE (op0);
	260
	261	/* If we can compute op0 and have multiple uses, keep the SSA
	262	name and use vcond_mask. */
	263	else if (can_compute_op0
	264	&& used_vec_cond_exprs >= 2
	265	&& (get_vcond_mask_icode (mode, TYPE_MODE (op0_type))
	266	!= CODE_FOR_nothing))
	267	tcode = TREE_CODE (op0);
502d63b6 ML	268	}
	269	else
	270	tcode = TREE_CODE (op0);
	271	}
	272	else
	273	tcode = TREE_CODE (op0);
	274
	275	if (TREE_CODE_CLASS (tcode) != tcc_comparison)
	276	{
	277	gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
	278	if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
	279	!= CODE_FOR_nothing)
	280	return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
	281	/* Fake op0 < 0. */
	282	else
	283	{
	284	gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
	285	== MODE_VECTOR_INT);
	286	op0a = op0;
	287	op0b = build_zero_cst (TREE_TYPE (op0));
	288	tcode = LT_EXPR;
	289	}
	290	}
	291	cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
	292	unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
	293
e29dd0eb RS	294	gcc_assert (known_eq (GET_MODE_NUNITS (mode),
e29dd0eb RS	295	GET_MODE_NUNITS (cmp_op_mode)));
502d63b6 ML	296
502d63b6 ML	297	icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
36f1de95 RB	298	/* Some targets do not have vcondeq and only vcond with NE/EQ
	299	but not vcondu, so make sure to also try vcond here as
	300	vcond_icode_p would canonicalize the optab query to. */
	301	if (icode == CODE_FOR_nothing
	302	&& (tcode == NE_EXPR \|\| tcode == EQ_EXPR)
	303	&& ((icode = get_vcond_icode (mode, cmp_op_mode, !unsignedp))
	304	!= CODE_FOR_nothing))
	305	unsignedp = !unsignedp;
502d63b6 ML	306	if (icode == CODE_FOR_nothing)
	307	{
	308	if (tcode == LT_EXPR
3457dae5	309	&& op0a == op0)
502d63b6 ML	310	{
	311	/* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
	312	into a constant when only get_vcond_eq_icode is supported.
3457dae5	313	Try changing it to NE_EXPR. */
502d63b6 ML	314	tcode = NE_EXPR;
502d63b6 ML	315	}
298e76e6 RS	316	if ((tcode == EQ_EXPR \|\| tcode == NE_EXPR)
	317	&& direct_internal_fn_supported_p (IFN_VCONDEQ, TREE_TYPE (lhs),
	318	TREE_TYPE (op0a),
	319	OPTIMIZE_FOR_BOTH))
502d63b6 ML	320	{
	321	tree tcode_tree = build_int_cst (integer_type_node, tcode);
	322	return gimple_build_call_internal (IFN_VCONDEQ, 5, op0a, op0b, op1,
	323	op2, tcode_tree);
	324	}
502d63b6	325
78595e91 RS	326	gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))
	327	&& can_compute_op0
	328	&& (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
	329	!= CODE_FOR_nothing));
	330	return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
	331	}
	332
502d63b6 ML	333	tree tcode_tree = build_int_cst (integer_type_node, tcode);
	334	return gimple_build_call_internal (unsignedp ? IFN_VCONDU : IFN_VCOND,
	335	5, op0a, op0b, op1, op2, tcode_tree);
	336	}
	337
	338
	339
cbe5f685 RB	340	namespace {
	341
	342	const pass_data pass_data_gimple_isel =
	343	{
	344	GIMPLE_PASS, /* type */
	345	"isel", /* name */
	346	OPTGROUP_VEC, /* optinfo_flags */
	347	TV_NONE, /* tv_id */
	348	PROP_cfg, /* properties_required */
	349	0, /* properties_provided */
	350	0, /* properties_destroyed */
	351	0, /* todo_flags_start */
	352	TODO_update_ssa, /* todo_flags_finish */
	353	};
502d63b6	354
cbe5f685 RB	355	class pass_gimple_isel : public gimple_opt_pass
	356	{
	357	public:
	358	pass_gimple_isel (gcc::context *ctxt)
	359	: gimple_opt_pass (pass_data_gimple_isel, ctxt)
	360	{}
	361
	362	/* opt_pass methods: */
	363	bool gate (function *) final override
	364	{
	365	return true;
	366	}
	367
	368	unsigned int execute (function *fun) final override;
	369	}; // class pass_gimple_isel
	370
	371
	372	/* Iterate all gimple statements and perform pre RTL expansion
	373	GIMPLE massaging to improve instruction selection. */
	374
	375	unsigned int
	376	pass_gimple_isel::execute (struct function *fun)
502d63b6 ML	377	{
	378	gimple_stmt_iterator gsi;
	379	basic_block bb;
502d63b6	380	hash_map<tree, unsigned int> vec_cond_ssa_name_uses;
a8d8caca	381	auto_bitmap dce_ssa_names;
e1a41143	382	bool cfg_changed = false;
502d63b6	383
871afdc5	384	FOR_EACH_BB_FN (bb, fun)
502d63b6 ML	385	{
	386	for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
	387	{
cbe5f685 RB	388	/* Pre-expand VEC_COND_EXPRs to .VCOND* internal function
cbe5f685 RB	389	calls mapping to supported optabs. */
871afdc5	390	gimple *g = gimple_expand_vec_cond_expr (fun, &gsi,
502d63b6 ML	391	&vec_cond_ssa_name_uses);
	392	if (g != NULL)
	393	{
	394	tree lhs = gimple_assign_lhs (gsi_stmt (gsi));
	395	gimple_set_lhs (g, lhs);
	396	gsi_replace (&gsi, g, false);
	397	}
683e55fa	398
cbe5f685	399	/* Recognize .VEC_SET and .VEC_EXTRACT patterns. */
c30efd8c	400	cfg_changed \|= gimple_expand_vec_set_extract_expr (fun, &gsi);
5877c544 ML	401	if (gsi_end_p (gsi))
5877c544 ML	402	break;
cbe5f685 RB	403
	404	gassign stmt = dyn_cast <gassign > (*gsi);
	405	if (!stmt)
	406	continue;
	407
	408	tree_code code = gimple_assign_rhs_code (stmt);
	409	tree lhs = gimple_assign_lhs (stmt);
	410	if (TREE_CODE_CLASS (code) == tcc_comparison
	411	&& !has_single_use (lhs))
	412	{
	413	/* Duplicate COND_EXPR condition defs when they are
	414	comparisons so RTL expansion with the help of TER
	415	can perform better if conversion. */
	416	imm_use_iterator imm_iter;
	417	use_operand_p use_p;
	418	auto_vec<gassign *, 4> cond_exprs;
	419	unsigned cnt = 0;
	420	FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
	421	{
	422	if (is_gimple_debug (USE_STMT (use_p)))
	423	continue;
	424	cnt++;
	425	if (gimple_bb (USE_STMT (use_p)) == bb
	426	&& is_gimple_assign (USE_STMT (use_p))
	427	&& gimple_assign_rhs1_ptr (USE_STMT (use_p)) == use_p->use
	428	&& gimple_assign_rhs_code (USE_STMT (use_p)) == COND_EXPR)
	429	cond_exprs.safe_push (as_a <gassign *> (USE_STMT (use_p)));
	430	}
	431	for (unsigned i = cond_exprs.length () == cnt ? 1 : 0;
	432	i < cond_exprs.length (); ++i)
	433	{
	434	gassign copy = as_a <gassign > (gimple_copy (stmt));
	435	tree new_def = duplicate_ssa_name (lhs, copy);
	436	gimple_assign_set_lhs (copy, new_def);
	437	auto gsi2 = gsi_for_stmt (cond_exprs[i]);
	438	gsi_insert_before (&gsi2, copy, GSI_SAME_STMT);
	439	gimple_assign_set_rhs1 (cond_exprs[i], new_def);
	440	update_stmt (cond_exprs[i]);
	441	}
	442	}
502d63b6 ML	443	}
	444	}
	445
cbe5f685	446	for (auto it = vec_cond_ssa_name_uses.begin ();
a8d8caca ML	447	it != vec_cond_ssa_name_uses.end (); ++it)
	448	bitmap_set_bit (dce_ssa_names, SSA_NAME_VERSION ((*it).first));
	449
	450	simple_dce_from_worklist (dce_ssa_names);
	451
e1a41143	452	return cfg_changed ? TODO_cleanup_cfg : 0;
502d63b6 ML	453	}
502d63b6 ML	454
502d63b6 ML	455	} // anon namespace
	456
	457	gimple_opt_pass *
	458	make_pass_gimple_isel (gcc::context *ctxt)
	459	{
	460	return new pass_gimple_isel (ctxt);
	461	}
	462