[thirdparty/gcc.git] / gcc / internal-fn.c

/* Internal functions.
   Copyright (C) 2011-2013 Free Software Foundation, Inc.

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 3, or (at your option) any later
version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.  */

#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "internal-fn.h"
#include "tree.h"
#include "stor-layout.h"
#include "expr.h"
#include "optabs.h"
#include "basic-block.h"
#include "tree-ssa-alias.h"
#include "internal-fn.h"
#include "gimple-expr.h"
#include "is-a.h"
#include "gimple.h"
#include "ubsan.h"
#include "target.h"
#include "predict.h"

/* The names of each internal function, indexed by function number.  */
const char *const internal_fn_name_array[] = {
#define DEF_INTERNAL_FN(CODE, FLAGS) #CODE,
#include "internal-fn.def"
#undef DEF_INTERNAL_FN
  "<invalid-fn>"
};

/* The ECF_* flags of each internal function, indexed by function number.  */
const int internal_fn_flags_array[] = {
#define DEF_INTERNAL_FN(CODE, FLAGS) FLAGS,
#include "internal-fn.def"
#undef DEF_INTERNAL_FN
  0
};

/* ARRAY_TYPE is an array of vector modes.  Return the associated insn
   for load-lanes-style optab OPTAB.  The insn must exist.  */

static enum insn_code
get_multi_vector_move (tree array_type, convert_optab optab)
{
  enum insn_code icode;
  enum machine_mode imode;
  enum machine_mode vmode;

  gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
  imode = TYPE_MODE (array_type);
  vmode = TYPE_MODE (TREE_TYPE (array_type));

  icode = convert_optab_handler (optab, imode, vmode);
  gcc_assert (icode != CODE_FOR_nothing);
  return icode;
}

/* Expand LOAD_LANES call STMT.  */

static void
expand_LOAD_LANES (gimple stmt)
{
  struct expand_operand ops[2];
  tree type, lhs, rhs;
  rtx target, mem;

  lhs = gimple_call_lhs (stmt);
  rhs = gimple_call_arg (stmt, 0);
  type = TREE_TYPE (lhs);

  target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
  mem = expand_normal (rhs);

  gcc_assert (MEM_P (mem));
  PUT_MODE (mem, TYPE_MODE (type));

  create_output_operand (&ops[0], target, TYPE_MODE (type));
  create_fixed_operand (&ops[1], mem);
  expand_insn (get_multi_vector_move (type, vec_load_lanes_optab), 2, ops);
}

/* Expand STORE_LANES call STMT.  */

static void
expand_STORE_LANES (gimple stmt)
{
  struct expand_operand ops[2];
  tree type, lhs, rhs;
  rtx target, reg;

  lhs = gimple_call_lhs (stmt);
  rhs = gimple_call_arg (stmt, 0);
  type = TREE_TYPE (rhs);

  target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
  reg = expand_normal (rhs);

  gcc_assert (MEM_P (target));
  PUT_MODE (target, TYPE_MODE (type));

  create_fixed_operand (&ops[0], target);
  create_input_operand (&ops[1], reg, TYPE_MODE (type));
  expand_insn (get_multi_vector_move (type, vec_store_lanes_optab), 2, ops);
}

static void
expand_ANNOTATE (gimple stmt ATTRIBUTE_UNUSED)
{
  gcc_unreachable ();
}

/* This should get expanded in adjust_simduid_builtins.  */

static void
expand_GOMP_SIMD_LANE (gimple stmt ATTRIBUTE_UNUSED)
{
  gcc_unreachable ();
}

/* This should get expanded in adjust_simduid_builtins.  */

static void
expand_GOMP_SIMD_VF (gimple stmt ATTRIBUTE_UNUSED)
{
  gcc_unreachable ();
}

/* This should get expanded in adjust_simduid_builtins.  */

static void
expand_GOMP_SIMD_LAST_LANE (gimple stmt ATTRIBUTE_UNUSED)
{
  gcc_unreachable ();
}

/* This should get expanded in the sanopt pass.  */

static void
expand_UBSAN_NULL (gimple stmt ATTRIBUTE_UNUSED)
{
  gcc_unreachable ();
}

/* Add sub/add overflow checking to the statement STMT.
   CODE says whether the operation is +, or -.  */

void
ubsan_expand_si_overflow_addsub_check (tree_code code, gimple stmt)
{
  rtx res, op0, op1;
  tree lhs, fn, arg0, arg1;
  rtx done_label, do_error, target = NULL_RTX;

  lhs = gimple_call_lhs (stmt);
  arg0 = gimple_call_arg (stmt, 0);
  arg1 = gimple_call_arg (stmt, 1);
  done_label = gen_label_rtx ();
  do_error = gen_label_rtx ();
  do_pending_stack_adjust ();
  op0 = expand_normal (arg0);
  op1 = expand_normal (arg1);

  enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
  if (lhs)
    target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);

  enum insn_code icode
    = optab_handler (code == PLUS_EXPR ? addv4_optab : subv4_optab, mode);
  if (icode != CODE_FOR_nothing)
    {
      struct expand_operand ops[4];
      rtx last = get_last_insn ();

      res = gen_reg_rtx (mode);
      create_output_operand (&ops[0], res, mode);
      create_input_operand (&ops[1], op0, mode);
      create_input_operand (&ops[2], op1, mode);
      create_fixed_operand (&ops[3], do_error);
      if (maybe_expand_insn (icode, 4, ops))
	{
	  last = get_last_insn ();
	  if (profile_status != PROFILE_ABSENT
	      && JUMP_P (last)
	      && any_condjump_p (last)
	      && !find_reg_note (last, REG_BR_PROB, 0))
	    add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
	  emit_jump (done_label);
        }
      else
	{
	  delete_insns_since (last);
	  icode = CODE_FOR_nothing;
	}
    }

  if (icode == CODE_FOR_nothing)
    {
      rtx sub_check = gen_label_rtx ();

      /* Compute the operation.  On RTL level, the addition is always
	 unsigned.  */
      res = expand_binop (mode, add_optab, op0, op1,
			  NULL_RTX, false, OPTAB_LIB_WIDEN);

      /* If the op1 is negative, we have to use a different check.  */
      emit_cmp_and_jump_insns (op1, const0_rtx, LT, NULL_RTX, mode,
			       false, sub_check, PROB_EVEN);

      /* Compare the result of the addition with one of the operands.  */
      emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? GE : LE,
			       NULL_RTX, mode, false, done_label,
			       PROB_VERY_LIKELY);
      /* If we get here, we have to print the error.  */
      emit_jump (do_error);

      emit_label (sub_check);
      /* We have k = a + b for b < 0 here.  k <= a must hold.  */
      emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? LE : GE,
			       NULL_RTX, mode, false, done_label,
			       PROB_VERY_LIKELY);
    }

  emit_label (do_error);
  /* Expand the ubsan builtin call.  */
  push_temp_slots ();
  fn = ubsan_build_overflow_builtin (code, gimple_location (stmt),
				     TREE_TYPE (arg0), arg0, arg1);
  expand_normal (fn);
  pop_temp_slots ();
  do_pending_stack_adjust ();

  /* We're done.  */
  emit_label (done_label);

  if (lhs)
    emit_move_insn (target, res);
}

/* Add negate overflow checking to the statement STMT.  */

void
ubsan_expand_si_overflow_neg_check (gimple stmt)
{
  rtx res, op1;
  tree lhs, fn, arg1;
  rtx done_label, do_error, target = NULL_RTX;

  lhs = gimple_call_lhs (stmt);
  arg1 = gimple_call_arg (stmt, 1);
  done_label = gen_label_rtx ();
  do_error = gen_label_rtx ();

  do_pending_stack_adjust ();
  op1 = expand_normal (arg1);

  enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg1));
  if (lhs)
    target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);

  enum insn_code icode = optab_handler (negv3_optab, mode);
  if (icode != CODE_FOR_nothing)
    {
      struct expand_operand ops[3];
      rtx last = get_last_insn ();

      res = gen_reg_rtx (mode);
      create_output_operand (&ops[0], res, mode);
      create_input_operand (&ops[1], op1, mode);
      create_fixed_operand (&ops[2], do_error);
      if (maybe_expand_insn (icode, 3, ops))
	{
	  last = get_last_insn ();
	  if (profile_status != PROFILE_ABSENT
	      && JUMP_P (last)
	      && any_condjump_p (last)
	      && !find_reg_note (last, REG_BR_PROB, 0))
	    add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
	  emit_jump (done_label);
        }
      else
	{
	  delete_insns_since (last);
	  icode = CODE_FOR_nothing;
	}
    }

  if (icode == CODE_FOR_nothing)
    {
      /* Compute the operation.  On RTL level, the addition is always
	 unsigned.  */
      res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);

      /* Compare the operand with the most negative value.  */
      rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
      emit_cmp_and_jump_insns (op1, minv, NE, NULL_RTX, mode, false,
			       done_label, PROB_VERY_LIKELY);
    }

  emit_label (do_error);
  /* Expand the ubsan builtin call.  */
  push_temp_slots ();
  fn = ubsan_build_overflow_builtin (NEGATE_EXPR, gimple_location (stmt),
				     TREE_TYPE (arg1), arg1, NULL_TREE);
  expand_normal (fn);
  pop_temp_slots ();
  do_pending_stack_adjust ();

  /* We're done.  */
  emit_label (done_label);

  if (lhs)
    emit_move_insn (target, res);
}

/* Add mul overflow checking to the statement STMT.  */

void
ubsan_expand_si_overflow_mul_check (gimple stmt)
{
  rtx res, op0, op1;
  tree lhs, fn, arg0, arg1;
  rtx done_label, do_error, target = NULL_RTX;

  lhs = gimple_call_lhs (stmt);
  arg0 = gimple_call_arg (stmt, 0);
  arg1 = gimple_call_arg (stmt, 1);
  done_label = gen_label_rtx ();
  do_error = gen_label_rtx ();

  do_pending_stack_adjust ();
  op0 = expand_normal (arg0);
  op1 = expand_normal (arg1);

  enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
  if (lhs)
    target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);

  enum insn_code icode = optab_handler (mulv4_optab, mode);
  if (icode != CODE_FOR_nothing)
    {
      struct expand_operand ops[4];
      rtx last = get_last_insn ();

      res = gen_reg_rtx (mode);
      create_output_operand (&ops[0], res, mode);
      create_input_operand (&ops[1], op0, mode);
      create_input_operand (&ops[2], op1, mode);
      create_fixed_operand (&ops[3], do_error);
      if (maybe_expand_insn (icode, 4, ops))
	{
	  last = get_last_insn ();
	  if (profile_status != PROFILE_ABSENT
	      && JUMP_P (last)
	      && any_condjump_p (last)
	      && !find_reg_note (last, REG_BR_PROB, 0))
	    add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
	  emit_jump (done_label);
        }
      else
	{
	  delete_insns_since (last);
	  icode = CODE_FOR_nothing;
	}
    }

  if (icode == CODE_FOR_nothing)
    {
      struct separate_ops ops;
      ops.op0 = arg0;
      ops.op1 = arg1;
      ops.op2 = NULL_TREE;
      ops.location = gimple_location (stmt);
      if (GET_MODE_2XWIDER_MODE (mode) != VOIDmode
	  && targetm.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode)))
	{
	  enum machine_mode wmode = GET_MODE_2XWIDER_MODE (mode);
	  ops.code = WIDEN_MULT_EXPR;
	  ops.type
	    = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), 0);

	  res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
	  rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res,
				     GET_MODE_PRECISION (mode), NULL_RTX, 0);
	  hipart = gen_lowpart (mode, hipart);
	  res = gen_lowpart (mode, res);
	  rtx signbit = expand_shift (RSHIFT_EXPR, mode, res,
				      GET_MODE_PRECISION (mode) - 1,
				      NULL_RTX, 0);
	  /* RES is low half of the double width result, HIPART
	     the high half.  There was overflow if
	     HIPART is different from RES < 0 ? -1 : 0.  */
	  emit_cmp_and_jump_insns (signbit, hipart, EQ, NULL_RTX, mode,
				   false, done_label, PROB_VERY_LIKELY);
	}
      else
	{
	  /* For now we don't instrument this.  See __mulvDI3 in libgcc2.c
	     for what could be done.  */
	  ops.code = MULT_EXPR;
	  ops.type = TREE_TYPE (arg0);
	  res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
	  emit_jump (done_label);
	}
    }

  emit_label (do_error);
  /* Expand the ubsan builtin call.  */
  push_temp_slots ();
  fn = ubsan_build_overflow_builtin (MULT_EXPR, gimple_location (stmt),
				     TREE_TYPE (arg0), arg0, arg1);
  expand_normal (fn);
  pop_temp_slots ();
  do_pending_stack_adjust ();

  /* We're done.  */
  emit_label (done_label);

  if (lhs)
    emit_move_insn (target, res);
}

/* Expand UBSAN_CHECK_ADD call STMT.  */

static void
expand_UBSAN_CHECK_ADD (gimple stmt)
{
  ubsan_expand_si_overflow_addsub_check (PLUS_EXPR, stmt);
}

/* Expand UBSAN_CHECK_SUB call STMT.  */

static void
expand_UBSAN_CHECK_SUB (gimple stmt)
{
  if (integer_zerop (gimple_call_arg (stmt, 0)))
    ubsan_expand_si_overflow_neg_check (stmt);
  else
    ubsan_expand_si_overflow_addsub_check (MINUS_EXPR, stmt);
}

/* Expand UBSAN_CHECK_MUL call STMT.  */

static void
expand_UBSAN_CHECK_MUL (gimple stmt)
{
  ubsan_expand_si_overflow_mul_check (stmt);
}

/* Routines to expand each internal function, indexed by function number.
   Each routine has the prototype:

       expand_<NAME> (gimple stmt)

   where STMT is the statement that performs the call. */
static void (*const internal_fn_expanders[]) (gimple) = {
#define DEF_INTERNAL_FN(CODE, FLAGS) expand_##CODE,
#include "internal-fn.def"
#undef DEF_INTERNAL_FN
  0
};

/* Expand STMT, which is a call to internal function FN.  */

void
expand_internal_call (gimple stmt)
{
  internal_fn_expanders[(int) gimple_call_internal_fn (stmt)] (stmt);
}
Commit	Line	Data
25583c4f	1	/* Internal functions.
d1e082c2	2	Copyright (C) 2011-2013 Free Software Foundation, Inc.
25583c4f RS	3
	4	This file is part of GCC.
	5
	6	GCC is free software; you can redistribute it and/or modify it under
	7	the terms of the GNU General Public License as published by the Free
	8	Software Foundation; either version 3, or (at your option) any later
	9	version.
	10
	11	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
	12	WARRANTY; without even the implied warranty of MERCHANTABILITY or
	13	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	14	for more details.
	15
	16	You should have received a copy of the GNU General Public License
	17	along with GCC; see the file COPYING3. If not see
	18	<http://www.gnu.org/licenses/>. */
	19
	20	#include "config.h"
	21	#include "system.h"
	22	#include "coretypes.h"
	23	#include "internal-fn.h"
	24	#include "tree.h"
d8a2d370	25	#include "stor-layout.h"
25583c4f RS	26	#include "expr.h"
25583c4f RS	27	#include "optabs.h"
2fb9a547 AM	28	#include "basic-block.h"
	29	#include "tree-ssa-alias.h"
	30	#include "internal-fn.h"
	31	#include "gimple-expr.h"
	32	#include "is-a.h"
25583c4f	33	#include "gimple.h"
31e071ae MP	34	#include "ubsan.h"
	35	#include "target.h"
	36	#include "predict.h"
25583c4f RS	37
	38	/* The names of each internal function, indexed by function number. */
	39	const char *const internal_fn_name_array[] = {
	40	#define DEF_INTERNAL_FN(CODE, FLAGS) #CODE,
	41	#include "internal-fn.def"
	42	#undef DEF_INTERNAL_FN
	43	"<invalid-fn>"
	44	};
	45
	46	/* The ECF_* flags of each internal function, indexed by function number. */
	47	const int internal_fn_flags_array[] = {
	48	#define DEF_INTERNAL_FN(CODE, FLAGS) FLAGS,
	49	#include "internal-fn.def"
	50	#undef DEF_INTERNAL_FN
	51	0
	52	};
	53
272c6793 RS	54	/* ARRAY_TYPE is an array of vector modes. Return the associated insn
	55	for load-lanes-style optab OPTAB. The insn must exist. */
	56
	57	static enum insn_code
	58	get_multi_vector_move (tree array_type, convert_optab optab)
	59	{
	60	enum insn_code icode;
	61	enum machine_mode imode;
	62	enum machine_mode vmode;
	63
	64	gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
	65	imode = TYPE_MODE (array_type);
	66	vmode = TYPE_MODE (TREE_TYPE (array_type));
	67
	68	icode = convert_optab_handler (optab, imode, vmode);
	69	gcc_assert (icode != CODE_FOR_nothing);
	70	return icode;
	71	}
	72
	73	/* Expand LOAD_LANES call STMT. */
	74
	75	static void
	76	expand_LOAD_LANES (gimple stmt)
	77	{
	78	struct expand_operand ops[2];
	79	tree type, lhs, rhs;
	80	rtx target, mem;
	81
	82	lhs = gimple_call_lhs (stmt);
	83	rhs = gimple_call_arg (stmt, 0);
	84	type = TREE_TYPE (lhs);
	85
	86	target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
	87	mem = expand_normal (rhs);
	88
	89	gcc_assert (MEM_P (mem));
	90	PUT_MODE (mem, TYPE_MODE (type));
	91
	92	create_output_operand (&ops[0], target, TYPE_MODE (type));
	93	create_fixed_operand (&ops[1], mem);
	94	expand_insn (get_multi_vector_move (type, vec_load_lanes_optab), 2, ops);
	95	}
	96
	97	/* Expand STORE_LANES call STMT. */
	98
	99	static void
	100	expand_STORE_LANES (gimple stmt)
	101	{
	102	struct expand_operand ops[2];
	103	tree type, lhs, rhs;
	104	rtx target, reg;
	105
	106	lhs = gimple_call_lhs (stmt);
	107	rhs = gimple_call_arg (stmt, 0);
	108	type = TREE_TYPE (rhs);
	109
	110	target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
	111	reg = expand_normal (rhs);
	112
	113	gcc_assert (MEM_P (target));
	114	PUT_MODE (target, TYPE_MODE (type));
	115
	116	create_fixed_operand (&ops[0], target);
	117	create_input_operand (&ops[1], reg, TYPE_MODE (type));
118	expand_insn (get_multi_vector_move (type, vec_store_lanes_optab), 2, ops);
119	}
120
8170608b TB	121	static void
	122	expand_ANNOTATE (gimple stmt ATTRIBUTE_UNUSED)
	123	{
	124	gcc_unreachable ();
	125	}
	126
74bf76ed JJ	127	/* This should get expanded in adjust_simduid_builtins. */
	128
	129	static void
	130	expand_GOMP_SIMD_LANE (gimple stmt ATTRIBUTE_UNUSED)
	131	{
	132	gcc_unreachable ();
	133	}
	134
	135	/* This should get expanded in adjust_simduid_builtins. */
	136
	137	static void
	138	expand_GOMP_SIMD_VF (gimple stmt ATTRIBUTE_UNUSED)
	139	{
	140	gcc_unreachable ();
	141	}
	142
	143	/* This should get expanded in adjust_simduid_builtins. */
	144
	145	static void
	146	expand_GOMP_SIMD_LAST_LANE (gimple stmt ATTRIBUTE_UNUSED)
	147	{
	148	gcc_unreachable ();
	149	}
	150
b9a55b13 MP	151	/* This should get expanded in the sanopt pass. */
	152
	153	static void
	154	expand_UBSAN_NULL (gimple stmt ATTRIBUTE_UNUSED)
	155	{
	156	gcc_unreachable ();
	157	}
	158
31e071ae MP	159	/* Add sub/add overflow checking to the statement STMT.
	160	CODE says whether the operation is +, or -. */
	161
	162	void
	163	ubsan_expand_si_overflow_addsub_check (tree_code code, gimple stmt)
	164	{
	165	rtx res, op0, op1;
	166	tree lhs, fn, arg0, arg1;
	167	rtx done_label, do_error, target = NULL_RTX;
	168
	169	lhs = gimple_call_lhs (stmt);
	170	arg0 = gimple_call_arg (stmt, 0);
	171	arg1 = gimple_call_arg (stmt, 1);
	172	done_label = gen_label_rtx ();
	173	do_error = gen_label_rtx ();
31e071ae MP	174	do_pending_stack_adjust ();
	175	op0 = expand_normal (arg0);
	176	op1 = expand_normal (arg1);
	177
	178	enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
	179	if (lhs)
	180	target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
	181
	182	enum insn_code icode
	183	= optab_handler (code == PLUS_EXPR ? addv4_optab : subv4_optab, mode);
	184	if (icode != CODE_FOR_nothing)
	185	{
	186	struct expand_operand ops[4];
	187	rtx last = get_last_insn ();
	188
	189	res = gen_reg_rtx (mode);
	190	create_output_operand (&ops[0], res, mode);
	191	create_input_operand (&ops[1], op0, mode);
	192	create_input_operand (&ops[2], op1, mode);
	193	create_fixed_operand (&ops[3], do_error);
	194	if (maybe_expand_insn (icode, 4, ops))
	195	{
	196	last = get_last_insn ();
	197	if (profile_status != PROFILE_ABSENT
	198	&& JUMP_P (last)
	199	&& any_condjump_p (last)
	200	&& !find_reg_note (last, REG_BR_PROB, 0))
	201	add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
	202	emit_jump (done_label);
	203	}
	204	else
	205	{
	206	delete_insns_since (last);
	207	icode = CODE_FOR_nothing;
	208	}
	209	}
	210
	211	if (icode == CODE_FOR_nothing)
	212	{
	213	rtx sub_check = gen_label_rtx ();
	214
	215	/* Compute the operation. On RTL level, the addition is always
	216	unsigned. */
	217	res = expand_binop (mode, add_optab, op0, op1,
	218	NULL_RTX, false, OPTAB_LIB_WIDEN);
	219
	220	/* If the op1 is negative, we have to use a different check. */
	221	emit_cmp_and_jump_insns (op1, const0_rtx, LT, NULL_RTX, mode,
	222	false, sub_check, PROB_EVEN);
	223
	224	/* Compare the result of the addition with one of the operands. */
	225	emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? GE : LE,
	226	NULL_RTX, mode, false, done_label,
	227	PROB_VERY_LIKELY);
	228	/* If we get here, we have to print the error. */
	229	emit_jump (do_error);
	230
	231	emit_label (sub_check);
	232	/* We have k = a + b for b < 0 here. k <= a must hold. */
	233	emit_cmp_and_jump_insns (res, op0, code == PLUS_EXPR ? LE : GE,
	234	NULL_RTX, mode, false, done_label,
	235	PROB_VERY_LIKELY);
	236	}
	237
1769415d MP	238	emit_label (do_error);
	239	/* Expand the ubsan builtin call. */
	240	push_temp_slots ();
	241	fn = ubsan_build_overflow_builtin (code, gimple_location (stmt),
	242	TREE_TYPE (arg0), arg0, arg1);
	243	expand_normal (fn);
	244	pop_temp_slots ();
	245	do_pending_stack_adjust ();
31e071ae	246
1769415d MP	247	/* We're done. */
1769415d MP	248	emit_label (done_label);
31e071ae MP	249
	250	if (lhs)
	251	emit_move_insn (target, res);
	252	}
	253
	254	/* Add negate overflow checking to the statement STMT. */
	255
	256	void
	257	ubsan_expand_si_overflow_neg_check (gimple stmt)
	258	{
	259	rtx res, op1;
	260	tree lhs, fn, arg1;
	261	rtx done_label, do_error, target = NULL_RTX;
	262
	263	lhs = gimple_call_lhs (stmt);
	264	arg1 = gimple_call_arg (stmt, 1);
	265	done_label = gen_label_rtx ();
	266	do_error = gen_label_rtx ();
31e071ae MP	267
	268	do_pending_stack_adjust ();
	269	op1 = expand_normal (arg1);
	270
	271	enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg1));
	272	if (lhs)
	273	target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
	274
	275	enum insn_code icode = optab_handler (negv3_optab, mode);
	276	if (icode != CODE_FOR_nothing)
	277	{
	278	struct expand_operand ops[3];
	279	rtx last = get_last_insn ();
	280
	281	res = gen_reg_rtx (mode);
	282	create_output_operand (&ops[0], res, mode);
	283	create_input_operand (&ops[1], op1, mode);
	284	create_fixed_operand (&ops[2], do_error);
	285	if (maybe_expand_insn (icode, 3, ops))
	286	{
	287	last = get_last_insn ();
	288	if (profile_status != PROFILE_ABSENT
	289	&& JUMP_P (last)
	290	&& any_condjump_p (last)
	291	&& !find_reg_note (last, REG_BR_PROB, 0))
	292	add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
	293	emit_jump (done_label);
	294	}
	295	else
	296	{
	297	delete_insns_since (last);
	298	icode = CODE_FOR_nothing;
	299	}
	300	}
	301
	302	if (icode == CODE_FOR_nothing)
	303	{
	304	/* Compute the operation. On RTL level, the addition is always
	305	unsigned. */
	306	res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
	307
	308	/* Compare the operand with the most negative value. */
	309	rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
	310	emit_cmp_and_jump_insns (op1, minv, NE, NULL_RTX, mode, false,
	311	done_label, PROB_VERY_LIKELY);
	312	}
	313
	314	emit_label (do_error);
	315	/* Expand the ubsan builtin call. */
1769415d MP	316	push_temp_slots ();
	317	fn = ubsan_build_overflow_builtin (NEGATE_EXPR, gimple_location (stmt),
	318	TREE_TYPE (arg1), arg1, NULL_TREE);
31e071ae	319	expand_normal (fn);
1769415d	320	pop_temp_slots ();
31e071ae MP	321	do_pending_stack_adjust ();
	322
	323	/* We're done. */
	324	emit_label (done_label);
	325
	326	if (lhs)
	327	emit_move_insn (target, res);
	328	}
	329
	330	/* Add mul overflow checking to the statement STMT. */
	331
	332	void
	333	ubsan_expand_si_overflow_mul_check (gimple stmt)
	334	{
	335	rtx res, op0, op1;
	336	tree lhs, fn, arg0, arg1;
	337	rtx done_label, do_error, target = NULL_RTX;
	338
	339	lhs = gimple_call_lhs (stmt);
	340	arg0 = gimple_call_arg (stmt, 0);
	341	arg1 = gimple_call_arg (stmt, 1);
	342	done_label = gen_label_rtx ();
	343	do_error = gen_label_rtx ();
31e071ae MP	344
	345	do_pending_stack_adjust ();
	346	op0 = expand_normal (arg0);
	347	op1 = expand_normal (arg1);
	348
	349	enum machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
	350	if (lhs)
	351	target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
	352
	353	enum insn_code icode = optab_handler (mulv4_optab, mode);
	354	if (icode != CODE_FOR_nothing)
	355	{
	356	struct expand_operand ops[4];
	357	rtx last = get_last_insn ();
	358
	359	res = gen_reg_rtx (mode);
	360	create_output_operand (&ops[0], res, mode);
	361	create_input_operand (&ops[1], op0, mode);
	362	create_input_operand (&ops[2], op1, mode);
	363	create_fixed_operand (&ops[3], do_error);
	364	if (maybe_expand_insn (icode, 4, ops))
	365	{
	366	last = get_last_insn ();
	367	if (profile_status != PROFILE_ABSENT
	368	&& JUMP_P (last)
	369	&& any_condjump_p (last)
	370	&& !find_reg_note (last, REG_BR_PROB, 0))
	371	add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
	372	emit_jump (done_label);
	373	}
	374	else
	375	{
	376	delete_insns_since (last);
	377	icode = CODE_FOR_nothing;
	378	}
	379	}
	380
	381	if (icode == CODE_FOR_nothing)
	382	{
	383	struct separate_ops ops;
	384	ops.op0 = arg0;
	385	ops.op1 = arg1;
	386	ops.op2 = NULL_TREE;
	387	ops.location = gimple_location (stmt);
	388	if (GET_MODE_2XWIDER_MODE (mode) != VOIDmode
	389	&& targetm.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode)))
	390	{
	391	enum machine_mode wmode = GET_MODE_2XWIDER_MODE (mode);
	392	ops.code = WIDEN_MULT_EXPR;
	393	ops.type
	394	= build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), 0);
	395
	396	res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
	397	rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res,
	398	GET_MODE_PRECISION (mode), NULL_RTX, 0);
	399	hipart = gen_lowpart (mode, hipart);
	400	res = gen_lowpart (mode, res);
	401	rtx signbit = expand_shift (RSHIFT_EXPR, mode, res,
	402	GET_MODE_PRECISION (mode) - 1,
	403	NULL_RTX, 0);
	404	/* RES is low half of the double width result, HIPART
	405	the high half. There was overflow if
	406	HIPART is different from RES < 0 ? -1 : 0. */
	407	emit_cmp_and_jump_insns (signbit, hipart, EQ, NULL_RTX, mode,
408	false, done_label, PROB_VERY_LIKELY);
409	}
410	else
411	{
412	/* For now we don't instrument this. See __mulvDI3 in libgcc2.c
413	for what could be done. */
414	ops.code = MULT_EXPR;
415	ops.type = TREE_TYPE (arg0);
416	res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
417	emit_jump (done_label);
418	}
419	}
420
421	emit_label (do_error);
422	/* Expand the ubsan builtin call. */
1769415d MP	423	push_temp_slots ();
	424	fn = ubsan_build_overflow_builtin (MULT_EXPR, gimple_location (stmt),
	425	TREE_TYPE (arg0), arg0, arg1);
31e071ae	426	expand_normal (fn);
1769415d	427	pop_temp_slots ();
31e071ae MP	428	do_pending_stack_adjust ();
	429
	430	/* We're done. */
	431	emit_label (done_label);
	432
	433	if (lhs)
	434	emit_move_insn (target, res);
	435	}
	436
	437	/* Expand UBSAN_CHECK_ADD call STMT. */
	438
	439	static void
	440	expand_UBSAN_CHECK_ADD (gimple stmt)
	441	{
	442	ubsan_expand_si_overflow_addsub_check (PLUS_EXPR, stmt);
	443	}
	444
	445	/* Expand UBSAN_CHECK_SUB call STMT. */
	446
	447	static void
	448	expand_UBSAN_CHECK_SUB (gimple stmt)
	449	{
	450	if (integer_zerop (gimple_call_arg (stmt, 0)))
	451	ubsan_expand_si_overflow_neg_check (stmt);
	452	else
	453	ubsan_expand_si_overflow_addsub_check (MINUS_EXPR, stmt);
	454	}
	455
	456	/* Expand UBSAN_CHECK_MUL call STMT. */
	457
	458	static void
	459	expand_UBSAN_CHECK_MUL (gimple stmt)
	460	{
	461	ubsan_expand_si_overflow_mul_check (stmt);
	462	}
	463
25583c4f RS	464	/* Routines to expand each internal function, indexed by function number.
	465	Each routine has the prototype:
	466
	467	expand_<NAME> (gimple stmt)
	468
	469	where STMT is the statement that performs the call. */
	470	static void (*const internal_fn_expanders[]) (gimple) = {
	471	#define DEF_INTERNAL_FN(CODE, FLAGS) expand_##CODE,
	472	#include "internal-fn.def"
	473	#undef DEF_INTERNAL_FN
	474	0
	475	};
	476
	477	/* Expand STMT, which is a call to internal function FN. */
	478
	479	void
	480	expand_internal_call (gimple stmt)
	481	{
	482	internal_fn_expanders[(int) gimple_call_internal_fn (stmt)] (stmt);
	483	}