1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2017 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
35 #include "diagnostic-core.h"
37 /* Include insn-config.h before expr.h so that HAVE_conditional_move
38 is properly defined. */
39 #include "stor-layout.h"
44 #include "optabs-tree.h"
47 static void prepare_float_lib_cmp (rtx
, rtx
, enum rtx_code
, rtx
*,
49 static rtx
expand_unop_direct (machine_mode
, optab
, rtx
, rtx
, int);
50 static void emit_libcall_block_1 (rtx_insn
*, rtx
, rtx
, rtx
, bool);
52 /* Debug facility for use in GDB. */
53 void debug_optab_libfuncs (void);
55 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
56 the result of operation CODE applied to OP0 (and OP1 if it is a binary
59 If the last insn does not set TARGET, don't do anything, but return 1.
61 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
62 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
63 try again, ensuring that TARGET is not one of the operands. */
66 add_equal_note (rtx_insn
*insns
, rtx target
, enum rtx_code code
, rtx op0
, rtx op1
)
72 gcc_assert (insns
&& INSN_P (insns
) && NEXT_INSN (insns
));
74 if (GET_RTX_CLASS (code
) != RTX_COMM_ARITH
75 && GET_RTX_CLASS (code
) != RTX_BIN_ARITH
76 && GET_RTX_CLASS (code
) != RTX_COMM_COMPARE
77 && GET_RTX_CLASS (code
) != RTX_COMPARE
78 && GET_RTX_CLASS (code
) != RTX_UNARY
)
81 if (GET_CODE (target
) == ZERO_EXTRACT
)
84 for (last_insn
= insns
;
85 NEXT_INSN (last_insn
) != NULL_RTX
;
86 last_insn
= NEXT_INSN (last_insn
))
89 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
90 a value changing in the insn, so the note would be invalid for CSE. */
91 if (reg_overlap_mentioned_p (target
, op0
)
92 || (op1
&& reg_overlap_mentioned_p (target
, op1
)))
95 && (rtx_equal_p (target
, op0
)
96 || (op1
&& rtx_equal_p (target
, op1
))))
98 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
99 over expanding it as temp = MEM op X, MEM = temp. If the target
100 supports MEM = MEM op X instructions, it is sometimes too hard
101 to reconstruct that form later, especially if X is also a memory,
102 and due to multiple occurrences of addresses the address might
103 be forced into register unnecessarily.
104 Note that not emitting the REG_EQUIV note might inhibit
105 CSE in some cases. */
106 set
= single_set (last_insn
);
108 && GET_CODE (SET_SRC (set
)) == code
109 && MEM_P (SET_DEST (set
))
110 && (rtx_equal_p (SET_DEST (set
), XEXP (SET_SRC (set
), 0))
111 || (op1
&& rtx_equal_p (SET_DEST (set
),
112 XEXP (SET_SRC (set
), 1)))))
118 set
= set_for_reg_notes (last_insn
);
122 if (! rtx_equal_p (SET_DEST (set
), target
)
123 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
124 && (GET_CODE (SET_DEST (set
)) != STRICT_LOW_PART
125 || ! rtx_equal_p (XEXP (SET_DEST (set
), 0), target
)))
128 if (GET_RTX_CLASS (code
) == RTX_UNARY
)
138 if (GET_MODE (op0
) != VOIDmode
&& GET_MODE (target
) != GET_MODE (op0
))
140 note
= gen_rtx_fmt_e (code
, GET_MODE (op0
), copy_rtx (op0
));
141 if (GET_MODE_UNIT_SIZE (GET_MODE (op0
))
142 > GET_MODE_UNIT_SIZE (GET_MODE (target
)))
143 note
= simplify_gen_unary (TRUNCATE
, GET_MODE (target
),
144 note
, GET_MODE (op0
));
146 note
= simplify_gen_unary (ZERO_EXTEND
, GET_MODE (target
),
147 note
, GET_MODE (op0
));
152 note
= gen_rtx_fmt_e (code
, GET_MODE (target
), copy_rtx (op0
));
156 note
= gen_rtx_fmt_ee (code
, GET_MODE (target
), copy_rtx (op0
), copy_rtx (op1
));
158 set_unique_reg_note (last_insn
, REG_EQUAL
, note
);
163 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
164 for a widening operation would be. In most cases this would be OP0, but if
165 that's a constant it'll be VOIDmode, which isn't useful. */
168 widened_mode (machine_mode to_mode
, rtx op0
, rtx op1
)
170 machine_mode m0
= GET_MODE (op0
);
171 machine_mode m1
= GET_MODE (op1
);
174 if (m0
== VOIDmode
&& m1
== VOIDmode
)
176 else if (m0
== VOIDmode
|| GET_MODE_UNIT_SIZE (m0
) < GET_MODE_UNIT_SIZE (m1
))
181 if (GET_MODE_UNIT_SIZE (result
) > GET_MODE_UNIT_SIZE (to_mode
))
187 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
188 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
189 not actually do a sign-extend or zero-extend, but can leave the
190 higher-order bits of the result rtx undefined, for example, in the case
191 of logical operations, but not right shifts. */
194 widen_operand (rtx op
, machine_mode mode
, machine_mode oldmode
,
195 int unsignedp
, int no_extend
)
198 scalar_int_mode int_mode
;
200 /* If we don't have to extend and this is a constant, return it. */
201 if (no_extend
&& GET_MODE (op
) == VOIDmode
)
204 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
205 extend since it will be more efficient to do so unless the signedness of
206 a promoted object differs from our extension. */
208 || !is_a
<scalar_int_mode
> (mode
, &int_mode
)
209 || (GET_CODE (op
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (op
)
210 && SUBREG_CHECK_PROMOTED_SIGN (op
, unsignedp
)))
211 return convert_modes (mode
, oldmode
, op
, unsignedp
);
213 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
215 if (GET_MODE_SIZE (int_mode
) <= UNITS_PER_WORD
)
216 return gen_lowpart (int_mode
, force_reg (GET_MODE (op
), op
));
218 /* Otherwise, get an object of MODE, clobber it, and set the low-order
221 result
= gen_reg_rtx (int_mode
);
222 emit_clobber (result
);
223 emit_move_insn (gen_lowpart (GET_MODE (op
), result
), op
);
227 /* Expand vector widening operations.
229 There are two different classes of operations handled here:
230 1) Operations whose result is wider than all the arguments to the operation.
231 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
232 In this case OP0 and optionally OP1 would be initialized,
233 but WIDE_OP wouldn't (not relevant for this case).
234 2) Operations whose result is of the same size as the last argument to the
235 operation, but wider than all the other arguments to the operation.
236 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
237 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
239 E.g, when called to expand the following operations, this is how
240 the arguments will be initialized:
242 widening-sum 2 oprnd0 - oprnd1
243 widening-dot-product 3 oprnd0 oprnd1 oprnd2
244 widening-mult 2 oprnd0 oprnd1 -
245 type-promotion (vec-unpack) 1 oprnd0 - - */
248 expand_widen_pattern_expr (sepops ops
, rtx op0
, rtx op1
, rtx wide_op
,
249 rtx target
, int unsignedp
)
251 struct expand_operand eops
[4];
252 tree oprnd0
, oprnd1
, oprnd2
;
253 machine_mode wmode
= VOIDmode
, tmode0
, tmode1
= VOIDmode
;
254 optab widen_pattern_optab
;
255 enum insn_code icode
;
256 int nops
= TREE_CODE_LENGTH (ops
->code
);
260 tmode0
= TYPE_MODE (TREE_TYPE (oprnd0
));
261 widen_pattern_optab
=
262 optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), optab_default
);
263 if (ops
->code
== WIDEN_MULT_PLUS_EXPR
264 || ops
->code
== WIDEN_MULT_MINUS_EXPR
)
265 icode
= find_widening_optab_handler (widen_pattern_optab
,
266 TYPE_MODE (TREE_TYPE (ops
->op2
)),
269 icode
= optab_handler (widen_pattern_optab
, tmode0
);
270 gcc_assert (icode
!= CODE_FOR_nothing
);
275 tmode1
= TYPE_MODE (TREE_TYPE (oprnd1
));
278 /* The last operand is of a wider mode than the rest of the operands. */
283 gcc_assert (tmode1
== tmode0
);
286 wmode
= TYPE_MODE (TREE_TYPE (oprnd2
));
290 create_output_operand (&eops
[op
++], target
, TYPE_MODE (ops
->type
));
291 create_convert_operand_from (&eops
[op
++], op0
, tmode0
, unsignedp
);
293 create_convert_operand_from (&eops
[op
++], op1
, tmode1
, unsignedp
);
295 create_convert_operand_from (&eops
[op
++], wide_op
, wmode
, unsignedp
);
296 expand_insn (icode
, op
, eops
);
297 return eops
[0].value
;
300 /* Generate code to perform an operation specified by TERNARY_OPTAB
301 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
303 UNSIGNEDP is for the case where we have to widen the operands
304 to perform the operation. It says to use zero-extension.
306 If TARGET is nonzero, the value
307 is generated there, if it is convenient to do so.
308 In all cases an rtx is returned for the locus of the value;
309 this may or may not be TARGET. */
312 expand_ternary_op (machine_mode mode
, optab ternary_optab
, rtx op0
,
313 rtx op1
, rtx op2
, rtx target
, int unsignedp
)
315 struct expand_operand ops
[4];
316 enum insn_code icode
= optab_handler (ternary_optab
, mode
);
318 gcc_assert (optab_handler (ternary_optab
, mode
) != CODE_FOR_nothing
);
320 create_output_operand (&ops
[0], target
, mode
);
321 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
322 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
323 create_convert_operand_from (&ops
[3], op2
, mode
, unsignedp
);
324 expand_insn (icode
, 4, ops
);
329 /* Like expand_binop, but return a constant rtx if the result can be
330 calculated at compile time. The arguments and return value are
331 otherwise the same as for expand_binop. */
334 simplify_expand_binop (machine_mode mode
, optab binoptab
,
335 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
336 enum optab_methods methods
)
338 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
340 rtx x
= simplify_binary_operation (optab_to_code (binoptab
),
346 return expand_binop (mode
, binoptab
, op0
, op1
, target
, unsignedp
, methods
);
349 /* Like simplify_expand_binop, but always put the result in TARGET.
350 Return true if the expansion succeeded. */
353 force_expand_binop (machine_mode mode
, optab binoptab
,
354 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
355 enum optab_methods methods
)
357 rtx x
= simplify_expand_binop (mode
, binoptab
, op0
, op1
,
358 target
, unsignedp
, methods
);
362 emit_move_insn (target
, x
);
366 /* Create a new vector value in VMODE with all elements set to OP. The
367 mode of OP must be the element mode of VMODE. If OP is a constant,
368 then the return value will be a constant. */
371 expand_vector_broadcast (machine_mode vmode
, rtx op
)
373 enum insn_code icode
;
378 gcc_checking_assert (VECTOR_MODE_P (vmode
));
381 return gen_const_vec_duplicate (vmode
, op
);
383 /* ??? If the target doesn't have a vec_init, then we have no easy way
384 of performing this operation. Most of this sort of generic support
385 is hidden away in the vector lowering support in gimple. */
386 icode
= convert_optab_handler (vec_init_optab
, vmode
,
387 GET_MODE_INNER (vmode
));
388 if (icode
== CODE_FOR_nothing
)
391 n
= GET_MODE_NUNITS (vmode
);
392 vec
= rtvec_alloc (n
);
393 for (i
= 0; i
< n
; ++i
)
394 RTVEC_ELT (vec
, i
) = op
;
395 ret
= gen_reg_rtx (vmode
);
396 emit_insn (GEN_FCN (icode
) (ret
, gen_rtx_PARALLEL (vmode
, vec
)));
401 /* This subroutine of expand_doubleword_shift handles the cases in which
402 the effective shift value is >= BITS_PER_WORD. The arguments and return
403 value are the same as for the parent routine, except that SUPERWORD_OP1
404 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
405 INTO_TARGET may be null if the caller has decided to calculate it. */
408 expand_superword_shift (optab binoptab
, rtx outof_input
, rtx superword_op1
,
409 rtx outof_target
, rtx into_target
,
410 int unsignedp
, enum optab_methods methods
)
412 if (into_target
!= 0)
413 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, superword_op1
,
414 into_target
, unsignedp
, methods
))
417 if (outof_target
!= 0)
419 /* For a signed right shift, we must fill OUTOF_TARGET with copies
420 of the sign bit, otherwise we must fill it with zeros. */
421 if (binoptab
!= ashr_optab
)
422 emit_move_insn (outof_target
, CONST0_RTX (word_mode
));
424 if (!force_expand_binop (word_mode
, binoptab
,
425 outof_input
, GEN_INT (BITS_PER_WORD
- 1),
426 outof_target
, unsignedp
, methods
))
432 /* This subroutine of expand_doubleword_shift handles the cases in which
433 the effective shift value is < BITS_PER_WORD. The arguments and return
434 value are the same as for the parent routine. */
437 expand_subword_shift (scalar_int_mode op1_mode
, optab binoptab
,
438 rtx outof_input
, rtx into_input
, rtx op1
,
439 rtx outof_target
, rtx into_target
,
440 int unsignedp
, enum optab_methods methods
,
441 unsigned HOST_WIDE_INT shift_mask
)
443 optab reverse_unsigned_shift
, unsigned_shift
;
446 reverse_unsigned_shift
= (binoptab
== ashl_optab
? lshr_optab
: ashl_optab
);
447 unsigned_shift
= (binoptab
== ashl_optab
? ashl_optab
: lshr_optab
);
449 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
450 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
451 the opposite direction to BINOPTAB. */
452 if (CONSTANT_P (op1
) || shift_mask
>= BITS_PER_WORD
)
454 carries
= outof_input
;
455 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
,
456 op1_mode
), op1_mode
);
457 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
462 /* We must avoid shifting by BITS_PER_WORD bits since that is either
463 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
464 has unknown behavior. Do a single shift first, then shift by the
465 remainder. It's OK to use ~OP1 as the remainder if shift counts
466 are truncated to the mode size. */
467 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
468 outof_input
, const1_rtx
, 0, unsignedp
, methods
);
469 if (shift_mask
== BITS_PER_WORD
- 1)
471 tmp
= immed_wide_int_const
472 (wi::minus_one (GET_MODE_PRECISION (op1_mode
)), op1_mode
);
473 tmp
= simplify_expand_binop (op1_mode
, xor_optab
, op1
, tmp
,
478 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
- 1,
479 op1_mode
), op1_mode
);
480 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
484 if (tmp
== 0 || carries
== 0)
486 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
487 carries
, tmp
, 0, unsignedp
, methods
);
491 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
492 so the result can go directly into INTO_TARGET if convenient. */
493 tmp
= expand_binop (word_mode
, unsigned_shift
, into_input
, op1
,
494 into_target
, unsignedp
, methods
);
498 /* Now OR in the bits carried over from OUTOF_INPUT. */
499 if (!force_expand_binop (word_mode
, ior_optab
, tmp
, carries
,
500 into_target
, unsignedp
, methods
))
503 /* Use a standard word_mode shift for the out-of half. */
504 if (outof_target
!= 0)
505 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
506 outof_target
, unsignedp
, methods
))
513 /* Try implementing expand_doubleword_shift using conditional moves.
514 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
515 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
516 are the shift counts to use in the former and latter case. All other
517 arguments are the same as the parent routine. */
520 expand_doubleword_shift_condmove (scalar_int_mode op1_mode
, optab binoptab
,
521 enum rtx_code cmp_code
, rtx cmp1
, rtx cmp2
,
522 rtx outof_input
, rtx into_input
,
523 rtx subword_op1
, rtx superword_op1
,
524 rtx outof_target
, rtx into_target
,
525 int unsignedp
, enum optab_methods methods
,
526 unsigned HOST_WIDE_INT shift_mask
)
528 rtx outof_superword
, into_superword
;
530 /* Put the superword version of the output into OUTOF_SUPERWORD and
532 outof_superword
= outof_target
!= 0 ? gen_reg_rtx (word_mode
) : 0;
533 if (outof_target
!= 0 && subword_op1
== superword_op1
)
535 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
536 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
537 into_superword
= outof_target
;
538 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
539 outof_superword
, 0, unsignedp
, methods
))
544 into_superword
= gen_reg_rtx (word_mode
);
545 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
546 outof_superword
, into_superword
,
551 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
552 if (!expand_subword_shift (op1_mode
, binoptab
,
553 outof_input
, into_input
, subword_op1
,
554 outof_target
, into_target
,
555 unsignedp
, methods
, shift_mask
))
558 /* Select between them. Do the INTO half first because INTO_SUPERWORD
559 might be the current value of OUTOF_TARGET. */
560 if (!emit_conditional_move (into_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
561 into_target
, into_superword
, word_mode
, false))
564 if (outof_target
!= 0)
565 if (!emit_conditional_move (outof_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
566 outof_target
, outof_superword
,
573 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
574 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
575 input operand; the shift moves bits in the direction OUTOF_INPUT->
576 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
577 of the target. OP1 is the shift count and OP1_MODE is its mode.
578 If OP1 is constant, it will have been truncated as appropriate
579 and is known to be nonzero.
581 If SHIFT_MASK is zero, the result of word shifts is undefined when the
582 shift count is outside the range [0, BITS_PER_WORD). This routine must
583 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
585 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
586 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
587 fill with zeros or sign bits as appropriate.
589 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
590 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
591 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
592 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
595 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
596 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
597 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
598 function wants to calculate it itself.
600 Return true if the shift could be successfully synthesized. */
603 expand_doubleword_shift (scalar_int_mode op1_mode
, optab binoptab
,
604 rtx outof_input
, rtx into_input
, rtx op1
,
605 rtx outof_target
, rtx into_target
,
606 int unsignedp
, enum optab_methods methods
,
607 unsigned HOST_WIDE_INT shift_mask
)
609 rtx superword_op1
, tmp
, cmp1
, cmp2
;
610 enum rtx_code cmp_code
;
612 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
613 fill the result with sign or zero bits as appropriate. If so, the value
614 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
615 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
616 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
618 This isn't worthwhile for constant shifts since the optimizers will
619 cope better with in-range shift counts. */
620 if (shift_mask
>= BITS_PER_WORD
622 && !CONSTANT_P (op1
))
624 if (!expand_doubleword_shift (op1_mode
, binoptab
,
625 outof_input
, into_input
, op1
,
627 unsignedp
, methods
, shift_mask
))
629 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
630 outof_target
, unsignedp
, methods
))
635 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
636 is true when the effective shift value is less than BITS_PER_WORD.
637 Set SUPERWORD_OP1 to the shift count that should be used to shift
638 OUTOF_INPUT into INTO_TARGET when the condition is false. */
639 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
, op1_mode
), op1_mode
);
640 if (!CONSTANT_P (op1
) && shift_mask
== BITS_PER_WORD
- 1)
642 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
643 is a subword shift count. */
644 cmp1
= simplify_expand_binop (op1_mode
, and_optab
, op1
, tmp
,
646 cmp2
= CONST0_RTX (op1_mode
);
652 /* Set CMP1 to OP1 - BITS_PER_WORD. */
653 cmp1
= simplify_expand_binop (op1_mode
, sub_optab
, op1
, tmp
,
655 cmp2
= CONST0_RTX (op1_mode
);
657 superword_op1
= cmp1
;
662 /* If we can compute the condition at compile time, pick the
663 appropriate subroutine. */
664 tmp
= simplify_relational_operation (cmp_code
, SImode
, op1_mode
, cmp1
, cmp2
);
665 if (tmp
!= 0 && CONST_INT_P (tmp
))
667 if (tmp
== const0_rtx
)
668 return expand_superword_shift (binoptab
, outof_input
, superword_op1
,
669 outof_target
, into_target
,
672 return expand_subword_shift (op1_mode
, binoptab
,
673 outof_input
, into_input
, op1
,
674 outof_target
, into_target
,
675 unsignedp
, methods
, shift_mask
);
678 /* Try using conditional moves to generate straight-line code. */
679 if (HAVE_conditional_move
)
681 rtx_insn
*start
= get_last_insn ();
682 if (expand_doubleword_shift_condmove (op1_mode
, binoptab
,
683 cmp_code
, cmp1
, cmp2
,
684 outof_input
, into_input
,
686 outof_target
, into_target
,
687 unsignedp
, methods
, shift_mask
))
689 delete_insns_since (start
);
692 /* As a last resort, use branches to select the correct alternative. */
693 rtx_code_label
*subword_label
= gen_label_rtx ();
694 rtx_code_label
*done_label
= gen_label_rtx ();
697 do_compare_rtx_and_jump (cmp1
, cmp2
, cmp_code
, false, op1_mode
,
699 profile_probability::uninitialized ());
702 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
703 outof_target
, into_target
,
707 emit_jump_insn (targetm
.gen_jump (done_label
));
709 emit_label (subword_label
);
711 if (!expand_subword_shift (op1_mode
, binoptab
,
712 outof_input
, into_input
, op1
,
713 outof_target
, into_target
,
714 unsignedp
, methods
, shift_mask
))
717 emit_label (done_label
);
721 /* Subroutine of expand_binop. Perform a double word multiplication of
722 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
723 as the target's word_mode. This function return NULL_RTX if anything
724 goes wrong, in which case it may have already emitted instructions
725 which need to be deleted.
727 If we want to multiply two two-word values and have normal and widening
728 multiplies of single-word values, we can do this with three smaller
731 The multiplication proceeds as follows:
732 _______________________
733 [__op0_high_|__op0_low__]
734 _______________________
735 * [__op1_high_|__op1_low__]
736 _______________________________________________
737 _______________________
738 (1) [__op0_low__*__op1_low__]
739 _______________________
740 (2a) [__op0_low__*__op1_high_]
741 _______________________
742 (2b) [__op0_high_*__op1_low__]
743 _______________________
744 (3) [__op0_high_*__op1_high_]
747 This gives a 4-word result. Since we are only interested in the
748 lower 2 words, partial result (3) and the upper words of (2a) and
749 (2b) don't need to be calculated. Hence (2a) and (2b) can be
750 calculated using non-widening multiplication.
752 (1), however, needs to be calculated with an unsigned widening
753 multiplication. If this operation is not directly supported we
754 try using a signed widening multiplication and adjust the result.
755 This adjustment works as follows:
757 If both operands are positive then no adjustment is needed.
759 If the operands have different signs, for example op0_low < 0 and
760 op1_low >= 0, the instruction treats the most significant bit of
761 op0_low as a sign bit instead of a bit with significance
762 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
763 with 2**BITS_PER_WORD - op0_low, and two's complements the
764 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
767 Similarly, if both operands are negative, we need to add
768 (op0_low + op1_low) * 2**BITS_PER_WORD.
770 We use a trick to adjust quickly. We logically shift op0_low right
771 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
772 op0_high (op1_high) before it is used to calculate 2b (2a). If no
773 logical shift exists, we do an arithmetic right shift and subtract
777 expand_doubleword_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
778 bool umulp
, enum optab_methods methods
)
780 int low
= (WORDS_BIG_ENDIAN
? 1 : 0);
781 int high
= (WORDS_BIG_ENDIAN
? 0 : 1);
782 rtx wordm1
= umulp
? NULL_RTX
: GEN_INT (BITS_PER_WORD
- 1);
783 rtx product
, adjust
, product_high
, temp
;
785 rtx op0_high
= operand_subword_force (op0
, high
, mode
);
786 rtx op0_low
= operand_subword_force (op0
, low
, mode
);
787 rtx op1_high
= operand_subword_force (op1
, high
, mode
);
788 rtx op1_low
= operand_subword_force (op1
, low
, mode
);
790 /* If we're using an unsigned multiply to directly compute the product
791 of the low-order words of the operands and perform any required
792 adjustments of the operands, we begin by trying two more multiplications
793 and then computing the appropriate sum.
795 We have checked above that the required addition is provided.
796 Full-word addition will normally always succeed, especially if
797 it is provided at all, so we don't worry about its failure. The
798 multiplication may well fail, however, so we do handle that. */
802 /* ??? This could be done with emit_store_flag where available. */
803 temp
= expand_binop (word_mode
, lshr_optab
, op0_low
, wordm1
,
804 NULL_RTX
, 1, methods
);
806 op0_high
= expand_binop (word_mode
, add_optab
, op0_high
, temp
,
807 NULL_RTX
, 0, OPTAB_DIRECT
);
810 temp
= expand_binop (word_mode
, ashr_optab
, op0_low
, wordm1
,
811 NULL_RTX
, 0, methods
);
814 op0_high
= expand_binop (word_mode
, sub_optab
, op0_high
, temp
,
815 NULL_RTX
, 0, OPTAB_DIRECT
);
822 adjust
= expand_binop (word_mode
, smul_optab
, op0_high
, op1_low
,
823 NULL_RTX
, 0, OPTAB_DIRECT
);
827 /* OP0_HIGH should now be dead. */
831 /* ??? This could be done with emit_store_flag where available. */
832 temp
= expand_binop (word_mode
, lshr_optab
, op1_low
, wordm1
,
833 NULL_RTX
, 1, methods
);
835 op1_high
= expand_binop (word_mode
, add_optab
, op1_high
, temp
,
836 NULL_RTX
, 0, OPTAB_DIRECT
);
839 temp
= expand_binop (word_mode
, ashr_optab
, op1_low
, wordm1
,
840 NULL_RTX
, 0, methods
);
843 op1_high
= expand_binop (word_mode
, sub_optab
, op1_high
, temp
,
844 NULL_RTX
, 0, OPTAB_DIRECT
);
851 temp
= expand_binop (word_mode
, smul_optab
, op1_high
, op0_low
,
852 NULL_RTX
, 0, OPTAB_DIRECT
);
856 /* OP1_HIGH should now be dead. */
858 adjust
= expand_binop (word_mode
, add_optab
, adjust
, temp
,
859 NULL_RTX
, 0, OPTAB_DIRECT
);
861 if (target
&& !REG_P (target
))
865 product
= expand_binop (mode
, umul_widen_optab
, op0_low
, op1_low
,
866 target
, 1, OPTAB_DIRECT
);
868 product
= expand_binop (mode
, smul_widen_optab
, op0_low
, op1_low
,
869 target
, 1, OPTAB_DIRECT
);
874 product_high
= operand_subword (product
, high
, 1, mode
);
875 adjust
= expand_binop (word_mode
, add_optab
, product_high
, adjust
,
876 NULL_RTX
, 0, OPTAB_DIRECT
);
877 emit_move_insn (product_high
, adjust
);
881 /* Wrapper around expand_binop which takes an rtx code to specify
882 the operation to perform, not an optab pointer. All other
883 arguments are the same. */
885 expand_simple_binop (machine_mode mode
, enum rtx_code code
, rtx op0
,
886 rtx op1
, rtx target
, int unsignedp
,
887 enum optab_methods methods
)
889 optab binop
= code_to_optab (code
);
892 return expand_binop (mode
, binop
, op0
, op1
, target
, unsignedp
, methods
);
895 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
896 binop. Order them according to commutative_operand_precedence and, if
897 possible, try to put TARGET or a pseudo first. */
899 swap_commutative_operands_with_target (rtx target
, rtx op0
, rtx op1
)
901 int op0_prec
= commutative_operand_precedence (op0
);
902 int op1_prec
= commutative_operand_precedence (op1
);
904 if (op0_prec
< op1_prec
)
907 if (op0_prec
> op1_prec
)
910 /* With equal precedence, both orders are ok, but it is better if the
911 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
912 if (target
== 0 || REG_P (target
))
913 return (REG_P (op1
) && !REG_P (op0
)) || target
== op1
;
915 return rtx_equal_p (op1
, target
);
918 /* Return true if BINOPTAB implements a shift operation. */
921 shift_optab_p (optab binoptab
)
923 switch (optab_to_code (binoptab
))
939 /* Return true if BINOPTAB implements a commutative binary operation. */
942 commutative_optab_p (optab binoptab
)
944 return (GET_RTX_CLASS (optab_to_code (binoptab
)) == RTX_COMM_ARITH
945 || binoptab
== smul_widen_optab
946 || binoptab
== umul_widen_optab
947 || binoptab
== smul_highpart_optab
948 || binoptab
== umul_highpart_optab
);
951 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
952 optimizing, and if the operand is a constant that costs more than
953 1 instruction, force the constant into a register and return that
954 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
957 avoid_expensive_constant (machine_mode mode
, optab binoptab
,
958 int opn
, rtx x
, bool unsignedp
)
960 bool speed
= optimize_insn_for_speed_p ();
965 && (rtx_cost (x
, mode
, optab_to_code (binoptab
), opn
, speed
)
966 > set_src_cost (x
, mode
, speed
)))
970 HOST_WIDE_INT intval
= trunc_int_for_mode (INTVAL (x
), mode
);
971 if (intval
!= INTVAL (x
))
972 x
= GEN_INT (intval
);
975 x
= convert_modes (mode
, VOIDmode
, x
, unsignedp
);
976 x
= force_reg (mode
, x
);
981 /* Helper function for expand_binop: handle the case where there
982 is an insn that directly implements the indicated operation.
983 Returns null if this is not possible. */
985 expand_binop_directly (machine_mode mode
, optab binoptab
,
987 rtx target
, int unsignedp
, enum optab_methods methods
,
990 machine_mode from_mode
= widened_mode (mode
, op0
, op1
);
991 enum insn_code icode
= find_widening_optab_handler (binoptab
, mode
,
993 machine_mode xmode0
= insn_data
[(int) icode
].operand
[1].mode
;
994 machine_mode xmode1
= insn_data
[(int) icode
].operand
[2].mode
;
995 machine_mode mode0
, mode1
, tmp_mode
;
996 struct expand_operand ops
[3];
999 rtx xop0
= op0
, xop1
= op1
;
1000 bool canonicalize_op1
= false;
1002 /* If it is a commutative operator and the modes would match
1003 if we would swap the operands, we can save the conversions. */
1004 commutative_p
= commutative_optab_p (binoptab
);
1006 && GET_MODE (xop0
) != xmode0
&& GET_MODE (xop1
) != xmode1
1007 && GET_MODE (xop0
) == xmode1
&& GET_MODE (xop1
) == xmode1
)
1008 std::swap (xop0
, xop1
);
1010 /* If we are optimizing, force expensive constants into a register. */
1011 xop0
= avoid_expensive_constant (xmode0
, binoptab
, 0, xop0
, unsignedp
);
1012 if (!shift_optab_p (binoptab
))
1013 xop1
= avoid_expensive_constant (xmode1
, binoptab
, 1, xop1
, unsignedp
);
1015 /* Shifts and rotates often use a different mode for op1 from op0;
1016 for VOIDmode constants we don't know the mode, so force it
1017 to be canonicalized using convert_modes. */
1018 canonicalize_op1
= true;
1020 /* In case the insn wants input operands in modes different from
1021 those of the actual operands, convert the operands. It would
1022 seem that we don't need to convert CONST_INTs, but we do, so
1023 that they're properly zero-extended, sign-extended or truncated
1026 mode0
= GET_MODE (xop0
) != VOIDmode
? GET_MODE (xop0
) : mode
;
1027 if (xmode0
!= VOIDmode
&& xmode0
!= mode0
)
1029 xop0
= convert_modes (xmode0
, mode0
, xop0
, unsignedp
);
1033 mode1
= ((GET_MODE (xop1
) != VOIDmode
|| canonicalize_op1
)
1034 ? GET_MODE (xop1
) : mode
);
1035 if (xmode1
!= VOIDmode
&& xmode1
!= mode1
)
1037 xop1
= convert_modes (xmode1
, mode1
, xop1
, unsignedp
);
1041 /* If operation is commutative,
1042 try to make the first operand a register.
1043 Even better, try to make it the same as the target.
1044 Also try to make the last operand a constant. */
1046 && swap_commutative_operands_with_target (target
, xop0
, xop1
))
1047 std::swap (xop0
, xop1
);
1049 /* Now, if insn's predicates don't allow our operands, put them into
1052 if (binoptab
== vec_pack_trunc_optab
1053 || binoptab
== vec_pack_usat_optab
1054 || binoptab
== vec_pack_ssat_optab
1055 || binoptab
== vec_pack_ufix_trunc_optab
1056 || binoptab
== vec_pack_sfix_trunc_optab
)
1058 /* The mode of the result is different then the mode of the
1060 tmp_mode
= insn_data
[(int) icode
].operand
[0].mode
;
1061 if (VECTOR_MODE_P (mode
)
1062 && GET_MODE_NUNITS (tmp_mode
) != 2 * GET_MODE_NUNITS (mode
))
1064 delete_insns_since (last
);
1071 create_output_operand (&ops
[0], target
, tmp_mode
);
1072 create_input_operand (&ops
[1], xop0
, mode0
);
1073 create_input_operand (&ops
[2], xop1
, mode1
);
1074 pat
= maybe_gen_insn (icode
, 3, ops
);
1077 /* If PAT is composed of more than one insn, try to add an appropriate
1078 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1079 operand, call expand_binop again, this time without a target. */
1080 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
1081 && ! add_equal_note (pat
, ops
[0].value
,
1082 optab_to_code (binoptab
),
1083 ops
[1].value
, ops
[2].value
))
1085 delete_insns_since (last
);
1086 return expand_binop (mode
, binoptab
, op0
, op1
, NULL_RTX
,
1087 unsignedp
, methods
);
1091 return ops
[0].value
;
1093 delete_insns_since (last
);
1097 /* Generate code to perform an operation specified by BINOPTAB
1098 on operands OP0 and OP1, with result having machine-mode MODE.
1100 UNSIGNEDP is for the case where we have to widen the operands
1101 to perform the operation. It says to use zero-extension.
1103 If TARGET is nonzero, the value
1104 is generated there, if it is convenient to do so.
1105 In all cases an rtx is returned for the locus of the value;
1106 this may or may not be TARGET. */
1109 expand_binop (machine_mode mode
, optab binoptab
, rtx op0
, rtx op1
,
1110 rtx target
, int unsignedp
, enum optab_methods methods
)
1112 enum optab_methods next_methods
1113 = (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
1114 ? OPTAB_WIDEN
: methods
);
1115 enum mode_class mclass
;
1116 machine_mode wider_mode
;
1117 scalar_int_mode int_mode
;
1120 rtx_insn
*entry_last
= get_last_insn ();
1123 mclass
= GET_MODE_CLASS (mode
);
1125 /* If subtracting an integer constant, convert this into an addition of
1126 the negated constant. */
1128 if (binoptab
== sub_optab
&& CONST_INT_P (op1
))
1130 op1
= negate_rtx (mode
, op1
);
1131 binoptab
= add_optab
;
1133 /* For shifts, constant invalid op1 might be expanded from different
1134 mode than MODE. As those are invalid, force them to a register
1135 to avoid further problems during expansion. */
1136 else if (CONST_INT_P (op1
)
1137 && shift_optab_p (binoptab
)
1138 && UINTVAL (op1
) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode
)))
1140 op1
= gen_int_mode (INTVAL (op1
), GET_MODE_INNER (mode
));
1141 op1
= force_reg (GET_MODE_INNER (mode
), op1
);
1144 /* Record where to delete back to if we backtrack. */
1145 last
= get_last_insn ();
1147 /* If we can do it with a three-operand insn, do so. */
1149 if (methods
!= OPTAB_MUST_WIDEN
1150 && find_widening_optab_handler (binoptab
, mode
,
1151 widened_mode (mode
, op0
, op1
), 1)
1152 != CODE_FOR_nothing
)
1154 temp
= expand_binop_directly (mode
, binoptab
, op0
, op1
, target
,
1155 unsignedp
, methods
, last
);
1160 /* If we were trying to rotate, and that didn't work, try rotating
1161 the other direction before falling back to shifts and bitwise-or. */
1162 if (((binoptab
== rotl_optab
1163 && optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
1164 || (binoptab
== rotr_optab
1165 && optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
))
1166 && is_int_mode (mode
, &int_mode
))
1168 optab otheroptab
= (binoptab
== rotl_optab
? rotr_optab
: rotl_optab
);
1170 unsigned int bits
= GET_MODE_PRECISION (int_mode
);
1172 if (CONST_INT_P (op1
))
1173 newop1
= GEN_INT (bits
- INTVAL (op1
));
1174 else if (targetm
.shift_truncation_mask (int_mode
) == bits
- 1)
1175 newop1
= negate_rtx (GET_MODE (op1
), op1
);
1177 newop1
= expand_binop (GET_MODE (op1
), sub_optab
,
1178 gen_int_mode (bits
, GET_MODE (op1
)), op1
,
1179 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1181 temp
= expand_binop_directly (int_mode
, otheroptab
, op0
, newop1
,
1182 target
, unsignedp
, methods
, last
);
1187 /* If this is a multiply, see if we can do a widening operation that
1188 takes operands of this mode and makes a wider mode. */
1190 if (binoptab
== smul_optab
1191 && GET_MODE_2XWIDER_MODE (mode
).exists (&wider_mode
)
1192 && (convert_optab_handler ((unsignedp
1194 : smul_widen_optab
),
1195 wider_mode
, mode
) != CODE_FOR_nothing
))
1197 temp
= expand_binop (wider_mode
,
1198 unsignedp
? umul_widen_optab
: smul_widen_optab
,
1199 op0
, op1
, NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1203 if (GET_MODE_CLASS (mode
) == MODE_INT
1204 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, GET_MODE (temp
)))
1205 return gen_lowpart (mode
, temp
);
1207 return convert_to_mode (mode
, temp
, unsignedp
);
1211 /* If this is a vector shift by a scalar, see if we can do a vector
1212 shift by a vector. If so, broadcast the scalar into a vector. */
1213 if (mclass
== MODE_VECTOR_INT
)
1215 optab otheroptab
= unknown_optab
;
1217 if (binoptab
== ashl_optab
)
1218 otheroptab
= vashl_optab
;
1219 else if (binoptab
== ashr_optab
)
1220 otheroptab
= vashr_optab
;
1221 else if (binoptab
== lshr_optab
)
1222 otheroptab
= vlshr_optab
;
1223 else if (binoptab
== rotl_optab
)
1224 otheroptab
= vrotl_optab
;
1225 else if (binoptab
== rotr_optab
)
1226 otheroptab
= vrotr_optab
;
1228 if (otheroptab
&& optab_handler (otheroptab
, mode
) != CODE_FOR_nothing
)
1230 /* The scalar may have been extended to be too wide. Truncate
1231 it back to the proper size to fit in the broadcast vector. */
1232 scalar_mode inner_mode
= GET_MODE_INNER (mode
);
1233 if (!CONST_INT_P (op1
)
1234 && (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (op1
)))
1235 > GET_MODE_BITSIZE (inner_mode
)))
1236 op1
= force_reg (inner_mode
,
1237 simplify_gen_unary (TRUNCATE
, inner_mode
, op1
,
1239 rtx vop1
= expand_vector_broadcast (mode
, op1
);
1242 temp
= expand_binop_directly (mode
, otheroptab
, op0
, vop1
,
1243 target
, unsignedp
, methods
, last
);
1250 /* Look for a wider mode of the same class for which we think we
1251 can open-code the operation. Check for a widening multiply at the
1252 wider mode as well. */
1254 if (CLASS_HAS_WIDER_MODES_P (mclass
)
1255 && methods
!= OPTAB_DIRECT
&& methods
!= OPTAB_LIB
)
1256 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1258 machine_mode next_mode
;
1259 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
1260 || (binoptab
== smul_optab
1261 && GET_MODE_WIDER_MODE (wider_mode
).exists (&next_mode
)
1262 && (find_widening_optab_handler ((unsignedp
1264 : smul_widen_optab
),
1266 != CODE_FOR_nothing
)))
1268 rtx xop0
= op0
, xop1
= op1
;
1271 /* For certain integer operations, we need not actually extend
1272 the narrow operands, as long as we will truncate
1273 the results to the same narrowness. */
1275 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1276 || binoptab
== xor_optab
1277 || binoptab
== add_optab
|| binoptab
== sub_optab
1278 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1279 && mclass
== MODE_INT
)
1282 xop0
= avoid_expensive_constant (mode
, binoptab
, 0,
1284 if (binoptab
!= ashl_optab
)
1285 xop1
= avoid_expensive_constant (mode
, binoptab
, 1,
1289 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
, no_extend
);
1291 /* The second operand of a shift must always be extended. */
1292 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1293 no_extend
&& binoptab
!= ashl_optab
);
1295 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1296 unsignedp
, OPTAB_DIRECT
);
1299 if (mclass
!= MODE_INT
1300 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1303 target
= gen_reg_rtx (mode
);
1304 convert_move (target
, temp
, 0);
1308 return gen_lowpart (mode
, temp
);
1311 delete_insns_since (last
);
1315 /* If operation is commutative,
1316 try to make the first operand a register.
1317 Even better, try to make it the same as the target.
1318 Also try to make the last operand a constant. */
1319 if (commutative_optab_p (binoptab
)
1320 && swap_commutative_operands_with_target (target
, op0
, op1
))
1321 std::swap (op0
, op1
);
1323 /* These can be done a word at a time. */
1324 if ((binoptab
== and_optab
|| binoptab
== ior_optab
|| binoptab
== xor_optab
)
1325 && is_int_mode (mode
, &int_mode
)
1326 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
1327 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1332 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1333 won't be accurate, so use a new target. */
1337 || !valid_multiword_target_p (target
))
1338 target
= gen_reg_rtx (int_mode
);
1342 /* Do the actual arithmetic. */
1343 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
1345 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
1346 rtx x
= expand_binop (word_mode
, binoptab
,
1347 operand_subword_force (op0
, i
, int_mode
),
1348 operand_subword_force (op1
, i
, int_mode
),
1349 target_piece
, unsignedp
, next_methods
);
1354 if (target_piece
!= x
)
1355 emit_move_insn (target_piece
, x
);
1358 insns
= get_insns ();
1361 if (i
== GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
)
1368 /* Synthesize double word shifts from single word shifts. */
1369 if ((binoptab
== lshr_optab
|| binoptab
== ashl_optab
1370 || binoptab
== ashr_optab
)
1371 && is_int_mode (mode
, &int_mode
)
1372 && (CONST_INT_P (op1
) || optimize_insn_for_speed_p ())
1373 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1374 && GET_MODE_PRECISION (int_mode
) == GET_MODE_BITSIZE (int_mode
)
1375 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
1376 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1377 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1379 unsigned HOST_WIDE_INT shift_mask
, double_shift_mask
;
1380 scalar_int_mode op1_mode
;
1382 double_shift_mask
= targetm
.shift_truncation_mask (int_mode
);
1383 shift_mask
= targetm
.shift_truncation_mask (word_mode
);
1384 op1_mode
= (GET_MODE (op1
) != VOIDmode
1385 ? as_a
<scalar_int_mode
> (GET_MODE (op1
))
1388 /* Apply the truncation to constant shifts. */
1389 if (double_shift_mask
> 0 && CONST_INT_P (op1
))
1390 op1
= GEN_INT (INTVAL (op1
) & double_shift_mask
);
1392 if (op1
== CONST0_RTX (op1_mode
))
1395 /* Make sure that this is a combination that expand_doubleword_shift
1396 can handle. See the comments there for details. */
1397 if (double_shift_mask
== 0
1398 || (shift_mask
== BITS_PER_WORD
- 1
1399 && double_shift_mask
== BITS_PER_WORD
* 2 - 1))
1402 rtx into_target
, outof_target
;
1403 rtx into_input
, outof_input
;
1404 int left_shift
, outof_word
;
1406 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1407 won't be accurate, so use a new target. */
1411 || !valid_multiword_target_p (target
))
1412 target
= gen_reg_rtx (int_mode
);
1416 /* OUTOF_* is the word we are shifting bits away from, and
1417 INTO_* is the word that we are shifting bits towards, thus
1418 they differ depending on the direction of the shift and
1419 WORDS_BIG_ENDIAN. */
1421 left_shift
= binoptab
== ashl_optab
;
1422 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1424 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1425 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1427 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1428 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1430 if (expand_doubleword_shift (op1_mode
, binoptab
,
1431 outof_input
, into_input
, op1
,
1432 outof_target
, into_target
,
1433 unsignedp
, next_methods
, shift_mask
))
1435 insns
= get_insns ();
1445 /* Synthesize double word rotates from single word shifts. */
1446 if ((binoptab
== rotl_optab
|| binoptab
== rotr_optab
)
1447 && is_int_mode (mode
, &int_mode
)
1448 && CONST_INT_P (op1
)
1449 && GET_MODE_PRECISION (int_mode
) == 2 * BITS_PER_WORD
1450 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1451 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1454 rtx into_target
, outof_target
;
1455 rtx into_input
, outof_input
;
1457 int shift_count
, left_shift
, outof_word
;
1459 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1460 won't be accurate, so use a new target. Do this also if target is not
1461 a REG, first because having a register instead may open optimization
1462 opportunities, and second because if target and op0 happen to be MEMs
1463 designating the same location, we would risk clobbering it too early
1464 in the code sequence we generate below. */
1469 || !valid_multiword_target_p (target
))
1470 target
= gen_reg_rtx (int_mode
);
1474 shift_count
= INTVAL (op1
);
1476 /* OUTOF_* is the word we are shifting bits away from, and
1477 INTO_* is the word that we are shifting bits towards, thus
1478 they differ depending on the direction of the shift and
1479 WORDS_BIG_ENDIAN. */
1481 left_shift
= (binoptab
== rotl_optab
);
1482 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1484 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1485 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1487 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1488 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1490 if (shift_count
== BITS_PER_WORD
)
1492 /* This is just a word swap. */
1493 emit_move_insn (outof_target
, into_input
);
1494 emit_move_insn (into_target
, outof_input
);
1499 rtx into_temp1
, into_temp2
, outof_temp1
, outof_temp2
;
1500 rtx first_shift_count
, second_shift_count
;
1501 optab reverse_unsigned_shift
, unsigned_shift
;
1503 reverse_unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1504 ? lshr_optab
: ashl_optab
);
1506 unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1507 ? ashl_optab
: lshr_optab
);
1509 if (shift_count
> BITS_PER_WORD
)
1511 first_shift_count
= GEN_INT (shift_count
- BITS_PER_WORD
);
1512 second_shift_count
= GEN_INT (2 * BITS_PER_WORD
- shift_count
);
1516 first_shift_count
= GEN_INT (BITS_PER_WORD
- shift_count
);
1517 second_shift_count
= GEN_INT (shift_count
);
1520 into_temp1
= expand_binop (word_mode
, unsigned_shift
,
1521 outof_input
, first_shift_count
,
1522 NULL_RTX
, unsignedp
, next_methods
);
1523 into_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1524 into_input
, second_shift_count
,
1525 NULL_RTX
, unsignedp
, next_methods
);
1527 if (into_temp1
!= 0 && into_temp2
!= 0)
1528 inter
= expand_binop (word_mode
, ior_optab
, into_temp1
, into_temp2
,
1529 into_target
, unsignedp
, next_methods
);
1533 if (inter
!= 0 && inter
!= into_target
)
1534 emit_move_insn (into_target
, inter
);
1536 outof_temp1
= expand_binop (word_mode
, unsigned_shift
,
1537 into_input
, first_shift_count
,
1538 NULL_RTX
, unsignedp
, next_methods
);
1539 outof_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1540 outof_input
, second_shift_count
,
1541 NULL_RTX
, unsignedp
, next_methods
);
1543 if (inter
!= 0 && outof_temp1
!= 0 && outof_temp2
!= 0)
1544 inter
= expand_binop (word_mode
, ior_optab
,
1545 outof_temp1
, outof_temp2
,
1546 outof_target
, unsignedp
, next_methods
);
1548 if (inter
!= 0 && inter
!= outof_target
)
1549 emit_move_insn (outof_target
, inter
);
1552 insns
= get_insns ();
1562 /* These can be done a word at a time by propagating carries. */
1563 if ((binoptab
== add_optab
|| binoptab
== sub_optab
)
1564 && is_int_mode (mode
, &int_mode
)
1565 && GET_MODE_SIZE (int_mode
) >= 2 * UNITS_PER_WORD
1566 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1569 optab otheroptab
= binoptab
== add_optab
? sub_optab
: add_optab
;
1570 const unsigned int nwords
= GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
;
1571 rtx carry_in
= NULL_RTX
, carry_out
= NULL_RTX
;
1572 rtx xop0
, xop1
, xtarget
;
1574 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1575 value is one of those, use it. Otherwise, use 1 since it is the
1576 one easiest to get. */
1577 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1578 int normalizep
= STORE_FLAG_VALUE
;
1583 /* Prepare the operands. */
1584 xop0
= force_reg (int_mode
, op0
);
1585 xop1
= force_reg (int_mode
, op1
);
1587 xtarget
= gen_reg_rtx (int_mode
);
1589 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
1592 /* Indicate for flow that the entire target reg is being set. */
1594 emit_clobber (xtarget
);
1596 /* Do the actual arithmetic. */
1597 for (i
= 0; i
< nwords
; i
++)
1599 int index
= (WORDS_BIG_ENDIAN
? nwords
- i
- 1 : i
);
1600 rtx target_piece
= operand_subword (xtarget
, index
, 1, int_mode
);
1601 rtx op0_piece
= operand_subword_force (xop0
, index
, int_mode
);
1602 rtx op1_piece
= operand_subword_force (xop1
, index
, int_mode
);
1605 /* Main add/subtract of the input operands. */
1606 x
= expand_binop (word_mode
, binoptab
,
1607 op0_piece
, op1_piece
,
1608 target_piece
, unsignedp
, next_methods
);
1614 /* Store carry from main add/subtract. */
1615 carry_out
= gen_reg_rtx (word_mode
);
1616 carry_out
= emit_store_flag_force (carry_out
,
1617 (binoptab
== add_optab
1620 word_mode
, 1, normalizep
);
1627 /* Add/subtract previous carry to main result. */
1628 newx
= expand_binop (word_mode
,
1629 normalizep
== 1 ? binoptab
: otheroptab
,
1631 NULL_RTX
, 1, next_methods
);
1635 /* Get out carry from adding/subtracting carry in. */
1636 rtx carry_tmp
= gen_reg_rtx (word_mode
);
1637 carry_tmp
= emit_store_flag_force (carry_tmp
,
1638 (binoptab
== add_optab
1641 word_mode
, 1, normalizep
);
1643 /* Logical-ior the two poss. carry together. */
1644 carry_out
= expand_binop (word_mode
, ior_optab
,
1645 carry_out
, carry_tmp
,
1646 carry_out
, 0, next_methods
);
1650 emit_move_insn (target_piece
, newx
);
1654 if (x
!= target_piece
)
1655 emit_move_insn (target_piece
, x
);
1658 carry_in
= carry_out
;
1661 if (i
== GET_MODE_BITSIZE (int_mode
) / (unsigned) BITS_PER_WORD
)
1663 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
1664 || ! rtx_equal_p (target
, xtarget
))
1666 rtx_insn
*temp
= emit_move_insn (target
, xtarget
);
1668 set_dst_reg_note (temp
, REG_EQUAL
,
1669 gen_rtx_fmt_ee (optab_to_code (binoptab
),
1670 int_mode
, copy_rtx (xop0
),
1681 delete_insns_since (last
);
1684 /* Attempt to synthesize double word multiplies using a sequence of word
1685 mode multiplications. We first attempt to generate a sequence using a
1686 more efficient unsigned widening multiply, and if that fails we then
1687 try using a signed widening multiply. */
1689 if (binoptab
== smul_optab
1690 && is_int_mode (mode
, &int_mode
)
1691 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1692 && optab_handler (smul_optab
, word_mode
) != CODE_FOR_nothing
1693 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
)
1695 rtx product
= NULL_RTX
;
1696 if (widening_optab_handler (umul_widen_optab
, int_mode
, word_mode
)
1697 != CODE_FOR_nothing
)
1699 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
1702 delete_insns_since (last
);
1705 if (product
== NULL_RTX
1706 && (widening_optab_handler (smul_widen_optab
, int_mode
, word_mode
)
1707 != CODE_FOR_nothing
))
1709 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
1712 delete_insns_since (last
);
1715 if (product
!= NULL_RTX
)
1717 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
1719 rtx_insn
*move
= emit_move_insn (target
? target
: product
,
1721 set_dst_reg_note (move
,
1723 gen_rtx_fmt_ee (MULT
, int_mode
,
1726 target
? target
: product
);
1732 /* It can't be open-coded in this mode.
1733 Use a library call if one is available and caller says that's ok. */
1735 libfunc
= optab_libfunc (binoptab
, mode
);
1737 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
1741 machine_mode op1_mode
= mode
;
1746 if (shift_optab_p (binoptab
))
1748 op1_mode
= targetm
.libgcc_shift_count_mode ();
1749 /* Specify unsigned here,
1750 since negative shift counts are meaningless. */
1751 op1x
= convert_to_mode (op1_mode
, op1
, 1);
1754 if (GET_MODE (op0
) != VOIDmode
1755 && GET_MODE (op0
) != mode
)
1756 op0
= convert_to_mode (mode
, op0
, unsignedp
);
1758 /* Pass 1 for NO_QUEUE so we don't lose any increments
1759 if the libcall is cse'd or moved. */
1760 value
= emit_library_call_value (libfunc
,
1761 NULL_RTX
, LCT_CONST
, mode
,
1762 op0
, mode
, op1x
, op1_mode
);
1764 insns
= get_insns ();
1767 bool trapv
= trapv_binoptab_p (binoptab
);
1768 target
= gen_reg_rtx (mode
);
1769 emit_libcall_block_1 (insns
, target
, value
,
1771 : gen_rtx_fmt_ee (optab_to_code (binoptab
),
1772 mode
, op0
, op1
), trapv
);
1777 delete_insns_since (last
);
1779 /* It can't be done in this mode. Can we do it in a wider mode? */
1781 if (! (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
1782 || methods
== OPTAB_MUST_WIDEN
))
1784 /* Caller says, don't even try. */
1785 delete_insns_since (entry_last
);
1789 /* Compute the value of METHODS to pass to recursive calls.
1790 Don't allow widening to be tried recursively. */
1792 methods
= (methods
== OPTAB_LIB_WIDEN
? OPTAB_LIB
: OPTAB_DIRECT
);
1794 /* Look for a wider mode of the same class for which it appears we can do
1797 if (CLASS_HAS_WIDER_MODES_P (mclass
))
1799 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1801 if (find_widening_optab_handler (binoptab
, wider_mode
, mode
, 1)
1803 || (methods
== OPTAB_LIB
1804 && optab_libfunc (binoptab
, wider_mode
)))
1806 rtx xop0
= op0
, xop1
= op1
;
1809 /* For certain integer operations, we need not actually extend
1810 the narrow operands, as long as we will truncate
1811 the results to the same narrowness. */
1813 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1814 || binoptab
== xor_optab
1815 || binoptab
== add_optab
|| binoptab
== sub_optab
1816 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1817 && mclass
== MODE_INT
)
1820 xop0
= widen_operand (xop0
, wider_mode
, mode
,
1821 unsignedp
, no_extend
);
1823 /* The second operand of a shift must always be extended. */
1824 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1825 no_extend
&& binoptab
!= ashl_optab
);
1827 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1828 unsignedp
, methods
);
1831 if (mclass
!= MODE_INT
1832 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1835 target
= gen_reg_rtx (mode
);
1836 convert_move (target
, temp
, 0);
1840 return gen_lowpart (mode
, temp
);
1843 delete_insns_since (last
);
1848 delete_insns_since (entry_last
);
1852 /* Expand a binary operator which has both signed and unsigned forms.
1853 UOPTAB is the optab for unsigned operations, and SOPTAB is for
1856 If we widen unsigned operands, we may use a signed wider operation instead
1857 of an unsigned wider operation, since the result would be the same. */
1860 sign_expand_binop (machine_mode mode
, optab uoptab
, optab soptab
,
1861 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
1862 enum optab_methods methods
)
1865 optab direct_optab
= unsignedp
? uoptab
: soptab
;
1868 /* Do it without widening, if possible. */
1869 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
1870 unsignedp
, OPTAB_DIRECT
);
1871 if (temp
|| methods
== OPTAB_DIRECT
)
1874 /* Try widening to a signed int. Disable any direct use of any
1875 signed insn in the current mode. */
1876 save_enable
= swap_optab_enable (soptab
, mode
, false);
1878 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
1879 unsignedp
, OPTAB_WIDEN
);
1881 /* For unsigned operands, try widening to an unsigned int. */
1882 if (!temp
&& unsignedp
)
1883 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
1884 unsignedp
, OPTAB_WIDEN
);
1885 if (temp
|| methods
== OPTAB_WIDEN
)
1888 /* Use the right width libcall if that exists. */
1889 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
1890 unsignedp
, OPTAB_LIB
);
1891 if (temp
|| methods
== OPTAB_LIB
)
1894 /* Must widen and use a libcall, use either signed or unsigned. */
1895 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
1896 unsignedp
, methods
);
1897 if (!temp
&& unsignedp
)
1898 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
1899 unsignedp
, methods
);
1902 /* Undo the fiddling above. */
1904 swap_optab_enable (soptab
, mode
, true);
1908 /* Generate code to perform an operation specified by UNOPPTAB
1909 on operand OP0, with two results to TARG0 and TARG1.
1910 We assume that the order of the operands for the instruction
1911 is TARG0, TARG1, OP0.
1913 Either TARG0 or TARG1 may be zero, but what that means is that
1914 the result is not actually wanted. We will generate it into
1915 a dummy pseudo-reg and discard it. They may not both be zero.
1917 Returns 1 if this operation can be performed; 0 if not. */
1920 expand_twoval_unop (optab unoptab
, rtx op0
, rtx targ0
, rtx targ1
,
1923 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
1924 enum mode_class mclass
;
1925 machine_mode wider_mode
;
1926 rtx_insn
*entry_last
= get_last_insn ();
1929 mclass
= GET_MODE_CLASS (mode
);
1932 targ0
= gen_reg_rtx (mode
);
1934 targ1
= gen_reg_rtx (mode
);
1936 /* Record where to go back to if we fail. */
1937 last
= get_last_insn ();
1939 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
1941 struct expand_operand ops
[3];
1942 enum insn_code icode
= optab_handler (unoptab
, mode
);
1944 create_fixed_operand (&ops
[0], targ0
);
1945 create_fixed_operand (&ops
[1], targ1
);
1946 create_convert_operand_from (&ops
[2], op0
, mode
, unsignedp
);
1947 if (maybe_expand_insn (icode
, 3, ops
))
1951 /* It can't be done in this mode. Can we do it in a wider mode? */
1953 if (CLASS_HAS_WIDER_MODES_P (mclass
))
1955 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1957 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
1959 rtx t0
= gen_reg_rtx (wider_mode
);
1960 rtx t1
= gen_reg_rtx (wider_mode
);
1961 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
1963 if (expand_twoval_unop (unoptab
, cop0
, t0
, t1
, unsignedp
))
1965 convert_move (targ0
, t0
, unsignedp
);
1966 convert_move (targ1
, t1
, unsignedp
);
1970 delete_insns_since (last
);
1975 delete_insns_since (entry_last
);
1979 /* Generate code to perform an operation specified by BINOPTAB
1980 on operands OP0 and OP1, with two results to TARG1 and TARG2.
1981 We assume that the order of the operands for the instruction
1982 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
1983 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
1985 Either TARG0 or TARG1 may be zero, but what that means is that
1986 the result is not actually wanted. We will generate it into
1987 a dummy pseudo-reg and discard it. They may not both be zero.
1989 Returns 1 if this operation can be performed; 0 if not. */
1992 expand_twoval_binop (optab binoptab
, rtx op0
, rtx op1
, rtx targ0
, rtx targ1
,
1995 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
1996 enum mode_class mclass
;
1997 machine_mode wider_mode
;
1998 rtx_insn
*entry_last
= get_last_insn ();
2001 mclass
= GET_MODE_CLASS (mode
);
2004 targ0
= gen_reg_rtx (mode
);
2006 targ1
= gen_reg_rtx (mode
);
2008 /* Record where to go back to if we fail. */
2009 last
= get_last_insn ();
2011 if (optab_handler (binoptab
, mode
) != CODE_FOR_nothing
)
2013 struct expand_operand ops
[4];
2014 enum insn_code icode
= optab_handler (binoptab
, mode
);
2015 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2016 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2017 rtx xop0
= op0
, xop1
= op1
;
2019 /* If we are optimizing, force expensive constants into a register. */
2020 xop0
= avoid_expensive_constant (mode0
, binoptab
, 0, xop0
, unsignedp
);
2021 xop1
= avoid_expensive_constant (mode1
, binoptab
, 1, xop1
, unsignedp
);
2023 create_fixed_operand (&ops
[0], targ0
);
2024 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2025 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
2026 create_fixed_operand (&ops
[3], targ1
);
2027 if (maybe_expand_insn (icode
, 4, ops
))
2029 delete_insns_since (last
);
2032 /* It can't be done in this mode. Can we do it in a wider mode? */
2034 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2036 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2038 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
)
2040 rtx t0
= gen_reg_rtx (wider_mode
);
2041 rtx t1
= gen_reg_rtx (wider_mode
);
2042 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2043 rtx cop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
2045 if (expand_twoval_binop (binoptab
, cop0
, cop1
,
2048 convert_move (targ0
, t0
, unsignedp
);
2049 convert_move (targ1
, t1
, unsignedp
);
2053 delete_insns_since (last
);
2058 delete_insns_since (entry_last
);
2062 /* Expand the two-valued library call indicated by BINOPTAB, but
2063 preserve only one of the values. If TARG0 is non-NULL, the first
2064 value is placed into TARG0; otherwise the second value is placed
2065 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2066 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2067 This routine assumes that the value returned by the library call is
2068 as if the return value was of an integral mode twice as wide as the
2069 mode of OP0. Returns 1 if the call was successful. */
2072 expand_twoval_binop_libfunc (optab binoptab
, rtx op0
, rtx op1
,
2073 rtx targ0
, rtx targ1
, enum rtx_code code
)
2076 machine_mode libval_mode
;
2081 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2082 gcc_assert (!targ0
!= !targ1
);
2084 mode
= GET_MODE (op0
);
2085 libfunc
= optab_libfunc (binoptab
, mode
);
2089 /* The value returned by the library function will have twice as
2090 many bits as the nominal MODE. */
2091 libval_mode
= smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode
));
2093 libval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
2097 /* Get the part of VAL containing the value that we want. */
2098 libval
= simplify_gen_subreg (mode
, libval
, libval_mode
,
2099 targ0
? 0 : GET_MODE_SIZE (mode
));
2100 insns
= get_insns ();
2102 /* Move the into the desired location. */
2103 emit_libcall_block (insns
, targ0
? targ0
: targ1
, libval
,
2104 gen_rtx_fmt_ee (code
, mode
, op0
, op1
));
2110 /* Wrapper around expand_unop which takes an rtx code to specify
2111 the operation to perform, not an optab pointer. All other
2112 arguments are the same. */
2114 expand_simple_unop (machine_mode mode
, enum rtx_code code
, rtx op0
,
2115 rtx target
, int unsignedp
)
2117 optab unop
= code_to_optab (code
);
2120 return expand_unop (mode
, unop
, op0
, target
, unsignedp
);
2126 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2128 A similar operation can be used for clrsb. UNOPTAB says which operation
2129 we are trying to expand. */
2131 widen_leading (scalar_int_mode mode
, rtx op0
, rtx target
, optab unoptab
)
2133 opt_scalar_int_mode wider_mode_iter
;
2134 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2136 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2137 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2142 last
= get_last_insn ();
2145 target
= gen_reg_rtx (mode
);
2146 xop0
= widen_operand (op0
, wider_mode
, mode
,
2147 unoptab
!= clrsb_optab
, false);
2148 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2149 unoptab
!= clrsb_optab
);
2152 (wider_mode
, sub_optab
, temp
,
2153 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
2154 - GET_MODE_PRECISION (mode
),
2156 target
, true, OPTAB_DIRECT
);
2158 delete_insns_since (last
);
2166 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2167 quantities, choosing which based on whether the high word is nonzero. */
2169 expand_doubleword_clz (scalar_int_mode mode
, rtx op0
, rtx target
)
2171 rtx xop0
= force_reg (mode
, op0
);
2172 rtx subhi
= gen_highpart (word_mode
, xop0
);
2173 rtx sublo
= gen_lowpart (word_mode
, xop0
);
2174 rtx_code_label
*hi0_label
= gen_label_rtx ();
2175 rtx_code_label
*after_label
= gen_label_rtx ();
2179 /* If we were not given a target, use a word_mode register, not a
2180 'mode' register. The result will fit, and nobody is expecting
2181 anything bigger (the return type of __builtin_clz* is int). */
2183 target
= gen_reg_rtx (word_mode
);
2185 /* In any case, write to a word_mode scratch in both branches of the
2186 conditional, so we can ensure there is a single move insn setting
2187 'target' to tag a REG_EQUAL note on. */
2188 result
= gen_reg_rtx (word_mode
);
2192 /* If the high word is not equal to zero,
2193 then clz of the full value is clz of the high word. */
2194 emit_cmp_and_jump_insns (subhi
, CONST0_RTX (word_mode
), EQ
, 0,
2195 word_mode
, true, hi0_label
);
2197 temp
= expand_unop_direct (word_mode
, clz_optab
, subhi
, result
, true);
2202 convert_move (result
, temp
, true);
2204 emit_jump_insn (targetm
.gen_jump (after_label
));
2207 /* Else clz of the full value is clz of the low word plus the number
2208 of bits in the high word. */
2209 emit_label (hi0_label
);
2211 temp
= expand_unop_direct (word_mode
, clz_optab
, sublo
, 0, true);
2214 temp
= expand_binop (word_mode
, add_optab
, temp
,
2215 gen_int_mode (GET_MODE_BITSIZE (word_mode
), word_mode
),
2216 result
, true, OPTAB_DIRECT
);
2220 convert_move (result
, temp
, true);
2222 emit_label (after_label
);
2223 convert_move (target
, result
, true);
2228 add_equal_note (seq
, target
, CLZ
, xop0
, 0);
2237 /* Try calculating popcount of a double-word quantity as two popcount's of
2238 word-sized quantities and summing up the results. */
2240 expand_doubleword_popcount (scalar_int_mode mode
, rtx op0
, rtx target
)
2247 t0
= expand_unop_direct (word_mode
, popcount_optab
,
2248 operand_subword_force (op0
, 0, mode
), NULL_RTX
,
2250 t1
= expand_unop_direct (word_mode
, popcount_optab
,
2251 operand_subword_force (op0
, 1, mode
), NULL_RTX
,
2259 /* If we were not given a target, use a word_mode register, not a
2260 'mode' register. The result will fit, and nobody is expecting
2261 anything bigger (the return type of __builtin_popcount* is int). */
2263 target
= gen_reg_rtx (word_mode
);
2265 t
= expand_binop (word_mode
, add_optab
, t0
, t1
, target
, 0, OPTAB_DIRECT
);
2270 add_equal_note (seq
, t
, POPCOUNT
, op0
, 0);
2278 (parity:narrow (low (x) ^ high (x))) */
2280 expand_doubleword_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2282 rtx t
= expand_binop (word_mode
, xor_optab
,
2283 operand_subword_force (op0
, 0, mode
),
2284 operand_subword_force (op0
, 1, mode
),
2285 NULL_RTX
, 0, OPTAB_DIRECT
);
2286 return expand_unop (word_mode
, parity_optab
, t
, target
, true);
2292 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2294 widen_bswap (scalar_int_mode mode
, rtx op0
, rtx target
)
2298 opt_scalar_int_mode wider_mode_iter
;
2300 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2301 if (optab_handler (bswap_optab
, wider_mode_iter
.require ())
2302 != CODE_FOR_nothing
)
2305 if (!wider_mode_iter
.exists ())
2308 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2309 last
= get_last_insn ();
2311 x
= widen_operand (op0
, wider_mode
, mode
, true, true);
2312 x
= expand_unop (wider_mode
, bswap_optab
, x
, NULL_RTX
, true);
2314 gcc_assert (GET_MODE_PRECISION (wider_mode
) == GET_MODE_BITSIZE (wider_mode
)
2315 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
));
2317 x
= expand_shift (RSHIFT_EXPR
, wider_mode
, x
,
2318 GET_MODE_BITSIZE (wider_mode
)
2319 - GET_MODE_BITSIZE (mode
),
2325 target
= gen_reg_rtx (mode
);
2326 emit_move_insn (target
, gen_lowpart (mode
, x
));
2329 delete_insns_since (last
);
2334 /* Try calculating bswap as two bswaps of two word-sized operands. */
2337 expand_doubleword_bswap (machine_mode mode
, rtx op
, rtx target
)
2341 t1
= expand_unop (word_mode
, bswap_optab
,
2342 operand_subword_force (op
, 0, mode
), NULL_RTX
, true);
2343 t0
= expand_unop (word_mode
, bswap_optab
,
2344 operand_subword_force (op
, 1, mode
), NULL_RTX
, true);
2346 if (target
== 0 || !valid_multiword_target_p (target
))
2347 target
= gen_reg_rtx (mode
);
2349 emit_clobber (target
);
2350 emit_move_insn (operand_subword (target
, 0, 1, mode
), t0
);
2351 emit_move_insn (operand_subword (target
, 1, 1, mode
), t1
);
2356 /* Try calculating (parity x) as (and (popcount x) 1), where
2357 popcount can also be done in a wider mode. */
2359 expand_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2361 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2362 opt_scalar_int_mode wider_mode_iter
;
2363 FOR_EACH_MODE_FROM (wider_mode_iter
, mode
)
2365 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2366 if (optab_handler (popcount_optab
, wider_mode
) != CODE_FOR_nothing
)
2371 last
= get_last_insn ();
2373 if (target
== 0 || GET_MODE (target
) != wider_mode
)
2374 target
= gen_reg_rtx (wider_mode
);
2376 xop0
= widen_operand (op0
, wider_mode
, mode
, true, false);
2377 temp
= expand_unop (wider_mode
, popcount_optab
, xop0
, NULL_RTX
,
2380 temp
= expand_binop (wider_mode
, and_optab
, temp
, const1_rtx
,
2381 target
, true, OPTAB_DIRECT
);
2385 if (mclass
!= MODE_INT
2386 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2387 return convert_to_mode (mode
, temp
, 0);
2389 return gen_lowpart (mode
, temp
);
2392 delete_insns_since (last
);
2398 /* Try calculating ctz(x) as K - clz(x & -x) ,
2399 where K is GET_MODE_PRECISION(mode) - 1.
2401 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2402 don't have to worry about what the hardware does in that case. (If
2403 the clz instruction produces the usual value at 0, which is K, the
2404 result of this code sequence will be -1; expand_ffs, below, relies
2405 on this. It might be nice to have it be K instead, for consistency
2406 with the (very few) processors that provide a ctz with a defined
2407 value, but that would take one more instruction, and it would be
2408 less convenient for expand_ffs anyway. */
2411 expand_ctz (scalar_int_mode mode
, rtx op0
, rtx target
)
2416 if (optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2421 temp
= expand_unop_direct (mode
, neg_optab
, op0
, NULL_RTX
, true);
2423 temp
= expand_binop (mode
, and_optab
, op0
, temp
, NULL_RTX
,
2424 true, OPTAB_DIRECT
);
2426 temp
= expand_unop_direct (mode
, clz_optab
, temp
, NULL_RTX
, true);
2428 temp
= expand_binop (mode
, sub_optab
,
2429 gen_int_mode (GET_MODE_PRECISION (mode
) - 1, mode
),
2431 true, OPTAB_DIRECT
);
2441 add_equal_note (seq
, temp
, CTZ
, op0
, 0);
2447 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2448 else with the sequence used by expand_clz.
2450 The ffs builtin promises to return zero for a zero value and ctz/clz
2451 may have an undefined value in that case. If they do not give us a
2452 convenient value, we have to generate a test and branch. */
2454 expand_ffs (scalar_int_mode mode
, rtx op0
, rtx target
)
2456 HOST_WIDE_INT val
= 0;
2457 bool defined_at_zero
= false;
2461 if (optab_handler (ctz_optab
, mode
) != CODE_FOR_nothing
)
2465 temp
= expand_unop_direct (mode
, ctz_optab
, op0
, 0, true);
2469 defined_at_zero
= (CTZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2);
2471 else if (optab_handler (clz_optab
, mode
) != CODE_FOR_nothing
)
2474 temp
= expand_ctz (mode
, op0
, 0);
2478 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2)
2480 defined_at_zero
= true;
2481 val
= (GET_MODE_PRECISION (mode
) - 1) - val
;
2487 if (defined_at_zero
&& val
== -1)
2488 /* No correction needed at zero. */;
2491 /* We don't try to do anything clever with the situation found
2492 on some processors (eg Alpha) where ctz(0:mode) ==
2493 bitsize(mode). If someone can think of a way to send N to -1
2494 and leave alone all values in the range 0..N-1 (where N is a
2495 power of two), cheaper than this test-and-branch, please add it.
2497 The test-and-branch is done after the operation itself, in case
2498 the operation sets condition codes that can be recycled for this.
2499 (This is true on i386, for instance.) */
2501 rtx_code_label
*nonzero_label
= gen_label_rtx ();
2502 emit_cmp_and_jump_insns (op0
, CONST0_RTX (mode
), NE
, 0,
2503 mode
, true, nonzero_label
);
2505 convert_move (temp
, GEN_INT (-1), false);
2506 emit_label (nonzero_label
);
2509 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2510 to produce a value in the range 0..bitsize. */
2511 temp
= expand_binop (mode
, add_optab
, temp
, gen_int_mode (1, mode
),
2512 target
, false, OPTAB_DIRECT
);
2519 add_equal_note (seq
, temp
, FFS
, op0
, 0);
2528 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2529 conditions, VAL may already be a SUBREG against which we cannot generate
2530 a further SUBREG. In this case, we expect forcing the value into a
2531 register will work around the situation. */
2534 lowpart_subreg_maybe_copy (machine_mode omode
, rtx val
,
2538 ret
= lowpart_subreg (omode
, val
, imode
);
2541 val
= force_reg (imode
, val
);
2542 ret
= lowpart_subreg (omode
, val
, imode
);
2543 gcc_assert (ret
!= NULL
);
2548 /* Expand a floating point absolute value or negation operation via a
2549 logical operation on the sign bit. */
2552 expand_absneg_bit (enum rtx_code code
, scalar_float_mode mode
,
2553 rtx op0
, rtx target
)
2555 const struct real_format
*fmt
;
2556 int bitpos
, word
, nwords
, i
;
2557 scalar_int_mode imode
;
2561 /* The format has to have a simple sign bit. */
2562 fmt
= REAL_MODE_FORMAT (mode
);
2566 bitpos
= fmt
->signbit_rw
;
2570 /* Don't create negative zeros if the format doesn't support them. */
2571 if (code
== NEG
&& !fmt
->has_signed_zero
)
2574 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
2576 if (!int_mode_for_mode (mode
).exists (&imode
))
2585 if (FLOAT_WORDS_BIG_ENDIAN
)
2586 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
2588 word
= bitpos
/ BITS_PER_WORD
;
2589 bitpos
= bitpos
% BITS_PER_WORD
;
2590 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
2593 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
2599 || (nwords
> 1 && !valid_multiword_target_p (target
)))
2600 target
= gen_reg_rtx (mode
);
2606 for (i
= 0; i
< nwords
; ++i
)
2608 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
2609 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
2613 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2615 immed_wide_int_const (mask
, imode
),
2616 targ_piece
, 1, OPTAB_LIB_WIDEN
);
2617 if (temp
!= targ_piece
)
2618 emit_move_insn (targ_piece
, temp
);
2621 emit_move_insn (targ_piece
, op0_piece
);
2624 insns
= get_insns ();
2631 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2632 gen_lowpart (imode
, op0
),
2633 immed_wide_int_const (mask
, imode
),
2634 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
2635 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
2637 set_dst_reg_note (get_last_insn (), REG_EQUAL
,
2638 gen_rtx_fmt_e (code
, mode
, copy_rtx (op0
)),
2645 /* As expand_unop, but will fail rather than attempt the operation in a
2646 different mode or with a libcall. */
2648 expand_unop_direct (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2651 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2653 struct expand_operand ops
[2];
2654 enum insn_code icode
= optab_handler (unoptab
, mode
);
2655 rtx_insn
*last
= get_last_insn ();
2658 create_output_operand (&ops
[0], target
, mode
);
2659 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2660 pat
= maybe_gen_insn (icode
, 2, ops
);
2663 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
2664 && ! add_equal_note (pat
, ops
[0].value
,
2665 optab_to_code (unoptab
),
2666 ops
[1].value
, NULL_RTX
))
2668 delete_insns_since (last
);
2669 return expand_unop (mode
, unoptab
, op0
, NULL_RTX
, unsignedp
);
2674 return ops
[0].value
;
2680 /* Generate code to perform an operation specified by UNOPTAB
2681 on operand OP0, with result having machine-mode MODE.
2683 UNSIGNEDP is for the case where we have to widen the operands
2684 to perform the operation. It says to use zero-extension.
2686 If TARGET is nonzero, the value
2687 is generated there, if it is convenient to do so.
2688 In all cases an rtx is returned for the locus of the value;
2689 this may or may not be TARGET. */
2692 expand_unop (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2695 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2696 machine_mode wider_mode
;
2697 scalar_int_mode int_mode
;
2698 scalar_float_mode float_mode
;
2702 temp
= expand_unop_direct (mode
, unoptab
, op0
, target
, unsignedp
);
2706 /* It can't be done in this mode. Can we open-code it in a wider mode? */
2708 /* Widening (or narrowing) clz needs special treatment. */
2709 if (unoptab
== clz_optab
)
2711 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
2713 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
2717 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2718 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2720 temp
= expand_doubleword_clz (int_mode
, op0
, target
);
2729 if (unoptab
== clrsb_optab
)
2731 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
2733 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
2740 if (unoptab
== popcount_optab
2741 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
2742 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2743 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
2744 && optimize_insn_for_speed_p ())
2746 temp
= expand_doubleword_popcount (int_mode
, op0
, target
);
2751 if (unoptab
== parity_optab
2752 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
2753 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2754 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
2755 || optab_handler (popcount_optab
, word_mode
) != CODE_FOR_nothing
)
2756 && optimize_insn_for_speed_p ())
2758 temp
= expand_doubleword_parity (int_mode
, op0
, target
);
2763 /* Widening (or narrowing) bswap needs special treatment. */
2764 if (unoptab
== bswap_optab
)
2766 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
2767 or ROTATERT. First try these directly; if this fails, then try the
2768 obvious pair of shifts with allowed widening, as this will probably
2769 be always more efficient than the other fallback methods. */
2775 if (optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
)
2777 temp
= expand_binop (mode
, rotl_optab
, op0
, GEN_INT (8), target
,
2778 unsignedp
, OPTAB_DIRECT
);
2783 if (optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
2785 temp
= expand_binop (mode
, rotr_optab
, op0
, GEN_INT (8), target
,
2786 unsignedp
, OPTAB_DIRECT
);
2791 last
= get_last_insn ();
2793 temp1
= expand_binop (mode
, ashl_optab
, op0
, GEN_INT (8), NULL_RTX
,
2794 unsignedp
, OPTAB_WIDEN
);
2795 temp2
= expand_binop (mode
, lshr_optab
, op0
, GEN_INT (8), NULL_RTX
,
2796 unsignedp
, OPTAB_WIDEN
);
2799 temp
= expand_binop (mode
, ior_optab
, temp1
, temp2
, target
,
2800 unsignedp
, OPTAB_WIDEN
);
2805 delete_insns_since (last
);
2808 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
2810 temp
= widen_bswap (int_mode
, op0
, target
);
2814 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2815 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2817 temp
= expand_doubleword_bswap (mode
, op0
, target
);
2826 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2827 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2829 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2832 rtx_insn
*last
= get_last_insn ();
2834 /* For certain operations, we need not actually extend
2835 the narrow operand, as long as we will truncate the
2836 results to the same narrowness. */
2838 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
2839 (unoptab
== neg_optab
2840 || unoptab
== one_cmpl_optab
)
2841 && mclass
== MODE_INT
);
2843 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2848 if (mclass
!= MODE_INT
2849 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2852 target
= gen_reg_rtx (mode
);
2853 convert_move (target
, temp
, 0);
2857 return gen_lowpart (mode
, temp
);
2860 delete_insns_since (last
);
2864 /* These can be done a word at a time. */
2865 if (unoptab
== one_cmpl_optab
2866 && is_int_mode (mode
, &int_mode
)
2867 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
2868 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2873 if (target
== 0 || target
== op0
|| !valid_multiword_target_p (target
))
2874 target
= gen_reg_rtx (int_mode
);
2878 /* Do the actual arithmetic. */
2879 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
2881 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
2882 rtx x
= expand_unop (word_mode
, unoptab
,
2883 operand_subword_force (op0
, i
, int_mode
),
2884 target_piece
, unsignedp
);
2886 if (target_piece
!= x
)
2887 emit_move_insn (target_piece
, x
);
2890 insns
= get_insns ();
2897 if (optab_to_code (unoptab
) == NEG
)
2899 /* Try negating floating point values by flipping the sign bit. */
2900 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
2902 temp
= expand_absneg_bit (NEG
, float_mode
, op0
, target
);
2907 /* If there is no negation pattern, and we have no negative zero,
2908 try subtracting from zero. */
2909 if (!HONOR_SIGNED_ZEROS (mode
))
2911 temp
= expand_binop (mode
, (unoptab
== negv_optab
2912 ? subv_optab
: sub_optab
),
2913 CONST0_RTX (mode
), op0
, target
,
2914 unsignedp
, OPTAB_DIRECT
);
2920 /* Try calculating parity (x) as popcount (x) % 2. */
2921 if (unoptab
== parity_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
2923 temp
= expand_parity (int_mode
, op0
, target
);
2928 /* Try implementing ffs (x) in terms of clz (x). */
2929 if (unoptab
== ffs_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
2931 temp
= expand_ffs (int_mode
, op0
, target
);
2936 /* Try implementing ctz (x) in terms of clz (x). */
2937 if (unoptab
== ctz_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
2939 temp
= expand_ctz (int_mode
, op0
, target
);
2945 /* Now try a library call in this mode. */
2946 libfunc
= optab_libfunc (unoptab
, mode
);
2952 machine_mode outmode
= mode
;
2954 /* All of these functions return small values. Thus we choose to
2955 have them return something that isn't a double-word. */
2956 if (unoptab
== ffs_optab
|| unoptab
== clz_optab
|| unoptab
== ctz_optab
2957 || unoptab
== clrsb_optab
|| unoptab
== popcount_optab
2958 || unoptab
== parity_optab
)
2960 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node
),
2961 optab_libfunc (unoptab
, mode
)));
2965 /* Pass 1 for NO_QUEUE so we don't lose any increments
2966 if the libcall is cse'd or moved. */
2967 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, outmode
,
2969 insns
= get_insns ();
2972 target
= gen_reg_rtx (outmode
);
2973 bool trapv
= trapv_unoptab_p (unoptab
);
2975 eq_value
= NULL_RTX
;
2978 eq_value
= gen_rtx_fmt_e (optab_to_code (unoptab
), mode
, op0
);
2979 if (GET_MODE_UNIT_SIZE (outmode
) < GET_MODE_UNIT_SIZE (mode
))
2980 eq_value
= simplify_gen_unary (TRUNCATE
, outmode
, eq_value
, mode
);
2981 else if (GET_MODE_UNIT_SIZE (outmode
) > GET_MODE_UNIT_SIZE (mode
))
2982 eq_value
= simplify_gen_unary (ZERO_EXTEND
,
2983 outmode
, eq_value
, mode
);
2985 emit_libcall_block_1 (insns
, target
, value
, eq_value
, trapv
);
2990 /* It can't be done in this mode. Can we do it in a wider mode? */
2992 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2994 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2996 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
2997 || optab_libfunc (unoptab
, wider_mode
))
3000 rtx_insn
*last
= get_last_insn ();
3002 /* For certain operations, we need not actually extend
3003 the narrow operand, as long as we will truncate the
3004 results to the same narrowness. */
3005 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3006 (unoptab
== neg_optab
3007 || unoptab
== one_cmpl_optab
3008 || unoptab
== bswap_optab
)
3009 && mclass
== MODE_INT
);
3011 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3014 /* If we are generating clz using wider mode, adjust the
3015 result. Similarly for clrsb. */
3016 if ((unoptab
== clz_optab
|| unoptab
== clrsb_optab
)
3019 scalar_int_mode wider_int_mode
3020 = as_a
<scalar_int_mode
> (wider_mode
);
3021 int_mode
= as_a
<scalar_int_mode
> (mode
);
3023 (wider_mode
, sub_optab
, temp
,
3024 gen_int_mode (GET_MODE_PRECISION (wider_int_mode
)
3025 - GET_MODE_PRECISION (int_mode
),
3027 target
, true, OPTAB_DIRECT
);
3030 /* Likewise for bswap. */
3031 if (unoptab
== bswap_optab
&& temp
!= 0)
3033 scalar_int_mode wider_int_mode
3034 = as_a
<scalar_int_mode
> (wider_mode
);
3035 int_mode
= as_a
<scalar_int_mode
> (mode
);
3036 gcc_assert (GET_MODE_PRECISION (wider_int_mode
)
3037 == GET_MODE_BITSIZE (wider_int_mode
)
3038 && GET_MODE_PRECISION (int_mode
)
3039 == GET_MODE_BITSIZE (int_mode
));
3041 temp
= expand_shift (RSHIFT_EXPR
, wider_int_mode
, temp
,
3042 GET_MODE_BITSIZE (wider_int_mode
)
3043 - GET_MODE_BITSIZE (int_mode
),
3049 if (mclass
!= MODE_INT
)
3052 target
= gen_reg_rtx (mode
);
3053 convert_move (target
, temp
, 0);
3057 return gen_lowpart (mode
, temp
);
3060 delete_insns_since (last
);
3065 /* One final attempt at implementing negation via subtraction,
3066 this time allowing widening of the operand. */
3067 if (optab_to_code (unoptab
) == NEG
&& !HONOR_SIGNED_ZEROS (mode
))
3070 temp
= expand_binop (mode
,
3071 unoptab
== negv_optab
? subv_optab
: sub_optab
,
3072 CONST0_RTX (mode
), op0
,
3073 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3081 /* Emit code to compute the absolute value of OP0, with result to
3082 TARGET if convenient. (TARGET may be 0.) The return value says
3083 where the result actually is to be found.
3085 MODE is the mode of the operand; the mode of the result is
3086 different but can be deduced from MODE.
3091 expand_abs_nojump (machine_mode mode
, rtx op0
, rtx target
,
3092 int result_unsignedp
)
3096 if (GET_MODE_CLASS (mode
) != MODE_INT
3098 result_unsignedp
= 1;
3100 /* First try to do it with a special abs instruction. */
3101 temp
= expand_unop (mode
, result_unsignedp
? abs_optab
: absv_optab
,
3106 /* For floating point modes, try clearing the sign bit. */
3107 scalar_float_mode float_mode
;
3108 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3110 temp
= expand_absneg_bit (ABS
, float_mode
, op0
, target
);
3115 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3116 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
3117 && !HONOR_SIGNED_ZEROS (mode
))
3119 rtx_insn
*last
= get_last_insn ();
3121 temp
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3124 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3130 delete_insns_since (last
);
3133 /* If this machine has expensive jumps, we can do integer absolute
3134 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3135 where W is the width of MODE. */
3137 scalar_int_mode int_mode
;
3138 if (is_int_mode (mode
, &int_mode
)
3139 && BRANCH_COST (optimize_insn_for_speed_p (),
3142 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3143 GET_MODE_PRECISION (int_mode
) - 1,
3146 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3149 temp
= expand_binop (int_mode
,
3150 result_unsignedp
? sub_optab
: subv_optab
,
3151 temp
, extended
, target
, 0, OPTAB_LIB_WIDEN
);
3161 expand_abs (machine_mode mode
, rtx op0
, rtx target
,
3162 int result_unsignedp
, int safe
)
3165 rtx_code_label
*op1
;
3167 if (GET_MODE_CLASS (mode
) != MODE_INT
3169 result_unsignedp
= 1;
3171 temp
= expand_abs_nojump (mode
, op0
, target
, result_unsignedp
);
3175 /* If that does not win, use conditional jump and negate. */
3177 /* It is safe to use the target if it is the same
3178 as the source if this is also a pseudo register */
3179 if (op0
== target
&& REG_P (op0
)
3180 && REGNO (op0
) >= FIRST_PSEUDO_REGISTER
)
3183 op1
= gen_label_rtx ();
3184 if (target
== 0 || ! safe
3185 || GET_MODE (target
) != mode
3186 || (MEM_P (target
) && MEM_VOLATILE_P (target
))
3188 && REGNO (target
) < FIRST_PSEUDO_REGISTER
))
3189 target
= gen_reg_rtx (mode
);
3191 emit_move_insn (target
, op0
);
3194 do_compare_rtx_and_jump (target
, CONST0_RTX (mode
), GE
, 0, mode
,
3195 NULL_RTX
, NULL
, op1
,
3196 profile_probability::uninitialized ());
3198 op0
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3201 emit_move_insn (target
, op0
);
3207 /* Emit code to compute the one's complement absolute value of OP0
3208 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3209 (TARGET may be NULL_RTX.) The return value says where the result
3210 actually is to be found.
3212 MODE is the mode of the operand; the mode of the result is
3213 different but can be deduced from MODE. */
3216 expand_one_cmpl_abs_nojump (machine_mode mode
, rtx op0
, rtx target
)
3220 /* Not applicable for floating point modes. */
3221 if (FLOAT_MODE_P (mode
))
3224 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3225 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
)
3227 rtx_insn
*last
= get_last_insn ();
3229 temp
= expand_unop (mode
, one_cmpl_optab
, op0
, NULL_RTX
, 0);
3231 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3237 delete_insns_since (last
);
3240 /* If this machine has expensive jumps, we can do one's complement
3241 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3243 scalar_int_mode int_mode
;
3244 if (is_int_mode (mode
, &int_mode
)
3245 && BRANCH_COST (optimize_insn_for_speed_p (),
3248 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3249 GET_MODE_PRECISION (int_mode
) - 1,
3252 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3262 /* A subroutine of expand_copysign, perform the copysign operation using the
3263 abs and neg primitives advertised to exist on the target. The assumption
3264 is that we have a split register file, and leaving op0 in fp registers,
3265 and not playing with subregs so much, will help the register allocator. */
3268 expand_copysign_absneg (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3269 int bitpos
, bool op0_is_abs
)
3271 scalar_int_mode imode
;
3272 enum insn_code icode
;
3274 rtx_code_label
*label
;
3279 /* Check if the back end provides an insn that handles signbit for the
3281 icode
= optab_handler (signbit_optab
, mode
);
3282 if (icode
!= CODE_FOR_nothing
)
3284 imode
= as_a
<scalar_int_mode
> (insn_data
[(int) icode
].operand
[0].mode
);
3285 sign
= gen_reg_rtx (imode
);
3286 emit_unop_insn (icode
, sign
, op1
, UNKNOWN
);
3290 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3292 if (!int_mode_for_mode (mode
).exists (&imode
))
3294 op1
= gen_lowpart (imode
, op1
);
3301 if (FLOAT_WORDS_BIG_ENDIAN
)
3302 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3304 word
= bitpos
/ BITS_PER_WORD
;
3305 bitpos
= bitpos
% BITS_PER_WORD
;
3306 op1
= operand_subword_force (op1
, word
, mode
);
3309 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3310 sign
= expand_binop (imode
, and_optab
, op1
,
3311 immed_wide_int_const (mask
, imode
),
3312 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3317 op0
= expand_unop (mode
, abs_optab
, op0
, target
, 0);
3324 if (target
== NULL_RTX
)
3325 target
= copy_to_reg (op0
);
3327 emit_move_insn (target
, op0
);
3330 label
= gen_label_rtx ();
3331 emit_cmp_and_jump_insns (sign
, const0_rtx
, EQ
, NULL_RTX
, imode
, 1, label
);
3333 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3334 op0
= simplify_unary_operation (NEG
, mode
, op0
, mode
);
3336 op0
= expand_unop (mode
, neg_optab
, op0
, target
, 0);
3338 emit_move_insn (target
, op0
);
3346 /* A subroutine of expand_copysign, perform the entire copysign operation
3347 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3348 is true if op0 is known to have its sign bit clear. */
3351 expand_copysign_bit (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3352 int bitpos
, bool op0_is_abs
)
3354 scalar_int_mode imode
;
3355 int word
, nwords
, i
;
3359 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3361 if (!int_mode_for_mode (mode
).exists (&imode
))
3370 if (FLOAT_WORDS_BIG_ENDIAN
)
3371 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3373 word
= bitpos
/ BITS_PER_WORD
;
3374 bitpos
= bitpos
% BITS_PER_WORD
;
3375 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3378 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3383 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3384 target
= gen_reg_rtx (mode
);
3390 for (i
= 0; i
< nwords
; ++i
)
3392 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3393 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3399 = expand_binop (imode
, and_optab
, op0_piece
,
3400 immed_wide_int_const (~mask
, imode
),
3401 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3402 op1
= expand_binop (imode
, and_optab
,
3403 operand_subword_force (op1
, i
, mode
),
3404 immed_wide_int_const (mask
, imode
),
3405 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3407 temp
= expand_binop (imode
, ior_optab
, op0_piece
, op1
,
3408 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3409 if (temp
!= targ_piece
)
3410 emit_move_insn (targ_piece
, temp
);
3413 emit_move_insn (targ_piece
, op0_piece
);
3416 insns
= get_insns ();
3423 op1
= expand_binop (imode
, and_optab
, gen_lowpart (imode
, op1
),
3424 immed_wide_int_const (mask
, imode
),
3425 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3427 op0
= gen_lowpart (imode
, op0
);
3429 op0
= expand_binop (imode
, and_optab
, op0
,
3430 immed_wide_int_const (~mask
, imode
),
3431 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3433 temp
= expand_binop (imode
, ior_optab
, op0
, op1
,
3434 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3435 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3441 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3442 scalar floating point mode. Return NULL if we do not know how to
3443 expand the operation inline. */
3446 expand_copysign (rtx op0
, rtx op1
, rtx target
)
3448 scalar_float_mode mode
;
3449 const struct real_format
*fmt
;
3453 mode
= as_a
<scalar_float_mode
> (GET_MODE (op0
));
3454 gcc_assert (GET_MODE (op1
) == mode
);
3456 /* First try to do it with a special instruction. */
3457 temp
= expand_binop (mode
, copysign_optab
, op0
, op1
,
3458 target
, 0, OPTAB_DIRECT
);
3462 fmt
= REAL_MODE_FORMAT (mode
);
3463 if (fmt
== NULL
|| !fmt
->has_signed_zero
)
3467 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3469 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0
)))
3470 op0
= simplify_unary_operation (ABS
, mode
, op0
, mode
);
3474 if (fmt
->signbit_ro
>= 0
3475 && (CONST_DOUBLE_AS_FLOAT_P (op0
)
3476 || (optab_handler (neg_optab
, mode
) != CODE_FOR_nothing
3477 && optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)))
3479 temp
= expand_copysign_absneg (mode
, op0
, op1
, target
,
3480 fmt
->signbit_ro
, op0_is_abs
);
3485 if (fmt
->signbit_rw
< 0)
3487 return expand_copysign_bit (mode
, op0
, op1
, target
,
3488 fmt
->signbit_rw
, op0_is_abs
);
3491 /* Generate an instruction whose insn-code is INSN_CODE,
3492 with two operands: an output TARGET and an input OP0.
3493 TARGET *must* be nonzero, and the output is always stored there.
3494 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3495 the value that is stored into TARGET.
3497 Return false if expansion failed. */
3500 maybe_emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
,
3503 struct expand_operand ops
[2];
3506 create_output_operand (&ops
[0], target
, GET_MODE (target
));
3507 create_input_operand (&ops
[1], op0
, GET_MODE (op0
));
3508 pat
= maybe_gen_insn (icode
, 2, ops
);
3512 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
3514 add_equal_note (pat
, ops
[0].value
, code
, ops
[1].value
, NULL_RTX
);
3518 if (ops
[0].value
!= target
)
3519 emit_move_insn (target
, ops
[0].value
);
3522 /* Generate an instruction whose insn-code is INSN_CODE,
3523 with two operands: an output TARGET and an input OP0.
3524 TARGET *must* be nonzero, and the output is always stored there.
3525 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3526 the value that is stored into TARGET. */
3529 emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
, enum rtx_code code
)
3531 bool ok
= maybe_emit_unop_insn (icode
, target
, op0
, code
);
3535 struct no_conflict_data
3538 rtx_insn
*first
, *insn
;
3542 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3543 the currently examined clobber / store has to stay in the list of
3544 insns that constitute the actual libcall block. */
3546 no_conflict_move_test (rtx dest
, const_rtx set
, void *p0
)
3548 struct no_conflict_data
*p
= (struct no_conflict_data
*) p0
;
3550 /* If this inns directly contributes to setting the target, it must stay. */
3551 if (reg_overlap_mentioned_p (p
->target
, dest
))
3552 p
->must_stay
= true;
3553 /* If we haven't committed to keeping any other insns in the list yet,
3554 there is nothing more to check. */
3555 else if (p
->insn
== p
->first
)
3557 /* If this insn sets / clobbers a register that feeds one of the insns
3558 already in the list, this insn has to stay too. */
3559 else if (reg_overlap_mentioned_p (dest
, PATTERN (p
->first
))
3560 || (CALL_P (p
->first
) && (find_reg_fusage (p
->first
, USE
, dest
)))
3561 || reg_used_between_p (dest
, p
->first
, p
->insn
)
3562 /* Likewise if this insn depends on a register set by a previous
3563 insn in the list, or if it sets a result (presumably a hard
3564 register) that is set or clobbered by a previous insn.
3565 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3566 SET_DEST perform the former check on the address, and the latter
3567 check on the MEM. */
3568 || (GET_CODE (set
) == SET
3569 && (modified_in_p (SET_SRC (set
), p
->first
)
3570 || modified_in_p (SET_DEST (set
), p
->first
)
3571 || modified_between_p (SET_SRC (set
), p
->first
, p
->insn
)
3572 || modified_between_p (SET_DEST (set
), p
->first
, p
->insn
))))
3573 p
->must_stay
= true;
3577 /* Emit code to make a call to a constant function or a library call.
3579 INSNS is a list containing all insns emitted in the call.
3580 These insns leave the result in RESULT. Our block is to copy RESULT
3581 to TARGET, which is logically equivalent to EQUIV.
3583 We first emit any insns that set a pseudo on the assumption that these are
3584 loading constants into registers; doing so allows them to be safely cse'ed
3585 between blocks. Then we emit all the other insns in the block, followed by
3586 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3587 note with an operand of EQUIV. */
3590 emit_libcall_block_1 (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
,
3591 bool equiv_may_trap
)
3593 rtx final_dest
= target
;
3594 rtx_insn
*next
, *last
, *insn
;
3596 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3597 into a MEM later. Protect the libcall block from this change. */
3598 if (! REG_P (target
) || REG_USERVAR_P (target
))
3599 target
= gen_reg_rtx (GET_MODE (target
));
3601 /* If we're using non-call exceptions, a libcall corresponding to an
3602 operation that may trap may also trap. */
3603 /* ??? See the comment in front of make_reg_eh_region_note. */
3604 if (cfun
->can_throw_non_call_exceptions
3605 && (equiv_may_trap
|| may_trap_p (equiv
)))
3607 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3610 rtx note
= find_reg_note (insn
, REG_EH_REGION
, NULL_RTX
);
3613 int lp_nr
= INTVAL (XEXP (note
, 0));
3614 if (lp_nr
== 0 || lp_nr
== INT_MIN
)
3615 remove_note (insn
, note
);
3621 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3622 reg note to indicate that this call cannot throw or execute a nonlocal
3623 goto (unless there is already a REG_EH_REGION note, in which case
3625 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3627 make_reg_eh_region_note_nothrow_nononlocal (insn
);
3630 /* First emit all insns that set pseudos. Remove them from the list as
3631 we go. Avoid insns that set pseudos which were referenced in previous
3632 insns. These can be generated by move_by_pieces, for example,
3633 to update an address. Similarly, avoid insns that reference things
3634 set in previous insns. */
3636 for (insn
= insns
; insn
; insn
= next
)
3638 rtx set
= single_set (insn
);
3640 next
= NEXT_INSN (insn
);
3642 if (set
!= 0 && REG_P (SET_DEST (set
))
3643 && REGNO (SET_DEST (set
)) >= FIRST_PSEUDO_REGISTER
)
3645 struct no_conflict_data data
;
3647 data
.target
= const0_rtx
;
3651 note_stores (PATTERN (insn
), no_conflict_move_test
, &data
);
3652 if (! data
.must_stay
)
3654 if (PREV_INSN (insn
))
3655 SET_NEXT_INSN (PREV_INSN (insn
)) = next
;
3660 SET_PREV_INSN (next
) = PREV_INSN (insn
);
3666 /* Some ports use a loop to copy large arguments onto the stack.
3667 Don't move anything outside such a loop. */
3672 /* Write the remaining insns followed by the final copy. */
3673 for (insn
= insns
; insn
; insn
= next
)
3675 next
= NEXT_INSN (insn
);
3680 last
= emit_move_insn (target
, result
);
3682 set_dst_reg_note (last
, REG_EQUAL
, copy_rtx (equiv
), target
);
3684 if (final_dest
!= target
)
3685 emit_move_insn (final_dest
, target
);
3689 emit_libcall_block (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
)
3691 emit_libcall_block_1 (insns
, target
, result
, equiv
, false);
3694 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3695 PURPOSE describes how this comparison will be used. CODE is the rtx
3696 comparison code we will be using.
3698 ??? Actually, CODE is slightly weaker than that. A target is still
3699 required to implement all of the normal bcc operations, but not
3700 required to implement all (or any) of the unordered bcc operations. */
3703 can_compare_p (enum rtx_code code
, machine_mode mode
,
3704 enum can_compare_purpose purpose
)
3707 test
= gen_rtx_fmt_ee (code
, mode
, const0_rtx
, const0_rtx
);
3710 enum insn_code icode
;
3712 if (purpose
== ccp_jump
3713 && (icode
= optab_handler (cbranch_optab
, mode
)) != CODE_FOR_nothing
3714 && insn_operand_matches (icode
, 0, test
))
3716 if (purpose
== ccp_store_flag
3717 && (icode
= optab_handler (cstore_optab
, mode
)) != CODE_FOR_nothing
3718 && insn_operand_matches (icode
, 1, test
))
3720 if (purpose
== ccp_cmov
3721 && optab_handler (cmov_optab
, mode
) != CODE_FOR_nothing
)
3724 mode
= GET_MODE_WIDER_MODE (mode
).else_void ();
3725 PUT_MODE (test
, mode
);
3727 while (mode
!= VOIDmode
);
3732 /* This function is called when we are going to emit a compare instruction that
3733 compares the values found in X and Y, using the rtl operator COMPARISON.
3735 If they have mode BLKmode, then SIZE specifies the size of both operands.
3737 UNSIGNEDP nonzero says that the operands are unsigned;
3738 this matters if they need to be widened (as given by METHODS).
3740 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
3741 if we failed to produce one.
3743 *PMODE is the mode of the inputs (in case they are const_int).
3745 This function performs all the setup necessary so that the caller only has
3746 to emit a single comparison insn. This setup can involve doing a BLKmode
3747 comparison or emitting a library call to perform the comparison if no insn
3748 is available to handle it.
3749 The values which are passed in through pointers can be modified; the caller
3750 should perform the comparison on the modified values. Constant
3751 comparisons must have already been folded. */
3754 prepare_cmp_insn (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
3755 int unsignedp
, enum optab_methods methods
,
3756 rtx
*ptest
, machine_mode
*pmode
)
3758 machine_mode mode
= *pmode
;
3760 machine_mode cmp_mode
;
3761 enum mode_class mclass
;
3763 /* The other methods are not needed. */
3764 gcc_assert (methods
== OPTAB_DIRECT
|| methods
== OPTAB_WIDEN
3765 || methods
== OPTAB_LIB_WIDEN
);
3767 /* If we are optimizing, force expensive constants into a register. */
3768 if (CONSTANT_P (x
) && optimize
3769 && (rtx_cost (x
, mode
, COMPARE
, 0, optimize_insn_for_speed_p ())
3770 > COSTS_N_INSNS (1)))
3771 x
= force_reg (mode
, x
);
3773 if (CONSTANT_P (y
) && optimize
3774 && (rtx_cost (y
, mode
, COMPARE
, 1, optimize_insn_for_speed_p ())
3775 > COSTS_N_INSNS (1)))
3776 y
= force_reg (mode
, y
);
3779 /* Make sure if we have a canonical comparison. The RTL
3780 documentation states that canonical comparisons are required only
3781 for targets which have cc0. */
3782 gcc_assert (!CONSTANT_P (x
) || CONSTANT_P (y
));
3785 /* Don't let both operands fail to indicate the mode. */
3786 if (GET_MODE (x
) == VOIDmode
&& GET_MODE (y
) == VOIDmode
)
3787 x
= force_reg (mode
, x
);
3788 if (mode
== VOIDmode
)
3789 mode
= GET_MODE (x
) != VOIDmode
? GET_MODE (x
) : GET_MODE (y
);
3791 /* Handle all BLKmode compares. */
3793 if (mode
== BLKmode
)
3795 machine_mode result_mode
;
3796 enum insn_code cmp_code
;
3799 = GEN_INT (MIN (MEM_ALIGN (x
), MEM_ALIGN (y
)) / BITS_PER_UNIT
);
3803 /* Try to use a memory block compare insn - either cmpstr
3804 or cmpmem will do. */
3805 opt_scalar_int_mode cmp_mode_iter
;
3806 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
3808 scalar_int_mode cmp_mode
= cmp_mode_iter
.require ();
3809 cmp_code
= direct_optab_handler (cmpmem_optab
, cmp_mode
);
3810 if (cmp_code
== CODE_FOR_nothing
)
3811 cmp_code
= direct_optab_handler (cmpstr_optab
, cmp_mode
);
3812 if (cmp_code
== CODE_FOR_nothing
)
3813 cmp_code
= direct_optab_handler (cmpstrn_optab
, cmp_mode
);
3814 if (cmp_code
== CODE_FOR_nothing
)
3817 /* Must make sure the size fits the insn's mode. */
3818 if (CONST_INT_P (size
)
3819 ? INTVAL (size
) >= (1 << GET_MODE_BITSIZE (cmp_mode
))
3820 : (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (size
)))
3821 > GET_MODE_BITSIZE (cmp_mode
)))
3824 result_mode
= insn_data
[cmp_code
].operand
[0].mode
;
3825 result
= gen_reg_rtx (result_mode
);
3826 size
= convert_to_mode (cmp_mode
, size
, 1);
3827 emit_insn (GEN_FCN (cmp_code
) (result
, x
, y
, size
, opalign
));
3829 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, result
, const0_rtx
);
3830 *pmode
= result_mode
;
3834 if (methods
!= OPTAB_LIB
&& methods
!= OPTAB_LIB_WIDEN
)
3837 /* Otherwise call a library function. */
3838 result
= emit_block_comp_via_libcall (XEXP (x
, 0), XEXP (y
, 0), size
);
3842 mode
= TYPE_MODE (integer_type_node
);
3843 methods
= OPTAB_LIB_WIDEN
;
3847 /* Don't allow operands to the compare to trap, as that can put the
3848 compare and branch in different basic blocks. */
3849 if (cfun
->can_throw_non_call_exceptions
)
3852 x
= copy_to_reg (x
);
3854 y
= copy_to_reg (y
);
3857 if (GET_MODE_CLASS (mode
) == MODE_CC
)
3859 enum insn_code icode
= optab_handler (cbranch_optab
, CCmode
);
3860 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
3861 gcc_assert (icode
!= CODE_FOR_nothing
3862 && insn_operand_matches (icode
, 0, test
));
3867 mclass
= GET_MODE_CLASS (mode
);
3868 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
3869 FOR_EACH_MODE_FROM (cmp_mode
, mode
)
3871 enum insn_code icode
;
3872 icode
= optab_handler (cbranch_optab
, cmp_mode
);
3873 if (icode
!= CODE_FOR_nothing
3874 && insn_operand_matches (icode
, 0, test
))
3876 rtx_insn
*last
= get_last_insn ();
3877 rtx op0
= prepare_operand (icode
, x
, 1, mode
, cmp_mode
, unsignedp
);
3878 rtx op1
= prepare_operand (icode
, y
, 2, mode
, cmp_mode
, unsignedp
);
3880 && insn_operand_matches (icode
, 1, op0
)
3881 && insn_operand_matches (icode
, 2, op1
))
3883 XEXP (test
, 0) = op0
;
3884 XEXP (test
, 1) = op1
;
3889 delete_insns_since (last
);
3892 if (methods
== OPTAB_DIRECT
|| !CLASS_HAS_WIDER_MODES_P (mclass
))
3896 if (methods
!= OPTAB_LIB_WIDEN
)
3899 if (!SCALAR_FLOAT_MODE_P (mode
))
3902 machine_mode ret_mode
;
3904 /* Handle a libcall just for the mode we are using. */
3905 libfunc
= optab_libfunc (cmp_optab
, mode
);
3906 gcc_assert (libfunc
);
3908 /* If we want unsigned, and this mode has a distinct unsigned
3909 comparison routine, use that. */
3912 rtx ulibfunc
= optab_libfunc (ucmp_optab
, mode
);
3917 ret_mode
= targetm
.libgcc_cmp_return_mode ();
3918 result
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
3919 ret_mode
, x
, mode
, y
, mode
);
3921 /* There are two kinds of comparison routines. Biased routines
3922 return 0/1/2, and unbiased routines return -1/0/1. Other parts
3923 of gcc expect that the comparison operation is equivalent
3924 to the modified comparison. For signed comparisons compare the
3925 result against 1 in the biased case, and zero in the unbiased
3926 case. For unsigned comparisons always compare against 1 after
3927 biasing the unbiased result by adding 1. This gives us a way to
3929 The comparisons in the fixed-point helper library are always
3934 if (!TARGET_LIB_INT_CMP_BIASED
&& !ALL_FIXED_POINT_MODE_P (mode
))
3937 x
= plus_constant (ret_mode
, result
, 1);
3943 prepare_cmp_insn (x
, y
, comparison
, NULL_RTX
, unsignedp
, methods
,
3947 prepare_float_lib_cmp (x
, y
, comparison
, ptest
, pmode
);
3955 /* Before emitting an insn with code ICODE, make sure that X, which is going
3956 to be used for operand OPNUM of the insn, is converted from mode MODE to
3957 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
3958 that it is accepted by the operand predicate. Return the new value. */
3961 prepare_operand (enum insn_code icode
, rtx x
, int opnum
, machine_mode mode
,
3962 machine_mode wider_mode
, int unsignedp
)
3964 if (mode
!= wider_mode
)
3965 x
= convert_modes (wider_mode
, mode
, x
, unsignedp
);
3967 if (!insn_operand_matches (icode
, opnum
, x
))
3969 machine_mode op_mode
= insn_data
[(int) icode
].operand
[opnum
].mode
;
3970 if (reload_completed
)
3972 if (GET_MODE (x
) != op_mode
&& GET_MODE (x
) != VOIDmode
)
3974 x
= copy_to_mode_reg (op_mode
, x
);
3980 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
3981 we can do the branch. */
3984 emit_cmp_and_jump_insn_1 (rtx test
, machine_mode mode
, rtx label
,
3985 profile_probability prob
)
3987 machine_mode optab_mode
;
3988 enum mode_class mclass
;
3989 enum insn_code icode
;
3992 mclass
= GET_MODE_CLASS (mode
);
3993 optab_mode
= (mclass
== MODE_CC
) ? CCmode
: mode
;
3994 icode
= optab_handler (cbranch_optab
, optab_mode
);
3996 gcc_assert (icode
!= CODE_FOR_nothing
);
3997 gcc_assert (insn_operand_matches (icode
, 0, test
));
3998 insn
= emit_jump_insn (GEN_FCN (icode
) (test
, XEXP (test
, 0),
3999 XEXP (test
, 1), label
));
4000 if (prob
.initialized_p ()
4001 && profile_status_for_fn (cfun
) != PROFILE_ABSENT
4004 && any_condjump_p (insn
)
4005 && !find_reg_note (insn
, REG_BR_PROB
, 0))
4006 add_reg_br_prob_note (insn
, prob
);
4009 /* Generate code to compare X with Y so that the condition codes are
4010 set and to jump to LABEL if the condition is true. If X is a
4011 constant and Y is not a constant, then the comparison is swapped to
4012 ensure that the comparison RTL has the canonical form.
4014 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4015 need to be widened. UNSIGNEDP is also used to select the proper
4016 branch condition code.
4018 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4020 MODE is the mode of the inputs (in case they are const_int).
4022 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4023 It will be potentially converted into an unsigned variant based on
4024 UNSIGNEDP to select a proper jump instruction.
4026 PROB is the probability of jumping to LABEL. */
4029 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4030 machine_mode mode
, int unsignedp
, rtx label
,
4031 profile_probability prob
)
4033 rtx op0
= x
, op1
= y
;
4036 /* Swap operands and condition to ensure canonical RTL. */
4037 if (swap_commutative_operands_p (x
, y
)
4038 && can_compare_p (swap_condition (comparison
), mode
, ccp_jump
))
4041 comparison
= swap_condition (comparison
);
4044 /* If OP0 is still a constant, then both X and Y must be constants
4045 or the opposite comparison is not supported. Force X into a register
4046 to create canonical RTL. */
4047 if (CONSTANT_P (op0
))
4048 op0
= force_reg (mode
, op0
);
4051 comparison
= unsigned_condition (comparison
);
4053 prepare_cmp_insn (op0
, op1
, comparison
, size
, unsignedp
, OPTAB_LIB_WIDEN
,
4055 emit_cmp_and_jump_insn_1 (test
, mode
, label
, prob
);
4059 /* Emit a library call comparison between floating point X and Y.
4060 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4063 prepare_float_lib_cmp (rtx x
, rtx y
, enum rtx_code comparison
,
4064 rtx
*ptest
, machine_mode
*pmode
)
4066 enum rtx_code swapped
= swap_condition (comparison
);
4067 enum rtx_code reversed
= reverse_condition_maybe_unordered (comparison
);
4068 machine_mode orig_mode
= GET_MODE (x
);
4070 rtx true_rtx
, false_rtx
;
4071 rtx value
, target
, equiv
;
4074 bool reversed_p
= false;
4075 scalar_int_mode cmp_mode
= targetm
.libgcc_cmp_return_mode ();
4077 FOR_EACH_MODE_FROM (mode
, orig_mode
)
4079 if (code_to_optab (comparison
)
4080 && (libfunc
= optab_libfunc (code_to_optab (comparison
), mode
)))
4083 if (code_to_optab (swapped
)
4084 && (libfunc
= optab_libfunc (code_to_optab (swapped
), mode
)))
4087 comparison
= swapped
;
4091 if (code_to_optab (reversed
)
4092 && (libfunc
= optab_libfunc (code_to_optab (reversed
), mode
)))
4094 comparison
= reversed
;
4100 gcc_assert (mode
!= VOIDmode
);
4102 if (mode
!= orig_mode
)
4104 x
= convert_to_mode (mode
, x
, 0);
4105 y
= convert_to_mode (mode
, y
, 0);
4108 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4109 the RTL. The allows the RTL optimizers to delete the libcall if the
4110 condition can be determined at compile-time. */
4111 if (comparison
== UNORDERED
4112 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4114 true_rtx
= const_true_rtx
;
4115 false_rtx
= const0_rtx
;
4122 true_rtx
= const0_rtx
;
4123 false_rtx
= const_true_rtx
;
4127 true_rtx
= const_true_rtx
;
4128 false_rtx
= const0_rtx
;
4132 true_rtx
= const1_rtx
;
4133 false_rtx
= const0_rtx
;
4137 true_rtx
= const0_rtx
;
4138 false_rtx
= constm1_rtx
;
4142 true_rtx
= constm1_rtx
;
4143 false_rtx
= const0_rtx
;
4147 true_rtx
= const0_rtx
;
4148 false_rtx
= const1_rtx
;
4156 if (comparison
== UNORDERED
)
4158 rtx temp
= simplify_gen_relational (NE
, cmp_mode
, mode
, x
, x
);
4159 equiv
= simplify_gen_relational (NE
, cmp_mode
, mode
, y
, y
);
4160 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4161 temp
, const_true_rtx
, equiv
);
4165 equiv
= simplify_gen_relational (comparison
, cmp_mode
, mode
, x
, y
);
4166 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4167 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4168 equiv
, true_rtx
, false_rtx
);
4172 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4173 cmp_mode
, x
, mode
, y
, mode
);
4174 insns
= get_insns ();
4177 target
= gen_reg_rtx (cmp_mode
);
4178 emit_libcall_block (insns
, target
, value
, equiv
);
4180 if (comparison
== UNORDERED
4181 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
)
4183 *ptest
= gen_rtx_fmt_ee (reversed_p
? EQ
: NE
, VOIDmode
, target
, false_rtx
);
4185 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, target
, const0_rtx
);
4190 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4193 emit_indirect_jump (rtx loc
)
4195 if (!targetm
.have_indirect_jump ())
4196 sorry ("indirect jumps are not available on this target");
4199 struct expand_operand ops
[1];
4200 create_address_operand (&ops
[0], loc
);
4201 expand_jump_insn (targetm
.code_for_indirect_jump
, 1, ops
);
4207 /* Emit a conditional move instruction if the machine supports one for that
4208 condition and machine mode.
4210 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4211 the mode to use should they be constants. If it is VOIDmode, they cannot
4214 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4215 should be stored there. MODE is the mode to use should they be constants.
4216 If it is VOIDmode, they cannot both be constants.
4218 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4219 is not supported. */
4222 emit_conditional_move (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4223 machine_mode cmode
, rtx op2
, rtx op3
,
4224 machine_mode mode
, int unsignedp
)
4228 enum insn_code icode
;
4229 enum rtx_code reversed
;
4231 /* If the two source operands are identical, that's just a move. */
4233 if (rtx_equal_p (op2
, op3
))
4236 target
= gen_reg_rtx (mode
);
4238 emit_move_insn (target
, op3
);
4242 /* If one operand is constant, make it the second one. Only do this
4243 if the other operand is not constant as well. */
4245 if (swap_commutative_operands_p (op0
, op1
))
4247 std::swap (op0
, op1
);
4248 code
= swap_condition (code
);
4251 /* get_condition will prefer to generate LT and GT even if the old
4252 comparison was against zero, so undo that canonicalization here since
4253 comparisons against zero are cheaper. */
4254 if (code
== LT
&& op1
== const1_rtx
)
4255 code
= LE
, op1
= const0_rtx
;
4256 else if (code
== GT
&& op1
== constm1_rtx
)
4257 code
= GE
, op1
= const0_rtx
;
4259 if (cmode
== VOIDmode
)
4260 cmode
= GET_MODE (op0
);
4262 enum rtx_code orig_code
= code
;
4263 bool swapped
= false;
4264 if (swap_commutative_operands_p (op2
, op3
)
4265 && ((reversed
= reversed_comparison_code_parts (code
, op0
, op1
, NULL
))
4268 std::swap (op2
, op3
);
4273 if (mode
== VOIDmode
)
4274 mode
= GET_MODE (op2
);
4276 icode
= direct_optab_handler (movcc_optab
, mode
);
4278 if (icode
== CODE_FOR_nothing
)
4282 target
= gen_reg_rtx (mode
);
4284 for (int pass
= 0; ; pass
++)
4286 code
= unsignedp
? unsigned_condition (code
) : code
;
4287 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4289 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4290 punt and let the caller figure out how best to deal with this
4292 if (COMPARISON_P (comparison
))
4294 saved_pending_stack_adjust save
;
4295 save_pending_stack_adjust (&save
);
4296 last
= get_last_insn ();
4297 do_pending_stack_adjust ();
4298 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4299 GET_CODE (comparison
), NULL_RTX
, unsignedp
,
4300 OPTAB_WIDEN
, &comparison
, &cmode
);
4303 struct expand_operand ops
[4];
4305 create_output_operand (&ops
[0], target
, mode
);
4306 create_fixed_operand (&ops
[1], comparison
);
4307 create_input_operand (&ops
[2], op2
, mode
);
4308 create_input_operand (&ops
[3], op3
, mode
);
4309 if (maybe_expand_insn (icode
, 4, ops
))
4311 if (ops
[0].value
!= target
)
4312 convert_move (target
, ops
[0].value
, false);
4316 delete_insns_since (last
);
4317 restore_pending_stack_adjust (&save
);
4323 /* If the preferred op2/op3 order is not usable, retry with other
4324 operand order, perhaps it will expand successfully. */
4327 else if ((reversed
= reversed_comparison_code_parts (orig_code
, op0
, op1
,
4333 std::swap (op2
, op3
);
4338 /* Emit a conditional negate or bitwise complement using the
4339 negcc or notcc optabs if available. Return NULL_RTX if such operations
4340 are not available. Otherwise return the RTX holding the result.
4341 TARGET is the desired destination of the result. COMP is the comparison
4342 on which to negate. If COND is true move into TARGET the negation
4343 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
4344 CODE is either NEG or NOT. MODE is the machine mode in which the
4345 operation is performed. */
4348 emit_conditional_neg_or_complement (rtx target
, rtx_code code
,
4349 machine_mode mode
, rtx cond
, rtx op1
,
4352 optab op
= unknown_optab
;
4355 else if (code
== NOT
)
4360 insn_code icode
= direct_optab_handler (op
, mode
);
4362 if (icode
== CODE_FOR_nothing
)
4366 target
= gen_reg_rtx (mode
);
4368 rtx_insn
*last
= get_last_insn ();
4369 struct expand_operand ops
[4];
4371 create_output_operand (&ops
[0], target
, mode
);
4372 create_fixed_operand (&ops
[1], cond
);
4373 create_input_operand (&ops
[2], op1
, mode
);
4374 create_input_operand (&ops
[3], op2
, mode
);
4376 if (maybe_expand_insn (icode
, 4, ops
))
4378 if (ops
[0].value
!= target
)
4379 convert_move (target
, ops
[0].value
, false);
4383 delete_insns_since (last
);
4387 /* Emit a conditional addition instruction if the machine supports one for that
4388 condition and machine mode.
4390 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4391 the mode to use should they be constants. If it is VOIDmode, they cannot
4394 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4395 should be stored there. MODE is the mode to use should they be constants.
4396 If it is VOIDmode, they cannot both be constants.
4398 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4399 is not supported. */
4402 emit_conditional_add (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4403 machine_mode cmode
, rtx op2
, rtx op3
,
4404 machine_mode mode
, int unsignedp
)
4408 enum insn_code icode
;
4410 /* If one operand is constant, make it the second one. Only do this
4411 if the other operand is not constant as well. */
4413 if (swap_commutative_operands_p (op0
, op1
))
4415 std::swap (op0
, op1
);
4416 code
= swap_condition (code
);
4419 /* get_condition will prefer to generate LT and GT even if the old
4420 comparison was against zero, so undo that canonicalization here since
4421 comparisons against zero are cheaper. */
4422 if (code
== LT
&& op1
== const1_rtx
)
4423 code
= LE
, op1
= const0_rtx
;
4424 else if (code
== GT
&& op1
== constm1_rtx
)
4425 code
= GE
, op1
= const0_rtx
;
4427 if (cmode
== VOIDmode
)
4428 cmode
= GET_MODE (op0
);
4430 if (mode
== VOIDmode
)
4431 mode
= GET_MODE (op2
);
4433 icode
= optab_handler (addcc_optab
, mode
);
4435 if (icode
== CODE_FOR_nothing
)
4439 target
= gen_reg_rtx (mode
);
4441 code
= unsignedp
? unsigned_condition (code
) : code
;
4442 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4444 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4445 return NULL and let the caller figure out how best to deal with this
4447 if (!COMPARISON_P (comparison
))
4450 do_pending_stack_adjust ();
4451 last
= get_last_insn ();
4452 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4453 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4454 &comparison
, &cmode
);
4457 struct expand_operand ops
[4];
4459 create_output_operand (&ops
[0], target
, mode
);
4460 create_fixed_operand (&ops
[1], comparison
);
4461 create_input_operand (&ops
[2], op2
, mode
);
4462 create_input_operand (&ops
[3], op3
, mode
);
4463 if (maybe_expand_insn (icode
, 4, ops
))
4465 if (ops
[0].value
!= target
)
4466 convert_move (target
, ops
[0].value
, false);
4470 delete_insns_since (last
);
4474 /* These functions attempt to generate an insn body, rather than
4475 emitting the insn, but if the gen function already emits them, we
4476 make no attempt to turn them back into naked patterns. */
4478 /* Generate and return an insn body to add Y to X. */
4481 gen_add2_insn (rtx x
, rtx y
)
4483 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (x
));
4485 gcc_assert (insn_operand_matches (icode
, 0, x
));
4486 gcc_assert (insn_operand_matches (icode
, 1, x
));
4487 gcc_assert (insn_operand_matches (icode
, 2, y
));
4489 return GEN_FCN (icode
) (x
, x
, y
);
4492 /* Generate and return an insn body to add r1 and c,
4493 storing the result in r0. */
4496 gen_add3_insn (rtx r0
, rtx r1
, rtx c
)
4498 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (r0
));
4500 if (icode
== CODE_FOR_nothing
4501 || !insn_operand_matches (icode
, 0, r0
)
4502 || !insn_operand_matches (icode
, 1, r1
)
4503 || !insn_operand_matches (icode
, 2, c
))
4506 return GEN_FCN (icode
) (r0
, r1
, c
);
4510 have_add2_insn (rtx x
, rtx y
)
4512 enum insn_code icode
;
4514 gcc_assert (GET_MODE (x
) != VOIDmode
);
4516 icode
= optab_handler (add_optab
, GET_MODE (x
));
4518 if (icode
== CODE_FOR_nothing
)
4521 if (!insn_operand_matches (icode
, 0, x
)
4522 || !insn_operand_matches (icode
, 1, x
)
4523 || !insn_operand_matches (icode
, 2, y
))
4529 /* Generate and return an insn body to add Y to X. */
4532 gen_addptr3_insn (rtx x
, rtx y
, rtx z
)
4534 enum insn_code icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4536 gcc_assert (insn_operand_matches (icode
, 0, x
));
4537 gcc_assert (insn_operand_matches (icode
, 1, y
));
4538 gcc_assert (insn_operand_matches (icode
, 2, z
));
4540 return GEN_FCN (icode
) (x
, y
, z
);
4543 /* Return true if the target implements an addptr pattern and X, Y,
4544 and Z are valid for the pattern predicates. */
4547 have_addptr3_insn (rtx x
, rtx y
, rtx z
)
4549 enum insn_code icode
;
4551 gcc_assert (GET_MODE (x
) != VOIDmode
);
4553 icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4555 if (icode
== CODE_FOR_nothing
)
4558 if (!insn_operand_matches (icode
, 0, x
)
4559 || !insn_operand_matches (icode
, 1, y
)
4560 || !insn_operand_matches (icode
, 2, z
))
4566 /* Generate and return an insn body to subtract Y from X. */
4569 gen_sub2_insn (rtx x
, rtx y
)
4571 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (x
));
4573 gcc_assert (insn_operand_matches (icode
, 0, x
));
4574 gcc_assert (insn_operand_matches (icode
, 1, x
));
4575 gcc_assert (insn_operand_matches (icode
, 2, y
));
4577 return GEN_FCN (icode
) (x
, x
, y
);
4580 /* Generate and return an insn body to subtract r1 and c,
4581 storing the result in r0. */
4584 gen_sub3_insn (rtx r0
, rtx r1
, rtx c
)
4586 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (r0
));
4588 if (icode
== CODE_FOR_nothing
4589 || !insn_operand_matches (icode
, 0, r0
)
4590 || !insn_operand_matches (icode
, 1, r1
)
4591 || !insn_operand_matches (icode
, 2, c
))
4594 return GEN_FCN (icode
) (r0
, r1
, c
);
4598 have_sub2_insn (rtx x
, rtx y
)
4600 enum insn_code icode
;
4602 gcc_assert (GET_MODE (x
) != VOIDmode
);
4604 icode
= optab_handler (sub_optab
, GET_MODE (x
));
4606 if (icode
== CODE_FOR_nothing
)
4609 if (!insn_operand_matches (icode
, 0, x
)
4610 || !insn_operand_matches (icode
, 1, x
)
4611 || !insn_operand_matches (icode
, 2, y
))
4617 /* Generate the body of an insn to extend Y (with mode MFROM)
4618 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4621 gen_extend_insn (rtx x
, rtx y
, machine_mode mto
,
4622 machine_mode mfrom
, int unsignedp
)
4624 enum insn_code icode
= can_extend_p (mto
, mfrom
, unsignedp
);
4625 return GEN_FCN (icode
) (x
, y
);
4628 /* Generate code to convert FROM to floating point
4629 and store in TO. FROM must be fixed point and not VOIDmode.
4630 UNSIGNEDP nonzero means regard FROM as unsigned.
4631 Normally this is done by correcting the final value
4632 if it is negative. */
4635 expand_float (rtx to
, rtx from
, int unsignedp
)
4637 enum insn_code icode
;
4639 scalar_mode from_mode
, to_mode
;
4640 machine_mode fmode
, imode
;
4641 bool can_do_signed
= false;
4643 /* Crash now, because we won't be able to decide which mode to use. */
4644 gcc_assert (GET_MODE (from
) != VOIDmode
);
4646 /* Look for an insn to do the conversion. Do it in the specified
4647 modes if possible; otherwise convert either input, output or both to
4648 wider mode. If the integer mode is wider than the mode of FROM,
4649 we can do the conversion signed even if the input is unsigned. */
4651 FOR_EACH_MODE_FROM (fmode
, GET_MODE (to
))
4652 FOR_EACH_MODE_FROM (imode
, GET_MODE (from
))
4654 int doing_unsigned
= unsignedp
;
4656 if (fmode
!= GET_MODE (to
)
4657 && (significand_size (fmode
)
4658 < GET_MODE_UNIT_PRECISION (GET_MODE (from
))))
4661 icode
= can_float_p (fmode
, imode
, unsignedp
);
4662 if (icode
== CODE_FOR_nothing
&& unsignedp
)
4664 enum insn_code scode
= can_float_p (fmode
, imode
, 0);
4665 if (scode
!= CODE_FOR_nothing
)
4666 can_do_signed
= true;
4667 if (imode
!= GET_MODE (from
))
4668 icode
= scode
, doing_unsigned
= 0;
4671 if (icode
!= CODE_FOR_nothing
)
4673 if (imode
!= GET_MODE (from
))
4674 from
= convert_to_mode (imode
, from
, unsignedp
);
4676 if (fmode
!= GET_MODE (to
))
4677 target
= gen_reg_rtx (fmode
);
4679 emit_unop_insn (icode
, target
, from
,
4680 doing_unsigned
? UNSIGNED_FLOAT
: FLOAT
);
4683 convert_move (to
, target
, 0);
4688 /* Unsigned integer, and no way to convert directly. Convert as signed,
4689 then unconditionally adjust the result. */
4692 && is_a
<scalar_mode
> (GET_MODE (to
), &to_mode
)
4693 && is_a
<scalar_mode
> (GET_MODE (from
), &from_mode
))
4695 opt_scalar_mode fmode_iter
;
4696 rtx_code_label
*label
= gen_label_rtx ();
4698 REAL_VALUE_TYPE offset
;
4700 /* Look for a usable floating mode FMODE wider than the source and at
4701 least as wide as the target. Using FMODE will avoid rounding woes
4702 with unsigned values greater than the signed maximum value. */
4704 FOR_EACH_MODE_FROM (fmode_iter
, to_mode
)
4706 scalar_mode fmode
= fmode_iter
.require ();
4707 if (GET_MODE_PRECISION (from_mode
) < GET_MODE_BITSIZE (fmode
)
4708 && can_float_p (fmode
, from_mode
, 0) != CODE_FOR_nothing
)
4712 if (!fmode_iter
.exists (&fmode
))
4714 /* There is no such mode. Pretend the target is wide enough. */
4717 /* Avoid double-rounding when TO is narrower than FROM. */
4718 if ((significand_size (fmode
) + 1)
4719 < GET_MODE_PRECISION (from_mode
))
4722 rtx_code_label
*neglabel
= gen_label_rtx ();
4724 /* Don't use TARGET if it isn't a register, is a hard register,
4725 or is the wrong mode. */
4727 || REGNO (target
) < FIRST_PSEUDO_REGISTER
4728 || GET_MODE (target
) != fmode
)
4729 target
= gen_reg_rtx (fmode
);
4732 do_pending_stack_adjust ();
4734 /* Test whether the sign bit is set. */
4735 emit_cmp_and_jump_insns (from
, const0_rtx
, LT
, NULL_RTX
, imode
,
4738 /* The sign bit is not set. Convert as signed. */
4739 expand_float (target
, from
, 0);
4740 emit_jump_insn (targetm
.gen_jump (label
));
4743 /* The sign bit is set.
4744 Convert to a usable (positive signed) value by shifting right
4745 one bit, while remembering if a nonzero bit was shifted
4746 out; i.e., compute (from & 1) | (from >> 1). */
4748 emit_label (neglabel
);
4749 temp
= expand_binop (imode
, and_optab
, from
, const1_rtx
,
4750 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4751 temp1
= expand_shift (RSHIFT_EXPR
, imode
, from
, 1, NULL_RTX
, 1);
4752 temp
= expand_binop (imode
, ior_optab
, temp
, temp1
, temp
, 1,
4754 expand_float (target
, temp
, 0);
4756 /* Multiply by 2 to undo the shift above. */
4757 temp
= expand_binop (fmode
, add_optab
, target
, target
,
4758 target
, 0, OPTAB_LIB_WIDEN
);
4760 emit_move_insn (target
, temp
);
4762 do_pending_stack_adjust ();
4768 /* If we are about to do some arithmetic to correct for an
4769 unsigned operand, do it in a pseudo-register. */
4771 if (to_mode
!= fmode
4772 || !REG_P (to
) || REGNO (to
) < FIRST_PSEUDO_REGISTER
)
4773 target
= gen_reg_rtx (fmode
);
4775 /* Convert as signed integer to floating. */
4776 expand_float (target
, from
, 0);
4778 /* If FROM is negative (and therefore TO is negative),
4779 correct its value by 2**bitwidth. */
4781 do_pending_stack_adjust ();
4782 emit_cmp_and_jump_insns (from
, const0_rtx
, GE
, NULL_RTX
, from_mode
,
4786 real_2expN (&offset
, GET_MODE_PRECISION (from_mode
), fmode
);
4787 temp
= expand_binop (fmode
, add_optab
, target
,
4788 const_double_from_real_value (offset
, fmode
),
4789 target
, 0, OPTAB_LIB_WIDEN
);
4791 emit_move_insn (target
, temp
);
4793 do_pending_stack_adjust ();
4798 /* No hardware instruction available; call a library routine. */
4803 convert_optab tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
4805 if (GET_MODE_PRECISION (GET_MODE (from
)) < GET_MODE_PRECISION (SImode
))
4806 from
= convert_to_mode (SImode
, from
, unsignedp
);
4808 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
4809 gcc_assert (libfunc
);
4813 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4814 GET_MODE (to
), from
, GET_MODE (from
));
4815 insns
= get_insns ();
4818 emit_libcall_block (insns
, target
, value
,
4819 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FLOAT
: FLOAT
,
4820 GET_MODE (to
), from
));
4825 /* Copy result to requested destination
4826 if we have been computing in a temp location. */
4830 if (GET_MODE (target
) == GET_MODE (to
))
4831 emit_move_insn (to
, target
);
4833 convert_move (to
, target
, 0);
4837 /* Generate code to convert FROM to fixed point and store in TO. FROM
4838 must be floating point. */
4841 expand_fix (rtx to
, rtx from
, int unsignedp
)
4843 enum insn_code icode
;
4845 machine_mode fmode
, imode
;
4846 opt_scalar_mode fmode_iter
;
4847 bool must_trunc
= false;
4849 /* We first try to find a pair of modes, one real and one integer, at
4850 least as wide as FROM and TO, respectively, in which we can open-code
4851 this conversion. If the integer mode is wider than the mode of TO,
4852 we can do the conversion either signed or unsigned. */
4854 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
4855 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
4857 int doing_unsigned
= unsignedp
;
4859 icode
= can_fix_p (imode
, fmode
, unsignedp
, &must_trunc
);
4860 if (icode
== CODE_FOR_nothing
&& imode
!= GET_MODE (to
) && unsignedp
)
4861 icode
= can_fix_p (imode
, fmode
, 0, &must_trunc
), doing_unsigned
= 0;
4863 if (icode
!= CODE_FOR_nothing
)
4865 rtx_insn
*last
= get_last_insn ();
4866 if (fmode
!= GET_MODE (from
))
4867 from
= convert_to_mode (fmode
, from
, 0);
4871 rtx temp
= gen_reg_rtx (GET_MODE (from
));
4872 from
= expand_unop (GET_MODE (from
), ftrunc_optab
, from
,
4876 if (imode
!= GET_MODE (to
))
4877 target
= gen_reg_rtx (imode
);
4879 if (maybe_emit_unop_insn (icode
, target
, from
,
4880 doing_unsigned
? UNSIGNED_FIX
: FIX
))
4883 convert_move (to
, target
, unsignedp
);
4886 delete_insns_since (last
);
4890 /* For an unsigned conversion, there is one more way to do it.
4891 If we have a signed conversion, we generate code that compares
4892 the real value to the largest representable positive number. If if
4893 is smaller, the conversion is done normally. Otherwise, subtract
4894 one plus the highest signed number, convert, and add it back.
4896 We only need to check all real modes, since we know we didn't find
4897 anything with a wider integer mode.
4899 This code used to extend FP value into mode wider than the destination.
4900 This is needed for decimal float modes which cannot accurately
4901 represent one plus the highest signed number of the same size, but
4902 not for binary modes. Consider, for instance conversion from SFmode
4905 The hot path through the code is dealing with inputs smaller than 2^63
4906 and doing just the conversion, so there is no bits to lose.
4908 In the other path we know the value is positive in the range 2^63..2^64-1
4909 inclusive. (as for other input overflow happens and result is undefined)
4910 So we know that the most important bit set in mantissa corresponds to
4911 2^63. The subtraction of 2^63 should not generate any rounding as it
4912 simply clears out that bit. The rest is trivial. */
4914 scalar_int_mode to_mode
;
4916 && is_a
<scalar_int_mode
> (GET_MODE (to
), &to_mode
)
4917 && HWI_COMPUTABLE_MODE_P (to_mode
))
4918 FOR_EACH_MODE_FROM (fmode_iter
, as_a
<scalar_mode
> (GET_MODE (from
)))
4920 scalar_mode fmode
= fmode_iter
.require ();
4921 if (CODE_FOR_nothing
!= can_fix_p (to_mode
, fmode
,
4923 && (!DECIMAL_FLOAT_MODE_P (fmode
)
4924 || (GET_MODE_BITSIZE (fmode
) > GET_MODE_PRECISION (to_mode
))))
4927 REAL_VALUE_TYPE offset
;
4929 rtx_code_label
*lab1
, *lab2
;
4932 bitsize
= GET_MODE_PRECISION (to_mode
);
4933 real_2expN (&offset
, bitsize
- 1, fmode
);
4934 limit
= const_double_from_real_value (offset
, fmode
);
4935 lab1
= gen_label_rtx ();
4936 lab2
= gen_label_rtx ();
4938 if (fmode
!= GET_MODE (from
))
4939 from
= convert_to_mode (fmode
, from
, 0);
4941 /* See if we need to do the subtraction. */
4942 do_pending_stack_adjust ();
4943 emit_cmp_and_jump_insns (from
, limit
, GE
, NULL_RTX
,
4944 GET_MODE (from
), 0, lab1
);
4946 /* If not, do the signed "fix" and branch around fixup code. */
4947 expand_fix (to
, from
, 0);
4948 emit_jump_insn (targetm
.gen_jump (lab2
));
4951 /* Otherwise, subtract 2**(N-1), convert to signed number,
4952 then add 2**(N-1). Do the addition using XOR since this
4953 will often generate better code. */
4955 target
= expand_binop (GET_MODE (from
), sub_optab
, from
, limit
,
4956 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
4957 expand_fix (to
, target
, 0);
4958 target
= expand_binop (to_mode
, xor_optab
, to
,
4960 (HOST_WIDE_INT_1
<< (bitsize
- 1),
4962 to
, 1, OPTAB_LIB_WIDEN
);
4965 emit_move_insn (to
, target
);
4969 if (optab_handler (mov_optab
, to_mode
) != CODE_FOR_nothing
)
4971 /* Make a place for a REG_NOTE and add it. */
4972 insn
= emit_move_insn (to
, to
);
4973 set_dst_reg_note (insn
, REG_EQUAL
,
4974 gen_rtx_fmt_e (UNSIGNED_FIX
, to_mode
,
4983 /* We can't do it with an insn, so use a library call. But first ensure
4984 that the mode of TO is at least as wide as SImode, since those are the
4985 only library calls we know about. */
4987 if (GET_MODE_PRECISION (GET_MODE (to
)) < GET_MODE_PRECISION (SImode
))
4989 target
= gen_reg_rtx (SImode
);
4991 expand_fix (target
, from
, unsignedp
);
4999 convert_optab tab
= unsignedp
? ufix_optab
: sfix_optab
;
5000 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5001 gcc_assert (libfunc
);
5005 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5006 GET_MODE (to
), from
, GET_MODE (from
));
5007 insns
= get_insns ();
5010 emit_libcall_block (insns
, target
, value
,
5011 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FIX
: FIX
,
5012 GET_MODE (to
), from
));
5017 if (GET_MODE (to
) == GET_MODE (target
))
5018 emit_move_insn (to
, target
);
5020 convert_move (to
, target
, 0);
5025 /* Promote integer arguments for a libcall if necessary.
5026 emit_library_call_value cannot do the promotion because it does not
5027 know if it should do a signed or unsigned promotion. This is because
5028 there are no tree types defined for libcalls. */
5031 prepare_libcall_arg (rtx arg
, int uintp
)
5033 scalar_int_mode mode
;
5034 machine_mode arg_mode
;
5035 if (is_a
<scalar_int_mode
> (GET_MODE (arg
), &mode
))
5037 /* If we need to promote the integer function argument we need to do
5038 it here instead of inside emit_library_call_value because in
5039 emit_library_call_value we don't know if we should do a signed or
5040 unsigned promotion. */
5043 arg_mode
= promote_function_mode (NULL_TREE
, mode
,
5044 &unsigned_p
, NULL_TREE
, 0);
5045 if (arg_mode
!= mode
)
5046 return convert_to_mode (arg_mode
, arg
, uintp
);
5051 /* Generate code to convert FROM or TO a fixed-point.
5052 If UINTP is true, either TO or FROM is an unsigned integer.
5053 If SATP is true, we need to saturate the result. */
5056 expand_fixed_convert (rtx to
, rtx from
, int uintp
, int satp
)
5058 machine_mode to_mode
= GET_MODE (to
);
5059 machine_mode from_mode
= GET_MODE (from
);
5061 enum rtx_code this_code
;
5062 enum insn_code code
;
5067 if (to_mode
== from_mode
)
5069 emit_move_insn (to
, from
);
5075 tab
= satp
? satfractuns_optab
: fractuns_optab
;
5076 this_code
= satp
? UNSIGNED_SAT_FRACT
: UNSIGNED_FRACT_CONVERT
;
5080 tab
= satp
? satfract_optab
: fract_optab
;
5081 this_code
= satp
? SAT_FRACT
: FRACT_CONVERT
;
5083 code
= convert_optab_handler (tab
, to_mode
, from_mode
);
5084 if (code
!= CODE_FOR_nothing
)
5086 emit_unop_insn (code
, to
, from
, this_code
);
5090 libfunc
= convert_optab_libfunc (tab
, to_mode
, from_mode
);
5091 gcc_assert (libfunc
);
5093 from
= prepare_libcall_arg (from
, uintp
);
5094 from_mode
= GET_MODE (from
);
5097 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, to_mode
,
5099 insns
= get_insns ();
5102 emit_libcall_block (insns
, to
, value
,
5103 gen_rtx_fmt_e (optab_to_code (tab
), to_mode
, from
));
5106 /* Generate code to convert FROM to fixed point and store in TO. FROM
5107 must be floating point, TO must be signed. Use the conversion optab
5108 TAB to do the conversion. */
5111 expand_sfix_optab (rtx to
, rtx from
, convert_optab tab
)
5113 enum insn_code icode
;
5115 machine_mode fmode
, imode
;
5117 /* We first try to find a pair of modes, one real and one integer, at
5118 least as wide as FROM and TO, respectively, in which we can open-code
5119 this conversion. If the integer mode is wider than the mode of TO,
5120 we can do the conversion either signed or unsigned. */
5122 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5123 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5125 icode
= convert_optab_handler (tab
, imode
, fmode
);
5126 if (icode
!= CODE_FOR_nothing
)
5128 rtx_insn
*last
= get_last_insn ();
5129 if (fmode
!= GET_MODE (from
))
5130 from
= convert_to_mode (fmode
, from
, 0);
5132 if (imode
!= GET_MODE (to
))
5133 target
= gen_reg_rtx (imode
);
5135 if (!maybe_emit_unop_insn (icode
, target
, from
, UNKNOWN
))
5137 delete_insns_since (last
);
5141 convert_move (to
, target
, 0);
5149 /* Report whether we have an instruction to perform the operation
5150 specified by CODE on operands of mode MODE. */
5152 have_insn_for (enum rtx_code code
, machine_mode mode
)
5154 return (code_to_optab (code
)
5155 && (optab_handler (code_to_optab (code
), mode
)
5156 != CODE_FOR_nothing
));
5159 /* Print information about the current contents of the optabs on
5163 debug_optab_libfuncs (void)
5167 /* Dump the arithmetic optabs. */
5168 for (i
= FIRST_NORM_OPTAB
; i
<= LAST_NORMLIB_OPTAB
; ++i
)
5169 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5171 rtx l
= optab_libfunc ((optab
) i
, (machine_mode
) j
);
5174 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5175 fprintf (stderr
, "%s\t%s:\t%s\n",
5176 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5182 /* Dump the conversion optabs. */
5183 for (i
= FIRST_CONV_OPTAB
; i
<= LAST_CONVLIB_OPTAB
; ++i
)
5184 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5185 for (k
= 0; k
< NUM_MACHINE_MODES
; ++k
)
5187 rtx l
= convert_optab_libfunc ((optab
) i
, (machine_mode
) j
,
5191 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5192 fprintf (stderr
, "%s\t%s\t%s:\t%s\n",
5193 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5201 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5202 CODE. Return 0 on failure. */
5205 gen_cond_trap (enum rtx_code code
, rtx op1
, rtx op2
, rtx tcode
)
5207 machine_mode mode
= GET_MODE (op1
);
5208 enum insn_code icode
;
5212 if (mode
== VOIDmode
)
5215 icode
= optab_handler (ctrap_optab
, mode
);
5216 if (icode
== CODE_FOR_nothing
)
5219 /* Some targets only accept a zero trap code. */
5220 if (!insn_operand_matches (icode
, 3, tcode
))
5223 do_pending_stack_adjust ();
5225 prepare_cmp_insn (op1
, op2
, code
, NULL_RTX
, false, OPTAB_DIRECT
,
5230 insn
= GEN_FCN (icode
) (trap_rtx
, XEXP (trap_rtx
, 0), XEXP (trap_rtx
, 1),
5233 /* If that failed, then give up. */
5241 insn
= get_insns ();
5246 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
5247 or unsigned operation code. */
5250 get_rtx_code (enum tree_code tcode
, bool unsignedp
)
5262 code
= unsignedp
? LTU
: LT
;
5265 code
= unsignedp
? LEU
: LE
;
5268 code
= unsignedp
? GTU
: GT
;
5271 code
= unsignedp
? GEU
: GE
;
5274 case UNORDERED_EXPR
:
5313 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
5314 select signed or unsigned operators. OPNO holds the index of the
5315 first comparison operand for insn ICODE. Do not generate the
5316 compare instruction itself. */
5319 vector_compare_rtx (machine_mode cmp_mode
, enum tree_code tcode
,
5320 tree t_op0
, tree t_op1
, bool unsignedp
,
5321 enum insn_code icode
, unsigned int opno
)
5323 struct expand_operand ops
[2];
5324 rtx rtx_op0
, rtx_op1
;
5325 machine_mode m0
, m1
;
5326 enum rtx_code rcode
= get_rtx_code (tcode
, unsignedp
);
5328 gcc_assert (TREE_CODE_CLASS (tcode
) == tcc_comparison
);
5330 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
5331 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
5332 cases, use the original mode. */
5333 rtx_op0
= expand_expr (t_op0
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op0
)),
5335 m0
= GET_MODE (rtx_op0
);
5337 m0
= TYPE_MODE (TREE_TYPE (t_op0
));
5339 rtx_op1
= expand_expr (t_op1
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op1
)),
5341 m1
= GET_MODE (rtx_op1
);
5343 m1
= TYPE_MODE (TREE_TYPE (t_op1
));
5345 create_input_operand (&ops
[0], rtx_op0
, m0
);
5346 create_input_operand (&ops
[1], rtx_op1
, m1
);
5347 if (!maybe_legitimize_operands (icode
, opno
, 2, ops
))
5349 return gen_rtx_fmt_ee (rcode
, cmp_mode
, ops
[0].value
, ops
[1].value
);
5352 /* Checks if vec_perm mask SEL is a constant equivalent to a shift of the first
5353 vec_perm operand, assuming the second operand is a constant vector of zeroes.
5354 Return the shift distance in bits if so, or NULL_RTX if the vec_perm is not a
5357 shift_amt_for_vec_perm_mask (rtx sel
)
5359 unsigned int i
, first
, nelt
= GET_MODE_NUNITS (GET_MODE (sel
));
5360 unsigned int bitsize
= GET_MODE_UNIT_BITSIZE (GET_MODE (sel
));
5362 if (GET_CODE (sel
) != CONST_VECTOR
)
5365 first
= INTVAL (CONST_VECTOR_ELT (sel
, 0));
5368 for (i
= 1; i
< nelt
; i
++)
5370 int idx
= INTVAL (CONST_VECTOR_ELT (sel
, i
));
5371 unsigned int expected
= i
+ first
;
5372 /* Indices into the second vector are all equivalent. */
5373 if (idx
< 0 || (MIN (nelt
, (unsigned) idx
) != MIN (nelt
, expected
)))
5377 return GEN_INT (first
* bitsize
);
5380 /* A subroutine of expand_vec_perm for expanding one vec_perm insn. */
5383 expand_vec_perm_1 (enum insn_code icode
, rtx target
,
5384 rtx v0
, rtx v1
, rtx sel
)
5386 machine_mode tmode
= GET_MODE (target
);
5387 machine_mode smode
= GET_MODE (sel
);
5388 struct expand_operand ops
[4];
5390 create_output_operand (&ops
[0], target
, tmode
);
5391 create_input_operand (&ops
[3], sel
, smode
);
5393 /* Make an effort to preserve v0 == v1. The target expander is able to
5394 rely on this to determine if we're permuting a single input operand. */
5395 if (rtx_equal_p (v0
, v1
))
5397 if (!insn_operand_matches (icode
, 1, v0
))
5398 v0
= force_reg (tmode
, v0
);
5399 gcc_checking_assert (insn_operand_matches (icode
, 1, v0
));
5400 gcc_checking_assert (insn_operand_matches (icode
, 2, v0
));
5402 create_fixed_operand (&ops
[1], v0
);
5403 create_fixed_operand (&ops
[2], v0
);
5407 create_input_operand (&ops
[1], v0
, tmode
);
5408 create_input_operand (&ops
[2], v1
, tmode
);
5411 if (maybe_expand_insn (icode
, 4, ops
))
5412 return ops
[0].value
;
5416 /* Generate instructions for vec_perm optab given its mode
5417 and three operands. */
5420 expand_vec_perm (machine_mode mode
, rtx v0
, rtx v1
, rtx sel
, rtx target
)
5422 enum insn_code icode
;
5423 machine_mode qimode
;
5424 unsigned int i
, w
, e
, u
;
5425 rtx tmp
, sel_qi
= NULL
;
5428 if (!target
|| GET_MODE (target
) != mode
)
5429 target
= gen_reg_rtx (mode
);
5431 w
= GET_MODE_SIZE (mode
);
5432 e
= GET_MODE_NUNITS (mode
);
5433 u
= GET_MODE_UNIT_SIZE (mode
);
5435 /* Set QIMODE to a different vector mode with byte elements.
5436 If no such mode, or if MODE already has byte elements, use VOIDmode. */
5437 if (GET_MODE_INNER (mode
) == QImode
5438 || !mode_for_vector (QImode
, w
).exists (&qimode
)
5439 || !VECTOR_MODE_P (qimode
))
5442 /* If the input is a constant, expand it specially. */
5443 gcc_assert (GET_MODE_CLASS (GET_MODE (sel
)) == MODE_VECTOR_INT
);
5444 if (GET_CODE (sel
) == CONST_VECTOR
)
5446 /* See if this can be handled with a vec_shr. We only do this if the
5447 second vector is all zeroes. */
5448 enum insn_code shift_code
= optab_handler (vec_shr_optab
, mode
);
5449 enum insn_code shift_code_qi
= ((qimode
!= VOIDmode
&& qimode
!= mode
)
5450 ? optab_handler (vec_shr_optab
, qimode
)
5451 : CODE_FOR_nothing
);
5452 rtx shift_amt
= NULL_RTX
;
5453 if (v1
== CONST0_RTX (GET_MODE (v1
))
5454 && (shift_code
!= CODE_FOR_nothing
5455 || shift_code_qi
!= CODE_FOR_nothing
))
5457 shift_amt
= shift_amt_for_vec_perm_mask (sel
);
5460 struct expand_operand ops
[3];
5461 if (shift_code
!= CODE_FOR_nothing
)
5463 create_output_operand (&ops
[0], target
, mode
);
5464 create_input_operand (&ops
[1], v0
, mode
);
5465 create_convert_operand_from_type (&ops
[2], shift_amt
,
5467 if (maybe_expand_insn (shift_code
, 3, ops
))
5468 return ops
[0].value
;
5470 if (shift_code_qi
!= CODE_FOR_nothing
)
5472 tmp
= gen_reg_rtx (qimode
);
5473 create_output_operand (&ops
[0], tmp
, qimode
);
5474 create_input_operand (&ops
[1], gen_lowpart (qimode
, v0
),
5476 create_convert_operand_from_type (&ops
[2], shift_amt
,
5478 if (maybe_expand_insn (shift_code_qi
, 3, ops
))
5479 return gen_lowpart (mode
, ops
[0].value
);
5484 icode
= direct_optab_handler (vec_perm_const_optab
, mode
);
5485 if (icode
!= CODE_FOR_nothing
)
5487 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
5492 /* Fall back to a constant byte-based permutation. */
5493 if (qimode
!= VOIDmode
)
5495 vec
= rtvec_alloc (w
);
5496 for (i
= 0; i
< e
; ++i
)
5498 unsigned int j
, this_e
;
5500 this_e
= INTVAL (CONST_VECTOR_ELT (sel
, i
));
5501 this_e
&= 2 * e
- 1;
5504 for (j
= 0; j
< u
; ++j
)
5505 RTVEC_ELT (vec
, i
* u
+ j
) = GEN_INT (this_e
+ j
);
5507 sel_qi
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5509 icode
= direct_optab_handler (vec_perm_const_optab
, qimode
);
5510 if (icode
!= CODE_FOR_nothing
)
5512 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
5513 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
5514 gen_lowpart (qimode
, v1
), sel_qi
);
5516 return gen_lowpart (mode
, tmp
);
5521 /* Otherwise expand as a fully variable permuation. */
5522 icode
= direct_optab_handler (vec_perm_optab
, mode
);
5523 if (icode
!= CODE_FOR_nothing
)
5525 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
5530 /* As a special case to aid several targets, lower the element-based
5531 permutation to a byte-based permutation and try again. */
5532 if (qimode
== VOIDmode
)
5534 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
5535 if (icode
== CODE_FOR_nothing
)
5540 /* Multiply each element by its byte size. */
5541 machine_mode selmode
= GET_MODE (sel
);
5543 sel
= expand_simple_binop (selmode
, PLUS
, sel
, sel
,
5544 NULL
, 0, OPTAB_DIRECT
);
5546 sel
= expand_simple_binop (selmode
, ASHIFT
, sel
,
5547 GEN_INT (exact_log2 (u
)),
5548 NULL
, 0, OPTAB_DIRECT
);
5549 gcc_assert (sel
!= NULL
);
5551 /* Broadcast the low byte each element into each of its bytes. */
5552 vec
= rtvec_alloc (w
);
5553 for (i
= 0; i
< w
; ++i
)
5555 int this_e
= i
/ u
* u
;
5556 if (BYTES_BIG_ENDIAN
)
5558 RTVEC_ELT (vec
, i
) = GEN_INT (this_e
);
5560 tmp
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5561 sel
= gen_lowpart (qimode
, sel
);
5562 sel
= expand_vec_perm (qimode
, sel
, sel
, tmp
, NULL
);
5563 gcc_assert (sel
!= NULL
);
5565 /* Add the byte offset to each byte element. */
5566 /* Note that the definition of the indicies here is memory ordering,
5567 so there should be no difference between big and little endian. */
5568 vec
= rtvec_alloc (w
);
5569 for (i
= 0; i
< w
; ++i
)
5570 RTVEC_ELT (vec
, i
) = GEN_INT (i
% u
);
5571 tmp
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5572 sel_qi
= expand_simple_binop (qimode
, PLUS
, sel
, tmp
,
5573 sel
, 0, OPTAB_DIRECT
);
5574 gcc_assert (sel_qi
!= NULL
);
5577 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
5578 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
5579 gen_lowpart (qimode
, v1
), sel_qi
);
5581 tmp
= gen_lowpart (mode
, tmp
);
5585 /* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
5589 expand_vec_cond_mask_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
5592 struct expand_operand ops
[4];
5593 machine_mode mode
= TYPE_MODE (vec_cond_type
);
5594 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
5595 enum insn_code icode
= get_vcond_mask_icode (mode
, mask_mode
);
5596 rtx mask
, rtx_op1
, rtx_op2
;
5598 if (icode
== CODE_FOR_nothing
)
5601 mask
= expand_normal (op0
);
5602 rtx_op1
= expand_normal (op1
);
5603 rtx_op2
= expand_normal (op2
);
5605 mask
= force_reg (mask_mode
, mask
);
5606 rtx_op1
= force_reg (GET_MODE (rtx_op1
), rtx_op1
);
5608 create_output_operand (&ops
[0], target
, mode
);
5609 create_input_operand (&ops
[1], rtx_op1
, mode
);
5610 create_input_operand (&ops
[2], rtx_op2
, mode
);
5611 create_input_operand (&ops
[3], mask
, mask_mode
);
5612 expand_insn (icode
, 4, ops
);
5614 return ops
[0].value
;
5617 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
5621 expand_vec_cond_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
5624 struct expand_operand ops
[6];
5625 enum insn_code icode
;
5626 rtx comparison
, rtx_op1
, rtx_op2
;
5627 machine_mode mode
= TYPE_MODE (vec_cond_type
);
5628 machine_mode cmp_op_mode
;
5631 enum tree_code tcode
;
5633 if (COMPARISON_CLASS_P (op0
))
5635 op0a
= TREE_OPERAND (op0
, 0);
5636 op0b
= TREE_OPERAND (op0
, 1);
5637 tcode
= TREE_CODE (op0
);
5641 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0
)));
5642 if (get_vcond_mask_icode (mode
, TYPE_MODE (TREE_TYPE (op0
)))
5643 != CODE_FOR_nothing
)
5644 return expand_vec_cond_mask_expr (vec_cond_type
, op0
, op1
,
5649 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0
)))
5650 == MODE_VECTOR_INT
);
5652 op0b
= build_zero_cst (TREE_TYPE (op0
));
5656 cmp_op_mode
= TYPE_MODE (TREE_TYPE (op0a
));
5657 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
5660 gcc_assert (GET_MODE_SIZE (mode
) == GET_MODE_SIZE (cmp_op_mode
)
5661 && GET_MODE_NUNITS (mode
) == GET_MODE_NUNITS (cmp_op_mode
));
5663 icode
= get_vcond_icode (mode
, cmp_op_mode
, unsignedp
);
5664 if (icode
== CODE_FOR_nothing
)
5666 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
5667 icode
= get_vcond_eq_icode (mode
, cmp_op_mode
);
5668 if (icode
== CODE_FOR_nothing
)
5672 comparison
= vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
,
5674 rtx_op1
= expand_normal (op1
);
5675 rtx_op2
= expand_normal (op2
);
5677 create_output_operand (&ops
[0], target
, mode
);
5678 create_input_operand (&ops
[1], rtx_op1
, mode
);
5679 create_input_operand (&ops
[2], rtx_op2
, mode
);
5680 create_fixed_operand (&ops
[3], comparison
);
5681 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
5682 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
5683 expand_insn (icode
, 6, ops
);
5684 return ops
[0].value
;
5687 /* Generate insns for a vector comparison into a mask. */
5690 expand_vec_cmp_expr (tree type
, tree exp
, rtx target
)
5692 struct expand_operand ops
[4];
5693 enum insn_code icode
;
5695 machine_mode mask_mode
= TYPE_MODE (type
);
5699 enum tree_code tcode
;
5701 op0a
= TREE_OPERAND (exp
, 0);
5702 op0b
= TREE_OPERAND (exp
, 1);
5703 tcode
= TREE_CODE (exp
);
5705 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
5706 vmode
= TYPE_MODE (TREE_TYPE (op0a
));
5708 icode
= get_vec_cmp_icode (vmode
, mask_mode
, unsignedp
);
5709 if (icode
== CODE_FOR_nothing
)
5711 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
5712 icode
= get_vec_cmp_eq_icode (vmode
, mask_mode
);
5713 if (icode
== CODE_FOR_nothing
)
5717 comparison
= vector_compare_rtx (mask_mode
, tcode
, op0a
, op0b
,
5718 unsignedp
, icode
, 2);
5719 create_output_operand (&ops
[0], target
, mask_mode
);
5720 create_fixed_operand (&ops
[1], comparison
);
5721 create_fixed_operand (&ops
[2], XEXP (comparison
, 0));
5722 create_fixed_operand (&ops
[3], XEXP (comparison
, 1));
5723 expand_insn (icode
, 4, ops
);
5724 return ops
[0].value
;
5727 /* Expand a highpart multiply. */
5730 expand_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
5731 rtx target
, bool uns_p
)
5733 struct expand_operand eops
[3];
5734 enum insn_code icode
;
5735 int method
, i
, nunits
;
5741 method
= can_mult_highpart_p (mode
, uns_p
);
5747 tab1
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
5748 return expand_binop (mode
, tab1
, op0
, op1
, target
, uns_p
,
5751 tab1
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
5752 tab2
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
5755 tab1
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
5756 tab2
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
5757 if (BYTES_BIG_ENDIAN
)
5758 std::swap (tab1
, tab2
);
5764 icode
= optab_handler (tab1
, mode
);
5765 nunits
= GET_MODE_NUNITS (mode
);
5766 wmode
= insn_data
[icode
].operand
[0].mode
;
5767 gcc_checking_assert (2 * GET_MODE_NUNITS (wmode
) == nunits
);
5768 gcc_checking_assert (GET_MODE_SIZE (wmode
) == GET_MODE_SIZE (mode
));
5770 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
5771 create_input_operand (&eops
[1], op0
, mode
);
5772 create_input_operand (&eops
[2], op1
, mode
);
5773 expand_insn (icode
, 3, eops
);
5774 m1
= gen_lowpart (mode
, eops
[0].value
);
5776 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
5777 create_input_operand (&eops
[1], op0
, mode
);
5778 create_input_operand (&eops
[2], op1
, mode
);
5779 expand_insn (optab_handler (tab2
, mode
), 3, eops
);
5780 m2
= gen_lowpart (mode
, eops
[0].value
);
5782 v
= rtvec_alloc (nunits
);
5785 for (i
= 0; i
< nunits
; ++i
)
5786 RTVEC_ELT (v
, i
) = GEN_INT (!BYTES_BIG_ENDIAN
+ (i
& ~1)
5787 + ((i
& 1) ? nunits
: 0));
5791 for (i
= 0; i
< nunits
; ++i
)
5792 RTVEC_ELT (v
, i
) = GEN_INT (2 * i
+ (BYTES_BIG_ENDIAN
? 0 : 1));
5794 perm
= gen_rtx_CONST_VECTOR (mode
, v
);
5796 return expand_vec_perm (mode
, m1
, m2
, perm
, target
);
5799 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
5803 find_cc_set (rtx x
, const_rtx pat
, void *data
)
5805 if (REG_P (x
) && GET_MODE_CLASS (GET_MODE (x
)) == MODE_CC
5806 && GET_CODE (pat
) == SET
)
5808 rtx
*p_cc_reg
= (rtx
*) data
;
5809 gcc_assert (!*p_cc_reg
);
5814 /* This is a helper function for the other atomic operations. This function
5815 emits a loop that contains SEQ that iterates until a compare-and-swap
5816 operation at the end succeeds. MEM is the memory to be modified. SEQ is
5817 a set of instructions that takes a value from OLD_REG as an input and
5818 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
5819 set to the current contents of MEM. After SEQ, a compare-and-swap will
5820 attempt to update MEM with NEW_REG. The function returns true when the
5821 loop was generated successfully. */
5824 expand_compare_and_swap_loop (rtx mem
, rtx old_reg
, rtx new_reg
, rtx seq
)
5826 machine_mode mode
= GET_MODE (mem
);
5827 rtx_code_label
*label
;
5828 rtx cmp_reg
, success
, oldval
;
5830 /* The loop we want to generate looks like
5836 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
5840 Note that we only do the plain load from memory once. Subsequent
5841 iterations use the value loaded by the compare-and-swap pattern. */
5843 label
= gen_label_rtx ();
5844 cmp_reg
= gen_reg_rtx (mode
);
5846 emit_move_insn (cmp_reg
, mem
);
5848 emit_move_insn (old_reg
, cmp_reg
);
5854 if (!expand_atomic_compare_and_swap (&success
, &oldval
, mem
, old_reg
,
5855 new_reg
, false, MEMMODEL_SYNC_SEQ_CST
,
5859 if (oldval
!= cmp_reg
)
5860 emit_move_insn (cmp_reg
, oldval
);
5862 /* Mark this jump predicted not taken. */
5863 emit_cmp_and_jump_insns (success
, const0_rtx
, EQ
, const0_rtx
,
5864 GET_MODE (success
), 1, label
,
5865 profile_probability::guessed_never ());
5870 /* This function tries to emit an atomic_exchange intruction. VAL is written
5871 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
5872 using TARGET if possible. */
5875 maybe_emit_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
5877 machine_mode mode
= GET_MODE (mem
);
5878 enum insn_code icode
;
5880 /* If the target supports the exchange directly, great. */
5881 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
5882 if (icode
!= CODE_FOR_nothing
)
5884 struct expand_operand ops
[4];
5886 create_output_operand (&ops
[0], target
, mode
);
5887 create_fixed_operand (&ops
[1], mem
);
5888 create_input_operand (&ops
[2], val
, mode
);
5889 create_integer_operand (&ops
[3], model
);
5890 if (maybe_expand_insn (icode
, 4, ops
))
5891 return ops
[0].value
;
5897 /* This function tries to implement an atomic exchange operation using
5898 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
5899 The previous contents of *MEM are returned, using TARGET if possible.
5900 Since this instructionn is an acquire barrier only, stronger memory
5901 models may require additional barriers to be emitted. */
5904 maybe_emit_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
,
5905 enum memmodel model
)
5907 machine_mode mode
= GET_MODE (mem
);
5908 enum insn_code icode
;
5909 rtx_insn
*last_insn
= get_last_insn ();
5911 icode
= optab_handler (sync_lock_test_and_set_optab
, mode
);
5913 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
5914 exists, and the memory model is stronger than acquire, add a release
5915 barrier before the instruction. */
5917 if (is_mm_seq_cst (model
) || is_mm_release (model
) || is_mm_acq_rel (model
))
5918 expand_mem_thread_fence (model
);
5920 if (icode
!= CODE_FOR_nothing
)
5922 struct expand_operand ops
[3];
5923 create_output_operand (&ops
[0], target
, mode
);
5924 create_fixed_operand (&ops
[1], mem
);
5925 create_input_operand (&ops
[2], val
, mode
);
5926 if (maybe_expand_insn (icode
, 3, ops
))
5927 return ops
[0].value
;
5930 /* If an external test-and-set libcall is provided, use that instead of
5931 any external compare-and-swap that we might get from the compare-and-
5932 swap-loop expansion later. */
5933 if (!can_compare_and_swap_p (mode
, false))
5935 rtx libfunc
= optab_libfunc (sync_lock_test_and_set_optab
, mode
);
5936 if (libfunc
!= NULL
)
5940 addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
5941 return emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
5942 mode
, addr
, ptr_mode
,
5947 /* If the test_and_set can't be emitted, eliminate any barrier that might
5948 have been emitted. */
5949 delete_insns_since (last_insn
);
5953 /* This function tries to implement an atomic exchange operation using a
5954 compare_and_swap loop. VAL is written to *MEM. The previous contents of
5955 *MEM are returned, using TARGET if possible. No memory model is required
5956 since a compare_and_swap loop is seq-cst. */
5959 maybe_emit_compare_and_swap_exchange_loop (rtx target
, rtx mem
, rtx val
)
5961 machine_mode mode
= GET_MODE (mem
);
5963 if (can_compare_and_swap_p (mode
, true))
5965 if (!target
|| !register_operand (target
, mode
))
5966 target
= gen_reg_rtx (mode
);
5967 if (expand_compare_and_swap_loop (mem
, target
, val
, NULL_RTX
))
5974 /* This function tries to implement an atomic test-and-set operation
5975 using the atomic_test_and_set instruction pattern. A boolean value
5976 is returned from the operation, using TARGET if possible. */
5979 maybe_emit_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
5981 machine_mode pat_bool_mode
;
5982 struct expand_operand ops
[3];
5984 if (!targetm
.have_atomic_test_and_set ())
5987 /* While we always get QImode from __atomic_test_and_set, we get
5988 other memory modes from __sync_lock_test_and_set. Note that we
5989 use no endian adjustment here. This matches the 4.6 behavior
5990 in the Sparc backend. */
5991 enum insn_code icode
= targetm
.code_for_atomic_test_and_set
;
5992 gcc_checking_assert (insn_data
[icode
].operand
[1].mode
== QImode
);
5993 if (GET_MODE (mem
) != QImode
)
5994 mem
= adjust_address_nv (mem
, QImode
, 0);
5996 pat_bool_mode
= insn_data
[icode
].operand
[0].mode
;
5997 create_output_operand (&ops
[0], target
, pat_bool_mode
);
5998 create_fixed_operand (&ops
[1], mem
);
5999 create_integer_operand (&ops
[2], model
);
6001 if (maybe_expand_insn (icode
, 3, ops
))
6002 return ops
[0].value
;
6006 /* This function expands the legacy _sync_lock test_and_set operation which is
6007 generally an atomic exchange. Some limited targets only allow the
6008 constant 1 to be stored. This is an ACQUIRE operation.
6010 TARGET is an optional place to stick the return value.
6011 MEM is where VAL is stored. */
6014 expand_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
)
6018 /* Try an atomic_exchange first. */
6019 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, MEMMODEL_SYNC_ACQUIRE
);
6023 ret
= maybe_emit_sync_lock_test_and_set (target
, mem
, val
,
6024 MEMMODEL_SYNC_ACQUIRE
);
6028 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6032 /* If there are no other options, try atomic_test_and_set if the value
6033 being stored is 1. */
6034 if (val
== const1_rtx
)
6035 ret
= maybe_emit_atomic_test_and_set (target
, mem
, MEMMODEL_SYNC_ACQUIRE
);
6040 /* This function expands the atomic test_and_set operation:
6041 atomically store a boolean TRUE into MEM and return the previous value.
6043 MEMMODEL is the memory model variant to use.
6044 TARGET is an optional place to stick the return value. */
6047 expand_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6049 machine_mode mode
= GET_MODE (mem
);
6050 rtx ret
, trueval
, subtarget
;
6052 ret
= maybe_emit_atomic_test_and_set (target
, mem
, model
);
6056 /* Be binary compatible with non-default settings of trueval, and different
6057 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6058 another only has atomic-exchange. */
6059 if (targetm
.atomic_test_and_set_trueval
== 1)
6061 trueval
= const1_rtx
;
6062 subtarget
= target
? target
: gen_reg_rtx (mode
);
6066 trueval
= gen_int_mode (targetm
.atomic_test_and_set_trueval
, mode
);
6067 subtarget
= gen_reg_rtx (mode
);
6070 /* Try the atomic-exchange optab... */
6071 ret
= maybe_emit_atomic_exchange (subtarget
, mem
, trueval
, model
);
6073 /* ... then an atomic-compare-and-swap loop ... */
6075 ret
= maybe_emit_compare_and_swap_exchange_loop (subtarget
, mem
, trueval
);
6077 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6079 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, trueval
, model
);
6081 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6082 things with the value 1. Thus we try again without trueval. */
6083 if (!ret
&& targetm
.atomic_test_and_set_trueval
!= 1)
6084 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, const1_rtx
, model
);
6086 /* Failing all else, assume a single threaded environment and simply
6087 perform the operation. */
6090 /* If the result is ignored skip the move to target. */
6091 if (subtarget
!= const0_rtx
)
6092 emit_move_insn (subtarget
, mem
);
6094 emit_move_insn (mem
, trueval
);
6098 /* Recall that have to return a boolean value; rectify if trueval
6099 is not exactly one. */
6100 if (targetm
.atomic_test_and_set_trueval
!= 1)
6101 ret
= emit_store_flag_force (target
, NE
, ret
, const0_rtx
, mode
, 0, 1);
6106 /* This function expands the atomic exchange operation:
6107 atomically store VAL in MEM and return the previous value in MEM.
6109 MEMMODEL is the memory model variant to use.
6110 TARGET is an optional place to stick the return value. */
6113 expand_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6115 machine_mode mode
= GET_MODE (mem
);
6118 /* If loads are not atomic for the required size and we are not called to
6119 provide a __sync builtin, do not do anything so that we stay consistent
6120 with atomic loads of the same size. */
6121 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6124 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6126 /* Next try a compare-and-swap loop for the exchange. */
6128 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6133 /* This function expands the atomic compare exchange operation:
6135 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6136 *PTARGET_OVAL is an optional place to store the old value from memory.
6137 Both target parameters may be NULL or const0_rtx to indicate that we do
6138 not care about that return value. Both target parameters are updated on
6139 success to the actual location of the corresponding result.
6141 MEMMODEL is the memory model variant to use.
6143 The return value of the function is true for success. */
6146 expand_atomic_compare_and_swap (rtx
*ptarget_bool
, rtx
*ptarget_oval
,
6147 rtx mem
, rtx expected
, rtx desired
,
6148 bool is_weak
, enum memmodel succ_model
,
6149 enum memmodel fail_model
)
6151 machine_mode mode
= GET_MODE (mem
);
6152 struct expand_operand ops
[8];
6153 enum insn_code icode
;
6154 rtx target_oval
, target_bool
= NULL_RTX
;
6157 /* If loads are not atomic for the required size and we are not called to
6158 provide a __sync builtin, do not do anything so that we stay consistent
6159 with atomic loads of the same size. */
6160 if (!can_atomic_load_p (mode
) && !is_mm_sync (succ_model
))
6163 /* Load expected into a register for the compare and swap. */
6164 if (MEM_P (expected
))
6165 expected
= copy_to_reg (expected
);
6167 /* Make sure we always have some place to put the return oldval.
6168 Further, make sure that place is distinct from the input expected,
6169 just in case we need that path down below. */
6170 if (ptarget_oval
&& *ptarget_oval
== const0_rtx
)
6171 ptarget_oval
= NULL
;
6173 if (ptarget_oval
== NULL
6174 || (target_oval
= *ptarget_oval
) == NULL
6175 || reg_overlap_mentioned_p (expected
, target_oval
))
6176 target_oval
= gen_reg_rtx (mode
);
6178 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
6179 if (icode
!= CODE_FOR_nothing
)
6181 machine_mode bool_mode
= insn_data
[icode
].operand
[0].mode
;
6183 if (ptarget_bool
&& *ptarget_bool
== const0_rtx
)
6184 ptarget_bool
= NULL
;
6186 /* Make sure we always have a place for the bool operand. */
6187 if (ptarget_bool
== NULL
6188 || (target_bool
= *ptarget_bool
) == NULL
6189 || GET_MODE (target_bool
) != bool_mode
)
6190 target_bool
= gen_reg_rtx (bool_mode
);
6192 /* Emit the compare_and_swap. */
6193 create_output_operand (&ops
[0], target_bool
, bool_mode
);
6194 create_output_operand (&ops
[1], target_oval
, mode
);
6195 create_fixed_operand (&ops
[2], mem
);
6196 create_input_operand (&ops
[3], expected
, mode
);
6197 create_input_operand (&ops
[4], desired
, mode
);
6198 create_integer_operand (&ops
[5], is_weak
);
6199 create_integer_operand (&ops
[6], succ_model
);
6200 create_integer_operand (&ops
[7], fail_model
);
6201 if (maybe_expand_insn (icode
, 8, ops
))
6203 /* Return success/failure. */
6204 target_bool
= ops
[0].value
;
6205 target_oval
= ops
[1].value
;
6210 /* Otherwise fall back to the original __sync_val_compare_and_swap
6211 which is always seq-cst. */
6212 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
6213 if (icode
!= CODE_FOR_nothing
)
6217 create_output_operand (&ops
[0], target_oval
, mode
);
6218 create_fixed_operand (&ops
[1], mem
);
6219 create_input_operand (&ops
[2], expected
, mode
);
6220 create_input_operand (&ops
[3], desired
, mode
);
6221 if (!maybe_expand_insn (icode
, 4, ops
))
6224 target_oval
= ops
[0].value
;
6226 /* If the caller isn't interested in the boolean return value,
6227 skip the computation of it. */
6228 if (ptarget_bool
== NULL
)
6231 /* Otherwise, work out if the compare-and-swap succeeded. */
6233 if (have_insn_for (COMPARE
, CCmode
))
6234 note_stores (PATTERN (get_last_insn ()), find_cc_set
, &cc_reg
);
6237 target_bool
= emit_store_flag_force (target_bool
, EQ
, cc_reg
,
6238 const0_rtx
, VOIDmode
, 0, 1);
6241 goto success_bool_from_val
;
6244 /* Also check for library support for __sync_val_compare_and_swap. */
6245 libfunc
= optab_libfunc (sync_compare_and_swap_optab
, mode
);
6246 if (libfunc
!= NULL
)
6248 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6249 rtx target
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6250 mode
, addr
, ptr_mode
,
6251 expected
, mode
, desired
, mode
);
6252 emit_move_insn (target_oval
, target
);
6254 /* Compute the boolean return value only if requested. */
6256 goto success_bool_from_val
;
6264 success_bool_from_val
:
6265 target_bool
= emit_store_flag_force (target_bool
, EQ
, target_oval
,
6266 expected
, VOIDmode
, 1, 1);
6268 /* Make sure that the oval output winds up where the caller asked. */
6270 *ptarget_oval
= target_oval
;
6272 *ptarget_bool
= target_bool
;
6276 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
6279 expand_asm_memory_blockage (void)
6283 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
6284 rtvec_alloc (0), rtvec_alloc (0),
6285 rtvec_alloc (0), UNKNOWN_LOCATION
);
6286 MEM_VOLATILE_P (asm_op
) = 1;
6288 clob
= gen_rtx_SCRATCH (VOIDmode
);
6289 clob
= gen_rtx_MEM (BLKmode
, clob
);
6290 clob
= gen_rtx_CLOBBER (VOIDmode
, clob
);
6292 emit_insn (gen_rtx_PARALLEL (VOIDmode
, gen_rtvec (2, asm_op
, clob
)));
6295 /* Do not propagate memory accesses across this point. */
6298 expand_memory_blockage (void)
6300 if (targetm
.have_memory_blockage ())
6301 emit_insn (targetm
.gen_memory_blockage ());
6303 expand_asm_memory_blockage ();
6306 /* This routine will either emit the mem_thread_fence pattern or issue a
6307 sync_synchronize to generate a fence for memory model MEMMODEL. */
6310 expand_mem_thread_fence (enum memmodel model
)
6312 if (is_mm_relaxed (model
))
6314 if (targetm
.have_mem_thread_fence ())
6316 emit_insn (targetm
.gen_mem_thread_fence (GEN_INT (model
)));
6317 expand_memory_blockage ();
6319 else if (targetm
.have_memory_barrier ())
6320 emit_insn (targetm
.gen_memory_barrier ());
6321 else if (synchronize_libfunc
!= NULL_RTX
)
6322 emit_library_call (synchronize_libfunc
, LCT_NORMAL
, VOIDmode
);
6324 expand_memory_blockage ();
6327 /* Emit a signal fence with given memory model. */
6330 expand_mem_signal_fence (enum memmodel model
)
6332 /* No machine barrier is required to implement a signal fence, but
6333 a compiler memory barrier must be issued, except for relaxed MM. */
6334 if (!is_mm_relaxed (model
))
6335 expand_memory_blockage ();
6338 /* This function expands the atomic load operation:
6339 return the atomically loaded value in MEM.
6341 MEMMODEL is the memory model variant to use.
6342 TARGET is an option place to stick the return value. */
6345 expand_atomic_load (rtx target
, rtx mem
, enum memmodel model
)
6347 machine_mode mode
= GET_MODE (mem
);
6348 enum insn_code icode
;
6350 /* If the target supports the load directly, great. */
6351 icode
= direct_optab_handler (atomic_load_optab
, mode
);
6352 if (icode
!= CODE_FOR_nothing
)
6354 struct expand_operand ops
[3];
6355 rtx_insn
*last
= get_last_insn ();
6356 if (is_mm_seq_cst (model
))
6357 expand_memory_blockage ();
6359 create_output_operand (&ops
[0], target
, mode
);
6360 create_fixed_operand (&ops
[1], mem
);
6361 create_integer_operand (&ops
[2], model
);
6362 if (maybe_expand_insn (icode
, 3, ops
))
6364 if (!is_mm_relaxed (model
))
6365 expand_memory_blockage ();
6366 return ops
[0].value
;
6368 delete_insns_since (last
);
6371 /* If the size of the object is greater than word size on this target,
6372 then we assume that a load will not be atomic. We could try to
6373 emulate a load with a compare-and-swap operation, but the store that
6374 doing this could result in would be incorrect if this is a volatile
6375 atomic load or targetting read-only-mapped memory. */
6376 if (GET_MODE_PRECISION (mode
) > BITS_PER_WORD
)
6377 /* If there is no atomic load, leave the library call. */
6380 /* Otherwise assume loads are atomic, and emit the proper barriers. */
6381 if (!target
|| target
== const0_rtx
)
6382 target
= gen_reg_rtx (mode
);
6384 /* For SEQ_CST, emit a barrier before the load. */
6385 if (is_mm_seq_cst (model
))
6386 expand_mem_thread_fence (model
);
6388 emit_move_insn (target
, mem
);
6390 /* Emit the appropriate barrier after the load. */
6391 expand_mem_thread_fence (model
);
6396 /* This function expands the atomic store operation:
6397 Atomically store VAL in MEM.
6398 MEMMODEL is the memory model variant to use.
6399 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
6400 function returns const0_rtx if a pattern was emitted. */
6403 expand_atomic_store (rtx mem
, rtx val
, enum memmodel model
, bool use_release
)
6405 machine_mode mode
= GET_MODE (mem
);
6406 enum insn_code icode
;
6407 struct expand_operand ops
[3];
6409 /* If the target supports the store directly, great. */
6410 icode
= direct_optab_handler (atomic_store_optab
, mode
);
6411 if (icode
!= CODE_FOR_nothing
)
6413 rtx_insn
*last
= get_last_insn ();
6414 if (!is_mm_relaxed (model
))
6415 expand_memory_blockage ();
6416 create_fixed_operand (&ops
[0], mem
);
6417 create_input_operand (&ops
[1], val
, mode
);
6418 create_integer_operand (&ops
[2], model
);
6419 if (maybe_expand_insn (icode
, 3, ops
))
6421 if (is_mm_seq_cst (model
))
6422 expand_memory_blockage ();
6425 delete_insns_since (last
);
6428 /* If using __sync_lock_release is a viable alternative, try it.
6429 Note that this will not be set to true if we are expanding a generic
6430 __atomic_store_n. */
6433 icode
= direct_optab_handler (sync_lock_release_optab
, mode
);
6434 if (icode
!= CODE_FOR_nothing
)
6436 create_fixed_operand (&ops
[0], mem
);
6437 create_input_operand (&ops
[1], const0_rtx
, mode
);
6438 if (maybe_expand_insn (icode
, 2, ops
))
6440 /* lock_release is only a release barrier. */
6441 if (is_mm_seq_cst (model
))
6442 expand_mem_thread_fence (model
);
6448 /* If the size of the object is greater than word size on this target,
6449 a default store will not be atomic. */
6450 if (GET_MODE_PRECISION (mode
) > BITS_PER_WORD
)
6452 /* If loads are atomic or we are called to provide a __sync builtin,
6453 we can try a atomic_exchange and throw away the result. Otherwise,
6454 don't do anything so that we do not create an inconsistency between
6455 loads and stores. */
6456 if (can_atomic_load_p (mode
) || is_mm_sync (model
))
6458 rtx target
= maybe_emit_atomic_exchange (NULL_RTX
, mem
, val
, model
);
6460 target
= maybe_emit_compare_and_swap_exchange_loop (NULL_RTX
, mem
,
6468 /* Otherwise assume stores are atomic, and emit the proper barriers. */
6469 expand_mem_thread_fence (model
);
6471 emit_move_insn (mem
, val
);
6473 /* For SEQ_CST, also emit a barrier after the store. */
6474 if (is_mm_seq_cst (model
))
6475 expand_mem_thread_fence (model
);
6481 /* Structure containing the pointers and values required to process the
6482 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
6484 struct atomic_op_functions
6486 direct_optab mem_fetch_before
;
6487 direct_optab mem_fetch_after
;
6488 direct_optab mem_no_result
;
6491 direct_optab no_result
;
6492 enum rtx_code reverse_code
;
6496 /* Fill in structure pointed to by OP with the various optab entries for an
6497 operation of type CODE. */
6500 get_atomic_op_for_code (struct atomic_op_functions
*op
, enum rtx_code code
)
6502 gcc_assert (op
!= NULL
);
6504 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
6505 in the source code during compilation, and the optab entries are not
6506 computable until runtime. Fill in the values at runtime. */
6510 op
->mem_fetch_before
= atomic_fetch_add_optab
;
6511 op
->mem_fetch_after
= atomic_add_fetch_optab
;
6512 op
->mem_no_result
= atomic_add_optab
;
6513 op
->fetch_before
= sync_old_add_optab
;
6514 op
->fetch_after
= sync_new_add_optab
;
6515 op
->no_result
= sync_add_optab
;
6516 op
->reverse_code
= MINUS
;
6519 op
->mem_fetch_before
= atomic_fetch_sub_optab
;
6520 op
->mem_fetch_after
= atomic_sub_fetch_optab
;
6521 op
->mem_no_result
= atomic_sub_optab
;
6522 op
->fetch_before
= sync_old_sub_optab
;
6523 op
->fetch_after
= sync_new_sub_optab
;
6524 op
->no_result
= sync_sub_optab
;
6525 op
->reverse_code
= PLUS
;
6528 op
->mem_fetch_before
= atomic_fetch_xor_optab
;
6529 op
->mem_fetch_after
= atomic_xor_fetch_optab
;
6530 op
->mem_no_result
= atomic_xor_optab
;
6531 op
->fetch_before
= sync_old_xor_optab
;
6532 op
->fetch_after
= sync_new_xor_optab
;
6533 op
->no_result
= sync_xor_optab
;
6534 op
->reverse_code
= XOR
;
6537 op
->mem_fetch_before
= atomic_fetch_and_optab
;
6538 op
->mem_fetch_after
= atomic_and_fetch_optab
;
6539 op
->mem_no_result
= atomic_and_optab
;
6540 op
->fetch_before
= sync_old_and_optab
;
6541 op
->fetch_after
= sync_new_and_optab
;
6542 op
->no_result
= sync_and_optab
;
6543 op
->reverse_code
= UNKNOWN
;
6546 op
->mem_fetch_before
= atomic_fetch_or_optab
;
6547 op
->mem_fetch_after
= atomic_or_fetch_optab
;
6548 op
->mem_no_result
= atomic_or_optab
;
6549 op
->fetch_before
= sync_old_ior_optab
;
6550 op
->fetch_after
= sync_new_ior_optab
;
6551 op
->no_result
= sync_ior_optab
;
6552 op
->reverse_code
= UNKNOWN
;
6555 op
->mem_fetch_before
= atomic_fetch_nand_optab
;
6556 op
->mem_fetch_after
= atomic_nand_fetch_optab
;
6557 op
->mem_no_result
= atomic_nand_optab
;
6558 op
->fetch_before
= sync_old_nand_optab
;
6559 op
->fetch_after
= sync_new_nand_optab
;
6560 op
->no_result
= sync_nand_optab
;
6561 op
->reverse_code
= UNKNOWN
;
6568 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
6569 using memory order MODEL. If AFTER is true the operation needs to return
6570 the value of *MEM after the operation, otherwise the previous value.
6571 TARGET is an optional place to place the result. The result is unused if
6573 Return the result if there is a better sequence, otherwise NULL_RTX. */
6576 maybe_optimize_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
6577 enum memmodel model
, bool after
)
6579 /* If the value is prefetched, or not used, it may be possible to replace
6580 the sequence with a native exchange operation. */
6581 if (!after
|| target
== const0_rtx
)
6583 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
6584 if (code
== AND
&& val
== const0_rtx
)
6586 if (target
== const0_rtx
)
6587 target
= gen_reg_rtx (GET_MODE (mem
));
6588 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6591 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
6592 if (code
== IOR
&& val
== constm1_rtx
)
6594 if (target
== const0_rtx
)
6595 target
= gen_reg_rtx (GET_MODE (mem
));
6596 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6603 /* Try to emit an instruction for a specific operation varaition.
6604 OPTAB contains the OP functions.
6605 TARGET is an optional place to return the result. const0_rtx means unused.
6606 MEM is the memory location to operate on.
6607 VAL is the value to use in the operation.
6608 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
6609 MODEL is the memory model, if used.
6610 AFTER is true if the returned result is the value after the operation. */
6613 maybe_emit_op (const struct atomic_op_functions
*optab
, rtx target
, rtx mem
,
6614 rtx val
, bool use_memmodel
, enum memmodel model
, bool after
)
6616 machine_mode mode
= GET_MODE (mem
);
6617 struct expand_operand ops
[4];
6618 enum insn_code icode
;
6622 /* Check to see if there is a result returned. */
6623 if (target
== const0_rtx
)
6627 icode
= direct_optab_handler (optab
->mem_no_result
, mode
);
6628 create_integer_operand (&ops
[2], model
);
6633 icode
= direct_optab_handler (optab
->no_result
, mode
);
6637 /* Otherwise, we need to generate a result. */
6642 icode
= direct_optab_handler (after
? optab
->mem_fetch_after
6643 : optab
->mem_fetch_before
, mode
);
6644 create_integer_operand (&ops
[3], model
);
6649 icode
= optab_handler (after
? optab
->fetch_after
6650 : optab
->fetch_before
, mode
);
6653 create_output_operand (&ops
[op_counter
++], target
, mode
);
6655 if (icode
== CODE_FOR_nothing
)
6658 create_fixed_operand (&ops
[op_counter
++], mem
);
6659 /* VAL may have been promoted to a wider mode. Shrink it if so. */
6660 create_convert_operand_to (&ops
[op_counter
++], val
, mode
, true);
6662 if (maybe_expand_insn (icode
, num_ops
, ops
))
6663 return (target
== const0_rtx
? const0_rtx
: ops
[0].value
);
6669 /* This function expands an atomic fetch_OP or OP_fetch operation:
6670 TARGET is an option place to stick the return value. const0_rtx indicates
6671 the result is unused.
6672 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6673 CODE is the operation being performed (OP)
6674 MEMMODEL is the memory model variant to use.
6675 AFTER is true to return the result of the operation (OP_fetch).
6676 AFTER is false to return the value before the operation (fetch_OP).
6678 This function will *only* generate instructions if there is a direct
6679 optab. No compare and swap loops or libcalls will be generated. */
6682 expand_atomic_fetch_op_no_fallback (rtx target
, rtx mem
, rtx val
,
6683 enum rtx_code code
, enum memmodel model
,
6686 machine_mode mode
= GET_MODE (mem
);
6687 struct atomic_op_functions optab
;
6689 bool unused_result
= (target
== const0_rtx
);
6691 get_atomic_op_for_code (&optab
, code
);
6693 /* Check to see if there are any better instructions. */
6694 result
= maybe_optimize_fetch_op (target
, mem
, val
, code
, model
, after
);
6698 /* Check for the case where the result isn't used and try those patterns. */
6701 /* Try the memory model variant first. */
6702 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, true);
6706 /* Next try the old style withuot a memory model. */
6707 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, true);
6711 /* There is no no-result pattern, so try patterns with a result. */
6715 /* Try the __atomic version. */
6716 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, after
);
6720 /* Try the older __sync version. */
6721 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, after
);
6725 /* If the fetch value can be calculated from the other variation of fetch,
6726 try that operation. */
6727 if (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
)
6729 /* Try the __atomic version, then the older __sync version. */
6730 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, !after
);
6732 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, !after
);
6736 /* If the result isn't used, no need to do compensation code. */
6740 /* Issue compensation code. Fetch_after == fetch_before OP val.
6741 Fetch_before == after REVERSE_OP val. */
6743 code
= optab
.reverse_code
;
6746 result
= expand_simple_binop (mode
, AND
, result
, val
, NULL_RTX
,
6747 true, OPTAB_LIB_WIDEN
);
6748 result
= expand_simple_unop (mode
, NOT
, result
, target
, true);
6751 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
6752 true, OPTAB_LIB_WIDEN
);
6757 /* No direct opcode can be generated. */
6763 /* This function expands an atomic fetch_OP or OP_fetch operation:
6764 TARGET is an option place to stick the return value. const0_rtx indicates
6765 the result is unused.
6766 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6767 CODE is the operation being performed (OP)
6768 MEMMODEL is the memory model variant to use.
6769 AFTER is true to return the result of the operation (OP_fetch).
6770 AFTER is false to return the value before the operation (fetch_OP). */
6772 expand_atomic_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
6773 enum memmodel model
, bool after
)
6775 machine_mode mode
= GET_MODE (mem
);
6777 bool unused_result
= (target
== const0_rtx
);
6779 /* If loads are not atomic for the required size and we are not called to
6780 provide a __sync builtin, do not do anything so that we stay consistent
6781 with atomic loads of the same size. */
6782 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6785 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, val
, code
, model
,
6791 /* Add/sub can be implemented by doing the reverse operation with -(val). */
6792 if (code
== PLUS
|| code
== MINUS
)
6795 enum rtx_code reverse
= (code
== PLUS
? MINUS
: PLUS
);
6798 tmp
= expand_simple_unop (mode
, NEG
, val
, NULL_RTX
, true);
6799 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, tmp
, reverse
,
6803 /* PLUS worked so emit the insns and return. */
6810 /* PLUS did not work, so throw away the negation code and continue. */
6814 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
6815 if (!can_compare_and_swap_p (mode
, false))
6819 enum rtx_code orig_code
= code
;
6820 struct atomic_op_functions optab
;
6822 get_atomic_op_for_code (&optab
, code
);
6823 libfunc
= optab_libfunc (after
? optab
.fetch_after
6824 : optab
.fetch_before
, mode
);
6826 && (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
))
6830 code
= optab
.reverse_code
;
6831 libfunc
= optab_libfunc (after
? optab
.fetch_before
6832 : optab
.fetch_after
, mode
);
6834 if (libfunc
!= NULL
)
6836 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6837 result
= emit_library_call_value (libfunc
, NULL
, LCT_NORMAL
, mode
,
6838 addr
, ptr_mode
, val
, mode
);
6840 if (!unused_result
&& fixup
)
6841 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
6842 true, OPTAB_LIB_WIDEN
);
6846 /* We need the original code for any further attempts. */
6850 /* If nothing else has succeeded, default to a compare and swap loop. */
6851 if (can_compare_and_swap_p (mode
, true))
6854 rtx t0
= gen_reg_rtx (mode
), t1
;
6858 /* If the result is used, get a register for it. */
6861 if (!target
|| !register_operand (target
, mode
))
6862 target
= gen_reg_rtx (mode
);
6863 /* If fetch_before, copy the value now. */
6865 emit_move_insn (target
, t0
);
6868 target
= const0_rtx
;
6873 t1
= expand_simple_binop (mode
, AND
, t1
, val
, NULL_RTX
,
6874 true, OPTAB_LIB_WIDEN
);
6875 t1
= expand_simple_unop (mode
, code
, t1
, NULL_RTX
, true);
6878 t1
= expand_simple_binop (mode
, code
, t1
, val
, NULL_RTX
, true,
6881 /* For after, copy the value now. */
6882 if (!unused_result
&& after
)
6883 emit_move_insn (target
, t1
);
6884 insn
= get_insns ();
6887 if (t1
!= NULL
&& expand_compare_and_swap_loop (mem
, t0
, t1
, insn
))
6894 /* Return true if OPERAND is suitable for operand number OPNO of
6895 instruction ICODE. */
6898 insn_operand_matches (enum insn_code icode
, unsigned int opno
, rtx operand
)
6900 return (!insn_data
[(int) icode
].operand
[opno
].predicate
6901 || (insn_data
[(int) icode
].operand
[opno
].predicate
6902 (operand
, insn_data
[(int) icode
].operand
[opno
].mode
)));
6905 /* TARGET is a target of a multiword operation that we are going to
6906 implement as a series of word-mode operations. Return true if
6907 TARGET is suitable for this purpose. */
6910 valid_multiword_target_p (rtx target
)
6915 mode
= GET_MODE (target
);
6916 for (i
= 0; i
< GET_MODE_SIZE (mode
); i
+= UNITS_PER_WORD
)
6917 if (!validate_subreg (word_mode
, mode
, target
, i
))
6922 /* Like maybe_legitimize_operand, but do not change the code of the
6923 current rtx value. */
6926 maybe_legitimize_operand_same_code (enum insn_code icode
, unsigned int opno
,
6927 struct expand_operand
*op
)
6929 /* See if the operand matches in its current form. */
6930 if (insn_operand_matches (icode
, opno
, op
->value
))
6933 /* If the operand is a memory whose address has no side effects,
6934 try forcing the address into a non-virtual pseudo register.
6935 The check for side effects is important because copy_to_mode_reg
6936 cannot handle things like auto-modified addresses. */
6937 if (insn_data
[(int) icode
].operand
[opno
].allows_mem
&& MEM_P (op
->value
))
6942 addr
= XEXP (mem
, 0);
6943 if (!(REG_P (addr
) && REGNO (addr
) > LAST_VIRTUAL_REGISTER
)
6944 && !side_effects_p (addr
))
6949 last
= get_last_insn ();
6950 mode
= get_address_mode (mem
);
6951 mem
= replace_equiv_address (mem
, copy_to_mode_reg (mode
, addr
));
6952 if (insn_operand_matches (icode
, opno
, mem
))
6957 delete_insns_since (last
);
6964 /* Try to make OP match operand OPNO of instruction ICODE. Return true
6965 on success, storing the new operand value back in OP. */
6968 maybe_legitimize_operand (enum insn_code icode
, unsigned int opno
,
6969 struct expand_operand
*op
)
6971 machine_mode mode
, imode
;
6972 bool old_volatile_ok
, result
;
6978 old_volatile_ok
= volatile_ok
;
6980 result
= maybe_legitimize_operand_same_code (icode
, opno
, op
);
6981 volatile_ok
= old_volatile_ok
;
6985 gcc_assert (mode
!= VOIDmode
);
6987 && op
->value
!= const0_rtx
6988 && GET_MODE (op
->value
) == mode
6989 && maybe_legitimize_operand_same_code (icode
, opno
, op
))
6992 op
->value
= gen_reg_rtx (mode
);
6998 gcc_assert (mode
!= VOIDmode
);
6999 gcc_assert (GET_MODE (op
->value
) == VOIDmode
7000 || GET_MODE (op
->value
) == mode
);
7001 if (maybe_legitimize_operand_same_code (icode
, opno
, op
))
7004 op
->value
= copy_to_mode_reg (mode
, op
->value
);
7007 case EXPAND_CONVERT_TO
:
7008 gcc_assert (mode
!= VOIDmode
);
7009 op
->value
= convert_to_mode (mode
, op
->value
, op
->unsigned_p
);
7012 case EXPAND_CONVERT_FROM
:
7013 if (GET_MODE (op
->value
) != VOIDmode
)
7014 mode
= GET_MODE (op
->value
);
7016 /* The caller must tell us what mode this value has. */
7017 gcc_assert (mode
!= VOIDmode
);
7019 imode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7020 if (imode
!= VOIDmode
&& imode
!= mode
)
7022 op
->value
= convert_modes (imode
, mode
, op
->value
, op
->unsigned_p
);
7027 case EXPAND_ADDRESS
:
7028 op
->value
= convert_memory_address (as_a
<scalar_int_mode
> (mode
),
7032 case EXPAND_INTEGER
:
7033 mode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7034 if (mode
!= VOIDmode
&& const_int_operand (op
->value
, mode
))
7038 return insn_operand_matches (icode
, opno
, op
->value
);
7041 /* Make OP describe an input operand that should have the same value
7042 as VALUE, after any mode conversion that the target might request.
7043 TYPE is the type of VALUE. */
7046 create_convert_operand_from_type (struct expand_operand
*op
,
7047 rtx value
, tree type
)
7049 create_convert_operand_from (op
, value
, TYPE_MODE (type
),
7050 TYPE_UNSIGNED (type
));
7053 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7054 of instruction ICODE. Return true on success, leaving the new operand
7055 values in the OPS themselves. Emit no code on failure. */
7058 maybe_legitimize_operands (enum insn_code icode
, unsigned int opno
,
7059 unsigned int nops
, struct expand_operand
*ops
)
7064 last
= get_last_insn ();
7065 for (i
= 0; i
< nops
; i
++)
7066 if (!maybe_legitimize_operand (icode
, opno
+ i
, &ops
[i
]))
7068 delete_insns_since (last
);
7074 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7075 as its operands. Return the instruction pattern on success,
7076 and emit any necessary set-up code. Return null and emit no
7080 maybe_gen_insn (enum insn_code icode
, unsigned int nops
,
7081 struct expand_operand
*ops
)
7083 gcc_assert (nops
== (unsigned int) insn_data
[(int) icode
].n_generator_args
);
7084 if (!maybe_legitimize_operands (icode
, 0, nops
, ops
))
7090 return GEN_FCN (icode
) (ops
[0].value
);
7092 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
);
7094 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
);
7096 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7099 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7100 ops
[3].value
, ops
[4].value
);
7102 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7103 ops
[3].value
, ops
[4].value
, ops
[5].value
);
7105 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7106 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7109 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7110 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7111 ops
[6].value
, ops
[7].value
);
7113 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7114 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7115 ops
[6].value
, ops
[7].value
, ops
[8].value
);
7120 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7121 as its operands. Return true on success and emit no code on failure. */
7124 maybe_expand_insn (enum insn_code icode
, unsigned int nops
,
7125 struct expand_operand
*ops
)
7127 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7136 /* Like maybe_expand_insn, but for jumps. */
7139 maybe_expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7140 struct expand_operand
*ops
)
7142 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7145 emit_jump_insn (pat
);
7151 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
7155 expand_insn (enum insn_code icode
, unsigned int nops
,
7156 struct expand_operand
*ops
)
7158 if (!maybe_expand_insn (icode
, nops
, ops
))
7162 /* Like expand_insn, but for jumps. */
7165 expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7166 struct expand_operand
*ops
)
7168 if (!maybe_expand_jump_insn (icode
, nops
, ops
))