]> git.ipfire.org Git - thirdparty/gcc.git/blob - gcc/optabs.c
Merge from trunk.
[thirdparty/gcc.git] / gcc / optabs.c
1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2013 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
10
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "tm.h"
25 #include "diagnostic-core.h"
26
27 /* Include insn-config.h before expr.h so that HAVE_conditional_move
28 is properly defined. */
29 #include "insn-config.h"
30 #include "rtl.h"
31 #include "tree.h"
32 #include "stor-layout.h"
33 #include "stringpool.h"
34 #include "varasm.h"
35 #include "tm_p.h"
36 #include "flags.h"
37 #include "function.h"
38 #include "except.h"
39 #include "expr.h"
40 #include "optabs.h"
41 #include "libfuncs.h"
42 #include "recog.h"
43 #include "reload.h"
44 #include "ggc.h"
45 #include "basic-block.h"
46 #include "target.h"
47
48 struct target_optabs default_target_optabs;
49 struct target_libfuncs default_target_libfuncs;
50 struct target_optabs *this_fn_optabs = &default_target_optabs;
51 #if SWITCHABLE_TARGET
52 struct target_optabs *this_target_optabs = &default_target_optabs;
53 struct target_libfuncs *this_target_libfuncs = &default_target_libfuncs;
54 #endif
55
56 #define libfunc_hash \
57 (this_target_libfuncs->x_libfunc_hash)
58
59 static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
60 enum machine_mode *);
61 static rtx expand_unop_direct (enum machine_mode, optab, rtx, rtx, int);
62 static void emit_libcall_block_1 (rtx, rtx, rtx, rtx, bool);
63
64 /* Debug facility for use in GDB. */
65 void debug_optab_libfuncs (void);
66
67 /* Prefixes for the current version of decimal floating point (BID vs. DPD) */
68 #if ENABLE_DECIMAL_BID_FORMAT
69 #define DECIMAL_PREFIX "bid_"
70 #else
71 #define DECIMAL_PREFIX "dpd_"
72 #endif
73 \f
74 /* Used for libfunc_hash. */
75
76 static hashval_t
77 hash_libfunc (const void *p)
78 {
79 const struct libfunc_entry *const e = (const struct libfunc_entry *) p;
80 return ((e->mode1 + e->mode2 * NUM_MACHINE_MODES) ^ e->op);
81 }
82
83 /* Used for libfunc_hash. */
84
85 static int
86 eq_libfunc (const void *p, const void *q)
87 {
88 const struct libfunc_entry *const e1 = (const struct libfunc_entry *) p;
89 const struct libfunc_entry *const e2 = (const struct libfunc_entry *) q;
90 return e1->op == e2->op && e1->mode1 == e2->mode1 && e1->mode2 == e2->mode2;
91 }
92
93 /* Return libfunc corresponding operation defined by OPTAB converting
94 from MODE2 to MODE1. Trigger lazy initialization if needed, return NULL
95 if no libfunc is available. */
96 rtx
97 convert_optab_libfunc (convert_optab optab, enum machine_mode mode1,
98 enum machine_mode mode2)
99 {
100 struct libfunc_entry e;
101 struct libfunc_entry **slot;
102
103 /* ??? This ought to be an assert, but not all of the places
104 that we expand optabs know about the optabs that got moved
105 to being direct. */
106 if (!(optab >= FIRST_CONV_OPTAB && optab <= LAST_CONVLIB_OPTAB))
107 return NULL_RTX;
108
109 e.op = optab;
110 e.mode1 = mode1;
111 e.mode2 = mode2;
112 slot = (struct libfunc_entry **)
113 htab_find_slot (libfunc_hash, &e, NO_INSERT);
114 if (!slot)
115 {
116 const struct convert_optab_libcall_d *d
117 = &convlib_def[optab - FIRST_CONV_OPTAB];
118
119 if (d->libcall_gen == NULL)
120 return NULL;
121
122 d->libcall_gen (optab, d->libcall_basename, mode1, mode2);
123 slot = (struct libfunc_entry **)
124 htab_find_slot (libfunc_hash, &e, NO_INSERT);
125 if (!slot)
126 return NULL;
127 }
128 return (*slot)->libfunc;
129 }
130
131 /* Return libfunc corresponding operation defined by OPTAB in MODE.
132 Trigger lazy initialization if needed, return NULL if no libfunc is
133 available. */
134 rtx
135 optab_libfunc (optab optab, enum machine_mode mode)
136 {
137 struct libfunc_entry e;
138 struct libfunc_entry **slot;
139
140 /* ??? This ought to be an assert, but not all of the places
141 that we expand optabs know about the optabs that got moved
142 to being direct. */
143 if (!(optab >= FIRST_NORM_OPTAB && optab <= LAST_NORMLIB_OPTAB))
144 return NULL_RTX;
145
146 e.op = optab;
147 e.mode1 = mode;
148 e.mode2 = VOIDmode;
149 slot = (struct libfunc_entry **)
150 htab_find_slot (libfunc_hash, &e, NO_INSERT);
151 if (!slot)
152 {
153 const struct optab_libcall_d *d
154 = &normlib_def[optab - FIRST_NORM_OPTAB];
155
156 if (d->libcall_gen == NULL)
157 return NULL;
158
159 d->libcall_gen (optab, d->libcall_basename, d->libcall_suffix, mode);
160 slot = (struct libfunc_entry **)
161 htab_find_slot (libfunc_hash, &e, NO_INSERT);
162 if (!slot)
163 return NULL;
164 }
165 return (*slot)->libfunc;
166 }
167
168 \f
169 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
170 the result of operation CODE applied to OP0 (and OP1 if it is a binary
171 operation).
172
173 If the last insn does not set TARGET, don't do anything, but return 1.
174
175 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
176 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
177 try again, ensuring that TARGET is not one of the operands. */
178
179 static int
180 add_equal_note (rtx insns, rtx target, enum rtx_code code, rtx op0, rtx op1)
181 {
182 rtx last_insn, set;
183 rtx note;
184
185 gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
186
187 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
188 && GET_RTX_CLASS (code) != RTX_BIN_ARITH
189 && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
190 && GET_RTX_CLASS (code) != RTX_COMPARE
191 && GET_RTX_CLASS (code) != RTX_UNARY)
192 return 1;
193
194 if (GET_CODE (target) == ZERO_EXTRACT)
195 return 1;
196
197 for (last_insn = insns;
198 NEXT_INSN (last_insn) != NULL_RTX;
199 last_insn = NEXT_INSN (last_insn))
200 ;
201
202 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
203 a value changing in the insn, so the note would be invalid for CSE. */
204 if (reg_overlap_mentioned_p (target, op0)
205 || (op1 && reg_overlap_mentioned_p (target, op1)))
206 {
207 if (MEM_P (target)
208 && (rtx_equal_p (target, op0)
209 || (op1 && rtx_equal_p (target, op1))))
210 {
211 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
212 over expanding it as temp = MEM op X, MEM = temp. If the target
213 supports MEM = MEM op X instructions, it is sometimes too hard
214 to reconstruct that form later, especially if X is also a memory,
215 and due to multiple occurrences of addresses the address might
216 be forced into register unnecessarily.
217 Note that not emitting the REG_EQUIV note might inhibit
218 CSE in some cases. */
219 set = single_set (last_insn);
220 if (set
221 && GET_CODE (SET_SRC (set)) == code
222 && MEM_P (SET_DEST (set))
223 && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
224 || (op1 && rtx_equal_p (SET_DEST (set),
225 XEXP (SET_SRC (set), 1)))))
226 return 1;
227 }
228 return 0;
229 }
230
231 set = single_set (last_insn);
232 if (set == NULL_RTX)
233 return 1;
234
235 if (! rtx_equal_p (SET_DEST (set), target)
236 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
237 && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
238 || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
239 return 1;
240
241 if (GET_RTX_CLASS (code) == RTX_UNARY)
242 switch (code)
243 {
244 case FFS:
245 case CLZ:
246 case CTZ:
247 case CLRSB:
248 case POPCOUNT:
249 case PARITY:
250 case BSWAP:
251 if (GET_MODE (op0) != VOIDmode && GET_MODE (target) != GET_MODE (op0))
252 {
253 note = gen_rtx_fmt_e (code, GET_MODE (op0), copy_rtx (op0));
254 if (GET_MODE_SIZE (GET_MODE (op0))
255 > GET_MODE_SIZE (GET_MODE (target)))
256 note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
257 note, GET_MODE (op0));
258 else
259 note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
260 note, GET_MODE (op0));
261 break;
262 }
263 /* FALLTHRU */
264 default:
265 note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
266 break;
267 }
268 else
269 note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
270
271 set_unique_reg_note (last_insn, REG_EQUAL, note);
272
273 return 1;
274 }
275 \f
276 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
277 for a widening operation would be. In most cases this would be OP0, but if
278 that's a constant it'll be VOIDmode, which isn't useful. */
279
280 static enum machine_mode
281 widened_mode (enum machine_mode to_mode, rtx op0, rtx op1)
282 {
283 enum machine_mode m0 = GET_MODE (op0);
284 enum machine_mode m1 = GET_MODE (op1);
285 enum machine_mode result;
286
287 if (m0 == VOIDmode && m1 == VOIDmode)
288 return to_mode;
289 else if (m0 == VOIDmode || GET_MODE_SIZE (m0) < GET_MODE_SIZE (m1))
290 result = m1;
291 else
292 result = m0;
293
294 if (GET_MODE_SIZE (result) > GET_MODE_SIZE (to_mode))
295 return to_mode;
296
297 return result;
298 }
299 \f
300 /* Find a widening optab even if it doesn't widen as much as we want.
301 E.g. if from_mode is HImode, and to_mode is DImode, and there is no
302 direct HI->SI insn, then return SI->DI, if that exists.
303 If PERMIT_NON_WIDENING is non-zero then this can be used with
304 non-widening optabs also. */
305
306 enum insn_code
307 find_widening_optab_handler_and_mode (optab op, enum machine_mode to_mode,
308 enum machine_mode from_mode,
309 int permit_non_widening,
310 enum machine_mode *found_mode)
311 {
312 for (; (permit_non_widening || from_mode != to_mode)
313 && GET_MODE_SIZE (from_mode) <= GET_MODE_SIZE (to_mode)
314 && from_mode != VOIDmode;
315 from_mode = GET_MODE_WIDER_MODE (from_mode))
316 {
317 enum insn_code handler = widening_optab_handler (op, to_mode,
318 from_mode);
319
320 if (handler != CODE_FOR_nothing)
321 {
322 if (found_mode)
323 *found_mode = from_mode;
324 return handler;
325 }
326 }
327
328 return CODE_FOR_nothing;
329 }
330 \f
331 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
332 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
333 not actually do a sign-extend or zero-extend, but can leave the
334 higher-order bits of the result rtx undefined, for example, in the case
335 of logical operations, but not right shifts. */
336
337 static rtx
338 widen_operand (rtx op, enum machine_mode mode, enum machine_mode oldmode,
339 int unsignedp, int no_extend)
340 {
341 rtx result;
342
343 /* If we don't have to extend and this is a constant, return it. */
344 if (no_extend && GET_MODE (op) == VOIDmode)
345 return op;
346
347 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
348 extend since it will be more efficient to do so unless the signedness of
349 a promoted object differs from our extension. */
350 if (! no_extend
351 || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
352 && SUBREG_PROMOTED_UNSIGNED_P (op) == unsignedp))
353 return convert_modes (mode, oldmode, op, unsignedp);
354
355 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
356 SUBREG. */
357 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
358 return gen_lowpart (mode, force_reg (GET_MODE (op), op));
359
360 /* Otherwise, get an object of MODE, clobber it, and set the low-order
361 part to OP. */
362
363 result = gen_reg_rtx (mode);
364 emit_clobber (result);
365 emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
366 return result;
367 }
368 \f
369 /* Return the optab used for computing the operation given by the tree code,
370 CODE and the tree EXP. This function is not always usable (for example, it
371 cannot give complete results for multiplication or division) but probably
372 ought to be relied on more widely throughout the expander. */
373 optab
374 optab_for_tree_code (enum tree_code code, const_tree type,
375 enum optab_subtype subtype)
376 {
377 bool trapv;
378 switch (code)
379 {
380 case BIT_AND_EXPR:
381 return and_optab;
382
383 case BIT_IOR_EXPR:
384 return ior_optab;
385
386 case BIT_NOT_EXPR:
387 return one_cmpl_optab;
388
389 case BIT_XOR_EXPR:
390 return xor_optab;
391
392 case MULT_HIGHPART_EXPR:
393 return TYPE_UNSIGNED (type) ? umul_highpart_optab : smul_highpart_optab;
394
395 case TRUNC_MOD_EXPR:
396 case CEIL_MOD_EXPR:
397 case FLOOR_MOD_EXPR:
398 case ROUND_MOD_EXPR:
399 return TYPE_UNSIGNED (type) ? umod_optab : smod_optab;
400
401 case RDIV_EXPR:
402 case TRUNC_DIV_EXPR:
403 case CEIL_DIV_EXPR:
404 case FLOOR_DIV_EXPR:
405 case ROUND_DIV_EXPR:
406 case EXACT_DIV_EXPR:
407 if (TYPE_SATURATING (type))
408 return TYPE_UNSIGNED (type) ? usdiv_optab : ssdiv_optab;
409 return TYPE_UNSIGNED (type) ? udiv_optab : sdiv_optab;
410
411 case LSHIFT_EXPR:
412 if (TREE_CODE (type) == VECTOR_TYPE)
413 {
414 if (subtype == optab_vector)
415 return TYPE_SATURATING (type) ? unknown_optab : vashl_optab;
416
417 gcc_assert (subtype == optab_scalar);
418 }
419 if (TYPE_SATURATING (type))
420 return TYPE_UNSIGNED (type) ? usashl_optab : ssashl_optab;
421 return ashl_optab;
422
423 case RSHIFT_EXPR:
424 if (TREE_CODE (type) == VECTOR_TYPE)
425 {
426 if (subtype == optab_vector)
427 return TYPE_UNSIGNED (type) ? vlshr_optab : vashr_optab;
428
429 gcc_assert (subtype == optab_scalar);
430 }
431 return TYPE_UNSIGNED (type) ? lshr_optab : ashr_optab;
432
433 case LROTATE_EXPR:
434 if (TREE_CODE (type) == VECTOR_TYPE)
435 {
436 if (subtype == optab_vector)
437 return vrotl_optab;
438
439 gcc_assert (subtype == optab_scalar);
440 }
441 return rotl_optab;
442
443 case RROTATE_EXPR:
444 if (TREE_CODE (type) == VECTOR_TYPE)
445 {
446 if (subtype == optab_vector)
447 return vrotr_optab;
448
449 gcc_assert (subtype == optab_scalar);
450 }
451 return rotr_optab;
452
453 case MAX_EXPR:
454 return TYPE_UNSIGNED (type) ? umax_optab : smax_optab;
455
456 case MIN_EXPR:
457 return TYPE_UNSIGNED (type) ? umin_optab : smin_optab;
458
459 case REALIGN_LOAD_EXPR:
460 return vec_realign_load_optab;
461
462 case WIDEN_SUM_EXPR:
463 return TYPE_UNSIGNED (type) ? usum_widen_optab : ssum_widen_optab;
464
465 case DOT_PROD_EXPR:
466 return TYPE_UNSIGNED (type) ? udot_prod_optab : sdot_prod_optab;
467
468 case WIDEN_MULT_PLUS_EXPR:
469 return (TYPE_UNSIGNED (type)
470 ? (TYPE_SATURATING (type)
471 ? usmadd_widen_optab : umadd_widen_optab)
472 : (TYPE_SATURATING (type)
473 ? ssmadd_widen_optab : smadd_widen_optab));
474
475 case WIDEN_MULT_MINUS_EXPR:
476 return (TYPE_UNSIGNED (type)
477 ? (TYPE_SATURATING (type)
478 ? usmsub_widen_optab : umsub_widen_optab)
479 : (TYPE_SATURATING (type)
480 ? ssmsub_widen_optab : smsub_widen_optab));
481
482 case FMA_EXPR:
483 return fma_optab;
484
485 case REDUC_MAX_EXPR:
486 return TYPE_UNSIGNED (type) ? reduc_umax_optab : reduc_smax_optab;
487
488 case REDUC_MIN_EXPR:
489 return TYPE_UNSIGNED (type) ? reduc_umin_optab : reduc_smin_optab;
490
491 case REDUC_PLUS_EXPR:
492 return TYPE_UNSIGNED (type) ? reduc_uplus_optab : reduc_splus_optab;
493
494 case VEC_LSHIFT_EXPR:
495 return vec_shl_optab;
496
497 case VEC_RSHIFT_EXPR:
498 return vec_shr_optab;
499
500 case VEC_WIDEN_MULT_HI_EXPR:
501 return TYPE_UNSIGNED (type) ?
502 vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
503
504 case VEC_WIDEN_MULT_LO_EXPR:
505 return TYPE_UNSIGNED (type) ?
506 vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
507
508 case VEC_WIDEN_MULT_EVEN_EXPR:
509 return TYPE_UNSIGNED (type) ?
510 vec_widen_umult_even_optab : vec_widen_smult_even_optab;
511
512 case VEC_WIDEN_MULT_ODD_EXPR:
513 return TYPE_UNSIGNED (type) ?
514 vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
515
516 case VEC_WIDEN_LSHIFT_HI_EXPR:
517 return TYPE_UNSIGNED (type) ?
518 vec_widen_ushiftl_hi_optab : vec_widen_sshiftl_hi_optab;
519
520 case VEC_WIDEN_LSHIFT_LO_EXPR:
521 return TYPE_UNSIGNED (type) ?
522 vec_widen_ushiftl_lo_optab : vec_widen_sshiftl_lo_optab;
523
524 case VEC_UNPACK_HI_EXPR:
525 return TYPE_UNSIGNED (type) ?
526 vec_unpacku_hi_optab : vec_unpacks_hi_optab;
527
528 case VEC_UNPACK_LO_EXPR:
529 return TYPE_UNSIGNED (type) ?
530 vec_unpacku_lo_optab : vec_unpacks_lo_optab;
531
532 case VEC_UNPACK_FLOAT_HI_EXPR:
533 /* The signedness is determined from input operand. */
534 return TYPE_UNSIGNED (type) ?
535 vec_unpacku_float_hi_optab : vec_unpacks_float_hi_optab;
536
537 case VEC_UNPACK_FLOAT_LO_EXPR:
538 /* The signedness is determined from input operand. */
539 return TYPE_UNSIGNED (type) ?
540 vec_unpacku_float_lo_optab : vec_unpacks_float_lo_optab;
541
542 case VEC_PACK_TRUNC_EXPR:
543 return vec_pack_trunc_optab;
544
545 case VEC_PACK_SAT_EXPR:
546 return TYPE_UNSIGNED (type) ? vec_pack_usat_optab : vec_pack_ssat_optab;
547
548 case VEC_PACK_FIX_TRUNC_EXPR:
549 /* The signedness is determined from output operand. */
550 return TYPE_UNSIGNED (type) ?
551 vec_pack_ufix_trunc_optab : vec_pack_sfix_trunc_optab;
552
553 default:
554 break;
555 }
556
557 trapv = INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type);
558 switch (code)
559 {
560 case POINTER_PLUS_EXPR:
561 case PLUS_EXPR:
562 if (TYPE_SATURATING (type))
563 return TYPE_UNSIGNED (type) ? usadd_optab : ssadd_optab;
564 return trapv ? addv_optab : add_optab;
565
566 case MINUS_EXPR:
567 if (TYPE_SATURATING (type))
568 return TYPE_UNSIGNED (type) ? ussub_optab : sssub_optab;
569 return trapv ? subv_optab : sub_optab;
570
571 case MULT_EXPR:
572 if (TYPE_SATURATING (type))
573 return TYPE_UNSIGNED (type) ? usmul_optab : ssmul_optab;
574 return trapv ? smulv_optab : smul_optab;
575
576 case NEGATE_EXPR:
577 if (TYPE_SATURATING (type))
578 return TYPE_UNSIGNED (type) ? usneg_optab : ssneg_optab;
579 return trapv ? negv_optab : neg_optab;
580
581 case ABS_EXPR:
582 return trapv ? absv_optab : abs_optab;
583
584 default:
585 return unknown_optab;
586 }
587 }
588 \f
589
590 /* Expand vector widening operations.
591
592 There are two different classes of operations handled here:
593 1) Operations whose result is wider than all the arguments to the operation.
594 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
595 In this case OP0 and optionally OP1 would be initialized,
596 but WIDE_OP wouldn't (not relevant for this case).
597 2) Operations whose result is of the same size as the last argument to the
598 operation, but wider than all the other arguments to the operation.
599 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
600 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
601
602 E.g, when called to expand the following operations, this is how
603 the arguments will be initialized:
604 nops OP0 OP1 WIDE_OP
605 widening-sum 2 oprnd0 - oprnd1
606 widening-dot-product 3 oprnd0 oprnd1 oprnd2
607 widening-mult 2 oprnd0 oprnd1 -
608 type-promotion (vec-unpack) 1 oprnd0 - - */
609
610 rtx
611 expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op,
612 rtx target, int unsignedp)
613 {
614 struct expand_operand eops[4];
615 tree oprnd0, oprnd1, oprnd2;
616 enum machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
617 optab widen_pattern_optab;
618 enum insn_code icode;
619 int nops = TREE_CODE_LENGTH (ops->code);
620 int op;
621
622 oprnd0 = ops->op0;
623 tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
624 widen_pattern_optab =
625 optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
626 if (ops->code == WIDEN_MULT_PLUS_EXPR
627 || ops->code == WIDEN_MULT_MINUS_EXPR)
628 icode = find_widening_optab_handler (widen_pattern_optab,
629 TYPE_MODE (TREE_TYPE (ops->op2)),
630 tmode0, 0);
631 else
632 icode = optab_handler (widen_pattern_optab, tmode0);
633 gcc_assert (icode != CODE_FOR_nothing);
634
635 if (nops >= 2)
636 {
637 oprnd1 = ops->op1;
638 tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
639 }
640
641 /* The last operand is of a wider mode than the rest of the operands. */
642 if (nops == 2)
643 wmode = tmode1;
644 else if (nops == 3)
645 {
646 gcc_assert (tmode1 == tmode0);
647 gcc_assert (op1);
648 oprnd2 = ops->op2;
649 wmode = TYPE_MODE (TREE_TYPE (oprnd2));
650 }
651
652 op = 0;
653 create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
654 create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
655 if (op1)
656 create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
657 if (wide_op)
658 create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
659 expand_insn (icode, op, eops);
660 return eops[0].value;
661 }
662
663 /* Generate code to perform an operation specified by TERNARY_OPTAB
664 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
665
666 UNSIGNEDP is for the case where we have to widen the operands
667 to perform the operation. It says to use zero-extension.
668
669 If TARGET is nonzero, the value
670 is generated there, if it is convenient to do so.
671 In all cases an rtx is returned for the locus of the value;
672 this may or may not be TARGET. */
673
674 rtx
675 expand_ternary_op (enum machine_mode mode, optab ternary_optab, rtx op0,
676 rtx op1, rtx op2, rtx target, int unsignedp)
677 {
678 struct expand_operand ops[4];
679 enum insn_code icode = optab_handler (ternary_optab, mode);
680
681 gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
682
683 create_output_operand (&ops[0], target, mode);
684 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
685 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
686 create_convert_operand_from (&ops[3], op2, mode, unsignedp);
687 expand_insn (icode, 4, ops);
688 return ops[0].value;
689 }
690
691
692 /* Like expand_binop, but return a constant rtx if the result can be
693 calculated at compile time. The arguments and return value are
694 otherwise the same as for expand_binop. */
695
696 rtx
697 simplify_expand_binop (enum machine_mode mode, optab binoptab,
698 rtx op0, rtx op1, rtx target, int unsignedp,
699 enum optab_methods methods)
700 {
701 if (CONSTANT_P (op0) && CONSTANT_P (op1))
702 {
703 rtx x = simplify_binary_operation (optab_to_code (binoptab),
704 mode, op0, op1);
705 if (x)
706 return x;
707 }
708
709 return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
710 }
711
712 /* Like simplify_expand_binop, but always put the result in TARGET.
713 Return true if the expansion succeeded. */
714
715 bool
716 force_expand_binop (enum machine_mode mode, optab binoptab,
717 rtx op0, rtx op1, rtx target, int unsignedp,
718 enum optab_methods methods)
719 {
720 rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
721 target, unsignedp, methods);
722 if (x == 0)
723 return false;
724 if (x != target)
725 emit_move_insn (target, x);
726 return true;
727 }
728
729 /* Generate insns for VEC_LSHIFT_EXPR, VEC_RSHIFT_EXPR. */
730
731 rtx
732 expand_vec_shift_expr (sepops ops, rtx target)
733 {
734 struct expand_operand eops[3];
735 enum insn_code icode;
736 rtx rtx_op1, rtx_op2;
737 enum machine_mode mode = TYPE_MODE (ops->type);
738 tree vec_oprnd = ops->op0;
739 tree shift_oprnd = ops->op1;
740 optab shift_optab;
741
742 switch (ops->code)
743 {
744 case VEC_RSHIFT_EXPR:
745 shift_optab = vec_shr_optab;
746 break;
747 case VEC_LSHIFT_EXPR:
748 shift_optab = vec_shl_optab;
749 break;
750 default:
751 gcc_unreachable ();
752 }
753
754 icode = optab_handler (shift_optab, mode);
755 gcc_assert (icode != CODE_FOR_nothing);
756
757 rtx_op1 = expand_normal (vec_oprnd);
758 rtx_op2 = expand_normal (shift_oprnd);
759
760 create_output_operand (&eops[0], target, mode);
761 create_input_operand (&eops[1], rtx_op1, GET_MODE (rtx_op1));
762 create_convert_operand_from_type (&eops[2], rtx_op2, TREE_TYPE (shift_oprnd));
763 expand_insn (icode, 3, eops);
764
765 return eops[0].value;
766 }
767
768 /* Create a new vector value in VMODE with all elements set to OP. The
769 mode of OP must be the element mode of VMODE. If OP is a constant,
770 then the return value will be a constant. */
771
772 static rtx
773 expand_vector_broadcast (enum machine_mode vmode, rtx op)
774 {
775 enum insn_code icode;
776 rtvec vec;
777 rtx ret;
778 int i, n;
779
780 gcc_checking_assert (VECTOR_MODE_P (vmode));
781
782 n = GET_MODE_NUNITS (vmode);
783 vec = rtvec_alloc (n);
784 for (i = 0; i < n; ++i)
785 RTVEC_ELT (vec, i) = op;
786
787 if (CONSTANT_P (op))
788 return gen_rtx_CONST_VECTOR (vmode, vec);
789
790 /* ??? If the target doesn't have a vec_init, then we have no easy way
791 of performing this operation. Most of this sort of generic support
792 is hidden away in the vector lowering support in gimple. */
793 icode = optab_handler (vec_init_optab, vmode);
794 if (icode == CODE_FOR_nothing)
795 return NULL;
796
797 ret = gen_reg_rtx (vmode);
798 emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
799
800 return ret;
801 }
802
803 /* This subroutine of expand_doubleword_shift handles the cases in which
804 the effective shift value is >= BITS_PER_WORD. The arguments and return
805 value are the same as for the parent routine, except that SUPERWORD_OP1
806 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
807 INTO_TARGET may be null if the caller has decided to calculate it. */
808
809 static bool
810 expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
811 rtx outof_target, rtx into_target,
812 int unsignedp, enum optab_methods methods)
813 {
814 if (into_target != 0)
815 if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
816 into_target, unsignedp, methods))
817 return false;
818
819 if (outof_target != 0)
820 {
821 /* For a signed right shift, we must fill OUTOF_TARGET with copies
822 of the sign bit, otherwise we must fill it with zeros. */
823 if (binoptab != ashr_optab)
824 emit_move_insn (outof_target, CONST0_RTX (word_mode));
825 else
826 if (!force_expand_binop (word_mode, binoptab,
827 outof_input, GEN_INT (BITS_PER_WORD - 1),
828 outof_target, unsignedp, methods))
829 return false;
830 }
831 return true;
832 }
833
834 /* This subroutine of expand_doubleword_shift handles the cases in which
835 the effective shift value is < BITS_PER_WORD. The arguments and return
836 value are the same as for the parent routine. */
837
838 static bool
839 expand_subword_shift (enum machine_mode op1_mode, optab binoptab,
840 rtx outof_input, rtx into_input, rtx op1,
841 rtx outof_target, rtx into_target,
842 int unsignedp, enum optab_methods methods,
843 unsigned HOST_WIDE_INT shift_mask)
844 {
845 optab reverse_unsigned_shift, unsigned_shift;
846 rtx tmp, carries;
847
848 reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
849 unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
850
851 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
852 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
853 the opposite direction to BINOPTAB. */
854 if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
855 {
856 carries = outof_input;
857 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
858 op1_mode), op1_mode);
859 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
860 0, true, methods);
861 }
862 else
863 {
864 /* We must avoid shifting by BITS_PER_WORD bits since that is either
865 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
866 has unknown behavior. Do a single shift first, then shift by the
867 remainder. It's OK to use ~OP1 as the remainder if shift counts
868 are truncated to the mode size. */
869 carries = expand_binop (word_mode, reverse_unsigned_shift,
870 outof_input, const1_rtx, 0, unsignedp, methods);
871 if (shift_mask == BITS_PER_WORD - 1)
872 {
873 tmp = immed_wide_int_const
874 (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode);
875 tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp,
876 0, true, methods);
877 }
878 else
879 {
880 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
881 op1_mode), op1_mode);
882 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
883 0, true, methods);
884 }
885 }
886 if (tmp == 0 || carries == 0)
887 return false;
888 carries = expand_binop (word_mode, reverse_unsigned_shift,
889 carries, tmp, 0, unsignedp, methods);
890 if (carries == 0)
891 return false;
892
893 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
894 so the result can go directly into INTO_TARGET if convenient. */
895 tmp = expand_binop (word_mode, unsigned_shift, into_input, op1,
896 into_target, unsignedp, methods);
897 if (tmp == 0)
898 return false;
899
900 /* Now OR in the bits carried over from OUTOF_INPUT. */
901 if (!force_expand_binop (word_mode, ior_optab, tmp, carries,
902 into_target, unsignedp, methods))
903 return false;
904
905 /* Use a standard word_mode shift for the out-of half. */
906 if (outof_target != 0)
907 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
908 outof_target, unsignedp, methods))
909 return false;
910
911 return true;
912 }
913
914
915 #ifdef HAVE_conditional_move
916 /* Try implementing expand_doubleword_shift using conditional moves.
917 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
918 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
919 are the shift counts to use in the former and latter case. All other
920 arguments are the same as the parent routine. */
921
922 static bool
923 expand_doubleword_shift_condmove (enum machine_mode op1_mode, optab binoptab,
924 enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
925 rtx outof_input, rtx into_input,
926 rtx subword_op1, rtx superword_op1,
927 rtx outof_target, rtx into_target,
928 int unsignedp, enum optab_methods methods,
929 unsigned HOST_WIDE_INT shift_mask)
930 {
931 rtx outof_superword, into_superword;
932
933 /* Put the superword version of the output into OUTOF_SUPERWORD and
934 INTO_SUPERWORD. */
935 outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
936 if (outof_target != 0 && subword_op1 == superword_op1)
937 {
938 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
939 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
940 into_superword = outof_target;
941 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
942 outof_superword, 0, unsignedp, methods))
943 return false;
944 }
945 else
946 {
947 into_superword = gen_reg_rtx (word_mode);
948 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
949 outof_superword, into_superword,
950 unsignedp, methods))
951 return false;
952 }
953
954 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
955 if (!expand_subword_shift (op1_mode, binoptab,
956 outof_input, into_input, subword_op1,
957 outof_target, into_target,
958 unsignedp, methods, shift_mask))
959 return false;
960
961 /* Select between them. Do the INTO half first because INTO_SUPERWORD
962 might be the current value of OUTOF_TARGET. */
963 if (!emit_conditional_move (into_target, cmp_code, cmp1, cmp2, op1_mode,
964 into_target, into_superword, word_mode, false))
965 return false;
966
967 if (outof_target != 0)
968 if (!emit_conditional_move (outof_target, cmp_code, cmp1, cmp2, op1_mode,
969 outof_target, outof_superword,
970 word_mode, false))
971 return false;
972
973 return true;
974 }
975 #endif
976
977 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
978 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
979 input operand; the shift moves bits in the direction OUTOF_INPUT->
980 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
981 of the target. OP1 is the shift count and OP1_MODE is its mode.
982 If OP1 is constant, it will have been truncated as appropriate
983 and is known to be nonzero.
984
985 If SHIFT_MASK is zero, the result of word shifts is undefined when the
986 shift count is outside the range [0, BITS_PER_WORD). This routine must
987 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
988
989 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
990 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
991 fill with zeros or sign bits as appropriate.
992
993 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
994 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
995 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
996 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
997 are undefined.
998
999 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
1000 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
1001 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
1002 function wants to calculate it itself.
1003
1004 Return true if the shift could be successfully synthesized. */
1005
1006 static bool
1007 expand_doubleword_shift (enum machine_mode op1_mode, optab binoptab,
1008 rtx outof_input, rtx into_input, rtx op1,
1009 rtx outof_target, rtx into_target,
1010 int unsignedp, enum optab_methods methods,
1011 unsigned HOST_WIDE_INT shift_mask)
1012 {
1013 rtx superword_op1, tmp, cmp1, cmp2;
1014 rtx subword_label, done_label;
1015 enum rtx_code cmp_code;
1016
1017 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
1018 fill the result with sign or zero bits as appropriate. If so, the value
1019 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
1020 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
1021 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
1022
1023 This isn't worthwhile for constant shifts since the optimizers will
1024 cope better with in-range shift counts. */
1025 if (shift_mask >= BITS_PER_WORD
1026 && outof_target != 0
1027 && !CONSTANT_P (op1))
1028 {
1029 if (!expand_doubleword_shift (op1_mode, binoptab,
1030 outof_input, into_input, op1,
1031 0, into_target,
1032 unsignedp, methods, shift_mask))
1033 return false;
1034 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
1035 outof_target, unsignedp, methods))
1036 return false;
1037 return true;
1038 }
1039
1040 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
1041 is true when the effective shift value is less than BITS_PER_WORD.
1042 Set SUPERWORD_OP1 to the shift count that should be used to shift
1043 OUTOF_INPUT into INTO_TARGET when the condition is false. */
1044 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
1045 if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
1046 {
1047 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
1048 is a subword shift count. */
1049 cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
1050 0, true, methods);
1051 cmp2 = CONST0_RTX (op1_mode);
1052 cmp_code = EQ;
1053 superword_op1 = op1;
1054 }
1055 else
1056 {
1057 /* Set CMP1 to OP1 - BITS_PER_WORD. */
1058 cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
1059 0, true, methods);
1060 cmp2 = CONST0_RTX (op1_mode);
1061 cmp_code = LT;
1062 superword_op1 = cmp1;
1063 }
1064 if (cmp1 == 0)
1065 return false;
1066
1067 /* If we can compute the condition at compile time, pick the
1068 appropriate subroutine. */
1069 tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
1070 if (tmp != 0 && CONST_INT_P (tmp))
1071 {
1072 if (tmp == const0_rtx)
1073 return expand_superword_shift (binoptab, outof_input, superword_op1,
1074 outof_target, into_target,
1075 unsignedp, methods);
1076 else
1077 return expand_subword_shift (op1_mode, binoptab,
1078 outof_input, into_input, op1,
1079 outof_target, into_target,
1080 unsignedp, methods, shift_mask);
1081 }
1082
1083 #ifdef HAVE_conditional_move
1084 /* Try using conditional moves to generate straight-line code. */
1085 {
1086 rtx start = get_last_insn ();
1087 if (expand_doubleword_shift_condmove (op1_mode, binoptab,
1088 cmp_code, cmp1, cmp2,
1089 outof_input, into_input,
1090 op1, superword_op1,
1091 outof_target, into_target,
1092 unsignedp, methods, shift_mask))
1093 return true;
1094 delete_insns_since (start);
1095 }
1096 #endif
1097
1098 /* As a last resort, use branches to select the correct alternative. */
1099 subword_label = gen_label_rtx ();
1100 done_label = gen_label_rtx ();
1101
1102 NO_DEFER_POP;
1103 do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
1104 0, 0, subword_label, -1);
1105 OK_DEFER_POP;
1106
1107 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
1108 outof_target, into_target,
1109 unsignedp, methods))
1110 return false;
1111
1112 emit_jump_insn (gen_jump (done_label));
1113 emit_barrier ();
1114 emit_label (subword_label);
1115
1116 if (!expand_subword_shift (op1_mode, binoptab,
1117 outof_input, into_input, op1,
1118 outof_target, into_target,
1119 unsignedp, methods, shift_mask))
1120 return false;
1121
1122 emit_label (done_label);
1123 return true;
1124 }
1125 \f
1126 /* Subroutine of expand_binop. Perform a double word multiplication of
1127 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
1128 as the target's word_mode. This function return NULL_RTX if anything
1129 goes wrong, in which case it may have already emitted instructions
1130 which need to be deleted.
1131
1132 If we want to multiply two two-word values and have normal and widening
1133 multiplies of single-word values, we can do this with three smaller
1134 multiplications.
1135
1136 The multiplication proceeds as follows:
1137 _______________________
1138 [__op0_high_|__op0_low__]
1139 _______________________
1140 * [__op1_high_|__op1_low__]
1141 _______________________________________________
1142 _______________________
1143 (1) [__op0_low__*__op1_low__]
1144 _______________________
1145 (2a) [__op0_low__*__op1_high_]
1146 _______________________
1147 (2b) [__op0_high_*__op1_low__]
1148 _______________________
1149 (3) [__op0_high_*__op1_high_]
1150
1151
1152 This gives a 4-word result. Since we are only interested in the
1153 lower 2 words, partial result (3) and the upper words of (2a) and
1154 (2b) don't need to be calculated. Hence (2a) and (2b) can be
1155 calculated using non-widening multiplication.
1156
1157 (1), however, needs to be calculated with an unsigned widening
1158 multiplication. If this operation is not directly supported we
1159 try using a signed widening multiplication and adjust the result.
1160 This adjustment works as follows:
1161
1162 If both operands are positive then no adjustment is needed.
1163
1164 If the operands have different signs, for example op0_low < 0 and
1165 op1_low >= 0, the instruction treats the most significant bit of
1166 op0_low as a sign bit instead of a bit with significance
1167 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
1168 with 2**BITS_PER_WORD - op0_low, and two's complements the
1169 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
1170 the result.
1171
1172 Similarly, if both operands are negative, we need to add
1173 (op0_low + op1_low) * 2**BITS_PER_WORD.
1174
1175 We use a trick to adjust quickly. We logically shift op0_low right
1176 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
1177 op0_high (op1_high) before it is used to calculate 2b (2a). If no
1178 logical shift exists, we do an arithmetic right shift and subtract
1179 the 0 or -1. */
1180
1181 static rtx
1182 expand_doubleword_mult (enum machine_mode mode, rtx op0, rtx op1, rtx target,
1183 bool umulp, enum optab_methods methods)
1184 {
1185 int low = (WORDS_BIG_ENDIAN ? 1 : 0);
1186 int high = (WORDS_BIG_ENDIAN ? 0 : 1);
1187 rtx wordm1 = umulp ? NULL_RTX : GEN_INT (BITS_PER_WORD - 1);
1188 rtx product, adjust, product_high, temp;
1189
1190 rtx op0_high = operand_subword_force (op0, high, mode);
1191 rtx op0_low = operand_subword_force (op0, low, mode);
1192 rtx op1_high = operand_subword_force (op1, high, mode);
1193 rtx op1_low = operand_subword_force (op1, low, mode);
1194
1195 /* If we're using an unsigned multiply to directly compute the product
1196 of the low-order words of the operands and perform any required
1197 adjustments of the operands, we begin by trying two more multiplications
1198 and then computing the appropriate sum.
1199
1200 We have checked above that the required addition is provided.
1201 Full-word addition will normally always succeed, especially if
1202 it is provided at all, so we don't worry about its failure. The
1203 multiplication may well fail, however, so we do handle that. */
1204
1205 if (!umulp)
1206 {
1207 /* ??? This could be done with emit_store_flag where available. */
1208 temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
1209 NULL_RTX, 1, methods);
1210 if (temp)
1211 op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
1212 NULL_RTX, 0, OPTAB_DIRECT);
1213 else
1214 {
1215 temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
1216 NULL_RTX, 0, methods);
1217 if (!temp)
1218 return NULL_RTX;
1219 op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
1220 NULL_RTX, 0, OPTAB_DIRECT);
1221 }
1222
1223 if (!op0_high)
1224 return NULL_RTX;
1225 }
1226
1227 adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
1228 NULL_RTX, 0, OPTAB_DIRECT);
1229 if (!adjust)
1230 return NULL_RTX;
1231
1232 /* OP0_HIGH should now be dead. */
1233
1234 if (!umulp)
1235 {
1236 /* ??? This could be done with emit_store_flag where available. */
1237 temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
1238 NULL_RTX, 1, methods);
1239 if (temp)
1240 op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
1241 NULL_RTX, 0, OPTAB_DIRECT);
1242 else
1243 {
1244 temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
1245 NULL_RTX, 0, methods);
1246 if (!temp)
1247 return NULL_RTX;
1248 op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
1249 NULL_RTX, 0, OPTAB_DIRECT);
1250 }
1251
1252 if (!op1_high)
1253 return NULL_RTX;
1254 }
1255
1256 temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
1257 NULL_RTX, 0, OPTAB_DIRECT);
1258 if (!temp)
1259 return NULL_RTX;
1260
1261 /* OP1_HIGH should now be dead. */
1262
1263 adjust = expand_binop (word_mode, add_optab, adjust, temp,
1264 NULL_RTX, 0, OPTAB_DIRECT);
1265
1266 if (target && !REG_P (target))
1267 target = NULL_RTX;
1268
1269 if (umulp)
1270 product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
1271 target, 1, OPTAB_DIRECT);
1272 else
1273 product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
1274 target, 1, OPTAB_DIRECT);
1275
1276 if (!product)
1277 return NULL_RTX;
1278
1279 product_high = operand_subword (product, high, 1, mode);
1280 adjust = expand_binop (word_mode, add_optab, product_high, adjust,
1281 NULL_RTX, 0, OPTAB_DIRECT);
1282 emit_move_insn (product_high, adjust);
1283 return product;
1284 }
1285 \f
1286 /* Wrapper around expand_binop which takes an rtx code to specify
1287 the operation to perform, not an optab pointer. All other
1288 arguments are the same. */
1289 rtx
1290 expand_simple_binop (enum machine_mode mode, enum rtx_code code, rtx op0,
1291 rtx op1, rtx target, int unsignedp,
1292 enum optab_methods methods)
1293 {
1294 optab binop = code_to_optab (code);
1295 gcc_assert (binop);
1296
1297 return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
1298 }
1299
1300 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1301 binop. Order them according to commutative_operand_precedence and, if
1302 possible, try to put TARGET or a pseudo first. */
1303 static bool
1304 swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
1305 {
1306 int op0_prec = commutative_operand_precedence (op0);
1307 int op1_prec = commutative_operand_precedence (op1);
1308
1309 if (op0_prec < op1_prec)
1310 return true;
1311
1312 if (op0_prec > op1_prec)
1313 return false;
1314
1315 /* With equal precedence, both orders are ok, but it is better if the
1316 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1317 if (target == 0 || REG_P (target))
1318 return (REG_P (op1) && !REG_P (op0)) || target == op1;
1319 else
1320 return rtx_equal_p (op1, target);
1321 }
1322
1323 /* Return true if BINOPTAB implements a shift operation. */
1324
1325 static bool
1326 shift_optab_p (optab binoptab)
1327 {
1328 switch (optab_to_code (binoptab))
1329 {
1330 case ASHIFT:
1331 case SS_ASHIFT:
1332 case US_ASHIFT:
1333 case ASHIFTRT:
1334 case LSHIFTRT:
1335 case ROTATE:
1336 case ROTATERT:
1337 return true;
1338
1339 default:
1340 return false;
1341 }
1342 }
1343
1344 /* Return true if BINOPTAB implements a commutative binary operation. */
1345
1346 static bool
1347 commutative_optab_p (optab binoptab)
1348 {
1349 return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
1350 || binoptab == smul_widen_optab
1351 || binoptab == umul_widen_optab
1352 || binoptab == smul_highpart_optab
1353 || binoptab == umul_highpart_optab);
1354 }
1355
1356 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1357 optimizing, and if the operand is a constant that costs more than
1358 1 instruction, force the constant into a register and return that
1359 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1360
1361 static rtx
1362 avoid_expensive_constant (enum machine_mode mode, optab binoptab,
1363 int opn, rtx x, bool unsignedp)
1364 {
1365 bool speed = optimize_insn_for_speed_p ();
1366
1367 if (mode != VOIDmode
1368 && optimize
1369 && CONSTANT_P (x)
1370 && (rtx_cost (x, optab_to_code (binoptab), opn, speed)
1371 > set_src_cost (x, speed)))
1372 {
1373 if (CONST_INT_P (x))
1374 {
1375 HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1376 if (intval != INTVAL (x))
1377 x = GEN_INT (intval);
1378 }
1379 else
1380 x = convert_modes (mode, VOIDmode, x, unsignedp);
1381 x = force_reg (mode, x);
1382 }
1383 return x;
1384 }
1385
1386 /* Helper function for expand_binop: handle the case where there
1387 is an insn that directly implements the indicated operation.
1388 Returns null if this is not possible. */
1389 static rtx
1390 expand_binop_directly (enum machine_mode mode, optab binoptab,
1391 rtx op0, rtx op1,
1392 rtx target, int unsignedp, enum optab_methods methods,
1393 rtx last)
1394 {
1395 enum machine_mode from_mode = widened_mode (mode, op0, op1);
1396 enum insn_code icode = find_widening_optab_handler (binoptab, mode,
1397 from_mode, 1);
1398 enum machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1399 enum machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1400 enum machine_mode mode0, mode1, tmp_mode;
1401 struct expand_operand ops[3];
1402 bool commutative_p;
1403 rtx pat;
1404 rtx xop0 = op0, xop1 = op1;
1405 rtx swap;
1406
1407 /* If it is a commutative operator and the modes would match
1408 if we would swap the operands, we can save the conversions. */
1409 commutative_p = commutative_optab_p (binoptab);
1410 if (commutative_p
1411 && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1412 && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode1)
1413 {
1414 swap = xop0;
1415 xop0 = xop1;
1416 xop1 = swap;
1417 }
1418
1419 /* If we are optimizing, force expensive constants into a register. */
1420 xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1421 if (!shift_optab_p (binoptab))
1422 xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1423
1424 /* In case the insn wants input operands in modes different from
1425 those of the actual operands, convert the operands. It would
1426 seem that we don't need to convert CONST_INTs, but we do, so
1427 that they're properly zero-extended, sign-extended or truncated
1428 for their mode. */
1429
1430 mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1431 if (xmode0 != VOIDmode && xmode0 != mode0)
1432 {
1433 xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1434 mode0 = xmode0;
1435 }
1436
1437 mode1 = GET_MODE (xop1) != VOIDmode ? GET_MODE (xop1) : mode;
1438 if (xmode1 != VOIDmode && xmode1 != mode1)
1439 {
1440 xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1441 mode1 = xmode1;
1442 }
1443
1444 /* If operation is commutative,
1445 try to make the first operand a register.
1446 Even better, try to make it the same as the target.
1447 Also try to make the last operand a constant. */
1448 if (commutative_p
1449 && swap_commutative_operands_with_target (target, xop0, xop1))
1450 {
1451 swap = xop1;
1452 xop1 = xop0;
1453 xop0 = swap;
1454 }
1455
1456 /* Now, if insn's predicates don't allow our operands, put them into
1457 pseudo regs. */
1458
1459 if (binoptab == vec_pack_trunc_optab
1460 || binoptab == vec_pack_usat_optab
1461 || binoptab == vec_pack_ssat_optab
1462 || binoptab == vec_pack_ufix_trunc_optab
1463 || binoptab == vec_pack_sfix_trunc_optab)
1464 {
1465 /* The mode of the result is different then the mode of the
1466 arguments. */
1467 tmp_mode = insn_data[(int) icode].operand[0].mode;
1468 if (GET_MODE_NUNITS (tmp_mode) != 2 * GET_MODE_NUNITS (mode))
1469 {
1470 delete_insns_since (last);
1471 return NULL_RTX;
1472 }
1473 }
1474 else
1475 tmp_mode = mode;
1476
1477 create_output_operand (&ops[0], target, tmp_mode);
1478 create_input_operand (&ops[1], xop0, mode0);
1479 create_input_operand (&ops[2], xop1, mode1);
1480 pat = maybe_gen_insn (icode, 3, ops);
1481 if (pat)
1482 {
1483 /* If PAT is composed of more than one insn, try to add an appropriate
1484 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1485 operand, call expand_binop again, this time without a target. */
1486 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1487 && ! add_equal_note (pat, ops[0].value, optab_to_code (binoptab),
1488 ops[1].value, ops[2].value))
1489 {
1490 delete_insns_since (last);
1491 return expand_binop (mode, binoptab, op0, op1, NULL_RTX,
1492 unsignedp, methods);
1493 }
1494
1495 emit_insn (pat);
1496 return ops[0].value;
1497 }
1498 delete_insns_since (last);
1499 return NULL_RTX;
1500 }
1501
1502 /* Generate code to perform an operation specified by BINOPTAB
1503 on operands OP0 and OP1, with result having machine-mode MODE.
1504
1505 UNSIGNEDP is for the case where we have to widen the operands
1506 to perform the operation. It says to use zero-extension.
1507
1508 If TARGET is nonzero, the value
1509 is generated there, if it is convenient to do so.
1510 In all cases an rtx is returned for the locus of the value;
1511 this may or may not be TARGET. */
1512
1513 rtx
1514 expand_binop (enum machine_mode mode, optab binoptab, rtx op0, rtx op1,
1515 rtx target, int unsignedp, enum optab_methods methods)
1516 {
1517 enum optab_methods next_methods
1518 = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1519 ? OPTAB_WIDEN : methods);
1520 enum mode_class mclass;
1521 enum machine_mode wider_mode;
1522 rtx libfunc;
1523 rtx temp;
1524 rtx entry_last = get_last_insn ();
1525 rtx last;
1526
1527 mclass = GET_MODE_CLASS (mode);
1528
1529 /* If subtracting an integer constant, convert this into an addition of
1530 the negated constant. */
1531
1532 if (binoptab == sub_optab && CONST_INT_P (op1))
1533 {
1534 op1 = negate_rtx (mode, op1);
1535 binoptab = add_optab;
1536 }
1537
1538 /* Record where to delete back to if we backtrack. */
1539 last = get_last_insn ();
1540
1541 /* If we can do it with a three-operand insn, do so. */
1542
1543 if (methods != OPTAB_MUST_WIDEN
1544 && find_widening_optab_handler (binoptab, mode,
1545 widened_mode (mode, op0, op1), 1)
1546 != CODE_FOR_nothing)
1547 {
1548 temp = expand_binop_directly (mode, binoptab, op0, op1, target,
1549 unsignedp, methods, last);
1550 if (temp)
1551 return temp;
1552 }
1553
1554 /* If we were trying to rotate, and that didn't work, try rotating
1555 the other direction before falling back to shifts and bitwise-or. */
1556 if (((binoptab == rotl_optab
1557 && optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
1558 || (binoptab == rotr_optab
1559 && optab_handler (rotl_optab, mode) != CODE_FOR_nothing))
1560 && mclass == MODE_INT)
1561 {
1562 optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1563 rtx newop1;
1564 unsigned int bits = GET_MODE_PRECISION (mode);
1565
1566 if (CONST_INT_P (op1))
1567 newop1 = GEN_INT (bits - INTVAL (op1));
1568 else if (targetm.shift_truncation_mask (mode) == bits - 1)
1569 newop1 = negate_rtx (GET_MODE (op1), op1);
1570 else
1571 newop1 = expand_binop (GET_MODE (op1), sub_optab,
1572 gen_int_mode (bits, GET_MODE (op1)), op1,
1573 NULL_RTX, unsignedp, OPTAB_DIRECT);
1574
1575 temp = expand_binop_directly (mode, otheroptab, op0, newop1,
1576 target, unsignedp, methods, last);
1577 if (temp)
1578 return temp;
1579 }
1580
1581 /* If this is a multiply, see if we can do a widening operation that
1582 takes operands of this mode and makes a wider mode. */
1583
1584 if (binoptab == smul_optab
1585 && GET_MODE_2XWIDER_MODE (mode) != VOIDmode
1586 && (widening_optab_handler ((unsignedp ? umul_widen_optab
1587 : smul_widen_optab),
1588 GET_MODE_2XWIDER_MODE (mode), mode)
1589 != CODE_FOR_nothing))
1590 {
1591 temp = expand_binop (GET_MODE_2XWIDER_MODE (mode),
1592 unsignedp ? umul_widen_optab : smul_widen_optab,
1593 op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1594
1595 if (temp != 0)
1596 {
1597 if (GET_MODE_CLASS (mode) == MODE_INT
1598 && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1599 return gen_lowpart (mode, temp);
1600 else
1601 return convert_to_mode (mode, temp, unsignedp);
1602 }
1603 }
1604
1605 /* If this is a vector shift by a scalar, see if we can do a vector
1606 shift by a vector. If so, broadcast the scalar into a vector. */
1607 if (mclass == MODE_VECTOR_INT)
1608 {
1609 optab otheroptab = unknown_optab;
1610
1611 if (binoptab == ashl_optab)
1612 otheroptab = vashl_optab;
1613 else if (binoptab == ashr_optab)
1614 otheroptab = vashr_optab;
1615 else if (binoptab == lshr_optab)
1616 otheroptab = vlshr_optab;
1617 else if (binoptab == rotl_optab)
1618 otheroptab = vrotl_optab;
1619 else if (binoptab == rotr_optab)
1620 otheroptab = vrotr_optab;
1621
1622 if (otheroptab && optab_handler (otheroptab, mode) != CODE_FOR_nothing)
1623 {
1624 rtx vop1 = expand_vector_broadcast (mode, op1);
1625 if (vop1)
1626 {
1627 temp = expand_binop_directly (mode, otheroptab, op0, vop1,
1628 target, unsignedp, methods, last);
1629 if (temp)
1630 return temp;
1631 }
1632 }
1633 }
1634
1635 /* Look for a wider mode of the same class for which we think we
1636 can open-code the operation. Check for a widening multiply at the
1637 wider mode as well. */
1638
1639 if (CLASS_HAS_WIDER_MODES_P (mclass)
1640 && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1641 for (wider_mode = GET_MODE_WIDER_MODE (mode);
1642 wider_mode != VOIDmode;
1643 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
1644 {
1645 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1646 || (binoptab == smul_optab
1647 && GET_MODE_WIDER_MODE (wider_mode) != VOIDmode
1648 && (find_widening_optab_handler ((unsignedp
1649 ? umul_widen_optab
1650 : smul_widen_optab),
1651 GET_MODE_WIDER_MODE (wider_mode),
1652 mode, 0)
1653 != CODE_FOR_nothing)))
1654 {
1655 rtx xop0 = op0, xop1 = op1;
1656 int no_extend = 0;
1657
1658 /* For certain integer operations, we need not actually extend
1659 the narrow operands, as long as we will truncate
1660 the results to the same narrowness. */
1661
1662 if ((binoptab == ior_optab || binoptab == and_optab
1663 || binoptab == xor_optab
1664 || binoptab == add_optab || binoptab == sub_optab
1665 || binoptab == smul_optab || binoptab == ashl_optab)
1666 && mclass == MODE_INT)
1667 {
1668 no_extend = 1;
1669 xop0 = avoid_expensive_constant (mode, binoptab, 0,
1670 xop0, unsignedp);
1671 if (binoptab != ashl_optab)
1672 xop1 = avoid_expensive_constant (mode, binoptab, 1,
1673 xop1, unsignedp);
1674 }
1675
1676 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1677
1678 /* The second operand of a shift must always be extended. */
1679 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1680 no_extend && binoptab != ashl_optab);
1681
1682 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1683 unsignedp, OPTAB_DIRECT);
1684 if (temp)
1685 {
1686 if (mclass != MODE_INT
1687 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1688 {
1689 if (target == 0)
1690 target = gen_reg_rtx (mode);
1691 convert_move (target, temp, 0);
1692 return target;
1693 }
1694 else
1695 return gen_lowpart (mode, temp);
1696 }
1697 else
1698 delete_insns_since (last);
1699 }
1700 }
1701
1702 /* If operation is commutative,
1703 try to make the first operand a register.
1704 Even better, try to make it the same as the target.
1705 Also try to make the last operand a constant. */
1706 if (commutative_optab_p (binoptab)
1707 && swap_commutative_operands_with_target (target, op0, op1))
1708 {
1709 temp = op1;
1710 op1 = op0;
1711 op0 = temp;
1712 }
1713
1714 /* These can be done a word at a time. */
1715 if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1716 && mclass == MODE_INT
1717 && GET_MODE_SIZE (mode) > UNITS_PER_WORD
1718 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1719 {
1720 int i;
1721 rtx insns;
1722
1723 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1724 won't be accurate, so use a new target. */
1725 if (target == 0
1726 || target == op0
1727 || target == op1
1728 || !valid_multiword_target_p (target))
1729 target = gen_reg_rtx (mode);
1730
1731 start_sequence ();
1732
1733 /* Do the actual arithmetic. */
1734 for (i = 0; i < GET_MODE_BITSIZE (mode) / BITS_PER_WORD; i++)
1735 {
1736 rtx target_piece = operand_subword (target, i, 1, mode);
1737 rtx x = expand_binop (word_mode, binoptab,
1738 operand_subword_force (op0, i, mode),
1739 operand_subword_force (op1, i, mode),
1740 target_piece, unsignedp, next_methods);
1741
1742 if (x == 0)
1743 break;
1744
1745 if (target_piece != x)
1746 emit_move_insn (target_piece, x);
1747 }
1748
1749 insns = get_insns ();
1750 end_sequence ();
1751
1752 if (i == GET_MODE_BITSIZE (mode) / BITS_PER_WORD)
1753 {
1754 emit_insn (insns);
1755 return target;
1756 }
1757 }
1758
1759 /* Synthesize double word shifts from single word shifts. */
1760 if ((binoptab == lshr_optab || binoptab == ashl_optab
1761 || binoptab == ashr_optab)
1762 && mclass == MODE_INT
1763 && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1764 && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
1765 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode)
1766 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1767 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1768 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1769 {
1770 unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1771 enum machine_mode op1_mode;
1772
1773 double_shift_mask = targetm.shift_truncation_mask (mode);
1774 shift_mask = targetm.shift_truncation_mask (word_mode);
1775 op1_mode = GET_MODE (op1) != VOIDmode ? GET_MODE (op1) : word_mode;
1776
1777 /* Apply the truncation to constant shifts. */
1778 if (double_shift_mask > 0 && CONST_INT_P (op1))
1779 op1 = GEN_INT (INTVAL (op1) & double_shift_mask);
1780
1781 if (op1 == CONST0_RTX (op1_mode))
1782 return op0;
1783
1784 /* Make sure that this is a combination that expand_doubleword_shift
1785 can handle. See the comments there for details. */
1786 if (double_shift_mask == 0
1787 || (shift_mask == BITS_PER_WORD - 1
1788 && double_shift_mask == BITS_PER_WORD * 2 - 1))
1789 {
1790 rtx insns;
1791 rtx into_target, outof_target;
1792 rtx into_input, outof_input;
1793 int left_shift, outof_word;
1794
1795 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1796 won't be accurate, so use a new target. */
1797 if (target == 0
1798 || target == op0
1799 || target == op1
1800 || !valid_multiword_target_p (target))
1801 target = gen_reg_rtx (mode);
1802
1803 start_sequence ();
1804
1805 /* OUTOF_* is the word we are shifting bits away from, and
1806 INTO_* is the word that we are shifting bits towards, thus
1807 they differ depending on the direction of the shift and
1808 WORDS_BIG_ENDIAN. */
1809
1810 left_shift = binoptab == ashl_optab;
1811 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1812
1813 outof_target = operand_subword (target, outof_word, 1, mode);
1814 into_target = operand_subword (target, 1 - outof_word, 1, mode);
1815
1816 outof_input = operand_subword_force (op0, outof_word, mode);
1817 into_input = operand_subword_force (op0, 1 - outof_word, mode);
1818
1819 if (expand_doubleword_shift (op1_mode, binoptab,
1820 outof_input, into_input, op1,
1821 outof_target, into_target,
1822 unsignedp, next_methods, shift_mask))
1823 {
1824 insns = get_insns ();
1825 end_sequence ();
1826
1827 emit_insn (insns);
1828 return target;
1829 }
1830 end_sequence ();
1831 }
1832 }
1833
1834 /* Synthesize double word rotates from single word shifts. */
1835 if ((binoptab == rotl_optab || binoptab == rotr_optab)
1836 && mclass == MODE_INT
1837 && CONST_INT_P (op1)
1838 && GET_MODE_PRECISION (mode) == 2 * BITS_PER_WORD
1839 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1840 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1841 {
1842 rtx insns;
1843 rtx into_target, outof_target;
1844 rtx into_input, outof_input;
1845 rtx inter;
1846 int shift_count, left_shift, outof_word;
1847
1848 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1849 won't be accurate, so use a new target. Do this also if target is not
1850 a REG, first because having a register instead may open optimization
1851 opportunities, and second because if target and op0 happen to be MEMs
1852 designating the same location, we would risk clobbering it too early
1853 in the code sequence we generate below. */
1854 if (target == 0
1855 || target == op0
1856 || target == op1
1857 || !REG_P (target)
1858 || !valid_multiword_target_p (target))
1859 target = gen_reg_rtx (mode);
1860
1861 start_sequence ();
1862
1863 shift_count = INTVAL (op1);
1864
1865 /* OUTOF_* is the word we are shifting bits away from, and
1866 INTO_* is the word that we are shifting bits towards, thus
1867 they differ depending on the direction of the shift and
1868 WORDS_BIG_ENDIAN. */
1869
1870 left_shift = (binoptab == rotl_optab);
1871 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1872
1873 outof_target = operand_subword (target, outof_word, 1, mode);
1874 into_target = operand_subword (target, 1 - outof_word, 1, mode);
1875
1876 outof_input = operand_subword_force (op0, outof_word, mode);
1877 into_input = operand_subword_force (op0, 1 - outof_word, mode);
1878
1879 if (shift_count == BITS_PER_WORD)
1880 {
1881 /* This is just a word swap. */
1882 emit_move_insn (outof_target, into_input);
1883 emit_move_insn (into_target, outof_input);
1884 inter = const0_rtx;
1885 }
1886 else
1887 {
1888 rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1889 rtx first_shift_count, second_shift_count;
1890 optab reverse_unsigned_shift, unsigned_shift;
1891
1892 reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1893 ? lshr_optab : ashl_optab);
1894
1895 unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1896 ? ashl_optab : lshr_optab);
1897
1898 if (shift_count > BITS_PER_WORD)
1899 {
1900 first_shift_count = GEN_INT (shift_count - BITS_PER_WORD);
1901 second_shift_count = GEN_INT (2 * BITS_PER_WORD - shift_count);
1902 }
1903 else
1904 {
1905 first_shift_count = GEN_INT (BITS_PER_WORD - shift_count);
1906 second_shift_count = GEN_INT (shift_count);
1907 }
1908
1909 into_temp1 = expand_binop (word_mode, unsigned_shift,
1910 outof_input, first_shift_count,
1911 NULL_RTX, unsignedp, next_methods);
1912 into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1913 into_input, second_shift_count,
1914 NULL_RTX, unsignedp, next_methods);
1915
1916 if (into_temp1 != 0 && into_temp2 != 0)
1917 inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2,
1918 into_target, unsignedp, next_methods);
1919 else
1920 inter = 0;
1921
1922 if (inter != 0 && inter != into_target)
1923 emit_move_insn (into_target, inter);
1924
1925 outof_temp1 = expand_binop (word_mode, unsigned_shift,
1926 into_input, first_shift_count,
1927 NULL_RTX, unsignedp, next_methods);
1928 outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1929 outof_input, second_shift_count,
1930 NULL_RTX, unsignedp, next_methods);
1931
1932 if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1933 inter = expand_binop (word_mode, ior_optab,
1934 outof_temp1, outof_temp2,
1935 outof_target, unsignedp, next_methods);
1936
1937 if (inter != 0 && inter != outof_target)
1938 emit_move_insn (outof_target, inter);
1939 }
1940
1941 insns = get_insns ();
1942 end_sequence ();
1943
1944 if (inter != 0)
1945 {
1946 emit_insn (insns);
1947 return target;
1948 }
1949 }
1950
1951 /* These can be done a word at a time by propagating carries. */
1952 if ((binoptab == add_optab || binoptab == sub_optab)
1953 && mclass == MODE_INT
1954 && GET_MODE_SIZE (mode) >= 2 * UNITS_PER_WORD
1955 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1956 {
1957 unsigned int i;
1958 optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
1959 const unsigned int nwords = GET_MODE_BITSIZE (mode) / BITS_PER_WORD;
1960 rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
1961 rtx xop0, xop1, xtarget;
1962
1963 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1964 value is one of those, use it. Otherwise, use 1 since it is the
1965 one easiest to get. */
1966 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1967 int normalizep = STORE_FLAG_VALUE;
1968 #else
1969 int normalizep = 1;
1970 #endif
1971
1972 /* Prepare the operands. */
1973 xop0 = force_reg (mode, op0);
1974 xop1 = force_reg (mode, op1);
1975
1976 xtarget = gen_reg_rtx (mode);
1977
1978 if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1979 target = xtarget;
1980
1981 /* Indicate for flow that the entire target reg is being set. */
1982 if (REG_P (target))
1983 emit_clobber (xtarget);
1984
1985 /* Do the actual arithmetic. */
1986 for (i = 0; i < nwords; i++)
1987 {
1988 int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
1989 rtx target_piece = operand_subword (xtarget, index, 1, mode);
1990 rtx op0_piece = operand_subword_force (xop0, index, mode);
1991 rtx op1_piece = operand_subword_force (xop1, index, mode);
1992 rtx x;
1993
1994 /* Main add/subtract of the input operands. */
1995 x = expand_binop (word_mode, binoptab,
1996 op0_piece, op1_piece,
1997 target_piece, unsignedp, next_methods);
1998 if (x == 0)
1999 break;
2000
2001 if (i + 1 < nwords)
2002 {
2003 /* Store carry from main add/subtract. */
2004 carry_out = gen_reg_rtx (word_mode);
2005 carry_out = emit_store_flag_force (carry_out,
2006 (binoptab == add_optab
2007 ? LT : GT),
2008 x, op0_piece,
2009 word_mode, 1, normalizep);
2010 }
2011
2012 if (i > 0)
2013 {
2014 rtx newx;
2015
2016 /* Add/subtract previous carry to main result. */
2017 newx = expand_binop (word_mode,
2018 normalizep == 1 ? binoptab : otheroptab,
2019 x, carry_in,
2020 NULL_RTX, 1, next_methods);
2021
2022 if (i + 1 < nwords)
2023 {
2024 /* Get out carry from adding/subtracting carry in. */
2025 rtx carry_tmp = gen_reg_rtx (word_mode);
2026 carry_tmp = emit_store_flag_force (carry_tmp,
2027 (binoptab == add_optab
2028 ? LT : GT),
2029 newx, x,
2030 word_mode, 1, normalizep);
2031
2032 /* Logical-ior the two poss. carry together. */
2033 carry_out = expand_binop (word_mode, ior_optab,
2034 carry_out, carry_tmp,
2035 carry_out, 0, next_methods);
2036 if (carry_out == 0)
2037 break;
2038 }
2039 emit_move_insn (target_piece, newx);
2040 }
2041 else
2042 {
2043 if (x != target_piece)
2044 emit_move_insn (target_piece, x);
2045 }
2046
2047 carry_in = carry_out;
2048 }
2049
2050 if (i == GET_MODE_BITSIZE (mode) / (unsigned) BITS_PER_WORD)
2051 {
2052 if (optab_handler (mov_optab, mode) != CODE_FOR_nothing
2053 || ! rtx_equal_p (target, xtarget))
2054 {
2055 rtx temp = emit_move_insn (target, xtarget);
2056
2057 set_dst_reg_note (temp, REG_EQUAL,
2058 gen_rtx_fmt_ee (optab_to_code (binoptab),
2059 mode, copy_rtx (xop0),
2060 copy_rtx (xop1)),
2061 target);
2062 }
2063 else
2064 target = xtarget;
2065
2066 return target;
2067 }
2068
2069 else
2070 delete_insns_since (last);
2071 }
2072
2073 /* Attempt to synthesize double word multiplies using a sequence of word
2074 mode multiplications. We first attempt to generate a sequence using a
2075 more efficient unsigned widening multiply, and if that fails we then
2076 try using a signed widening multiply. */
2077
2078 if (binoptab == smul_optab
2079 && mclass == MODE_INT
2080 && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
2081 && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
2082 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
2083 {
2084 rtx product = NULL_RTX;
2085 if (widening_optab_handler (umul_widen_optab, mode, word_mode)
2086 != CODE_FOR_nothing)
2087 {
2088 product = expand_doubleword_mult (mode, op0, op1, target,
2089 true, methods);
2090 if (!product)
2091 delete_insns_since (last);
2092 }
2093
2094 if (product == NULL_RTX
2095 && widening_optab_handler (smul_widen_optab, mode, word_mode)
2096 != CODE_FOR_nothing)
2097 {
2098 product = expand_doubleword_mult (mode, op0, op1, target,
2099 false, methods);
2100 if (!product)
2101 delete_insns_since (last);
2102 }
2103
2104 if (product != NULL_RTX)
2105 {
2106 if (optab_handler (mov_optab, mode) != CODE_FOR_nothing)
2107 {
2108 temp = emit_move_insn (target ? target : product, product);
2109 set_dst_reg_note (temp,
2110 REG_EQUAL,
2111 gen_rtx_fmt_ee (MULT, mode,
2112 copy_rtx (op0),
2113 copy_rtx (op1)),
2114 target ? target : product);
2115 }
2116 return product;
2117 }
2118 }
2119
2120 /* It can't be open-coded in this mode.
2121 Use a library call if one is available and caller says that's ok. */
2122
2123 libfunc = optab_libfunc (binoptab, mode);
2124 if (libfunc
2125 && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
2126 {
2127 rtx insns;
2128 rtx op1x = op1;
2129 enum machine_mode op1_mode = mode;
2130 rtx value;
2131
2132 start_sequence ();
2133
2134 if (shift_optab_p (binoptab))
2135 {
2136 op1_mode = targetm.libgcc_shift_count_mode ();
2137 /* Specify unsigned here,
2138 since negative shift counts are meaningless. */
2139 op1x = convert_to_mode (op1_mode, op1, 1);
2140 }
2141
2142 if (GET_MODE (op0) != VOIDmode
2143 && GET_MODE (op0) != mode)
2144 op0 = convert_to_mode (mode, op0, unsignedp);
2145
2146 /* Pass 1 for NO_QUEUE so we don't lose any increments
2147 if the libcall is cse'd or moved. */
2148 value = emit_library_call_value (libfunc,
2149 NULL_RTX, LCT_CONST, mode, 2,
2150 op0, mode, op1x, op1_mode);
2151
2152 insns = get_insns ();
2153 end_sequence ();
2154
2155 target = gen_reg_rtx (mode);
2156 emit_libcall_block_1 (insns, target, value,
2157 gen_rtx_fmt_ee (optab_to_code (binoptab),
2158 mode, op0, op1),
2159 trapv_binoptab_p (binoptab));
2160
2161 return target;
2162 }
2163
2164 delete_insns_since (last);
2165
2166 /* It can't be done in this mode. Can we do it in a wider mode? */
2167
2168 if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
2169 || methods == OPTAB_MUST_WIDEN))
2170 {
2171 /* Caller says, don't even try. */
2172 delete_insns_since (entry_last);
2173 return 0;
2174 }
2175
2176 /* Compute the value of METHODS to pass to recursive calls.
2177 Don't allow widening to be tried recursively. */
2178
2179 methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
2180
2181 /* Look for a wider mode of the same class for which it appears we can do
2182 the operation. */
2183
2184 if (CLASS_HAS_WIDER_MODES_P (mclass))
2185 {
2186 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2187 wider_mode != VOIDmode;
2188 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2189 {
2190 if (find_widening_optab_handler (binoptab, wider_mode, mode, 1)
2191 != CODE_FOR_nothing
2192 || (methods == OPTAB_LIB
2193 && optab_libfunc (binoptab, wider_mode)))
2194 {
2195 rtx xop0 = op0, xop1 = op1;
2196 int no_extend = 0;
2197
2198 /* For certain integer operations, we need not actually extend
2199 the narrow operands, as long as we will truncate
2200 the results to the same narrowness. */
2201
2202 if ((binoptab == ior_optab || binoptab == and_optab
2203 || binoptab == xor_optab
2204 || binoptab == add_optab || binoptab == sub_optab
2205 || binoptab == smul_optab || binoptab == ashl_optab)
2206 && mclass == MODE_INT)
2207 no_extend = 1;
2208
2209 xop0 = widen_operand (xop0, wider_mode, mode,
2210 unsignedp, no_extend);
2211
2212 /* The second operand of a shift must always be extended. */
2213 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
2214 no_extend && binoptab != ashl_optab);
2215
2216 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
2217 unsignedp, methods);
2218 if (temp)
2219 {
2220 if (mclass != MODE_INT
2221 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2222 {
2223 if (target == 0)
2224 target = gen_reg_rtx (mode);
2225 convert_move (target, temp, 0);
2226 return target;
2227 }
2228 else
2229 return gen_lowpart (mode, temp);
2230 }
2231 else
2232 delete_insns_since (last);
2233 }
2234 }
2235 }
2236
2237 delete_insns_since (entry_last);
2238 return 0;
2239 }
2240 \f
2241 /* Expand a binary operator which has both signed and unsigned forms.
2242 UOPTAB is the optab for unsigned operations, and SOPTAB is for
2243 signed operations.
2244
2245 If we widen unsigned operands, we may use a signed wider operation instead
2246 of an unsigned wider operation, since the result would be the same. */
2247
2248 rtx
2249 sign_expand_binop (enum machine_mode mode, optab uoptab, optab soptab,
2250 rtx op0, rtx op1, rtx target, int unsignedp,
2251 enum optab_methods methods)
2252 {
2253 rtx temp;
2254 optab direct_optab = unsignedp ? uoptab : soptab;
2255 bool save_enable;
2256
2257 /* Do it without widening, if possible. */
2258 temp = expand_binop (mode, direct_optab, op0, op1, target,
2259 unsignedp, OPTAB_DIRECT);
2260 if (temp || methods == OPTAB_DIRECT)
2261 return temp;
2262
2263 /* Try widening to a signed int. Disable any direct use of any
2264 signed insn in the current mode. */
2265 save_enable = swap_optab_enable (soptab, mode, false);
2266
2267 temp = expand_binop (mode, soptab, op0, op1, target,
2268 unsignedp, OPTAB_WIDEN);
2269
2270 /* For unsigned operands, try widening to an unsigned int. */
2271 if (!temp && unsignedp)
2272 temp = expand_binop (mode, uoptab, op0, op1, target,
2273 unsignedp, OPTAB_WIDEN);
2274 if (temp || methods == OPTAB_WIDEN)
2275 goto egress;
2276
2277 /* Use the right width libcall if that exists. */
2278 temp = expand_binop (mode, direct_optab, op0, op1, target,
2279 unsignedp, OPTAB_LIB);
2280 if (temp || methods == OPTAB_LIB)
2281 goto egress;
2282
2283 /* Must widen and use a libcall, use either signed or unsigned. */
2284 temp = expand_binop (mode, soptab, op0, op1, target,
2285 unsignedp, methods);
2286 if (!temp && unsignedp)
2287 temp = expand_binop (mode, uoptab, op0, op1, target,
2288 unsignedp, methods);
2289
2290 egress:
2291 /* Undo the fiddling above. */
2292 if (save_enable)
2293 swap_optab_enable (soptab, mode, true);
2294 return temp;
2295 }
2296 \f
2297 /* Generate code to perform an operation specified by UNOPPTAB
2298 on operand OP0, with two results to TARG0 and TARG1.
2299 We assume that the order of the operands for the instruction
2300 is TARG0, TARG1, OP0.
2301
2302 Either TARG0 or TARG1 may be zero, but what that means is that
2303 the result is not actually wanted. We will generate it into
2304 a dummy pseudo-reg and discard it. They may not both be zero.
2305
2306 Returns 1 if this operation can be performed; 0 if not. */
2307
2308 int
2309 expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2310 int unsignedp)
2311 {
2312 enum machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2313 enum mode_class mclass;
2314 enum machine_mode wider_mode;
2315 rtx entry_last = get_last_insn ();
2316 rtx last;
2317
2318 mclass = GET_MODE_CLASS (mode);
2319
2320 if (!targ0)
2321 targ0 = gen_reg_rtx (mode);
2322 if (!targ1)
2323 targ1 = gen_reg_rtx (mode);
2324
2325 /* Record where to go back to if we fail. */
2326 last = get_last_insn ();
2327
2328 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2329 {
2330 struct expand_operand ops[3];
2331 enum insn_code icode = optab_handler (unoptab, mode);
2332
2333 create_fixed_operand (&ops[0], targ0);
2334 create_fixed_operand (&ops[1], targ1);
2335 create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2336 if (maybe_expand_insn (icode, 3, ops))
2337 return 1;
2338 }
2339
2340 /* It can't be done in this mode. Can we do it in a wider mode? */
2341
2342 if (CLASS_HAS_WIDER_MODES_P (mclass))
2343 {
2344 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2345 wider_mode != VOIDmode;
2346 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2347 {
2348 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2349 {
2350 rtx t0 = gen_reg_rtx (wider_mode);
2351 rtx t1 = gen_reg_rtx (wider_mode);
2352 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2353
2354 if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2355 {
2356 convert_move (targ0, t0, unsignedp);
2357 convert_move (targ1, t1, unsignedp);
2358 return 1;
2359 }
2360 else
2361 delete_insns_since (last);
2362 }
2363 }
2364 }
2365
2366 delete_insns_since (entry_last);
2367 return 0;
2368 }
2369 \f
2370 /* Generate code to perform an operation specified by BINOPTAB
2371 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2372 We assume that the order of the operands for the instruction
2373 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2374 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2375
2376 Either TARG0 or TARG1 may be zero, but what that means is that
2377 the result is not actually wanted. We will generate it into
2378 a dummy pseudo-reg and discard it. They may not both be zero.
2379
2380 Returns 1 if this operation can be performed; 0 if not. */
2381
2382 int
2383 expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2384 int unsignedp)
2385 {
2386 enum machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2387 enum mode_class mclass;
2388 enum machine_mode wider_mode;
2389 rtx entry_last = get_last_insn ();
2390 rtx last;
2391
2392 mclass = GET_MODE_CLASS (mode);
2393
2394 if (!targ0)
2395 targ0 = gen_reg_rtx (mode);
2396 if (!targ1)
2397 targ1 = gen_reg_rtx (mode);
2398
2399 /* Record where to go back to if we fail. */
2400 last = get_last_insn ();
2401
2402 if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2403 {
2404 struct expand_operand ops[4];
2405 enum insn_code icode = optab_handler (binoptab, mode);
2406 enum machine_mode mode0 = insn_data[icode].operand[1].mode;
2407 enum machine_mode mode1 = insn_data[icode].operand[2].mode;
2408 rtx xop0 = op0, xop1 = op1;
2409
2410 /* If we are optimizing, force expensive constants into a register. */
2411 xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2412 xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2413
2414 create_fixed_operand (&ops[0], targ0);
2415 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2416 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
2417 create_fixed_operand (&ops[3], targ1);
2418 if (maybe_expand_insn (icode, 4, ops))
2419 return 1;
2420 delete_insns_since (last);
2421 }
2422
2423 /* It can't be done in this mode. Can we do it in a wider mode? */
2424
2425 if (CLASS_HAS_WIDER_MODES_P (mclass))
2426 {
2427 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2428 wider_mode != VOIDmode;
2429 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2430 {
2431 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2432 {
2433 rtx t0 = gen_reg_rtx (wider_mode);
2434 rtx t1 = gen_reg_rtx (wider_mode);
2435 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2436 rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2437
2438 if (expand_twoval_binop (binoptab, cop0, cop1,
2439 t0, t1, unsignedp))
2440 {
2441 convert_move (targ0, t0, unsignedp);
2442 convert_move (targ1, t1, unsignedp);
2443 return 1;
2444 }
2445 else
2446 delete_insns_since (last);
2447 }
2448 }
2449 }
2450
2451 delete_insns_since (entry_last);
2452 return 0;
2453 }
2454
2455 /* Expand the two-valued library call indicated by BINOPTAB, but
2456 preserve only one of the values. If TARG0 is non-NULL, the first
2457 value is placed into TARG0; otherwise the second value is placed
2458 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2459 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2460 This routine assumes that the value returned by the library call is
2461 as if the return value was of an integral mode twice as wide as the
2462 mode of OP0. Returns 1 if the call was successful. */
2463
2464 bool
2465 expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2466 rtx targ0, rtx targ1, enum rtx_code code)
2467 {
2468 enum machine_mode mode;
2469 enum machine_mode libval_mode;
2470 rtx libval;
2471 rtx insns;
2472 rtx libfunc;
2473
2474 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2475 gcc_assert (!targ0 != !targ1);
2476
2477 mode = GET_MODE (op0);
2478 libfunc = optab_libfunc (binoptab, mode);
2479 if (!libfunc)
2480 return false;
2481
2482 /* The value returned by the library function will have twice as
2483 many bits as the nominal MODE. */
2484 libval_mode = smallest_mode_for_size (2 * GET_MODE_BITSIZE (mode),
2485 MODE_INT);
2486 start_sequence ();
2487 libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2488 libval_mode, 2,
2489 op0, mode,
2490 op1, mode);
2491 /* Get the part of VAL containing the value that we want. */
2492 libval = simplify_gen_subreg (mode, libval, libval_mode,
2493 targ0 ? 0 : GET_MODE_SIZE (mode));
2494 insns = get_insns ();
2495 end_sequence ();
2496 /* Move the into the desired location. */
2497 emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2498 gen_rtx_fmt_ee (code, mode, op0, op1));
2499
2500 return true;
2501 }
2502
2503 \f
2504 /* Wrapper around expand_unop which takes an rtx code to specify
2505 the operation to perform, not an optab pointer. All other
2506 arguments are the same. */
2507 rtx
2508 expand_simple_unop (enum machine_mode mode, enum rtx_code code, rtx op0,
2509 rtx target, int unsignedp)
2510 {
2511 optab unop = code_to_optab (code);
2512 gcc_assert (unop);
2513
2514 return expand_unop (mode, unop, op0, target, unsignedp);
2515 }
2516
2517 /* Try calculating
2518 (clz:narrow x)
2519 as
2520 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2521
2522 A similar operation can be used for clrsb. UNOPTAB says which operation
2523 we are trying to expand. */
2524 static rtx
2525 widen_leading (enum machine_mode mode, rtx op0, rtx target, optab unoptab)
2526 {
2527 enum mode_class mclass = GET_MODE_CLASS (mode);
2528 if (CLASS_HAS_WIDER_MODES_P (mclass))
2529 {
2530 enum machine_mode wider_mode;
2531 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2532 wider_mode != VOIDmode;
2533 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2534 {
2535 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2536 {
2537 rtx xop0, temp, last;
2538
2539 last = get_last_insn ();
2540
2541 if (target == 0)
2542 target = gen_reg_rtx (mode);
2543 xop0 = widen_operand (op0, wider_mode, mode,
2544 unoptab != clrsb_optab, false);
2545 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2546 unoptab != clrsb_optab);
2547 if (temp != 0)
2548 temp = expand_binop
2549 (wider_mode, sub_optab, temp,
2550 gen_int_mode (GET_MODE_PRECISION (wider_mode)
2551 - GET_MODE_PRECISION (mode),
2552 wider_mode),
2553 target, true, OPTAB_DIRECT);
2554 if (temp == 0)
2555 delete_insns_since (last);
2556
2557 return temp;
2558 }
2559 }
2560 }
2561 return 0;
2562 }
2563
2564 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2565 quantities, choosing which based on whether the high word is nonzero. */
2566 static rtx
2567 expand_doubleword_clz (enum machine_mode mode, rtx op0, rtx target)
2568 {
2569 rtx xop0 = force_reg (mode, op0);
2570 rtx subhi = gen_highpart (word_mode, xop0);
2571 rtx sublo = gen_lowpart (word_mode, xop0);
2572 rtx hi0_label = gen_label_rtx ();
2573 rtx after_label = gen_label_rtx ();
2574 rtx seq, temp, result;
2575
2576 /* If we were not given a target, use a word_mode register, not a
2577 'mode' register. The result will fit, and nobody is expecting
2578 anything bigger (the return type of __builtin_clz* is int). */
2579 if (!target)
2580 target = gen_reg_rtx (word_mode);
2581
2582 /* In any case, write to a word_mode scratch in both branches of the
2583 conditional, so we can ensure there is a single move insn setting
2584 'target' to tag a REG_EQUAL note on. */
2585 result = gen_reg_rtx (word_mode);
2586
2587 start_sequence ();
2588
2589 /* If the high word is not equal to zero,
2590 then clz of the full value is clz of the high word. */
2591 emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2592 word_mode, true, hi0_label);
2593
2594 temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true);
2595 if (!temp)
2596 goto fail;
2597
2598 if (temp != result)
2599 convert_move (result, temp, true);
2600
2601 emit_jump_insn (gen_jump (after_label));
2602 emit_barrier ();
2603
2604 /* Else clz of the full value is clz of the low word plus the number
2605 of bits in the high word. */
2606 emit_label (hi0_label);
2607
2608 temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true);
2609 if (!temp)
2610 goto fail;
2611 temp = expand_binop (word_mode, add_optab, temp,
2612 gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode),
2613 result, true, OPTAB_DIRECT);
2614 if (!temp)
2615 goto fail;
2616 if (temp != result)
2617 convert_move (result, temp, true);
2618
2619 emit_label (after_label);
2620 convert_move (target, result, true);
2621
2622 seq = get_insns ();
2623 end_sequence ();
2624
2625 add_equal_note (seq, target, CLZ, xop0, 0);
2626 emit_insn (seq);
2627 return target;
2628
2629 fail:
2630 end_sequence ();
2631 return 0;
2632 }
2633
2634 /* Try calculating
2635 (bswap:narrow x)
2636 as
2637 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2638 static rtx
2639 widen_bswap (enum machine_mode mode, rtx op0, rtx target)
2640 {
2641 enum mode_class mclass = GET_MODE_CLASS (mode);
2642 enum machine_mode wider_mode;
2643 rtx x, last;
2644
2645 if (!CLASS_HAS_WIDER_MODES_P (mclass))
2646 return NULL_RTX;
2647
2648 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2649 wider_mode != VOIDmode;
2650 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2651 if (optab_handler (bswap_optab, wider_mode) != CODE_FOR_nothing)
2652 goto found;
2653 return NULL_RTX;
2654
2655 found:
2656 last = get_last_insn ();
2657
2658 x = widen_operand (op0, wider_mode, mode, true, true);
2659 x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2660
2661 gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2662 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2663 if (x != 0)
2664 x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2665 GET_MODE_BITSIZE (wider_mode)
2666 - GET_MODE_BITSIZE (mode),
2667 NULL_RTX, true);
2668
2669 if (x != 0)
2670 {
2671 if (target == 0)
2672 target = gen_reg_rtx (mode);
2673 emit_move_insn (target, gen_lowpart (mode, x));
2674 }
2675 else
2676 delete_insns_since (last);
2677
2678 return target;
2679 }
2680
2681 /* Try calculating bswap as two bswaps of two word-sized operands. */
2682
2683 static rtx
2684 expand_doubleword_bswap (enum machine_mode mode, rtx op, rtx target)
2685 {
2686 rtx t0, t1;
2687
2688 t1 = expand_unop (word_mode, bswap_optab,
2689 operand_subword_force (op, 0, mode), NULL_RTX, true);
2690 t0 = expand_unop (word_mode, bswap_optab,
2691 operand_subword_force (op, 1, mode), NULL_RTX, true);
2692
2693 if (target == 0 || !valid_multiword_target_p (target))
2694 target = gen_reg_rtx (mode);
2695 if (REG_P (target))
2696 emit_clobber (target);
2697 emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2698 emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2699
2700 return target;
2701 }
2702
2703 /* Try calculating (parity x) as (and (popcount x) 1), where
2704 popcount can also be done in a wider mode. */
2705 static rtx
2706 expand_parity (enum machine_mode mode, rtx op0, rtx target)
2707 {
2708 enum mode_class mclass = GET_MODE_CLASS (mode);
2709 if (CLASS_HAS_WIDER_MODES_P (mclass))
2710 {
2711 enum machine_mode wider_mode;
2712 for (wider_mode = mode; wider_mode != VOIDmode;
2713 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2714 {
2715 if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2716 {
2717 rtx xop0, temp, last;
2718
2719 last = get_last_insn ();
2720
2721 if (target == 0)
2722 target = gen_reg_rtx (mode);
2723 xop0 = widen_operand (op0, wider_mode, mode, true, false);
2724 temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2725 true);
2726 if (temp != 0)
2727 temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2728 target, true, OPTAB_DIRECT);
2729 if (temp == 0)
2730 delete_insns_since (last);
2731
2732 return temp;
2733 }
2734 }
2735 }
2736 return 0;
2737 }
2738
2739 /* Try calculating ctz(x) as K - clz(x & -x) ,
2740 where K is GET_MODE_PRECISION(mode) - 1.
2741
2742 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2743 don't have to worry about what the hardware does in that case. (If
2744 the clz instruction produces the usual value at 0, which is K, the
2745 result of this code sequence will be -1; expand_ffs, below, relies
2746 on this. It might be nice to have it be K instead, for consistency
2747 with the (very few) processors that provide a ctz with a defined
2748 value, but that would take one more instruction, and it would be
2749 less convenient for expand_ffs anyway. */
2750
2751 static rtx
2752 expand_ctz (enum machine_mode mode, rtx op0, rtx target)
2753 {
2754 rtx seq, temp;
2755
2756 if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2757 return 0;
2758
2759 start_sequence ();
2760
2761 temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
2762 if (temp)
2763 temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
2764 true, OPTAB_DIRECT);
2765 if (temp)
2766 temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
2767 if (temp)
2768 temp = expand_binop (mode, sub_optab,
2769 gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
2770 temp, target,
2771 true, OPTAB_DIRECT);
2772 if (temp == 0)
2773 {
2774 end_sequence ();
2775 return 0;
2776 }
2777
2778 seq = get_insns ();
2779 end_sequence ();
2780
2781 add_equal_note (seq, temp, CTZ, op0, 0);
2782 emit_insn (seq);
2783 return temp;
2784 }
2785
2786
2787 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2788 else with the sequence used by expand_clz.
2789
2790 The ffs builtin promises to return zero for a zero value and ctz/clz
2791 may have an undefined value in that case. If they do not give us a
2792 convenient value, we have to generate a test and branch. */
2793 static rtx
2794 expand_ffs (enum machine_mode mode, rtx op0, rtx target)
2795 {
2796 HOST_WIDE_INT val = 0;
2797 bool defined_at_zero = false;
2798 rtx temp, seq;
2799
2800 if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
2801 {
2802 start_sequence ();
2803
2804 temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
2805 if (!temp)
2806 goto fail;
2807
2808 defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
2809 }
2810 else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
2811 {
2812 start_sequence ();
2813 temp = expand_ctz (mode, op0, 0);
2814 if (!temp)
2815 goto fail;
2816
2817 if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
2818 {
2819 defined_at_zero = true;
2820 val = (GET_MODE_PRECISION (mode) - 1) - val;
2821 }
2822 }
2823 else
2824 return 0;
2825
2826 if (defined_at_zero && val == -1)
2827 /* No correction needed at zero. */;
2828 else
2829 {
2830 /* We don't try to do anything clever with the situation found
2831 on some processors (eg Alpha) where ctz(0:mode) ==
2832 bitsize(mode). If someone can think of a way to send N to -1
2833 and leave alone all values in the range 0..N-1 (where N is a
2834 power of two), cheaper than this test-and-branch, please add it.
2835
2836 The test-and-branch is done after the operation itself, in case
2837 the operation sets condition codes that can be recycled for this.
2838 (This is true on i386, for instance.) */
2839
2840 rtx nonzero_label = gen_label_rtx ();
2841 emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
2842 mode, true, nonzero_label);
2843
2844 convert_move (temp, GEN_INT (-1), false);
2845 emit_label (nonzero_label);
2846 }
2847
2848 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2849 to produce a value in the range 0..bitsize. */
2850 temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
2851 target, false, OPTAB_DIRECT);
2852 if (!temp)
2853 goto fail;
2854
2855 seq = get_insns ();
2856 end_sequence ();
2857
2858 add_equal_note (seq, temp, FFS, op0, 0);
2859 emit_insn (seq);
2860 return temp;
2861
2862 fail:
2863 end_sequence ();
2864 return 0;
2865 }
2866
2867 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2868 conditions, VAL may already be a SUBREG against which we cannot generate
2869 a further SUBREG. In this case, we expect forcing the value into a
2870 register will work around the situation. */
2871
2872 static rtx
2873 lowpart_subreg_maybe_copy (enum machine_mode omode, rtx val,
2874 enum machine_mode imode)
2875 {
2876 rtx ret;
2877 ret = lowpart_subreg (omode, val, imode);
2878 if (ret == NULL)
2879 {
2880 val = force_reg (imode, val);
2881 ret = lowpart_subreg (omode, val, imode);
2882 gcc_assert (ret != NULL);
2883 }
2884 return ret;
2885 }
2886
2887 /* Expand a floating point absolute value or negation operation via a
2888 logical operation on the sign bit. */
2889
2890 static rtx
2891 expand_absneg_bit (enum rtx_code code, enum machine_mode mode,
2892 rtx op0, rtx target)
2893 {
2894 const struct real_format *fmt;
2895 int bitpos, word, nwords, i;
2896 enum machine_mode imode;
2897 rtx temp, insns;
2898
2899 /* The format has to have a simple sign bit. */
2900 fmt = REAL_MODE_FORMAT (mode);
2901 if (fmt == NULL)
2902 return NULL_RTX;
2903
2904 bitpos = fmt->signbit_rw;
2905 if (bitpos < 0)
2906 return NULL_RTX;
2907
2908 /* Don't create negative zeros if the format doesn't support them. */
2909 if (code == NEG && !fmt->has_signed_zero)
2910 return NULL_RTX;
2911
2912 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
2913 {
2914 imode = int_mode_for_mode (mode);
2915 if (imode == BLKmode)
2916 return NULL_RTX;
2917 word = 0;
2918 nwords = 1;
2919 }
2920 else
2921 {
2922 imode = word_mode;
2923
2924 if (FLOAT_WORDS_BIG_ENDIAN)
2925 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
2926 else
2927 word = bitpos / BITS_PER_WORD;
2928 bitpos = bitpos % BITS_PER_WORD;
2929 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
2930 }
2931
2932 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
2933 if (code == ABS)
2934 mask = ~mask;
2935
2936 if (target == 0
2937 || target == op0
2938 || (nwords > 1 && !valid_multiword_target_p (target)))
2939 target = gen_reg_rtx (mode);
2940
2941 if (nwords > 1)
2942 {
2943 start_sequence ();
2944
2945 for (i = 0; i < nwords; ++i)
2946 {
2947 rtx targ_piece = operand_subword (target, i, 1, mode);
2948 rtx op0_piece = operand_subword_force (op0, i, mode);
2949
2950 if (i == word)
2951 {
2952 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2953 op0_piece,
2954 immed_wide_int_const (mask, imode),
2955 targ_piece, 1, OPTAB_LIB_WIDEN);
2956 if (temp != targ_piece)
2957 emit_move_insn (targ_piece, temp);
2958 }
2959 else
2960 emit_move_insn (targ_piece, op0_piece);
2961 }
2962
2963 insns = get_insns ();
2964 end_sequence ();
2965
2966 emit_insn (insns);
2967 }
2968 else
2969 {
2970 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2971 gen_lowpart (imode, op0),
2972 immed_wide_int_const (mask, imode),
2973 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
2974 target = lowpart_subreg_maybe_copy (mode, temp, imode);
2975
2976 set_dst_reg_note (get_last_insn (), REG_EQUAL,
2977 gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
2978 target);
2979 }
2980
2981 return target;
2982 }
2983
2984 /* As expand_unop, but will fail rather than attempt the operation in a
2985 different mode or with a libcall. */
2986 static rtx
2987 expand_unop_direct (enum machine_mode mode, optab unoptab, rtx op0, rtx target,
2988 int unsignedp)
2989 {
2990 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2991 {
2992 struct expand_operand ops[2];
2993 enum insn_code icode = optab_handler (unoptab, mode);
2994 rtx last = get_last_insn ();
2995 rtx pat;
2996
2997 create_output_operand (&ops[0], target, mode);
2998 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2999 pat = maybe_gen_insn (icode, 2, ops);
3000 if (pat)
3001 {
3002 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3003 && ! add_equal_note (pat, ops[0].value, optab_to_code (unoptab),
3004 ops[1].value, NULL_RTX))
3005 {
3006 delete_insns_since (last);
3007 return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
3008 }
3009
3010 emit_insn (pat);
3011
3012 return ops[0].value;
3013 }
3014 }
3015 return 0;
3016 }
3017
3018 /* Generate code to perform an operation specified by UNOPTAB
3019 on operand OP0, with result having machine-mode MODE.
3020
3021 UNSIGNEDP is for the case where we have to widen the operands
3022 to perform the operation. It says to use zero-extension.
3023
3024 If TARGET is nonzero, the value
3025 is generated there, if it is convenient to do so.
3026 In all cases an rtx is returned for the locus of the value;
3027 this may or may not be TARGET. */
3028
3029 rtx
3030 expand_unop (enum machine_mode mode, optab unoptab, rtx op0, rtx target,
3031 int unsignedp)
3032 {
3033 enum mode_class mclass = GET_MODE_CLASS (mode);
3034 enum machine_mode wider_mode;
3035 rtx temp;
3036 rtx libfunc;
3037
3038 temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
3039 if (temp)
3040 return temp;
3041
3042 /* It can't be done in this mode. Can we open-code it in a wider mode? */
3043
3044 /* Widening (or narrowing) clz needs special treatment. */
3045 if (unoptab == clz_optab)
3046 {
3047 temp = widen_leading (mode, op0, target, unoptab);
3048 if (temp)
3049 return temp;
3050
3051 if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
3052 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3053 {
3054 temp = expand_doubleword_clz (mode, op0, target);
3055 if (temp)
3056 return temp;
3057 }
3058
3059 goto try_libcall;
3060 }
3061
3062 if (unoptab == clrsb_optab)
3063 {
3064 temp = widen_leading (mode, op0, target, unoptab);
3065 if (temp)
3066 return temp;
3067 goto try_libcall;
3068 }
3069
3070 /* Widening (or narrowing) bswap needs special treatment. */
3071 if (unoptab == bswap_optab)
3072 {
3073 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3074 or ROTATERT. First try these directly; if this fails, then try the
3075 obvious pair of shifts with allowed widening, as this will probably
3076 be always more efficient than the other fallback methods. */
3077 if (mode == HImode)
3078 {
3079 rtx last, temp1, temp2;
3080
3081 if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
3082 {
3083 temp = expand_binop (mode, rotl_optab, op0, GEN_INT (8), target,
3084 unsignedp, OPTAB_DIRECT);
3085 if (temp)
3086 return temp;
3087 }
3088
3089 if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
3090 {
3091 temp = expand_binop (mode, rotr_optab, op0, GEN_INT (8), target,
3092 unsignedp, OPTAB_DIRECT);
3093 if (temp)
3094 return temp;
3095 }
3096
3097 last = get_last_insn ();
3098
3099 temp1 = expand_binop (mode, ashl_optab, op0, GEN_INT (8), NULL_RTX,
3100 unsignedp, OPTAB_WIDEN);
3101 temp2 = expand_binop (mode, lshr_optab, op0, GEN_INT (8), NULL_RTX,
3102 unsignedp, OPTAB_WIDEN);
3103 if (temp1 && temp2)
3104 {
3105 temp = expand_binop (mode, ior_optab, temp1, temp2, target,
3106 unsignedp, OPTAB_WIDEN);
3107 if (temp)
3108 return temp;
3109 }
3110
3111 delete_insns_since (last);
3112 }
3113
3114 temp = widen_bswap (mode, op0, target);
3115 if (temp)
3116 return temp;
3117
3118 if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
3119 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3120 {
3121 temp = expand_doubleword_bswap (mode, op0, target);
3122 if (temp)
3123 return temp;
3124 }
3125
3126 goto try_libcall;
3127 }
3128
3129 if (CLASS_HAS_WIDER_MODES_P (mclass))
3130 for (wider_mode = GET_MODE_WIDER_MODE (mode);
3131 wider_mode != VOIDmode;
3132 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
3133 {
3134 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
3135 {
3136 rtx xop0 = op0;
3137 rtx last = get_last_insn ();
3138
3139 /* For certain operations, we need not actually extend
3140 the narrow operand, as long as we will truncate the
3141 results to the same narrowness. */
3142
3143 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3144 (unoptab == neg_optab
3145 || unoptab == one_cmpl_optab)
3146 && mclass == MODE_INT);
3147
3148 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3149 unsignedp);
3150
3151 if (temp)
3152 {
3153 if (mclass != MODE_INT
3154 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3155 {
3156 if (target == 0)
3157 target = gen_reg_rtx (mode);
3158 convert_move (target, temp, 0);
3159 return target;
3160 }
3161 else
3162 return gen_lowpart (mode, temp);
3163 }
3164 else
3165 delete_insns_since (last);
3166 }
3167 }
3168
3169 /* These can be done a word at a time. */
3170 if (unoptab == one_cmpl_optab
3171 && mclass == MODE_INT
3172 && GET_MODE_SIZE (mode) > UNITS_PER_WORD
3173 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3174 {
3175 int i;
3176 rtx insns;
3177
3178 if (target == 0 || target == op0 || !valid_multiword_target_p (target))
3179 target = gen_reg_rtx (mode);
3180
3181 start_sequence ();
3182
3183 /* Do the actual arithmetic. */
3184 for (i = 0; i < GET_MODE_BITSIZE (mode) / BITS_PER_WORD; i++)
3185 {
3186 rtx target_piece = operand_subword (target, i, 1, mode);
3187 rtx x = expand_unop (word_mode, unoptab,
3188 operand_subword_force (op0, i, mode),
3189 target_piece, unsignedp);
3190
3191 if (target_piece != x)
3192 emit_move_insn (target_piece, x);
3193 }
3194
3195 insns = get_insns ();
3196 end_sequence ();
3197
3198 emit_insn (insns);
3199 return target;
3200 }
3201
3202 if (optab_to_code (unoptab) == NEG)
3203 {
3204 /* Try negating floating point values by flipping the sign bit. */
3205 if (SCALAR_FLOAT_MODE_P (mode))
3206 {
3207 temp = expand_absneg_bit (NEG, mode, op0, target);
3208 if (temp)
3209 return temp;
3210 }
3211
3212 /* If there is no negation pattern, and we have no negative zero,
3213 try subtracting from zero. */
3214 if (!HONOR_SIGNED_ZEROS (mode))
3215 {
3216 temp = expand_binop (mode, (unoptab == negv_optab
3217 ? subv_optab : sub_optab),
3218 CONST0_RTX (mode), op0, target,
3219 unsignedp, OPTAB_DIRECT);
3220 if (temp)
3221 return temp;
3222 }
3223 }
3224
3225 /* Try calculating parity (x) as popcount (x) % 2. */
3226 if (unoptab == parity_optab)
3227 {
3228 temp = expand_parity (mode, op0, target);
3229 if (temp)
3230 return temp;
3231 }
3232
3233 /* Try implementing ffs (x) in terms of clz (x). */
3234 if (unoptab == ffs_optab)
3235 {
3236 temp = expand_ffs (mode, op0, target);
3237 if (temp)
3238 return temp;
3239 }
3240
3241 /* Try implementing ctz (x) in terms of clz (x). */
3242 if (unoptab == ctz_optab)
3243 {
3244 temp = expand_ctz (mode, op0, target);
3245 if (temp)
3246 return temp;
3247 }
3248
3249 try_libcall:
3250 /* Now try a library call in this mode. */
3251 libfunc = optab_libfunc (unoptab, mode);
3252 if (libfunc)
3253 {
3254 rtx insns;
3255 rtx value;
3256 rtx eq_value;
3257 enum machine_mode outmode = mode;
3258
3259 /* All of these functions return small values. Thus we choose to
3260 have them return something that isn't a double-word. */
3261 if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3262 || unoptab == clrsb_optab || unoptab == popcount_optab
3263 || unoptab == parity_optab)
3264 outmode
3265 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3266 optab_libfunc (unoptab, mode)));
3267
3268 start_sequence ();
3269
3270 /* Pass 1 for NO_QUEUE so we don't lose any increments
3271 if the libcall is cse'd or moved. */
3272 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3273 1, op0, mode);
3274 insns = get_insns ();
3275 end_sequence ();
3276
3277 target = gen_reg_rtx (outmode);
3278 eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3279 if (GET_MODE_SIZE (outmode) < GET_MODE_SIZE (mode))
3280 eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3281 else if (GET_MODE_SIZE (outmode) > GET_MODE_SIZE (mode))
3282 eq_value = simplify_gen_unary (ZERO_EXTEND, outmode, eq_value, mode);
3283 emit_libcall_block_1 (insns, target, value, eq_value,
3284 trapv_unoptab_p (unoptab));
3285
3286 return target;
3287 }
3288
3289 /* It can't be done in this mode. Can we do it in a wider mode? */
3290
3291 if (CLASS_HAS_WIDER_MODES_P (mclass))
3292 {
3293 for (wider_mode = GET_MODE_WIDER_MODE (mode);
3294 wider_mode != VOIDmode;
3295 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
3296 {
3297 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3298 || optab_libfunc (unoptab, wider_mode))
3299 {
3300 rtx xop0 = op0;
3301 rtx last = get_last_insn ();
3302
3303 /* For certain operations, we need not actually extend
3304 the narrow operand, as long as we will truncate the
3305 results to the same narrowness. */
3306 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3307 (unoptab == neg_optab
3308 || unoptab == one_cmpl_optab
3309 || unoptab == bswap_optab)
3310 && mclass == MODE_INT);
3311
3312 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3313 unsignedp);
3314
3315 /* If we are generating clz using wider mode, adjust the
3316 result. Similarly for clrsb. */
3317 if ((unoptab == clz_optab || unoptab == clrsb_optab)
3318 && temp != 0)
3319 temp = expand_binop
3320 (wider_mode, sub_optab, temp,
3321 gen_int_mode (GET_MODE_PRECISION (wider_mode)
3322 - GET_MODE_PRECISION (mode),
3323 wider_mode),
3324 target, true, OPTAB_DIRECT);
3325
3326 /* Likewise for bswap. */
3327 if (unoptab == bswap_optab && temp != 0)
3328 {
3329 gcc_assert (GET_MODE_PRECISION (wider_mode)
3330 == GET_MODE_BITSIZE (wider_mode)
3331 && GET_MODE_PRECISION (mode)
3332 == GET_MODE_BITSIZE (mode));
3333
3334 temp = expand_shift (RSHIFT_EXPR, wider_mode, temp,
3335 GET_MODE_BITSIZE (wider_mode)
3336 - GET_MODE_BITSIZE (mode),
3337 NULL_RTX, true);
3338 }
3339
3340 if (temp)
3341 {
3342 if (mclass != MODE_INT)
3343 {
3344 if (target == 0)
3345 target = gen_reg_rtx (mode);
3346 convert_move (target, temp, 0);
3347 return target;
3348 }
3349 else
3350 return gen_lowpart (mode, temp);
3351 }
3352 else
3353 delete_insns_since (last);
3354 }
3355 }
3356 }
3357
3358 /* One final attempt at implementing negation via subtraction,
3359 this time allowing widening of the operand. */
3360 if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3361 {
3362 rtx temp;
3363 temp = expand_binop (mode,
3364 unoptab == negv_optab ? subv_optab : sub_optab,
3365 CONST0_RTX (mode), op0,
3366 target, unsignedp, OPTAB_LIB_WIDEN);
3367 if (temp)
3368 return temp;
3369 }
3370
3371 return 0;
3372 }
3373 \f
3374 /* Emit code to compute the absolute value of OP0, with result to
3375 TARGET if convenient. (TARGET may be 0.) The return value says
3376 where the result actually is to be found.
3377
3378 MODE is the mode of the operand; the mode of the result is
3379 different but can be deduced from MODE.
3380
3381 */
3382
3383 rtx
3384 expand_abs_nojump (enum machine_mode mode, rtx op0, rtx target,
3385 int result_unsignedp)
3386 {
3387 rtx temp;
3388
3389 if (! flag_trapv)
3390 result_unsignedp = 1;
3391
3392 /* First try to do it with a special abs instruction. */
3393 temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3394 op0, target, 0);
3395 if (temp != 0)
3396 return temp;
3397
3398 /* For floating point modes, try clearing the sign bit. */
3399 if (SCALAR_FLOAT_MODE_P (mode))
3400 {
3401 temp = expand_absneg_bit (ABS, mode, op0, target);
3402 if (temp)
3403 return temp;
3404 }
3405
3406 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3407 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3408 && !HONOR_SIGNED_ZEROS (mode))
3409 {
3410 rtx last = get_last_insn ();
3411
3412 temp = expand_unop (mode, neg_optab, op0, NULL_RTX, 0);
3413 if (temp != 0)
3414 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3415 OPTAB_WIDEN);
3416
3417 if (temp != 0)
3418 return temp;
3419
3420 delete_insns_since (last);
3421 }
3422
3423 /* If this machine has expensive jumps, we can do integer absolute
3424 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3425 where W is the width of MODE. */
3426
3427 if (GET_MODE_CLASS (mode) == MODE_INT
3428 && BRANCH_COST (optimize_insn_for_speed_p (),
3429 false) >= 2)
3430 {
3431 rtx extended = expand_shift (RSHIFT_EXPR, mode, op0,
3432 GET_MODE_PRECISION (mode) - 1,
3433 NULL_RTX, 0);
3434
3435 temp = expand_binop (mode, xor_optab, extended, op0, target, 0,
3436 OPTAB_LIB_WIDEN);
3437 if (temp != 0)
3438 temp = expand_binop (mode, result_unsignedp ? sub_optab : subv_optab,
3439 temp, extended, target, 0, OPTAB_LIB_WIDEN);
3440
3441 if (temp != 0)
3442 return temp;
3443 }
3444
3445 return NULL_RTX;
3446 }
3447
3448 rtx
3449 expand_abs (enum machine_mode mode, rtx op0, rtx target,
3450 int result_unsignedp, int safe)
3451 {
3452 rtx temp, op1;
3453
3454 if (! flag_trapv)
3455 result_unsignedp = 1;
3456
3457 temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3458 if (temp != 0)
3459 return temp;
3460
3461 /* If that does not win, use conditional jump and negate. */
3462
3463 /* It is safe to use the target if it is the same
3464 as the source if this is also a pseudo register */
3465 if (op0 == target && REG_P (op0)
3466 && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3467 safe = 1;
3468
3469 op1 = gen_label_rtx ();
3470 if (target == 0 || ! safe
3471 || GET_MODE (target) != mode
3472 || (MEM_P (target) && MEM_VOLATILE_P (target))
3473 || (REG_P (target)
3474 && REGNO (target) < FIRST_PSEUDO_REGISTER))
3475 target = gen_reg_rtx (mode);
3476
3477 emit_move_insn (target, op0);
3478 NO_DEFER_POP;
3479
3480 do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3481 NULL_RTX, NULL_RTX, op1, -1);
3482
3483 op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3484 target, target, 0);
3485 if (op0 != target)
3486 emit_move_insn (target, op0);
3487 emit_label (op1);
3488 OK_DEFER_POP;
3489 return target;
3490 }
3491
3492 /* Emit code to compute the one's complement absolute value of OP0
3493 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3494 (TARGET may be NULL_RTX.) The return value says where the result
3495 actually is to be found.
3496
3497 MODE is the mode of the operand; the mode of the result is
3498 different but can be deduced from MODE. */
3499
3500 rtx
3501 expand_one_cmpl_abs_nojump (enum machine_mode mode, rtx op0, rtx target)
3502 {
3503 rtx temp;
3504
3505 /* Not applicable for floating point modes. */
3506 if (FLOAT_MODE_P (mode))
3507 return NULL_RTX;
3508
3509 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3510 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3511 {
3512 rtx last = get_last_insn ();
3513
3514 temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3515 if (temp != 0)
3516 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3517 OPTAB_WIDEN);
3518
3519 if (temp != 0)
3520 return temp;
3521
3522 delete_insns_since (last);
3523 }
3524
3525 /* If this machine has expensive jumps, we can do one's complement
3526 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3527
3528 if (GET_MODE_CLASS (mode) == MODE_INT
3529 && BRANCH_COST (optimize_insn_for_speed_p (),
3530 false) >= 2)
3531 {
3532 rtx extended = expand_shift (RSHIFT_EXPR, mode, op0,
3533 GET_MODE_PRECISION (mode) - 1,
3534 NULL_RTX, 0);
3535
3536 temp = expand_binop (mode, xor_optab, extended, op0, target, 0,
3537 OPTAB_LIB_WIDEN);
3538
3539 if (temp != 0)
3540 return temp;
3541 }
3542
3543 return NULL_RTX;
3544 }
3545
3546 /* A subroutine of expand_copysign, perform the copysign operation using the
3547 abs and neg primitives advertised to exist on the target. The assumption
3548 is that we have a split register file, and leaving op0 in fp registers,
3549 and not playing with subregs so much, will help the register allocator. */
3550
3551 static rtx
3552 expand_copysign_absneg (enum machine_mode mode, rtx op0, rtx op1, rtx target,
3553 int bitpos, bool op0_is_abs)
3554 {
3555 enum machine_mode imode;
3556 enum insn_code icode;
3557 rtx sign, label;
3558
3559 if (target == op1)
3560 target = NULL_RTX;
3561
3562 /* Check if the back end provides an insn that handles signbit for the
3563 argument's mode. */
3564 icode = optab_handler (signbit_optab, mode);
3565 if (icode != CODE_FOR_nothing)
3566 {
3567 imode = insn_data[(int) icode].operand[0].mode;
3568 sign = gen_reg_rtx (imode);
3569 emit_unop_insn (icode, sign, op1, UNKNOWN);
3570 }
3571 else
3572 {
3573 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3574 {
3575 imode = int_mode_for_mode (mode);
3576 if (imode == BLKmode)
3577 return NULL_RTX;
3578 op1 = gen_lowpart (imode, op1);
3579 }
3580 else
3581 {
3582 int word;
3583
3584 imode = word_mode;
3585 if (FLOAT_WORDS_BIG_ENDIAN)
3586 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3587 else
3588 word = bitpos / BITS_PER_WORD;
3589 bitpos = bitpos % BITS_PER_WORD;
3590 op1 = operand_subword_force (op1, word, mode);
3591 }
3592
3593 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3594 sign = expand_binop (imode, and_optab, op1,
3595 immed_wide_int_const (mask, imode),
3596 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3597 }
3598
3599 if (!op0_is_abs)
3600 {
3601 op0 = expand_unop (mode, abs_optab, op0, target, 0);
3602 if (op0 == NULL)
3603 return NULL_RTX;
3604 target = op0;
3605 }
3606 else
3607 {
3608 if (target == NULL_RTX)
3609 target = copy_to_reg (op0);
3610 else
3611 emit_move_insn (target, op0);
3612 }
3613
3614 label = gen_label_rtx ();
3615 emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3616
3617 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3618 op0 = simplify_unary_operation (NEG, mode, op0, mode);
3619 else
3620 op0 = expand_unop (mode, neg_optab, op0, target, 0);
3621 if (op0 != target)
3622 emit_move_insn (target, op0);
3623
3624 emit_label (label);
3625
3626 return target;
3627 }
3628
3629
3630 /* A subroutine of expand_copysign, perform the entire copysign operation
3631 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3632 is true if op0 is known to have its sign bit clear. */
3633
3634 static rtx
3635 expand_copysign_bit (enum machine_mode mode, rtx op0, rtx op1, rtx target,
3636 int bitpos, bool op0_is_abs)
3637 {
3638 enum machine_mode imode;
3639 int word, nwords, i;
3640 rtx temp, insns;
3641
3642 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3643 {
3644 imode = int_mode_for_mode (mode);
3645 if (imode == BLKmode)
3646 return NULL_RTX;
3647 word = 0;
3648 nwords = 1;
3649 }
3650 else
3651 {
3652 imode = word_mode;
3653
3654 if (FLOAT_WORDS_BIG_ENDIAN)
3655 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3656 else
3657 word = bitpos / BITS_PER_WORD;
3658 bitpos = bitpos % BITS_PER_WORD;
3659 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3660 }
3661
3662 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3663
3664 if (target == 0
3665 || target == op0
3666 || target == op1
3667 || (nwords > 1 && !valid_multiword_target_p (target)))
3668 target = gen_reg_rtx (mode);
3669
3670 if (nwords > 1)
3671 {
3672 start_sequence ();
3673
3674 for (i = 0; i < nwords; ++i)
3675 {
3676 rtx targ_piece = operand_subword (target, i, 1, mode);
3677 rtx op0_piece = operand_subword_force (op0, i, mode);
3678
3679 if (i == word)
3680 {
3681 if (!op0_is_abs)
3682 op0_piece
3683 = expand_binop (imode, and_optab, op0_piece,
3684 immed_wide_int_const (~mask, imode),
3685 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3686 op1 = expand_binop (imode, and_optab,
3687 operand_subword_force (op1, i, mode),
3688 immed_wide_int_const (mask, imode),
3689 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3690
3691 temp = expand_binop (imode, ior_optab, op0_piece, op1,
3692 targ_piece, 1, OPTAB_LIB_WIDEN);
3693 if (temp != targ_piece)
3694 emit_move_insn (targ_piece, temp);
3695 }
3696 else
3697 emit_move_insn (targ_piece, op0_piece);
3698 }
3699
3700 insns = get_insns ();
3701 end_sequence ();
3702
3703 emit_insn (insns);
3704 }
3705 else
3706 {
3707 op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
3708 immed_wide_int_const (mask, imode),
3709 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3710
3711 op0 = gen_lowpart (imode, op0);
3712 if (!op0_is_abs)
3713 op0 = expand_binop (imode, and_optab, op0,
3714 immed_wide_int_const (~mask, imode),
3715 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3716
3717 temp = expand_binop (imode, ior_optab, op0, op1,
3718 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
3719 target = lowpart_subreg_maybe_copy (mode, temp, imode);
3720 }
3721
3722 return target;
3723 }
3724
3725 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3726 scalar floating point mode. Return NULL if we do not know how to
3727 expand the operation inline. */
3728
3729 rtx
3730 expand_copysign (rtx op0, rtx op1, rtx target)
3731 {
3732 enum machine_mode mode = GET_MODE (op0);
3733 const struct real_format *fmt;
3734 bool op0_is_abs;
3735 rtx temp;
3736
3737 gcc_assert (SCALAR_FLOAT_MODE_P (mode));
3738 gcc_assert (GET_MODE (op1) == mode);
3739
3740 /* First try to do it with a special instruction. */
3741 temp = expand_binop (mode, copysign_optab, op0, op1,
3742 target, 0, OPTAB_DIRECT);
3743 if (temp)
3744 return temp;
3745
3746 fmt = REAL_MODE_FORMAT (mode);
3747 if (fmt == NULL || !fmt->has_signed_zero)
3748 return NULL_RTX;
3749
3750 op0_is_abs = false;
3751 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3752 {
3753 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
3754 op0 = simplify_unary_operation (ABS, mode, op0, mode);
3755 op0_is_abs = true;
3756 }
3757
3758 if (fmt->signbit_ro >= 0
3759 && (CONST_DOUBLE_AS_FLOAT_P (op0)
3760 || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
3761 && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
3762 {
3763 temp = expand_copysign_absneg (mode, op0, op1, target,
3764 fmt->signbit_ro, op0_is_abs);
3765 if (temp)
3766 return temp;
3767 }
3768
3769 if (fmt->signbit_rw < 0)
3770 return NULL_RTX;
3771 return expand_copysign_bit (mode, op0, op1, target,
3772 fmt->signbit_rw, op0_is_abs);
3773 }
3774 \f
3775 /* Generate an instruction whose insn-code is INSN_CODE,
3776 with two operands: an output TARGET and an input OP0.
3777 TARGET *must* be nonzero, and the output is always stored there.
3778 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3779 the value that is stored into TARGET.
3780
3781 Return false if expansion failed. */
3782
3783 bool
3784 maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
3785 enum rtx_code code)
3786 {
3787 struct expand_operand ops[2];
3788 rtx pat;
3789
3790 create_output_operand (&ops[0], target, GET_MODE (target));
3791 create_input_operand (&ops[1], op0, GET_MODE (op0));
3792 pat = maybe_gen_insn (icode, 2, ops);
3793 if (!pat)
3794 return false;
3795
3796 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX && code != UNKNOWN)
3797 add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX);
3798
3799 emit_insn (pat);
3800
3801 if (ops[0].value != target)
3802 emit_move_insn (target, ops[0].value);
3803 return true;
3804 }
3805 /* Generate an instruction whose insn-code is INSN_CODE,
3806 with two operands: an output TARGET and an input OP0.
3807 TARGET *must* be nonzero, and the output is always stored there.
3808 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3809 the value that is stored into TARGET. */
3810
3811 void
3812 emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
3813 {
3814 bool ok = maybe_emit_unop_insn (icode, target, op0, code);
3815 gcc_assert (ok);
3816 }
3817 \f
3818 struct no_conflict_data
3819 {
3820 rtx target, first, insn;
3821 bool must_stay;
3822 };
3823
3824 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3825 the currently examined clobber / store has to stay in the list of
3826 insns that constitute the actual libcall block. */
3827 static void
3828 no_conflict_move_test (rtx dest, const_rtx set, void *p0)
3829 {
3830 struct no_conflict_data *p= (struct no_conflict_data *) p0;
3831
3832 /* If this inns directly contributes to setting the target, it must stay. */
3833 if (reg_overlap_mentioned_p (p->target, dest))
3834 p->must_stay = true;
3835 /* If we haven't committed to keeping any other insns in the list yet,
3836 there is nothing more to check. */
3837 else if (p->insn == p->first)
3838 return;
3839 /* If this insn sets / clobbers a register that feeds one of the insns
3840 already in the list, this insn has to stay too. */
3841 else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
3842 || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
3843 || reg_used_between_p (dest, p->first, p->insn)
3844 /* Likewise if this insn depends on a register set by a previous
3845 insn in the list, or if it sets a result (presumably a hard
3846 register) that is set or clobbered by a previous insn.
3847 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3848 SET_DEST perform the former check on the address, and the latter
3849 check on the MEM. */
3850 || (GET_CODE (set) == SET
3851 && (modified_in_p (SET_SRC (set), p->first)
3852 || modified_in_p (SET_DEST (set), p->first)
3853 || modified_between_p (SET_SRC (set), p->first, p->insn)
3854 || modified_between_p (SET_DEST (set), p->first, p->insn))))
3855 p->must_stay = true;
3856 }
3857
3858 \f
3859 /* Emit code to make a call to a constant function or a library call.
3860
3861 INSNS is a list containing all insns emitted in the call.
3862 These insns leave the result in RESULT. Our block is to copy RESULT
3863 to TARGET, which is logically equivalent to EQUIV.
3864
3865 We first emit any insns that set a pseudo on the assumption that these are
3866 loading constants into registers; doing so allows them to be safely cse'ed
3867 between blocks. Then we emit all the other insns in the block, followed by
3868 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3869 note with an operand of EQUIV. */
3870
3871 static void
3872 emit_libcall_block_1 (rtx insns, rtx target, rtx result, rtx equiv,
3873 bool equiv_may_trap)
3874 {
3875 rtx final_dest = target;
3876 rtx next, last, insn;
3877
3878 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3879 into a MEM later. Protect the libcall block from this change. */
3880 if (! REG_P (target) || REG_USERVAR_P (target))
3881 target = gen_reg_rtx (GET_MODE (target));
3882
3883 /* If we're using non-call exceptions, a libcall corresponding to an
3884 operation that may trap may also trap. */
3885 /* ??? See the comment in front of make_reg_eh_region_note. */
3886 if (cfun->can_throw_non_call_exceptions
3887 && (equiv_may_trap || may_trap_p (equiv)))
3888 {
3889 for (insn = insns; insn; insn = NEXT_INSN (insn))
3890 if (CALL_P (insn))
3891 {
3892 rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3893 if (note)
3894 {
3895 int lp_nr = INTVAL (XEXP (note, 0));
3896 if (lp_nr == 0 || lp_nr == INT_MIN)
3897 remove_note (insn, note);
3898 }
3899 }
3900 }
3901 else
3902 {
3903 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3904 reg note to indicate that this call cannot throw or execute a nonlocal
3905 goto (unless there is already a REG_EH_REGION note, in which case
3906 we update it). */
3907 for (insn = insns; insn; insn = NEXT_INSN (insn))
3908 if (CALL_P (insn))
3909 make_reg_eh_region_note_nothrow_nononlocal (insn);
3910 }
3911
3912 /* First emit all insns that set pseudos. Remove them from the list as
3913 we go. Avoid insns that set pseudos which were referenced in previous
3914 insns. These can be generated by move_by_pieces, for example,
3915 to update an address. Similarly, avoid insns that reference things
3916 set in previous insns. */
3917
3918 for (insn = insns; insn; insn = next)
3919 {
3920 rtx set = single_set (insn);
3921
3922 next = NEXT_INSN (insn);
3923
3924 if (set != 0 && REG_P (SET_DEST (set))
3925 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
3926 {
3927 struct no_conflict_data data;
3928
3929 data.target = const0_rtx;
3930 data.first = insns;
3931 data.insn = insn;
3932 data.must_stay = 0;
3933 note_stores (PATTERN (insn), no_conflict_move_test, &data);
3934 if (! data.must_stay)
3935 {
3936 if (PREV_INSN (insn))
3937 NEXT_INSN (PREV_INSN (insn)) = next;
3938 else
3939 insns = next;
3940
3941 if (next)
3942 PREV_INSN (next) = PREV_INSN (insn);
3943
3944 add_insn (insn);
3945 }
3946 }
3947
3948 /* Some ports use a loop to copy large arguments onto the stack.
3949 Don't move anything outside such a loop. */
3950 if (LABEL_P (insn))
3951 break;
3952 }
3953
3954 /* Write the remaining insns followed by the final copy. */
3955 for (insn = insns; insn; insn = next)
3956 {
3957 next = NEXT_INSN (insn);
3958
3959 add_insn (insn);
3960 }
3961
3962 last = emit_move_insn (target, result);
3963 set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
3964
3965 if (final_dest != target)
3966 emit_move_insn (final_dest, target);
3967 }
3968
3969 void
3970 emit_libcall_block (rtx insns, rtx target, rtx result, rtx equiv)
3971 {
3972 emit_libcall_block_1 (insns, target, result, equiv, false);
3973 }
3974 \f
3975 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3976 PURPOSE describes how this comparison will be used. CODE is the rtx
3977 comparison code we will be using.
3978
3979 ??? Actually, CODE is slightly weaker than that. A target is still
3980 required to implement all of the normal bcc operations, but not
3981 required to implement all (or any) of the unordered bcc operations. */
3982
3983 int
3984 can_compare_p (enum rtx_code code, enum machine_mode mode,
3985 enum can_compare_purpose purpose)
3986 {
3987 rtx test;
3988 test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
3989 do
3990 {
3991 enum insn_code icode;
3992
3993 if (purpose == ccp_jump
3994 && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
3995 && insn_operand_matches (icode, 0, test))
3996 return 1;
3997 if (purpose == ccp_store_flag
3998 && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
3999 && insn_operand_matches (icode, 1, test))
4000 return 1;
4001 if (purpose == ccp_cmov
4002 && optab_handler (cmov_optab, mode) != CODE_FOR_nothing)
4003 return 1;
4004
4005 mode = GET_MODE_WIDER_MODE (mode);
4006 PUT_MODE (test, mode);
4007 }
4008 while (mode != VOIDmode);
4009
4010 return 0;
4011 }
4012
4013 /* This function is called when we are going to emit a compare instruction that
4014 compares the values found in *PX and *PY, using the rtl operator COMPARISON.
4015
4016 *PMODE is the mode of the inputs (in case they are const_int).
4017 *PUNSIGNEDP nonzero says that the operands are unsigned;
4018 this matters if they need to be widened (as given by METHODS).
4019
4020 If they have mode BLKmode, then SIZE specifies the size of both operands.
4021
4022 This function performs all the setup necessary so that the caller only has
4023 to emit a single comparison insn. This setup can involve doing a BLKmode
4024 comparison or emitting a library call to perform the comparison if no insn
4025 is available to handle it.
4026 The values which are passed in through pointers can be modified; the caller
4027 should perform the comparison on the modified values. Constant
4028 comparisons must have already been folded. */
4029
4030 static void
4031 prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size,
4032 int unsignedp, enum optab_methods methods,
4033 rtx *ptest, enum machine_mode *pmode)
4034 {
4035 enum machine_mode mode = *pmode;
4036 rtx libfunc, test;
4037 enum machine_mode cmp_mode;
4038 enum mode_class mclass;
4039
4040 /* The other methods are not needed. */
4041 gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
4042 || methods == OPTAB_LIB_WIDEN);
4043
4044 /* If we are optimizing, force expensive constants into a register. */
4045 if (CONSTANT_P (x) && optimize
4046 && (rtx_cost (x, COMPARE, 0, optimize_insn_for_speed_p ())
4047 > COSTS_N_INSNS (1)))
4048 x = force_reg (mode, x);
4049
4050 if (CONSTANT_P (y) && optimize
4051 && (rtx_cost (y, COMPARE, 1, optimize_insn_for_speed_p ())
4052 > COSTS_N_INSNS (1)))
4053 y = force_reg (mode, y);
4054
4055 #ifdef HAVE_cc0
4056 /* Make sure if we have a canonical comparison. The RTL
4057 documentation states that canonical comparisons are required only
4058 for targets which have cc0. */
4059 gcc_assert (!CONSTANT_P (x) || CONSTANT_P (y));
4060 #endif
4061
4062 /* Don't let both operands fail to indicate the mode. */
4063 if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
4064 x = force_reg (mode, x);
4065 if (mode == VOIDmode)
4066 mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
4067
4068 /* Handle all BLKmode compares. */
4069
4070 if (mode == BLKmode)
4071 {
4072 enum machine_mode result_mode;
4073 enum insn_code cmp_code;
4074 tree length_type;
4075 rtx libfunc;
4076 rtx result;
4077 rtx opalign
4078 = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
4079
4080 gcc_assert (size);
4081
4082 /* Try to use a memory block compare insn - either cmpstr
4083 or cmpmem will do. */
4084 for (cmp_mode = GET_CLASS_NARROWEST_MODE (MODE_INT);
4085 cmp_mode != VOIDmode;
4086 cmp_mode = GET_MODE_WIDER_MODE (cmp_mode))
4087 {
4088 cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
4089 if (cmp_code == CODE_FOR_nothing)
4090 cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
4091 if (cmp_code == CODE_FOR_nothing)
4092 cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
4093 if (cmp_code == CODE_FOR_nothing)
4094 continue;
4095
4096 /* Must make sure the size fits the insn's mode. */
4097 if ((CONST_INT_P (size)
4098 && INTVAL (size) >= (1 << GET_MODE_BITSIZE (cmp_mode)))
4099 || (GET_MODE_BITSIZE (GET_MODE (size))
4100 > GET_MODE_BITSIZE (cmp_mode)))
4101 continue;
4102
4103 result_mode = insn_data[cmp_code].operand[0].mode;
4104 result = gen_reg_rtx (result_mode);
4105 size = convert_to_mode (cmp_mode, size, 1);
4106 emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
4107
4108 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
4109 *pmode = result_mode;
4110 return;
4111 }
4112
4113 if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
4114 goto fail;
4115
4116 /* Otherwise call a library function, memcmp. */
4117 libfunc = memcmp_libfunc;
4118 length_type = sizetype;
4119 result_mode = TYPE_MODE (integer_type_node);
4120 cmp_mode = TYPE_MODE (length_type);
4121 size = convert_to_mode (TYPE_MODE (length_type), size,
4122 TYPE_UNSIGNED (length_type));
4123
4124 result = emit_library_call_value (libfunc, 0, LCT_PURE,
4125 result_mode, 3,
4126 XEXP (x, 0), Pmode,
4127 XEXP (y, 0), Pmode,
4128 size, cmp_mode);
4129 x = result;
4130 y = const0_rtx;
4131 mode = result_mode;
4132 methods = OPTAB_LIB_WIDEN;
4133 unsignedp = false;
4134 }
4135
4136 /* Don't allow operands to the compare to trap, as that can put the
4137 compare and branch in different basic blocks. */
4138 if (cfun->can_throw_non_call_exceptions)
4139 {
4140 if (may_trap_p (x))
4141 x = force_reg (mode, x);
4142 if (may_trap_p (y))
4143 y = force_reg (mode, y);
4144 }
4145
4146 if (GET_MODE_CLASS (mode) == MODE_CC)
4147 {
4148 gcc_assert (can_compare_p (comparison, CCmode, ccp_jump));
4149 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4150 return;
4151 }
4152
4153 mclass = GET_MODE_CLASS (mode);
4154 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4155 cmp_mode = mode;
4156 do
4157 {
4158 enum insn_code icode;
4159 icode = optab_handler (cbranch_optab, cmp_mode);
4160 if (icode != CODE_FOR_nothing
4161 && insn_operand_matches (icode, 0, test))
4162 {
4163 rtx last = get_last_insn ();
4164 rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
4165 rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
4166 if (op0 && op1
4167 && insn_operand_matches (icode, 1, op0)
4168 && insn_operand_matches (icode, 2, op1))
4169 {
4170 XEXP (test, 0) = op0;
4171 XEXP (test, 1) = op1;
4172 *ptest = test;
4173 *pmode = cmp_mode;
4174 return;
4175 }
4176 delete_insns_since (last);
4177 }
4178
4179 if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass))
4180 break;
4181 cmp_mode = GET_MODE_WIDER_MODE (cmp_mode);
4182 }
4183 while (cmp_mode != VOIDmode);
4184
4185 if (methods != OPTAB_LIB_WIDEN)
4186 goto fail;
4187
4188 if (!SCALAR_FLOAT_MODE_P (mode))
4189 {
4190 rtx result;
4191 enum machine_mode ret_mode;
4192
4193 /* Handle a libcall just for the mode we are using. */
4194 libfunc = optab_libfunc (cmp_optab, mode);
4195 gcc_assert (libfunc);
4196
4197 /* If we want unsigned, and this mode has a distinct unsigned
4198 comparison routine, use that. */
4199 if (unsignedp)
4200 {
4201 rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4202 if (ulibfunc)
4203 libfunc = ulibfunc;
4204 }
4205
4206 ret_mode = targetm.libgcc_cmp_return_mode ();
4207 result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4208 ret_mode, 2, x, mode, y, mode);
4209
4210 /* There are two kinds of comparison routines. Biased routines
4211 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4212 of gcc expect that the comparison operation is equivalent
4213 to the modified comparison. For signed comparisons compare the
4214 result against 1 in the biased case, and zero in the unbiased
4215 case. For unsigned comparisons always compare against 1 after
4216 biasing the unbiased result by adding 1. This gives us a way to
4217 represent LTU.
4218 The comparisons in the fixed-point helper library are always
4219 biased. */
4220 x = result;
4221 y = const1_rtx;
4222
4223 if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4224 {
4225 if (unsignedp)
4226 x = plus_constant (ret_mode, result, 1);
4227 else
4228 y = const0_rtx;
4229 }
4230
4231 *pmode = word_mode;
4232 prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods,
4233 ptest, pmode);
4234 }
4235 else
4236 prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4237
4238 return;
4239
4240 fail:
4241 *ptest = NULL_RTX;
4242 }
4243
4244 /* Before emitting an insn with code ICODE, make sure that X, which is going
4245 to be used for operand OPNUM of the insn, is converted from mode MODE to
4246 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4247 that it is accepted by the operand predicate. Return the new value. */
4248
4249 rtx
4250 prepare_operand (enum insn_code icode, rtx x, int opnum, enum machine_mode mode,
4251 enum machine_mode wider_mode, int unsignedp)
4252 {
4253 if (mode != wider_mode)
4254 x = convert_modes (wider_mode, mode, x, unsignedp);
4255
4256 if (!insn_operand_matches (icode, opnum, x))
4257 {
4258 if (reload_completed)
4259 return NULL_RTX;
4260 x = copy_to_mode_reg (insn_data[(int) icode].operand[opnum].mode, x);
4261 }
4262
4263 return x;
4264 }
4265
4266 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4267 we can do the branch. */
4268
4269 static void
4270 emit_cmp_and_jump_insn_1 (rtx test, enum machine_mode mode, rtx label, int prob)
4271 {
4272 enum machine_mode optab_mode;
4273 enum mode_class mclass;
4274 enum insn_code icode;
4275 rtx insn;
4276
4277 mclass = GET_MODE_CLASS (mode);
4278 optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4279 icode = optab_handler (cbranch_optab, optab_mode);
4280
4281 gcc_assert (icode != CODE_FOR_nothing);
4282 gcc_assert (insn_operand_matches (icode, 0, test));
4283 insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4284 XEXP (test, 1), label));
4285 if (prob != -1
4286 && profile_status != PROFILE_ABSENT
4287 && insn
4288 && JUMP_P (insn)
4289 && any_condjump_p (insn)
4290 && !find_reg_note (insn, REG_BR_PROB, 0))
4291 add_int_reg_note (insn, REG_BR_PROB, prob);
4292 }
4293
4294 /* Generate code to compare X with Y so that the condition codes are
4295 set and to jump to LABEL if the condition is true. If X is a
4296 constant and Y is not a constant, then the comparison is swapped to
4297 ensure that the comparison RTL has the canonical form.
4298
4299 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4300 need to be widened. UNSIGNEDP is also used to select the proper
4301 branch condition code.
4302
4303 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4304
4305 MODE is the mode of the inputs (in case they are const_int).
4306
4307 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4308 It will be potentially converted into an unsigned variant based on
4309 UNSIGNEDP to select a proper jump instruction.
4310
4311 PROB is the probability of jumping to LABEL. */
4312
4313 void
4314 emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4315 enum machine_mode mode, int unsignedp, rtx label,
4316 int prob)
4317 {
4318 rtx op0 = x, op1 = y;
4319 rtx test;
4320
4321 /* Swap operands and condition to ensure canonical RTL. */
4322 if (swap_commutative_operands_p (x, y)
4323 && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4324 {
4325 op0 = y, op1 = x;
4326 comparison = swap_condition (comparison);
4327 }
4328
4329 /* If OP0 is still a constant, then both X and Y must be constants
4330 or the opposite comparison is not supported. Force X into a register
4331 to create canonical RTL. */
4332 if (CONSTANT_P (op0))
4333 op0 = force_reg (mode, op0);
4334
4335 if (unsignedp)
4336 comparison = unsigned_condition (comparison);
4337
4338 prepare_cmp_insn (op0, op1, comparison, size, unsignedp, OPTAB_LIB_WIDEN,
4339 &test, &mode);
4340 emit_cmp_and_jump_insn_1 (test, mode, label, prob);
4341 }
4342
4343 \f
4344 /* Emit a library call comparison between floating point X and Y.
4345 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4346
4347 static void
4348 prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
4349 rtx *ptest, enum machine_mode *pmode)
4350 {
4351 enum rtx_code swapped = swap_condition (comparison);
4352 enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
4353 enum machine_mode orig_mode = GET_MODE (x);
4354 enum machine_mode mode, cmp_mode;
4355 rtx true_rtx, false_rtx;
4356 rtx value, target, insns, equiv;
4357 rtx libfunc = 0;
4358 bool reversed_p = false;
4359 cmp_mode = targetm.libgcc_cmp_return_mode ();
4360
4361 for (mode = orig_mode;
4362 mode != VOIDmode;
4363 mode = GET_MODE_WIDER_MODE (mode))
4364 {
4365 if (code_to_optab (comparison)
4366 && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
4367 break;
4368
4369 if (code_to_optab (swapped)
4370 && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
4371 {
4372 rtx tmp;
4373 tmp = x; x = y; y = tmp;
4374 comparison = swapped;
4375 break;
4376 }
4377
4378 if (code_to_optab (reversed)
4379 && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
4380 {
4381 comparison = reversed;
4382 reversed_p = true;
4383 break;
4384 }
4385 }
4386
4387 gcc_assert (mode != VOIDmode);
4388
4389 if (mode != orig_mode)
4390 {
4391 x = convert_to_mode (mode, x, 0);
4392 y = convert_to_mode (mode, y, 0);
4393 }
4394
4395 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4396 the RTL. The allows the RTL optimizers to delete the libcall if the
4397 condition can be determined at compile-time. */
4398 if (comparison == UNORDERED
4399 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4400 {
4401 true_rtx = const_true_rtx;
4402 false_rtx = const0_rtx;
4403 }
4404 else
4405 {
4406 switch (comparison)
4407 {
4408 case EQ:
4409 true_rtx = const0_rtx;
4410 false_rtx = const_true_rtx;
4411 break;
4412
4413 case NE:
4414 true_rtx = const_true_rtx;
4415 false_rtx = const0_rtx;
4416 break;
4417
4418 case GT:
4419 true_rtx = const1_rtx;
4420 false_rtx = const0_rtx;
4421 break;
4422
4423 case GE:
4424 true_rtx = const0_rtx;
4425 false_rtx = constm1_rtx;
4426 break;
4427
4428 case LT:
4429 true_rtx = constm1_rtx;
4430 false_rtx = const0_rtx;
4431 break;
4432
4433 case LE:
4434 true_rtx = const0_rtx;
4435 false_rtx = const1_rtx;
4436 break;
4437
4438 default:
4439 gcc_unreachable ();
4440 }
4441 }
4442
4443 if (comparison == UNORDERED)
4444 {
4445 rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
4446 equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
4447 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4448 temp, const_true_rtx, equiv);
4449 }
4450 else
4451 {
4452 equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
4453 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4454 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4455 equiv, true_rtx, false_rtx);
4456 }
4457
4458 start_sequence ();
4459 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4460 cmp_mode, 2, x, mode, y, mode);
4461 insns = get_insns ();
4462 end_sequence ();
4463
4464 target = gen_reg_rtx (cmp_mode);
4465 emit_libcall_block (insns, target, value, equiv);
4466
4467 if (comparison == UNORDERED
4468 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
4469 || reversed_p)
4470 *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
4471 else
4472 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
4473
4474 *pmode = cmp_mode;
4475 }
4476 \f
4477 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4478
4479 void
4480 emit_indirect_jump (rtx loc)
4481 {
4482 struct expand_operand ops[1];
4483
4484 create_address_operand (&ops[0], loc);
4485 expand_jump_insn (CODE_FOR_indirect_jump, 1, ops);
4486 emit_barrier ();
4487 }
4488 \f
4489 #ifdef HAVE_conditional_move
4490
4491 /* Emit a conditional move instruction if the machine supports one for that
4492 condition and machine mode.
4493
4494 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4495 the mode to use should they be constants. If it is VOIDmode, they cannot
4496 both be constants.
4497
4498 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4499 should be stored there. MODE is the mode to use should they be constants.
4500 If it is VOIDmode, they cannot both be constants.
4501
4502 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4503 is not supported. */
4504
4505 rtx
4506 emit_conditional_move (rtx target, enum rtx_code code, rtx op0, rtx op1,
4507 enum machine_mode cmode, rtx op2, rtx op3,
4508 enum machine_mode mode, int unsignedp)
4509 {
4510 rtx tem, comparison, last;
4511 enum insn_code icode;
4512 enum rtx_code reversed;
4513
4514 /* If one operand is constant, make it the second one. Only do this
4515 if the other operand is not constant as well. */
4516
4517 if (swap_commutative_operands_p (op0, op1))
4518 {
4519 tem = op0;
4520 op0 = op1;
4521 op1 = tem;
4522 code = swap_condition (code);
4523 }
4524
4525 /* get_condition will prefer to generate LT and GT even if the old
4526 comparison was against zero, so undo that canonicalization here since
4527 comparisons against zero are cheaper. */
4528 if (code == LT && op1 == const1_rtx)
4529 code = LE, op1 = const0_rtx;
4530 else if (code == GT && op1 == constm1_rtx)
4531 code = GE, op1 = const0_rtx;
4532
4533 if (cmode == VOIDmode)
4534 cmode = GET_MODE (op0);
4535
4536 if (swap_commutative_operands_p (op2, op3)
4537 && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL))
4538 != UNKNOWN))
4539 {
4540 tem = op2;
4541 op2 = op3;
4542 op3 = tem;
4543 code = reversed;
4544 }
4545
4546 if (mode == VOIDmode)
4547 mode = GET_MODE (op2);
4548
4549 icode = direct_optab_handler (movcc_optab, mode);
4550
4551 if (icode == CODE_FOR_nothing)
4552 return 0;
4553
4554 if (!target)
4555 target = gen_reg_rtx (mode);
4556
4557 code = unsignedp ? unsigned_condition (code) : code;
4558 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4559
4560 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4561 return NULL and let the caller figure out how best to deal with this
4562 situation. */
4563 if (!COMPARISON_P (comparison))
4564 return NULL_RTX;
4565
4566 do_pending_stack_adjust ();
4567 last = get_last_insn ();
4568 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4569 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4570 &comparison, &cmode);
4571 if (comparison)
4572 {
4573 struct expand_operand ops[4];
4574
4575 create_output_operand (&ops[0], target, mode);
4576 create_fixed_operand (&ops[1], comparison);
4577 create_input_operand (&ops[2], op2, mode);
4578 create_input_operand (&ops[3], op3, mode);
4579 if (maybe_expand_insn (icode, 4, ops))
4580 {
4581 if (ops[0].value != target)
4582 convert_move (target, ops[0].value, false);
4583 return target;
4584 }
4585 }
4586 delete_insns_since (last);
4587 return NULL_RTX;
4588 }
4589
4590 /* Return nonzero if a conditional move of mode MODE is supported.
4591
4592 This function is for combine so it can tell whether an insn that looks
4593 like a conditional move is actually supported by the hardware. If we
4594 guess wrong we lose a bit on optimization, but that's it. */
4595 /* ??? sparc64 supports conditionally moving integers values based on fp
4596 comparisons, and vice versa. How do we handle them? */
4597
4598 int
4599 can_conditionally_move_p (enum machine_mode mode)
4600 {
4601 if (direct_optab_handler (movcc_optab, mode) != CODE_FOR_nothing)
4602 return 1;
4603
4604 return 0;
4605 }
4606
4607 #endif /* HAVE_conditional_move */
4608
4609 /* Emit a conditional addition instruction if the machine supports one for that
4610 condition and machine mode.
4611
4612 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4613 the mode to use should they be constants. If it is VOIDmode, they cannot
4614 both be constants.
4615
4616 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4617 should be stored there. MODE is the mode to use should they be constants.
4618 If it is VOIDmode, they cannot both be constants.
4619
4620 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4621 is not supported. */
4622
4623 rtx
4624 emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
4625 enum machine_mode cmode, rtx op2, rtx op3,
4626 enum machine_mode mode, int unsignedp)
4627 {
4628 rtx tem, comparison, last;
4629 enum insn_code icode;
4630
4631 /* If one operand is constant, make it the second one. Only do this
4632 if the other operand is not constant as well. */
4633
4634 if (swap_commutative_operands_p (op0, op1))
4635 {
4636 tem = op0;
4637 op0 = op1;
4638 op1 = tem;
4639 code = swap_condition (code);
4640 }
4641
4642 /* get_condition will prefer to generate LT and GT even if the old
4643 comparison was against zero, so undo that canonicalization here since
4644 comparisons against zero are cheaper. */
4645 if (code == LT && op1 == const1_rtx)
4646 code = LE, op1 = const0_rtx;
4647 else if (code == GT && op1 == constm1_rtx)
4648 code = GE, op1 = const0_rtx;
4649
4650 if (cmode == VOIDmode)
4651 cmode = GET_MODE (op0);
4652
4653 if (mode == VOIDmode)
4654 mode = GET_MODE (op2);
4655
4656 icode = optab_handler (addcc_optab, mode);
4657
4658 if (icode == CODE_FOR_nothing)
4659 return 0;
4660
4661 if (!target)
4662 target = gen_reg_rtx (mode);
4663
4664 code = unsignedp ? unsigned_condition (code) : code;
4665 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4666
4667 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4668 return NULL and let the caller figure out how best to deal with this
4669 situation. */
4670 if (!COMPARISON_P (comparison))
4671 return NULL_RTX;
4672
4673 do_pending_stack_adjust ();
4674 last = get_last_insn ();
4675 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4676 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4677 &comparison, &cmode);
4678 if (comparison)
4679 {
4680 struct expand_operand ops[4];
4681
4682 create_output_operand (&ops[0], target, mode);
4683 create_fixed_operand (&ops[1], comparison);
4684 create_input_operand (&ops[2], op2, mode);
4685 create_input_operand (&ops[3], op3, mode);
4686 if (maybe_expand_insn (icode, 4, ops))
4687 {
4688 if (ops[0].value != target)
4689 convert_move (target, ops[0].value, false);
4690 return target;
4691 }
4692 }
4693 delete_insns_since (last);
4694 return NULL_RTX;
4695 }
4696 \f
4697 /* These functions attempt to generate an insn body, rather than
4698 emitting the insn, but if the gen function already emits them, we
4699 make no attempt to turn them back into naked patterns. */
4700
4701 /* Generate and return an insn body to add Y to X. */
4702
4703 rtx
4704 gen_add2_insn (rtx x, rtx y)
4705 {
4706 enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
4707
4708 gcc_assert (insn_operand_matches (icode, 0, x));
4709 gcc_assert (insn_operand_matches (icode, 1, x));
4710 gcc_assert (insn_operand_matches (icode, 2, y));
4711
4712 return GEN_FCN (icode) (x, x, y);
4713 }
4714
4715 /* Generate and return an insn body to add r1 and c,
4716 storing the result in r0. */
4717
4718 rtx
4719 gen_add3_insn (rtx r0, rtx r1, rtx c)
4720 {
4721 enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
4722
4723 if (icode == CODE_FOR_nothing
4724 || !insn_operand_matches (icode, 0, r0)
4725 || !insn_operand_matches (icode, 1, r1)
4726 || !insn_operand_matches (icode, 2, c))
4727 return NULL_RTX;
4728
4729 return GEN_FCN (icode) (r0, r1, c);
4730 }
4731
4732 int
4733 have_add2_insn (rtx x, rtx y)
4734 {
4735 enum insn_code icode;
4736
4737 gcc_assert (GET_MODE (x) != VOIDmode);
4738
4739 icode = optab_handler (add_optab, GET_MODE (x));
4740
4741 if (icode == CODE_FOR_nothing)
4742 return 0;
4743
4744 if (!insn_operand_matches (icode, 0, x)
4745 || !insn_operand_matches (icode, 1, x)
4746 || !insn_operand_matches (icode, 2, y))
4747 return 0;
4748
4749 return 1;
4750 }
4751
4752 /* Generate and return an insn body to subtract Y from X. */
4753
4754 rtx
4755 gen_sub2_insn (rtx x, rtx y)
4756 {
4757 enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
4758
4759 gcc_assert (insn_operand_matches (icode, 0, x));
4760 gcc_assert (insn_operand_matches (icode, 1, x));
4761 gcc_assert (insn_operand_matches (icode, 2, y));
4762
4763 return GEN_FCN (icode) (x, x, y);
4764 }
4765
4766 /* Generate and return an insn body to subtract r1 and c,
4767 storing the result in r0. */
4768
4769 rtx
4770 gen_sub3_insn (rtx r0, rtx r1, rtx c)
4771 {
4772 enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
4773
4774 if (icode == CODE_FOR_nothing
4775 || !insn_operand_matches (icode, 0, r0)
4776 || !insn_operand_matches (icode, 1, r1)
4777 || !insn_operand_matches (icode, 2, c))
4778 return NULL_RTX;
4779
4780 return GEN_FCN (icode) (r0, r1, c);
4781 }
4782
4783 int
4784 have_sub2_insn (rtx x, rtx y)
4785 {
4786 enum insn_code icode;
4787
4788 gcc_assert (GET_MODE (x) != VOIDmode);
4789
4790 icode = optab_handler (sub_optab, GET_MODE (x));
4791
4792 if (icode == CODE_FOR_nothing)
4793 return 0;
4794
4795 if (!insn_operand_matches (icode, 0, x)
4796 || !insn_operand_matches (icode, 1, x)
4797 || !insn_operand_matches (icode, 2, y))
4798 return 0;
4799
4800 return 1;
4801 }
4802
4803 /* Generate the body of an instruction to copy Y into X.
4804 It may be a list of insns, if one insn isn't enough. */
4805
4806 rtx
4807 gen_move_insn (rtx x, rtx y)
4808 {
4809 rtx seq;
4810
4811 start_sequence ();
4812 emit_move_insn_1 (x, y);
4813 seq = get_insns ();
4814 end_sequence ();
4815 return seq;
4816 }
4817 \f
4818 /* Return the insn code used to extend FROM_MODE to TO_MODE.
4819 UNSIGNEDP specifies zero-extension instead of sign-extension. If
4820 no such operation exists, CODE_FOR_nothing will be returned. */
4821
4822 enum insn_code
4823 can_extend_p (enum machine_mode to_mode, enum machine_mode from_mode,
4824 int unsignedp)
4825 {
4826 convert_optab tab;
4827 #ifdef HAVE_ptr_extend
4828 if (unsignedp < 0)
4829 return CODE_FOR_ptr_extend;
4830 #endif
4831
4832 tab = unsignedp ? zext_optab : sext_optab;
4833 return convert_optab_handler (tab, to_mode, from_mode);
4834 }
4835
4836 /* Generate the body of an insn to extend Y (with mode MFROM)
4837 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4838
4839 rtx
4840 gen_extend_insn (rtx x, rtx y, enum machine_mode mto,
4841 enum machine_mode mfrom, int unsignedp)
4842 {
4843 enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
4844 return GEN_FCN (icode) (x, y);
4845 }
4846 \f
4847 /* can_fix_p and can_float_p say whether the target machine
4848 can directly convert a given fixed point type to
4849 a given floating point type, or vice versa.
4850 The returned value is the CODE_FOR_... value to use,
4851 or CODE_FOR_nothing if these modes cannot be directly converted.
4852
4853 *TRUNCP_PTR is set to 1 if it is necessary to output
4854 an explicit FTRUNC insn before the fix insn; otherwise 0. */
4855
4856 static enum insn_code
4857 can_fix_p (enum machine_mode fixmode, enum machine_mode fltmode,
4858 int unsignedp, int *truncp_ptr)
4859 {
4860 convert_optab tab;
4861 enum insn_code icode;
4862
4863 tab = unsignedp ? ufixtrunc_optab : sfixtrunc_optab;
4864 icode = convert_optab_handler (tab, fixmode, fltmode);
4865 if (icode != CODE_FOR_nothing)
4866 {
4867 *truncp_ptr = 0;
4868 return icode;
4869 }
4870
4871 /* FIXME: This requires a port to define both FIX and FTRUNC pattern
4872 for this to work. We need to rework the fix* and ftrunc* patterns
4873 and documentation. */
4874 tab = unsignedp ? ufix_optab : sfix_optab;
4875 icode = convert_optab_handler (tab, fixmode, fltmode);
4876 if (icode != CODE_FOR_nothing
4877 && optab_handler (ftrunc_optab, fltmode) != CODE_FOR_nothing)
4878 {
4879 *truncp_ptr = 1;
4880 return icode;
4881 }
4882
4883 *truncp_ptr = 0;
4884 return CODE_FOR_nothing;
4885 }
4886
4887 enum insn_code
4888 can_float_p (enum machine_mode fltmode, enum machine_mode fixmode,
4889 int unsignedp)
4890 {
4891 convert_optab tab;
4892
4893 tab = unsignedp ? ufloat_optab : sfloat_optab;
4894 return convert_optab_handler (tab, fltmode, fixmode);
4895 }
4896
4897 /* Function supportable_convert_operation
4898
4899 Check whether an operation represented by the code CODE is a
4900 convert operation that is supported by the target platform in
4901 vector form (i.e., when operating on arguments of type VECTYPE_IN
4902 producing a result of type VECTYPE_OUT).
4903
4904 Convert operations we currently support directly are FIX_TRUNC and FLOAT.
4905 This function checks if these operations are supported
4906 by the target platform either directly (via vector tree-codes), or via
4907 target builtins.
4908
4909 Output:
4910 - CODE1 is code of vector operation to be used when
4911 vectorizing the operation, if available.
4912 - DECL is decl of target builtin functions to be used
4913 when vectorizing the operation, if available. In this case,
4914 CODE1 is CALL_EXPR. */
4915
4916 bool
4917 supportable_convert_operation (enum tree_code code,
4918 tree vectype_out, tree vectype_in,
4919 tree *decl, enum tree_code *code1)
4920 {
4921 enum machine_mode m1,m2;
4922 int truncp;
4923
4924 m1 = TYPE_MODE (vectype_out);
4925 m2 = TYPE_MODE (vectype_in);
4926
4927 /* First check if we can done conversion directly. */
4928 if ((code == FIX_TRUNC_EXPR
4929 && can_fix_p (m1,m2,TYPE_UNSIGNED (vectype_out), &truncp)
4930 != CODE_FOR_nothing)
4931 || (code == FLOAT_EXPR
4932 && can_float_p (m1,m2,TYPE_UNSIGNED (vectype_in))
4933 != CODE_FOR_nothing))
4934 {
4935 *code1 = code;
4936 return true;
4937 }
4938
4939 /* Now check for builtin. */
4940 if (targetm.vectorize.builtin_conversion
4941 && targetm.vectorize.builtin_conversion (code, vectype_out, vectype_in))
4942 {
4943 *code1 = CALL_EXPR;
4944 *decl = targetm.vectorize.builtin_conversion (code, vectype_out, vectype_in);
4945 return true;
4946 }
4947 return false;
4948 }
4949
4950 \f
4951 /* Generate code to convert FROM to floating point
4952 and store in TO. FROM must be fixed point and not VOIDmode.
4953 UNSIGNEDP nonzero means regard FROM as unsigned.
4954 Normally this is done by correcting the final value
4955 if it is negative. */
4956
4957 void
4958 expand_float (rtx to, rtx from, int unsignedp)
4959 {
4960 enum insn_code icode;
4961 rtx target = to;
4962 enum machine_mode fmode, imode;
4963 bool can_do_signed = false;
4964
4965 /* Crash now, because we won't be able to decide which mode to use. */
4966 gcc_assert (GET_MODE (from) != VOIDmode);
4967
4968 /* Look for an insn to do the conversion. Do it in the specified
4969 modes if possible; otherwise convert either input, output or both to
4970 wider mode. If the integer mode is wider than the mode of FROM,
4971 we can do the conversion signed even if the input is unsigned. */
4972
4973 for (fmode = GET_MODE (to); fmode != VOIDmode;
4974 fmode = GET_MODE_WIDER_MODE (fmode))
4975 for (imode = GET_MODE (from); imode != VOIDmode;
4976 imode = GET_MODE_WIDER_MODE (imode))
4977 {
4978 int doing_unsigned = unsignedp;
4979
4980 if (fmode != GET_MODE (to)
4981 && significand_size (fmode) < GET_MODE_PRECISION (GET_MODE (from)))
4982 continue;
4983
4984 icode = can_float_p (fmode, imode, unsignedp);
4985 if (icode == CODE_FOR_nothing && unsignedp)
4986 {
4987 enum insn_code scode = can_float_p (fmode, imode, 0);
4988 if (scode != CODE_FOR_nothing)
4989 can_do_signed = true;
4990 if (imode != GET_MODE (from))
4991 icode = scode, doing_unsigned = 0;
4992 }
4993
4994 if (icode != CODE_FOR_nothing)
4995 {
4996 if (imode != GET_MODE (from))
4997 from = convert_to_mode (imode, from, unsignedp);
4998
4999 if (fmode != GET_MODE (to))
5000 target = gen_reg_rtx (fmode);
5001
5002 emit_unop_insn (icode, target, from,
5003 doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
5004
5005 if (target != to)
5006 convert_move (to, target, 0);
5007 return;
5008 }
5009 }
5010
5011 /* Unsigned integer, and no way to convert directly. Convert as signed,
5012 then unconditionally adjust the result. */
5013 if (unsignedp && can_do_signed)
5014 {
5015 rtx label = gen_label_rtx ();
5016 rtx temp;
5017 REAL_VALUE_TYPE offset;
5018
5019 /* Look for a usable floating mode FMODE wider than the source and at
5020 least as wide as the target. Using FMODE will avoid rounding woes
5021 with unsigned values greater than the signed maximum value. */
5022
5023 for (fmode = GET_MODE (to); fmode != VOIDmode;
5024 fmode = GET_MODE_WIDER_MODE (fmode))
5025 if (GET_MODE_PRECISION (GET_MODE (from)) < GET_MODE_BITSIZE (fmode)
5026 && can_float_p (fmode, GET_MODE (from), 0) != CODE_FOR_nothing)
5027 break;
5028
5029 if (fmode == VOIDmode)
5030 {
5031 /* There is no such mode. Pretend the target is wide enough. */
5032 fmode = GET_MODE (to);
5033
5034 /* Avoid double-rounding when TO is narrower than FROM. */
5035 if ((significand_size (fmode) + 1)
5036 < GET_MODE_PRECISION (GET_MODE (from)))
5037 {
5038 rtx temp1;
5039 rtx neglabel = gen_label_rtx ();
5040
5041 /* Don't use TARGET if it isn't a register, is a hard register,
5042 or is the wrong mode. */
5043 if (!REG_P (target)
5044 || REGNO (target) < FIRST_PSEUDO_REGISTER
5045 || GET_MODE (target) != fmode)
5046 target = gen_reg_rtx (fmode);
5047
5048 imode = GET_MODE (from);
5049 do_pending_stack_adjust ();
5050
5051 /* Test whether the sign bit is set. */
5052 emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
5053 0, neglabel);
5054
5055 /* The sign bit is not set. Convert as signed. */
5056 expand_float (target, from, 0);
5057 emit_jump_insn (gen_jump (label));
5058 emit_barrier ();
5059
5060 /* The sign bit is set.
5061 Convert to a usable (positive signed) value by shifting right
5062 one bit, while remembering if a nonzero bit was shifted
5063 out; i.e., compute (from & 1) | (from >> 1). */
5064
5065 emit_label (neglabel);
5066 temp = expand_binop (imode, and_optab, from, const1_rtx,
5067 NULL_RTX, 1, OPTAB_LIB_WIDEN);
5068 temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
5069 temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
5070 OPTAB_LIB_WIDEN);
5071 expand_float (target, temp, 0);
5072
5073 /* Multiply by 2 to undo the shift above. */
5074 temp = expand_binop (fmode, add_optab, target, target,
5075 target, 0, OPTAB_LIB_WIDEN);
5076 if (temp != target)
5077 emit_move_insn (target, temp);
5078
5079 do_pending_stack_adjust ();
5080 emit_label (label);
5081 goto done;
5082 }
5083 }
5084
5085 /* If we are about to do some arithmetic to correct for an
5086 unsigned operand, do it in a pseudo-register. */
5087
5088 if (GET_MODE (to) != fmode
5089 || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
5090 target = gen_reg_rtx (fmode);
5091
5092 /* Convert as signed integer to floating. */
5093 expand_float (target, from, 0);
5094
5095 /* If FROM is negative (and therefore TO is negative),
5096 correct its value by 2**bitwidth. */
5097
5098 do_pending_stack_adjust ();
5099 emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, GET_MODE (from),
5100 0, label);
5101
5102
5103 real_2expN (&offset, GET_MODE_PRECISION (GET_MODE (from)), fmode);
5104 temp = expand_binop (fmode, add_optab, target,
5105 CONST_DOUBLE_FROM_REAL_VALUE (offset, fmode),
5106 target, 0, OPTAB_LIB_WIDEN);
5107 if (temp != target)
5108 emit_move_insn (target, temp);
5109
5110 do_pending_stack_adjust ();
5111 emit_label (label);
5112 goto done;
5113 }
5114
5115 /* No hardware instruction available; call a library routine. */
5116 {
5117 rtx libfunc;
5118 rtx insns;
5119 rtx value;
5120 convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
5121
5122 if (GET_MODE_SIZE (GET_MODE (from)) < GET_MODE_SIZE (SImode))
5123 from = convert_to_mode (SImode, from, unsignedp);
5124
5125 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5126 gcc_assert (libfunc);
5127
5128 start_sequence ();
5129
5130 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5131 GET_MODE (to), 1, from,
5132 GET_MODE (from));
5133 insns = get_insns ();
5134 end_sequence ();
5135
5136 emit_libcall_block (insns, target, value,
5137 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
5138 GET_MODE (to), from));
5139 }
5140
5141 done:
5142
5143 /* Copy result to requested destination
5144 if we have been computing in a temp location. */
5145
5146 if (target != to)
5147 {
5148 if (GET_MODE (target) == GET_MODE (to))
5149 emit_move_insn (to, target);
5150 else
5151 convert_move (to, target, 0);
5152 }
5153 }
5154 \f
5155 /* Generate code to convert FROM to fixed point and store in TO. FROM
5156 must be floating point. */
5157
5158 void
5159 expand_fix (rtx to, rtx from, int unsignedp)
5160 {
5161 enum insn_code icode;
5162 rtx target = to;
5163 enum machine_mode fmode, imode;
5164 int must_trunc = 0;
5165
5166 /* We first try to find a pair of modes, one real and one integer, at
5167 least as wide as FROM and TO, respectively, in which we can open-code
5168 this conversion. If the integer mode is wider than the mode of TO,
5169 we can do the conversion either signed or unsigned. */
5170
5171 for (fmode = GET_MODE (from); fmode != VOIDmode;
5172 fmode = GET_MODE_WIDER_MODE (fmode))
5173 for (imode = GET_MODE (to); imode != VOIDmode;
5174 imode = GET_MODE_WIDER_MODE (imode))
5175 {
5176 int doing_unsigned = unsignedp;
5177
5178 icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
5179 if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
5180 icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
5181
5182 if (icode != CODE_FOR_nothing)
5183 {
5184 rtx last = get_last_insn ();
5185 if (fmode != GET_MODE (from))
5186 from = convert_to_mode (fmode, from, 0);
5187
5188 if (must_trunc)
5189 {
5190 rtx temp = gen_reg_rtx (GET_MODE (from));
5191 from = expand_unop (GET_MODE (from), ftrunc_optab, from,
5192 temp, 0);
5193 }
5194
5195 if (imode != GET_MODE (to))
5196 target = gen_reg_rtx (imode);
5197
5198 if (maybe_emit_unop_insn (icode, target, from,
5199 doing_unsigned ? UNSIGNED_FIX : FIX))
5200 {
5201 if (target != to)
5202 convert_move (to, target, unsignedp);
5203 return;
5204 }
5205 delete_insns_since (last);
5206 }
5207 }
5208
5209 /* For an unsigned conversion, there is one more way to do it.
5210 If we have a signed conversion, we generate code that compares
5211 the real value to the largest representable positive number. If if
5212 is smaller, the conversion is done normally. Otherwise, subtract
5213 one plus the highest signed number, convert, and add it back.
5214
5215 We only need to check all real modes, since we know we didn't find
5216 anything with a wider integer mode.
5217
5218 This code used to extend FP value into mode wider than the destination.
5219 This is needed for decimal float modes which cannot accurately
5220 represent one plus the highest signed number of the same size, but
5221 not for binary modes. Consider, for instance conversion from SFmode
5222 into DImode.
5223
5224 The hot path through the code is dealing with inputs smaller than 2^63
5225 and doing just the conversion, so there is no bits to lose.
5226
5227 In the other path we know the value is positive in the range 2^63..2^64-1
5228 inclusive. (as for other input overflow happens and result is undefined)
5229 So we know that the most important bit set in mantissa corresponds to
5230 2^63. The subtraction of 2^63 should not generate any rounding as it
5231 simply clears out that bit. The rest is trivial. */
5232
5233 if (unsignedp && GET_MODE_PRECISION (GET_MODE (to)) <= HOST_BITS_PER_WIDE_INT)
5234 for (fmode = GET_MODE (from); fmode != VOIDmode;
5235 fmode = GET_MODE_WIDER_MODE (fmode))
5236 if (CODE_FOR_nothing != can_fix_p (GET_MODE (to), fmode, 0, &must_trunc)
5237 && (!DECIMAL_FLOAT_MODE_P (fmode)
5238 || GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (GET_MODE (to))))
5239 {
5240 int bitsize;
5241 REAL_VALUE_TYPE offset;
5242 rtx limit, lab1, lab2, insn;
5243
5244 bitsize = GET_MODE_PRECISION (GET_MODE (to));
5245 real_2expN (&offset, bitsize - 1, fmode);
5246 limit = CONST_DOUBLE_FROM_REAL_VALUE (offset, fmode);
5247 lab1 = gen_label_rtx ();
5248 lab2 = gen_label_rtx ();
5249
5250 if (fmode != GET_MODE (from))
5251 from = convert_to_mode (fmode, from, 0);
5252
5253 /* See if we need to do the subtraction. */
5254 do_pending_stack_adjust ();
5255 emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX, GET_MODE (from),
5256 0, lab1);
5257
5258 /* If not, do the signed "fix" and branch around fixup code. */
5259 expand_fix (to, from, 0);
5260 emit_jump_insn (gen_jump (lab2));
5261 emit_barrier ();
5262
5263 /* Otherwise, subtract 2**(N-1), convert to signed number,
5264 then add 2**(N-1). Do the addition using XOR since this
5265 will often generate better code. */
5266 emit_label (lab1);
5267 target = expand_binop (GET_MODE (from), sub_optab, from, limit,
5268 NULL_RTX, 0, OPTAB_LIB_WIDEN);
5269 expand_fix (to, target, 0);
5270 target = expand_binop (GET_MODE (to), xor_optab, to,
5271 gen_int_mode
5272 ((HOST_WIDE_INT) 1 << (bitsize - 1),
5273 GET_MODE (to)),
5274 to, 1, OPTAB_LIB_WIDEN);
5275
5276 if (target != to)
5277 emit_move_insn (to, target);
5278
5279 emit_label (lab2);
5280
5281 if (optab_handler (mov_optab, GET_MODE (to)) != CODE_FOR_nothing)
5282 {
5283 /* Make a place for a REG_NOTE and add it. */
5284 insn = emit_move_insn (to, to);
5285 set_dst_reg_note (insn, REG_EQUAL,
5286 gen_rtx_fmt_e (UNSIGNED_FIX, GET_MODE (to),
5287 copy_rtx (from)),
5288 to);
5289 }
5290
5291 return;
5292 }
5293
5294 /* We can't do it with an insn, so use a library call. But first ensure
5295 that the mode of TO is at least as wide as SImode, since those are the
5296 only library calls we know about. */
5297
5298 if (GET_MODE_SIZE (GET_MODE (to)) < GET_MODE_SIZE (SImode))
5299 {
5300 target = gen_reg_rtx (SImode);
5301
5302 expand_fix (target, from, unsignedp);
5303 }
5304 else
5305 {
5306 rtx insns;
5307 rtx value;
5308 rtx libfunc;
5309
5310 convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
5311 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5312 gcc_assert (libfunc);
5313
5314 start_sequence ();
5315
5316 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5317 GET_MODE (to), 1, from,
5318 GET_MODE (from));
5319 insns = get_insns ();
5320 end_sequence ();
5321
5322 emit_libcall_block (insns, target, value,
5323 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
5324 GET_MODE (to), from));
5325 }
5326
5327 if (target != to)
5328 {
5329 if (GET_MODE (to) == GET_MODE (target))
5330 emit_move_insn (to, target);
5331 else
5332 convert_move (to, target, 0);
5333 }
5334 }
5335
5336 /* Generate code to convert FROM or TO a fixed-point.
5337 If UINTP is true, either TO or FROM is an unsigned integer.
5338 If SATP is true, we need to saturate the result. */
5339
5340 void
5341 expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
5342 {
5343 enum machine_mode to_mode = GET_MODE (to);
5344 enum machine_mode from_mode = GET_MODE (from);
5345 convert_optab tab;
5346 enum rtx_code this_code;
5347 enum insn_code code;
5348 rtx insns, value;
5349 rtx libfunc;
5350
5351 if (to_mode == from_mode)
5352 {
5353 emit_move_insn (to, from);
5354 return;
5355 }
5356
5357 if (uintp)
5358 {
5359 tab = satp ? satfractuns_optab : fractuns_optab;
5360 this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
5361 }
5362 else
5363 {
5364 tab = satp ? satfract_optab : fract_optab;
5365 this_code = satp ? SAT_FRACT : FRACT_CONVERT;
5366 }
5367 code = convert_optab_handler (tab, to_mode, from_mode);
5368 if (code != CODE_FOR_nothing)
5369 {
5370 emit_unop_insn (code, to, from, this_code);
5371 return;
5372 }
5373
5374 libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
5375 gcc_assert (libfunc);
5376
5377 start_sequence ();
5378 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
5379 1, from, from_mode);
5380 insns = get_insns ();
5381 end_sequence ();
5382
5383 emit_libcall_block (insns, to, value,
5384 gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
5385 }
5386
5387 /* Generate code to convert FROM to fixed point and store in TO. FROM
5388 must be floating point, TO must be signed. Use the conversion optab
5389 TAB to do the conversion. */
5390
5391 bool
5392 expand_sfix_optab (rtx to, rtx from, convert_optab tab)
5393 {
5394 enum insn_code icode;
5395 rtx target = to;
5396 enum machine_mode fmode, imode;
5397
5398 /* We first try to find a pair of modes, one real and one integer, at
5399 least as wide as FROM and TO, respectively, in which we can open-code
5400 this conversion. If the integer mode is wider than the mode of TO,
5401 we can do the conversion either signed or unsigned. */
5402
5403 for (fmode = GET_MODE (from); fmode != VOIDmode;
5404 fmode = GET_MODE_WIDER_MODE (fmode))
5405 for (imode = GET_MODE (to); imode != VOIDmode;
5406 imode = GET_MODE_WIDER_MODE (imode))
5407 {
5408 icode = convert_optab_handler (tab, imode, fmode);
5409 if (icode != CODE_FOR_nothing)
5410 {
5411 rtx last = get_last_insn ();
5412 if (fmode != GET_MODE (from))
5413 from = convert_to_mode (fmode, from, 0);
5414
5415 if (imode != GET_MODE (to))
5416 target = gen_reg_rtx (imode);
5417
5418 if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
5419 {
5420 delete_insns_since (last);
5421 continue;
5422 }
5423 if (target != to)
5424 convert_move (to, target, 0);
5425 return true;
5426 }
5427 }
5428
5429 return false;
5430 }
5431 \f
5432 /* Report whether we have an instruction to perform the operation
5433 specified by CODE on operands of mode MODE. */
5434 int
5435 have_insn_for (enum rtx_code code, enum machine_mode mode)
5436 {
5437 return (code_to_optab (code)
5438 && (optab_handler (code_to_optab (code), mode)
5439 != CODE_FOR_nothing));
5440 }
5441
5442 /* Initialize the libfunc fields of an entire group of entries in some
5443 optab. Each entry is set equal to a string consisting of a leading
5444 pair of underscores followed by a generic operation name followed by
5445 a mode name (downshifted to lowercase) followed by a single character
5446 representing the number of operands for the given operation (which is
5447 usually one of the characters '2', '3', or '4').
5448
5449 OPTABLE is the table in which libfunc fields are to be initialized.
5450 OPNAME is the generic (string) name of the operation.
5451 SUFFIX is the character which specifies the number of operands for
5452 the given generic operation.
5453 MODE is the mode to generate for.
5454 */
5455
5456 static void
5457 gen_libfunc (optab optable, const char *opname, int suffix,
5458 enum machine_mode mode)
5459 {
5460 unsigned opname_len = strlen (opname);
5461 const char *mname = GET_MODE_NAME (mode);
5462 unsigned mname_len = strlen (mname);
5463 int prefix_len = targetm.libfunc_gnu_prefix ? 6 : 2;
5464 int len = prefix_len + opname_len + mname_len + 1 + 1;
5465 char *libfunc_name = XALLOCAVEC (char, len);
5466 char *p;
5467 const char *q;
5468
5469 p = libfunc_name;
5470 *p++ = '_';
5471 *p++ = '_';
5472 if (targetm.libfunc_gnu_prefix)
5473 {
5474 *p++ = 'g';
5475 *p++ = 'n';
5476 *p++ = 'u';
5477 *p++ = '_';
5478 }
5479 for (q = opname; *q; )
5480 *p++ = *q++;
5481 for (q = mname; *q; q++)
5482 *p++ = TOLOWER (*q);
5483 *p++ = suffix;
5484 *p = '\0';
5485
5486 set_optab_libfunc (optable, mode,
5487 ggc_alloc_string (libfunc_name, p - libfunc_name));
5488 }
5489
5490 /* Like gen_libfunc, but verify that integer operation is involved. */
5491
5492 void
5493 gen_int_libfunc (optab optable, const char *opname, char suffix,
5494 enum machine_mode mode)
5495 {
5496 int maxsize = 2 * BITS_PER_WORD;
5497
5498 if (GET_MODE_CLASS (mode) != MODE_INT)
5499 return;
5500 if (maxsize < LONG_LONG_TYPE_SIZE)
5501 maxsize = LONG_LONG_TYPE_SIZE;
5502 if (GET_MODE_CLASS (mode) != MODE_INT
5503 || mode < word_mode || GET_MODE_BITSIZE (mode) > maxsize)
5504 return;
5505 gen_libfunc (optable, opname, suffix, mode);
5506 }
5507
5508 /* Like gen_libfunc, but verify that FP and set decimal prefix if needed. */
5509
5510 void
5511 gen_fp_libfunc (optab optable, const char *opname, char suffix,
5512 enum machine_mode mode)
5513 {
5514 char *dec_opname;
5515
5516 if (GET_MODE_CLASS (mode) == MODE_FLOAT)
5517 gen_libfunc (optable, opname, suffix, mode);
5518 if (DECIMAL_FLOAT_MODE_P (mode))
5519 {
5520 dec_opname = XALLOCAVEC (char, sizeof (DECIMAL_PREFIX) + strlen (opname));
5521 /* For BID support, change the name to have either a bid_ or dpd_ prefix
5522 depending on the low level floating format used. */
5523 memcpy (dec_opname, DECIMAL_PREFIX, sizeof (DECIMAL_PREFIX) - 1);
5524 strcpy (dec_opname + sizeof (DECIMAL_PREFIX) - 1, opname);
5525 gen_libfunc (optable, dec_opname, suffix, mode);
5526 }
5527 }
5528
5529 /* Like gen_libfunc, but verify that fixed-point operation is involved. */
5530
5531 void
5532 gen_fixed_libfunc (optab optable, const char *opname, char suffix,
5533 enum machine_mode mode)
5534 {
5535 if (!ALL_FIXED_POINT_MODE_P (mode))
5536 return;
5537 gen_libfunc (optable, opname, suffix, mode);
5538 }
5539
5540 /* Like gen_libfunc, but verify that signed fixed-point operation is
5541 involved. */
5542
5543 void
5544 gen_signed_fixed_libfunc (optab optable, const char *opname, char suffix,
5545 enum machine_mode mode)
5546 {
5547 if (!SIGNED_FIXED_POINT_MODE_P (mode))
5548 return;
5549 gen_libfunc (optable, opname, suffix, mode);
5550 }
5551
5552 /* Like gen_libfunc, but verify that unsigned fixed-point operation is
5553 involved. */
5554
5555 void
5556 gen_unsigned_fixed_libfunc (optab optable, const char *opname, char suffix,
5557 enum machine_mode mode)
5558 {
5559 if (!UNSIGNED_FIXED_POINT_MODE_P (mode))
5560 return;
5561 gen_libfunc (optable, opname, suffix, mode);
5562 }
5563
5564 /* Like gen_libfunc, but verify that FP or INT operation is involved. */
5565
5566 void
5567 gen_int_fp_libfunc (optab optable, const char *name, char suffix,
5568 enum machine_mode mode)
5569 {
5570 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5571 gen_fp_libfunc (optable, name, suffix, mode);
5572 if (INTEGRAL_MODE_P (mode))
5573 gen_int_libfunc (optable, name, suffix, mode);
5574 }
5575
5576 /* Like gen_libfunc, but verify that FP or INT operation is involved
5577 and add 'v' suffix for integer operation. */
5578
5579 void
5580 gen_intv_fp_libfunc (optab optable, const char *name, char suffix,
5581 enum machine_mode mode)
5582 {
5583 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5584 gen_fp_libfunc (optable, name, suffix, mode);
5585 if (GET_MODE_CLASS (mode) == MODE_INT)
5586 {
5587 int len = strlen (name);
5588 char *v_name = XALLOCAVEC (char, len + 2);
5589 strcpy (v_name, name);
5590 v_name[len] = 'v';
5591 v_name[len + 1] = 0;
5592 gen_int_libfunc (optable, v_name, suffix, mode);
5593 }
5594 }
5595
5596 /* Like gen_libfunc, but verify that FP or INT or FIXED operation is
5597 involved. */
5598
5599 void
5600 gen_int_fp_fixed_libfunc (optab optable, const char *name, char suffix,
5601 enum machine_mode mode)
5602 {
5603 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5604 gen_fp_libfunc (optable, name, suffix, mode);
5605 if (INTEGRAL_MODE_P (mode))
5606 gen_int_libfunc (optable, name, suffix, mode);
5607 if (ALL_FIXED_POINT_MODE_P (mode))
5608 gen_fixed_libfunc (optable, name, suffix, mode);
5609 }
5610
5611 /* Like gen_libfunc, but verify that FP or INT or signed FIXED operation is
5612 involved. */
5613
5614 void
5615 gen_int_fp_signed_fixed_libfunc (optab optable, const char *name, char suffix,
5616 enum machine_mode mode)
5617 {
5618 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5619 gen_fp_libfunc (optable, name, suffix, mode);
5620 if (INTEGRAL_MODE_P (mode))
5621 gen_int_libfunc (optable, name, suffix, mode);
5622 if (SIGNED_FIXED_POINT_MODE_P (mode))
5623 gen_signed_fixed_libfunc (optable, name, suffix, mode);
5624 }
5625
5626 /* Like gen_libfunc, but verify that INT or FIXED operation is
5627 involved. */
5628
5629 void
5630 gen_int_fixed_libfunc (optab optable, const char *name, char suffix,
5631 enum machine_mode mode)
5632 {
5633 if (INTEGRAL_MODE_P (mode))
5634 gen_int_libfunc (optable, name, suffix, mode);
5635 if (ALL_FIXED_POINT_MODE_P (mode))
5636 gen_fixed_libfunc (optable, name, suffix, mode);
5637 }
5638
5639 /* Like gen_libfunc, but verify that INT or signed FIXED operation is
5640 involved. */
5641
5642 void
5643 gen_int_signed_fixed_libfunc (optab optable, const char *name, char suffix,
5644 enum machine_mode mode)
5645 {
5646 if (INTEGRAL_MODE_P (mode))
5647 gen_int_libfunc (optable, name, suffix, mode);
5648 if (SIGNED_FIXED_POINT_MODE_P (mode))
5649 gen_signed_fixed_libfunc (optable, name, suffix, mode);
5650 }
5651
5652 /* Like gen_libfunc, but verify that INT or unsigned FIXED operation is
5653 involved. */
5654
5655 void
5656 gen_int_unsigned_fixed_libfunc (optab optable, const char *name, char suffix,
5657 enum machine_mode mode)
5658 {
5659 if (INTEGRAL_MODE_P (mode))
5660 gen_int_libfunc (optable, name, suffix, mode);
5661 if (UNSIGNED_FIXED_POINT_MODE_P (mode))
5662 gen_unsigned_fixed_libfunc (optable, name, suffix, mode);
5663 }
5664
5665 /* Initialize the libfunc fields of an entire group of entries of an
5666 inter-mode-class conversion optab. The string formation rules are
5667 similar to the ones for init_libfuncs, above, but instead of having
5668 a mode name and an operand count these functions have two mode names
5669 and no operand count. */
5670
5671 void
5672 gen_interclass_conv_libfunc (convert_optab tab,
5673 const char *opname,
5674 enum machine_mode tmode,
5675 enum machine_mode fmode)
5676 {
5677 size_t opname_len = strlen (opname);
5678 size_t mname_len = 0;
5679
5680 const char *fname, *tname;
5681 const char *q;
5682 int prefix_len = targetm.libfunc_gnu_prefix ? 6 : 2;
5683 char *libfunc_name, *suffix;
5684 char *nondec_name, *dec_name, *nondec_suffix, *dec_suffix;
5685 char *p;
5686
5687 /* If this is a decimal conversion, add the current BID vs. DPD prefix that
5688 depends on which underlying decimal floating point format is used. */
5689 const size_t dec_len = sizeof (DECIMAL_PREFIX) - 1;
5690
5691 mname_len = strlen (GET_MODE_NAME (tmode)) + strlen (GET_MODE_NAME (fmode));
5692
5693 nondec_name = XALLOCAVEC (char, prefix_len + opname_len + mname_len + 1 + 1);
5694 nondec_name[0] = '_';
5695 nondec_name[1] = '_';
5696 if (targetm.libfunc_gnu_prefix)
5697 {
5698 nondec_name[2] = 'g';
5699 nondec_name[3] = 'n';
5700 nondec_name[4] = 'u';
5701 nondec_name[5] = '_';
5702 }
5703
5704 memcpy (&nondec_name[prefix_len], opname, opname_len);
5705 nondec_suffix = nondec_name + opname_len + prefix_len;
5706
5707 dec_name = XALLOCAVEC (char, 2 + dec_len + opname_len + mname_len + 1 + 1);
5708 dec_name[0] = '_';
5709 dec_name[1] = '_';
5710 memcpy (&dec_name[2], DECIMAL_PREFIX, dec_len);
5711 memcpy (&dec_name[2+dec_len], opname, opname_len);
5712 dec_suffix = dec_name + dec_len + opname_len + 2;
5713
5714 fname = GET_MODE_NAME (fmode);
5715 tname = GET_MODE_NAME (tmode);
5716
5717 if (DECIMAL_FLOAT_MODE_P (fmode) || DECIMAL_FLOAT_MODE_P (tmode))
5718 {
5719 libfunc_name = dec_name;
5720 suffix = dec_suffix;
5721 }
5722 else
5723 {
5724 libfunc_name = nondec_name;
5725 suffix = nondec_suffix;
5726 }
5727
5728 p = suffix;
5729 for (q = fname; *q; p++, q++)
5730 *p = TOLOWER (*q);
5731 for (q = tname; *q; p++, q++)
5732 *p = TOLOWER (*q);
5733
5734 *p = '\0';
5735
5736 set_conv_libfunc (tab, tmode, fmode,
5737 ggc_alloc_string (libfunc_name, p - libfunc_name));
5738 }
5739
5740 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5741 int->fp conversion. */
5742
5743 void
5744 gen_int_to_fp_conv_libfunc (convert_optab tab,
5745 const char *opname,
5746 enum machine_mode tmode,
5747 enum machine_mode fmode)
5748 {
5749 if (GET_MODE_CLASS (fmode) != MODE_INT)
5750 return;
5751 if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode))
5752 return;
5753 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5754 }
5755
5756 /* ufloat_optab is special by using floatun for FP and floatuns decimal fp
5757 naming scheme. */
5758
5759 void
5760 gen_ufloat_conv_libfunc (convert_optab tab,
5761 const char *opname ATTRIBUTE_UNUSED,
5762 enum machine_mode tmode,
5763 enum machine_mode fmode)
5764 {
5765 if (DECIMAL_FLOAT_MODE_P (tmode))
5766 gen_int_to_fp_conv_libfunc (tab, "floatuns", tmode, fmode);
5767 else
5768 gen_int_to_fp_conv_libfunc (tab, "floatun", tmode, fmode);
5769 }
5770
5771 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5772 fp->int conversion. */
5773
5774 void
5775 gen_int_to_fp_nondecimal_conv_libfunc (convert_optab tab,
5776 const char *opname,
5777 enum machine_mode tmode,
5778 enum machine_mode fmode)
5779 {
5780 if (GET_MODE_CLASS (fmode) != MODE_INT)
5781 return;
5782 if (GET_MODE_CLASS (tmode) != MODE_FLOAT)
5783 return;
5784 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5785 }
5786
5787 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5788 fp->int conversion with no decimal floating point involved. */
5789
5790 void
5791 gen_fp_to_int_conv_libfunc (convert_optab tab,
5792 const char *opname,
5793 enum machine_mode tmode,
5794 enum machine_mode fmode)
5795 {
5796 if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode))
5797 return;
5798 if (GET_MODE_CLASS (tmode) != MODE_INT)
5799 return;
5800 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5801 }
5802
5803 /* Initialize the libfunc fields of an of an intra-mode-class conversion optab.
5804 The string formation rules are
5805 similar to the ones for init_libfunc, above. */
5806
5807 void
5808 gen_intraclass_conv_libfunc (convert_optab tab, const char *opname,
5809 enum machine_mode tmode, enum machine_mode fmode)
5810 {
5811 size_t opname_len = strlen (opname);
5812 size_t mname_len = 0;
5813
5814 const char *fname, *tname;
5815 const char *q;
5816 int prefix_len = targetm.libfunc_gnu_prefix ? 6 : 2;
5817 char *nondec_name, *dec_name, *nondec_suffix, *dec_suffix;
5818 char *libfunc_name, *suffix;
5819 char *p;
5820
5821 /* If this is a decimal conversion, add the current BID vs. DPD prefix that
5822 depends on which underlying decimal floating point format is used. */
5823 const size_t dec_len = sizeof (DECIMAL_PREFIX) - 1;
5824
5825 mname_len = strlen (GET_MODE_NAME (tmode)) + strlen (GET_MODE_NAME (fmode));
5826
5827 nondec_name = XALLOCAVEC (char, 2 + opname_len + mname_len + 1 + 1);
5828 nondec_name[0] = '_';
5829 nondec_name[1] = '_';
5830 if (targetm.libfunc_gnu_prefix)
5831 {
5832 nondec_name[2] = 'g';
5833 nondec_name[3] = 'n';
5834 nondec_name[4] = 'u';
5835 nondec_name[5] = '_';
5836 }
5837 memcpy (&nondec_name[prefix_len], opname, opname_len);
5838 nondec_suffix = nondec_name + opname_len + prefix_len;
5839
5840 dec_name = XALLOCAVEC (char, 2 + dec_len + opname_len + mname_len + 1 + 1);
5841 dec_name[0] = '_';
5842 dec_name[1] = '_';
5843 memcpy (&dec_name[2], DECIMAL_PREFIX, dec_len);
5844 memcpy (&dec_name[2 + dec_len], opname, opname_len);
5845 dec_suffix = dec_name + dec_len + opname_len + 2;
5846
5847 fname = GET_MODE_NAME (fmode);
5848 tname = GET_MODE_NAME (tmode);
5849
5850 if (DECIMAL_FLOAT_MODE_P (fmode) || DECIMAL_FLOAT_MODE_P (tmode))
5851 {
5852 libfunc_name = dec_name;
5853 suffix = dec_suffix;
5854 }
5855 else
5856 {
5857 libfunc_name = nondec_name;
5858 suffix = nondec_suffix;
5859 }
5860
5861 p = suffix;
5862 for (q = fname; *q; p++, q++)
5863 *p = TOLOWER (*q);
5864 for (q = tname; *q; p++, q++)
5865 *p = TOLOWER (*q);
5866
5867 *p++ = '2';
5868 *p = '\0';
5869
5870 set_conv_libfunc (tab, tmode, fmode,
5871 ggc_alloc_string (libfunc_name, p - libfunc_name));
5872 }
5873
5874 /* Pick proper libcall for trunc_optab. We need to chose if we do
5875 truncation or extension and interclass or intraclass. */
5876
5877 void
5878 gen_trunc_conv_libfunc (convert_optab tab,
5879 const char *opname,
5880 enum machine_mode tmode,
5881 enum machine_mode fmode)
5882 {
5883 if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode))
5884 return;
5885 if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode))
5886 return;
5887 if (tmode == fmode)
5888 return;
5889
5890 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (fmode))
5891 || (GET_MODE_CLASS (fmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (tmode)))
5892 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5893
5894 if (GET_MODE_PRECISION (fmode) <= GET_MODE_PRECISION (tmode))
5895 return;
5896
5897 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT
5898 && GET_MODE_CLASS (fmode) == MODE_FLOAT)
5899 || (DECIMAL_FLOAT_MODE_P (fmode) && DECIMAL_FLOAT_MODE_P (tmode)))
5900 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
5901 }
5902
5903 /* Pick proper libcall for extend_optab. We need to chose if we do
5904 truncation or extension and interclass or intraclass. */
5905
5906 void
5907 gen_extend_conv_libfunc (convert_optab tab,
5908 const char *opname ATTRIBUTE_UNUSED,
5909 enum machine_mode tmode,
5910 enum machine_mode fmode)
5911 {
5912 if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode))
5913 return;
5914 if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode))
5915 return;
5916 if (tmode == fmode)
5917 return;
5918
5919 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (fmode))
5920 || (GET_MODE_CLASS (fmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (tmode)))
5921 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5922
5923 if (GET_MODE_PRECISION (fmode) > GET_MODE_PRECISION (tmode))
5924 return;
5925
5926 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT
5927 && GET_MODE_CLASS (fmode) == MODE_FLOAT)
5928 || (DECIMAL_FLOAT_MODE_P (fmode) && DECIMAL_FLOAT_MODE_P (tmode)))
5929 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
5930 }
5931
5932 /* Pick proper libcall for fract_optab. We need to chose if we do
5933 interclass or intraclass. */
5934
5935 void
5936 gen_fract_conv_libfunc (convert_optab tab,
5937 const char *opname,
5938 enum machine_mode tmode,
5939 enum machine_mode fmode)
5940 {
5941 if (tmode == fmode)
5942 return;
5943 if (!(ALL_FIXED_POINT_MODE_P (tmode) || ALL_FIXED_POINT_MODE_P (fmode)))
5944 return;
5945
5946 if (GET_MODE_CLASS (tmode) == GET_MODE_CLASS (fmode))
5947 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
5948 else
5949 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5950 }
5951
5952 /* Pick proper libcall for fractuns_optab. */
5953
5954 void
5955 gen_fractuns_conv_libfunc (convert_optab tab,
5956 const char *opname,
5957 enum machine_mode tmode,
5958 enum machine_mode fmode)
5959 {
5960 if (tmode == fmode)
5961 return;
5962 /* One mode must be a fixed-point mode, and the other must be an integer
5963 mode. */
5964 if (!((ALL_FIXED_POINT_MODE_P (tmode) && GET_MODE_CLASS (fmode) == MODE_INT)
5965 || (ALL_FIXED_POINT_MODE_P (fmode)
5966 && GET_MODE_CLASS (tmode) == MODE_INT)))
5967 return;
5968
5969 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5970 }
5971
5972 /* Pick proper libcall for satfract_optab. We need to chose if we do
5973 interclass or intraclass. */
5974
5975 void
5976 gen_satfract_conv_libfunc (convert_optab tab,
5977 const char *opname,
5978 enum machine_mode tmode,
5979 enum machine_mode fmode)
5980 {
5981 if (tmode == fmode)
5982 return;
5983 /* TMODE must be a fixed-point mode. */
5984 if (!ALL_FIXED_POINT_MODE_P (tmode))
5985 return;
5986
5987 if (GET_MODE_CLASS (tmode) == GET_MODE_CLASS (fmode))
5988 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
5989 else
5990 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5991 }
5992
5993 /* Pick proper libcall for satfractuns_optab. */
5994
5995 void
5996 gen_satfractuns_conv_libfunc (convert_optab tab,
5997 const char *opname,
5998 enum machine_mode tmode,
5999 enum machine_mode fmode)
6000 {
6001 if (tmode == fmode)
6002 return;
6003 /* TMODE must be a fixed-point mode, and FMODE must be an integer mode. */
6004 if (!(ALL_FIXED_POINT_MODE_P (tmode) && GET_MODE_CLASS (fmode) == MODE_INT))
6005 return;
6006
6007 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
6008 }
6009
6010 /* A table of previously-created libfuncs, hashed by name. */
6011 static GTY ((param_is (union tree_node))) htab_t libfunc_decls;
6012
6013 /* Hashtable callbacks for libfunc_decls. */
6014
6015 static hashval_t
6016 libfunc_decl_hash (const void *entry)
6017 {
6018 return IDENTIFIER_HASH_VALUE (DECL_NAME ((const_tree) entry));
6019 }
6020
6021 static int
6022 libfunc_decl_eq (const void *entry1, const void *entry2)
6023 {
6024 return DECL_NAME ((const_tree) entry1) == (const_tree) entry2;
6025 }
6026
6027 /* Build a decl for a libfunc named NAME. */
6028
6029 tree
6030 build_libfunc_function (const char *name)
6031 {
6032 tree decl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL,
6033 get_identifier (name),
6034 build_function_type (integer_type_node, NULL_TREE));
6035 /* ??? We don't have any type information except for this is
6036 a function. Pretend this is "int foo()". */
6037 DECL_ARTIFICIAL (decl) = 1;
6038 DECL_EXTERNAL (decl) = 1;
6039 TREE_PUBLIC (decl) = 1;
6040 gcc_assert (DECL_ASSEMBLER_NAME (decl));
6041
6042 /* Zap the nonsensical SYMBOL_REF_DECL for this. What we're left with
6043 are the flags assigned by targetm.encode_section_info. */
6044 SET_SYMBOL_REF_DECL (XEXP (DECL_RTL (decl), 0), NULL);
6045
6046 return decl;
6047 }
6048
6049 rtx
6050 init_one_libfunc (const char *name)
6051 {
6052 tree id, decl;
6053 void **slot;
6054 hashval_t hash;
6055
6056 if (libfunc_decls == NULL)
6057 libfunc_decls = htab_create_ggc (37, libfunc_decl_hash,
6058 libfunc_decl_eq, NULL);
6059
6060 /* See if we have already created a libfunc decl for this function. */
6061 id = get_identifier (name);
6062 hash = IDENTIFIER_HASH_VALUE (id);
6063 slot = htab_find_slot_with_hash (libfunc_decls, id, hash, INSERT);
6064 decl = (tree) *slot;
6065 if (decl == NULL)
6066 {
6067 /* Create a new decl, so that it can be passed to
6068 targetm.encode_section_info. */
6069 decl = build_libfunc_function (name);
6070 *slot = decl;
6071 }
6072 return XEXP (DECL_RTL (decl), 0);
6073 }
6074
6075 /* Adjust the assembler name of libfunc NAME to ASMSPEC. */
6076
6077 rtx
6078 set_user_assembler_libfunc (const char *name, const char *asmspec)
6079 {
6080 tree id, decl;
6081 void **slot;
6082 hashval_t hash;
6083
6084 id = get_identifier (name);
6085 hash = IDENTIFIER_HASH_VALUE (id);
6086 slot = htab_find_slot_with_hash (libfunc_decls, id, hash, NO_INSERT);
6087 gcc_assert (slot);
6088 decl = (tree) *slot;
6089 set_user_assembler_name (decl, asmspec);
6090 return XEXP (DECL_RTL (decl), 0);
6091 }
6092
6093 /* Call this to reset the function entry for one optab (OPTABLE) in mode
6094 MODE to NAME, which should be either 0 or a string constant. */
6095 void
6096 set_optab_libfunc (optab op, enum machine_mode mode, const char *name)
6097 {
6098 rtx val;
6099 struct libfunc_entry e;
6100 struct libfunc_entry **slot;
6101
6102 e.op = op;
6103 e.mode1 = mode;
6104 e.mode2 = VOIDmode;
6105
6106 if (name)
6107 val = init_one_libfunc (name);
6108 else
6109 val = 0;
6110 slot = (struct libfunc_entry **) htab_find_slot (libfunc_hash, &e, INSERT);
6111 if (*slot == NULL)
6112 *slot = ggc_alloc_libfunc_entry ();
6113 (*slot)->op = op;
6114 (*slot)->mode1 = mode;
6115 (*slot)->mode2 = VOIDmode;
6116 (*slot)->libfunc = val;
6117 }
6118
6119 /* Call this to reset the function entry for one conversion optab
6120 (OPTABLE) from mode FMODE to mode TMODE to NAME, which should be
6121 either 0 or a string constant. */
6122 void
6123 set_conv_libfunc (convert_optab optab, enum machine_mode tmode,
6124 enum machine_mode fmode, const char *name)
6125 {
6126 rtx val;
6127 struct libfunc_entry e;
6128 struct libfunc_entry **slot;
6129
6130 e.op = optab;
6131 e.mode1 = tmode;
6132 e.mode2 = fmode;
6133
6134 if (name)
6135 val = init_one_libfunc (name);
6136 else
6137 val = 0;
6138 slot = (struct libfunc_entry **) htab_find_slot (libfunc_hash, &e, INSERT);
6139 if (*slot == NULL)
6140 *slot = ggc_alloc_libfunc_entry ();
6141 (*slot)->op = optab;
6142 (*slot)->mode1 = tmode;
6143 (*slot)->mode2 = fmode;
6144 (*slot)->libfunc = val;
6145 }
6146
6147 /* Call this to initialize the contents of the optabs
6148 appropriately for the current target machine. */
6149
6150 void
6151 init_optabs (void)
6152 {
6153 if (libfunc_hash)
6154 htab_empty (libfunc_hash);
6155 else
6156 libfunc_hash = htab_create_ggc (10, hash_libfunc, eq_libfunc, NULL);
6157
6158 /* Fill in the optabs with the insns we support. */
6159 init_all_optabs (this_fn_optabs);
6160
6161 /* The ffs function operates on `int'. Fall back on it if we do not
6162 have a libgcc2 function for that width. */
6163 if (INT_TYPE_SIZE < BITS_PER_WORD)
6164 set_optab_libfunc (ffs_optab, mode_for_size (INT_TYPE_SIZE, MODE_INT, 0),
6165 "ffs");
6166
6167 /* Explicitly initialize the bswap libfuncs since we need them to be
6168 valid for things other than word_mode. */
6169 if (targetm.libfunc_gnu_prefix)
6170 {
6171 set_optab_libfunc (bswap_optab, SImode, "__gnu_bswapsi2");
6172 set_optab_libfunc (bswap_optab, DImode, "__gnu_bswapdi2");
6173 }
6174 else
6175 {
6176 set_optab_libfunc (bswap_optab, SImode, "__bswapsi2");
6177 set_optab_libfunc (bswap_optab, DImode, "__bswapdi2");
6178 }
6179
6180 /* Use cabs for double complex abs, since systems generally have cabs.
6181 Don't define any libcall for float complex, so that cabs will be used. */
6182 if (complex_double_type_node)
6183 set_optab_libfunc (abs_optab, TYPE_MODE (complex_double_type_node),
6184 "cabs");
6185
6186 abort_libfunc = init_one_libfunc ("abort");
6187 memcpy_libfunc = init_one_libfunc ("memcpy");
6188 memmove_libfunc = init_one_libfunc ("memmove");
6189 memcmp_libfunc = init_one_libfunc ("memcmp");
6190 memset_libfunc = init_one_libfunc ("memset");
6191 setbits_libfunc = init_one_libfunc ("__setbits");
6192
6193 #ifndef DONT_USE_BUILTIN_SETJMP
6194 setjmp_libfunc = init_one_libfunc ("__builtin_setjmp");
6195 longjmp_libfunc = init_one_libfunc ("__builtin_longjmp");
6196 #else
6197 setjmp_libfunc = init_one_libfunc ("setjmp");
6198 longjmp_libfunc = init_one_libfunc ("longjmp");
6199 #endif
6200 unwind_sjlj_register_libfunc = init_one_libfunc ("_Unwind_SjLj_Register");
6201 unwind_sjlj_unregister_libfunc
6202 = init_one_libfunc ("_Unwind_SjLj_Unregister");
6203
6204 /* For function entry/exit instrumentation. */
6205 profile_function_entry_libfunc
6206 = init_one_libfunc ("__cyg_profile_func_enter");
6207 profile_function_exit_libfunc
6208 = init_one_libfunc ("__cyg_profile_func_exit");
6209
6210 gcov_flush_libfunc = init_one_libfunc ("__gcov_flush");
6211
6212 /* Allow the target to add more libcalls or rename some, etc. */
6213 targetm.init_libfuncs ();
6214 }
6215
6216 /* Use the current target and options to initialize
6217 TREE_OPTIMIZATION_OPTABS (OPTNODE). */
6218
6219 void
6220 init_tree_optimization_optabs (tree optnode)
6221 {
6222 /* Quick exit if we have already computed optabs for this target. */
6223 if (TREE_OPTIMIZATION_BASE_OPTABS (optnode) == this_target_optabs)
6224 return;
6225
6226 /* Forget any previous information and set up for the current target. */
6227 TREE_OPTIMIZATION_BASE_OPTABS (optnode) = this_target_optabs;
6228 struct target_optabs *tmp_optabs = (struct target_optabs *)
6229 TREE_OPTIMIZATION_OPTABS (optnode);
6230 if (tmp_optabs)
6231 memset (tmp_optabs, 0, sizeof (struct target_optabs));
6232 else
6233 tmp_optabs = (struct target_optabs *)
6234 ggc_alloc_atomic (sizeof (struct target_optabs));
6235
6236 /* Generate a new set of optabs into tmp_optabs. */
6237 init_all_optabs (tmp_optabs);
6238
6239 /* If the optabs changed, record it. */
6240 if (memcmp (tmp_optabs, this_target_optabs, sizeof (struct target_optabs)))
6241 TREE_OPTIMIZATION_OPTABS (optnode) = (unsigned char *) tmp_optabs;
6242 else
6243 {
6244 TREE_OPTIMIZATION_OPTABS (optnode) = NULL;
6245 ggc_free (tmp_optabs);
6246 }
6247 }
6248
6249 /* A helper function for init_sync_libfuncs. Using the basename BASE,
6250 install libfuncs into TAB for BASE_N for 1 <= N <= MAX. */
6251
6252 static void
6253 init_sync_libfuncs_1 (optab tab, const char *base, int max)
6254 {
6255 enum machine_mode mode;
6256 char buf[64];
6257 size_t len = strlen (base);
6258 int i;
6259
6260 gcc_assert (max <= 8);
6261 gcc_assert (len + 3 < sizeof (buf));
6262
6263 memcpy (buf, base, len);
6264 buf[len] = '_';
6265 buf[len + 1] = '0';
6266 buf[len + 2] = '\0';
6267
6268 mode = QImode;
6269 for (i = 1; i <= max; i *= 2)
6270 {
6271 buf[len + 1] = '0' + i;
6272 set_optab_libfunc (tab, mode, buf);
6273 mode = GET_MODE_2XWIDER_MODE (mode);
6274 }
6275 }
6276
6277 void
6278 init_sync_libfuncs (int max)
6279 {
6280 if (!flag_sync_libcalls)
6281 return;
6282
6283 init_sync_libfuncs_1 (sync_compare_and_swap_optab,
6284 "__sync_val_compare_and_swap", max);
6285 init_sync_libfuncs_1 (sync_lock_test_and_set_optab,
6286 "__sync_lock_test_and_set", max);
6287
6288 init_sync_libfuncs_1 (sync_old_add_optab, "__sync_fetch_and_add", max);
6289 init_sync_libfuncs_1 (sync_old_sub_optab, "__sync_fetch_and_sub", max);
6290 init_sync_libfuncs_1 (sync_old_ior_optab, "__sync_fetch_and_or", max);
6291 init_sync_libfuncs_1 (sync_old_and_optab, "__sync_fetch_and_and", max);
6292 init_sync_libfuncs_1 (sync_old_xor_optab, "__sync_fetch_and_xor", max);
6293 init_sync_libfuncs_1 (sync_old_nand_optab, "__sync_fetch_and_nand", max);
6294
6295 init_sync_libfuncs_1 (sync_new_add_optab, "__sync_add_and_fetch", max);
6296 init_sync_libfuncs_1 (sync_new_sub_optab, "__sync_sub_and_fetch", max);
6297 init_sync_libfuncs_1 (sync_new_ior_optab, "__sync_or_and_fetch", max);
6298 init_sync_libfuncs_1 (sync_new_and_optab, "__sync_and_and_fetch", max);
6299 init_sync_libfuncs_1 (sync_new_xor_optab, "__sync_xor_and_fetch", max);
6300 init_sync_libfuncs_1 (sync_new_nand_optab, "__sync_nand_and_fetch", max);
6301 }
6302
6303 /* Print information about the current contents of the optabs on
6304 STDERR. */
6305
6306 DEBUG_FUNCTION void
6307 debug_optab_libfuncs (void)
6308 {
6309 int i, j, k;
6310
6311 /* Dump the arithmetic optabs. */
6312 for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
6313 for (j = 0; j < NUM_MACHINE_MODES; ++j)
6314 {
6315 rtx l = optab_libfunc ((optab) i, (enum machine_mode) j);
6316 if (l)
6317 {
6318 gcc_assert (GET_CODE (l) == SYMBOL_REF);
6319 fprintf (stderr, "%s\t%s:\t%s\n",
6320 GET_RTX_NAME (optab_to_code ((optab) i)),
6321 GET_MODE_NAME (j),
6322 XSTR (l, 0));
6323 }
6324 }
6325
6326 /* Dump the conversion optabs. */
6327 for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
6328 for (j = 0; j < NUM_MACHINE_MODES; ++j)
6329 for (k = 0; k < NUM_MACHINE_MODES; ++k)
6330 {
6331 rtx l = convert_optab_libfunc ((optab) i, (enum machine_mode) j,
6332 (enum machine_mode) k);
6333 if (l)
6334 {
6335 gcc_assert (GET_CODE (l) == SYMBOL_REF);
6336 fprintf (stderr, "%s\t%s\t%s:\t%s\n",
6337 GET_RTX_NAME (optab_to_code ((optab) i)),
6338 GET_MODE_NAME (j),
6339 GET_MODE_NAME (k),
6340 XSTR (l, 0));
6341 }
6342 }
6343 }
6344
6345 \f
6346 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6347 CODE. Return 0 on failure. */
6348
6349 rtx
6350 gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
6351 {
6352 enum machine_mode mode = GET_MODE (op1);
6353 enum insn_code icode;
6354 rtx insn;
6355 rtx trap_rtx;
6356
6357 if (mode == VOIDmode)
6358 return 0;
6359
6360 icode = optab_handler (ctrap_optab, mode);
6361 if (icode == CODE_FOR_nothing)
6362 return 0;
6363
6364 /* Some targets only accept a zero trap code. */
6365 if (!insn_operand_matches (icode, 3, tcode))
6366 return 0;
6367
6368 do_pending_stack_adjust ();
6369 start_sequence ();
6370 prepare_cmp_insn (op1, op2, code, NULL_RTX, false, OPTAB_DIRECT,
6371 &trap_rtx, &mode);
6372 if (!trap_rtx)
6373 insn = NULL_RTX;
6374 else
6375 insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
6376 tcode);
6377
6378 /* If that failed, then give up. */
6379 if (insn == 0)
6380 {
6381 end_sequence ();
6382 return 0;
6383 }
6384
6385 emit_insn (insn);
6386 insn = get_insns ();
6387 end_sequence ();
6388 return insn;
6389 }
6390
6391 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6392 or unsigned operation code. */
6393
6394 static enum rtx_code
6395 get_rtx_code (enum tree_code tcode, bool unsignedp)
6396 {
6397 enum rtx_code code;
6398 switch (tcode)
6399 {
6400 case EQ_EXPR:
6401 code = EQ;
6402 break;
6403 case NE_EXPR:
6404 code = NE;
6405 break;
6406 case LT_EXPR:
6407 code = unsignedp ? LTU : LT;
6408 break;
6409 case LE_EXPR:
6410 code = unsignedp ? LEU : LE;
6411 break;
6412 case GT_EXPR:
6413 code = unsignedp ? GTU : GT;
6414 break;
6415 case GE_EXPR:
6416 code = unsignedp ? GEU : GE;
6417 break;
6418
6419 case UNORDERED_EXPR:
6420 code = UNORDERED;
6421 break;
6422 case ORDERED_EXPR:
6423 code = ORDERED;
6424 break;
6425 case UNLT_EXPR:
6426 code = UNLT;
6427 break;
6428 case UNLE_EXPR:
6429 code = UNLE;
6430 break;
6431 case UNGT_EXPR:
6432 code = UNGT;
6433 break;
6434 case UNGE_EXPR:
6435 code = UNGE;
6436 break;
6437 case UNEQ_EXPR:
6438 code = UNEQ;
6439 break;
6440 case LTGT_EXPR:
6441 code = LTGT;
6442 break;
6443
6444 default:
6445 gcc_unreachable ();
6446 }
6447 return code;
6448 }
6449
6450 /* Return comparison rtx for COND. Use UNSIGNEDP to select signed or
6451 unsigned operators. Do not generate compare instruction. */
6452
6453 static rtx
6454 vector_compare_rtx (enum tree_code tcode, tree t_op0, tree t_op1,
6455 bool unsignedp, enum insn_code icode)
6456 {
6457 struct expand_operand ops[2];
6458 rtx rtx_op0, rtx_op1;
6459 enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
6460
6461 gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
6462
6463 /* Expand operands. */
6464 rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
6465 EXPAND_STACK_PARM);
6466 rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
6467 EXPAND_STACK_PARM);
6468
6469 create_input_operand (&ops[0], rtx_op0, GET_MODE (rtx_op0));
6470 create_input_operand (&ops[1], rtx_op1, GET_MODE (rtx_op1));
6471 if (!maybe_legitimize_operands (icode, 4, 2, ops))
6472 gcc_unreachable ();
6473 return gen_rtx_fmt_ee (rcode, VOIDmode, ops[0].value, ops[1].value);
6474 }
6475
6476 /* Return true if VEC_PERM_EXPR can be expanded using SIMD extensions
6477 of the CPU. SEL may be NULL, which stands for an unknown constant. */
6478
6479 bool
6480 can_vec_perm_p (enum machine_mode mode, bool variable,
6481 const unsigned char *sel)
6482 {
6483 enum machine_mode qimode;
6484
6485 /* If the target doesn't implement a vector mode for the vector type,
6486 then no operations are supported. */
6487 if (!VECTOR_MODE_P (mode))
6488 return false;
6489
6490 if (!variable)
6491 {
6492 if (direct_optab_handler (vec_perm_const_optab, mode) != CODE_FOR_nothing
6493 && (sel == NULL
6494 || targetm.vectorize.vec_perm_const_ok == NULL
6495 || targetm.vectorize.vec_perm_const_ok (mode, sel)))
6496 return true;
6497 }
6498
6499 if (direct_optab_handler (vec_perm_optab, mode) != CODE_FOR_nothing)
6500 return true;
6501
6502 /* We allow fallback to a QI vector mode, and adjust the mask. */
6503 if (GET_MODE_INNER (mode) == QImode)
6504 return false;
6505 qimode = mode_for_vector (QImode, GET_MODE_SIZE (mode));
6506 if (!VECTOR_MODE_P (qimode))
6507 return false;
6508
6509 /* ??? For completeness, we ought to check the QImode version of
6510 vec_perm_const_optab. But all users of this implicit lowering
6511 feature implement the variable vec_perm_optab. */
6512 if (direct_optab_handler (vec_perm_optab, qimode) == CODE_FOR_nothing)
6513 return false;
6514
6515 /* In order to support the lowering of variable permutations,
6516 we need to support shifts and adds. */
6517 if (variable)
6518 {
6519 if (GET_MODE_UNIT_SIZE (mode) > 2
6520 && optab_handler (ashl_optab, mode) == CODE_FOR_nothing
6521 && optab_handler (vashl_optab, mode) == CODE_FOR_nothing)
6522 return false;
6523 if (optab_handler (add_optab, qimode) == CODE_FOR_nothing)
6524 return false;
6525 }
6526
6527 return true;
6528 }
6529
6530 /* A subroutine of expand_vec_perm for expanding one vec_perm insn. */
6531
6532 static rtx
6533 expand_vec_perm_1 (enum insn_code icode, rtx target,
6534 rtx v0, rtx v1, rtx sel)
6535 {
6536 enum machine_mode tmode = GET_MODE (target);
6537 enum machine_mode smode = GET_MODE (sel);
6538 struct expand_operand ops[4];
6539
6540 create_output_operand (&ops[0], target, tmode);
6541 create_input_operand (&ops[3], sel, smode);
6542
6543 /* Make an effort to preserve v0 == v1. The target expander is able to
6544 rely on this to determine if we're permuting a single input operand. */
6545 if (rtx_equal_p (v0, v1))
6546 {
6547 if (!insn_operand_matches (icode, 1, v0))
6548 v0 = force_reg (tmode, v0);
6549 gcc_checking_assert (insn_operand_matches (icode, 1, v0));
6550 gcc_checking_assert (insn_operand_matches (icode, 2, v0));
6551
6552 create_fixed_operand (&ops[1], v0);
6553 create_fixed_operand (&ops[2], v0);
6554 }
6555 else
6556 {
6557 create_input_operand (&ops[1], v0, tmode);
6558 create_input_operand (&ops[2], v1, tmode);
6559 }
6560
6561 if (maybe_expand_insn (icode, 4, ops))
6562 return ops[0].value;
6563 return NULL_RTX;
6564 }
6565
6566 /* Generate instructions for vec_perm optab given its mode
6567 and three operands. */
6568
6569 rtx
6570 expand_vec_perm (enum machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
6571 {
6572 enum insn_code icode;
6573 enum machine_mode qimode;
6574 unsigned int i, w, e, u;
6575 rtx tmp, sel_qi = NULL;
6576 rtvec vec;
6577
6578 if (!target || GET_MODE (target) != mode)
6579 target = gen_reg_rtx (mode);
6580
6581 w = GET_MODE_SIZE (mode);
6582 e = GET_MODE_NUNITS (mode);
6583 u = GET_MODE_UNIT_SIZE (mode);
6584
6585 /* Set QIMODE to a different vector mode with byte elements.
6586 If no such mode, or if MODE already has byte elements, use VOIDmode. */
6587 qimode = VOIDmode;
6588 if (GET_MODE_INNER (mode) != QImode)
6589 {
6590 qimode = mode_for_vector (QImode, w);
6591 if (!VECTOR_MODE_P (qimode))
6592 qimode = VOIDmode;
6593 }
6594
6595 /* If the input is a constant, expand it specially. */
6596 gcc_assert (GET_MODE_CLASS (GET_MODE (sel)) == MODE_VECTOR_INT);
6597 if (GET_CODE (sel) == CONST_VECTOR)
6598 {
6599 icode = direct_optab_handler (vec_perm_const_optab, mode);
6600 if (icode != CODE_FOR_nothing)
6601 {
6602 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6603 if (tmp)
6604 return tmp;
6605 }
6606
6607 /* Fall back to a constant byte-based permutation. */
6608 if (qimode != VOIDmode)
6609 {
6610 vec = rtvec_alloc (w);
6611 for (i = 0; i < e; ++i)
6612 {
6613 unsigned int j, this_e;
6614
6615 this_e = INTVAL (CONST_VECTOR_ELT (sel, i));
6616 this_e &= 2 * e - 1;
6617 this_e *= u;
6618
6619 for (j = 0; j < u; ++j)
6620 RTVEC_ELT (vec, i * u + j) = GEN_INT (this_e + j);
6621 }
6622 sel_qi = gen_rtx_CONST_VECTOR (qimode, vec);
6623
6624 icode = direct_optab_handler (vec_perm_const_optab, qimode);
6625 if (icode != CODE_FOR_nothing)
6626 {
6627 tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
6628 tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
6629 gen_lowpart (qimode, v1), sel_qi);
6630 if (tmp)
6631 return gen_lowpart (mode, tmp);
6632 }
6633 }
6634 }
6635
6636 /* Otherwise expand as a fully variable permuation. */
6637 icode = direct_optab_handler (vec_perm_optab, mode);
6638 if (icode != CODE_FOR_nothing)
6639 {
6640 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6641 if (tmp)
6642 return tmp;
6643 }
6644
6645 /* As a special case to aid several targets, lower the element-based
6646 permutation to a byte-based permutation and try again. */
6647 if (qimode == VOIDmode)
6648 return NULL_RTX;
6649 icode = direct_optab_handler (vec_perm_optab, qimode);
6650 if (icode == CODE_FOR_nothing)
6651 return NULL_RTX;
6652
6653 if (sel_qi == NULL)
6654 {
6655 /* Multiply each element by its byte size. */
6656 enum machine_mode selmode = GET_MODE (sel);
6657 if (u == 2)
6658 sel = expand_simple_binop (selmode, PLUS, sel, sel,
6659 sel, 0, OPTAB_DIRECT);
6660 else
6661 sel = expand_simple_binop (selmode, ASHIFT, sel,
6662 GEN_INT (exact_log2 (u)),
6663 sel, 0, OPTAB_DIRECT);
6664 gcc_assert (sel != NULL);
6665
6666 /* Broadcast the low byte each element into each of its bytes. */
6667 vec = rtvec_alloc (w);
6668 for (i = 0; i < w; ++i)
6669 {
6670 int this_e = i / u * u;
6671 if (BYTES_BIG_ENDIAN)
6672 this_e += u - 1;
6673 RTVEC_ELT (vec, i) = GEN_INT (this_e);
6674 }
6675 tmp = gen_rtx_CONST_VECTOR (qimode, vec);
6676 sel = gen_lowpart (qimode, sel);
6677 sel = expand_vec_perm (qimode, sel, sel, tmp, NULL);
6678 gcc_assert (sel != NULL);
6679
6680 /* Add the byte offset to each byte element. */
6681 /* Note that the definition of the indicies here is memory ordering,
6682 so there should be no difference between big and little endian. */
6683 vec = rtvec_alloc (w);
6684 for (i = 0; i < w; ++i)
6685 RTVEC_ELT (vec, i) = GEN_INT (i % u);
6686 tmp = gen_rtx_CONST_VECTOR (qimode, vec);
6687 sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
6688 sel, 0, OPTAB_DIRECT);
6689 gcc_assert (sel_qi != NULL);
6690 }
6691
6692 tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
6693 tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
6694 gen_lowpart (qimode, v1), sel_qi);
6695 if (tmp)
6696 tmp = gen_lowpart (mode, tmp);
6697 return tmp;
6698 }
6699
6700 /* Return insn code for a conditional operator with a comparison in
6701 mode CMODE, unsigned if UNS is true, resulting in a value of mode VMODE. */
6702
6703 static inline enum insn_code
6704 get_vcond_icode (enum machine_mode vmode, enum machine_mode cmode, bool uns)
6705 {
6706 enum insn_code icode = CODE_FOR_nothing;
6707 if (uns)
6708 icode = convert_optab_handler (vcondu_optab, vmode, cmode);
6709 else
6710 icode = convert_optab_handler (vcond_optab, vmode, cmode);
6711 return icode;
6712 }
6713
6714 /* Return TRUE iff, appropriate vector insns are available
6715 for vector cond expr with vector type VALUE_TYPE and a comparison
6716 with operand vector types in CMP_OP_TYPE. */
6717
6718 bool
6719 expand_vec_cond_expr_p (tree value_type, tree cmp_op_type)
6720 {
6721 enum machine_mode value_mode = TYPE_MODE (value_type);
6722 enum machine_mode cmp_op_mode = TYPE_MODE (cmp_op_type);
6723 if (GET_MODE_SIZE (value_mode) != GET_MODE_SIZE (cmp_op_mode)
6724 || GET_MODE_NUNITS (value_mode) != GET_MODE_NUNITS (cmp_op_mode)
6725 || get_vcond_icode (TYPE_MODE (value_type), TYPE_MODE (cmp_op_type),
6726 TYPE_UNSIGNED (cmp_op_type)) == CODE_FOR_nothing)
6727 return false;
6728 return true;
6729 }
6730
6731 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
6732 three operands. */
6733
6734 rtx
6735 expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
6736 rtx target)
6737 {
6738 struct expand_operand ops[6];
6739 enum insn_code icode;
6740 rtx comparison, rtx_op1, rtx_op2;
6741 enum machine_mode mode = TYPE_MODE (vec_cond_type);
6742 enum machine_mode cmp_op_mode;
6743 bool unsignedp;
6744 tree op0a, op0b;
6745 enum tree_code tcode;
6746
6747 if (COMPARISON_CLASS_P (op0))
6748 {
6749 op0a = TREE_OPERAND (op0, 0);
6750 op0b = TREE_OPERAND (op0, 1);
6751 tcode = TREE_CODE (op0);
6752 }
6753 else
6754 {
6755 /* Fake op0 < 0. */
6756 gcc_assert (!TYPE_UNSIGNED (TREE_TYPE (op0)));
6757 op0a = op0;
6758 op0b = build_zero_cst (TREE_TYPE (op0));
6759 tcode = LT_EXPR;
6760 }
6761 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
6762 cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
6763
6764
6765 gcc_assert (GET_MODE_SIZE (mode) == GET_MODE_SIZE (cmp_op_mode)
6766 && GET_MODE_NUNITS (mode) == GET_MODE_NUNITS (cmp_op_mode));
6767
6768 icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
6769 if (icode == CODE_FOR_nothing)
6770 return 0;
6771
6772 comparison = vector_compare_rtx (tcode, op0a, op0b, unsignedp, icode);
6773 rtx_op1 = expand_normal (op1);
6774 rtx_op2 = expand_normal (op2);
6775
6776 create_output_operand (&ops[0], target, mode);
6777 create_input_operand (&ops[1], rtx_op1, mode);
6778 create_input_operand (&ops[2], rtx_op2, mode);
6779 create_fixed_operand (&ops[3], comparison);
6780 create_fixed_operand (&ops[4], XEXP (comparison, 0));
6781 create_fixed_operand (&ops[5], XEXP (comparison, 1));
6782 expand_insn (icode, 6, ops);
6783 return ops[0].value;
6784 }
6785
6786 /* Return non-zero if a highpart multiply is supported of can be synthisized.
6787 For the benefit of expand_mult_highpart, the return value is 1 for direct,
6788 2 for even/odd widening, and 3 for hi/lo widening. */
6789
6790 int
6791 can_mult_highpart_p (enum machine_mode mode, bool uns_p)
6792 {
6793 optab op;
6794 unsigned char *sel;
6795 unsigned i, nunits;
6796
6797 op = uns_p ? umul_highpart_optab : smul_highpart_optab;
6798 if (optab_handler (op, mode) != CODE_FOR_nothing)
6799 return 1;
6800
6801 /* If the mode is an integral vector, synth from widening operations. */
6802 if (GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
6803 return 0;
6804
6805 nunits = GET_MODE_NUNITS (mode);
6806 sel = XALLOCAVEC (unsigned char, nunits);
6807
6808 op = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6809 if (optab_handler (op, mode) != CODE_FOR_nothing)
6810 {
6811 op = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6812 if (optab_handler (op, mode) != CODE_FOR_nothing)
6813 {
6814 for (i = 0; i < nunits; ++i)
6815 sel[i] = !BYTES_BIG_ENDIAN + (i & ~1) + ((i & 1) ? nunits : 0);
6816 if (can_vec_perm_p (mode, false, sel))
6817 return 2;
6818 }
6819 }
6820
6821 op = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6822 if (optab_handler (op, mode) != CODE_FOR_nothing)
6823 {
6824 op = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6825 if (optab_handler (op, mode) != CODE_FOR_nothing)
6826 {
6827 for (i = 0; i < nunits; ++i)
6828 sel[i] = 2 * i + (BYTES_BIG_ENDIAN ? 0 : 1);
6829 if (can_vec_perm_p (mode, false, sel))
6830 return 3;
6831 }
6832 }
6833
6834 return 0;
6835 }
6836
6837 /* Expand a highpart multiply. */
6838
6839 rtx
6840 expand_mult_highpart (enum machine_mode mode, rtx op0, rtx op1,
6841 rtx target, bool uns_p)
6842 {
6843 struct expand_operand eops[3];
6844 enum insn_code icode;
6845 int method, i, nunits;
6846 enum machine_mode wmode;
6847 rtx m1, m2, perm;
6848 optab tab1, tab2;
6849 rtvec v;
6850
6851 method = can_mult_highpart_p (mode, uns_p);
6852 switch (method)
6853 {
6854 case 0:
6855 return NULL_RTX;
6856 case 1:
6857 tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
6858 return expand_binop (mode, tab1, op0, op1, target, uns_p,
6859 OPTAB_LIB_WIDEN);
6860 case 2:
6861 tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6862 tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6863 break;
6864 case 3:
6865 tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6866 tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6867 if (BYTES_BIG_ENDIAN)
6868 {
6869 optab t = tab1;
6870 tab1 = tab2;
6871 tab2 = t;
6872 }
6873 break;
6874 default:
6875 gcc_unreachable ();
6876 }
6877
6878 icode = optab_handler (tab1, mode);
6879 nunits = GET_MODE_NUNITS (mode);
6880 wmode = insn_data[icode].operand[0].mode;
6881 gcc_checking_assert (2 * GET_MODE_NUNITS (wmode) == nunits);
6882 gcc_checking_assert (GET_MODE_SIZE (wmode) == GET_MODE_SIZE (mode));
6883
6884 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6885 create_input_operand (&eops[1], op0, mode);
6886 create_input_operand (&eops[2], op1, mode);
6887 expand_insn (icode, 3, eops);
6888 m1 = gen_lowpart (mode, eops[0].value);
6889
6890 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6891 create_input_operand (&eops[1], op0, mode);
6892 create_input_operand (&eops[2], op1, mode);
6893 expand_insn (optab_handler (tab2, mode), 3, eops);
6894 m2 = gen_lowpart (mode, eops[0].value);
6895
6896 v = rtvec_alloc (nunits);
6897 if (method == 2)
6898 {
6899 for (i = 0; i < nunits; ++i)
6900 RTVEC_ELT (v, i) = GEN_INT (!BYTES_BIG_ENDIAN + (i & ~1)
6901 + ((i & 1) ? nunits : 0));
6902 }
6903 else
6904 {
6905 for (i = 0; i < nunits; ++i)
6906 RTVEC_ELT (v, i) = GEN_INT (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
6907 }
6908 perm = gen_rtx_CONST_VECTOR (mode, v);
6909
6910 return expand_vec_perm (mode, m1, m2, perm, target);
6911 }
6912 \f
6913 /* Return true if there is a compare_and_swap pattern. */
6914
6915 bool
6916 can_compare_and_swap_p (enum machine_mode mode, bool allow_libcall)
6917 {
6918 enum insn_code icode;
6919
6920 /* Check for __atomic_compare_and_swap. */
6921 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
6922 if (icode != CODE_FOR_nothing)
6923 return true;
6924
6925 /* Check for __sync_compare_and_swap. */
6926 icode = optab_handler (sync_compare_and_swap_optab, mode);
6927 if (icode != CODE_FOR_nothing)
6928 return true;
6929 if (allow_libcall && optab_libfunc (sync_compare_and_swap_optab, mode))
6930 return true;
6931
6932 /* No inline compare and swap. */
6933 return false;
6934 }
6935
6936 /* Return true if an atomic exchange can be performed. */
6937
6938 bool
6939 can_atomic_exchange_p (enum machine_mode mode, bool allow_libcall)
6940 {
6941 enum insn_code icode;
6942
6943 /* Check for __atomic_exchange. */
6944 icode = direct_optab_handler (atomic_exchange_optab, mode);
6945 if (icode != CODE_FOR_nothing)
6946 return true;
6947
6948 /* Don't check __sync_test_and_set, as on some platforms that
6949 has reduced functionality. Targets that really do support
6950 a proper exchange should simply be updated to the __atomics. */
6951
6952 return can_compare_and_swap_p (mode, allow_libcall);
6953 }
6954
6955
6956 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
6957 pattern. */
6958
6959 static void
6960 find_cc_set (rtx x, const_rtx pat, void *data)
6961 {
6962 if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
6963 && GET_CODE (pat) == SET)
6964 {
6965 rtx *p_cc_reg = (rtx *) data;
6966 gcc_assert (!*p_cc_reg);
6967 *p_cc_reg = x;
6968 }
6969 }
6970
6971 /* This is a helper function for the other atomic operations. This function
6972 emits a loop that contains SEQ that iterates until a compare-and-swap
6973 operation at the end succeeds. MEM is the memory to be modified. SEQ is
6974 a set of instructions that takes a value from OLD_REG as an input and
6975 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
6976 set to the current contents of MEM. After SEQ, a compare-and-swap will
6977 attempt to update MEM with NEW_REG. The function returns true when the
6978 loop was generated successfully. */
6979
6980 static bool
6981 expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
6982 {
6983 enum machine_mode mode = GET_MODE (mem);
6984 rtx label, cmp_reg, success, oldval;
6985
6986 /* The loop we want to generate looks like
6987
6988 cmp_reg = mem;
6989 label:
6990 old_reg = cmp_reg;
6991 seq;
6992 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6993 if (success)
6994 goto label;
6995
6996 Note that we only do the plain load from memory once. Subsequent
6997 iterations use the value loaded by the compare-and-swap pattern. */
6998
6999 label = gen_label_rtx ();
7000 cmp_reg = gen_reg_rtx (mode);
7001
7002 emit_move_insn (cmp_reg, mem);
7003 emit_label (label);
7004 emit_move_insn (old_reg, cmp_reg);
7005 if (seq)
7006 emit_insn (seq);
7007
7008 success = NULL_RTX;
7009 oldval = cmp_reg;
7010 if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
7011 new_reg, false, MEMMODEL_SEQ_CST,
7012 MEMMODEL_RELAXED))
7013 return false;
7014
7015 if (oldval != cmp_reg)
7016 emit_move_insn (cmp_reg, oldval);
7017
7018 /* Mark this jump predicted not taken. */
7019 emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
7020 GET_MODE (success), 1, label, 0);
7021 return true;
7022 }
7023
7024
7025 /* This function tries to emit an atomic_exchange intruction. VAL is written
7026 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
7027 using TARGET if possible. */
7028
7029 static rtx
7030 maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7031 {
7032 enum machine_mode mode = GET_MODE (mem);
7033 enum insn_code icode;
7034
7035 /* If the target supports the exchange directly, great. */
7036 icode = direct_optab_handler (atomic_exchange_optab, mode);
7037 if (icode != CODE_FOR_nothing)
7038 {
7039 struct expand_operand ops[4];
7040
7041 create_output_operand (&ops[0], target, mode);
7042 create_fixed_operand (&ops[1], mem);
7043 create_input_operand (&ops[2], val, mode);
7044 create_integer_operand (&ops[3], model);
7045 if (maybe_expand_insn (icode, 4, ops))
7046 return ops[0].value;
7047 }
7048
7049 return NULL_RTX;
7050 }
7051
7052 /* This function tries to implement an atomic exchange operation using
7053 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
7054 The previous contents of *MEM are returned, using TARGET if possible.
7055 Since this instructionn is an acquire barrier only, stronger memory
7056 models may require additional barriers to be emitted. */
7057
7058 static rtx
7059 maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
7060 enum memmodel model)
7061 {
7062 enum machine_mode mode = GET_MODE (mem);
7063 enum insn_code icode;
7064 rtx last_insn = get_last_insn ();
7065
7066 icode = optab_handler (sync_lock_test_and_set_optab, mode);
7067
7068 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
7069 exists, and the memory model is stronger than acquire, add a release
7070 barrier before the instruction. */
7071
7072 if ((model & MEMMODEL_MASK) == MEMMODEL_SEQ_CST
7073 || (model & MEMMODEL_MASK) == MEMMODEL_RELEASE
7074 || (model & MEMMODEL_MASK) == MEMMODEL_ACQ_REL)
7075 expand_mem_thread_fence (model);
7076
7077 if (icode != CODE_FOR_nothing)
7078 {
7079 struct expand_operand ops[3];
7080 create_output_operand (&ops[0], target, mode);
7081 create_fixed_operand (&ops[1], mem);
7082 create_input_operand (&ops[2], val, mode);
7083 if (maybe_expand_insn (icode, 3, ops))
7084 return ops[0].value;
7085 }
7086
7087 /* If an external test-and-set libcall is provided, use that instead of
7088 any external compare-and-swap that we might get from the compare-and-
7089 swap-loop expansion later. */
7090 if (!can_compare_and_swap_p (mode, false))
7091 {
7092 rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
7093 if (libfunc != NULL)
7094 {
7095 rtx addr;
7096
7097 addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7098 return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7099 mode, 2, addr, ptr_mode,
7100 val, mode);
7101 }
7102 }
7103
7104 /* If the test_and_set can't be emitted, eliminate any barrier that might
7105 have been emitted. */
7106 delete_insns_since (last_insn);
7107 return NULL_RTX;
7108 }
7109
7110 /* This function tries to implement an atomic exchange operation using a
7111 compare_and_swap loop. VAL is written to *MEM. The previous contents of
7112 *MEM are returned, using TARGET if possible. No memory model is required
7113 since a compare_and_swap loop is seq-cst. */
7114
7115 static rtx
7116 maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
7117 {
7118 enum machine_mode mode = GET_MODE (mem);
7119
7120 if (can_compare_and_swap_p (mode, true))
7121 {
7122 if (!target || !register_operand (target, mode))
7123 target = gen_reg_rtx (mode);
7124 if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
7125 return target;
7126 }
7127
7128 return NULL_RTX;
7129 }
7130
7131 /* This function tries to implement an atomic test-and-set operation
7132 using the atomic_test_and_set instruction pattern. A boolean value
7133 is returned from the operation, using TARGET if possible. */
7134
7135 #ifndef HAVE_atomic_test_and_set
7136 #define HAVE_atomic_test_and_set 0
7137 #define CODE_FOR_atomic_test_and_set CODE_FOR_nothing
7138 #endif
7139
7140 static rtx
7141 maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7142 {
7143 enum machine_mode pat_bool_mode;
7144 struct expand_operand ops[3];
7145
7146 if (!HAVE_atomic_test_and_set)
7147 return NULL_RTX;
7148
7149 /* While we always get QImode from __atomic_test_and_set, we get
7150 other memory modes from __sync_lock_test_and_set. Note that we
7151 use no endian adjustment here. This matches the 4.6 behavior
7152 in the Sparc backend. */
7153 gcc_checking_assert
7154 (insn_data[CODE_FOR_atomic_test_and_set].operand[1].mode == QImode);
7155 if (GET_MODE (mem) != QImode)
7156 mem = adjust_address_nv (mem, QImode, 0);
7157
7158 pat_bool_mode = insn_data[CODE_FOR_atomic_test_and_set].operand[0].mode;
7159 create_output_operand (&ops[0], target, pat_bool_mode);
7160 create_fixed_operand (&ops[1], mem);
7161 create_integer_operand (&ops[2], model);
7162
7163 if (maybe_expand_insn (CODE_FOR_atomic_test_and_set, 3, ops))
7164 return ops[0].value;
7165 return NULL_RTX;
7166 }
7167
7168 /* This function expands the legacy _sync_lock test_and_set operation which is
7169 generally an atomic exchange. Some limited targets only allow the
7170 constant 1 to be stored. This is an ACQUIRE operation.
7171
7172 TARGET is an optional place to stick the return value.
7173 MEM is where VAL is stored. */
7174
7175 rtx
7176 expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
7177 {
7178 rtx ret;
7179
7180 /* Try an atomic_exchange first. */
7181 ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_ACQUIRE);
7182 if (ret)
7183 return ret;
7184
7185 ret = maybe_emit_sync_lock_test_and_set (target, mem, val, MEMMODEL_ACQUIRE);
7186 if (ret)
7187 return ret;
7188
7189 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7190 if (ret)
7191 return ret;
7192
7193 /* If there are no other options, try atomic_test_and_set if the value
7194 being stored is 1. */
7195 if (val == const1_rtx)
7196 ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_ACQUIRE);
7197
7198 return ret;
7199 }
7200
7201 /* This function expands the atomic test_and_set operation:
7202 atomically store a boolean TRUE into MEM and return the previous value.
7203
7204 MEMMODEL is the memory model variant to use.
7205 TARGET is an optional place to stick the return value. */
7206
7207 rtx
7208 expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7209 {
7210 enum machine_mode mode = GET_MODE (mem);
7211 rtx ret, trueval, subtarget;
7212
7213 ret = maybe_emit_atomic_test_and_set (target, mem, model);
7214 if (ret)
7215 return ret;
7216
7217 /* Be binary compatible with non-default settings of trueval, and different
7218 cpu revisions. E.g. one revision may have atomic-test-and-set, but
7219 another only has atomic-exchange. */
7220 if (targetm.atomic_test_and_set_trueval == 1)
7221 {
7222 trueval = const1_rtx;
7223 subtarget = target ? target : gen_reg_rtx (mode);
7224 }
7225 else
7226 {
7227 trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
7228 subtarget = gen_reg_rtx (mode);
7229 }
7230
7231 /* Try the atomic-exchange optab... */
7232 ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
7233
7234 /* ... then an atomic-compare-and-swap loop ... */
7235 if (!ret)
7236 ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
7237
7238 /* ... before trying the vaguely defined legacy lock_test_and_set. */
7239 if (!ret)
7240 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
7241
7242 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
7243 things with the value 1. Thus we try again without trueval. */
7244 if (!ret && targetm.atomic_test_and_set_trueval != 1)
7245 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
7246
7247 /* Failing all else, assume a single threaded environment and simply
7248 perform the operation. */
7249 if (!ret)
7250 {
7251 emit_move_insn (subtarget, mem);
7252 emit_move_insn (mem, trueval);
7253 ret = subtarget;
7254 }
7255
7256 /* Recall that have to return a boolean value; rectify if trueval
7257 is not exactly one. */
7258 if (targetm.atomic_test_and_set_trueval != 1)
7259 ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
7260
7261 return ret;
7262 }
7263
7264 /* This function expands the atomic exchange operation:
7265 atomically store VAL in MEM and return the previous value in MEM.
7266
7267 MEMMODEL is the memory model variant to use.
7268 TARGET is an optional place to stick the return value. */
7269
7270 rtx
7271 expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7272 {
7273 rtx ret;
7274
7275 ret = maybe_emit_atomic_exchange (target, mem, val, model);
7276
7277 /* Next try a compare-and-swap loop for the exchange. */
7278 if (!ret)
7279 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7280
7281 return ret;
7282 }
7283
7284 /* This function expands the atomic compare exchange operation:
7285
7286 *PTARGET_BOOL is an optional place to store the boolean success/failure.
7287 *PTARGET_OVAL is an optional place to store the old value from memory.
7288 Both target parameters may be NULL to indicate that we do not care about
7289 that return value. Both target parameters are updated on success to
7290 the actual location of the corresponding result.
7291
7292 MEMMODEL is the memory model variant to use.
7293
7294 The return value of the function is true for success. */
7295
7296 bool
7297 expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
7298 rtx mem, rtx expected, rtx desired,
7299 bool is_weak, enum memmodel succ_model,
7300 enum memmodel fail_model)
7301 {
7302 enum machine_mode mode = GET_MODE (mem);
7303 struct expand_operand ops[8];
7304 enum insn_code icode;
7305 rtx target_oval, target_bool = NULL_RTX;
7306 rtx libfunc;
7307
7308 /* Load expected into a register for the compare and swap. */
7309 if (MEM_P (expected))
7310 expected = copy_to_reg (expected);
7311
7312 /* Make sure we always have some place to put the return oldval.
7313 Further, make sure that place is distinct from the input expected,
7314 just in case we need that path down below. */
7315 if (ptarget_oval == NULL
7316 || (target_oval = *ptarget_oval) == NULL
7317 || reg_overlap_mentioned_p (expected, target_oval))
7318 target_oval = gen_reg_rtx (mode);
7319
7320 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
7321 if (icode != CODE_FOR_nothing)
7322 {
7323 enum machine_mode bool_mode = insn_data[icode].operand[0].mode;
7324
7325 /* Make sure we always have a place for the bool operand. */
7326 if (ptarget_bool == NULL
7327 || (target_bool = *ptarget_bool) == NULL
7328 || GET_MODE (target_bool) != bool_mode)
7329 target_bool = gen_reg_rtx (bool_mode);
7330
7331 /* Emit the compare_and_swap. */
7332 create_output_operand (&ops[0], target_bool, bool_mode);
7333 create_output_operand (&ops[1], target_oval, mode);
7334 create_fixed_operand (&ops[2], mem);
7335 create_input_operand (&ops[3], expected, mode);
7336 create_input_operand (&ops[4], desired, mode);
7337 create_integer_operand (&ops[5], is_weak);
7338 create_integer_operand (&ops[6], succ_model);
7339 create_integer_operand (&ops[7], fail_model);
7340 expand_insn (icode, 8, ops);
7341
7342 /* Return success/failure. */
7343 target_bool = ops[0].value;
7344 target_oval = ops[1].value;
7345 goto success;
7346 }
7347
7348 /* Otherwise fall back to the original __sync_val_compare_and_swap
7349 which is always seq-cst. */
7350 icode = optab_handler (sync_compare_and_swap_optab, mode);
7351 if (icode != CODE_FOR_nothing)
7352 {
7353 rtx cc_reg;
7354
7355 create_output_operand (&ops[0], target_oval, mode);
7356 create_fixed_operand (&ops[1], mem);
7357 create_input_operand (&ops[2], expected, mode);
7358 create_input_operand (&ops[3], desired, mode);
7359 if (!maybe_expand_insn (icode, 4, ops))
7360 return false;
7361
7362 target_oval = ops[0].value;
7363
7364 /* If the caller isn't interested in the boolean return value,
7365 skip the computation of it. */
7366 if (ptarget_bool == NULL)
7367 goto success;
7368
7369 /* Otherwise, work out if the compare-and-swap succeeded. */
7370 cc_reg = NULL_RTX;
7371 if (have_insn_for (COMPARE, CCmode))
7372 note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg);
7373 if (cc_reg)
7374 {
7375 target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
7376 const0_rtx, VOIDmode, 0, 1);
7377 goto success;
7378 }
7379 goto success_bool_from_val;
7380 }
7381
7382 /* Also check for library support for __sync_val_compare_and_swap. */
7383 libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
7384 if (libfunc != NULL)
7385 {
7386 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7387 target_oval = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7388 mode, 3, addr, ptr_mode,
7389 expected, mode, desired, mode);
7390
7391 /* Compute the boolean return value only if requested. */
7392 if (ptarget_bool)
7393 goto success_bool_from_val;
7394 else
7395 goto success;
7396 }
7397
7398 /* Failure. */
7399 return false;
7400
7401 success_bool_from_val:
7402 target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
7403 expected, VOIDmode, 1, 1);
7404 success:
7405 /* Make sure that the oval output winds up where the caller asked. */
7406 if (ptarget_oval)
7407 *ptarget_oval = target_oval;
7408 if (ptarget_bool)
7409 *ptarget_bool = target_bool;
7410 return true;
7411 }
7412
7413 /* Generate asm volatile("" : : : "memory") as the memory barrier. */
7414
7415 static void
7416 expand_asm_memory_barrier (void)
7417 {
7418 rtx asm_op, clob;
7419
7420 asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, empty_string, empty_string, 0,
7421 rtvec_alloc (0), rtvec_alloc (0),
7422 rtvec_alloc (0), UNKNOWN_LOCATION);
7423 MEM_VOLATILE_P (asm_op) = 1;
7424
7425 clob = gen_rtx_SCRATCH (VOIDmode);
7426 clob = gen_rtx_MEM (BLKmode, clob);
7427 clob = gen_rtx_CLOBBER (VOIDmode, clob);
7428
7429 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
7430 }
7431
7432 /* This routine will either emit the mem_thread_fence pattern or issue a
7433 sync_synchronize to generate a fence for memory model MEMMODEL. */
7434
7435 #ifndef HAVE_mem_thread_fence
7436 # define HAVE_mem_thread_fence 0
7437 # define gen_mem_thread_fence(x) (gcc_unreachable (), NULL_RTX)
7438 #endif
7439 #ifndef HAVE_memory_barrier
7440 # define HAVE_memory_barrier 0
7441 # define gen_memory_barrier() (gcc_unreachable (), NULL_RTX)
7442 #endif
7443
7444 void
7445 expand_mem_thread_fence (enum memmodel model)
7446 {
7447 if (HAVE_mem_thread_fence)
7448 emit_insn (gen_mem_thread_fence (GEN_INT (model)));
7449 else if ((model & MEMMODEL_MASK) != MEMMODEL_RELAXED)
7450 {
7451 if (HAVE_memory_barrier)
7452 emit_insn (gen_memory_barrier ());
7453 else if (synchronize_libfunc != NULL_RTX)
7454 emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode, 0);
7455 else
7456 expand_asm_memory_barrier ();
7457 }
7458 }
7459
7460 /* This routine will either emit the mem_signal_fence pattern or issue a
7461 sync_synchronize to generate a fence for memory model MEMMODEL. */
7462
7463 #ifndef HAVE_mem_signal_fence
7464 # define HAVE_mem_signal_fence 0
7465 # define gen_mem_signal_fence(x) (gcc_unreachable (), NULL_RTX)
7466 #endif
7467
7468 void
7469 expand_mem_signal_fence (enum memmodel model)
7470 {
7471 if (HAVE_mem_signal_fence)
7472 emit_insn (gen_mem_signal_fence (GEN_INT (model)));
7473 else if ((model & MEMMODEL_MASK) != MEMMODEL_RELAXED)
7474 {
7475 /* By default targets are coherent between a thread and the signal
7476 handler running on the same thread. Thus this really becomes a
7477 compiler barrier, in that stores must not be sunk past
7478 (or raised above) a given point. */
7479 expand_asm_memory_barrier ();
7480 }
7481 }
7482
7483 /* This function expands the atomic load operation:
7484 return the atomically loaded value in MEM.
7485
7486 MEMMODEL is the memory model variant to use.
7487 TARGET is an option place to stick the return value. */
7488
7489 rtx
7490 expand_atomic_load (rtx target, rtx mem, enum memmodel model)
7491 {
7492 enum machine_mode mode = GET_MODE (mem);
7493 enum insn_code icode;
7494
7495 /* If the target supports the load directly, great. */
7496 icode = direct_optab_handler (atomic_load_optab, mode);
7497 if (icode != CODE_FOR_nothing)
7498 {
7499 struct expand_operand ops[3];
7500
7501 create_output_operand (&ops[0], target, mode);
7502 create_fixed_operand (&ops[1], mem);
7503 create_integer_operand (&ops[2], model);
7504 if (maybe_expand_insn (icode, 3, ops))
7505 return ops[0].value;
7506 }
7507
7508 /* If the size of the object is greater than word size on this target,
7509 then we assume that a load will not be atomic. */
7510 if (GET_MODE_PRECISION (mode) > BITS_PER_WORD)
7511 {
7512 /* Issue val = compare_and_swap (mem, 0, 0).
7513 This may cause the occasional harmless store of 0 when the value is
7514 already 0, but it seems to be OK according to the standards guys. */
7515 if (expand_atomic_compare_and_swap (NULL, &target, mem, const0_rtx,
7516 const0_rtx, false, model, model))
7517 return target;
7518 else
7519 /* Otherwise there is no atomic load, leave the library call. */
7520 return NULL_RTX;
7521 }
7522
7523 /* Otherwise assume loads are atomic, and emit the proper barriers. */
7524 if (!target || target == const0_rtx)
7525 target = gen_reg_rtx (mode);
7526
7527 /* For SEQ_CST, emit a barrier before the load. */
7528 if ((model & MEMMODEL_MASK) == MEMMODEL_SEQ_CST)
7529 expand_mem_thread_fence (model);
7530
7531 emit_move_insn (target, mem);
7532
7533 /* Emit the appropriate barrier after the load. */
7534 expand_mem_thread_fence (model);
7535
7536 return target;
7537 }
7538
7539 /* This function expands the atomic store operation:
7540 Atomically store VAL in MEM.
7541 MEMMODEL is the memory model variant to use.
7542 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7543 function returns const0_rtx if a pattern was emitted. */
7544
7545 rtx
7546 expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
7547 {
7548 enum machine_mode mode = GET_MODE (mem);
7549 enum insn_code icode;
7550 struct expand_operand ops[3];
7551
7552 /* If the target supports the store directly, great. */
7553 icode = direct_optab_handler (atomic_store_optab, mode);
7554 if (icode != CODE_FOR_nothing)
7555 {
7556 create_fixed_operand (&ops[0], mem);
7557 create_input_operand (&ops[1], val, mode);
7558 create_integer_operand (&ops[2], model);
7559 if (maybe_expand_insn (icode, 3, ops))
7560 return const0_rtx;
7561 }
7562
7563 /* If using __sync_lock_release is a viable alternative, try it. */
7564 if (use_release)
7565 {
7566 icode = direct_optab_handler (sync_lock_release_optab, mode);
7567 if (icode != CODE_FOR_nothing)
7568 {
7569 create_fixed_operand (&ops[0], mem);
7570 create_input_operand (&ops[1], const0_rtx, mode);
7571 if (maybe_expand_insn (icode, 2, ops))
7572 {
7573 /* lock_release is only a release barrier. */
7574 if ((model & MEMMODEL_MASK) == MEMMODEL_SEQ_CST)
7575 expand_mem_thread_fence (model);
7576 return const0_rtx;
7577 }
7578 }
7579 }
7580
7581 /* If the size of the object is greater than word size on this target,
7582 a default store will not be atomic, Try a mem_exchange and throw away
7583 the result. If that doesn't work, don't do anything. */
7584 if (GET_MODE_PRECISION (mode) > BITS_PER_WORD)
7585 {
7586 rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
7587 if (!target)
7588 target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem, val);
7589 if (target)
7590 return const0_rtx;
7591 else
7592 return NULL_RTX;
7593 }
7594
7595 /* Otherwise assume stores are atomic, and emit the proper barriers. */
7596 expand_mem_thread_fence (model);
7597
7598 emit_move_insn (mem, val);
7599
7600 /* For SEQ_CST, also emit a barrier after the store. */
7601 if ((model & MEMMODEL_MASK) == MEMMODEL_SEQ_CST)
7602 expand_mem_thread_fence (model);
7603
7604 return const0_rtx;
7605 }
7606
7607
7608 /* Structure containing the pointers and values required to process the
7609 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7610
7611 struct atomic_op_functions
7612 {
7613 direct_optab mem_fetch_before;
7614 direct_optab mem_fetch_after;
7615 direct_optab mem_no_result;
7616 optab fetch_before;
7617 optab fetch_after;
7618 direct_optab no_result;
7619 enum rtx_code reverse_code;
7620 };
7621
7622
7623 /* Fill in structure pointed to by OP with the various optab entries for an
7624 operation of type CODE. */
7625
7626 static void
7627 get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
7628 {
7629 gcc_assert (op!= NULL);
7630
7631 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7632 in the source code during compilation, and the optab entries are not
7633 computable until runtime. Fill in the values at runtime. */
7634 switch (code)
7635 {
7636 case PLUS:
7637 op->mem_fetch_before = atomic_fetch_add_optab;
7638 op->mem_fetch_after = atomic_add_fetch_optab;
7639 op->mem_no_result = atomic_add_optab;
7640 op->fetch_before = sync_old_add_optab;
7641 op->fetch_after = sync_new_add_optab;
7642 op->no_result = sync_add_optab;
7643 op->reverse_code = MINUS;
7644 break;
7645 case MINUS:
7646 op->mem_fetch_before = atomic_fetch_sub_optab;
7647 op->mem_fetch_after = atomic_sub_fetch_optab;
7648 op->mem_no_result = atomic_sub_optab;
7649 op->fetch_before = sync_old_sub_optab;
7650 op->fetch_after = sync_new_sub_optab;
7651 op->no_result = sync_sub_optab;
7652 op->reverse_code = PLUS;
7653 break;
7654 case XOR:
7655 op->mem_fetch_before = atomic_fetch_xor_optab;
7656 op->mem_fetch_after = atomic_xor_fetch_optab;
7657 op->mem_no_result = atomic_xor_optab;
7658 op->fetch_before = sync_old_xor_optab;
7659 op->fetch_after = sync_new_xor_optab;
7660 op->no_result = sync_xor_optab;
7661 op->reverse_code = XOR;
7662 break;
7663 case AND:
7664 op->mem_fetch_before = atomic_fetch_and_optab;
7665 op->mem_fetch_after = atomic_and_fetch_optab;
7666 op->mem_no_result = atomic_and_optab;
7667 op->fetch_before = sync_old_and_optab;
7668 op->fetch_after = sync_new_and_optab;
7669 op->no_result = sync_and_optab;
7670 op->reverse_code = UNKNOWN;
7671 break;
7672 case IOR:
7673 op->mem_fetch_before = atomic_fetch_or_optab;
7674 op->mem_fetch_after = atomic_or_fetch_optab;
7675 op->mem_no_result = atomic_or_optab;
7676 op->fetch_before = sync_old_ior_optab;
7677 op->fetch_after = sync_new_ior_optab;
7678 op->no_result = sync_ior_optab;
7679 op->reverse_code = UNKNOWN;
7680 break;
7681 case NOT:
7682 op->mem_fetch_before = atomic_fetch_nand_optab;
7683 op->mem_fetch_after = atomic_nand_fetch_optab;
7684 op->mem_no_result = atomic_nand_optab;
7685 op->fetch_before = sync_old_nand_optab;
7686 op->fetch_after = sync_new_nand_optab;
7687 op->no_result = sync_nand_optab;
7688 op->reverse_code = UNKNOWN;
7689 break;
7690 default:
7691 gcc_unreachable ();
7692 }
7693 }
7694
7695 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7696 using memory order MODEL. If AFTER is true the operation needs to return
7697 the value of *MEM after the operation, otherwise the previous value.
7698 TARGET is an optional place to place the result. The result is unused if
7699 it is const0_rtx.
7700 Return the result if there is a better sequence, otherwise NULL_RTX. */
7701
7702 static rtx
7703 maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7704 enum memmodel model, bool after)
7705 {
7706 /* If the value is prefetched, or not used, it may be possible to replace
7707 the sequence with a native exchange operation. */
7708 if (!after || target == const0_rtx)
7709 {
7710 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7711 if (code == AND && val == const0_rtx)
7712 {
7713 if (target == const0_rtx)
7714 target = gen_reg_rtx (GET_MODE (mem));
7715 return maybe_emit_atomic_exchange (target, mem, val, model);
7716 }
7717
7718 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7719 if (code == IOR && val == constm1_rtx)
7720 {
7721 if (target == const0_rtx)
7722 target = gen_reg_rtx (GET_MODE (mem));
7723 return maybe_emit_atomic_exchange (target, mem, val, model);
7724 }
7725 }
7726
7727 return NULL_RTX;
7728 }
7729
7730 /* Try to emit an instruction for a specific operation varaition.
7731 OPTAB contains the OP functions.
7732 TARGET is an optional place to return the result. const0_rtx means unused.
7733 MEM is the memory location to operate on.
7734 VAL is the value to use in the operation.
7735 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7736 MODEL is the memory model, if used.
7737 AFTER is true if the returned result is the value after the operation. */
7738
7739 static rtx
7740 maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
7741 rtx val, bool use_memmodel, enum memmodel model, bool after)
7742 {
7743 enum machine_mode mode = GET_MODE (mem);
7744 struct expand_operand ops[4];
7745 enum insn_code icode;
7746 int op_counter = 0;
7747 int num_ops;
7748
7749 /* Check to see if there is a result returned. */
7750 if (target == const0_rtx)
7751 {
7752 if (use_memmodel)
7753 {
7754 icode = direct_optab_handler (optab->mem_no_result, mode);
7755 create_integer_operand (&ops[2], model);
7756 num_ops = 3;
7757 }
7758 else
7759 {
7760 icode = direct_optab_handler (optab->no_result, mode);
7761 num_ops = 2;
7762 }
7763 }
7764 /* Otherwise, we need to generate a result. */
7765 else
7766 {
7767 if (use_memmodel)
7768 {
7769 icode = direct_optab_handler (after ? optab->mem_fetch_after
7770 : optab->mem_fetch_before, mode);
7771 create_integer_operand (&ops[3], model);
7772 num_ops = 4;
7773 }
7774 else
7775 {
7776 icode = optab_handler (after ? optab->fetch_after
7777 : optab->fetch_before, mode);
7778 num_ops = 3;
7779 }
7780 create_output_operand (&ops[op_counter++], target, mode);
7781 }
7782 if (icode == CODE_FOR_nothing)
7783 return NULL_RTX;
7784
7785 create_fixed_operand (&ops[op_counter++], mem);
7786 /* VAL may have been promoted to a wider mode. Shrink it if so. */
7787 create_convert_operand_to (&ops[op_counter++], val, mode, true);
7788
7789 if (maybe_expand_insn (icode, num_ops, ops))
7790 return (target == const0_rtx ? const0_rtx : ops[0].value);
7791
7792 return NULL_RTX;
7793 }
7794
7795
7796 /* This function expands an atomic fetch_OP or OP_fetch operation:
7797 TARGET is an option place to stick the return value. const0_rtx indicates
7798 the result is unused.
7799 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7800 CODE is the operation being performed (OP)
7801 MEMMODEL is the memory model variant to use.
7802 AFTER is true to return the result of the operation (OP_fetch).
7803 AFTER is false to return the value before the operation (fetch_OP).
7804
7805 This function will *only* generate instructions if there is a direct
7806 optab. No compare and swap loops or libcalls will be generated. */
7807
7808 static rtx
7809 expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
7810 enum rtx_code code, enum memmodel model,
7811 bool after)
7812 {
7813 enum machine_mode mode = GET_MODE (mem);
7814 struct atomic_op_functions optab;
7815 rtx result;
7816 bool unused_result = (target == const0_rtx);
7817
7818 get_atomic_op_for_code (&optab, code);
7819
7820 /* Check to see if there are any better instructions. */
7821 result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
7822 if (result)
7823 return result;
7824
7825 /* Check for the case where the result isn't used and try those patterns. */
7826 if (unused_result)
7827 {
7828 /* Try the memory model variant first. */
7829 result = maybe_emit_op (&optab, target, mem, val, true, model, true);
7830 if (result)
7831 return result;
7832
7833 /* Next try the old style withuot a memory model. */
7834 result = maybe_emit_op (&optab, target, mem, val, false, model, true);
7835 if (result)
7836 return result;
7837
7838 /* There is no no-result pattern, so try patterns with a result. */
7839 target = NULL_RTX;
7840 }
7841
7842 /* Try the __atomic version. */
7843 result = maybe_emit_op (&optab, target, mem, val, true, model, after);
7844 if (result)
7845 return result;
7846
7847 /* Try the older __sync version. */
7848 result = maybe_emit_op (&optab, target, mem, val, false, model, after);
7849 if (result)
7850 return result;
7851
7852 /* If the fetch value can be calculated from the other variation of fetch,
7853 try that operation. */
7854 if (after || unused_result || optab.reverse_code != UNKNOWN)
7855 {
7856 /* Try the __atomic version, then the older __sync version. */
7857 result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
7858 if (!result)
7859 result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
7860
7861 if (result)
7862 {
7863 /* If the result isn't used, no need to do compensation code. */
7864 if (unused_result)
7865 return result;
7866
7867 /* Issue compensation code. Fetch_after == fetch_before OP val.
7868 Fetch_before == after REVERSE_OP val. */
7869 if (!after)
7870 code = optab.reverse_code;
7871 if (code == NOT)
7872 {
7873 result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
7874 true, OPTAB_LIB_WIDEN);
7875 result = expand_simple_unop (mode, NOT, result, target, true);
7876 }
7877 else
7878 result = expand_simple_binop (mode, code, result, val, target,
7879 true, OPTAB_LIB_WIDEN);
7880 return result;
7881 }
7882 }
7883
7884 /* No direct opcode can be generated. */
7885 return NULL_RTX;
7886 }
7887
7888
7889
7890 /* This function expands an atomic fetch_OP or OP_fetch operation:
7891 TARGET is an option place to stick the return value. const0_rtx indicates
7892 the result is unused.
7893 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7894 CODE is the operation being performed (OP)
7895 MEMMODEL is the memory model variant to use.
7896 AFTER is true to return the result of the operation (OP_fetch).
7897 AFTER is false to return the value before the operation (fetch_OP). */
7898 rtx
7899 expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7900 enum memmodel model, bool after)
7901 {
7902 enum machine_mode mode = GET_MODE (mem);
7903 rtx result;
7904 bool unused_result = (target == const0_rtx);
7905
7906 result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
7907 after);
7908
7909 if (result)
7910 return result;
7911
7912 /* Add/sub can be implemented by doing the reverse operation with -(val). */
7913 if (code == PLUS || code == MINUS)
7914 {
7915 rtx tmp;
7916 enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
7917
7918 start_sequence ();
7919 tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
7920 result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
7921 model, after);
7922 if (result)
7923 {
7924 /* PLUS worked so emit the insns and return. */
7925 tmp = get_insns ();
7926 end_sequence ();
7927 emit_insn (tmp);
7928 return result;
7929 }
7930
7931 /* PLUS did not work, so throw away the negation code and continue. */
7932 end_sequence ();
7933 }
7934
7935 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
7936 if (!can_compare_and_swap_p (mode, false))
7937 {
7938 rtx libfunc;
7939 bool fixup = false;
7940 enum rtx_code orig_code = code;
7941 struct atomic_op_functions optab;
7942
7943 get_atomic_op_for_code (&optab, code);
7944 libfunc = optab_libfunc (after ? optab.fetch_after
7945 : optab.fetch_before, mode);
7946 if (libfunc == NULL
7947 && (after || unused_result || optab.reverse_code != UNKNOWN))
7948 {
7949 fixup = true;
7950 if (!after)
7951 code = optab.reverse_code;
7952 libfunc = optab_libfunc (after ? optab.fetch_before
7953 : optab.fetch_after, mode);
7954 }
7955 if (libfunc != NULL)
7956 {
7957 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7958 result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
7959 2, addr, ptr_mode, val, mode);
7960
7961 if (!unused_result && fixup)
7962 result = expand_simple_binop (mode, code, result, val, target,
7963 true, OPTAB_LIB_WIDEN);
7964 return result;
7965 }
7966
7967 /* We need the original code for any further attempts. */
7968 code = orig_code;
7969 }
7970
7971 /* If nothing else has succeeded, default to a compare and swap loop. */
7972 if (can_compare_and_swap_p (mode, true))
7973 {
7974 rtx insn;
7975 rtx t0 = gen_reg_rtx (mode), t1;
7976
7977 start_sequence ();
7978
7979 /* If the result is used, get a register for it. */
7980 if (!unused_result)
7981 {
7982 if (!target || !register_operand (target, mode))
7983 target = gen_reg_rtx (mode);
7984 /* If fetch_before, copy the value now. */
7985 if (!after)
7986 emit_move_insn (target, t0);
7987 }
7988 else
7989 target = const0_rtx;
7990
7991 t1 = t0;
7992 if (code == NOT)
7993 {
7994 t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
7995 true, OPTAB_LIB_WIDEN);
7996 t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
7997 }
7998 else
7999 t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
8000 OPTAB_LIB_WIDEN);
8001
8002 /* For after, copy the value now. */
8003 if (!unused_result && after)
8004 emit_move_insn (target, t1);
8005 insn = get_insns ();
8006 end_sequence ();
8007
8008 if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
8009 return target;
8010 }
8011
8012 return NULL_RTX;
8013 }
8014 \f
8015 /* Return true if OPERAND is suitable for operand number OPNO of
8016 instruction ICODE. */
8017
8018 bool
8019 insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
8020 {
8021 return (!insn_data[(int) icode].operand[opno].predicate
8022 || (insn_data[(int) icode].operand[opno].predicate
8023 (operand, insn_data[(int) icode].operand[opno].mode)));
8024 }
8025 \f
8026 /* TARGET is a target of a multiword operation that we are going to
8027 implement as a series of word-mode operations. Return true if
8028 TARGET is suitable for this purpose. */
8029
8030 bool
8031 valid_multiword_target_p (rtx target)
8032 {
8033 enum machine_mode mode;
8034 int i;
8035
8036 mode = GET_MODE (target);
8037 for (i = 0; i < GET_MODE_SIZE (mode); i += UNITS_PER_WORD)
8038 if (!validate_subreg (word_mode, mode, target, i))
8039 return false;
8040 return true;
8041 }
8042
8043 /* Like maybe_legitimize_operand, but do not change the code of the
8044 current rtx value. */
8045
8046 static bool
8047 maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
8048 struct expand_operand *op)
8049 {
8050 /* See if the operand matches in its current form. */
8051 if (insn_operand_matches (icode, opno, op->value))
8052 return true;
8053
8054 /* If the operand is a memory whose address has no side effects,
8055 try forcing the address into a non-virtual pseudo register.
8056 The check for side effects is important because copy_to_mode_reg
8057 cannot handle things like auto-modified addresses. */
8058 if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
8059 {
8060 rtx addr, mem;
8061
8062 mem = op->value;
8063 addr = XEXP (mem, 0);
8064 if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
8065 && !side_effects_p (addr))
8066 {
8067 rtx last;
8068 enum machine_mode mode;
8069
8070 last = get_last_insn ();
8071 mode = get_address_mode (mem);
8072 mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
8073 if (insn_operand_matches (icode, opno, mem))
8074 {
8075 op->value = mem;
8076 return true;
8077 }
8078 delete_insns_since (last);
8079 }
8080 }
8081
8082 return false;
8083 }
8084
8085 /* Try to make OP match operand OPNO of instruction ICODE. Return true
8086 on success, storing the new operand value back in OP. */
8087
8088 static bool
8089 maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
8090 struct expand_operand *op)
8091 {
8092 enum machine_mode mode, imode;
8093 bool old_volatile_ok, result;
8094
8095 mode = op->mode;
8096 switch (op->type)
8097 {
8098 case EXPAND_FIXED:
8099 old_volatile_ok = volatile_ok;
8100 volatile_ok = true;
8101 result = maybe_legitimize_operand_same_code (icode, opno, op);
8102 volatile_ok = old_volatile_ok;
8103 return result;
8104
8105 case EXPAND_OUTPUT:
8106 gcc_assert (mode != VOIDmode);
8107 if (op->value
8108 && op->value != const0_rtx
8109 && GET_MODE (op->value) == mode
8110 && maybe_legitimize_operand_same_code (icode, opno, op))
8111 return true;
8112
8113 op->value = gen_reg_rtx (mode);
8114 break;
8115
8116 case EXPAND_INPUT:
8117 input:
8118 gcc_assert (mode != VOIDmode);
8119 gcc_assert (GET_MODE (op->value) == VOIDmode
8120 || GET_MODE (op->value) == mode);
8121 if (maybe_legitimize_operand_same_code (icode, opno, op))
8122 return true;
8123
8124 op->value = copy_to_mode_reg (mode, op->value);
8125 break;
8126
8127 case EXPAND_CONVERT_TO:
8128 gcc_assert (mode != VOIDmode);
8129 op->value = convert_to_mode (mode, op->value, op->unsigned_p);
8130 goto input;
8131
8132 case EXPAND_CONVERT_FROM:
8133 if (GET_MODE (op->value) != VOIDmode)
8134 mode = GET_MODE (op->value);
8135 else
8136 /* The caller must tell us what mode this value has. */
8137 gcc_assert (mode != VOIDmode);
8138
8139 imode = insn_data[(int) icode].operand[opno].mode;
8140 if (imode != VOIDmode && imode != mode)
8141 {
8142 op->value = convert_modes (imode, mode, op->value, op->unsigned_p);
8143 mode = imode;
8144 }
8145 goto input;
8146
8147 case EXPAND_ADDRESS:
8148 gcc_assert (mode != VOIDmode);
8149 op->value = convert_memory_address (mode, op->value);
8150 goto input;
8151
8152 case EXPAND_INTEGER:
8153 mode = insn_data[(int) icode].operand[opno].mode;
8154 if (mode != VOIDmode && const_int_operand (op->value, mode))
8155 goto input;
8156 break;
8157 }
8158 return insn_operand_matches (icode, opno, op->value);
8159 }
8160
8161 /* Make OP describe an input operand that should have the same value
8162 as VALUE, after any mode conversion that the target might request.
8163 TYPE is the type of VALUE. */
8164
8165 void
8166 create_convert_operand_from_type (struct expand_operand *op,
8167 rtx value, tree type)
8168 {
8169 create_convert_operand_from (op, value, TYPE_MODE (type),
8170 TYPE_UNSIGNED (type));
8171 }
8172
8173 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8174 of instruction ICODE. Return true on success, leaving the new operand
8175 values in the OPS themselves. Emit no code on failure. */
8176
8177 bool
8178 maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
8179 unsigned int nops, struct expand_operand *ops)
8180 {
8181 rtx last;
8182 unsigned int i;
8183
8184 last = get_last_insn ();
8185 for (i = 0; i < nops; i++)
8186 if (!maybe_legitimize_operand (icode, opno + i, &ops[i]))
8187 {
8188 delete_insns_since (last);
8189 return false;
8190 }
8191 return true;
8192 }
8193
8194 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8195 as its operands. Return the instruction pattern on success,
8196 and emit any necessary set-up code. Return null and emit no
8197 code on failure. */
8198
8199 rtx
8200 maybe_gen_insn (enum insn_code icode, unsigned int nops,
8201 struct expand_operand *ops)
8202 {
8203 gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
8204 if (!maybe_legitimize_operands (icode, 0, nops, ops))
8205 return NULL_RTX;
8206
8207 switch (nops)
8208 {
8209 case 1:
8210 return GEN_FCN (icode) (ops[0].value);
8211 case 2:
8212 return GEN_FCN (icode) (ops[0].value, ops[1].value);
8213 case 3:
8214 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
8215 case 4:
8216 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8217 ops[3].value);
8218 case 5:
8219 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8220 ops[3].value, ops[4].value);
8221 case 6:
8222 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8223 ops[3].value, ops[4].value, ops[5].value);
8224 case 7:
8225 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8226 ops[3].value, ops[4].value, ops[5].value,
8227 ops[6].value);
8228 case 8:
8229 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8230 ops[3].value, ops[4].value, ops[5].value,
8231 ops[6].value, ops[7].value);
8232 case 9:
8233 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8234 ops[3].value, ops[4].value, ops[5].value,
8235 ops[6].value, ops[7].value, ops[8].value);
8236 }
8237 gcc_unreachable ();
8238 }
8239
8240 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8241 as its operands. Return true on success and emit no code on failure. */
8242
8243 bool
8244 maybe_expand_insn (enum insn_code icode, unsigned int nops,
8245 struct expand_operand *ops)
8246 {
8247 rtx pat = maybe_gen_insn (icode, nops, ops);
8248 if (pat)
8249 {
8250 emit_insn (pat);
8251 return true;
8252 }
8253 return false;
8254 }
8255
8256 /* Like maybe_expand_insn, but for jumps. */
8257
8258 bool
8259 maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
8260 struct expand_operand *ops)
8261 {
8262 rtx pat = maybe_gen_insn (icode, nops, ops);
8263 if (pat)
8264 {
8265 emit_jump_insn (pat);
8266 return true;
8267 }
8268 return false;
8269 }
8270
8271 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8272 as its operands. */
8273
8274 void
8275 expand_insn (enum insn_code icode, unsigned int nops,
8276 struct expand_operand *ops)
8277 {
8278 if (!maybe_expand_insn (icode, nops, ops))
8279 gcc_unreachable ();
8280 }
8281
8282 /* Like expand_insn, but for jumps. */
8283
8284 void
8285 expand_jump_insn (enum insn_code icode, unsigned int nops,
8286 struct expand_operand *ops)
8287 {
8288 if (!maybe_expand_jump_insn (icode, nops, ops))
8289 gcc_unreachable ();
8290 }
8291
8292 /* Reduce conditional compilation elsewhere. */
8293 #ifndef HAVE_insv
8294 #define HAVE_insv 0
8295 #define CODE_FOR_insv CODE_FOR_nothing
8296 #endif
8297 #ifndef HAVE_extv
8298 #define HAVE_extv 0
8299 #define CODE_FOR_extv CODE_FOR_nothing
8300 #endif
8301 #ifndef HAVE_extzv
8302 #define HAVE_extzv 0
8303 #define CODE_FOR_extzv CODE_FOR_nothing
8304 #endif
8305
8306 /* Enumerates the possible types of structure operand to an
8307 extraction_insn. */
8308 enum extraction_type { ET_unaligned_mem, ET_reg };
8309
8310 /* Check whether insv, extv or extzv pattern ICODE can be used for an
8311 insertion or extraction of type TYPE on a structure of mode MODE.
8312 Return true if so and fill in *INSN accordingly. STRUCT_OP is the
8313 operand number of the structure (the first sign_extract or zero_extract
8314 operand) and FIELD_OP is the operand number of the field (the other
8315 side of the set from the sign_extract or zero_extract). */
8316
8317 static bool
8318 get_traditional_extraction_insn (extraction_insn *insn,
8319 enum extraction_type type,
8320 enum machine_mode mode,
8321 enum insn_code icode,
8322 int struct_op, int field_op)
8323 {
8324 const struct insn_data_d *data = &insn_data[icode];
8325
8326 enum machine_mode struct_mode = data->operand[struct_op].mode;
8327 if (struct_mode == VOIDmode)
8328 struct_mode = word_mode;
8329 if (mode != struct_mode)
8330 return false;
8331
8332 enum machine_mode field_mode = data->operand[field_op].mode;
8333 if (field_mode == VOIDmode)
8334 field_mode = word_mode;
8335
8336 enum machine_mode pos_mode = data->operand[struct_op + 2].mode;
8337 if (pos_mode == VOIDmode)
8338 pos_mode = word_mode;
8339
8340 insn->icode = icode;
8341 insn->field_mode = field_mode;
8342 insn->struct_mode = (type == ET_unaligned_mem ? byte_mode : struct_mode);
8343 insn->pos_mode = pos_mode;
8344 return true;
8345 }
8346
8347 /* Return true if an optab exists to perform an insertion or extraction
8348 of type TYPE in mode MODE. Describe the instruction in *INSN if so.
8349
8350 REG_OPTAB is the optab to use for register structures and
8351 MISALIGN_OPTAB is the optab to use for misaligned memory structures.
8352 POS_OP is the operand number of the bit position. */
8353
8354 static bool
8355 get_optab_extraction_insn (struct extraction_insn *insn,
8356 enum extraction_type type,
8357 enum machine_mode mode, direct_optab reg_optab,
8358 direct_optab misalign_optab, int pos_op)
8359 {
8360 direct_optab optab = (type == ET_unaligned_mem ? misalign_optab : reg_optab);
8361 enum insn_code icode = direct_optab_handler (optab, mode);
8362 if (icode == CODE_FOR_nothing)
8363 return false;
8364
8365 const struct insn_data_d *data = &insn_data[icode];
8366
8367 insn->icode = icode;
8368 insn->field_mode = mode;
8369 insn->struct_mode = (type == ET_unaligned_mem ? BLKmode : mode);
8370 insn->pos_mode = data->operand[pos_op].mode;
8371 if (insn->pos_mode == VOIDmode)
8372 insn->pos_mode = word_mode;
8373 return true;
8374 }
8375
8376 /* Return true if an instruction exists to perform an insertion or
8377 extraction (PATTERN says which) of type TYPE in mode MODE.
8378 Describe the instruction in *INSN if so. */
8379
8380 static bool
8381 get_extraction_insn (extraction_insn *insn,
8382 enum extraction_pattern pattern,
8383 enum extraction_type type,
8384 enum machine_mode mode)
8385 {
8386 switch (pattern)
8387 {
8388 case EP_insv:
8389 if (HAVE_insv
8390 && get_traditional_extraction_insn (insn, type, mode,
8391 CODE_FOR_insv, 0, 3))
8392 return true;
8393 return get_optab_extraction_insn (insn, type, mode, insv_optab,
8394 insvmisalign_optab, 2);
8395
8396 case EP_extv:
8397 if (HAVE_extv
8398 && get_traditional_extraction_insn (insn, type, mode,
8399 CODE_FOR_extv, 1, 0))
8400 return true;
8401 return get_optab_extraction_insn (insn, type, mode, extv_optab,
8402 extvmisalign_optab, 3);
8403
8404 case EP_extzv:
8405 if (HAVE_extzv
8406 && get_traditional_extraction_insn (insn, type, mode,
8407 CODE_FOR_extzv, 1, 0))
8408 return true;
8409 return get_optab_extraction_insn (insn, type, mode, extzv_optab,
8410 extzvmisalign_optab, 3);
8411
8412 default:
8413 gcc_unreachable ();
8414 }
8415 }
8416
8417 /* Return true if an instruction exists to access a field of mode
8418 FIELDMODE in a structure that has STRUCT_BITS significant bits.
8419 Describe the "best" such instruction in *INSN if so. PATTERN and
8420 TYPE describe the type of insertion or extraction we want to perform.
8421
8422 For an insertion, the number of significant structure bits includes
8423 all bits of the target. For an extraction, it need only include the
8424 most significant bit of the field. Larger widths are acceptable
8425 in both cases. */
8426
8427 static bool
8428 get_best_extraction_insn (extraction_insn *insn,
8429 enum extraction_pattern pattern,
8430 enum extraction_type type,
8431 unsigned HOST_WIDE_INT struct_bits,
8432 enum machine_mode field_mode)
8433 {
8434 enum machine_mode mode = smallest_mode_for_size (struct_bits, MODE_INT);
8435 while (mode != VOIDmode)
8436 {
8437 if (get_extraction_insn (insn, pattern, type, mode))
8438 {
8439 while (mode != VOIDmode
8440 && GET_MODE_SIZE (mode) <= GET_MODE_SIZE (field_mode)
8441 && !TRULY_NOOP_TRUNCATION_MODES_P (insn->field_mode,
8442 field_mode))
8443 {
8444 get_extraction_insn (insn, pattern, type, mode);
8445 mode = GET_MODE_WIDER_MODE (mode);
8446 }
8447 return true;
8448 }
8449 mode = GET_MODE_WIDER_MODE (mode);
8450 }
8451 return false;
8452 }
8453
8454 /* Return true if an instruction exists to access a field of mode
8455 FIELDMODE in a register structure that has STRUCT_BITS significant bits.
8456 Describe the "best" such instruction in *INSN if so. PATTERN describes
8457 the type of insertion or extraction we want to perform.
8458
8459 For an insertion, the number of significant structure bits includes
8460 all bits of the target. For an extraction, it need only include the
8461 most significant bit of the field. Larger widths are acceptable
8462 in both cases. */
8463
8464 bool
8465 get_best_reg_extraction_insn (extraction_insn *insn,
8466 enum extraction_pattern pattern,
8467 unsigned HOST_WIDE_INT struct_bits,
8468 enum machine_mode field_mode)
8469 {
8470 return get_best_extraction_insn (insn, pattern, ET_reg, struct_bits,
8471 field_mode);
8472 }
8473
8474 /* Return true if an instruction exists to access a field of BITSIZE
8475 bits starting BITNUM bits into a memory structure. Describe the
8476 "best" such instruction in *INSN if so. PATTERN describes the type
8477 of insertion or extraction we want to perform and FIELDMODE is the
8478 natural mode of the extracted field.
8479
8480 The instructions considered here only access bytes that overlap
8481 the bitfield; they do not touch any surrounding bytes. */
8482
8483 bool
8484 get_best_mem_extraction_insn (extraction_insn *insn,
8485 enum extraction_pattern pattern,
8486 HOST_WIDE_INT bitsize, HOST_WIDE_INT bitnum,
8487 enum machine_mode field_mode)
8488 {
8489 unsigned HOST_WIDE_INT struct_bits = (bitnum % BITS_PER_UNIT
8490 + bitsize
8491 + BITS_PER_UNIT - 1);
8492 struct_bits -= struct_bits % BITS_PER_UNIT;
8493 return get_best_extraction_insn (insn, pattern, ET_unaligned_mem,
8494 struct_bits, field_mode);
8495 }
8496
8497 #include "gt-optabs.h"