print-rtl.c (debug_call_placeholder_verbose): New variable.

[thirdparty/gcc.git] / gcc / config / ia64 / ia64.md

;; IA-64 Machine description template
;; Copyright (C) 1999, 2000 Free Software Foundation, Inc.
;; Contributed by James E. Wilson <wilson@cygnus.com> and
;;		  David Mosberger <davidm@hpl.hp.com>.

;; This file is part of GNU CC.

;; GNU CC is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 2, or (at your option)
;; any later version.

;; GNU CC is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;; GNU General Public License for more details.

;; You should have received a copy of the GNU General Public License
;; along with GNU CC; see the file COPYING.  If not, write to
;; the Free Software Foundation, 59 Temple Place - Suite 330,
;; Boston, MA 02111-1307, USA.

;;- See file "rtl.def" for documentation on define_insn, match_*, et. al.

;; ??? register_operand accepts (subreg:DI (mem:SI X)) which forces later
;; reload.  This will be fixed once scheduling support is turned on.

;; ??? Optimize for post-increment addressing modes.

;; ??? fselect is not supported, because there is no integer register
;; equivalent.

;; ??? fp abs/min/max instructions may also work for integer values.

;; ??? Would a predicate_reg_operand predicate be useful?  The HP one is buggy,
;; it assumes the operand is a register and takes REGNO of it without checking.

;; ??? Would a branch_reg_operand predicate be useful?  The HP one is buggy,
;; it assumes the operand is a register and takes REGNO of it without checking.

;; ??? Go through list of documented named patterns and look for more to
;; implement.

;; ??? Go through instruction manual and look for more instructions that
;; can be emitted.

;; ??? Add function unit scheduling info for Itanium (TM) processor.

;; ??? The explicit stop in the flushrs pattern is not ideal.  It
;; would be better if rtx_needs_barrier took care of this, but this is
;; something that can be fixed later.

;; Unspec usage:
;;
;; unspec:
;;	1	gr_spill
;;	2	gr_restore
;;	3	fr_spill
;;	4	fr_restore
;;	8	popcnt
;;	12	mf
;;	13	cmpxchg_acq
;;	19	fetchadd_acq
;;	20	bsp_value
;;	21	flushrs
;;
;; unspec_volatile:
;;	0	alloc
;;	1	blockage
;;	2	insn_group_barrier
;;	5	set_bsp
;;	7	pred.rel.mutex
;;	8	pred.safe_across_calls all
;;	9	pred.safe_across_calls normal
\f
;; ::::::::::::::::::::
;; ::
;; :: Attributes
;; ::
;; ::::::::::::::::::::

;; Instruction type.  This primarily determines how instructions can be
;; packed in bundles, and secondarily affects scheduling to function units.

;; A alu, can go in I or M syllable of a bundle
;; I integer
;; M memory
;; F floating-point
;; B branch
;; L long immediate, takes two syllables
;; S stop bit

;; ??? Should not have any pattern with type unknown.  Perhaps add code to
;; check this in md_reorg?  Currently use unknown for patterns which emit
;; multiple instructions, patterns which emit 0 instructions, and patterns
;; which emit instruction that can go in any slot (e.g. nop).

(define_attr "type" "unknown,A,I,M,F,B,L,S" (const_string "unknown"))

;; Predication.  True iff this instruction can be predicated.

(define_attr "predicable" "no,yes" (const_string "yes"))

\f
;; ::::::::::::::::::::
;; ::
;; :: Function Units
;; ::
;; ::::::::::::::::::::

;; Each usage of a function units by a class of insns is specified with a
;; `define_function_unit' expression, which looks like this:
;; (define_function_unit NAME MULTIPLICITY SIMULTANEITY TEST READY-DELAY
;;   ISSUE-DELAY [CONFLICT-LIST])

;; This default scheduling info seeks to pack instructions into bundles
;; efficiently to reduce code size, so we just list how many of each
;; instruction type can go in a bundle.  ISSUE_RATE is set to 3.

;; ??? Add scheduler ready-list hook (MD_SCHED_REORDER) that orders
;; instructions, so that the next instruction can fill the next bundle slot.
;; This really needs to know where the stop bits are though.

;; ??? Use MD_SCHED_REORDER to put alloc first instead of using an unspec
;; volatile.  Use ADJUST_PRIORITY to set the priority of alloc very high to
;; make it schedule first.

;; ??? Modify the md_reorg code that emits stop bits so that instead of putting
;; them in the last possible place, we put them in places where bundles allow
;; them.  This should reduce code size, but may decrease performance if we end
;; up with more stop bits than the minimum we need.

;; Alu instructions can execute on either the integer or memory function
;; unit.  We indicate this by defining an alu function unit, and then marking
;; it as busy everytime we issue a integer or memory type instruction.

(define_function_unit "alu" 3 1 (eq_attr "type" "A,I,M") 1 0)

(define_function_unit "integer" 2 1 (eq_attr "type" "I") 1 0)

(define_function_unit "memory" 3 1 (eq_attr "type" "M") 1 0)

(define_function_unit "floating_point" 1 1 (eq_attr "type" "F") 1 0)

(define_function_unit "branch" 3 1 (eq_attr "type" "B") 1 0)

;; ??? This isn't quite right, because we can only fit two insns in a bundle
;; when using an L type instruction.  That isn't modeled currently.

(define_function_unit "long_immediate" 1 1 (eq_attr "type" "L") 1 0)

\f
;; ::::::::::::::::::::
;; ::
;; :: Moves
;; ::
;; ::::::::::::::::::::

(define_expand "movqi"
  [(set (match_operand:QI 0 "general_operand" "")
	(match_operand:QI 1 "general_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && ! ia64_move_ok (operands[0], operands[1]))
    operands[1] = force_reg (QImode, operands[1]);
}")

;; Errata 72 implies that we cannot use predicated loads and stores
;; on affected systems.  Reuse TARGET_A_STEP for convenience.

;; ??? It would be convenient at this point if the cond_exec pattern
;; expander understood non-constant conditions on attributes.  Failing
;; that we have to replicate patterns.

(define_insn "*movqicc_astep"
  [(cond_exec
     (match_operator 2 "predicate_operator"
       [(match_operand:CC 3 "register_operand" "c,c,c,c,c")
        (const_int 0)])
     (set (match_operand:QI 0 "register_operand"  "=r,r, r,*f,*f")
	  (match_operand:QI 1 "nonmemory_operand" "rO,J,*f,rO,*f")))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
   (%J2) mov %0 = %r1
   (%J2) addl %0 = %1, r0
   (%J2) getf.sig %0 = %1
   (%J2) setf.sig %0 = %r1
   (%J2) mov %0 = %1"
  [(set_attr "type" "A,A,M,M,F")
   (set_attr "predicable" "no")])

(define_insn "*movqi_internal_astep"
  [(set (match_operand:QI 0 "destination_operand" "=r,r,r, m, r,*f,*f")
	(match_operand:QI 1 "move_operand"        "rO,J,m,rO,*f,rO,*f"))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
   mov %0 = %r1
   addl %0 = %1, r0
   ld1%O1 %0 = %1%P1
   st1%Q0 %0 = %r1%P0
   getf.sig %0 = %1
   setf.sig %0 = %r1
   mov %0 = %1"
  [(set_attr "type" "A,A,M,M,M,M,F")
   (set_attr "predicable" "no")])

(define_insn "*movqi_internal"
  [(set (match_operand:QI 0 "destination_operand" "=r,r,r, m, r,*f,*f")
	(match_operand:QI 1 "move_operand"        "rO,J,m,rO,*f,rO,*f"))]
  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
   mov %0 = %r1
   addl %0 = %1, r0
   ld1%O1 %0 = %1%P1
   st1%Q0 %0 = %r1%P0
   getf.sig %0 = %1
   setf.sig %0 = %r1
   mov %0 = %1"
  [(set_attr "type" "A,A,M,M,M,M,F")])

(define_expand "movhi"
  [(set (match_operand:HI 0 "general_operand" "")
	(match_operand:HI 1 "general_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && ! ia64_move_ok (operands[0], operands[1]))
    operands[1] = force_reg (HImode, operands[1]);
}")

;; Errata 72 workaround.
(define_insn "*movhicc_astep"
  [(cond_exec
     (match_operator 2 "predicate_operator"
       [(match_operand:CC 3 "register_operand" "c,c,c,c,c")
        (const_int 0)])
     (set (match_operand:HI 0 "register_operand"  "=r,r, r,*f,*f")
	  (match_operand:HI 1 "nonmemory_operand" "rO,J,*f,rO,*f")))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
   (%J2) mov %0 = %r1
   (%J2) addl %0 = %1, r0
   (%J2) getf.sig %0 = %1
   (%J2) setf.sig %0 = %r1
   (%J2) mov %0 = %1"
  [(set_attr "type" "A,A,M,M,F")
   (set_attr "predicable" "no")])

(define_insn "*movhi_internal_astep"
  [(set (match_operand:HI 0 "destination_operand" "=r,r,r, m, r,*f,*f")
	(match_operand:HI 1 "move_operand"        "rO,J,m,rO,*f,rO,*f"))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
   mov %0 = %r1
   addl %0 = %1, r0
   ld2%O1 %0 = %1%P1
   st2%Q0 %0 = %r1%P0
   getf.sig %0 = %1
   setf.sig %0 = %r1
   mov %0 = %1"
  [(set_attr "type" "A,A,M,M,M,M,F")
   (set_attr "predicable" "no")])

(define_insn "*movhi_internal"
  [(set (match_operand:HI 0 "destination_operand" "=r,r,r, m, r,*f,*f")
	(match_operand:HI 1 "move_operand"        "rO,J,m,rO,*f,rO,*f"))]
  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
   mov %0 = %r1
   addl %0 = %1, r0
   ld2%O1 %0 = %1%P1
   st2%Q0 %0 = %r1%P0
   getf.sig %0 = %1
   setf.sig %0 = %r1
   mov %0 = %1"
  [(set_attr "type" "A,A,M,M,M,M,F")])

(define_expand "movsi"
  [(set (match_operand:SI 0 "general_operand" "")
	(match_operand:SI 1 "general_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && ! ia64_move_ok (operands[0], operands[1]))
    operands[1] = force_reg (SImode, operands[1]);
}")

;; Errata 72 workaround.
(define_insn "*movsicc_astep"
  [(cond_exec
     (match_operator 2 "predicate_operator"
       [(match_operand:CC 3 "register_operand" "c,c,c,c,c,c,c,c")
        (const_int 0)])
     (set (match_operand:SI 0 "register_operand"  "=r,r,r, r,*f,*f, r,*d")
	  (match_operand:SI 1 "nonmemory_operand" "rO,J,i,*f,rO,*f,*d,rK")))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
   (%J2) mov %0 = %r1
   (%J2) addl %0 = %1, r0
   (%J2) movl %0 = %1
   (%J2) getf.sig %0 = %1
   (%J2) setf.sig %0 = %r1
   (%J2) mov %0 = %1
   (%J2) mov %0 = %1
   (%J2) mov %0 = %r1"
  [(set_attr "type" "A,A,L,M,M,F,M,M")
   (set_attr "predicable" "no")])

(define_insn "*movsi_internal_astep"
  [(set (match_operand:SI 0 "destination_operand" "=r,r,r,r, m, r,*f,*f, r,*d")
	(match_operand:SI 1 "move_operand"        "rO,J,i,m,rO,*f,rO,*f,*d,rK"))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
  mov %0 = %r1
  addl %0 = %1, r0
  movl %0 = %1
  ld4%O1 %0 = %1%P1
  st4%Q0 %0 = %r1%P0
  getf.sig %0 = %1
  setf.sig %0 = %r1
  mov %0 = %1
  mov %0 = %1
  mov %0 = %r1"
  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M")
   (set_attr "predicable" "no")])

(define_insn "*movsi_internal"
  [(set (match_operand:SI 0 "destination_operand" "=r,r,r,r, m, r,*f,*f, r,*d")
	(match_operand:SI 1 "move_operand"        "rO,J,i,m,rO,*f,rO,*f,*d,rK"))]
  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
  mov %0 = %r1
  addl %0 = %1, r0
  movl %0 = %1
  ld4%O1 %0 = %1%P1
  st4%Q0 %0 = %r1%P0
  getf.sig %0 = %1
  setf.sig %0 = %r1
  mov %0 = %1
  mov %0 = %1
  mov %0 = %r1"
  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M")])

(define_expand "movdi"
  [(set (match_operand:DI 0 "general_operand" "")
	(match_operand:DI 1 "general_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && ! ia64_move_ok (operands[0], operands[1]))
    operands[1] = force_reg (DImode, operands[1]);
  if (! TARGET_NO_PIC && symbolic_operand (operands[1], DImode))
    {
      /* Before optimization starts, delay committing to any particular
	 type of PIC address load.  If this function gets deferred, we
	 may acquire information that changes the value of the
	 sdata_symbolic_operand predicate.  */
      /* But don't delay for function pointers.  Loading a function address
	 actually loads the address of the descriptor not the function.
	 If we represent these as SYMBOL_REFs, then they get cse'd with
	 calls, and we end up with calls to the descriptor address instead of
	 calls to the function address.  Functions are not candidates for
	 sdata anyways.  */
      if (rtx_equal_function_value_matters
	  && ! (GET_CODE (operands[1]) == SYMBOL_REF
		&& SYMBOL_REF_FLAG (operands[1])))
	emit_insn (gen_movdi_symbolic (operands[0], operands[1]));
      else
        ia64_expand_load_address (operands[0], operands[1]);
      DONE;
    }
}")

;; Errata 72 workaround.
(define_insn ""
  [(cond_exec
     (match_operator 2 "predicate_operator"
       [(match_operand:CC 3 "register_operand" "c,c,c,c,c,c,c,c,c,c,c")
        (const_int 0)])
     (set (match_operand:DI 0 "register_operand"
			      "=r,r,r, r,*f,*f,   r,*b,*e, r,*d")
	  (match_operand:DI 1 "nonmemory_operand"
			      "rO,J,i,*f,rO,*f,*b*e,rO,rK,*d,rK")))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "*
{
  static const char * const alt[] = {
    \"(%J2) mov %0 = %r1\",
    \"(%J2) addl %0 = %1, r0\",
    \"(%J2) movl %0 = %1\",
    \"(%J2) getf.sig %0 = %1\",
    \"(%J2) setf.sig %0 = %r1\",
    \"(%J2) mov %0 = %1\",
    \"(%J2) mov %0 = %1\",
    \"(%J2) mov %0 = %r1\",
    \"(%J2) mov %0 = %1\",
    \"(%J2) mov %0 = %1\",
    \"(%J2) mov %0 = %1\"
  };

  /* We use 'i' for alternative 2 despite possible PIC problems.

     If we define LEGITIMATE_CONSTANT_P such that symbols are not
     allowed, then the compiler dumps the data into constant memory
     instead of letting us read the values from the GOT.  Similarly
     if we use 'n' instead of 'i'.

     Instead, we allow such insns through reload and then split them
     afterward (even without optimization).  Therefore, we should
     never get so far with a symbolic operand.  */

  if (which_alternative == 2 && ! TARGET_NO_PIC
      && symbolic_operand (operands[1], VOIDmode))
    abort ();

  return alt[which_alternative];
}"
  [(set_attr "type" "A,A,L,M,M,F,I,I,I,M,M")
   (set_attr "predicable" "no")])

;; This is used during early compilation to delay the decision on
;; how to refer to a variable as long as possible.  This is especially
;; important between initial rtl generation and optimization for
;; deferred functions, since we may acquire additional information
;; on the variables used in the meantime.

(define_insn_and_split "movdi_symbolic"
  [(set (match_operand:DI 0 "register_operand" "=r")
	(match_operand:DI 1 "symbolic_operand" "s"))
   (use (reg:DI 1))]
  ""
  "* abort ();"
  ""
  [(const_int 0)]
  "ia64_expand_load_address (operands[0], operands[1]); DONE;")

(define_insn "*movdi_internal_astep"
  [(set (match_operand:DI 0 "destination_operand"
		    "=r,r,r,r, m, r,*f,*f,*f, Q,   r,*b,*e, r,*d, r,*c")
	(match_operand:DI 1 "move_operand"
		    "rO,J,i,m,rO,*f,rO,*f, Q,*f,*b*e,rO,rK,*d,rK,*c,rO"))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "*
{
  static const char * const alt[] = {
    \"mov %0 = %r1\",
    \"addl %0 = %1, r0\",
    \"movl %0 = %1\",
    \"ld8%O1 %0 = %1%P1\",
    \"st8%Q0 %0 = %r1%P0\",
    \"getf.sig %0 = %1\",
    \"setf.sig %0 = %r1\",
    \"mov %0 = %1\",
    \"ldf8 %0 = %1%P1\",
    \"stf8 %0 = %1%P0\",
    \"mov %0 = %1\",
    \"mov %0 = %r1\",
    \"mov %0 = %1\",
    \"mov %0 = %1\",
    \"mov %0 = %1\",
    \"mov %0 = pr\",
    \"mov pr = %1, -1\"
  };

  if (which_alternative == 2 && ! TARGET_NO_PIC
      && symbolic_operand (operands[1], VOIDmode))
    abort ();

  return alt[which_alternative];
}"
  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M,I,I,I,M,M,I,I")
   (set_attr "predicable" "no")])

(define_insn "*movdi_internal"
  [(set (match_operand:DI 0 "destination_operand"
		    "=r,r,r,r, m, r,*f,*f,*f, Q,   r,*b,*e, r,*d, r,*c")
	(match_operand:DI 1 "move_operand"
		    "rO,J,i,m,rO,*f,rO,*f, Q,*f,*b*e,rO,rK,*d,rK,*c,rO"))]
  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "*
{
  static const char * const alt[] = {
    \"%,mov %0 = %r1\",
    \"%,addl %0 = %1, r0\",
    \"%,movl %0 = %1\",
    \"%,ld8%O1 %0 = %1%P1\",
    \"%,st8%Q0 %0 = %r1%P0\",
    \"%,getf.sig %0 = %1\",
    \"%,setf.sig %0 = %r1\",
    \"%,mov %0 = %1\",
    \"%,ldf8 %0 = %1%P1\",
    \"%,stf8 %0 = %1%P0\",
    \"%,mov %0 = %1\",
    \"%,mov %0 = %r1\",
    \"%,mov %0 = %1\",
    \"%,mov %0 = %1\",
    \"%,mov %0 = %1\",
    \"mov %0 = pr\",
    \"mov pr = %1, -1\"
  };

  if (which_alternative == 2 && ! TARGET_NO_PIC
      && symbolic_operand (operands[1], VOIDmode))
    abort ();

  return alt[which_alternative];
}"
  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M,I,I,I,M,M,I,I")])

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
	(match_operand:DI 1 "symbolic_operand" ""))]
  "reload_completed && ! TARGET_NO_PIC"
  [(const_int 0)]
  "
{
  ia64_expand_load_address (operands[0], operands[1]);
  DONE;
}")

(define_expand "load_fptr"
  [(set (match_dup 2)
	(plus:DI (reg:DI 1) (match_operand:DI 1 "function_operand" "")))
   (set (match_operand:DI 0 "register_operand" "") (match_dup 3))]
  ""
  "
{
  operands[2] = no_new_pseudos ? operands[0] : gen_reg_rtx (DImode);
  operands[3] = gen_rtx_MEM (DImode, operands[2]);
  RTX_UNCHANGING_P (operands[3]) = 1;
}")

(define_insn "*load_fptr_internal1"
  [(set (match_operand:DI 0 "register_operand" "=r")
	(plus:DI (reg:DI 1) (match_operand:DI 1 "function_operand" "s")))]
  ""
  "addl %0 = @ltoff(@fptr(%1)), gp"
  [(set_attr "type" "A")])

(define_insn "load_gprel"
  [(set (match_operand:DI 0 "register_operand" "=r")
	(plus:DI (reg:DI 1) (match_operand:DI 1 "sdata_symbolic_operand" "s")))]
  ""
  "addl %0 = @gprel(%1), gp"
  [(set_attr "type" "A")])

(define_insn "gprel64_offset"
  [(set (match_operand:DI 0 "register_operand" "=r")
	(minus:DI (match_operand:DI 1 "symbolic_operand" "") (reg:DI 1)))]
  ""
  "movl %0 = @gprel(%1)"
  [(set_attr "type" "L")])

(define_expand "load_gprel64"
  [(set (match_dup 2)
	(minus:DI (match_operand:DI 1 "symbolic_operand" "") (reg:DI 1)))
   (set (match_operand:DI 0 "register_operand" "")
	(plus:DI (reg:DI 1) (match_dup 2)))]
  ""
  "
{
  operands[2] = no_new_pseudos ? operands[0] : gen_reg_rtx (DImode);
}")

(define_expand "load_symptr"
  [(set (match_dup 2)
	(plus:DI (reg:DI 1) (match_operand:DI 1 "got_symbolic_operand" "")))
   (set (match_operand:DI 0 "register_operand" "") (match_dup 3))]
  ""
  "
{
  operands[2] = no_new_pseudos ? operands[0] : gen_reg_rtx (DImode);
  operands[3] = gen_rtx_MEM (DImode, operands[2]);
  RTX_UNCHANGING_P (operands[3]) = 1;
}")

(define_insn "*load_symptr_internal1"
  [(set (match_operand:DI 0 "register_operand" "=r")
	(plus:DI (reg:DI 1) (match_operand:DI 1 "got_symbolic_operand" "s")))]
  ""
  "addl %0 = @ltoff(%1), gp"
  [(set_attr "type" "A")])

;; With no offsettable memory references, we've got to have a scratch
;; around to play with the second word.
(define_expand "movti"
  [(parallel [(set (match_operand:TI 0 "general_operand" "")
		   (match_operand:TI 1 "general_operand" ""))
	      (clobber (match_scratch:DI 2 ""))])]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && ! ia64_move_ok (operands[0], operands[1]))
    operands[1] = force_reg (TImode, operands[1]);
}")

(define_insn_and_split "*movti_internal"
  [(set (match_operand:TI 0 "nonimmediate_operand" "=r,r,m")
	(match_operand:TI 1 "general_operand"      "ri,m,r"))
   (clobber (match_scratch:DI 2 "=X,&r,&r"))]
  "ia64_move_ok (operands[0], operands[1])"
  "#"
  "reload_completed"
  [(const_int 0)]
  "
{
  rtx adj1, adj2, in[2], out[2];
  int first;

  adj1 = ia64_split_timode (in, operands[1], operands[2]);
  adj2 = ia64_split_timode (out, operands[0], operands[2]);

  first = 0;
  if (reg_overlap_mentioned_p (out[0], in[1]))
    {
      if (reg_overlap_mentioned_p (out[1], in[0]))
	abort ();
      first = 1;
    }

  if (adj1 && adj2)
    abort ();
  if (adj1)
    emit_insn (adj1);
  if (adj2)
    emit_insn (adj2);
  emit_insn (gen_rtx_SET (VOIDmode, out[first], in[first]));
  emit_insn (gen_rtx_SET (VOIDmode, out[!first], in[!first]));
  DONE;
}"
  [(set_attr "type" "unknown")
   (set_attr "predicable" "no")])

;; ??? SSA creates these.  Can't allow memories since we don't have
;; the scratch register.  Fortunately combine will know how to add
;; the clobber and scratch.
(define_insn_and_split "*movti_internal_reg"
  [(set (match_operand:TI 0 "register_operand"  "=r")
	(match_operand:TI 1 "nonmemory_operand" "ri"))]
  ""
  "#"
  "reload_completed"
  [(const_int 0)]
  "
{
  rtx in[2], out[2];
  int first;

  ia64_split_timode (in, operands[1], NULL_RTX);
  ia64_split_timode (out, operands[0], NULL_RTX);

  first = 0;
  if (reg_overlap_mentioned_p (out[0], in[1]))
    {
      if (reg_overlap_mentioned_p (out[1], in[0]))
	abort ();
      first = 1;
    }

  emit_insn (gen_rtx_SET (VOIDmode, out[first], in[first]));
  emit_insn (gen_rtx_SET (VOIDmode, out[!first], in[!first]));
  DONE;
}"
  [(set_attr "type" "unknown")
   (set_attr "predicable" "no")])

(define_expand "reload_inti"
  [(parallel [(set (match_operand:TI 0 "register_operand" "=r")
		   (match_operand:TI 1 "" "m"))
	      (clobber (match_operand:TI 2 "register_operand" "=&r"))])]
  ""
  "
{
  unsigned int s_regno = REGNO (operands[2]);
  if (s_regno == REGNO (operands[0]))
    s_regno += 1;
  operands[2] = gen_rtx_REG (DImode, s_regno);
}")

(define_expand "reload_outti"
  [(parallel [(set (match_operand:TI 0 "" "=m")
		   (match_operand:TI 1 "register_operand" "r"))
	      (clobber (match_operand:TI 2 "register_operand" "=&r"))])]
  ""
  "
{
  unsigned int s_regno = REGNO (operands[2]);
  if (s_regno == REGNO (operands[1]))
    s_regno += 1;
  operands[2] = gen_rtx_REG (DImode, s_regno);
}")

;; Floating Point Moves
;;
;; Note - Patterns for SF mode moves are compulsory, but
;; patterns for DF are optional, as GCC can synthesise them.

(define_expand "movsf"
  [(set (match_operand:SF 0 "general_operand" "")
	(match_operand:SF 1 "general_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && ! ia64_move_ok (operands[0], operands[1]))
    operands[1] = force_reg (SFmode, operands[1]);
}")

;; Errata 72 workaround.
(define_insn "*movsfcc_astep"
  [(cond_exec
     (match_operator 2 "predicate_operator"
       [(match_operand:CC 3 "register_operand" "c,c,c,c")
        (const_int 0)])
     (set (match_operand:SF 0 "register_operand"  "=f,*r, f,*r")
	  (match_operand:SF 1 "nonmemory_operand" "fG,fG,*r,*r")))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
  (%J2) mov %0 = %F1
  (%J2) getf.s %0 = %F1
  (%J2) setf.s %0 = %1
  (%J2) mov %0 = %1"
  [(set_attr "type" "F,M,M,A")
   (set_attr "predicable" "no")])

(define_insn "*movsf_internal_astep"
  [(set (match_operand:SF 0 "destination_operand" "=f,f, Q,*r, f,*r,*r, m")
	(match_operand:SF 1 "general_operand"     "fG,Q,fG,fG,*r,*r, m,*r"))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
  mov %0 = %F1
  ldfs %0 = %1%P1
  stfs %0 = %F1%P0
  getf.s %0 = %F1
  setf.s %0 = %1
  mov %0 = %1
  ld4%O1 %0 = %1%P1
  st4%Q0 %0 = %1%P0"
  [(set_attr "type" "F,M,M,M,M,A,M,M")
   (set_attr "predicable" "no")])

(define_insn "*movsf_internal"
  [(set (match_operand:SF 0 "destination_operand" "=f,f, Q,*r, f,*r,*r, m")
	(match_operand:SF 1 "general_operand"     "fG,Q,fG,fG,*r,*r, m,*r"))]
  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
  mov %0 = %F1
  ldfs %0 = %1%P1
  stfs %0 = %F1%P0
  getf.s %0 = %F1
  setf.s %0 = %1
  mov %0 = %1
  ld4%O1 %0 = %1%P1
  st4%Q0 %0 = %1%P0"
  [(set_attr "type" "F,M,M,M,M,A,M,M")])

(define_expand "movdf"
  [(set (match_operand:DF 0 "general_operand" "")
	(match_operand:DF 1 "general_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed
      && ! ia64_move_ok (operands[0], operands[1]))
    operands[1] = force_reg (DFmode, operands[1]);
}")

;; Errata 72 workaround.
(define_insn "*movdfcc_astep"
  [(cond_exec
     (match_operator 2 "predicate_operator"
       [(match_operand:CC 3 "register_operand" "c,c,c,c")
        (const_int 0)])
     (set (match_operand:DF 0 "register_operand"  "=f,*r, f,*r")
	  (match_operand:DF 1 "nonmemory_operand" "fG,fG,*r,*r")))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
  (%J2) mov %0 = %F1
  (%J2) getf.d %0 = %F1
  (%J2) setf.d %0 = %1
  (%J2) mov %0 = %1"
  [(set_attr "type" "F,M,M,A")
   (set_attr "predicable" "no")])

(define_insn "*movdf_internal_astep"
  [(set (match_operand:DF 0 "destination_operand" "=f,f, Q,*r, f,*r,*r, m")
	(match_operand:DF 1 "general_operand"     "fG,Q,fG,fG,*r,*r, m,*r"))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
  mov %0 = %F1
  ldfd %0 = %1%P1
  stfd %0 = %F1%P0
  getf.d %0 = %F1
  setf.d %0 = %1
  mov %0 = %1
  ld8%O1 %0 = %1%P1
  st8%Q0 %0 = %1%P0"
  [(set_attr "type" "F,M,M,M,M,A,M,M")
   (set_attr "predicable" "no")])

(define_insn "*movdf_internal"
  [(set (match_operand:DF 0 "destination_operand" "=f,f, Q,*r, f,*r,*r, m")
	(match_operand:DF 1 "general_operand"     "fG,Q,fG,fG,*r,*r, m,*r"))]
  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
  mov %0 = %F1
  ldfd %0 = %1%P1
  stfd %0 = %F1%P0
  getf.d %0 = %F1
  setf.d %0 = %1
  mov %0 = %1
  ld8%O1 %0 = %1%P1
  st8%Q0 %0 = %1%P0"
  [(set_attr "type" "F,M,M,M,M,A,M,M")])

;; With no offsettable memory references, we've got to have a scratch
;; around to play with the second word if the variable winds up in GRs.
(define_expand "movtf"
  [(set (match_operand:TF 0 "general_operand" "")
	(match_operand:TF 1 "general_operand" ""))]
  ""
  "
{
  /* We must support TFmode loads into general registers for stdarg/vararg
     and unprototyped calls.  We split them into DImode loads for convenience.
     We don't need TFmode stores from general regs, because a stdarg/vararg
     routine does a block store to memory of unnamed arguments.  */
  if (GET_CODE (operands[0]) == REG
      && GR_REGNO_P (REGNO (operands[0])))
    {
      /* We're hoping to transform everything that deals with TFmode
	 quantities and GR registers early in the compiler.  */
      if (no_new_pseudos)
	abort ();

      /* Struct to register can just use TImode instead.  */
      if ((GET_CODE (operands[1]) == SUBREG
	   && GET_MODE (SUBREG_REG (operands[1])) == TImode)
	  || (GET_CODE (operands[1]) == REG
	      && GR_REGNO_P (REGNO (operands[1]))))
	{
	  emit_move_insn (gen_rtx_REG (TImode, REGNO (operands[0])),
			  SUBREG_REG (operands[1]));
	  DONE;
	}

      if (GET_CODE (operands[1]) == CONST_DOUBLE)
	{
	  emit_move_insn (gen_rtx_REG (DImode, REGNO (operands[0])),
			  operand_subword (operands[1], 0, 0, DImode));
	  emit_move_insn (gen_rtx_REG (DImode, REGNO (operands[0]) + 1),
			  operand_subword (operands[1], 1, 0, DImode));
	  DONE;
	}

      /* If the quantity is in a register not known to be GR, spill it.  */
      if (register_operand (operands[1], TFmode))
	operands[1] = spill_tfmode_operand (operands[1], 1);

      if (GET_CODE (operands[1]) == MEM)
	{
	  rtx out[2];

	  out[WORDS_BIG_ENDIAN] = gen_rtx_REG (DImode, REGNO (operands[0]));
	  out[!WORDS_BIG_ENDIAN] = gen_rtx_REG (DImode, REGNO (operands[0])+1);

	  emit_move_insn (out[0], change_address (operands[1], DImode, NULL));
	  emit_move_insn (out[1],
			  change_address (operands[1], DImode,
					  plus_constant (XEXP (operands[1], 0),
							 8)));
	  DONE;
	}

      abort ();
    }

  if (! reload_in_progress && ! reload_completed)
    {
      operands[0] = spill_tfmode_operand (operands[0], 0);
      operands[1] = spill_tfmode_operand (operands[1], 0);

      if (! ia64_move_ok (operands[0], operands[1]))
	operands[1] = force_reg (TFmode, operands[1]);
    }
}")

;; ??? There's no easy way to mind volatile acquire/release semantics.

;; Errata 72 workaround.
(define_insn "*movtfcc_astep"
  [(cond_exec
     (match_operator 2 "predicate_operator"
       [(match_operand:CC 3 "register_operand" "c")
        (const_int 0)])
     (set (match_operand:TF 0 "register_operand"  "=f")
	  (match_operand:TF 1 "nonmemory_operand" "fG")))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "(%J2) mov %0 = %F1"
  [(set_attr "type" "F")
   (set_attr "predicable" "no")])

(define_insn "*movtf_internal_astep"
  [(set (match_operand:TF 0 "destination_tfmode_operand" "=f,f, m")
	(match_operand:TF 1 "general_tfmode_operand"     "fG,m,fG"))]
  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
  mov %0 = %F1
  ldfe %0 = %1%P1
  stfe %0 = %F1%P0"
  [(set_attr "type" "F,M,M")
   (set_attr "predicable" "no")])

(define_insn "*movtf_internal"
  [(set (match_operand:TF 0 "destination_tfmode_operand" "=f,f, m")
	(match_operand:TF 1 "general_tfmode_operand"     "fG,m,fG"))]
  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
  "@
  mov %0 = %F1
  ldfe %0 = %1%P1
  stfe %0 = %F1%P0"
  [(set_attr "type" "F,M,M")])
\f
;; ::::::::::::::::::::
;; ::
;; :: Conversions
;; ::
;; ::::::::::::::::::::

;; Signed conversions from a smaller integer to a larger integer

(define_insn "extendqidi2"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(sign_extend:DI (match_operand:QI 1 "gr_register_operand" "r")))]
  ""
  "sxt1 %0 = %1"
  [(set_attr "type" "I")])

(define_insn "extendhidi2"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(sign_extend:DI (match_operand:HI 1 "gr_register_operand" "r")))]
  ""
  "sxt2 %0 = %1"
  [(set_attr "type" "I")])

(define_insn "extendsidi2"
  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
	(sign_extend:DI (match_operand:SI 1 "grfr_register_operand" "r,*f")))]
  ""
  "@
   sxt4 %0 = %1
   fsxt.r %0 = %1, %1%B0"
  [(set_attr "type" "I,F")])

;; Unsigned conversions from a smaller integer to a larger integer

(define_insn "zero_extendqidi2"
  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
	(zero_extend:DI (match_operand:QI 1 "gr_nonimmediate_operand" "r,m")))]
  ""
  "@
   zxt1 %0 = %1
   ld1%O1 %0 = %1%P1"
  [(set_attr "type" "I,M")])

(define_insn "zero_extendhidi2"
  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
	(zero_extend:DI (match_operand:HI 1 "gr_nonimmediate_operand" "r,m")))]
  ""
  "@
   zxt2 %0 = %1
   ld2%O1 %0 = %1%P1"
  [(set_attr "type" "I,M")])

(define_insn "zero_extendsidi2"
  [(set (match_operand:DI 0 "grfr_register_operand" "=r,r,*f")
	(zero_extend:DI
	  (match_operand:SI 1 "grfr_nonimmediate_operand" "r,m,*f")))]
  ""
  "@
   zxt4 %0 = %1
   ld4%O1 %0 = %1%P1
   fsxt.r %0 = f1, %1%B0"
  [(set_attr "type" "I,M,F")])

;; Convert between floating point types of different sizes.

;; ??? Optimization opportunity here.  Get rid of the insn altogether
;; when we can.  Should probably use a scheme like has been proposed
;; for ia32 in dealing with operands that match unary operators.  This
;; would let combine merge the thing into adjacent insns.

(define_insn_and_split "extendsfdf2"
  [(set (match_operand:DF 0 "fr_register_operand" "=f,f")
	(float_extend:DF (match_operand:SF 1 "fr_register_operand" "0,f")))]
  ""
  "mov %0 = %1"
  "reload_completed"
  [(set (match_dup 0) (float_extend:DF (match_dup 1)))]
  "if (true_regnum (operands[0]) == true_regnum (operands[1])) DONE;"
  [(set_attr "type" "F")])

(define_insn_and_split "extendsftf2"
  [(set (match_operand:TF 0 "fr_register_operand" "=f,f")
	(float_extend:TF (match_operand:SF 1 "fr_register_operand" "0,f")))]
  ""
  "mov %0 = %1"
  "reload_completed"
  [(set (match_dup 0) (float_extend:TF (match_dup 1)))]
  "if (true_regnum (operands[0]) == true_regnum (operands[1])) DONE;"
  [(set_attr "type" "F")])

(define_insn_and_split "extenddftf2"
  [(set (match_operand:TF 0 "fr_register_operand" "=f,f")
	(float_extend:TF (match_operand:DF 1 "fr_register_operand" "0,f")))]
  ""
  "mov %0 = %1"
  "reload_completed"
  [(set (match_dup 0) (float_extend:TF (match_dup 1)))]
  "if (true_regnum (operands[0]) == true_regnum (operands[1])) DONE;"
  [(set_attr "type" "F")])

(define_insn "truncdfsf2"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(float_truncate:SF (match_operand:DF 1 "fr_register_operand" "f")))]
  ""
  "fnorm.s %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "trunctfsf2"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(float_truncate:SF (match_operand:TF 1 "fr_register_operand" "f")))]
  ""
  "fnorm.s %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "trunctfdf2"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(float_truncate:DF (match_operand:TF 1 "fr_register_operand" "f")))]
  ""
  "fnorm.d %0 = %1%B0"
  [(set_attr "type" "F")])

;; Convert between signed integer types and floating point.

(define_insn "floatditf2"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(float:TF (match_operand:DI 1 "fr_register_operand" "f")))]
  ""
  "fcvt.xf %0 = %1"
  [(set_attr "type" "F")])

(define_insn "fix_truncsfdi2"
  [(set (match_operand:DI 0 "fr_register_operand" "=f")
	(fix:DI (match_operand:SF 1 "fr_register_operand" "f")))]
  ""
  "fcvt.fx.trunc %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "fix_truncdfdi2"
  [(set (match_operand:DI 0 "fr_register_operand" "=f")
	(fix:DI (match_operand:DF 1 "fr_register_operand" "f")))]
  ""
  "fcvt.fx.trunc %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "fix_trunctfdi2"
  [(set (match_operand:DI 0 "fr_register_operand" "=f")
	(fix:DI (match_operand:TF 1 "fr_register_operand" "f")))]
  ""
  "fcvt.fx.trunc %0 = %1%B0"
  [(set_attr "type" "F")])

;; Convert between unsigned integer types and floating point.

(define_insn "floatunsdisf2"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(unsigned_float:SF (match_operand:DI 1 "fr_register_operand" "f")))]
  ""
  "fcvt.xuf.s %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "floatunsdidf2"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(unsigned_float:DF (match_operand:DI 1 "fr_register_operand" "f")))]
  ""
  "fcvt.xuf.d %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "floatunsditf2"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(unsigned_float:TF (match_operand:DI 1 "fr_register_operand" "f")))]
  ""
  "fcvt.xuf %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "fixuns_truncsfdi2"
  [(set (match_operand:DI 0 "fr_register_operand" "=f")
	(unsigned_fix:DI (match_operand:SF 1 "fr_register_operand" "f")))]
  ""
  "fcvt.fxu.trunc %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "fixuns_truncdfdi2"
  [(set (match_operand:DI 0 "fr_register_operand" "=f")
	(unsigned_fix:DI (match_operand:DF 1 "fr_register_operand" "f")))]
  ""
  "fcvt.fxu.trunc %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "fixuns_trunctfdi2"
  [(set (match_operand:DI 0 "fr_register_operand" "=f")
	(unsigned_fix:DI (match_operand:TF 1 "fr_register_operand" "f")))]
  ""
  "fcvt.fxu.trunc %0 = %1%B0"
  [(set_attr "type" "F")])
\f
;; ::::::::::::::::::::
;; ::
;; :: Bit field extraction
;; ::
;; ::::::::::::::::::::

(define_insn "extv"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(sign_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
			 (match_operand:DI 2 "const_int_operand" "n")
			 (match_operand:DI 3 "const_int_operand" "n")))]
  ""
  "extr %0 = %1, %3, %2"
  [(set_attr "type" "I")])

(define_insn "extzv"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(zero_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
			 (match_operand:DI 2 "const_int_operand" "n")
			 (match_operand:DI 3 "const_int_operand" "n")))]
  ""
  "extr.u %0 = %1, %3, %2"
  [(set_attr "type" "I")])

;; Insert a bit field.
;; Can have 3 operands, source1 (inserter), source2 (insertee), dest.
;; Source1 can be 0 or -1.
;; Source2 can be 0.

;; ??? Actual dep instruction is more powerful than what these insv
;; patterns support.  Unfortunately, combine is unable to create patterns
;; where source2 != dest.

(define_expand "insv"
  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "")
			 (match_operand:DI 1 "const_int_operand" "")
			 (match_operand:DI 2 "const_int_operand" ""))
	(match_operand:DI 3 "nonmemory_operand" ""))]
  ""
  "
{
  int width = INTVAL (operands[1]);
  int shift = INTVAL (operands[2]);

  /* If operand[3] is a constant, and isn't 0 or -1, then load it into a
     pseudo.  */
  if (! register_operand (operands[3], DImode)
      && operands[3] != const0_rtx && operands[3] != constm1_rtx)
    operands[3] = force_reg (DImode, operands[3]);

  /* If this is a single dep instruction, we have nothing to do.  */
  if (! ((register_operand (operands[3], DImode) && width <= 16)
	 || operands[3] == const0_rtx || operands[3] == constm1_rtx))
    {
      /* Check for cases that can be implemented with a mix instruction.  */
      if (width == 32 && shift == 0)
	{
	  /* Directly generating the mix4left instruction confuses
	     optimize_bit_field in function.c.  Since this is performing
	     a useful optimization, we defer generation of the complicated
	     mix4left RTL to the first splitting phase.  */
	  rtx tmp = gen_reg_rtx (DImode);
	  emit_insn (gen_shift_mix4left (operands[0], operands[3], tmp));
	  DONE;
	}
      else if (width == 32 && shift == 32)
	{
	  emit_insn (gen_mix4right (operands[0], operands[3]));
	  DONE;
	}

      /* We could handle remaining cases by emitting multiple dep
	 instructions.

	 If we need more than two dep instructions then we lose.  A 6
	 insn sequence mov mask1,mov mask2,shl;;and,and;;or is better than
	 mov;;dep,shr;;dep,shr;;dep.  The former can be executed in 3 cycles,
	 the latter is 6 cycles on an Itanium (TM) processor, because there is
	 only one function unit that can execute dep and shr immed.

	 If we only need two dep instruction, then we still lose.
	 mov;;dep,shr;;dep is still 4 cycles.  Even if we optimize away
	 the unnecessary mov, this is still undesirable because it will be
	 hard to optimize, and it creates unnecessary pressure on the I0
	 function unit.  */

      FAIL;

#if 0
      /* This code may be useful for other IA-64 processors, so we leave it in
	 for now.  */
      while (width > 16)
	{
	  rtx tmp;

	  emit_insn (gen_insv (operands[0], GEN_INT (16), GEN_INT (shift),
			       operands[3]));
	  shift += 16;
	  width -= 16;
	  tmp = gen_reg_rtx (DImode);
	  emit_insn (gen_lshrdi3 (tmp, operands[3], GEN_INT (16)));
	  operands[3] = tmp;
	}
      operands[1] = GEN_INT (width);
      operands[2] = GEN_INT (shift);
#endif
    }
}")

(define_insn "*insv_internal"
  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "+r")
			 (match_operand:DI 1 "const_int_operand" "n")
			 (match_operand:DI 2 "const_int_operand" "n"))
	(match_operand:DI 3 "nonmemory_operand" "rP"))]
  "(gr_register_operand (operands[3], DImode) && INTVAL (operands[1]) <= 16)
   || operands[3] == const0_rtx || operands[3] == constm1_rtx"
  "dep %0 = %3, %0, %2, %1"
  [(set_attr "type" "I")])

;; Combine doesn't like to create bitfield insertions into zero.
(define_insn "*depz_internal"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(and:DI (ashift:DI (match_operand:DI 1 "gr_register_operand" "r")
			   (match_operand:DI 2 "const_int_operand" "n"))
		(match_operand:DI 3 "const_int_operand" "n")))]
  "CONST_OK_FOR_M (INTVAL (operands[2]))
   && ia64_depz_field_mask (operands[3], operands[2]) > 0"
  "*
{
  operands[3] = GEN_INT (ia64_depz_field_mask (operands[3], operands[2]));
  return \"%,dep.z %0 = %1, %2, %3\";
}"
  [(set_attr "type" "I")])

(define_insn "shift_mix4left"
  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "+r")
			 (const_int 32) (const_int 0))
	(match_operand:DI 1 "gr_register_operand" "r"))
   (clobber (match_operand:DI 2 "gr_register_operand" "=r"))]
  ""
  "#"
  [(set_attr "type" "unknown")])

(define_split
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "")
			 (const_int 32) (const_int 0))
	(match_operand:DI 1 "register_operand" ""))
   (clobber (match_operand:DI 2 "register_operand" ""))]
  "reload_completed"
  [(set (match_dup 3) (ashift:DI (match_dup 1) (const_int 32)))
   (unspec_volatile [(const_int 0)] 2)
   (set (zero_extract:DI (match_dup 0) (const_int 32) (const_int 0))
	(lshiftrt:DI (match_dup 3) (const_int 32)))]
  "operands[3] = operands[2];")

(define_split
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "")
			 (const_int 32) (const_int 0))
	(match_operand:DI 1 "register_operand" ""))
   (clobber (match_operand:DI 2 "register_operand" ""))]
  "! reload_completed"
  [(set (match_dup 3) (ashift:DI (match_dup 1) (const_int 32)))
   (set (zero_extract:DI (match_dup 0) (const_int 32) (const_int 0))
	(lshiftrt:DI (match_dup 3) (const_int 32)))]
  "operands[3] = operands[2];")

(define_insn "*mix4left"
  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "+r")
			 (const_int 32) (const_int 0))
	(lshiftrt:DI (match_operand:DI 1 "gr_register_operand" "r")
		     (const_int 32)))]
  ""
  "mix4.l %0 = %0, %r1"
  [(set_attr "type" "I")])

(define_insn "mix4right"
  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "+r")
			 (const_int 32) (const_int 32))
	(match_operand:DI 1 "gr_reg_or_0_operand" "rO"))]
  ""
  "mix4.r %0 = %r1, %0"
  [(set_attr "type" "I")])

;; This is used by the rotrsi3 pattern.

(define_insn "*mix4right_3op"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(ior:DI (zero_extend:DI (match_operand:SI 1 "gr_register_operand" "r"))
		(ashift:DI (zero_extend:DI
			     (match_operand:SI 2 "gr_register_operand" "r"))
			   (const_int 32))))]
  ""
  "mix4.r %0 = %2, %1"
  [(set_attr "type" "I")])

\f
;; ::::::::::::::::::::
;; ::
;; :: 16 bit Integer arithmetic
;; ::
;; ::::::::::::::::::::

(define_insn "mulhi3"
  [(set (match_operand:HI 0 "gr_register_operand" "=r")
	(mult:HI (match_operand:HI 1 "gr_register_operand" "r")
		 (match_operand:HI 2 "gr_register_operand" "r")))]
  ""
  "pmpy2.r %0 = %1, %2"
  [(set_attr "type" "I")])

\f
;; ::::::::::::::::::::
;; ::
;; :: 32 bit Integer arithmetic
;; ::
;; ::::::::::::::::::::

(define_insn "addsi3"
  [(set (match_operand:SI 0 "gr_register_operand" "=r,r,r")
	(plus:SI (match_operand:SI 1 "gr_register_operand" "%r,r,a")
		 (match_operand:SI 2 "gr_reg_or_22bit_operand" "r,I,J")))]
  ""
  "@
  add %0 = %1, %2
  adds %0 = %2, %1
  addl %0 = %2, %1"
  [(set_attr "type" "A")])

(define_insn "*addsi3_plus1"
  [(set (match_operand:SI 0 "gr_register_operand" "=r")
	(plus:SI (plus:SI (match_operand:SI 1 "gr_register_operand" "r")
			  (match_operand:SI 2 "gr_register_operand" "r"))
		 (const_int 1)))]
  ""
  "add %0 = %1, %2, 1"
  [(set_attr "type" "A")])

(define_insn "*addsi3_plus1_alt"
  [(set (match_operand:SI 0 "gr_register_operand" "=r")
	(plus:SI (mult:SI (match_operand:SI 1 "gr_register_operand" "r")
			  (const_int 2))
		 (const_int 1)))]
  ""
  "add %0 = %1, %1, 1"
  [(set_attr "type" "A")])

(define_insn "*addsi3_shladd"
  [(set (match_operand:SI 0 "gr_register_operand" "=r")
	(plus:SI (mult:SI (match_operand:SI 1 "gr_register_operand" "r")
			  (match_operand:SI 2 "shladd_operand" "n"))
		 (match_operand:SI 3 "gr_register_operand" "r")))]
  ""
  "shladd %0 = %1, %S2, %3"
  [(set_attr "type" "A")])

(define_insn "subsi3"
  [(set (match_operand:SI 0 "gr_register_operand" "=r")
	(minus:SI (match_operand:SI 1 "gr_reg_or_8bit_operand" "rK")
		  (match_operand:SI 2 "gr_register_operand" "r")))]
  ""
  "sub %0 = %1, %2"
  [(set_attr "type" "A")])

(define_insn "*subsi3_minus1"
  [(set (match_operand:SI 0 "gr_register_operand" "=r")
	(plus:SI (not:SI (match_operand:SI 1 "gr_register_operand" "r"))
		 (match_operand:SI 2 "gr_register_operand" "r")))]
  ""
  "sub %0 = %2, %1, 1"
  [(set_attr "type" "A")])

(define_insn "mulsi3"
  [(set (match_operand:SI 0 "fr_register_operand" "=f")
	(mult:SI (match_operand:SI 1 "grfr_register_operand" "f")
		 (match_operand:SI 2 "grfr_register_operand" "f")))]
  ""
  "xma.l %0 = %1, %2, f0%B0"
  [(set_attr "type" "F")])

(define_insn "*maddsi3"
  [(set (match_operand:SI 0 "fr_register_operand" "=f")
	(plus:SI (mult:SI (match_operand:SI 1 "grfr_register_operand" "f")
			  (match_operand:SI 2 "grfr_register_operand" "f"))
		 (match_operand:SI 3 "grfr_register_operand" "f")))]
  ""
  "xma.l %0 = %1, %2, %3%B0"
  [(set_attr "type" "F")])

(define_insn "negsi2"
  [(set (match_operand:SI 0 "gr_register_operand" "=r")
	(neg:SI (match_operand:SI 1 "gr_register_operand" "r")))]
  ""
  "sub %0 = r0, %1"
  [(set_attr "type" "A")])

(define_expand "abssi2"
  [(set (match_dup 2)
	(ge:CC (match_operand:SI 1 "gr_register_operand" "") (const_int 0)))
   (set (match_operand:SI 0 "gr_register_operand" "")
	(if_then_else:SI (eq:CC (match_dup 2) (const_int 0))
			 (neg:SI (match_dup 1))
			 (match_dup 1)))]
  ""
  "
{
  operands[2] = gen_reg_rtx (CCmode);
}")

(define_expand "sminsi3"
  [(set (match_dup 3)
	(ge:CC (match_operand:SI 1 "gr_register_operand" "")
	       (match_operand:SI 2 "gr_register_operand" "")))
   (set (match_operand:SI 0 "gr_register_operand" "")
	(if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
			 (match_dup 2) (match_dup 1)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "smaxsi3"
  [(set (match_dup 3)
	(ge:CC (match_operand:SI 1 "gr_register_operand" "")
	       (match_operand:SI 2 "gr_register_operand" "")))
   (set (match_operand:SI 0 "gr_register_operand" "")
	(if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
			 (match_dup 1) (match_dup 2)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "uminsi3"
  [(set (match_dup 3)
	(geu:CC (match_operand:SI 1 "gr_register_operand" "")
		(match_operand:SI 2 "gr_register_operand" "")))
   (set (match_operand:SI 0 "gr_register_operand" "")
	(if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
			 (match_dup 2) (match_dup 1)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "umaxsi3"
  [(set (match_dup 3)
	(geu:CC (match_operand:SI 1 "gr_register_operand" "")
		(match_operand:SI 2 "gr_register_operand" "")))
   (set (match_operand:SI 0 "gr_register_operand" "")
	(if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
			 (match_dup 1) (match_dup 2)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

\f
;; ::::::::::::::::::::
;; ::
;; :: 64 bit Integer arithmetic
;; ::
;; ::::::::::::::::::::

(define_insn "adddi3"
  [(set (match_operand:DI 0 "gr_register_operand" "=r,r,r")
	(plus:DI (match_operand:DI 1 "gr_register_operand" "%r,r,a")
		 (match_operand:DI 2 "gr_reg_or_22bit_operand" "r,I,J")))]
  ""
  "@
  add %0 = %1, %2
  adds %0 = %2, %1
  addl %0 = %2, %1"
  [(set_attr "type" "A")])

(define_insn "*adddi3_plus1"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(plus:DI (plus:DI (match_operand:DI 1 "gr_register_operand" "r")
			  (match_operand:DI 2 "gr_register_operand" "r"))
		 (const_int 1)))]
  ""
  "add %0 = %1, %2, 1"
  [(set_attr "type" "A")])

;; This has some of the same problems as shladd.  We let the shladd
;; eliminator hack handle it, which results in the 1 being forced into
;; a register, but not more ugliness here.
(define_insn "*adddi3_plus1_alt"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(plus:DI (mult:DI (match_operand:DI 1 "gr_register_operand" "r")
			  (const_int 2))
		 (const_int 1)))]
  ""
  "add %0 = %1, %1, 1"
  [(set_attr "type" "A")])

(define_insn "subdi3"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(minus:DI (match_operand:DI 1 "gr_reg_or_8bit_operand" "rK")
		  (match_operand:DI 2 "gr_register_operand" "r")))]
  ""
  "sub %0 = %1, %2"
  [(set_attr "type" "A")])

(define_insn "*subdi3_minus1"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(plus:DI (not:DI (match_operand:DI 1 "gr_register_operand" "r"))
		 (match_operand:DI 2 "gr_register_operand" "r")))]
  ""
  "sub %0 = %2, %1, 1"
  [(set_attr "type" "A")])

;; ??? Use grfr instead of fr because of virtual register elimination
;; and silly test cases multiplying by the frame pointer.
(define_insn "muldi3"
  [(set (match_operand:DI 0 "fr_register_operand" "=f")
	(mult:DI (match_operand:DI 1 "grfr_register_operand" "f")
		 (match_operand:DI 2 "grfr_register_operand" "f")))]
  ""
  "xma.l %0 = %1, %2, f0%B0"
  [(set_attr "type" "F")])

;; ??? If operand 3 is an eliminable reg, then register elimination causes the
;; same problem that we have with shladd below.  Unfortunately, this case is
;; much harder to fix because the multiply puts the result in an FP register,
;; but the add needs inputs from a general register.  We add a spurious clobber
;; here so that it will be present just in case register elimination gives us
;; the funny result.

;; ??? Maybe validate_changes should try adding match_scratch clobbers?

;; ??? Maybe we should change how adds are canonicalized.

(define_insn "*madddi3"
  [(set (match_operand:DI 0 "fr_register_operand" "=f")
	(plus:DI (mult:DI (match_operand:DI 1 "grfr_register_operand" "f")
			  (match_operand:DI 2 "grfr_register_operand" "f"))
		 (match_operand:DI 3 "grfr_register_operand" "f")))
   (clobber (match_scratch:DI 4 "=X"))]
  ""
  "xma.l %0 = %1, %2, %3%B0"
  [(set_attr "type" "F")])

;; This can be created by register elimination if operand3 of shladd is an
;; eliminable register or has reg_equiv_constant set.

;; We have to use nonmemory_operand for operand 4, to ensure that the
;; validate_changes call inside eliminate_regs will always succeed.  If it
;; doesn't succeed, then this remain a madddi3 pattern, and will be reloaded
;; incorrectly.

(define_insn "*madddi3_elim"
  [(set (match_operand:DI 0 "register_operand" "=&r")
	(plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "f")
				   (match_operand:DI 2 "register_operand" "f"))
			  (match_operand:DI 3 "register_operand" "f"))
		 (match_operand:DI 4 "nonmemory_operand" "rI")))
   (clobber (match_scratch:DI 5 "=f"))]
  "reload_in_progress"
  "#"
  [(set_attr "type" "unknown")])

;; ??? Need to emit an instruction group barrier here because this gets split
;; after md_reorg.

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
	(plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "")
				   (match_operand:DI 2 "register_operand" ""))
			  (match_operand:DI 3 "register_operand" ""))
		 (match_operand:DI 4 "gr_reg_or_14bit_operand" "")))
   (clobber (match_scratch:DI 5 ""))]
  "reload_completed"
  [(parallel [(set (match_dup 5) (plus:DI (mult:DI (match_dup 1) (match_dup 2))
					  (match_dup 3)))
	      (clobber (match_dup 0))])
   (unspec_volatile [(const_int 0)] 2)
   (set (match_dup 0) (match_dup 5))
   (unspec_volatile [(const_int 0)] 2)
   (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 4)))]
  "")

;; ??? There are highpart multiply and add instructions, but we have no way
;; to generate them.

(define_insn "smuldi3_highpart"
  [(set (match_operand:DI 0 "fr_register_operand" "=f")
	(truncate:DI
	 (lshiftrt:TI
	  (mult:TI (sign_extend:TI
		     (match_operand:DI 1 "fr_register_operand" "f"))
		   (sign_extend:TI
		     (match_operand:DI 2 "fr_register_operand" "f")))
	  (const_int 64))))]
  ""
  "xma.h %0 = %1, %2, f0%B0"
  [(set_attr "type" "F")])

(define_insn "umuldi3_highpart"
  [(set (match_operand:DI 0 "fr_register_operand" "=f")
	(truncate:DI
	 (lshiftrt:TI
	  (mult:TI (zero_extend:TI
		     (match_operand:DI 1 "fr_register_operand" "f"))
		   (zero_extend:TI
		     (match_operand:DI 2 "fr_register_operand" "f")))
	  (const_int 64))))]
  ""
  "xma.hu %0 = %1, %2, f0%B0"
  [(set_attr "type" "F")])

(define_insn "negdi2"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(neg:DI (match_operand:DI 1 "gr_register_operand" "r")))]
  ""
  "sub %0 = r0, %1"
  [(set_attr "type" "A")])

(define_expand "absdi2"
  [(set (match_dup 2)
	(ge:CC (match_operand:DI 1 "gr_register_operand" "") (const_int 0)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(if_then_else:DI (eq:CC (match_dup 2) (const_int 0))
			 (neg:DI (match_dup 1))
			 (match_dup 1)))]
  ""
  "
{
  operands[2] = gen_reg_rtx (CCmode);
}")

(define_expand "smindi3"
  [(set (match_dup 3)
	(ge:CC (match_operand:DI 1 "gr_register_operand" "")
	       (match_operand:DI 2 "gr_register_operand" "")))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
			 (match_dup 2) (match_dup 1)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "smaxdi3"
  [(set (match_dup 3)
	(ge:CC (match_operand:DI 1 "gr_register_operand" "")
	       (match_operand:DI 2 "gr_register_operand" "")))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
			 (match_dup 1) (match_dup 2)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "umindi3"
  [(set (match_dup 3)
	(geu:CC (match_operand:DI 1 "gr_register_operand" "")
		(match_operand:DI 2 "gr_register_operand" "")))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
			 (match_dup 2) (match_dup 1)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "umaxdi3"
  [(set (match_dup 3)
	(geu:CC (match_operand:DI 1 "gr_register_operand" "")
		(match_operand:DI 2 "gr_register_operand" "")))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
			 (match_dup 1) (match_dup 2)))]
  ""
  "
{
  operands[3] = gen_reg_rtx (CCmode);
}")

(define_expand "ffsdi2"
  [(set (match_dup 6)
	(eq:CC (match_operand:DI 1 "gr_register_operand" "") (const_int 0)))
   (set (match_dup 2) (plus:DI (match_dup 1) (const_int -1)))
   (set (match_dup 5) (const_int 0))
   (set (match_dup 3) (xor:DI (match_dup 1) (match_dup 2)))
   (set (match_dup 4) (unspec:DI [(match_dup 3)] 8))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(if_then_else:DI (ne:CC (match_dup 6) (const_int 0))
			 (match_dup 5) (match_dup 4)))]
  ""
  "
{
  operands[2] = gen_reg_rtx (DImode);
  operands[3] = gen_reg_rtx (DImode);
  operands[4] = gen_reg_rtx (DImode);
  operands[5] = gen_reg_rtx (DImode);
  operands[6] = gen_reg_rtx (CCmode);
}")

(define_insn "*popcnt"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(unspec:DI [(match_operand:DI 1 "gr_register_operand" "r")] 8))]
  ""
  "popcnt %0 = %1"
  [(set_attr "type" "I")])

\f
;; ::::::::::::::::::::
;; ::
;; :: 32 bit floating point arithmetic
;; ::
;; ::::::::::::::::::::

(define_insn "addsf3"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(plus:SF (match_operand:SF 1 "fr_register_operand" "%f")
		 (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fadd.s %0 = %1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "subsf3"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(minus:SF (match_operand:SF 1 "fr_reg_or_fp01_operand" "fG")
		  (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fsub.s %0 = %F1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "mulsf3"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(mult:SF (match_operand:SF 1 "fr_register_operand" "%f")
		 (match_operand:SF 2 "fr_register_operand" "f")))]
  ""
  "fmpy.s %0 = %1, %2%B0"
  [(set_attr "type" "F")])

(define_insn "abssf2"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(abs:SF (match_operand:SF 1 "fr_register_operand" "f")))]
  ""
  "fabs %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "negsf2"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(neg:SF (match_operand:SF 1 "fr_register_operand" "f")))]
  ""
  "fneg %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "*nabssf2"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(neg:SF (abs:SF (match_operand:SF 1 "fr_register_operand" "f"))))]
  ""
  "fnegabs %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "minsf3"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(smin:SF (match_operand:SF 1 "fr_register_operand" "f")
		 (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fmin %0 = %1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "maxsf3"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(smax:SF (match_operand:SF 1 "fr_register_operand" "f")
		 (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fmax %0 = %1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "*maddsf3"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(plus:SF (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
			  (match_operand:SF 2 "fr_register_operand" "f"))
		 (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fma.s %0 = %1, %2, %F3%B0"
  [(set_attr "type" "F")])

(define_insn "*msubsf3"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(minus:SF (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
			   (match_operand:SF 2 "fr_register_operand" "f"))
		  (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fms.s %0 = %1, %2, %F3%B0"
  [(set_attr "type" "F")])

(define_insn "*nmulsf3"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(neg:SF (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
			 (match_operand:SF 2 "fr_register_operand" "f"))))]
  ""
  "fnmpy.s %0 = %1, %2%B0"
  [(set_attr "type" "F")])

;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?

(define_insn "*nmaddsf3"
  [(set (match_operand:SF 0 "fr_register_operand" "=f")
	(plus:SF (neg:SF (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
				  (match_operand:SF 2 "fr_register_operand" "f")))
		 (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fnma.s %0 = %1, %2, %F3%B0"
  [(set_attr "type" "F")])

\f
;; ::::::::::::::::::::
;; ::
;; :: 64 bit floating point arithmetic
;; ::
;; ::::::::::::::::::::

(define_insn "adddf3"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(plus:DF (match_operand:DF 1 "fr_register_operand" "%f")
		 (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fadd.d %0 = %1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "subdf3"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(minus:DF (match_operand:DF 1 "fr_reg_or_fp01_operand" "fG")
		  (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fsub.d %0 = %F1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "muldf3"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(mult:DF (match_operand:DF 1 "fr_register_operand" "f")
		 (match_operand:DF 2 "fr_register_operand" "f")))]
  ""
  "fmpy.d %0 = %1, %2%B0"
  [(set_attr "type" "F")])

(define_insn "absdf2"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(abs:DF (match_operand:DF 1 "fr_register_operand" "f")))]
  ""
  "fabs %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "negdf2"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(neg:DF (match_operand:DF 1 "fr_register_operand" "f")))]
  ""
  "fneg %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "*nabsdf2"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(neg:DF (abs:DF (match_operand:DF 1 "fr_register_operand" "f"))))]
  ""
  "fnegabs %0 = %1%B0"
  [(set_attr "type" "F")])

(define_insn "mindf3"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(smin:DF (match_operand:DF 1 "fr_register_operand" "f")
		 (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fmin %0 = %1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "maxdf3"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(smax:DF (match_operand:DF 1 "fr_register_operand" "f")
		 (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fmax %0 = %1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "*madddf3"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(plus:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
			  (match_operand:DF 2 "fr_register_operand" "f"))
		 (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fma.d %0 = %1, %2, %F3%B0"
  [(set_attr "type" "F")])

(define_insn "*msubdf3"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(minus:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
			   (match_operand:DF 2 "fr_register_operand" "f"))
		  (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fms.d %0 = %1, %2, %F3%B0"
  [(set_attr "type" "F")])

(define_insn "*nmuldf3"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(neg:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
			 (match_operand:DF 2 "fr_register_operand" "f"))))]
  ""
  "fnmpy.d %0 = %1, %2%B0"
  [(set_attr "type" "F")])

;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?

(define_insn "*nmadddf3"
  [(set (match_operand:DF 0 "fr_register_operand" "=f")
	(plus:DF (neg:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
				  (match_operand:DF 2 "fr_register_operand" "f")))
		 (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")))]
  ""
  "fnma.d %0 = %1, %2, %F3%B0"
  [(set_attr "type" "F")])
\f
;; ::::::::::::::::::::
;; ::
;; :: 80 bit floating point arithmetic
;; ::
;; ::::::::::::::::::::

(define_insn "addtf3"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(plus:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
		 (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
  ""
  "fadd %0 = %F1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "subtf3"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(minus:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
		  (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
  ""
  "fsub %0 = %F1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "multf3"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(mult:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
		 (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
  ""
  "fmpy %0 = %F1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "abstf2"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(abs:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")))]
  ""
  "fabs %0 = %F1%B0"
  [(set_attr "type" "F")])

(define_insn "negtf2"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(neg:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")))]
  ""
  "fneg %0 = %F1%B0"
  [(set_attr "type" "F")])

(define_insn "*nabstf2"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(neg:TF (abs:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG"))))]
  ""
  "fnegabs %0 = %F1%B0"
  [(set_attr "type" "F")])

(define_insn "mintf3"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(smin:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
		 (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
  ""
  "fmin %0 = %F1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "maxtf3"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(smax:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
		 (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
  ""
  "fmax %0 = %F1, %F2%B0"
  [(set_attr "type" "F")])

(define_insn "*maddtf3"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(plus:TF (mult:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
			  (match_operand:TF 2 "tfreg_or_fp01_operand" "fG"))
		 (match_operand:TF 3 "tfreg_or_fp01_operand" "fG")))]
  ""
  "fma %0 = %F1, %F2, %F3%B0"
  [(set_attr "type" "F")])

(define_insn "*msubtf3"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(minus:TF (mult:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
			   (match_operand:TF 2 "tfreg_or_fp01_operand" "fG"))
		  (match_operand:TF 3 "tfreg_or_fp01_operand" "fG")))]
  ""
  "fms %0 = %F1, %F2, %F3%B0"
  [(set_attr "type" "F")])

(define_insn "*nmultf3"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(neg:TF (mult:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
			 (match_operand:TF 2 "tfreg_or_fp01_operand" "fG"))))]
  ""
  "fnmpy %0 = %F1, %F2%B0"
  [(set_attr "type" "F")])

;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?

(define_insn "*nmaddtf3"
  [(set (match_operand:TF 0 "fr_register_operand" "=f")
	(plus:TF (neg:TF (mult:TF
			  (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
			  (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))
		 (match_operand:TF 3 "tfreg_or_fp01_operand" "fG")))]
  ""
  "fnma %0 = %F1, %F2, %F3%B0"
  [(set_attr "type" "F")])
\f
;; ::::::::::::::::::::
;; ::
;; :: 32 bit Integer Shifts and Rotates
;; ::
;; ::::::::::::::::::::

(define_expand "ashlsi3"
  [(set (match_operand:SI 0 "gr_register_operand" "")
	(ashift:SI (match_operand:SI 1 "gr_register_operand" "")
		   (match_operand:SI 2 "gr_reg_or_5bit_operand" "")))]
  ""
  "
{
  if (GET_CODE (operands[2]) != CONST_INT)
    {
      /* Why oh why didn't Intel arrange for SHIFT_COUNT_TRUNCATED?  Now
	 we've got to get rid of stray bits outside the SImode register.  */
      rtx subshift = gen_reg_rtx (DImode);
      emit_insn (gen_zero_extendsidi2 (subshift, operands[2]));
      operands[2] = subshift;
    }
}")

(define_insn "*ashlsi3_internal"
  [(set (match_operand:SI 0 "gr_register_operand" "=r,r,r")
	(ashift:SI (match_operand:SI 1 "gr_register_operand" "r,r,r")
		   (match_operand:DI 2 "gr_reg_or_5bit_operand" "R,n,r")))]
  ""
  "@
   shladd %0 = %1, %2, r0
   dep.z %0 = %1, %2, %E2
   shl %0 = %1, %2"
  [(set_attr "type" "A,I,I")])

(define_expand "ashrsi3"
  [(set (match_operand:SI 0 "gr_register_operand" "")
	(ashiftrt:SI (match_operand:SI 1 "gr_register_operand" "")
		     (match_operand:SI 2 "gr_reg_or_5bit_operand" "")))]
  ""
  "
{
  rtx subtarget = gen_reg_rtx (DImode);
  if (GET_CODE (operands[2]) == CONST_INT)
    emit_insn (gen_extv (subtarget, gen_lowpart (DImode, operands[1]),
			 GEN_INT (32 - INTVAL (operands[2])), operands[2]));
  else
    {
      rtx subshift = gen_reg_rtx (DImode);
      emit_insn (gen_extendsidi2 (subtarget, operands[1]));
      emit_insn (gen_zero_extendsidi2 (subshift, operands[2]));
      emit_insn (gen_ashrdi3 (subtarget, subtarget, subshift));
    }
  emit_move_insn (gen_lowpart (DImode, operands[0]), subtarget);
  DONE;
}")

(define_expand "lshrsi3"
  [(set (match_operand:SI 0 "gr_register_operand" "")
	(lshiftrt:SI (match_operand:SI 1 "gr_register_operand" "")
		     (match_operand:SI 2 "gr_reg_or_5bit_operand" "")))]
  ""
  "
{
  rtx subtarget = gen_reg_rtx (DImode);
  if (GET_CODE (operands[2]) == CONST_INT)
    emit_insn (gen_extzv (subtarget, gen_lowpart (DImode, operands[1]),
			  GEN_INT (32 - INTVAL (operands[2])), operands[2]));
  else
    {
      rtx subshift = gen_reg_rtx (DImode);
      emit_insn (gen_zero_extendsidi2 (subtarget, operands[1]));
      emit_insn (gen_zero_extendsidi2 (subshift, operands[2]));
      emit_insn (gen_lshrdi3 (subtarget, subtarget, subshift));
    }
  emit_move_insn (gen_lowpart (DImode, operands[0]), subtarget);
  DONE;
}")

;; Use mix4.r/shr to implement rotrsi3.  We only get 32 bits of valid result
;; here, instead of 64 like the patterns above.

(define_expand "rotrsi3"
  [(set (match_dup 3)
	(ior:DI (zero_extend:DI (match_operand:SI 1 "gr_register_operand" ""))
		(ashift:DI (zero_extend:DI (match_dup 1)) (const_int 32))))
   (set (match_dup 3)
	(lshiftrt:DI (match_dup 3)
		     (match_operand:DI 2 "nonmemory_operand" "")))
   (set (match_operand:SI 0 "gr_register_operand" "") (match_dup 4))]
  ""
  "
{
  if (! shift_32bit_count_operand (operands[2], SImode))
    FAIL;
  operands[3] = gen_reg_rtx (DImode);
  operands[4] = gen_lowpart (SImode, operands[3]);
}")
\f
;; ::::::::::::::::::::
;; ::
;; :: 64 bit Integer Shifts and Rotates
;; ::
;; ::::::::::::::::::::

(define_insn "ashldi3"
  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
	(ashift:DI (match_operand:DI 1 "gr_register_operand" "r,r")
		   (match_operand:DI 2 "gr_reg_or_6bit_operand" "R,rM")))]
  ""
  "@
   shladd %0 = %1, %2, r0
   shl %0 = %1, %2"
  [(set_attr "type" "A,I")])

;; ??? Maybe combine this with the multiply and add instruction?

(define_insn "*shladd"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(plus:DI (mult:DI (match_operand:DI 1 "gr_register_operand" "r")
			  (match_operand:DI 2 "shladd_operand" "n"))
		 (match_operand:DI 3 "gr_register_operand" "r")))]
  ""
  "shladd %0 = %1, %S2, %3"
  [(set_attr "type" "A")])

;; This can be created by register elimination if operand3 of shladd is an
;; eliminable register or has reg_equiv_constant set.

;; We have to use nonmemory_operand for operand 4, to ensure that the
;; validate_changes call inside eliminate_regs will always succeed.  If it
;; doesn't succeed, then this remain a shladd pattern, and will be reloaded
;; incorrectly.

(define_insn_and_split "*shladd_elim"
  [(set (match_operand:DI 0 "gr_register_operand" "=&r")
	(plus:DI (plus:DI (mult:DI (match_operand:DI 1 "gr_register_operand" "r")
				   (match_operand:DI 2 "shladd_operand" "n"))
			  (match_operand:DI 3 "nonmemory_operand" "r"))
		 (match_operand:DI 4 "nonmemory_operand" "rI")))]
  "reload_in_progress"
  "* abort ();"
  "reload_completed"
  [(set (match_dup 0) (plus:DI (mult:DI (match_dup 1) (match_dup 2))
			       (match_dup 3)))
   (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 4)))]
  ""
  [(set_attr "type" "unknown")])

(define_insn "ashrdi3"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(ashiftrt:DI (match_operand:DI 1 "gr_register_operand" "r")
		     (match_operand:DI 2 "gr_reg_or_6bit_operand" "rM")))]
  ""
  "shr %0 = %1, %2"
  [(set_attr "type" "I")])

(define_insn "lshrdi3"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(lshiftrt:DI (match_operand:DI 1 "gr_register_operand" "r")
		     (match_operand:DI 2 "gr_reg_or_6bit_operand" "rM")))]
  ""
  "shr.u %0 = %1, %2"
  [(set_attr "type" "I")])

;; Using a predicate that accepts only constants doesn't work, because optabs
;; will load the operand into a register and call the pattern if the predicate
;; did not accept it on the first try.  So we use nonmemory_operand and then
;; verify that we have an appropriate constant in the expander.

(define_expand "rotrdi3"
  [(set (match_operand:DI 0 "gr_register_operand" "")
	(rotatert:DI (match_operand:DI 1 "gr_register_operand" "")
		     (match_operand:DI 2 "nonmemory_operand" "")))]
  ""
  "
{
  if (! shift_count_operand (operands[2], DImode))
    FAIL;
}")

(define_insn "*rotrdi3_internal"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(rotatert:DI (match_operand:DI 1 "gr_register_operand" "r")
		     (match_operand:DI 2 "shift_count_operand" "M")))]
  ""
  "shrp %0 = %1, %1, %2"
  [(set_attr "type" "I")])

\f
;; ::::::::::::::::::::
;; ::
;; :: 32 bit Integer Logical operations
;; ::
;; ::::::::::::::::::::

;; We don't seem to need any other 32-bit logical operations, because gcc
;; generates zero-extend;zero-extend;DImode-op, which combine optimizes to
;; DImode-op;zero-extend, and then we can optimize away the zero-extend.
;; This doesn't work for unary logical operations, because we don't call
;; apply_distributive_law for them.

;; ??? Likewise, this doesn't work for andnot, which isn't handled by
;; apply_distributive_law.  We get inefficient code for
;; int sub4 (int i, int j) { return i & ~j; }
;; We could convert (and (not (sign_extend A)) (sign_extend B)) to
;; (zero_extend (and (not A) B)) in combine.
;; Or maybe fix this by adding andsi3/iorsi3/xorsi3 patterns like the
;; one_cmplsi2 pattern.

(define_insn "one_cmplsi2"
  [(set (match_operand:SI 0 "gr_register_operand" "=r")
	(not:SI (match_operand:SI 1 "gr_register_operand" "r")))]
  ""
  "andcm %0 = -1, %1"
  [(set_attr "type" "A")])
\f
;; ::::::::::::::::::::
;; ::
;; :: 64 bit Integer Logical operations
;; ::
;; ::::::::::::::::::::

(define_insn "anddi3"
  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
	(and:DI (match_operand:DI 1 "grfr_register_operand" "%r,*f")
		(match_operand:DI 2 "grfr_reg_or_8bit_operand" "rK,*f")))]
  ""
  "@
   and %0 = %2, %1
   fand %0 = %2, %1%B0"
  [(set_attr "type" "A,F")])

(define_insn "*andnot"
  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
	(and:DI (not:DI (match_operand:DI 1 "grfr_register_operand" "r,*f"))
		(match_operand:DI 2 "grfr_reg_or_8bit_operand" "rK,*f")))]
  ""
  "@
   andcm %0 = %2, %1
   fandcm %0 = %2, %1%B0"
  [(set_attr "type" "A,F")])

(define_insn "iordi3"
  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
	(ior:DI (match_operand:DI 1 "grfr_register_operand" "%r,*f")
		(match_operand:DI 2 "grfr_reg_or_8bit_operand" "rK,*f")))]
  ""
  "@
   or %0 = %2, %1
   for %0 = %2, %1%B0"
  [(set_attr "type" "A,F")])

(define_insn "xordi3"
  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
	(xor:DI (match_operand:DI 1 "grfr_register_operand" "%r,*f")
		(match_operand:DI 2 "grfr_reg_or_8bit_operand" "rK,*f")))]
  ""
  "@
   xor %0 = %2, %1
   fxor %0 = %2, %1%B0"
  [(set_attr "type" "A,F")])

(define_insn "one_cmpldi2"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(not:DI (match_operand:DI 1 "gr_register_operand" "r")))]
  ""
  "andcm %0 = -1, %1"
  [(set_attr "type" "A")])
\f
;; ::::::::::::::::::::
;; ::
;; :: Comparisons
;; ::
;; ::::::::::::::::::::

(define_expand "cmpsi"
  [(set (cc0)
        (compare (match_operand:SI 0 "gr_register_operand" "")
  		 (match_operand:SI 1 "gr_reg_or_8bit_and_adjusted_operand" "")))]
  ""
  "
{
  ia64_compare_op0 = operands[0];
  ia64_compare_op1 = operands[1];
  DONE;
}")

(define_expand "cmpdi"
  [(set (cc0)
        (compare (match_operand:DI 0 "gr_register_operand" "")
  		 (match_operand:DI 1 "gr_reg_or_8bit_and_adjusted_operand" "")))]
  ""
  "
{
  ia64_compare_op0 = operands[0];
  ia64_compare_op1 = operands[1];
  DONE;
}")

(define_expand "cmpsf"
  [(set (cc0)
        (compare (match_operand:SF 0 "fr_reg_or_fp01_operand" "")
  		 (match_operand:SF 1 "fr_reg_or_fp01_operand" "")))]
  ""
  "
{
  ia64_compare_op0 = operands[0];
  ia64_compare_op1 = operands[1];
  DONE;
}")

(define_expand "cmpdf"
  [(set (cc0)
        (compare (match_operand:DF 0 "fr_reg_or_fp01_operand" "")
  		 (match_operand:DF 1 "fr_reg_or_fp01_operand" "")))]
  ""
  "
{
  ia64_compare_op0 = operands[0];
  ia64_compare_op1 = operands[1];
  DONE;
}")

(define_expand "cmptf"
  [(set (cc0)
        (compare (match_operand:TF 0 "tfreg_or_fp01_operand" "")
  		 (match_operand:TF 1 "tfreg_or_fp01_operand" "")))]
  ""
  "
{
  ia64_compare_op0 = operands[0];
  ia64_compare_op1 = operands[1];
  DONE;
}")

(define_insn "*cmpsi_normal"
  [(set (match_operand:CC 0 "register_operand" "=c")
	(match_operator:CC 1 "normal_comparison_operator"
	   [(match_operand:SI 2 "gr_register_operand" "r")
	    (match_operand:SI 3 "gr_reg_or_8bit_operand" "rK")]))]
  ""
  "cmp4.%C1 %0, %I0 = %3, %2"
  [(set_attr "type" "A")])

(define_insn "*cmpsi_adjusted"
  [(set (match_operand:CC 0 "register_operand" "=c")
	(match_operator:CC 1 "adjusted_comparison_operator"
	   [(match_operand:SI 2 "gr_register_operand" "r")
	    (match_operand:SI 3 "gr_reg_or_8bit_adjusted_operand" "rL")]))]
  ""
  "cmp4.%C1 %0, %I0 = %3, %2"
  [(set_attr "type" "A")])

(define_insn "*cmpdi_normal"
  [(set (match_operand:CC 0 "register_operand" "=c")
	(match_operator:CC 1 "normal_comparison_operator"
	   [(match_operand:DI 2 "gr_register_operand" "r")
	    (match_operand:DI 3 "gr_reg_or_8bit_operand" "rK")]))]
  ""
  "cmp.%C1 %0, %I0 = %3, %2"
  [(set_attr "type" "A")])

(define_insn "*cmpdi_adjusted"
  [(set (match_operand:CC 0 "register_operand" "=c")
	(match_operator:CC 1 "adjusted_comparison_operator"
	   [(match_operand:DI 2 "gr_register_operand" "r")
	    (match_operand:DI 3 "gr_reg_or_8bit_adjusted_operand" "rL")]))]
  ""
  "cmp.%C1 %0, %I0 = %3, %2"
  [(set_attr "type" "A")])

(define_insn "*cmpsf_internal"
  [(set (match_operand:CC 0 "register_operand" "=c")
	(match_operator:CC 1 "comparison_operator"
	   [(match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")
	    (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG")]))]
  ""
  "fcmp.%D1 %0, %I0 = %F2, %F3"
  [(set_attr "type" "F")])

(define_insn "*cmpdf_internal"
  [(set (match_operand:CC 0 "register_operand" "=c")
	(match_operator:CC 1 "comparison_operator"
	   [(match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")
	    (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")]))]
  ""
  "fcmp.%D1 %0, %I0 = %F2, %F3"
  [(set_attr "type" "F")])

(define_insn "*cmptf_internal"
  [(set (match_operand:CC 0 "register_operand" "=c")
	(match_operator:CC 1 "comparison_operator"
		   [(match_operand:TF 2 "tfreg_or_fp01_operand" "fG")
		    (match_operand:TF 3 "tfreg_or_fp01_operand" "fG")]))]
  ""
  "fcmp.%D1 %0, %I0 = %F2, %F3"
  [(set_attr "type" "F")])

;; ??? Can this pattern be generated?

(define_insn "*bit_zero"
  [(set (match_operand:CC 0 "register_operand" "=c")
	(eq:CC (zero_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
				(const_int 1)
				(match_operand:DI 2 "immediate_operand" "n"))
	       (const_int 0)))]
  ""
  "tbit.z %0, %I0 = %1, %2"
  [(set_attr "type" "I")])

(define_insn "*bit_one"
  [(set (match_operand:CC 0 "register_operand" "=c")
	(ne:CC (zero_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
				(const_int 1)
				(match_operand:DI 2 "immediate_operand" "n"))
	       (const_int 0)))]
  ""
  "tbit.nz %0, %I0 = %1, %2"
  [(set_attr "type" "I")])

;; ??? We also need this if we run out of PR regs and need to spill some.

;; ??? We need this if a CCmode value does not get allocated to a hard
;; register.  This happens if we cse/gcse a CCmode value across a call, and the
;; function has a nonlocal goto.  This is because global does not allocate
;; call crossing pseudos to hard registers when current_function_has_
;; nonlocal_goto is true.  This is relatively common for C++ programs that
;; use exceptions.  See ia64_secondary_reload_class.

;; We use a define_expand here so that cse/gcse/combine can't accidentally
;; create movcc insns.  If this was a named define_insn, we would not be able
;; to make it conditional on reload.

(define_expand "movcc"
  [(set (match_operand:CC 0 "nonimmediate_operand" "")
	(match_operand:CC 1 "move_operand" ""))]
  ""
  "
{
  if (! reload_in_progress && ! reload_completed)
    FAIL;
}")

(define_insn "*movcc_internal"
  [(set (match_operand:CC 0 "nonimmediate_operand" "=r,c,r,m")
	(match_operand:CC 1 "move_operand" "c,r,m,r"))]
  "reload_in_progress || reload_completed"
  "@
   #
   cmp4.ne %0, %I0 = %1, r0
   ld4%O1 %0 = %1%P1
   st4%Q0 %0 = %1%P0"
  [(set_attr "type" "unknown,A,M,M")])

(define_split
  [(set (match_operand:CC 0 "register_operand" "")
	(match_operand:CC 1 "register_operand" ""))]
  "reload_completed
   && GET_CODE (operands[0]) == REG && GR_REGNO_P (REGNO (operands[0]))
   && GET_CODE (operands[1]) == REG && PR_REGNO_P (REGNO (operands[1]))"
  [(set (match_dup 2)
	(if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
			 (const_int 1)
			 (match_dup 2)))
   (set (match_dup 2)
	(if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
			 (match_dup 2)
			 (const_int 0)))]
  "operands[2] = gen_rtx_SUBREG (DImode, operands[0], 0);")

\f
;; ::::::::::::::::::::
;; ::
;; :: Branches
;; ::
;; ::::::::::::::::::::

(define_expand "beq"
  [(set (match_dup 1)
	(eq:CC (match_dup 2)
	       (match_dup 3)))
   (set (pc)
	(if_then_else (ne:CC (match_dup 1)
			     (const_int 0))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bne"
  [(set (match_dup 1)
	(ne:CC (match_dup 2)
	       (match_dup 3)))
   (set (pc)
	(if_then_else (ne:CC (match_dup 1)
			     (const_int 0))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "blt"
  [(set (match_dup 1)
	(lt:CC (match_dup 2)
	       (match_dup 3)))
   (set (pc)
	(if_then_else (ne:CC (match_dup 1)
			     (const_int 0))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "ble"
  [(set (match_dup 1)
	(le:CC (match_dup 2)
	       (match_dup 3)))
   (set (pc)
	(if_then_else (ne:CC (match_dup 1)
			     (const_int 0))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bgt"
  [(set (match_dup 1)
	(gt:CC (match_dup 2)
	       (match_dup 3)))
   (set (pc)
	(if_then_else (ne:CC (match_dup 1)
			     (const_int 0))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bge"
  [(set (match_dup 1)
	(ge:CC (match_dup 2)
	       (match_dup 3)))
   (set (pc)
	(if_then_else (ne:CC (match_dup 1)
			     (const_int 0))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bltu"
  [(set (match_dup 1)
	(ltu:CC (match_dup 2)
		(match_dup 3)))
   (set (pc)
	(if_then_else (ne:CC (match_dup 1)
			     (const_int 0))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bleu"
  [(set (match_dup 1)
	(leu:CC (match_dup 2)
		(match_dup 3)))
   (set (pc)
	(if_then_else (ne:CC (match_dup 1)
			     (const_int 0))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bgtu"
  [(set (match_dup 1)
	(gtu:CC (match_dup 2)
		(match_dup 3)))
   (set (pc)
	(if_then_else (ne:CC (match_dup 1)
			     (const_int 0))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bgeu"
  [(set (match_dup 1)
	(geu:CC (match_dup 2)
		(match_dup 3)))
   (set (pc)
	(if_then_else (ne:CC (match_dup 1)
			     (const_int 0))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bunordered"
  [(set (match_dup 1)
	(unordered:CC (match_dup 2)
		      (match_dup 3)))
   (set (pc)
	(if_then_else (ne:CC (match_dup 1)
			     (const_int 0))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "bordered"
  [(set (match_dup 1)
	(ordered:CC (match_dup 2)
		      (match_dup 3)))
   (set (pc)
	(if_then_else (ne:CC (match_dup 1)
			     (const_int 0))
		      (label_ref (match_operand 0 "" ""))
		      (pc)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_insn "*br_true"
  [(set (pc)
	(if_then_else (match_operator 0 "predicate_operator"
			[(match_operand:CC 1 "register_operand" "c")
			 (const_int 0)])
		      (label_ref (match_operand 2 "" ""))
		      (pc)))]
  ""
  "(%J0) br.cond%+ %l2"
  [(set_attr "type" "B")
   (set_attr "predicable" "no")])

(define_insn "*br_false"
  [(set (pc)
	(if_then_else (match_operator 0 "predicate_operator"
			[(match_operand:CC 1 "register_operand" "c")
			 (const_int 0)])
		      (pc)
		      (label_ref (match_operand 2 "" ""))))]
  ""
  "(%j0) br.cond%+ %l2"
  [(set_attr "type" "B")
   (set_attr "predicable" "no")])
\f
;; ::::::::::::::::::::
;; ::
;; :: Counted loop operations
;; ::
;; ::::::::::::::::::::

(define_expand "doloop_end"
  [(use (match_operand 0 "" ""))	; loop pseudo
   (use (match_operand 1 "" ""))	; iterations; zero if unknown
   (use (match_operand 2 "" ""))	; max iterations
   (use (match_operand 3 "" ""))	; loop level
   (use (match_operand 4 "" ""))]	; label
  ""
  "
{
  /* Only use cloop on innermost loops.  */
  if (INTVAL (operands[3]) > 1)
    FAIL;
  emit_jump_insn (gen_doloop_end_internal (gen_rtx_REG (DImode, AR_LC_REGNUM),
					   operands[4]));
  DONE;
}")

(define_insn "doloop_end_internal"
  [(set (pc) (if_then_else (ne (match_operand:DI 0 "ar_lc_reg_operand" "")
			       (const_int 0))
		(label_ref (match_operand 1 "" ""))
		(pc)))
   (set (match_dup 0) (if_then_else:DI (ne (match_dup 0) (const_int 0))
			 (match_dup 0)
			 (plus:DI (match_dup 0) (const_int -1))))]
  ""
  "br.cloop.sptk.few %l1"
  [(set_attr "type" "B")
   (set_attr "predicable" "no")])
\f
;; ::::::::::::::::::::
;; ::
;; :: Set flag operations
;; ::
;; ::::::::::::::::::::

(define_expand "seq"
  [(set (match_dup 1)
	(eq:CC (match_dup 2)
	       (match_dup 3)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sne"
  [(set (match_dup 1)
	(ne:CC (match_dup 2)
	       (match_dup 3)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "slt"
  [(set (match_dup 1)
	(lt:CC (match_dup 2)
	       (match_dup 3)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sle"
  [(set (match_dup 1)
	(le:CC (match_dup 2)
	       (match_dup 3)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sgt"
  [(set (match_dup 1)
	(gt:CC (match_dup 2)
	       (match_dup 3)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sge"
  [(set (match_dup 1)
	(ge:CC (match_dup 2)
	       (match_dup 3)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sltu"
  [(set (match_dup 1)
	(ltu:CC (match_dup 2)
		(match_dup 3)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sleu"
  [(set (match_dup 1)
	(leu:CC (match_dup 2)
		(match_dup 3)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sgtu"
  [(set (match_dup 1)
	(gtu:CC (match_dup 2)
		(match_dup 3)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sgeu"
  [(set (match_dup 1)
	(geu:CC (match_dup 2)
		(match_dup 3)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sunordered"
  [(set (match_dup 1)
	(unordered:CC (match_dup 2)
		      (match_dup 3)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

(define_expand "sordered"
  [(set (match_dup 1)
	(ordered:CC (match_dup 2)
		      (match_dup 3)))
   (set (match_operand:DI 0 "gr_register_operand" "")
	(ne:DI (match_dup 1) (const_int 0)))]
  ""
  "
{
  operands[1] = gen_reg_rtx (CCmode);
  operands[2] = ia64_compare_op0;
  operands[3] = ia64_compare_op1;
}")

;; Don't allow memory as destination here, because cmov/cmov/st is more
;; efficient than mov/mov/cst/cst.

(define_insn_and_split "*sne_internal"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(ne:DI (match_operand:CC 1 "register_operand" "c")
	       (const_int 0)))]
  ""
  "#"
  "reload_completed"
  [(set (match_dup 0)
	(if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
			 (const_int 1)
			 (match_dup 0)))
   (set (match_dup 0)
	(if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
			 (match_dup 0)
			 (const_int 0)))]
  ""
  [(set_attr "type" "unknown")])

;; ??? Unknown if this can be matched.

(define_insn_and_split "*seq_internal"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(eq:DI (match_operand:CC 1 "register_operand" "c")
	       (const_int 0)))]
  ""
  "#"
  "reload_completed"
  [(set (match_dup 0)
	(if_then_else:DI (eq:CC (match_dup 1) (const_int 0))
			 (const_int 1)
			 (match_dup 0)))
   (set (match_dup 0)
	(if_then_else:DI (eq:CC (match_dup 1) (const_int 0))
			 (match_dup 0)
			 (const_int 0)))]
  ""
  [(set_attr "type" "unknown")])

\f
;; ::::::::::::::::::::
;; ::
;; :: Conditional move instructions.
;; ::
;; ::::::::::::::::::::

;; ??? Add movXXcc patterns?

;;
;; DImode if_then_else patterns.
;;

;; Errata 72 workaround.
(define_insn "*cmovdi_internal_astep"
  [(set (match_operand:DI 0 "nonimmediate_operand"
		"=r,*f,Q,*b,r,*f,Q,*b,r,*f,Q,*b")
	(if_then_else:DI
	  (match_operator:CC 4 "predicate_operator"
	    [(match_operand:CC 1 "register_operand"
			       "c,c,c,c,c,c,c,c,c,c,c,c")
	     (const_int 0)])
	  (match_operand:DI 2 "general_operand"
		"0,0,0,0,ri*f*b,rO,*f,r,ri*f*b,rO,*f,r")
	  (match_operand:DI 3 "general_operand"
		"ri*f*b,rO,*f,r,0,0,0,0,ri*f*b,rO,*f,r")))]
  "TARGET_A_STEP"
  "* abort ();"
  [(set_attr "predicable" "no")])

(define_insn "*cmovdi_internal"
  [(set (match_operand:DI 0 "nonimmediate_operand"
		"=r,m,*f,Q,*b,*d*e,r,m,*f,Q,*b,*d*e,r,m,*f,Q,*b,*d*e")
	(if_then_else:DI
	  (match_operator:CC 4 "predicate_operator"
	    [(match_operand:CC 1 "register_operand"
			       "c,c,c,c,c,c,c,c,c,c,c,c,c,c,c,c,c,c")
	     (const_int 0)])
	  (match_operand:DI 2 "general_operand"
		"0,0,0,0,0,0,rim*f*b*d*e,rO,rOQ,*f,rO,rK,rim*f*b*d*e,rO,rOQ,*f,rO,rK")
	  (match_operand:DI 3 "general_operand"
		"rim*f*b*d*e,rO,rOQ,*f,rO,rK,0,0,0,0,0,0,rim*f*b*d*e,rO,rOQ,*f,rO,rK")))]
  "! TARGET_A_STEP"
  "* abort ();"
  [(set_attr "predicable" "no")])

(define_split
  [(set (match_operand 0 "nonimmediate_operand" "")
	(if_then_else
	  (match_operator:CC 4 "predicate_operator"
	    [(match_operand:CC 1 "register_operand" "")
	     (const_int 0)])
	  (match_operand 2 "general_operand" "")
	  (match_operand 3 "general_operand" "")))]
  "reload_completed"
  [(const_int 0)]
  "
{
  rtx tmp;
  if (! rtx_equal_p (operands[0], operands[2]))
    {
      tmp = gen_rtx_SET (VOIDmode, operands[0], operands[2]);
      tmp = gen_rtx_COND_EXEC (VOIDmode, operands[4], tmp);
      emit_insn (tmp);
    }
  if (! rtx_equal_p (operands[0], operands[3]))
    {
      tmp = gen_rtx_fmt_ee (GET_CODE (operands[4]) == NE ? EQ : NE,
			    CCmode, operands[1], const0_rtx);
      tmp = gen_rtx_COND_EXEC (VOIDmode, tmp,
			       gen_rtx_SET (VOIDmode, operands[0],
					    operands[3]));
      emit_insn (tmp);
    }
  DONE;
}")

;; Absolute value pattern.

(define_insn "*absdi2_internal"
  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
	(if_then_else:DI
	  (match_operator:CC 4 "predicate_operator"
	    [(match_operand:CC 1 "register_operand" "c,c")
	     (const_int 0)])
	  (neg:DI (match_operand:DI 2 "gr_reg_or_22bit_operand" "rI,rI"))
	  (match_operand:DI 3 "gr_reg_or_22bit_operand" "0,rI")))]
  ""
  "#"
  [(set_attr "type" "A,unknown")
   (set_attr "predicable" "no")])

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
	(if_then_else:DI
	  (match_operator:CC 4 "predicate_operator"
	    [(match_operand:CC 1 "register_operand" "c,c")
	     (const_int 0)])
	  (neg:DI (match_operand:DI 2 "gr_reg_or_22bit_operand" ""))
	  (match_operand:DI 3 "gr_reg_or_22bit_operand" "")))]
  "reload_completed && rtx_equal_p (operands[0], operands[3])"
  [(cond_exec
     (match_dup 4)
     (set (match_dup 0)
	  (neg:DI (match_dup 2))))]
  "")

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
	(if_then_else:DI
	  (match_operator:CC 4 "predicate_operator"
	    [(match_operand:CC 1 "register_operand" "c,c")
	     (const_int 0)])
	  (neg:DI (match_operand:DI 2 "gr_reg_or_22bit_operand" ""))
	  (match_operand:DI 3 "gr_reg_or_22bit_operand" "")))]
  "reload_completed"
  [(cond_exec
     (match_dup 4)
     (set (match_dup 0) (neg:DI (match_dup 2))))
   (cond_exec
     (match_dup 5)
     (set (match_dup 0) (match_dup 3)))]
  "
{
  operands[5] = gen_rtx_fmt_ee (GET_CODE (operands[4]) == NE ? EQ : NE,
				CCmode, operands[1], const0_rtx);
}")

;;
;; SImode if_then_else patterns.
;;

(define_insn "*cmovsi_internal_astep"
  [(set (match_operand:SI 0 "nonimmediate_operand" "=r,*f,r,*f,r,*f")
	(if_then_else:SI
	  (match_operator:CC 4 "predicate_operator"
	    [(match_operand:CC 1 "register_operand" "c,c,c,c,c,c")
	     (const_int 0)])
	  (match_operand:SI 2 "general_operand"
		    "0,0,ri*f,rO,ri*f,rO")
	  (match_operand:SI 3 "general_operand"
		    "ri*f,rO,0,0,ri*f,rO")))]
  "TARGET_A_STEP"
  "* abort ();"
  [(set_attr "predicable" "no")])

(define_insn "*cmovsi_internal"
  [(set (match_operand:SI 0 "nonimmediate_operand" "=r,m,*f,r,m,*f,r,m,*f")
	(if_then_else:SI
	  (match_operator:CC 4 "predicate_operator"
	    [(match_operand:CC 1 "register_operand" "c,c,c,c,c,c,c,c,c")
	     (const_int 0)])
	  (match_operand:SI 2 "general_operand"
		    "0,0,0,rim*f,rO,rO,rim*f,rO,rO")
	  (match_operand:SI 3 "general_operand"
		    "rim*f,rO,rO,0,0,0,rim*f,rO,rO")))]
  "! TARGET_A_STEP"
  "* abort ();"
  [(set_attr "predicable" "no")])

(define_insn "*abssi2_internal"
  [(set (match_operand:SI 0 "gr_register_operand" "=r,r")
	(if_then_else:SI
	  (match_operator:CC 4 "predicate_operator"
	    [(match_operand:CC 1 "register_operand" "c,c")
	     (const_int 0)])
	  (neg:SI (match_operand:SI 3 "gr_reg_or_22bit_operand" "rI,rI"))
	  (match_operand:SI 2 "gr_reg_or_22bit_operand" "0,rI")))]
  ""
  "#"
  [(set_attr "type" "A,unknown")
   (set_attr "predicable" "no")])

(define_split
  [(set (match_operand:SI 0 "register_operand" "")
	(if_then_else:SI
	  (match_operator:CC 4 "predicate_operator"
	    [(match_operand:CC 1 "register_operand" "c,c")
	     (const_int 0)])
	  (neg:SI (match_operand:SI 2 "gr_reg_or_22bit_operand" ""))
	  (match_operand:SI 3 "gr_reg_or_22bit_operand" "")))]
  "reload_completed && rtx_equal_p (operands[0], operands[3])"
  [(cond_exec
     (match_dup 4)
     (set (match_dup 0)
	  (neg:SI (match_dup 2))))]
  "")

(define_split
  [(set (match_operand:SI 0 "register_operand" "")
	(if_then_else:SI
	  (match_operator:CC 4 "predicate_operator"
	    [(match_operand:CC 1 "register_operand" "c,c")
	     (const_int 0)])
	  (neg:SI (match_operand:SI 2 "gr_reg_or_22bit_operand" ""))
	  (match_operand:SI 3 "gr_reg_or_22bit_operand" "")))]
  "reload_completed"
  [(cond_exec
     (match_dup 4)
     (set (match_dup 0) (neg:SI (match_dup 2))))
   (cond_exec
     (match_dup 5)
     (set (match_dup 0) (match_dup 3)))]
  "
{
  operands[5] = gen_rtx_fmt_ee (GET_CODE (operands[4]) == NE ? EQ : NE,
				CCmode, operands[1], const0_rtx);
}")

\f
;; ::::::::::::::::::::
;; ::
;; :: Call and branch instructions
;; ::
;; ::::::::::::::::::::

;; Subroutine call instruction returning no value.  Operand 0 is the function
;; to call; operand 1 is the number of bytes of arguments pushed (in mode
;; `SImode', except it is normally a `const_int'); operand 2 is the number of
;; registers used as operands.

;; On most machines, operand 2 is not actually stored into the RTL pattern.  It
;; is supplied for the sake of some RISC machines which need to put this
;; information into the assembler code; they can put it in the RTL instead of
;; operand 1.

(define_expand "call"
  [(use (match_operand:DI 0 "" ""))
   (use (match_operand 1 "" ""))
   (use (match_operand 2 "" ""))
   (use (match_operand 3 "" ""))]
  ""
  "
{
  /* ??? Stripping off the MEM isn't correct.  Will lose alias info.  */
  rtx addr = XEXP (operands[0], 0);
  enum machine_mode mode = GET_MODE (addr);

  if (TARGET_NO_PIC || TARGET_AUTO_PIC)
    emit_call_insn (gen_call_internal (addr, operands[1],
				       gen_rtx_REG (DImode, R_BR (0))));

  /* If this is an indirect call, then we have the address of a descriptor.  */
  else if (! symbolic_operand (addr, mode))
    emit_insn (gen_indirect_call_pic (addr, operands[1]));
  else if (TARGET_CONST_GP)
    emit_call_insn (gen_call_internal (addr, operands[1],
				       gen_rtx_REG (DImode, R_BR (0))));
  else
    emit_insn (gen_call_pic (addr, operands[1]));

  DONE;
}")

(define_expand "indirect_call_pic"
  [(set (match_dup 2) (reg:DI 1))
   (set (match_dup 3) (mem:DI (match_operand 0 "" "")))
   (set (match_dup 4) (plus:DI (match_dup 0) (const_int 8)))
   (set (reg:DI 1) (mem:DI (match_dup 4)))
   (parallel [(call (mem:DI (match_dup 3)) (match_operand 1 "" ""))
	      (use (reg:DI 1))
	      (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 2))]
  ""
  "
{
  operands[2] = ia64_gp_save_reg (0);
  operands[3] = gen_reg_rtx (DImode);
  operands[4] = gen_reg_rtx (DImode);
}")

;; ??? Saving/restoring the GP register is not needed if we are calling
;; a function in the same module.

(define_expand "call_pic"
  [(set (match_dup 2) (reg:DI 1))
   (parallel [(call (mem:DI (match_operand 0 "" "")) (match_operand 1 "" ""))
	      (use (reg:DI 1))
	      (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 2))]
  ""
  "
{
  /* ??? Using setjmp_operand is an unsatisfying solution.  Should rethink.  */
  operands[2] = ia64_gp_save_reg (setjmp_operand (XEXP (operands[0], 0),
						  VOIDmode));
}")

(define_insn "call_internal"
  [(call (mem:DI (match_operand:DI 0 "call_operand" "bi"))
	 (match_operand 1 "" ""))
   (clobber (match_operand:DI 2 "register_operand" "=b"))]
  ""
  "br.call%+.many %2 = %0"
  [(set_attr "type" "B")])

(define_insn "*call_internal1"
  [(call (mem:DI (match_operand:DI 0 "call_operand" "bi"))
	 (match_operand 1 "" ""))
   (use (reg:DI 1))
   (clobber (match_operand:DI 2 "register_operand" "=b"))]
  ""
  "br.call%+.many %2 = %0"
  [(set_attr "type" "B")])

;; Subroutine call instruction returning a value.  Operand 0 is the hard
;; register in which the value is returned.  There are three more operands, the
;; same as the three operands of the `call' instruction (but with numbers
;; increased by one).

;; Subroutines that return `BLKmode' objects use the `call' insn.

(define_expand "call_value"
  [(use (match_operand 0 "" ""))
   (use (match_operand:DI 1 "" ""))
   (use (match_operand 2 "" ""))
   (use (match_operand 3 "" ""))
   (use (match_operand 4 "" ""))]
  ""
  "
{
  /* ??? Stripping off the MEM isn't correct.  Will lose alias info.  */
  rtx addr = XEXP (operands[1], 0);
  enum machine_mode mode = GET_MODE (addr);

  if (TARGET_NO_PIC || TARGET_AUTO_PIC)
    emit_call_insn (gen_call_value_internal (operands[0], addr, operands[2],
					     gen_rtx_REG (DImode, R_BR (0))));

  /* If this is an indirect call, then we have the address of a descriptor.  */
  else if (! symbolic_operand (addr, mode))
    {
      /* This is for HFA returns.  */
      if (GET_CODE (operands[0]) == PARALLEL)
	emit_insn (gen_indirect_call_multiple_values_pic (operands[0], addr,
							  operands[2]));
      else
	emit_insn (gen_indirect_call_value_pic (operands[0], addr,
						operands[2]));
    }
  else if (TARGET_CONST_GP)
    emit_call_insn (gen_call_value_internal (operands[0], addr, operands[2],
					     gen_rtx_REG (DImode, R_BR (0))));
  /* This is for HFA returns.  */
  else if (GET_CODE (operands[0]) == PARALLEL)
    emit_insn (gen_call_multiple_values_pic (operands[0], addr, operands[2]));
  else
    emit_insn (gen_call_value_pic (operands[0], addr, operands[2]));

  DONE;
}")

(define_expand "indirect_call_value_pic"
  [(set (match_dup 3) (reg:DI 1))
   (set (match_dup 4) (mem:DI (match_operand 1 "" "")))
   (set (match_dup 5) (plus:DI (match_dup 1) (const_int 8)))
   (set (reg:DI 1) (mem:DI (match_dup 5)))
   (parallel [(set (match_operand 0 "" "")
		   (call (mem:DI (match_dup 4)) (match_operand 2 "" "")))
	      (use (reg:DI 1))
	      (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 3))]
  ""
  "
{
  operands[3] = ia64_gp_save_reg (0);
  operands[4] = gen_reg_rtx (DImode);
  operands[5] = gen_reg_rtx (DImode);
}")

(define_expand "indirect_call_multiple_values_pic"
  [(set (match_dup 3) (reg:DI 1))
   (set (match_dup 4) (mem:DI (match_operand 1 "" "")))
   (set (match_dup 5) (plus:DI (match_dup 1) (const_int 8)))
   (set (reg:DI 1) (mem:DI (match_dup 5)))
   (match_par_dup 6 [(set (match_operand 0 "" "")
			  (call (mem:DI (match_dup 4))
				(match_operand 2 "" "")))
		     (use (reg:DI 1))
		     (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 3))]
  ""
  "
{
  int count;
  int i;
  rtx call;

  operands[3] = ia64_gp_save_reg (0);
  operands[4] = gen_reg_rtx (DImode);
  operands[5] = gen_reg_rtx (DImode);

  /* This code is the same as the code in call_multiple_values_pic, except
     that op3 was replaced with op6 and op1 was replaced with op4.  */
  call = gen_rtx_CALL (VOIDmode, gen_rtx_MEM (DImode, operands[4]),
		       operands[2]);

  count = XVECLEN (operands[0], 0);
  operands[6] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count + 2));
  
  XVECEXP (operands[6], 0, 0)
    = gen_rtx_SET (VOIDmode, XEXP (XVECEXP (operands[0], 0, 0), 0), call);

  XVECEXP (operands[6], 0, 1)
    = gen_rtx_USE (DImode, gen_rtx_REG (DImode, GR_REG (1)));
  XVECEXP (operands[6], 0, 2)
    = gen_rtx_CLOBBER (DImode, gen_rtx_REG (DImode, BR_REG (0)));

  for (i = 1; i < count; i++)
    XVECEXP (operands[6], 0, i + 2)
      = gen_rtx_SET (VOIDmode, XEXP (XVECEXP (operands[0], 0, i), 0), call);

}")

;; ??? Saving/restoring the GP register is not needed if we are calling
;; a function in the same module.

(define_expand "call_value_pic"
  [(set (match_dup 3) (reg:DI 1))
   (parallel [(set (match_operand 0 "" "")
		   (call (mem:DI (match_operand 1 "" ""))
			 (match_operand 2 "" "")))
	      (use (reg:DI 1))
	      (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 3))]
  ""
  "
{
  /* ??? Using setjmp_operand is an unsatisfying solution.  Should rethink.  */
  operands[3] = ia64_gp_save_reg (setjmp_operand (XEXP (operands[1], 0),
						  VOIDmode));
}")

;; ??? Saving/restoring the GP register is not needed if we are calling
;; a function in the same module.

(define_expand "call_multiple_values_pic"
  [(set (match_dup 4) (reg:DI 1))
   (match_par_dup 3 [(set (match_operand 0 "" "")
			  (call (mem:DI (match_operand 1 "" ""))
				(match_operand 2 "" "")))
		     (use (reg:DI 1))
		     (clobber (reg:DI 320))])
   (set (reg:DI 1) (match_dup 4))]
  ""
  "
{
  int count;
  int i;
  rtx call;

  operands[4] = ia64_gp_save_reg (0);

  call = gen_rtx_CALL (VOIDmode, gen_rtx_MEM (DImode, operands[1]),
		       operands[2]);

  count = XVECLEN (operands[0], 0);
  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count + 2));
  
  XVECEXP (operands[3], 0, 0)
    = gen_rtx_SET (VOIDmode, XEXP (XVECEXP (operands[0], 0, 0), 0), call);

  XVECEXP (operands[3], 0, 1)
    = gen_rtx_USE (DImode, gen_rtx_REG (DImode, GR_REG (1)));
  XVECEXP (operands[3], 0, 2)
    = gen_rtx_CLOBBER (DImode, gen_rtx_REG (DImode, BR_REG (0)));

  for (i = 1; i < count; i++)
    XVECEXP (operands[3], 0, i + 2)
      = gen_rtx_SET (VOIDmode, XEXP (XVECEXP (operands[0], 0, i), 0), call);
}")

(define_insn "call_value_internal"
  [(set (match_operand 0 "register_operand" "=rf")
	(call (mem:DI (match_operand:DI 1 "call_operand" "bi"))
	      (match_operand 2 "" "")))
   (clobber (match_operand:DI 3 "register_operand" "=b"))]
  ""
  "br.call%+.many %3 = %1"
  [(set_attr "type" "B")])

(define_insn "*call_value_internal1"
  [(set (match_operand 0 "register_operand" "=rf")
	(call (mem:DI (match_operand:DI 1 "call_operand" "bi"))
	      (match_operand 2 "" "")))
   (use (reg:DI 1))
   (clobber (match_operand:DI 3 "register_operand" "=b"))]
  ""
  "br.call%+.many %3 = %1"
  [(set_attr "type" "B")])

(define_insn "*call_multiple_values_internal1"
  [(match_parallel 0 "call_multiple_values_operation"
		   [(set (match_operand 1 "register_operand" "=rf")
			 (call (mem:DI (match_operand:DI 2 "call_operand" "bi"))
			       (match_operand 3 "" "")))
		    (use (reg:DI 1))
		    (clobber (match_operand:DI 4 "register_operand" "=b"))])]
  ""
  "br.call%+.many %4 = %2"
  [(set_attr "type" "B")])

;; Call subroutine returning any type.

(define_expand "untyped_call"
  [(parallel [(call (match_operand 0 "" "")
		    (const_int 0))
	      (match_operand 1 "" "")
	      (match_operand 2 "" "")])]
  ""
  "
{
  int i;

  emit_call_insn (gen_call (operands[0], const0_rtx, NULL, const0_rtx));

  for (i = 0; i < XVECLEN (operands[2], 0); i++)
    {
      rtx set = XVECEXP (operands[2], 0, i);
      emit_move_insn (SET_DEST (set), SET_SRC (set));
    }

  /* The optimizer does not know that the call sets the function value
     registers we stored in the result block.  We avoid problems by
     claiming that all hard registers are used and clobbered at this
     point.  */
  emit_insn (gen_blockage ());

  DONE;
}")

(define_insn "return_internal"
  [(return)
   (use (match_operand:DI 0 "register_operand" "b"))]
  ""
  "br.ret.sptk.many %0"
  [(set_attr "type" "B")])

(define_insn "return"
  [(return)]
  "ia64_direct_return ()"
  "br.ret.sptk.many rp"
  [(set_attr "type" "B")])

(define_insn "*return_true"
  [(set (pc)
	(if_then_else (match_operator 0 "predicate_operator"
			[(match_operand:CC 1 "register_operand" "c")
			 (const_int 0)])
		      (return)
		      (pc)))]
  "ia64_direct_return ()"
  "(%J0) br.ret%+.many rp"
  [(set_attr "type" "B")
   (set_attr "predicable" "no")])

(define_insn "*return_false"
  [(set (pc)
	(if_then_else (match_operator 0 "predicate_operator"
			[(match_operand:CC 1 "register_operand" "c")
			 (const_int 0)])
		      (pc)
		      (return)))]
  "ia64_direct_return ()"
  "(%j0) br.ret%+.many rp"
  [(set_attr "type" "B")
   (set_attr "predicable" "no")])

(define_insn "jump"
  [(set (pc) (label_ref (match_operand 0 "" "")))]
  ""
  "br %l0"
  [(set_attr "type" "B")])

(define_insn "indirect_jump"
  [(set (pc) (match_operand:DI 0 "register_operand" "b"))]
  ""
  "br %0"
  [(set_attr "type" "B")])

(define_expand "tablejump"
  [(match_operand:DI 0 "register_operand" "")
   (match_operand 1 "" "")]
  ""
  "
{
  rtx tmp1 = gen_reg_rtx (DImode);
  rtx tmp2 = gen_reg_rtx (DImode);

  emit_move_insn (tmp1, gen_rtx_LABEL_REF (Pmode, operands[1]));
  emit_insn (gen_adddi3 (tmp2, operands[0], tmp1));
  emit_jump_insn (gen_tablejump_internal (tmp2, operands[1]));
  DONE;
}")

(define_insn "tablejump_internal"
  [(set (pc) (match_operand:DI 0 "register_operand" "b"))
   (use (label_ref (match_operand 1 "" "")))]
  ""
  "br %0"
  [(set_attr "type" "B")])

\f
;; ::::::::::::::::::::
;; ::
;; :: Prologue and Epilogue instructions
;; ::
;; ::::::::::::::::::::

(define_expand "prologue"
  [(const_int 1)]
  ""
  "
{
  ia64_expand_prologue ();
  DONE;
}")

(define_expand "epilogue"
  [(const_int 2)]
  ""
  "
{
  ia64_expand_epilogue ();
  DONE;
}")

;; This prevents the scheduler from moving the SP decrement past FP-relative
;; stack accesses.  This is the same as adddi3 plus the extra set.

(define_insn "prologue_allocate_stack"
  [(set (match_operand:DI 0 "register_operand" "=r,r,r")
	(plus:DI (match_operand:DI 1 "register_operand" "%r,r,a")
		 (match_operand:DI 2 "gr_reg_or_22bit_operand" "r,I,J")))
   (set (match_operand:DI 3 "register_operand" "=r,r,r")
	(match_dup 3))]
  ""
  "@
  add %0 = %1, %2
  adds %0 = %2, %1
  addl %0 = %2, %1"
  [(set_attr "type" "A")])

;; This prevents the scheduler from moving the SP restore past FP-relative
;; stack accesses.  This is similar to movdi plus the extra set.

(define_insn "epilogue_deallocate_stack"
  [(set (match_operand:DI 0 "register_operand" "=r")
	(match_operand:DI 1 "register_operand" "+r"))
   (set (match_dup 1) (match_dup 1))]
  ""
  "mov %0 = %1"
  [(set_attr "type" "A")])

;; Allocate a new register frame.

(define_insn "alloc"
  [(set (match_operand:DI 0 "register_operand" "=r")
	(unspec_volatile:DI [(const_int 0)] 0))
   (use (match_operand:DI 1 "const_int_operand" "i"))
   (use (match_operand:DI 2 "const_int_operand" "i"))
   (use (match_operand:DI 3 "const_int_operand" "i"))
   (use (match_operand:DI 4 "const_int_operand" "i"))]
  ""
  "alloc %0 = ar.pfs, %1, %2, %3, %4"
  [(set_attr "type" "M")
   (set_attr "predicable" "no")])

;; Modifies ar.unat
(define_expand "gr_spill"
  [(parallel [(set (match_operand:DI 0 "memory_operand" "=m")
		   (unspec:DI [(match_operand:DI 1 "register_operand" "r")
			       (match_operand:DI 2 "const_int_operand" "")] 1))
	      (clobber (match_dup 3))])]
  ""
  "operands[3] = gen_rtx_REG (DImode, AR_UNAT_REGNUM);")

(define_insn "gr_spill_internal"
  [(set (match_operand:DI 0 "memory_operand" "=m")
	(unspec:DI [(match_operand:DI 1 "register_operand" "r")
		    (match_operand:DI 2 "const_int_operand" "")] 1))
   (clobber (match_operand:DI 3 "register_operand" ""))]
  ""
  ".mem.offset %2, 0\;st8.spill %0 = %1%P0"
  [(set_attr "type" "M")])

;; Reads ar.unat
(define_expand "gr_restore"
  [(parallel [(set (match_operand:DI 0 "register_operand" "=r")
		   (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
			       (match_operand:DI 2 "const_int_operand" "")] 2))
	      (use (match_dup 3))])]
  ""
  "operands[3] = gen_rtx_REG (DImode, AR_UNAT_REGNUM);")

(define_insn "gr_restore_internal"
  [(set (match_operand:DI 0 "register_operand" "=r")
	(unspec:DI [(match_operand:DI 1 "memory_operand" "m")
		    (match_operand:DI 2 "const_int_operand" "")] 2))
   (use (match_operand:DI 3 "register_operand" ""))]
  ""
  ".mem.offset %2, 0\;ld8.fill %0 = %1%P1"
  [(set_attr "type" "M")])

(define_insn "fr_spill"
  [(set (match_operand:TF 0 "memory_operand" "=m")
	(unspec:TF [(match_operand:TF 1 "register_operand" "f")] 3))]
  ""
  "stf.spill %0 = %1%P0"
  [(set_attr "type" "M")])

(define_insn "fr_restore"
  [(set (match_operand:TF 0 "register_operand" "=f")
	(unspec:TF [(match_operand:TF 1 "memory_operand" "m")] 4))]
  ""
  "ldf.fill %0 = %1%P1"
  [(set_attr "type" "M")])

(define_insn "bsp_value"
  [(set (match_operand:DI 0 "register_operand" "=r")
	(unspec:DI [(const_int 0)] 20))]
  ""
  "mov %0 = ar.bsp"
  [(set_attr "type" "I")])

(define_insn "set_bsp"
  [(unspec_volatile [(const_int 0)] 5)
   (use (match_operand:DI 0 "register_operand" "r"))]
  ""
  "flushrs\;mov r19=ar.rsc\;;;\;and r19=0x1c,r19\;;;\;mov ar.rsc=r19\;;;\;mov ar.bspstore=%0\;;;\;or r19=0x3,r19\;;;\;loadrs\;invala\;;;\;mov ar.rsc=r19"
  [(set_attr "type" "unknown")
   (set_attr "predicable" "no")])

(define_insn "flushrs"
  [(unspec [(const_int 0)] 21)]
  ""
  ";;\;flushrs"
  [(set_attr "type" "M")])
\f
;; ::::::::::::::::::::
;; ::
;; :: Miscellaneous instructions
;; ::
;; ::::::::::::::::::::

;; ??? Emiting a NOP instruction isn't very useful.  This should probably
;; be emitting ";;" to force a break in the instruction packing.

;; No operation, needed in case the user uses -g but not -O.
(define_insn "nop"
  [(const_int 0)]
  ""
  "nop 0"
  [(set_attr "type" "unknown")])

;; Pseudo instruction that prevents the scheduler from moving code above this
;; point.
(define_insn "blockage"
  [(unspec_volatile [(const_int 0)] 1)]
  ""
  ""
  [(set_attr "type" "unknown")
   (set_attr "predicable" "no")])

(define_insn "insn_group_barrier"
  [(unspec_volatile [(const_int 0)] 2)]
  ""
  ";;"
  [(set_attr "type" "S")
   (set_attr "predicable" "no")])

\f
;; Non-local goto support.

(define_expand "save_stack_nonlocal"
  [(use (match_operand:OI 0 "memory_operand" ""))
   (use (match_operand:DI 1 "register_operand" ""))]
  ""
  "
{
  emit_library_call (gen_rtx_SYMBOL_REF (Pmode,
					 \"__ia64_save_stack_nonlocal\"),
		     0, VOIDmode, 2, XEXP (operands[0], 0), Pmode,
		     operands[1], Pmode);
  DONE;
}")

(define_expand "nonlocal_goto"
  [(use (match_operand 0 "general_operand" ""))
   (use (match_operand 1 "general_operand" ""))
   (use (match_operand 2 "general_operand" ""))
   (use (match_operand 3 "general_operand" ""))]
  ""
  "
{
  emit_library_call (gen_rtx_SYMBOL_REF (Pmode, \"__ia64_nonlocal_goto\"),
		     0, VOIDmode, 3,
		     operands[1], Pmode,
		     copy_to_reg (XEXP (operands[2], 0)), Pmode,
		     operands[3], Pmode);
  emit_barrier ();
  DONE;
}")

;; Restore the GP after the exception/longjmp.  The preceeding call will
;; have tucked it away.
(define_expand "exception_receiver"
  [(set (reg:DI 1) (match_dup 0))]
  ""
  "operands[0] = ia64_gp_save_reg (0);")

;; The rest of the setjmp processing happens with the nonlocal_goto expander.
;; ??? This is not tested.
(define_expand "builtin_setjmp_setup"
  [(use (match_operand:DI 0 "" ""))]
  ""
  "
{
  emit_move_insn (ia64_gp_save_reg (0), gen_rtx_REG (DImode, GR_REG (1)));
  DONE;
}")

(define_expand "builtin_setjmp_receiver"
  [(use (match_operand:DI 0 "" ""))]
  ""
  "
{
  emit_move_insn (gen_rtx_REG (DImode, GR_REG (1)), ia64_gp_save_reg (0));
  DONE;
}")

(define_expand "eh_epilogue"
  [(use (match_operand:DI 0 "register_operand" "r"))
   (use (match_operand:DI 1 "register_operand" "r"))
   (use (match_operand:DI 2 "register_operand" "r"))]
  ""
  "
{
  rtx bsp = gen_rtx_REG (Pmode, 10);
  rtx sp = gen_rtx_REG (Pmode, 9);

  if (GET_CODE (operands[0]) != REG || REGNO (operands[0]) != 10)
    {
      emit_move_insn (bsp, operands[0]);
      operands[0] = bsp;
    }
  if (GET_CODE (operands[2]) != REG || REGNO (operands[2]) != 9)
    {
      emit_move_insn (sp, operands[2]);
      operands[2] = sp;
    }
  emit_insn (gen_rtx_USE (VOIDmode, sp));
  emit_insn (gen_rtx_USE (VOIDmode, bsp));

  cfun->machine->ia64_eh_epilogue_sp = sp;
  cfun->machine->ia64_eh_epilogue_bsp = bsp;
}")
\f
;; Builtin apply support.

(define_expand "restore_stack_nonlocal"
  [(use (match_operand:DI 0 "register_operand" ""))
   (use (match_operand:OI 1 "memory_operand" ""))]
  ""
  "
{
  emit_library_call (gen_rtx_SYMBOL_REF (Pmode,
					 \"__ia64_restore_stack_nonlocal\"),
		     0, VOIDmode, 1,
		     copy_to_reg (XEXP (operands[1], 0)), Pmode);
  DONE;
}")

\f
;;; Intrinsics support.

(define_expand "mf"
  [(set (mem:BLK (match_dup 0))
	(unspec:BLK [(mem:BLK (match_dup 0))] 12))]
  ""
  "
{
  operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (DImode));
  MEM_VOLATILE_P (operands[0]) = 1;
}")

(define_insn "*mf_internal"
  [(set (match_operand:BLK 0 "" "")
	(unspec:BLK [(match_operand:BLK 1 "" "")] 12))]
  ""
  "mf"
  [(set_attr "type" "M")])

(define_insn "fetchadd_acq_si"
  [(set (match_operand:SI 0 "gr_register_operand" "=r")
	(match_dup 1))
   (set (match_operand:SI 1 "not_postinc_memory_operand" "+S")
	(unspec:SI [(match_dup 1)
		    (match_operand:SI 2 "fetchadd_operand" "n")] 19))]
  ""
  "fetchadd4.acq %0 = %1, %2"
  [(set_attr "type" "M")])

(define_insn "fetchadd_acq_di"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(match_dup 1))
   (set (match_operand:DI 1 "not_postinc_memory_operand" "+S")
	(unspec:DI [(match_dup 1)
		    (match_operand:DI 2 "fetchadd_operand" "n")] 19))]
  ""
  "fetchadd8.acq %0 = %1, %2"
  [(set_attr "type" "M")])

(define_insn "cmpxchg_acq_si"
  [(set (match_operand:SI 0 "gr_register_operand" "=r")
	(match_dup 1))
   (set (match_operand:SI 1 "not_postinc_memory_operand" "+S")
        (unspec:SI [(match_dup 1)
                    (match_operand:SI 2 "gr_register_operand" "r")
		    (match_operand:SI 3 "ar_ccv_reg_operand" "")] 13))]
  ""
  "cmpxchg4.acq %0 = %1, %2, %3"
  [(set_attr "type" "M")])

(define_insn "cmpxchg_acq_di"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
	(match_dup 1))
   (set (match_operand:DI 1 "not_postinc_memory_operand" "+S")
        (unspec:DI [(match_dup 1)
                    (match_operand:DI 2 "gr_register_operand" "r")
		    (match_operand:DI 3 "ar_ccv_reg_operand" "")] 13))]
  ""
  "cmpxchg8.acq %0 = %1, %2, %3"
  [(set_attr "type" "M")])

(define_insn "xchgsi"
  [(set (match_operand:SI 0 "gr_register_operand" "=r")
        (match_operand:SI 1 "not_postinc_memory_operand" "+S"))
   (set (match_dup 1)
        (match_operand:SI 2 "gr_register_operand" "r"))]
  ""
  "xchg4 %0 = %1, %2"
  [(set_attr "type" "M")])

(define_insn "xchgdi"
  [(set (match_operand:DI 0 "gr_register_operand" "=r")
        (match_operand:DI 1 "not_postinc_memory_operand" "+S"))
   (set (match_dup 1)
        (match_operand:DI 2 "gr_register_operand" "r"))]
  ""
  "xchg8 %0 = %1, %2"
  [(set_attr "type" "M")])
\f
;; Predication.

(define_cond_exec
  [(match_operator 0 "predicate_operator"
     [(match_operand:CC 1 "register_operand" "c")
      (const_int 0)])]
  ""
  "(%J0)")

(define_insn "pred_rel_mutex"
  [(unspec_volatile [(match_operand:CC 0 "register_operand" "c")] 7)]
  ""
  ".pred.rel.mutex %0, %I0"
  [(set_attr "type" "unknown")
   (set_attr "predicable" "no")])

(define_insn "safe_across_calls_all"
  [(unspec_volatile [(const_int 0)] 8)]
  ""
  ".pred.safe_across_calls p1-p63"
  [(set_attr "type" "unknown")
   (set_attr "predicable" "no")])

(define_insn "safe_across_calls_normal"
  [(unspec_volatile [(const_int 0)] 9)]
  ""
  "*
{
  emit_safe_across_calls (asm_out_file);
  return \"\";
}"
  [(set_attr "type" "unknown")
   (set_attr "predicable" "no")])