[i386] Fix ambiguous .md attribute uses

[thirdparty/gcc.git] / gcc / config / s390 / s390.md

;;- Machine description for GNU compiler -- S/390 / zSeries version.
;;  Copyright (C) 1999-2019 Free Software Foundation, Inc.
;;  Contributed by Hartmut Penner (hpenner@de.ibm.com) and
;;                 Ulrich Weigand (uweigand@de.ibm.com) and
;;                 Andreas Krebbel (Andreas.Krebbel@de.ibm.com)

;; This file is part of GCC.

;; GCC is free software; you can redistribute it and/or modify it under
;; the terms of the GNU General Public License as published by the Free
;; Software Foundation; either version 3, or (at your option) any later
;; version.

;; GCC is distributed in the hope that it will be useful, but WITHOUT ANY
;; WARRANTY; without even the implied warranty of MERCHANTABILITY or
;; FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
;; for more details.

;; You should have received a copy of the GNU General Public License
;; along with GCC; see the file COPYING3.  If not see
;; <http://www.gnu.org/licenses/>.

;;
;; See constraints.md for a description of constraints specific to s390.
;;

;; Special formats used for outputting 390 instructions.
;;
;;     %C: print opcode suffix for branch condition.
;;     %D: print opcode suffix for inverse branch condition.
;;     %J: print tls_load/tls_gdcall/tls_ldcall suffix
;;     %G: print the size of the operand in bytes.
;;     %O: print only the displacement of a memory reference.
;;     %R: print only the base register of a memory reference.
;;     %S: print S-type memory reference (base+displacement).
;;     %N: print the second word of a DImode operand.
;;     %M: print the second word of a TImode operand.
;;     %Y: print shift count operand.
;;
;;     %b: print integer X as if it's an unsigned byte.
;;     %c: print integer X as if it's an signed byte.
;;     %x: print integer X as if it's an unsigned halfword.
;;     %h: print integer X as if it's a signed halfword.
;;     %i: print the first nonzero HImode part of X.
;;     %j: print the first HImode part unequal to -1 of X.
;;     %k: print the first nonzero SImode part of X.
;;     %m: print the first SImode part unequal to -1 of X.
;;     %o: print integer X as if it's an unsigned 32bit word.
;;
;; We have a special constraint for pattern matching.
;;
;;   s_operand -- Matches a valid S operand in a RS, SI or SS type instruction.
;;

;;
;; UNSPEC usage
;;

(define_c_enum "unspec" [
   ; Miscellaneous
   UNSPEC_ROUND
   UNSPEC_ICM
   UNSPEC_TIE

   ; Convert CC into a str comparison result and copy it into an
   ; integer register
   ; cc0->0, cc1->1, cc2->-1, (cc3->-1)
   UNSPEC_STRCMPCC_TO_INT

   ; Copy CC as is into the lower 2 bits of an integer register
   UNSPEC_CC_TO_INT

   ; The right hand side of an setmem
   UNSPEC_REPLICATE_BYTE

   ; GOT/PLT and lt-relative accesses
   UNSPEC_LTREL_OFFSET
   UNSPEC_POOL_OFFSET
   UNSPEC_GOTENT
   UNSPEC_GOT
   UNSPEC_GOTOFF
   UNSPEC_PLT
   UNSPEC_PLTOFF

   ; Literal pool
   UNSPEC_RELOAD_BASE
   UNSPEC_MAIN_BASE
   UNSPEC_LTREF
   UNSPEC_INSN
   UNSPEC_EXECUTE
   UNSPEC_EXECUTE_JUMP

   ; Atomic Support
   UNSPEC_MB
   UNSPEC_MOVA

   ; TLS relocation specifiers
   UNSPEC_TLSGD
   UNSPEC_TLSLDM
   UNSPEC_NTPOFF
   UNSPEC_DTPOFF
   UNSPEC_GOTNTPOFF
   UNSPEC_INDNTPOFF

   ; TLS support
   UNSPEC_TLSLDM_NTPOFF
   UNSPEC_TLS_LOAD

   ; String Functions
   UNSPEC_SRST
   UNSPEC_MVST

   ; Stack Smashing Protector
   UNSPEC_SP_SET
   UNSPEC_SP_TEST

   ; Split stack support
   UNSPEC_STACK_CHECK

   ; Test Data Class (TDC)
   UNSPEC_TDC_INSN

   ; Byte-wise Population Count
   UNSPEC_POPCNT
   UNSPEC_COPYSIGN

   ; Load FP Integer
   UNSPEC_FPINT_FLOOR
   UNSPEC_FPINT_BTRUNC
   UNSPEC_FPINT_ROUND
   UNSPEC_FPINT_CEIL
   UNSPEC_FPINT_NEARBYINT
   UNSPEC_FPINT_RINT

   UNSPEC_LCBB

   ; Vector
   UNSPEC_VEC_SMULT_HI
   UNSPEC_VEC_UMULT_HI
   UNSPEC_VEC_SMULT_LO
   UNSPEC_VEC_SMULT_EVEN
   UNSPEC_VEC_UMULT_EVEN
   UNSPEC_VEC_SMULT_ODD
   UNSPEC_VEC_UMULT_ODD

   UNSPEC_VEC_VMAL
   UNSPEC_VEC_VMAH
   UNSPEC_VEC_VMALH
   UNSPEC_VEC_VMAE
   UNSPEC_VEC_VMALE
   UNSPEC_VEC_VMAO
   UNSPEC_VEC_VMALO

   UNSPEC_VEC_GATHER
   UNSPEC_VEC_EXTRACT
   UNSPEC_VEC_INSERT_AND_ZERO
   UNSPEC_VEC_LOAD_BNDRY
   UNSPEC_VEC_LOAD_LEN
   UNSPEC_VEC_LOAD_LEN_R
   UNSPEC_VEC_MERGEH
   UNSPEC_VEC_MERGEL
   UNSPEC_VEC_PACK
   UNSPEC_VEC_PACK_SATURATE
   UNSPEC_VEC_PACK_SATURATE_CC
   UNSPEC_VEC_PACK_SATURATE_GENCC
   UNSPEC_VEC_PACK_UNSIGNED_SATURATE
   UNSPEC_VEC_PACK_UNSIGNED_SATURATE_CC
   UNSPEC_VEC_PACK_UNSIGNED_SATURATE_GENCC
   UNSPEC_VEC_PERM
   UNSPEC_VEC_PERMI
   UNSPEC_VEC_EXTEND
   UNSPEC_VEC_STORE_LEN
   UNSPEC_VEC_STORE_LEN_R
   UNSPEC_VEC_VBPERM
   UNSPEC_VEC_UNPACKH
   UNSPEC_VEC_UNPACKH_L
   UNSPEC_VEC_UNPACKL
   UNSPEC_VEC_UNPACKL_L
   UNSPEC_VEC_ADDC
   UNSPEC_VEC_ADDE_U128
   UNSPEC_VEC_ADDEC_U128
   UNSPEC_VEC_AVG
   UNSPEC_VEC_AVGU
   UNSPEC_VEC_CHECKSUM
   UNSPEC_VEC_GFMSUM
   UNSPEC_VEC_GFMSUM_128
   UNSPEC_VEC_GFMSUM_ACCUM
   UNSPEC_VEC_GFMSUM_ACCUM_128
   UNSPEC_VEC_SET

   UNSPEC_VEC_VSUMG
   UNSPEC_VEC_VSUMQ
   UNSPEC_VEC_VSUM
   UNSPEC_VEC_RL_MASK
   UNSPEC_VEC_SLL
   UNSPEC_VEC_SLB
   UNSPEC_VEC_SLDBYTE
   UNSPEC_VEC_SLDBIT
   UNSPEC_VEC_SRDBIT
   UNSPEC_VEC_SRAL
   UNSPEC_VEC_SRAB
   UNSPEC_VEC_SRL
   UNSPEC_VEC_SRLB

   UNSPEC_VEC_SUBC
   UNSPEC_VEC_SUBE_U128
   UNSPEC_VEC_SUBEC_U128

   UNSPEC_VEC_TEST_MASK

   UNSPEC_VEC_VFAE
   UNSPEC_VEC_VFAECC

   UNSPEC_VEC_VFEE
   UNSPEC_VEC_VFEECC
   UNSPEC_VEC_VFENE
   UNSPEC_VEC_VFENECC

   UNSPEC_VEC_VISTR
   UNSPEC_VEC_VISTRCC

   UNSPEC_VEC_VSTRC
   UNSPEC_VEC_VSTRCCC

   UNSPEC_VEC_VSTRS
   UNSPEC_VEC_VSTRSCC

   UNSPEC_VEC_VCDGB
   UNSPEC_VEC_VCDLGB

   UNSPEC_VEC_VCGDB
   UNSPEC_VEC_VCLGDB

   UNSPEC_VEC_VFI

   UNSPEC_VEC_VFLL        ; vector fp load lengthened
   UNSPEC_VEC_VFLR        ; vector fp load rounded

   UNSPEC_VEC_VFTCI
   UNSPEC_VEC_VFTCICC

   UNSPEC_VEC_MSUM

   UNSPEC_VEC_VFMIN
   UNSPEC_VEC_VFMAX

   UNSPEC_VEC_ELTSWAP
])

;;
;; UNSPEC_VOLATILE usage
;;

(define_c_enum "unspecv" [
   ; Blockage
   UNSPECV_BLOCKAGE

   ; TPF Support
   UNSPECV_TPF_PROLOGUE
   UNSPECV_TPF_EPILOGUE

   ; Literal pool
   UNSPECV_POOL
   UNSPECV_POOL_SECTION
   UNSPECV_POOL_ALIGN
   UNSPECV_POOL_ENTRY
   UNSPECV_MAIN_POOL

   ; TLS support
   UNSPECV_SET_TP

   ; Atomic Support
   UNSPECV_CAS
   UNSPECV_ATOMIC_OP

   ; Non-branch nops used for compare-and-branch adjustments on z10
   UNSPECV_NOP_LR_0
   UNSPECV_NOP_LR_1

   ; Hotpatching (unremovable NOPs)
   UNSPECV_NOP_2_BYTE
   UNSPECV_NOP_4_BYTE
   UNSPECV_NOP_6_BYTE

   ; Transactional Execution support
   UNSPECV_TBEGIN
   UNSPECV_TBEGIN_TDB
   UNSPECV_TBEGINC
   UNSPECV_TEND
   UNSPECV_TABORT
   UNSPECV_ETND
   UNSPECV_NTSTG
   UNSPECV_PPA

   ; Set and get floating point control register
   UNSPECV_SFPC
   UNSPECV_EFPC

   ; Split stack support
   UNSPECV_SPLIT_STACK_CALL
   UNSPECV_SPLIT_STACK_DATA

   UNSPECV_OSC_BREAK
  ])

;;
;; Registers
;;

; Registers with special meaning

(define_constants
  [
   ; Sibling call register.
   (SIBCALL_REGNUM		 1)
   ; A call-clobbered reg which can be used in indirect branch thunks
   (INDIRECT_BRANCH_THUNK_REGNUM 1)
   ; Literal pool base register.
   (BASE_REGNUM			13)
   ; Return address register.
   (RETURN_REGNUM		14)
   ; Stack pointer register.
   (STACK_REGNUM		15)
   ; Condition code register.
   (CC_REGNUM			33)
   ; Thread local storage pointer register.
   (TP_REGNUM			36)
  ])

; Hardware register names

(define_constants
  [
   ; General purpose registers
   (GPR0_REGNUM                  0)
   (GPR1_REGNUM                  1)
   (GPR2_REGNUM                  2)
   (GPR6_REGNUM                  6)
   ; Floating point registers.
   (FPR0_REGNUM                 16)
   (FPR1_REGNUM                 20)
   (FPR2_REGNUM                 17)
   (FPR3_REGNUM                 21)
   (FPR4_REGNUM                 18)
   (FPR5_REGNUM                 22)
   (FPR6_REGNUM                 19)
   (FPR7_REGNUM                 23)
   (FPR8_REGNUM                 24)
   (FPR9_REGNUM                 28)
   (FPR10_REGNUM                25)
   (FPR11_REGNUM                29)
   (FPR12_REGNUM                26)
   (FPR13_REGNUM                30)
   (FPR14_REGNUM                27)
   (FPR15_REGNUM                31)
   (VR0_REGNUM                  16)
   (VR16_REGNUM                 38)
   (VR23_REGNUM                 45)
   (VR24_REGNUM                 46)
   (VR31_REGNUM                 53)
  ])

; Rounding modes for binary floating point numbers
(define_constants
  [(BFP_RND_CURRENT                 0)
   (BFP_RND_NEAREST_TIE_AWAY_FROM_0 1)
   (BFP_RND_PREP_FOR_SHORT_PREC     3)
   (BFP_RND_NEAREST_TIE_TO_EVEN     4)
   (BFP_RND_TOWARD_0                5)
   (BFP_RND_TOWARD_INF              6)
   (BFP_RND_TOWARD_MINF             7)])

; Rounding modes for decimal floating point numbers
; 1-7 were introduced with the floating point extension facility
; available with z196
; With these rounding modes (1-7) a quantum exception might occur
; which is suppressed for the other modes.
(define_constants
  [(DFP_RND_CURRENT                          0)
   (DFP_RND_NEAREST_TIE_AWAY_FROM_0_QUANTEXC 1)
   (DFP_RND_CURRENT_QUANTEXC                 2)
   (DFP_RND_PREP_FOR_SHORT_PREC_QUANTEXC     3)
   (DFP_RND_NEAREST_TIE_TO_EVEN_QUANTEXC     4)
   (DFP_RND_TOWARD_0_QUANTEXC                5)
   (DFP_RND_TOWARD_INF_QUANTEXC              6)
   (DFP_RND_TOWARD_MINF_QUANTEXC             7)
   (DFP_RND_NEAREST_TIE_TO_EVEN              8)
   (DFP_RND_TOWARD_0                         9)
   (DFP_RND_TOWARD_INF                      10)
   (DFP_RND_TOWARD_MINF                     11)
   (DFP_RND_NEAREST_TIE_AWAY_FROM_0         12)
   (DFP_RND_NEAREST_TIE_TO_0                13)
   (DFP_RND_AWAY_FROM_0                     14)
   (DFP_RND_PREP_FOR_SHORT_PREC             15)])

;;
;; PFPO GPR0 argument format
;;

(define_constants
  [
   ; PFPO operation type
   (PFPO_CONVERT          0x1000000)
   ; PFPO operand types
   (PFPO_OP_TYPE_SF             0x5)
   (PFPO_OP_TYPE_DF             0x6)
   (PFPO_OP_TYPE_TF             0x7)
   (PFPO_OP_TYPE_SD             0x8)
   (PFPO_OP_TYPE_DD             0x9)
   (PFPO_OP_TYPE_TD             0xa)
   ; Bitposition of operand types
   (PFPO_OP0_TYPE_SHIFT          16)
   (PFPO_OP1_TYPE_SHIFT           8)
   ; Decide whether current DFP or BFD rounding mode should be used
   ; for the conversion.
   (PFPO_RND_MODE_DFP             0)
   (PFPO_RND_MODE_BFP             1)
  ])

;; PPA constants

; Immediate values which can be used as the third operand to the
; perform processor assist instruction

(define_constants
  [(PPA_TX_ABORT                 1)
   (PPA_OOO_BARRIER             15)])

; Immediate operands for tbegin and tbeginc
(define_constants [(TBEGIN_MASK  65292)]) ; 0xff0c
(define_constants [(TBEGINC_MASK 65288)]) ; 0xff08

;; Instruction operand type as used in the Principles of Operation.
;; Used to determine defaults for length and other attribute values.

(define_attr "op_type"
  "NN,E,RR,RRE,RX,RS,RSI,RI,SI,S,SS,SSE,RXE,RSE,RIL,RIE,RXY,RSY,SIY,RRF,SIL,RRS,RIS,VRI,VRR,VRS,VRV,VRX,VSI"
  (const_string "NN"))

;; Instruction type attribute used for scheduling.

(define_attr "type" "none,integer,load,lr,la,larl,lm,stm,
	             cs,vs,store,sem,idiv,
                     imulhi,imulsi,imuldi,
		     branch,jsr,fsimptf,fsimpdf,fsimpsf,fhex,
		     floadtf,floaddf,floadsf,fstoredf,fstoresf,
		     fmultf,fmuldf,fmulsf,fdivtf,fdivdf,fdivsf,
		     ftoi,fsqrttf,fsqrtdf,fsqrtsf,
		     fmadddf,fmaddsf,
                     ftrunctf,ftruncdf, ftruncsd, ftruncdd,
                     itoftf, itofdf, itofsf, itofdd, itoftd,
                     fdivdd, fdivtd, floaddd, floadsd, fmuldd, fmultd,
                     fsimpdd, fsimpsd, fsimptd, fstoredd, fstoresd,
                     ftoidfp, other"
  (cond [(eq_attr "op_type" "NN")  (const_string "other")
         (eq_attr "op_type" "SS")  (const_string "cs")]
    (const_string "integer")))

;; Another attribute used for scheduling purposes:
;;   agen: Instruction uses the address generation unit
;;   reg: Instruction does not use the agen unit

(define_attr "atype" "agen,reg"
  (if_then_else (eq_attr "op_type" "E,RR,RI,RRE,RSI,RIL,RIE,RRF")
		(const_string "reg")
		(const_string "agen")))

;; Properties concerning Z10 execution grouping and value forwarding.
;; z10_super: instruction is superscalar.
;; z10_super_c: instruction is superscalar and meets the condition of z10_c.
;; z10_fwd: The instruction reads the value of an operand and stores it into a
;;   target register.  It can forward this value to a second instruction that reads
;;   the same register if that second instruction is issued in the same group.
;; z10_rec: The instruction is in the T pipeline and reads a register. If the
;;   instruction in the S pipe writes to the register, then the T instruction
;;   can immediately read the new value.
;; z10_fr: union of Z10_fwd and z10_rec.
;; z10_c: second operand of instruction is a register and read with complemented bits.
;;
;; An additional suffix A1, A3, or E1 indicates the respective AGI bypass.


(define_attr "z10prop" "none,
                        z10_super, z10_super_E1, z10_super_A1, z10_super_c, z10_super_c_E1,
                        z10_fwd, z10_fwd_A1, z10_fwd_A3, z10_fwd_E1,
                        z10_rec,
                        z10_fr, z10_fr_A3, z10_fr_E1,
                        z10_c"
             (const_string "none"))

;; Properties concerning Z196 decoding
;; z196_alone: must group alone
;; z196_end: ends a group
;; z196_cracked: instruction is cracked or expanded
(define_attr "z196prop" "none,
                         z196_alone, z196_ends,
                         z196_cracked"
             (const_string "none"))

; mnemonics which only get defined through if_then_else currently
; don't get added to the list values automatically and hence need to
; be listed here.
(define_attr "mnemonic" "b,bas,basr,bc,bcr_flush,unknown" (const_string "unknown"))

;; Length in bytes.

(define_attr "length" ""
  (cond [(eq_attr "op_type" "E,RR")		          (const_int 2)
         (eq_attr "op_type" "RX,RI,RRE,RS,RSI,S,SI,RRF")  (const_int 4)]
    (const_int 6)))


;; Processor type.  This attribute must exactly match the processor_type
;; enumeration in s390.h.

(define_attr "cpu" "z900,z990,z9_109,z9_ec,z10,z196,zEC12,z13,z14,arch13"
  (const (symbol_ref "s390_tune_attr")))

(define_attr "cpu_facility"
  "standard,ieee,zarch,cpu_zarch,longdisp,extimm,dfp,z10,z196,zEC12,vx,z13,z14,vxe,arch13,vxe2"
  (const_string "standard"))

(define_attr "enabled" ""
  (cond [(eq_attr "cpu_facility" "standard")
	 (const_int 1)

         (and (eq_attr "cpu_facility" "ieee")
	      (match_test "TARGET_CPU_IEEE_FLOAT"))
	 (const_int 1)

	 (and (eq_attr "cpu_facility" "zarch")
	      (match_test "TARGET_ZARCH"))
	 (const_int 1)

	 (and (eq_attr "cpu_facility" "longdisp")
	      (match_test "TARGET_LONG_DISPLACEMENT"))
	 (const_int 1)

         (and (eq_attr "cpu_facility" "extimm")
	      (match_test "TARGET_EXTIMM"))
	 (const_int 1)

         (and (eq_attr "cpu_facility" "dfp")
	      (match_test "TARGET_DFP"))
	 (const_int 1)

         (eq_attr "cpu_facility" "cpu_zarch")
	 (const_int 1)

         (and (eq_attr "cpu_facility" "z10")
              (match_test "TARGET_Z10"))
	 (const_int 1)

         (and (eq_attr "cpu_facility" "z196")
              (match_test "TARGET_Z196"))
	 (const_int 1)

         (and (eq_attr "cpu_facility" "zEC12")
              (match_test "TARGET_ZEC12"))
	 (const_int 1)

         (and (eq_attr "cpu_facility" "vx")
              (match_test "TARGET_VX"))
	 (const_int 1)

         (and (eq_attr "cpu_facility" "z13")
              (match_test "TARGET_Z13"))
	 (const_int 1)

         (and (eq_attr "cpu_facility" "z14")
              (match_test "TARGET_Z14"))
	 (const_int 1)

         (and (eq_attr "cpu_facility" "vxe")
	      (match_test "TARGET_VXE"))
	 (const_int 1)

	 (and (eq_attr "cpu_facility" "arch13")
	      (match_test "TARGET_ARCH13"))
	 (const_int 1)

         (and (eq_attr "cpu_facility" "vxe2")
	      (match_test "TARGET_VXE2"))
	 (const_int 1)
	 ]
	(const_int 0)))

;; Whether an instruction supports relative long addressing.
;; Currently this corresponds to RIL-b and RIL-c instruction formats,
;; but having a separate attribute, as opposed to reusing op_type,
;; provides additional flexibility.

(define_attr "relative_long" "no,yes" (const_string "no"))

;; Pipeline description for z900.
(include "2064.md")

;; Pipeline description for z990, z9-109 and z9-ec.
(include "2084.md")

;; Pipeline description for z10
(include "2097.md")

;; Pipeline description for z196
(include "2817.md")

;; Pipeline description for zEC12
(include "2827.md")

;; Pipeline description for z13
(include "2964.md")

;; Pipeline description for z14
(include "3906.md")

;; Pipeline description for arch13
(include "8561.md")

;; Predicates
(include "predicates.md")

;; Constraint definitions
(include "constraints.md")

;; Other includes
(include "tpf.md")

;; Iterators

(define_mode_iterator ALL [TI DI SI HI QI TF DF SF TD DD SD V1QI V2QI V4QI V8QI V16QI V1HI V2HI V4HI V8HI V1SI V2SI V4SI V1DI V2DI V1SF V2SF V4SF V1TI V1DF V2DF V1TF])

;; These mode iterators allow floating point patterns to be generated from the
;; same template.
(define_mode_iterator FP_ALL [TF DF SF (TD "TARGET_HARD_DFP") (DD "TARGET_HARD_DFP")
                              (SD "TARGET_HARD_DFP")])
(define_mode_iterator FP [TF DF SF (TD "TARGET_HARD_DFP") (DD "TARGET_HARD_DFP")])
(define_mode_iterator BFP [TF DF SF])
(define_mode_iterator DFP [TD DD])
(define_mode_iterator DFP_ALL [TD DD SD])
(define_mode_iterator DSF [DF SF])
(define_mode_iterator SD_SF [SF SD])
(define_mode_iterator DD_DF [DF DD])
(define_mode_iterator TD_TF [TF TD])

; 32 bit int<->fp conversion instructions are available since VXE2 (arch13).
(define_mode_iterator VX_CONV_BFP [DF (SF "TARGET_VXE2")])
(define_mode_iterator VX_CONV_INT [DI (SI "TARGET_VXE2")])

;; These mode iterators allow 31-bit and 64-bit GPR patterns to be generated
;; from the same template.
(define_mode_iterator GPR [(DI "TARGET_ZARCH") SI])
(define_mode_iterator DGPR [(TI "TARGET_ZARCH") DI SI])
(define_mode_iterator DSI [DI SI])
(define_mode_iterator TDI [TI DI])

;; These mode iterators allow :P to be used for patterns that operate on
;; pointer-sized quantities.  Exactly one of the two alternatives will match.
(define_mode_iterator P [(DI "TARGET_64BIT") (SI "!TARGET_64BIT")])

;; These macros refer to the actual word_mode of the configuration.
;; This is equal to Pmode except on 31-bit machines in zarch mode.
(define_mode_iterator DW [(TI "TARGET_ZARCH") (DI "!TARGET_ZARCH")])
(define_mode_iterator W  [(DI "TARGET_ZARCH") (SI "!TARGET_ZARCH")])

;; Used by the umul pattern to express modes having half the size.
(define_mode_attr DWH [(TI "DI") (DI "SI")])
(define_mode_attr dwh [(TI "di") (DI "si")])

;; This mode iterator allows the QI and HI patterns to be defined from
;; the same template.
(define_mode_iterator HQI [HI QI])

;; This mode iterator allows the integer patterns to be defined from the
;; same template.
(define_mode_iterator INT [(DI "TARGET_ZARCH") SI HI QI])
(define_mode_iterator DINT [(TI "TARGET_ZARCH") DI SI HI QI])
(define_mode_iterator SINT [SI HI QI])

;; This iterator allows some 'ashift' and 'lshiftrt' pattern to be defined from
;; the same template.
(define_code_iterator SHIFT [ashift lshiftrt])

;; This iterator allows r[ox]sbg to be defined with the same template
(define_code_iterator IXOR [ior xor])

;; This is used for merging the nand/nor and and/or with complement patterns
(define_code_iterator ANDOR [and ior])
(define_code_attr bitops_name [(and "and") (ior "or")])
(define_code_attr inv_bitops_name [(and "or") (ior "and")])
(define_code_attr inv_no [(and "o") (ior "n")])

;; This iterator is used to expand the patterns for the nearest
;; integer functions.
(define_int_iterator FPINT [UNSPEC_FPINT_FLOOR UNSPEC_FPINT_BTRUNC
			    UNSPEC_FPINT_ROUND UNSPEC_FPINT_CEIL
			    UNSPEC_FPINT_NEARBYINT])
(define_int_attr fpint_name [(UNSPEC_FPINT_FLOOR "floor")
			     (UNSPEC_FPINT_BTRUNC "btrunc")
			     (UNSPEC_FPINT_ROUND "round")
			     (UNSPEC_FPINT_CEIL "ceil")
			     (UNSPEC_FPINT_NEARBYINT "nearbyint")])
(define_int_attr fpint_roundingmode [(UNSPEC_FPINT_FLOOR "7")
				     (UNSPEC_FPINT_BTRUNC "5")
				     (UNSPEC_FPINT_ROUND "1")
				     (UNSPEC_FPINT_CEIL "6")
				     (UNSPEC_FPINT_NEARBYINT "0")])

;; This iterator and attribute allow to combine most atomic operations.
(define_code_iterator ATOMIC [and ior xor plus minus mult])
(define_code_iterator ATOMIC_Z196 [and ior xor plus])
(define_code_attr atomic [(and "and") (ior "or") (xor "xor")
			  (plus "add") (minus "sub") (mult "nand")])
(define_code_attr noxa [(and "n") (ior "o") (xor "x") (plus "a")])

;; In FP templates, a string like "lt<de>br" will expand to "ltxbr" in
;; TF/TDmode, "ltdbr" in DF/DDmode, and "ltebr" in SF/SDmode.
(define_mode_attr xde [(TF "x") (DF "d") (SF "e") (TD "x") (DD "d") (SD "e") (V4SF "e") (V2DF "d")])

;; In FP templates, a <dee> in "m<dee><bt>r" will expand to "mx<bt>r" in
;; TF/TDmode, "md<bt>r" in DF/DDmode, "mee<bt>r" in SFmode and "me<bt>r in
;; SDmode.
(define_mode_attr xdee [(TF "x") (DF "d") (SF "ee") (TD "x") (DD "d") (SD "e")])

;; In FP templates, "<RRe>" will expand to "RRE" in TFmode and "RR" otherwise.
;; Likewise for "<RXe>".
(define_mode_attr RRe [(TF "RRE") (DF "RR") (SF "RR")])
(define_mode_attr RXe [(TF "RXE") (DF "RX") (SF "RX")])

;; The decimal floating point variants of add, sub, div and mul support 3
;; fp register operands.  The following attributes allow to merge the bfp and
;; dfp variants in a single insn definition.

;; These mode attributes are supposed to be used in the `enabled' insn
;; attribute to disable certain alternatives for certain modes.
(define_mode_attr nBFP [(TF "0") (DF "0") (SF "0") (TD "*") (DD "*") (DD "*")])
(define_mode_attr nDFP [(TF "*") (DF "*") (SF "*") (TD "0") (DD "0") (DD "0")])
(define_mode_attr DSF [(TF "0") (DF "*") (SF "*") (TD "0") (DD "0") (SD "0")])
(define_mode_attr DFDI [(TF "0") (DF "*") (SF "0")
			(TD "0") (DD "0") (DD "0")
			(TI "0") (DI "*") (SI "0")])
(define_mode_attr SFSI [(TF "0") (DF "0") (SF "*")
			(TD "0") (DD "0") (DD "0")
			(TI "0") (DI "0") (SI "*")])
(define_mode_attr DF [(TF "0") (DF "*") (SF "0")
		      (TD "0") (DD "0") (DD "0")
		      (TI "0") (DI "0") (SI "0")])
(define_mode_attr SF [(TF "0") (DF "0") (SF "*")
		      (TD "0") (DD "0") (DD "0")
		      (TI "0") (DI "0") (SI "0")])

;; This attribute is used in the operand constraint list
;; for instructions dealing with the sign bit of 32 or 64bit fp values.
;; TFmode values are represented by a fp register pair.  Since the
;; sign bit instructions only handle single source and target fp registers
;; these instructions can only be used for TFmode values if the source and
;; target operand uses the same fp register.
(define_mode_attr fT0 [(TF "0") (DF "f") (SF "f")])

;; This attribute adds b for bfp instructions and t for dfp instructions and is used
;; within instruction mnemonics.
(define_mode_attr bt [(TF "b") (DF "b") (SF "b") (TD "t") (DD "t") (SD "t")])

;; This attribute is used within instruction mnemonics.  It evaluates to d for dfp
;; modes and to an empty string for bfp modes.
(define_mode_attr _d [(TF "") (DF "") (SF "") (TD "d") (DD "d") (SD "d")])

;; In GPR and P templates, a constraint like "<d0>" will expand to "d" in DImode
;; and "0" in SImode. This allows to combine instructions of which the 31bit
;; version only operates on one register.
(define_mode_attr d0 [(DI "d") (SI "0")])

;; In combination with d0 this allows to combine instructions of which the 31bit
;; version only operates on one register. The DImode version needs an additional
;; register for the assembler output.
(define_mode_attr 1 [(DI "%1,") (SI "")])

;; In SHIFT templates, a string like "s<lr>dl" will expand to "sldl" in
;; 'ashift' and "srdl" in 'lshiftrt'.
(define_code_attr lr [(ashift "l") (lshiftrt "r")])

;; In SHIFT templates, this attribute holds the correct standard name for the
;; pattern itself and the corresponding function calls.
(define_code_attr shift [(ashift "ashl") (lshiftrt "lshr")])

;; This attribute handles differences in the instruction 'type' and will result
;; in "RRE" for DImode and "RR" for SImode.
(define_mode_attr E [(DI "E") (SI "")])

;; This attribute handles differences in the instruction 'type' and makes RX<Y>
;; to result in "RXY" for DImode and "RX" for SImode.
(define_mode_attr Y [(DI "Y") (SI "")])

;; This attribute handles differences in the instruction 'type' and will result
;; in "RSE" for TImode and "RS" for DImode.
(define_mode_attr TE [(TI "E") (DI "")])

;; In GPR templates, a string like "lc<g>r" will expand to "lcgr" in DImode
;; and "lcr" in SImode.
(define_mode_attr g [(DI "g") (SI "")])

;; In GPR templates, a string like "sl<y>" will expand to "slg" in DImode
;; and "sly" in SImode. This is useful because on 64bit the ..g instructions
;; were enhanced with long displacements whereas 31bit instructions got a ..y
;; variant for long displacements.
(define_mode_attr y [(DI "g") (SI "y")])

;; In DW templates, a string like "cds<g>" will expand to "cdsg" in TImode
;; and "cds" in DImode.
(define_mode_attr tg [(TI "g") (DI "")])

;; In TDI templates, a string like "c<d>sg".
(define_mode_attr td [(TI "d") (DI "")])

;; In GPR templates, a string like "c<gf>dbr" will expand to "cgdbr" in DImode
;; and "cfdbr" in SImode.
(define_mode_attr gf [(DI "g") (SI "f")])

;; In GPR templates, a string like sll<gk> will expand to sllg for DI
;; and sllk for SI.  This way it is possible to merge the new z196 SI
;; 3 operands shift instructions into the existing patterns.
(define_mode_attr gk [(DI "g") (SI "k")])

;; ICM mask required to load MODE value into the lowest subreg
;; of a SImode register.
(define_mode_attr icm_lo [(HI "3") (QI "1")])

;; In HQI templates, a string like "llg<hc>" will expand to "llgh" in
;; HImode and "llgc" in QImode.
(define_mode_attr hc [(HI "h") (QI "c")])

;; In P templates, the mode <DBL> will expand to "TI" in DImode and "DI"
;; in SImode.
(define_mode_attr DBL [(DI "TI") (SI "DI")])

;; This attribute expands to DF for TFmode and to DD for TDmode .  It is
;; used for Txmode splitters splitting a Txmode copy into 2 Dxmode copies.
(define_mode_attr HALF_TMODE [(TF "DF") (TD "DD")])

;; Maximum unsigned integer that fits in MODE.
(define_mode_attr max_uint [(HI "65535") (QI "255")])

;; Start and end field computations for RISBG et al.
(define_mode_attr bfstart [(DI "s") (SI "t")])
(define_mode_attr bfend   [(DI "e") (SI "f")])

;; In place of GET_MODE_BITSIZE (<MODE>mode)
(define_mode_attr bitsize [(DI "64") (SI "32") (HI "16") (QI "8")])
;; 64 - bitsize
(define_mode_attr bitoff [(DI "0") (SI "32") (HI "48") (QI "56")])
(define_mode_attr bitoff_plus [(DI "") (SI "32+") (HI "48+") (QI "56+")])

;; In place of GET_MODE_SIZE (<MODE>mode)
(define_mode_attr modesize [(DI "8") (SI "4")])

;; Allow return and simple_return to be defined from a single template.
(define_code_iterator ANY_RETURN [return simple_return])



; Condition code modes generated by vector fp comparisons.  These will
; be used also in single element mode.
(define_mode_iterator VFCMP [CCVEQ CCVFH CCVFHE])
; Used with VFCMP to expand part of the mnemonic
; For fp we have a mismatch: eq in the insn name - e in asm
(define_mode_attr asm_fcmp [(CCVEQ "e") (CCVFH "h") (CCVFHE "he")])
(define_mode_attr insn_cmp [(CCVEQ "eq") (CCVIH "h") (CCVIHU "hl") (CCVFH "h") (CCVFHE "he")])

;; Subst pattern definitions
(include "subst.md")

(include "vector.md")

;;
;;- Compare instructions.
;;

; Test-under-Mask instructions

(define_insn "*tmqi_mem"
  [(set (reg CC_REGNUM)
        (compare (and:QI (match_operand:QI 0 "memory_operand" "Q,S")
                         (match_operand:QI 1 "immediate_operand" "n,n"))
                 (match_operand:QI 2 "immediate_operand" "n,n")))]
  "s390_match_ccmode (insn, s390_tm_ccmode (operands[1], operands[2], false))"
  "@
   tm\t%S0,%b1
   tmy\t%S0,%b1"
  [(set_attr "op_type" "SI,SIY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_super,z10_super")])

(define_insn "*tmdi_reg"
  [(set (reg CC_REGNUM)
        (compare (and:DI (match_operand:DI 0 "nonimmediate_operand" "d,d,d,d")
                         (match_operand:DI 1 "immediate_operand"
					     "N0HD0,N1HD0,N2HD0,N3HD0"))
                 (match_operand:DI 2 "immediate_operand" "n,n,n,n")))]
  "TARGET_ZARCH
   && s390_match_ccmode (insn, s390_tm_ccmode (operands[1], operands[2], true))
   && s390_single_part (operands[1], DImode, HImode, 0) >= 0"
  "@
   tmhh\t%0,%i1
   tmhl\t%0,%i1
   tmlh\t%0,%i1
   tmll\t%0,%i1"
  [(set_attr "op_type" "RI")
   (set_attr "z10prop" "z10_super,z10_super,z10_super,z10_super")])

(define_insn "*tmsi_reg"
  [(set (reg CC_REGNUM)
        (compare (and:SI (match_operand:SI 0 "nonimmediate_operand" "d,d")
                         (match_operand:SI 1 "immediate_operand" "N0HS0,N1HS0"))
                 (match_operand:SI 2 "immediate_operand" "n,n")))]
  "s390_match_ccmode (insn, s390_tm_ccmode (operands[1], operands[2], true))
   && s390_single_part (operands[1], SImode, HImode, 0) >= 0"
  "@
   tmh\t%0,%i1
   tml\t%0,%i1"
  [(set_attr "op_type" "RI")
   (set_attr "z10prop" "z10_super,z10_super")])

(define_insn "*tm<mode>_full"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HQI 0 "register_operand" "d")
                 (match_operand:HQI 1 "immediate_operand" "n")))]
  "s390_match_ccmode (insn, s390_tm_ccmode (constm1_rtx, operands[1], true))"
  "tml\t%0,<max_uint>"
  [(set_attr "op_type" "RI")
   (set_attr "z10prop" "z10_super")])


;
; Load-and-Test instructions
;

; tst(di|si) instruction pattern(s).

(define_insn "*tstdi_sign"
  [(set (reg CC_REGNUM)
        (compare
          (ashiftrt:DI
            (ashift:DI
              (subreg:DI (match_operand:SI 0 "nonimmediate_operand" "d,T") 0)
	      (const_int 32)) (const_int 32))
	  (match_operand:DI 1 "const0_operand" "")))
   (set (match_operand:DI 2 "register_operand" "=d,d")
        (sign_extend:DI (match_dup 0)))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_ZARCH"
  "ltgfr\t%2,%0
   ltgf\t%2,%0"
  [(set_attr "op_type"      "RRE,RXY")
   (set_attr "cpu_facility" "*,z10")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1") ])

; ltr, lt, ltgr, ltg
(define_insn "*tst<mode>_extimm"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 0 "nonimmediate_operand" "d,T")
                 (match_operand:GPR 1 "const0_operand" "")))
   (set (match_operand:GPR 2 "register_operand" "=d,d")
        (match_dup 0))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_EXTIMM"
  "@
   lt<g>r\t%2,%0
   lt<g>\t%2,%0"
  [(set_attr "op_type" "RR<E>,RXY")
   (set_attr "z10prop" "z10_fr_E1,z10_fwd_A3") ])

; Peephole to combine a load-and-test from volatile memory which combine does
; not do.
(define_peephole2
  [(set (match_operand:GPR 0 "register_operand")
	(match_operand:GPR 2 "memory_operand"))
   (set (reg CC_REGNUM)
	(compare (match_dup 0) (match_operand:GPR 1 "const0_operand")))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_EXTIMM
   && GENERAL_REG_P (operands[0])
   && satisfies_constraint_T (operands[2])
   && !contains_constant_pool_address_p (operands[2])"
  [(parallel
    [(set (reg:CCS CC_REGNUM)
	  (compare:CCS (match_dup 2) (match_dup 1)))
     (set (match_dup 0) (match_dup 2))])])

; ltr, lt, ltgr, ltg
(define_insn "*tst<mode>_cconly_extimm"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 0 "nonimmediate_operand" "d,T")
                 (match_operand:GPR 1 "const0_operand" "")))
   (clobber (match_scratch:GPR 2 "=X,d"))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_EXTIMM"
  "@
   lt<g>r\t%0,%0
   lt<g>\t%2,%0"
  [(set_attr "op_type" "RR<E>,RXY")
   (set_attr "z10prop" "z10_fr_E1,z10_fwd_A3")])

(define_insn "*tstdi"
  [(set (reg CC_REGNUM)
        (compare (match_operand:DI 0 "register_operand" "d")
                 (match_operand:DI 1 "const0_operand" "")))
   (set (match_operand:DI 2 "register_operand" "=d")
        (match_dup 0))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_ZARCH && !TARGET_EXTIMM"
  "ltgr\t%2,%0"
  [(set_attr "op_type" "RRE")
   (set_attr "z10prop" "z10_fr_E1")])

(define_insn "*tstsi"
  [(set (reg CC_REGNUM)
        (compare (match_operand:SI 0 "nonimmediate_operand" "d,Q,S")
                 (match_operand:SI 1 "const0_operand" "")))
   (set (match_operand:SI 2 "register_operand" "=d,d,d")
        (match_dup 0))]
  "s390_match_ccmode(insn, CCSmode) && !TARGET_EXTIMM"
  "@
   ltr\t%2,%0
   icm\t%2,15,%S0
   icmy\t%2,15,%S0"
  [(set_attr "op_type" "RR,RS,RSY")
   (set_attr "cpu_facility" "*,*,longdisp")
   (set_attr "z10prop" "z10_fr_E1,z10_super_E1,z10_super_E1")])

(define_insn "*tstsi_cconly"
  [(set (reg CC_REGNUM)
        (compare (match_operand:SI 0 "nonimmediate_operand" "d,Q,S")
                 (match_operand:SI 1 "const0_operand" "")))
   (clobber (match_scratch:SI 2 "=X,d,d"))]
  "s390_match_ccmode(insn, CCSmode)"
  "@
   ltr\t%0,%0
   icm\t%2,15,%S0
   icmy\t%2,15,%S0"
  [(set_attr "op_type" "RR,RS,RSY")
   (set_attr "cpu_facility" "*,*,longdisp")
   (set_attr "z10prop" "z10_fr_E1,z10_super_E1,z10_super_E1")])

(define_insn "*tstdi_cconly_31"
  [(set (reg CC_REGNUM)
        (compare (match_operand:DI 0 "register_operand" "d")
                 (match_operand:DI 1 "const0_operand" "")))]
  "s390_match_ccmode(insn, CCSmode) && !TARGET_ZARCH"
  "srda\t%0,0"
  [(set_attr "op_type" "RS")
   (set_attr "atype"   "reg")])

; ltr, ltgr
(define_insn "*tst<mode>_cconly2"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 0 "register_operand" "d")
                 (match_operand:GPR 1 "const0_operand" "")))]
  "s390_match_ccmode(insn, CCSmode)"
  "lt<g>r\t%0,%0"
  [(set_attr "op_type" "RR<E>")
   (set_attr "z10prop" "z10_fr_E1")])

; tst(hi|qi) instruction pattern(s).

(define_insn "*tst<mode>CCT"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HQI 0 "nonimmediate_operand" "?Q,?S,d")
                 (match_operand:HQI 1 "const0_operand" "")))
   (set (match_operand:HQI 2 "register_operand" "=d,d,0")
        (match_dup 0))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   icm\t%2,<icm_lo>,%S0
   icmy\t%2,<icm_lo>,%S0
   tml\t%0,<max_uint>"
  [(set_attr "op_type" "RS,RSY,RI")
   (set_attr "cpu_facility" "*,longdisp,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,z10_super")])

(define_insn "*tsthiCCT_cconly"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HI 0 "nonimmediate_operand" "Q,S,d")
                 (match_operand:HI 1 "const0_operand" "")))
   (clobber (match_scratch:HI 2 "=d,d,X"))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   icm\t%2,3,%S0
   icmy\t%2,3,%S0
   tml\t%0,65535"
  [(set_attr "op_type" "RS,RSY,RI")
   (set_attr "cpu_facility" "*,longdisp,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,z10_super")])

(define_insn "*tstqiCCT_cconly"
  [(set (reg CC_REGNUM)
        (compare (match_operand:QI 0 "nonimmediate_operand" "?Q,?S,d")
                 (match_operand:QI 1 "const0_operand" "")))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   cli\t%S0,0
   cliy\t%S0,0
   tml\t%0,255"
  [(set_attr "op_type" "SI,SIY,RI")
   (set_attr "cpu_facility" "*,longdisp,*")
   (set_attr "z10prop" "z10_super,z10_super,z10_super")])

(define_insn "*tst<mode>"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HQI 0 "s_operand" "Q,S")
                 (match_operand:HQI 1 "const0_operand" "")))
   (set (match_operand:HQI 2 "register_operand" "=d,d")
        (match_dup 0))]
  "s390_match_ccmode(insn, CCSmode)"
  "@
   icm\t%2,<icm_lo>,%S0
   icmy\t%2,<icm_lo>,%S0"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_insn "*tst<mode>_cconly"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HQI 0 "s_operand" "Q,S")
                 (match_operand:HQI 1 "const0_operand" "")))
   (clobber (match_scratch:HQI 2 "=d,d"))]
  "s390_match_ccmode(insn, CCSmode)"
  "@
   icm\t%2,<icm_lo>,%S0
   icmy\t%2,<icm_lo>,%S0"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])


; Compare (equality) instructions

(define_insn "*cmpdi_cct"
  [(set (reg CC_REGNUM)
        (compare (match_operand:DI 0 "nonimmediate_operand" "%d,d,d,d,Q")
                 (match_operand:DI 1 "general_operand" "d,K,Os,T,BQ")))]
  "s390_match_ccmode (insn, CCTmode) && TARGET_ZARCH"
  "@
   cgr\t%0,%1
   cghi\t%0,%h1
   cgfi\t%0,%1
   cg\t%0,%1
   #"
  [(set_attr "op_type" "RRE,RI,RIL,RXY,SS")
   (set_attr "z10prop" "z10_super_c,z10_super,z10_super,z10_super,*")])

(define_insn "*cmpsi_cct"
  [(set (reg CC_REGNUM)
        (compare (match_operand:SI 0 "nonimmediate_operand" "%d,d,d,d,d,Q")
                 (match_operand:SI 1 "general_operand" "d,K,Os,R,T,BQ")))]
  "s390_match_ccmode (insn, CCTmode)"
  "@
   cr\t%0,%1
   chi\t%0,%h1
   cfi\t%0,%1
   c\t%0,%1
   cy\t%0,%1
   #"
  [(set_attr "op_type" "RR,RI,RIL,RX,RXY,SS")
   (set_attr "cpu_facility" "*,*,*,*,longdisp,*")
   (set_attr "z10prop" "z10_super_c,z10_super,z10_super,z10_super,z10_super,*")])

; Compare (signed) instructions

(define_insn "*cmpdi_ccs_sign"
  [(set (reg CC_REGNUM)
        (compare (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand"
						     "d,T,b"))
                 (match_operand:DI 0 "register_operand" "d, d,d")))]
  "s390_match_ccmode(insn, CCSRmode) && TARGET_ZARCH"
  "@
   cgfr\t%0,%1
   cgf\t%0,%1
   cgfrl\t%0,%1"
  [(set_attr "op_type"      "RRE,RXY,RIL")
   (set_attr "z10prop" "z10_c,*,*")
   (set_attr "type"         "*,*,larl")
   (set_attr "relative_long" "*,*,yes")])



(define_insn "*cmpsi_ccs_sign"
  [(set (reg CC_REGNUM)
        (compare (sign_extend:SI (match_operand:HI 1 "memory_operand" "R,T,b"))
                 (match_operand:SI 0 "register_operand" "d,d,d")))]
  "s390_match_ccmode(insn, CCSRmode)"
  "@
   ch\t%0,%1
   chy\t%0,%1
   chrl\t%0,%1"
  [(set_attr "op_type"      "RX,RXY,RIL")
   (set_attr "cpu_facility" "*,longdisp,z10")
   (set_attr "type"         "*,*,larl")
   (set_attr "z196prop" "z196_cracked,z196_cracked,z196_cracked")
   (set_attr "relative_long" "*,*,yes")])

(define_insn "*cmphi_ccs_z10"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HI 0 "s_operand"         "Q")
                 (match_operand:HI 1 "immediate_operand" "K")))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_Z10"
  "chhsi\t%0,%1"
  [(set_attr "op_type" "SIL")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*cmpdi_ccs_signhi_rl"
  [(set (reg CC_REGNUM)
	(compare (sign_extend:DI (match_operand:HI 1 "memory_operand" "T,b"))
		 (match_operand:GPR 0 "register_operand"  "d,d")))]
  "s390_match_ccmode(insn, CCSRmode) && TARGET_Z10"
  "@
   cgh\t%0,%1
   cghrl\t%0,%1"
  [(set_attr "op_type" "RXY,RIL")
   (set_attr "type"    "*,larl")
   (set_attr "relative_long" "*,yes")])

; cr, chi, cfi, c, cy, cgr, cghi, cgfi, cg, chsi, cghsi, crl, cgrl
(define_insn "*cmp<mode>_ccs"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 0 "nonimmediate_operand"
                                      "d,d,Q, d,d,d,d")
                 (match_operand:GPR 1 "general_operand"
                                      "d,K,K,Os,R,T,b")))]
  "s390_match_ccmode(insn, CCSmode)"
  "@
   c<g>r\t%0,%1
   c<g>hi\t%0,%h1
   c<g>hsi\t%0,%h1
   c<g>fi\t%0,%1
   c<g>\t%0,%1
   c<y>\t%0,%1
   c<g>rl\t%0,%1"
  [(set_attr "op_type" "RR<E>,RI,SIL,RIL,RX<Y>,RXY,RIL")
   (set_attr "cpu_facility" "*,*,z10,extimm,*,longdisp,z10")
   (set_attr "type" "*,*,*,*,*,*,larl")
   (set_attr "z10prop" "z10_super_c,z10_super,z10_super,z10_super,z10_super,z10_super,z10_super")
   (set_attr "relative_long" "*,*,*,*,*,*,yes")])


; Compare (unsigned) instructions

(define_insn "*cmpsi_ccu_zerohi_rlsi"
  [(set (reg CC_REGNUM)
 	(compare (zero_extend:SI (mem:HI (match_operand:SI 1
					  "larl_operand" "X")))
		 (match_operand:SI 0 "register_operand" "d")))]
  "s390_match_ccmode(insn, CCURmode) && TARGET_Z10"
  "clhrl\t%0,%1"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "larl")
   (set_attr "z10prop" "z10_super")
   (set_attr "relative_long" "yes")])

; clhrl, clghrl
(define_insn "*cmp<GPR:mode>_ccu_zerohi_rldi"
  [(set (reg CC_REGNUM)
 	(compare (zero_extend:GPR (mem:HI (match_operand:DI 1
					  "larl_operand" "X")))
		 (match_operand:GPR 0 "register_operand" "d")))]
  "s390_match_ccmode(insn, CCURmode) && TARGET_Z10"
  "cl<g>hrl\t%0,%1"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "larl")
   (set_attr "z10prop" "z10_super")
   (set_attr "relative_long" "yes")])

(define_insn "*cmpdi_ccu_zero"
  [(set (reg CC_REGNUM)
        (compare (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand"
                                                        "d,T,b"))
                 (match_operand:DI 0 "register_operand" "d,d,d")))]
  "s390_match_ccmode (insn, CCURmode) && TARGET_ZARCH"
  "@
   clgfr\t%0,%1
   clgf\t%0,%1
   clgfrl\t%0,%1"
  [(set_attr "op_type"      "RRE,RXY,RIL")
   (set_attr "cpu_facility" "*,*,z10")
   (set_attr "type"         "*,*,larl")
   (set_attr "z10prop" "z10_super_c,z10_super_E1,z10_super")
   (set_attr "relative_long" "*,*,yes")])

(define_insn "*cmpdi_ccu"
  [(set (reg CC_REGNUM)
        (compare (match_operand:DI 0 "nonimmediate_operand"
                                     "d, d,d,Q,d, Q,BQ")
                 (match_operand:DI 1 "general_operand"
                                     "d,Op,b,D,T,BQ,Q")))]
  "s390_match_ccmode (insn, CCUmode) && TARGET_ZARCH"
  "@
   clgr\t%0,%1
   clgfi\t%0,%1
   clgrl\t%0,%1
   clghsi\t%0,%x1
   clg\t%0,%1
   #
   #"
  [(set_attr "op_type" "RRE,RIL,RIL,SIL,RXY,SS,SS")
   (set_attr "cpu_facility" "*,extimm,z10,z10,*,*,*")
   (set_attr "type"         "*,*,larl,*,*,*,*")
   (set_attr "z10prop" "z10_super_c,z10_super,z10_super,z10_super,z10_super,*,*")
   (set_attr "relative_long" "*,*,yes,*,*,*,*")])

(define_insn "*cmpsi_ccu"
  [(set (reg CC_REGNUM)
        (compare (match_operand:SI 0 "nonimmediate_operand" "d, d,d,Q,d,d, Q,BQ")
                 (match_operand:SI 1 "general_operand"      "d,Os,b,D,R,T,BQ, Q")))]
  "s390_match_ccmode (insn, CCUmode)"
  "@
   clr\t%0,%1
   clfi\t%0,%o1
   clrl\t%0,%1
   clfhsi\t%0,%x1
   cl\t%0,%1
   cly\t%0,%1
   #
   #"
  [(set_attr "op_type" "RR,RIL,RIL,SIL,RX,RXY,SS,SS")
   (set_attr "cpu_facility" "*,extimm,z10,z10,*,longdisp,*,*")
   (set_attr "type"         "*,*,larl,*,*,*,*,*")
   (set_attr "z10prop" "z10_super_c,z10_super,z10_super,z10_super,z10_super,z10_super,*,*")
   (set_attr "relative_long" "*,*,yes,*,*,*,*,*")])

(define_insn "*cmphi_ccu"
  [(set (reg CC_REGNUM)
        (compare (match_operand:HI 0 "nonimmediate_operand" "d,d,Q,Q,BQ")
                 (match_operand:HI 1 "general_operand"      "Q,S,D,BQ,Q")))]
  "s390_match_ccmode (insn, CCUmode)
   && !register_operand (operands[1], HImode)"
  "@
   clm\t%0,3,%S1
   clmy\t%0,3,%S1
   clhhsi\t%0,%1
   #
   #"
  [(set_attr "op_type" "RS,RSY,SIL,SS,SS")
   (set_attr "cpu_facility" "*,longdisp,z10,*,*")
   (set_attr "z10prop" "*,*,z10_super,*,*")])

(define_insn "*cmpqi_ccu"
  [(set (reg CC_REGNUM)
        (compare (match_operand:QI 0 "nonimmediate_operand" "d,d,Q,S,Q,BQ")
                 (match_operand:QI 1 "general_operand" "Q,S,n,n,BQ,Q")))]
  "s390_match_ccmode (insn, CCUmode)
   && !register_operand (operands[1], QImode)"
  "@
   clm\t%0,1,%S1
   clmy\t%0,1,%S1
   cli\t%S0,%b1
   cliy\t%S0,%b1
   #
   #"
  [(set_attr "op_type" "RS,RSY,SI,SIY,SS,SS")
   (set_attr "cpu_facility" "*,longdisp,*,longdisp,*,*")
   (set_attr "z10prop" "*,*,z10_super,z10_super,*,*")])


; Block compare (CLC) instruction patterns.

(define_insn "*clc"
  [(set (reg CC_REGNUM)
        (compare (match_operand:BLK 0 "memory_operand" "Q")
                 (match_operand:BLK 1 "memory_operand" "Q")))
   (use (match_operand 2 "const_int_operand" "n"))]
  "s390_match_ccmode (insn, CCUmode)
   && INTVAL (operands[2]) >= 1 && INTVAL (operands[2]) <= 256"
  "clc\t%O0(%2,%R0),%S1"
  [(set_attr "op_type" "SS")])

(define_split
  [(set (reg CC_REGNUM)
        (compare (match_operand 0 "memory_operand" "")
                 (match_operand 1 "memory_operand" "")))]
  "reload_completed
   && s390_match_ccmode (insn, CCUmode)
   && GET_MODE (operands[0]) == GET_MODE (operands[1])
   && GET_MODE_SIZE (GET_MODE (operands[0])) > 0"
  [(parallel
    [(set (match_dup 0) (match_dup 1))
     (use (match_dup 2))])]
{
  operands[2] = GEN_INT (GET_MODE_SIZE (GET_MODE (operands[0])));
  operands[0] = adjust_address (operands[0], BLKmode, 0);
  operands[1] = adjust_address (operands[1], BLKmode, 0);

  operands[1] = gen_rtx_COMPARE (GET_MODE (SET_DEST (PATTERN (curr_insn))),
				 operands[0], operands[1]);
  operands[0] = SET_DEST (PATTERN (curr_insn));
})


; (TF|DF|SF|TD|DD|SD) instructions


; FIXME: load and test instructions turn SNaN into QNaN what is not
; acceptable if the target will be used afterwards.  On the other hand
; they are quite convenient for implementing comparisons with 0.0. So
; try to enable them via splitter/peephole if the value isn't needed anymore.
; See testcases: load-and-test-fp-1.c and load-and-test-fp-2.c

; ltxbr, ltdbr, ltebr, ltxtr, ltdtr
(define_insn "*cmp<mode>_ccs_0"
  [(set (reg CC_REGNUM)
	(compare (match_operand:FP 0 "register_operand"  "f")
		 (match_operand:FP 1 "const0_operand"    "")))
   (clobber (match_operand:FP      2 "register_operand" "=0"))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT"
  "lt<xde><bt>r\t%0,%0"
   [(set_attr "op_type" "RRE")
    (set_attr "type"  "fsimp<mode>")])

; VX: TFmode in FPR pairs: use cxbr instead of wfcxb
; cxtr, cdtr, cxbr, cdbr, cebr, cdb, ceb, wfcsb, wfcdb
(define_insn "*cmp<mode>_ccs"
  [(set (reg CC_REGNUM)
        (compare (match_operand:FP 0 "register_operand" "f,f,v,v")
                 (match_operand:FP 1 "general_operand"  "f,R,v,v")))]
  "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT"
  "@
   c<xde><bt>r\t%0,%1
   c<xde>b\t%0,%1
   wfcdb\t%0,%1
   wfcsb\t%0,%1"
  [(set_attr "op_type" "RRE,RXE,VRR,VRR")
   (set_attr "cpu_facility" "*,*,vx,vxe")
   (set_attr "enabled" "*,<DSF>,<DF>,<SF>")])

; Compare and Branch instructions

; cij, cgij, crj, cgrj, cfi, cgfi, cr, cgr
; The following instructions do a complementary access of their second
; operand (z01 only): crj_c, cgrjc, cr, cgr
(define_insn "*cmp_and_br_signed_<mode>"
  [(set (pc)
	(if_then_else (match_operator 0 "s390_signed_integer_comparison"
			[(match_operand:GPR 1 "register_operand"  "d,d")
			 (match_operand:GPR 2 "nonmemory_operand" "d,C")])
		      (label_ref (match_operand 3 "" ""))
		      (pc)))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10 && !TARGET_AVOID_CMP_AND_BRANCH"
{
  if (get_attr_length (insn) == 6)
    return which_alternative ?
      "c<g>ij%C0\t%1,%c2,%l3" : "c<g>rj%C0\t%1,%2,%l3";
  else
    return which_alternative ?
      "c<g>fi\t%1,%c2\;jg%C0\t%l3" : "c<g>r\t%1,%2\;jg%C0\t%l3";
}
  [(set_attr "op_type" "RIE")
   (set_attr "type"    "branch")
   (set_attr "z10prop" "z10_super_c,z10_super")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 3))) (const_int 60000))
                      (const_int 6) (const_int 12)))]) ; 8 byte for cr/jg
                                                       ; 10 byte for cgr/jg

; clij, clgij, clrj, clgrj, clfi, clgfi, clr, clgr
; The following instructions do a complementary access of their second
; operand (z10 only): clrj, clgrj, clr, clgr
(define_insn "*cmp_and_br_unsigned_<mode>"
  [(set (pc)
	(if_then_else (match_operator 0 "s390_unsigned_integer_comparison"
			[(match_operand:GPR 1 "register_operand"  "d,d")
			 (match_operand:GPR 2 "nonmemory_operand" "d,I")])
		      (label_ref (match_operand 3 "" ""))
		      (pc)))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10 && !TARGET_AVOID_CMP_AND_BRANCH"
{
  if (get_attr_length (insn) == 6)
    return which_alternative ?
      "cl<g>ij%C0\t%1,%b2,%l3" : "cl<g>rj%C0\t%1,%2,%l3";
  else
    return which_alternative ?
      "cl<g>fi\t%1,%b2\;jg%C0\t%l3" : "cl<g>r\t%1,%2\;jg%C0\t%l3";
}
  [(set_attr "op_type" "RIE")
   (set_attr "type"    "branch")
   (set_attr "z10prop" "z10_super_c,z10_super")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 3))) (const_int 60000))
                      (const_int 6) (const_int 12)))]) ; 8 byte for clr/jg
                                                       ; 10 byte for clgr/jg

; And now the same two patterns as above but with a negated CC mask.

; cij, cgij, crj, cgrj, cfi, cgfi, cr, cgr
; The following instructions do a complementary access of their second
; operand (z01 only): crj_c, cgrjc, cr, cgr
(define_insn "*icmp_and_br_signed_<mode>"
  [(set (pc)
	(if_then_else (match_operator 0 "s390_signed_integer_comparison"
			[(match_operand:GPR 1 "register_operand"  "d,d")
			 (match_operand:GPR 2 "nonmemory_operand" "d,C")])
		      (pc)
		      (label_ref (match_operand 3 "" ""))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10 && !TARGET_AVOID_CMP_AND_BRANCH"
{
  if (get_attr_length (insn) == 6)
    return which_alternative ?
      "c<g>ij%D0\t%1,%c2,%l3" : "c<g>rj%D0\t%1,%2,%l3";
  else
    return which_alternative ?
      "c<g>fi\t%1,%c2\;jg%D0\t%l3" : "c<g>r\t%1,%2\;jg%D0\t%l3";
}
  [(set_attr "op_type" "RIE")
   (set_attr "type"    "branch")
   (set_attr "z10prop" "z10_super_c,z10_super")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 3))) (const_int 60000))
                      (const_int 6) (const_int 12)))]) ; 8 byte for cr/jg
                                                       ; 10 byte for cgr/jg

; clij, clgij, clrj, clgrj, clfi, clgfi, clr, clgr
; The following instructions do a complementary access of their second
; operand (z10 only): clrj, clgrj, clr, clgr
(define_insn "*icmp_and_br_unsigned_<mode>"
  [(set (pc)
	(if_then_else (match_operator 0 "s390_unsigned_integer_comparison"
			[(match_operand:GPR 1 "register_operand"  "d,d")
			 (match_operand:GPR 2 "nonmemory_operand" "d,I")])
		      (pc)
		      (label_ref (match_operand 3 "" ""))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10 && !TARGET_AVOID_CMP_AND_BRANCH"
{
  if (get_attr_length (insn) == 6)
    return which_alternative ?
      "cl<g>ij%D0\t%1,%b2,%l3" : "cl<g>rj%D0\t%1,%2,%l3";
  else
    return which_alternative ?
      "cl<g>fi\t%1,%b2\;jg%D0\t%l3" : "cl<g>r\t%1,%2\;jg%D0\t%l3";
}
  [(set_attr "op_type" "RIE")
   (set_attr "type"    "branch")
   (set_attr "z10prop" "z10_super_c,z10_super")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 3))) (const_int 60000))
                      (const_int 6) (const_int 12)))]) ; 8 byte for clr/jg
                                                       ; 10 byte for clgr/jg

;;
;;- Move instructions.
;;

;
; movti instruction pattern(s).
;


; Separate out the register pair alternative since constraints (P) are
; not able to deal with const_wide_int's.  But predicates do.
(define_insn "*movti_bigconst"
  [(set (match_operand:TI 0 "register_operand"              "=d")
        (match_operand:TI 1 "reload_const_wide_int_operand" ""))]
  "TARGET_ZARCH"
  "#")

; FIXME: More constants are possible by enabling jxx, jyy constraints
; for TImode (use double-int for the calculations)
(define_insn "movti"
  [(set (match_operand:TI 0 "nonimmediate_operand" "=d,S,v,  v,  v,v,d,v,R,d,    d, d,    d, d,o")
        (match_operand:TI 1 "general_operand"      " S,d,v,j00,jm1,d,v,R,v,K,NxHD0,Os,NxSD0,dT,d"))]
  "TARGET_ZARCH"
  "@
   lmg\t%0,%N0,%S1
   stmg\t%1,%N1,%S0
   vlr\t%v0,%v1
   vzero\t%v0
   vone\t%v0
   vlvgp\t%v0,%1,%N1
   #
   vl\t%v0,%1%A1
   vst\t%v1,%0%A0
   #
   #
   #
   #
   #
   #"
  [(set_attr "op_type" "RSY,RSY,VRR,VRI,VRI,VRR,*,VRX,VRX,*,*,*,*,*,*")
   (set_attr "type" "lm,stm,*,*,*,*,*,*,*,*,*,*,*,*,*")
   (set_attr "cpu_facility" "*,*,vx,vx,vx,vx,vx,vx,vx,*,*,*,extimm,*,*")])

(define_split
  [(set (match_operand:TI 0 "nonimmediate_operand" "")
        (match_operand:TI 1 "general_operand" ""))]
  "TARGET_ZARCH && reload_completed
   && !s_operand (operands[0], TImode)
   && !s_operand (operands[1], TImode)
   && s390_split_ok_p (operands[0], operands[1], TImode, 0)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 0, 0, TImode);
  operands[3] = operand_subword (operands[0], 1, 0, TImode);
  operands[4] = operand_subword (operands[1], 0, 0, TImode);
  operands[5] = operand_subword (operands[1], 1, 0, TImode);
})

(define_split
  [(set (match_operand:TI 0 "nonimmediate_operand" "")
        (match_operand:TI 1 "general_operand" ""))]
  "TARGET_ZARCH && reload_completed
   && !s_operand (operands[0], TImode)
   && !s_operand (operands[1], TImode)
   && s390_split_ok_p (operands[0], operands[1], TImode, 1)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 1, 0, TImode);
  operands[3] = operand_subword (operands[0], 0, 0, TImode);
  operands[4] = operand_subword (operands[1], 1, 0, TImode);
  operands[5] = operand_subword (operands[1], 0, 0, TImode);
})

; Use part of the TImode target reg to perform the address
; calculation.  If the TImode value is supposed to be copied into a VR
; this splitter is not necessary.
(define_split
  [(set (match_operand:TI 0 "register_operand" "")
        (match_operand:TI 1 "memory_operand" ""))]
  "TARGET_ZARCH && reload_completed
   && !VECTOR_REG_P (operands[0])
   && !s_operand (operands[1], VOIDmode)"
  [(set (match_dup 0) (match_dup 1))]
{
  rtx addr = operand_subword (operands[0], 1, 0, TImode);
  addr = gen_lowpart (Pmode, addr);
  s390_load_address (addr, XEXP (operands[1], 0));
  operands[1] = replace_equiv_address (operands[1], addr);
})


; Split a VR -> GPR TImode move into 2 vector load GR from VR element.
; For the higher order bits we do simply a DImode move while the
; second part is done via vec extract.  Both will end up as vlgvg.
(define_split
  [(set (match_operand:TI 0 "register_operand" "")
        (match_operand:TI 1 "register_operand" ""))]
  "TARGET_VX && reload_completed
   && GENERAL_REG_P (operands[0])
   && VECTOR_REG_P (operands[1])"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (unspec:DI [(match_dup 5) (const_int 1)]
				 UNSPEC_VEC_EXTRACT))]
{
  operands[2] = operand_subword (operands[0], 0, 0, TImode);
  operands[3] = operand_subword (operands[0], 1, 0, TImode);
  operands[4] = gen_rtx_REG (DImode, REGNO (operands[1]));
  operands[5] = gen_rtx_REG (V2DImode, REGNO (operands[1]));
})

;
; Patterns used for secondary reloads
;

; z10 provides move instructions accepting larl memory operands.
; Unfortunately there is no such variant for QI, TI and FP mode moves.
; These patterns are also used for unaligned SI and DI accesses.

(define_expand "reload<ALL:mode><P:mode>_tomem_z10"
  [(parallel [(match_operand:ALL 0 "memory_operand"   "")
	      (match_operand:ALL 1 "register_operand" "=d")
	      (match_operand:P   2 "register_operand" "=&a")])]
  "TARGET_Z10"
{
  s390_reload_symref_address (operands[1], operands[0], operands[2], 1);
  DONE;
})

(define_expand "reload<ALL:mode><P:mode>_toreg_z10"
  [(parallel [(match_operand:ALL 0 "register_operand" "=d")
	      (match_operand:ALL 1 "memory_operand"   "")
	      (match_operand:P   2 "register_operand" "=a")])]
  "TARGET_Z10"
{
  s390_reload_symref_address (operands[0], operands[1], operands[2], 0);
  DONE;
})

(define_expand "reload<P:mode>_larl_odd_addend_z10"
  [(parallel [(match_operand:P 0 "register_operand" "=d")
	      (match_operand:P 1 "larl_operand"     "")
	      (match_operand:P 2 "register_operand" "=a")])]
  "TARGET_Z10"
{
  s390_reload_larl_operand (operands[0], operands[1], operands[2]);
  DONE;
})

; Handles loading a PLUS (load address) expression

(define_expand "reload<mode>_plus"
  [(parallel [(match_operand:P 0 "register_operand"  "=a")
              (match_operand:P 1 "s390_plus_operand" "")
              (match_operand:P 2 "register_operand"  "=&a")])]
  ""
{
  s390_expand_plus_operand (operands[0], operands[1], operands[2]);
  DONE;
})

; Not all the indirect memory access instructions support the full
; format (long disp + index + base).  So whenever a move from/to such
; an address is required and the instruction cannot deal with it we do
; a load address into a scratch register first and use this as the new
; base register.
; This in particular is used for:
; - non-offsetable memory accesses for multiword moves
; - full vector reg moves with long displacements

(define_expand "reload<mode>_la_in"
  [(parallel [(match_operand 0   "register_operand" "")
              (match_operand 1   "" "")
              (match_operand:P 2 "register_operand" "=&a")])]
  ""
{
  gcc_assert (MEM_P (operands[1]));
  s390_load_address (operands[2], find_replacement (&XEXP (operands[1], 0)));
  operands[1] = replace_equiv_address (operands[1], operands[2]);
  emit_move_insn (operands[0], operands[1]);
  DONE;
})

(define_expand "reload<mode>_la_out"
  [(parallel [(match_operand   0 "" "")
              (match_operand   1 "register_operand" "")
              (match_operand:P 2 "register_operand" "=&a")])]
  ""
{
  gcc_assert (MEM_P (operands[0]));
  s390_load_address (operands[2], find_replacement (&XEXP (operands[0], 0)));
  operands[0] = replace_equiv_address (operands[0], operands[2]);
  emit_move_insn (operands[0], operands[1]);
  DONE;
})

(define_expand "reload<mode>_PIC_addr"
  [(parallel [(match_operand   0 "register_operand" "=d")
	      (match_operand   1 "larl_operand"     "")
	      (match_operand:P 2 "register_operand" "=a")])]
  ""
{
  rtx new_rtx = legitimize_pic_address (operands[1], operands[2]);
  emit_move_insn (operands[0], new_rtx);
})

;
; movdi instruction pattern(s).
;

(define_expand "movdi"
  [(set (match_operand:DI 0 "general_operand" "")
        (match_operand:DI 1 "general_operand" ""))]
  ""
{
  /* Handle symbolic constants.  */
  if (TARGET_64BIT
      && (SYMBOLIC_CONST (operands[1])
	  || (GET_CODE (operands[1]) == PLUS
	      && XEXP (operands[1], 0) == pic_offset_table_rtx
	      && SYMBOLIC_CONST (XEXP (operands[1], 1)))))
    emit_symbolic_move (operands);
})

(define_insn "*movdi_64"
  [(set (match_operand:DI 0 "nonimmediate_operand"
         "=d,    d,    d,    d,    d, d,    d,    d,f,d,d,d,d,d,T,!*f,!*f,!*f,!R,!T,b,Q,d,t,Q,t,v,v,v,d,v,R,d")
        (match_operand:DI 1 "general_operand"
         " K,N0HD0,N1HD0,N2HD0,N3HD0,Os,N0SD0,N1SD0,d,f,L,b,d,T,d, *f,  R,  T,*f,*f,d,K,t,d,t,Q,K,v,d,v,R,v,ZL"))]
  "TARGET_ZARCH"
  "@
   lghi\t%0,%h1
   llihh\t%0,%i1
   llihl\t%0,%i1
   llilh\t%0,%i1
   llill\t%0,%i1
   lgfi\t%0,%1
   llihf\t%0,%k1
   llilf\t%0,%k1
   ldgr\t%0,%1
   lgdr\t%0,%1
   lay\t%0,%a1
   lgrl\t%0,%1
   lgr\t%0,%1
   lg\t%0,%1
   stg\t%1,%0
   ldr\t%0,%1
   ld\t%0,%1
   ldy\t%0,%1
   std\t%1,%0
   stdy\t%1,%0
   stgrl\t%1,%0
   mvghi\t%0,%1
   #
   #
   stam\t%1,%N1,%S0
   lam\t%0,%N0,%S1
   vleig\t%v0,%h1,0
   vlr\t%v0,%v1
   vlvgg\t%v0,%1,0
   vlgvg\t%0,%v1,0
   vleg\t%v0,%1,0
   vsteg\t%v1,%0,0
   larl\t%0,%1"
  [(set_attr "op_type" "RI,RI,RI,RI,RI,RIL,RIL,RIL,RRE,RRE,RXY,RIL,RRE,RXY,
                        RXY,RR,RX,RXY,RX,RXY,RIL,SIL,*,*,RS,RS,VRI,VRR,VRS,VRS,
                        VRX,VRX,RIL")
   (set_attr "type" "*,*,*,*,*,*,*,*,floaddf,floaddf,la,larl,lr,load,store,
                     floaddf,floaddf,floaddf,fstoredf,fstoredf,larl,*,*,*,*,
                     *,*,*,*,*,*,*,larl")
   (set_attr "cpu_facility" "*,*,*,*,*,extimm,extimm,extimm,dfp,dfp,longdisp,
                             z10,*,*,*,*,*,longdisp,*,longdisp,
                             z10,z10,*,*,*,*,vx,vx,vx,vx,vx,vx,*")
   (set_attr "z10prop" "z10_fwd_A1,
                        z10_fwd_E1,
                        z10_fwd_E1,
                        z10_fwd_E1,
                        z10_fwd_E1,
                        z10_fwd_A1,
                        z10_fwd_E1,
                        z10_fwd_E1,
                        *,
                        *,
                        z10_fwd_A1,
                        z10_fwd_A3,
                        z10_fr_E1,
                        z10_fwd_A3,
                        z10_rec,
                        *,
                        *,
                        *,
                        *,
                        *,
                        z10_rec,
                        z10_super,
                        *,
                        *,
                        *,
                        *,*,*,*,*,*,*,
                        z10_super_A1")
   (set_attr "relative_long" "*,*,*,*,*,*,*,*,*,*,
                              *,yes,*,*,*,*,*,*,*,*,
                              yes,*,*,*,*,*,*,*,*,*,
                              *,*,yes")
])

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (match_operand:DI 1 "register_operand" ""))]
  "TARGET_ZARCH && ACCESS_REG_P (operands[1])"
  [(set (match_dup 2) (match_dup 3))
   (set (match_dup 0) (ashift:DI (match_dup 0) (const_int 32)))
   (set (strict_low_part (match_dup 2)) (match_dup 4))]
  "operands[2] = gen_lowpart (SImode, operands[0]);
   s390_split_access_reg (operands[1], &operands[4], &operands[3]);")

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (match_operand:DI 1 "register_operand" ""))]
  "TARGET_ZARCH && ACCESS_REG_P (operands[0])
   && dead_or_set_p (insn, operands[1])"
  [(set (match_dup 3) (match_dup 2))
   (set (match_dup 1) (lshiftrt:DI (match_dup 1) (const_int 32)))
   (set (match_dup 4) (match_dup 2))]
  "operands[2] = gen_lowpart (SImode, operands[1]);
   s390_split_access_reg (operands[0], &operands[3], &operands[4]);")

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (match_operand:DI 1 "register_operand" ""))]
  "TARGET_ZARCH && ACCESS_REG_P (operands[0])
   && !dead_or_set_p (insn, operands[1])"
  [(set (match_dup 3) (match_dup 2))
   (set (match_dup 1) (rotate:DI (match_dup 1) (const_int 32)))
   (set (match_dup 4) (match_dup 2))
   (set (match_dup 1) (rotate:DI (match_dup 1) (const_int 32)))]
  "operands[2] = gen_lowpart (SImode, operands[1]);
   s390_split_access_reg (operands[0], &operands[3], &operands[4]);")

(define_insn "*movdi_31"
  [(set (match_operand:DI 0 "nonimmediate_operand"
                            "=d,d,Q,S,d  ,o,!*f,!*f,!*f,!R,!T,d")
        (match_operand:DI 1 "general_operand"
                            " Q,S,d,d,dPT,d, *f,  R,  T,*f,*f,b"))]
  "!TARGET_ZARCH"
  "@
   lm\t%0,%N0,%S1
   lmy\t%0,%N0,%S1
   stm\t%1,%N1,%S0
   stmy\t%1,%N1,%S0
   #
   #
   ldr\t%0,%1
   ld\t%0,%1
   ldy\t%0,%1
   std\t%1,%0
   stdy\t%1,%0
   #"
  [(set_attr "op_type" "RS,RSY,RS,RSY,*,*,RR,RX,RXY,RX,RXY,*")
   (set_attr "type" "lm,lm,stm,stm,*,*,floaddf,floaddf,floaddf,fstoredf,fstoredf,*")
   (set_attr "cpu_facility" "*,longdisp,*,longdisp,*,*,*,*,longdisp,*,longdisp,z10")])

; For a load from a symbol ref we can use one of the target registers
; together with larl to load the address.
(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (match_operand:DI 1 "memory_operand" ""))]
  "!TARGET_ZARCH && reload_completed && TARGET_Z10
   && larl_operand (XEXP (operands[1], 0), SImode)"
  [(set (match_dup 2) (match_dup 3))
   (set (match_dup 0) (match_dup 1))]
{
  operands[2] = operand_subword (operands[0], 1, 0, DImode);
  operands[3] = XEXP (operands[1], 0);
  operands[1] = replace_equiv_address (operands[1], operands[2]);
})

(define_split
  [(set (match_operand:DI 0 "nonimmediate_operand" "")
        (match_operand:DI 1 "general_operand" ""))]
  "!TARGET_ZARCH && reload_completed
   && !s_operand (operands[0], DImode)
   && !s_operand (operands[1], DImode)
   && s390_split_ok_p (operands[0], operands[1], DImode, 0)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 0, 0, DImode);
  operands[3] = operand_subword (operands[0], 1, 0, DImode);
  operands[4] = operand_subword (operands[1], 0, 0, DImode);
  operands[5] = operand_subword (operands[1], 1, 0, DImode);
})

(define_split
  [(set (match_operand:DI 0 "nonimmediate_operand" "")
        (match_operand:DI 1 "general_operand" ""))]
  "!TARGET_ZARCH && reload_completed
   && !s_operand (operands[0], DImode)
   && !s_operand (operands[1], DImode)
   && s390_split_ok_p (operands[0], operands[1], DImode, 1)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 1, 0, DImode);
  operands[3] = operand_subword (operands[0], 0, 0, DImode);
  operands[4] = operand_subword (operands[1], 1, 0, DImode);
  operands[5] = operand_subword (operands[1], 0, 0, DImode);
})

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (match_operand:DI 1 "memory_operand" ""))]
  "!TARGET_ZARCH && reload_completed
   && !FP_REG_P (operands[0])
   && !s_operand (operands[1], VOIDmode)"
  [(set (match_dup 0) (match_dup 1))]
{
  rtx addr = operand_subword (operands[0], 1, 0, DImode);
  s390_load_address (addr, XEXP (operands[1], 0));
  operands[1] = replace_equiv_address (operands[1], addr);
})

(define_peephole2
  [(set (match_operand:DI 0 "register_operand" "")
        (mem:DI (match_operand 1 "address_operand" "")))]
  "TARGET_ZARCH
   && !FP_REG_P (operands[0])
   && GET_CODE (operands[1]) == SYMBOL_REF
   && CONSTANT_POOL_ADDRESS_P (operands[1])
   && get_pool_mode (operands[1]) == DImode
   && legitimate_reload_constant_p (get_pool_constant (operands[1]))"
  [(set (match_dup 0) (match_dup 2))]
  "operands[2] = get_pool_constant (operands[1]);")

(define_insn "*la_64"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (match_operand:QI 1 "address_operand" "ZR,ZT"))]
  "TARGET_64BIT"
  "@
   la\t%0,%a1
   lay\t%0,%a1"
  [(set_attr "op_type" "RX,RXY")
   (set_attr "type"    "la")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_fwd_A1,z10_fwd_A1")])

(define_peephole2
  [(parallel
    [(set (match_operand:DI 0 "register_operand" "")
          (match_operand:QI 1 "address_operand" ""))
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_64BIT
   && preferred_la_operand_p (operands[1], const0_rtx)"
  [(set (match_dup 0) (match_dup 1))]
  "")

(define_peephole2
  [(set (match_operand:DI 0 "register_operand" "")
        (match_operand:DI 1 "register_operand" ""))
   (parallel
    [(set (match_dup 0)
          (plus:DI (match_dup 0)
                   (match_operand:DI 2 "nonmemory_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_64BIT
   && !reg_overlap_mentioned_p (operands[0], operands[2])
   && preferred_la_operand_p (operands[1], operands[2])"
  [(set (match_dup 0) (plus:DI (match_dup 1) (match_dup 2)))]
  "")

;
; movsi instruction pattern(s).
;

(define_expand "movsi"
  [(set (match_operand:SI 0 "general_operand" "")
        (match_operand:SI 1 "general_operand" ""))]
  ""
{
  /* Handle symbolic constants.  */
  if (!TARGET_64BIT
      && (SYMBOLIC_CONST (operands[1])
	  || (GET_CODE (operands[1]) == PLUS
	      && XEXP (operands[1], 0) == pic_offset_table_rtx
	      && SYMBOLIC_CONST (XEXP(operands[1], 1)))))
    emit_symbolic_move (operands);
})

(define_insn "*movsi_larl"
  [(set (match_operand:SI 0 "register_operand" "=d")
        (match_operand:SI 1 "larl_operand" "X"))]
  "!TARGET_64BIT
   && !FP_REG_P (operands[0])"
  "larl\t%0,%1"
   [(set_attr "op_type" "RIL")
    (set_attr "type"    "larl")
    (set_attr "z10prop" "z10_fwd_A1")
    (set_attr "relative_long" "yes")])

(define_insn "*movsi_zarch"
  [(set (match_operand:SI 0 "nonimmediate_operand"
	 "=d,    d,    d, d,d,d,d,d,d,R,T,!*f,!*f,!*f,!*f,!*f,!R,!T,d,t,Q,b,Q,t,v,v,v,d,v,R")
        (match_operand:SI 1 "general_operand"
	 " K,N0HS0,N1HS0,Os,L,b,d,R,T,d,d, *f, *f,  R,  R,  T,*f,*f,t,d,t,d,K,Q,K,v,d,v,R,v"))]
  "TARGET_ZARCH"
  "@
   lhi\t%0,%h1
   llilh\t%0,%i1
   llill\t%0,%i1
   iilf\t%0,%o1
   lay\t%0,%a1
   lrl\t%0,%1
   lr\t%0,%1
   l\t%0,%1
   ly\t%0,%1
   st\t%1,%0
   sty\t%1,%0
   ldr\t%0,%1
   ler\t%0,%1
   lde\t%0,%1
   le\t%0,%1
   ley\t%0,%1
   ste\t%1,%0
   stey\t%1,%0
   ear\t%0,%1
   sar\t%0,%1
   stam\t%1,%1,%S0
   strl\t%1,%0
   mvhi\t%0,%1
   lam\t%0,%0,%S1
   vleif\t%v0,%h1,0
   vlr\t%v0,%v1
   vlvgf\t%v0,%1,0
   vlgvf\t%0,%v1,0
   vlef\t%v0,%1,0
   vstef\t%v1,%0,0"
  [(set_attr "op_type" "RI,RI,RI,RIL,RXY,RIL,RR,RX,RXY,RX,RXY,
                        RR,RR,RXE,RX,RXY,RX,RXY,RRE,RRE,RS,RIL,SIL,RS,VRI,VRR,VRS,VRS,VRX,VRX")
   (set_attr "type" "*,
                     *,
                     *,
                     *,
                     la,
                     larl,
                     lr,
                     load,
                     load,
                     store,
                     store,
                     floadsf,
                     floadsf,
                     floadsf,
                     floadsf,
                     floadsf,
                     fstoresf,
                     fstoresf,
                     *,
                     *,
                     *,
                     larl,
                     *,
                     *,*,*,*,*,*,*")
   (set_attr "cpu_facility" "*,*,*,extimm,longdisp,z10,*,*,longdisp,*,longdisp,
                             vx,*,vx,*,longdisp,*,longdisp,*,*,*,z10,z10,*,vx,vx,vx,vx,vx,vx")
   (set_attr "z10prop" "z10_fwd_A1,
                        z10_fwd_E1,
                        z10_fwd_E1,
                        z10_fwd_A1,
                        z10_fwd_A1,
                        z10_fwd_A3,
                        z10_fr_E1,
                        z10_fwd_A3,
                        z10_fwd_A3,
                        z10_rec,
                        z10_rec,
                        *,
                        *,
                        *,
                        *,
                        *,
                        *,
                        *,
                        z10_super_E1,
                        z10_super,
                        *,
                        z10_rec,
                        z10_super,
                        *,*,*,*,*,*,*")
   (set_attr "relative_long" "*,*,*,*,*,yes,*,*,*,*,
                              *,*,*,*,*,*,*,*,*,*,
                              *,yes,*,*,*,*,*,*,*,*")])

(define_insn "*movsi_esa"
  [(set (match_operand:SI 0 "nonimmediate_operand" "=d,d,d,R,!*f,!*f,!*f,!*f,!R,d,t,Q,t")
        (match_operand:SI 1 "general_operand"       "K,d,R,d, *f, *f,  R,  R,*f,t,d,t,Q"))]
  "!TARGET_ZARCH"
  "@
   lhi\t%0,%h1
   lr\t%0,%1
   l\t%0,%1
   st\t%1,%0
   ldr\t%0,%1
   ler\t%0,%1
   lde\t%0,%1
   le\t%0,%1
   ste\t%1,%0
   ear\t%0,%1
   sar\t%0,%1
   stam\t%1,%1,%S0
   lam\t%0,%0,%S1"
  [(set_attr "op_type" "RI,RR,RX,RX,RR,RR,RXE,RX,RX,RRE,RRE,RS,RS")
   (set_attr "type" "*,lr,load,store,floadsf,floadsf,floadsf,floadsf,fstoresf,*,*,*,*")
   (set_attr "z10prop" "z10_fwd_A1,z10_fr_E1,z10_fwd_A3,z10_rec,*,*,*,*,*,z10_super_E1,
                        z10_super,*,*")
   (set_attr "cpu_facility" "*,*,*,*,vx,*,vx,*,*,*,*,*,*")
])

(define_peephole2
  [(set (match_operand:SI 0 "register_operand" "")
        (mem:SI (match_operand 1 "address_operand" "")))]
  "!FP_REG_P (operands[0])
   && GET_CODE (operands[1]) == SYMBOL_REF
   && CONSTANT_POOL_ADDRESS_P (operands[1])
   && get_pool_mode (operands[1]) == SImode
   && legitimate_reload_constant_p (get_pool_constant (operands[1]))"
  [(set (match_dup 0) (match_dup 2))]
  "operands[2] = get_pool_constant (operands[1]);")

(define_insn "*la_31"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (match_operand:QI 1 "address_operand" "ZR,ZT"))]
  "!TARGET_64BIT && legitimate_la_operand_p (operands[1])"
  "@
   la\t%0,%a1
   lay\t%0,%a1"
  [(set_attr "op_type"  "RX,RXY")
   (set_attr "type"     "la")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_fwd_A1,z10_fwd_A1")])

(define_peephole2
  [(parallel
    [(set (match_operand:SI 0 "register_operand" "")
          (match_operand:QI 1 "address_operand" ""))
     (clobber (reg:CC CC_REGNUM))])]
  "!TARGET_64BIT
   && preferred_la_operand_p (operands[1], const0_rtx)"
  [(set (match_dup 0) (match_dup 1))]
  "")

(define_peephole2
  [(set (match_operand:SI 0 "register_operand" "")
        (match_operand:SI 1 "register_operand" ""))
   (parallel
    [(set (match_dup 0)
          (plus:SI (match_dup 0)
                   (match_operand:SI 2 "nonmemory_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  "!TARGET_64BIT
   && !reg_overlap_mentioned_p (operands[0], operands[2])
   && preferred_la_operand_p (operands[1], operands[2])"
  [(set (match_dup 0) (plus:SI (match_dup 1) (match_dup 2)))]
  "")

(define_insn "*la_31_and"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (and:SI (match_operand:QI 1 "address_operand" "ZR,ZT")
                (const_int 2147483647)))]
  "!TARGET_64BIT"
  "@
   la\t%0,%a1
   lay\t%0,%a1"
  [(set_attr "op_type"  "RX,RXY")
   (set_attr "type"     "la")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_fwd_A1,z10_fwd_A1")])

(define_insn_and_split "*la_31_and_cc"
  [(set (match_operand:SI 0 "register_operand" "=d")
        (and:SI (match_operand:QI 1 "address_operand" "p")
                (const_int 2147483647)))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_64BIT"
  "#"
  "&& reload_completed"
  [(set (match_dup 0)
        (and:SI (match_dup 1) (const_int 2147483647)))]
  ""
  [(set_attr "op_type"  "RX")
   (set_attr "type"     "la")])

(define_insn "force_la_31"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (match_operand:QI 1 "address_operand" "ZR,ZT"))
   (use (const_int 0))]
  "!TARGET_64BIT"
  "@
   la\t%0,%a1
   lay\t%0,%a1"
  [(set_attr "op_type"  "RX")
   (set_attr "type"     "la")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_fwd_A1,z10_fwd_A1")])

;
; movhi instruction pattern(s).
;

(define_expand "movhi"
  [(set (match_operand:HI 0 "nonimmediate_operand" "")
        (match_operand:HI 1 "general_operand" ""))]
  ""
{
  /* Make it explicit that loading a register from memory
     always sign-extends (at least) to SImode.  */
  if (optimize && can_create_pseudo_p ()
      && register_operand (operands[0], VOIDmode)
      && GET_CODE (operands[1]) == MEM)
    {
      rtx tmp = gen_reg_rtx (SImode);
      rtx ext = gen_rtx_SIGN_EXTEND (SImode, operands[1]);
      emit_insn (gen_rtx_SET (tmp, ext));
      operands[1] = gen_lowpart (HImode, tmp);
    }
})

(define_insn "*movhi"
  [(set (match_operand:HI 0 "nonimmediate_operand" "=d,d,d,d,d,R,T,b,Q,v,v,v,d,v,R")
        (match_operand:HI 1 "general_operand"      " d,n,R,T,b,d,d,d,K,K,v,d,v,R,v"))]
  ""
  "@
   lr\t%0,%1
   lhi\t%0,%h1
   lh\t%0,%1
   lhy\t%0,%1
   lhrl\t%0,%1
   sth\t%1,%0
   sthy\t%1,%0
   sthrl\t%1,%0
   mvhhi\t%0,%1
   vleih\t%v0,%h1,0
   vlr\t%v0,%v1
   vlvgh\t%v0,%1,0
   vlgvh\t%0,%v1,0
   vleh\t%v0,%1,0
   vsteh\t%v1,%0,0"
  [(set_attr "op_type"      "RR,RI,RX,RXY,RIL,RX,RXY,RIL,SIL,VRI,VRR,VRS,VRS,VRX,VRX")
   (set_attr "type"         "lr,*,*,*,larl,store,store,store,*,*,*,*,*,*,*")
   (set_attr "cpu_facility" "*,*,*,longdisp,z10,*,longdisp,z10,z10,vx,vx,vx,vx,vx,vx")
   (set_attr "z10prop" "z10_fr_E1,
                       z10_fwd_A1,
                       z10_super_E1,
                       z10_super_E1,
                       z10_super_E1,
                       z10_rec,
                       z10_rec,
                       z10_rec,
                       z10_super,*,*,*,*,*,*")
   (set_attr "relative_long" "*,*,*,*,yes,*,*,yes,*,*,*,*,*,*,*")])

(define_peephole2
  [(set (match_operand:HI 0 "register_operand" "")
        (mem:HI (match_operand 1 "address_operand" "")))]
  "GET_CODE (operands[1]) == SYMBOL_REF
   && CONSTANT_POOL_ADDRESS_P (operands[1])
   && get_pool_mode (operands[1]) == HImode
   && GET_CODE (get_pool_constant (operands[1])) == CONST_INT"
  [(set (match_dup 0) (match_dup 2))]
  "operands[2] = get_pool_constant (operands[1]);")

;
; movqi instruction pattern(s).
;

(define_expand "movqi"
  [(set (match_operand:QI 0 "nonimmediate_operand" "")
        (match_operand:QI 1 "general_operand" ""))]
  ""
{
  /* On z/Architecture, zero-extending from memory to register
     is just as fast as a QImode load.  */
  if (TARGET_ZARCH && optimize && can_create_pseudo_p ()
      && register_operand (operands[0], VOIDmode)
      && GET_CODE (operands[1]) == MEM)
    {
      rtx tmp = gen_reg_rtx (DImode);
      rtx ext = gen_rtx_ZERO_EXTEND (DImode, operands[1]);
      emit_insn (gen_rtx_SET (tmp, ext));
      operands[1] = gen_lowpart (QImode, tmp);
    }
})

(define_insn "*movqi"
  [(set (match_operand:QI 0 "nonimmediate_operand" "=d,d,d,d,R,T,Q,S,?Q,v,v,v,d,v,R")
        (match_operand:QI 1 "general_operand"      " d,n,R,T,d,d,n,n,?Q,K,v,d,v,R,v"))]
  ""
  "@
   lr\t%0,%1
   lhi\t%0,%b1
   ic\t%0,%1
   icy\t%0,%1
   stc\t%1,%0
   stcy\t%1,%0
   mvi\t%S0,%b1
   mviy\t%S0,%b1
   #
   vleib\t%v0,%b1,0
   vlr\t%v0,%v1
   vlvgb\t%v0,%1,0
   vlgvb\t%0,%v1,0
   vleb\t%v0,%1,0
   vsteb\t%v1,%0,0"
  [(set_attr "op_type" "RR,RI,RX,RXY,RX,RXY,SI,SIY,SS,VRI,VRR,VRS,VRS,VRX,VRX")
   (set_attr "type" "lr,*,*,*,store,store,store,store,*,*,*,*,*,*,*")
   (set_attr "cpu_facility" "*,*,*,longdisp,*,longdisp,*,longdisp,*,vx,vx,vx,vx,vx,vx")
   (set_attr "z10prop" "z10_fr_E1,
                        z10_fwd_A1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_rec,
                        z10_rec,
                        z10_super,
                        z10_super,
                        *,*,*,*,*,*,*")])

(define_peephole2
  [(set (match_operand:QI 0 "nonimmediate_operand" "")
        (mem:QI (match_operand 1 "address_operand" "")))]
  "GET_CODE (operands[1]) == SYMBOL_REF
   && CONSTANT_POOL_ADDRESS_P (operands[1])
   && get_pool_mode (operands[1]) == QImode
   && GET_CODE (get_pool_constant (operands[1])) == CONST_INT"
  [(set (match_dup 0) (match_dup 2))]
  "operands[2] = get_pool_constant (operands[1]);")

;
; movstrictqi instruction pattern(s).
;

(define_insn "*movstrictqi"
  [(set (strict_low_part (match_operand:QI 0 "register_operand" "+d,d"))
                         (match_operand:QI 1 "memory_operand" "R,T"))]
  ""
  "@
   ic\t%0,%1
   icy\t%0,%1"
  [(set_attr "op_type"  "RX,RXY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

;
; movstricthi instruction pattern(s).
;

(define_insn "*movstricthi"
  [(set (strict_low_part (match_operand:HI 0 "register_operand" "+d,d"))
                         (match_operand:HI 1 "memory_operand" "Q,S"))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   icm\t%0,3,%S1
   icmy\t%0,3,%S1"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

;
; movstrictsi instruction pattern(s).
;

(define_insn "movstrictsi"
  [(set (strict_low_part (match_operand:SI 0 "register_operand" "+d,d,d,d"))
                         (match_operand:SI 1 "general_operand" "d,R,T,t"))]
  "TARGET_ZARCH"
  "@
   lr\t%0,%1
   l\t%0,%1
   ly\t%0,%1
   ear\t%0,%1"
  [(set_attr "op_type" "RR,RX,RXY,RRE")
   (set_attr "type" "lr,load,load,*")
   (set_attr "cpu_facility" "*,*,longdisp,*")
   (set_attr "z10prop" "z10_fr_E1,z10_fwd_A3,z10_fwd_A3,z10_super_E1")])

;
; mov(tf|td) instruction pattern(s).
;

(define_expand "mov<mode>"
  [(set (match_operand:TD_TF 0 "nonimmediate_operand" "")
        (match_operand:TD_TF 1 "general_operand"      ""))]
  ""
  "")

(define_insn "*mov<mode>_64"
  [(set (match_operand:TD_TF 0 "nonimmediate_operand" "=f,f,f,o,d,S, d,o")
        (match_operand:TD_TF 1 "general_operand"      " G,f,o,f,S,d,dT,d"))]
  "TARGET_ZARCH"
  "@
   lzxr\t%0
   lxr\t%0,%1
   #
   #
   lmg\t%0,%N0,%S1
   stmg\t%1,%N1,%S0
   #
   #"
  [(set_attr "op_type"      "RRE,RRE,*,*,RSY,RSY,*,*")
   (set_attr "type"         "fsimptf,fsimptf,*,*,lm,stm,*,*")
   (set_attr "cpu_facility" "z196,*,*,*,*,*,*,*")])

(define_insn "*mov<mode>_31"
  [(set (match_operand:TD_TF 0 "nonimmediate_operand" "=f,f,f,o")
        (match_operand:TD_TF 1 "general_operand"      " G,f,o,f"))]
  "!TARGET_ZARCH"
  "@
   lzxr\t%0
   lxr\t%0,%1
   #
   #"
  [(set_attr "op_type"      "RRE,RRE,*,*")
   (set_attr "type"         "fsimptf,fsimptf,*,*")
   (set_attr "cpu_facility" "z196,*,*,*")])

; TFmode in GPRs splitters

(define_split
  [(set (match_operand:TD_TF 0 "nonimmediate_operand" "")
        (match_operand:TD_TF 1 "general_operand"      ""))]
  "TARGET_ZARCH && reload_completed
   && !s_operand (operands[0], <MODE>mode)
   && !s_operand (operands[1], <MODE>mode)
   && s390_split_ok_p (operands[0], operands[1], <MODE>mode, 0)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 0, 0, <MODE>mode);
  operands[3] = operand_subword (operands[0], 1, 0, <MODE>mode);
  operands[4] = operand_subword (operands[1], 0, 0, <MODE>mode);
  operands[5] = operand_subword (operands[1], 1, 0, <MODE>mode);
})

(define_split
  [(set (match_operand:TD_TF 0 "nonimmediate_operand" "")
        (match_operand:TD_TF 1 "general_operand"      ""))]
  "TARGET_ZARCH && reload_completed
   && !s_operand (operands[0], <MODE>mode)
   && !s_operand (operands[1], <MODE>mode)
   && s390_split_ok_p (operands[0], operands[1], <MODE>mode, 1)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 1, 0, <MODE>mode);
  operands[3] = operand_subword (operands[0], 0, 0, <MODE>mode);
  operands[4] = operand_subword (operands[1], 1, 0, <MODE>mode);
  operands[5] = operand_subword (operands[1], 0, 0, <MODE>mode);
})

(define_split
  [(set (match_operand:TD_TF 0 "register_operand" "")
        (match_operand:TD_TF 1 "memory_operand"   ""))]
  "TARGET_ZARCH && reload_completed
   && GENERAL_REG_P (operands[0])
   && !s_operand (operands[1], VOIDmode)"
  [(set (match_dup 0) (match_dup 1))]
{
  rtx addr = operand_subword (operands[0], 1, 0, <MODE>mode);
  addr = gen_lowpart (Pmode, addr);
  s390_load_address (addr, XEXP (operands[1], 0));
  operands[1] = replace_equiv_address (operands[1], addr);
})

; TFmode in BFPs splitters

(define_split
  [(set (match_operand:TD_TF 0 "register_operand" "")
        (match_operand:TD_TF 1 "memory_operand" ""))]
  "reload_completed && offsettable_memref_p (operands[1])
   && FP_REG_P (operands[0])"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = simplify_gen_subreg (<HALF_TMODE>mode, operands[0],
                                     <MODE>mode, 0);
  operands[3] = simplify_gen_subreg (<HALF_TMODE>mode, operands[0],
                                     <MODE>mode, 8);
  operands[4] = adjust_address_nv (operands[1], <HALF_TMODE>mode, 0);
  operands[5] = adjust_address_nv (operands[1], <HALF_TMODE>mode, 8);
})

(define_split
  [(set (match_operand:TD_TF 0 "memory_operand" "")
        (match_operand:TD_TF 1 "register_operand" ""))]
  "reload_completed && offsettable_memref_p (operands[0])
   && FP_REG_P (operands[1])"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = adjust_address_nv (operands[0], <HALF_TMODE>mode, 0);
  operands[3] = adjust_address_nv (operands[0], <HALF_TMODE>mode, 8);
  operands[4] = simplify_gen_subreg (<HALF_TMODE>mode, operands[1],
				     <MODE>mode, 0);
  operands[5] = simplify_gen_subreg (<HALF_TMODE>mode, operands[1],
                                     <MODE>mode, 8);
})

;
; mov(df|dd) instruction pattern(s).
;

(define_expand "mov<mode>"
  [(set (match_operand:DD_DF 0 "nonimmediate_operand" "")
        (match_operand:DD_DF 1 "general_operand"  ""))]
  ""
  "")

(define_insn "*mov<mode>_64dfp"
  [(set (match_operand:DD_DF 0 "nonimmediate_operand"
			       "=f,f,f,d,f,f,R,T,d,d,d,d,b,T,v,v,v,d,v,R")
        (match_operand:DD_DF 1 "general_operand"
			       " G,f,d,f,R,T,f,f,G,d,b,T,d,d,v,G,d,v,R,v"))]
  "TARGET_DFP"
  "@
   lzdr\t%0
   ldr\t%0,%1
   ldgr\t%0,%1
   lgdr\t%0,%1
   ld\t%0,%1
   ldy\t%0,%1
   std\t%1,%0
   stdy\t%1,%0
   lghi\t%0,0
   lgr\t%0,%1
   lgrl\t%0,%1
   lg\t%0,%1
   stgrl\t%1,%0
   stg\t%1,%0
   vlr\t%v0,%v1
   vleig\t%v0,0,0
   vlvgg\t%v0,%1,0
   vlgvg\t%0,%v1,0
   vleg\t%0,%1,0
   vsteg\t%1,%0,0"
  [(set_attr "op_type" "RRE,RR,RRE,RRE,RX,RXY,RX,RXY,RI,RRE,RIL,RXY,RIL,RXY,VRR,VRI,VRS,VRS,VRX,VRX")
   (set_attr "type" "fsimpdf,floaddf,floaddf,floaddf,floaddf,floaddf,
                     fstoredf,fstoredf,*,lr,load,load,store,store,*,*,*,*,load,store")
   (set_attr "z10prop" "*,*,*,*,*,*,*,*,z10_fwd_A1,z10_fr_E1,z10_fwd_A3,z10_fwd_A3,z10_rec,z10_rec,*,*,*,*,*,*")
   (set_attr "cpu_facility" "z196,*,*,*,*,longdisp,*,longdisp,*,*,z10,*,z10,*,vx,vx,vx,vx,vx,vx")
   (set_attr "relative_long" "*,*,*,*,*,*,*,*,*,*,yes,*,yes,*,*,*,*,*,*,*")])

(define_insn "*mov<mode>_64"
  [(set (match_operand:DD_DF 0 "nonimmediate_operand" "=f,f,f,f,R,T,d,d,d,d,b,T")
        (match_operand:DD_DF 1 "general_operand"      " G,f,R,T,f,f,G,d,b,T,d,d"))]
  "TARGET_ZARCH"
  "@
   lzdr\t%0
   ldr\t%0,%1
   ld\t%0,%1
   ldy\t%0,%1
   std\t%1,%0
   stdy\t%1,%0
   lghi\t%0,0
   lgr\t%0,%1
   lgrl\t%0,%1
   lg\t%0,%1
   stgrl\t%1,%0
   stg\t%1,%0"
  [(set_attr "op_type" "RRE,RR,RX,RXY,RX,RXY,RI,RRE,RIL,RXY,RIL,RXY")
   (set_attr "type"    "fsimpdf,fload<mode>,fload<mode>,fload<mode>,
                        fstore<mode>,fstore<mode>,*,lr,load,load,store,store")
   (set_attr "z10prop" "*,*,*,*,*,*,z10_fwd_A1,z10_fr_E1,z10_fwd_A3,z10_fwd_A3,z10_rec,z10_rec")
   (set_attr "cpu_facility" "z196,*,*,longdisp,*,longdisp,*,*,z10,*,z10,*")
   (set_attr "relative_long" "*,*,*,*,*,*,*,*,yes,*,*,*")])

(define_insn "*mov<mode>_31"
  [(set (match_operand:DD_DF 0 "nonimmediate_operand"
                               "=f,f,f,f,R,T,d,d,Q,S,  d,o")
        (match_operand:DD_DF 1 "general_operand"
                               " G,f,R,T,f,f,Q,S,d,d,dPT,d"))]
  "!TARGET_ZARCH"
  "@
   lzdr\t%0
   ldr\t%0,%1
   ld\t%0,%1
   ldy\t%0,%1
   std\t%1,%0
   stdy\t%1,%0
   lm\t%0,%N0,%S1
   lmy\t%0,%N0,%S1
   stm\t%1,%N1,%S0
   stmy\t%1,%N1,%S0
   #
   #"
  [(set_attr "op_type" "RRE,RR,RX,RXY,RX,RXY,RS,RSY,RS,RSY,*,*")
   (set_attr "type"    "fsimpdf,fload<mode>,fload<mode>,fload<mode>,
                        fstore<mode>,fstore<mode>,lm,lm,stm,stm,*,*")
   (set_attr "cpu_facility" "z196,*,*,longdisp,*,longdisp,*,longdisp,*,longdisp,*,*")])

(define_split
  [(set (match_operand:DD_DF 0 "nonimmediate_operand" "")
        (match_operand:DD_DF 1 "general_operand" ""))]
  "!TARGET_ZARCH && reload_completed
   && !s_operand (operands[0], <MODE>mode)
   && !s_operand (operands[1], <MODE>mode)
   && s390_split_ok_p (operands[0], operands[1], <MODE>mode, 0)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 0, 0, <MODE>mode);
  operands[3] = operand_subword (operands[0], 1, 0, <MODE>mode);
  operands[4] = operand_subword (operands[1], 0, 0, <MODE>mode);
  operands[5] = operand_subword (operands[1], 1, 0, <MODE>mode);
})

(define_split
  [(set (match_operand:DD_DF 0 "nonimmediate_operand" "")
        (match_operand:DD_DF 1 "general_operand" ""))]
  "!TARGET_ZARCH && reload_completed
   && !s_operand (operands[0], <MODE>mode)
   && !s_operand (operands[1], <MODE>mode)
   && s390_split_ok_p (operands[0], operands[1], <MODE>mode, 1)"
  [(set (match_dup 2) (match_dup 4))
   (set (match_dup 3) (match_dup 5))]
{
  operands[2] = operand_subword (operands[0], 1, 0, <MODE>mode);
  operands[3] = operand_subword (operands[0], 0, 0, <MODE>mode);
  operands[4] = operand_subword (operands[1], 1, 0, <MODE>mode);
  operands[5] = operand_subword (operands[1], 0, 0, <MODE>mode);
})

(define_split
  [(set (match_operand:DD_DF 0 "register_operand" "")
        (match_operand:DD_DF 1 "memory_operand" ""))]
  "!TARGET_ZARCH && reload_completed
   && !FP_REG_P (operands[0])
   && !s_operand (operands[1], VOIDmode)"
  [(set (match_dup 0) (match_dup 1))]
{
  rtx addr = operand_subword (operands[0], 1, 0, <MODE>mode);
  s390_load_address (addr, XEXP (operands[1], 0));
  operands[1] = replace_equiv_address (operands[1], addr);
})

;
; mov(sf|sd) instruction pattern(s).
;

(define_insn "mov<mode>"
  [(set (match_operand:SD_SF 0 "nonimmediate_operand"
			       "=f,f,f,f,f,f,R,T,d,d,d,d,d,b,R,T,v,v,v,d,v,R")
        (match_operand:SD_SF 1 "general_operand"
			       " G,f,f,R,R,T,f,f,G,d,b,R,T,d,d,d,v,G,d,v,R,v"))]
  ""
  "@
   lzer\t%0
   ldr\t%0,%1
   ler\t%0,%1
   lde\t%0,%1
   le\t%0,%1
   ley\t%0,%1
   ste\t%1,%0
   stey\t%1,%0
   lhi\t%0,0
   lr\t%0,%1
   lrl\t%0,%1
   l\t%0,%1
   ly\t%0,%1
   strl\t%1,%0
   st\t%1,%0
   sty\t%1,%0
   vlr\t%v0,%v1
   vleif\t%v0,0,0
   vlvgf\t%v0,%1,0
   vlgvf\t%0,%v1,0
   vlef\t%0,%1,0
   vstef\t%1,%0,0"
  [(set_attr "op_type" "RRE,RR,RR,RXE,RX,RXY,RX,RXY,RI,RR,RIL,RX,RXY,RIL,RX,RXY,VRR,VRI,VRS,VRS,VRX,VRX")
   (set_attr "type"    "fsimpsf,fsimpsf,fload<mode>,fload<mode>,fload<mode>,fload<mode>,
                        fstore<mode>,fstore<mode>,*,lr,load,load,load,store,store,store,*,*,*,*,load,store")
   (set_attr "z10prop" "*,*,*,*,*,*,*,*,z10_fwd_A1,z10_fr_E1,z10_fr_E1,z10_fwd_A3,z10_fwd_A3,z10_rec,z10_rec,z10_rec,*,*,*,*,*,*")
   (set_attr "cpu_facility" "z196,vx,*,vx,*,longdisp,*,longdisp,*,*,z10,*,longdisp,z10,*,longdisp,vx,vx,vx,vx,vx,vx")
   (set_attr "relative_long" "*,*,*,*,*,*,*,*,*,*,yes,*,*,yes,*,*,*,*,*,*,*,*")])

;
; movcc instruction pattern
;

(define_insn "movcc"
  [(set (match_operand:CC 0 "nonimmediate_operand" "=d,c,d,d,d,R,T")
	(match_operand:CC 1 "nonimmediate_operand" " d,d,c,R,T,d,d"))]
  ""
  "@
   lr\t%0,%1
   tmh\t%1,12288
   ipm\t%0
   l\t%0,%1
   ly\t%0,%1
   st\t%1,%0
   sty\t%1,%0"
  [(set_attr "op_type" "RR,RI,RRE,RX,RXY,RX,RXY")
   (set_attr "type" "lr,*,*,load,load,store,store")
   (set_attr "cpu_facility" "*,*,*,*,longdisp,*,longdisp")
   (set_attr "z10prop" "z10_fr_E1,z10_super,*,z10_fwd_A3,z10_fwd_A3,z10_rec,z10_rec")
   (set_attr "z196prop" "*,*,z196_ends,*,*,*,*")])

;
; Block move (MVC) patterns.
;

(define_insn "*mvc"
  [(set (match_operand:BLK 0 "memory_operand" "=Q")
        (match_operand:BLK 1 "memory_operand" "Q"))
   (use (match_operand 2 "const_int_operand" "n"))]
  "INTVAL (operands[2]) >= 1 && INTVAL (operands[2]) <= 256"
  "mvc\t%O0(%2,%R0),%S1"
  [(set_attr "op_type" "SS")])

; This splitter converts a QI to QI mode copy into a BLK mode copy in
; order to have it implemented with mvc.

(define_split
  [(set (match_operand:QI 0 "memory_operand" "")
        (match_operand:QI 1 "memory_operand" ""))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (match_dup 1))
     (use (const_int 1))])]
{
  operands[0] = adjust_address (operands[0], BLKmode, 0);
  operands[1] = adjust_address (operands[1], BLKmode, 0);
})


(define_peephole2
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (match_operand:BLK 1 "memory_operand" ""))
     (use (match_operand 2 "const_int_operand" ""))])
   (parallel
    [(set (match_operand:BLK 3 "memory_operand" "")
          (match_operand:BLK 4 "memory_operand" ""))
     (use (match_operand 5 "const_int_operand" ""))])]
  "((INTVAL (operands[2]) > 16 && INTVAL (operands[5]) > 16)
    || (INTVAL (operands[2]) + INTVAL (operands[5]) <= 16))
   && s390_offset_p (operands[0], operands[3], operands[2])
   && s390_offset_p (operands[1], operands[4], operands[2])
   && !s390_overlap_p (operands[0], operands[1],
                       INTVAL (operands[2]) + INTVAL (operands[5]))
   && INTVAL (operands[2]) + INTVAL (operands[5]) <= 256"
  [(parallel
    [(set (match_dup 6) (match_dup 7))
     (use (match_dup 8))])]
  "operands[6] = gen_rtx_MEM (BLKmode, XEXP (operands[0], 0));
   operands[7] = gen_rtx_MEM (BLKmode, XEXP (operands[1], 0));
   operands[8] = GEN_INT (INTVAL (operands[2]) + INTVAL (operands[5]));")

(define_peephole2
  [(parallel
    [(set (match_operand:BLK 0 "plus16_Q_operand" "")
          (match_operand:BLK 1 "plus16_Q_operand" ""))
     (use (match_operand 2 "const_int_operand" ""))])]
  "INTVAL (operands[2]) > 16 && INTVAL (operands[2]) <= 32"
  [(parallel
    [(set (match_dup 0) (match_dup 1))
     (use (const_int 16))])
   (parallel
    [(set (match_dup 3) (match_dup 4))
     (use (match_dup 5))])]
  "operands[3] = change_address (operands[0], VOIDmode,
                                 plus_constant (Pmode, XEXP (operands[0], 0), 16));
   operands[4] = change_address (operands[1], VOIDmode,
                                 plus_constant (Pmode, XEXP (operands[1], 0), 16));
   operands[5] = GEN_INT (INTVAL (operands[2]) - 16);")


;
; load_multiple pattern(s).
;
; ??? Due to reload problems with replacing registers inside match_parallel
; we currently support load_multiple/store_multiple only after reload.
;

(define_expand "load_multiple"
  [(match_par_dup 3 [(set (match_operand 0 "" "")
			  (match_operand 1 "" ""))
		     (use (match_operand 2 "" ""))])]
  "reload_completed"
{
  machine_mode mode;
  int regno;
  int count;
  rtx from;
  int i, off;

  /* Support only loading a constant number of fixed-point registers from
     memory and only bother with this if more than two */
  if (GET_CODE (operands[2]) != CONST_INT
      || INTVAL (operands[2]) < 2
      || INTVAL (operands[2]) > 16
      || GET_CODE (operands[1]) != MEM
      || GET_CODE (operands[0]) != REG
      || REGNO (operands[0]) >= 16)
    FAIL;

  count = INTVAL (operands[2]);
  regno = REGNO (operands[0]);
  mode = GET_MODE (operands[0]);
  if (mode != SImode && (!TARGET_ZARCH || mode != DImode))
    FAIL;

  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count));
  if (!can_create_pseudo_p ())
    {
      if (GET_CODE (XEXP (operands[1], 0)) == REG)
	{
	  from = XEXP (operands[1], 0);
	  off = 0;
	}
      else if (GET_CODE (XEXP (operands[1], 0)) == PLUS
	       && GET_CODE (XEXP (XEXP (operands[1], 0), 0)) == REG
	       && GET_CODE (XEXP (XEXP (operands[1], 0), 1)) == CONST_INT)
	{
	  from = XEXP (XEXP (operands[1], 0), 0);
	  off = INTVAL (XEXP (XEXP (operands[1], 0), 1));
	}
      else
	FAIL;
    }
  else
    {
      from = force_reg (Pmode, XEXP (operands[1], 0));
      off = 0;
    }

  for (i = 0; i < count; i++)
    XVECEXP (operands[3], 0, i)
      = gen_rtx_SET (gen_rtx_REG (mode, regno + i),
		     change_address (operands[1], mode,
		       plus_constant (Pmode, from,
				      off + i * GET_MODE_SIZE (mode))));
})

(define_insn "*load_multiple_di"
  [(match_parallel 0 "load_multiple_operation"
		   [(set (match_operand:DI 1 "register_operand" "=r")
			 (match_operand:DI 2 "s_operand" "S"))])]
  "reload_completed && TARGET_ZARCH"
{
  int words = XVECLEN (operands[0], 0);
  operands[0] = gen_rtx_REG (DImode, REGNO (operands[1]) + words - 1);
  return "lmg\t%1,%0,%S2";
}
   [(set_attr "op_type" "RSY")
    (set_attr "type"    "lm")])

(define_insn "*load_multiple_si"
  [(match_parallel 0 "load_multiple_operation"
		   [(set (match_operand:SI 1 "register_operand" "=r,r")
			 (match_operand:SI 2 "s_operand" "Q,S"))])]
  "reload_completed"
{
  int words = XVECLEN (operands[0], 0);
  operands[0] = gen_rtx_REG (SImode, REGNO (operands[1]) + words - 1);
  return which_alternative == 0 ? "lm\t%1,%0,%S2" : "lmy\t%1,%0,%S2";
}
   [(set_attr "op_type" "RS,RSY")
    (set_attr "cpu_facility" "*,longdisp")
    (set_attr "type"    "lm")])

;
; store multiple pattern(s).
;

(define_expand "store_multiple"
  [(match_par_dup 3 [(set (match_operand 0 "" "")
			  (match_operand 1 "" ""))
		     (use (match_operand 2 "" ""))])]
  "reload_completed"
{
  machine_mode mode;
  int regno;
  int count;
  rtx to;
  int i, off;

  /* Support only storing a constant number of fixed-point registers to
     memory and only bother with this if more than two.  */
  if (GET_CODE (operands[2]) != CONST_INT
      || INTVAL (operands[2]) < 2
      || INTVAL (operands[2]) > 16
      || GET_CODE (operands[0]) != MEM
      || GET_CODE (operands[1]) != REG
      || REGNO (operands[1]) >= 16)
    FAIL;

  count = INTVAL (operands[2]);
  regno = REGNO (operands[1]);
  mode = GET_MODE (operands[1]);
  if (mode != SImode && (!TARGET_ZARCH || mode != DImode))
    FAIL;

  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count));

  if (!can_create_pseudo_p ())
    {
      if (GET_CODE (XEXP (operands[0], 0)) == REG)
	{
	  to = XEXP (operands[0], 0);
	  off = 0;
	}
      else if (GET_CODE (XEXP (operands[0], 0)) == PLUS
	       && GET_CODE (XEXP (XEXP (operands[0], 0), 0)) == REG
	       && GET_CODE (XEXP (XEXP (operands[0], 0), 1)) == CONST_INT)
	{
	  to = XEXP (XEXP (operands[0], 0), 0);
	  off = INTVAL (XEXP (XEXP (operands[0], 0), 1));
	}
      else
	FAIL;
    }
  else
    {
      to = force_reg (Pmode, XEXP (operands[0], 0));
      off = 0;
    }

  for (i = 0; i < count; i++)
    XVECEXP (operands[3], 0, i)
      = gen_rtx_SET (change_address (operands[0], mode,
		       plus_constant (Pmode, to,
				      off + i * GET_MODE_SIZE (mode))),
		     gen_rtx_REG (mode, regno + i));
})

(define_insn "*store_multiple_di"
  [(match_parallel 0 "store_multiple_operation"
		   [(set (match_operand:DI 1 "s_operand" "=S")
			 (match_operand:DI 2 "register_operand" "r"))])]
  "reload_completed && TARGET_ZARCH"
{
  int words = XVECLEN (operands[0], 0);
  operands[0] = gen_rtx_REG (DImode, REGNO (operands[2]) + words - 1);
  return "stmg\t%2,%0,%S1";
}
   [(set_attr "op_type" "RSY")
    (set_attr "type"    "stm")])


(define_insn "*store_multiple_si"
  [(match_parallel 0 "store_multiple_operation"
		   [(set (match_operand:SI 1 "s_operand" "=Q,S")
			 (match_operand:SI 2 "register_operand" "r,r"))])]
  "reload_completed"
{
  int words = XVECLEN (operands[0], 0);
  operands[0] = gen_rtx_REG (SImode, REGNO (operands[2]) + words - 1);
  return which_alternative == 0 ? "stm\t%2,%0,%S1" : "stmy\t%2,%0,%S1";
}
   [(set_attr "op_type" "RS,RSY")
    (set_attr "cpu_facility" "*,longdisp")
    (set_attr "type"    "stm")])

;;
;; String instructions.
;;

(define_insn "*execute_rl"
  [(match_parallel 0 "execute_operation"
    [(unspec [(match_operand 1    "register_operand" "a")
	      (match_operand 2    "" "")
              (match_operand:SI 3 "larl_operand" "X")] UNSPEC_EXECUTE)])]
  "TARGET_Z10 && GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
   && GET_MODE_SIZE (GET_MODE (operands[1])) <= UNITS_PER_WORD"
  "exrl\t%1,%3"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "cs")
   (set_attr "relative_long" "yes")])

(define_insn "*execute"
  [(match_parallel 0 "execute_operation"
    [(unspec [(match_operand 1 "register_operand" "a")
              (match_operand:BLK 2 "memory_operand" "R")
              (match_operand 3 "" "")] UNSPEC_EXECUTE)])]
  "GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
   && GET_MODE_SIZE (GET_MODE (operands[1])) <= UNITS_PER_WORD"
  "ex\t%1,%2"
  [(set_attr "op_type" "RX")
   (set_attr "type" "cs")])


;
; strlenM instruction pattern(s).
;

(define_expand "strlen<mode>"
  [(match_operand:P   0 "register_operand" "")  ; result
   (match_operand:BLK 1 "memory_operand" "")    ; input string
   (match_operand:SI  2 "immediate_operand" "") ; search character
   (match_operand:SI  3 "immediate_operand" "")] ; known alignment
  ""
{
  if (!TARGET_VX || operands[2] != const0_rtx)
    emit_insn (gen_strlen_srst<mode> (operands[0], operands[1],
				      operands[2], operands[3]));
  else
    s390_expand_vec_strlen (operands[0], operands[1], operands[3]);

  DONE;
})

(define_expand "strlen_srst<mode>"
  [(set (reg:SI 0) (match_operand:SI 2 "immediate_operand" ""))
   (parallel
    [(set (match_dup 4)
	  (unspec:P [(const_int 0)
		      (match_operand:BLK 1 "memory_operand" "")
		      (reg:SI 0)
		      (match_operand 3 "immediate_operand" "")] UNSPEC_SRST))
     (clobber (scratch:P))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_operand:P 0 "register_operand" "")
          (minus:P (match_dup 4) (match_dup 5)))
     (clobber (reg:CC CC_REGNUM))])]
  ""
{
  operands[4] = gen_reg_rtx (Pmode);
  operands[5] = gen_reg_rtx (Pmode);
  emit_move_insn (operands[5], force_operand (XEXP (operands[1], 0), NULL_RTX));
  operands[1] = replace_equiv_address (operands[1], operands[5]);
})

(define_insn "*strlen<mode>"
  [(set (match_operand:P 0 "register_operand" "=a")
	(unspec:P [(match_operand:P 2 "general_operand" "0")
		    (mem:BLK (match_operand:P 3 "register_operand" "1"))
		    (reg:SI 0)
		    (match_operand 4 "immediate_operand" "")] UNSPEC_SRST))
   (clobber (match_scratch:P 1 "=a"))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "srst\t%0,%1\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

;
; cmpstrM instruction pattern(s).
;

(define_expand "cmpstrsi"
  [(set (reg:SI 0) (const_int 0))
   (parallel
    [(clobber (match_operand 3 "" ""))
     (clobber (match_dup 4))
     (set (reg:CCU CC_REGNUM)
	  (compare:CCU (match_operand:BLK 1 "memory_operand" "")
	 	       (match_operand:BLK 2 "memory_operand" "")))
     (use (reg:SI 0))])
   (parallel
    [(set (match_operand:SI 0 "register_operand" "=d")
	  (unspec:SI [(reg:CCU CC_REGNUM)] UNSPEC_STRCMPCC_TO_INT))
     (clobber (reg:CC CC_REGNUM))])]
  ""
{
  /* As the result of CMPINT is inverted compared to what we need,
     we have to swap the operands.  */
  rtx op1 = operands[2];
  rtx op2 = operands[1];
  rtx addr1 = gen_reg_rtx (Pmode);
  rtx addr2 = gen_reg_rtx (Pmode);

  emit_move_insn (addr1, force_operand (XEXP (op1, 0), NULL_RTX));
  emit_move_insn (addr2, force_operand (XEXP (op2, 0), NULL_RTX));
  operands[1] = replace_equiv_address_nv (op1, addr1);
  operands[2] = replace_equiv_address_nv (op2, addr2);
  operands[3] = addr1;
  operands[4] = addr2;
})

(define_insn "*cmpstr<mode>"
  [(clobber (match_operand:P 0 "register_operand" "=d"))
   (clobber (match_operand:P 1 "register_operand" "=d"))
   (set (reg:CCU CC_REGNUM)
	(compare:CCU (mem:BLK (match_operand:P 2 "register_operand" "0"))
		     (mem:BLK (match_operand:P 3 "register_operand" "1"))))
   (use (reg:SI 0))]
  ""
  "clst\t%0,%1\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

;
; movstr instruction pattern.
;

(define_expand "movstr"
  [(match_operand 0 "register_operand" "")
   (match_operand 1 "memory_operand" "")
   (match_operand 2 "memory_operand" "")]
  ""
{
  if (TARGET_64BIT)
    emit_insn (gen_movstrdi (operands[0], operands[1], operands[2]));
  else
    emit_insn (gen_movstrsi (operands[0], operands[1], operands[2]));
  DONE;
})

(define_expand "movstr<P:mode>"
  [(set (reg:SI 0) (const_int 0))
   (parallel
    [(clobber (match_dup 3))
     (set (match_operand:BLK 1 "memory_operand" "")
	  (match_operand:BLK 2 "memory_operand" ""))
     (set (match_operand:P 0 "register_operand" "")
	  (unspec:P [(match_dup 1)
		   (match_dup 2)
		   (reg:SI 0)] UNSPEC_MVST))
     (clobber (reg:CC CC_REGNUM))])]
  ""
{
  rtx addr1, addr2;

  if (TARGET_VX && optimize_function_for_speed_p (cfun))
    {
      s390_expand_vec_movstr (operands[0], operands[1], operands[2]);
      DONE;
    }

  addr1 = gen_reg_rtx (Pmode);
  addr2 = gen_reg_rtx (Pmode);

  emit_move_insn (addr1, force_operand (XEXP (operands[1], 0), NULL_RTX));
  emit_move_insn (addr2, force_operand (XEXP (operands[2], 0), NULL_RTX));
  operands[1] = replace_equiv_address_nv (operands[1], addr1);
  operands[2] = replace_equiv_address_nv (operands[2], addr2);
  operands[3] = addr2;
})

(define_insn "*movstr"
  [(clobber (match_operand:P 2 "register_operand" "=d"))
   (set (mem:BLK (match_operand:P 1 "register_operand" "0"))
	(mem:BLK (match_operand:P 3 "register_operand" "2")))
   (set (match_operand:P 0 "register_operand" "=d")
	(unspec:P [(mem:BLK (match_dup 1))
		 (mem:BLK (match_dup 3))
		 (reg:SI 0)] UNSPEC_MVST))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "mvst\t%1,%2\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])


;
; cpymemM instruction pattern(s).
;

(define_expand "cpymem<mode>"
  [(set (match_operand:BLK 0 "memory_operand" "")   ; destination
        (match_operand:BLK 1 "memory_operand" ""))  ; source
   (use (match_operand:GPR 2 "general_operand" "")) ; count
   (match_operand 3 "" "")]
  ""
{
  if (s390_expand_cpymem (operands[0], operands[1], operands[2]))
    DONE;
  else
    FAIL;
})

; Move a block that is up to 256 bytes in length.
; The block length is taken as (operands[2] % 256) + 1.

(define_expand "cpymem_short"
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (match_operand:BLK 1 "memory_operand" ""))
     (use (match_operand 2 "nonmemory_operand" ""))
     (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
     (clobber (match_dup 3))])]
  ""
  "operands[3] = gen_rtx_SCRATCH (Pmode);")

(define_insn "*cpymem_short"
  [(set (match_operand:BLK 0 "memory_operand" "=Q,Q,Q,Q")
        (match_operand:BLK 1 "memory_operand" "Q,Q,Q,Q"))
   (use (match_operand 2 "nonmemory_operand" "n,a,a,a"))
   (use (match_operand 3 "immediate_operand" "X,R,X,X"))
   (clobber (match_scratch:P 4 "=X,X,X,&a"))]
  "(GET_MODE (operands[2]) == Pmode || GET_MODE (operands[2]) == VOIDmode)"
  "#"
  [(set_attr "type"         "cs")
   (set_attr "cpu_facility" "*,*,z10,cpu_zarch")])

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (match_operand:BLK 1 "memory_operand" ""))
   (use (match_operand 2 "const_int_operand" ""))
   (use (match_operand 3 "immediate_operand" ""))
   (clobber (scratch))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (match_dup 1))
     (use (match_dup 2))])]
  "operands[2] = GEN_INT ((INTVAL (operands[2]) & 0xff) + 1);")

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (match_operand:BLK 1 "memory_operand" ""))
   (use (match_operand 2 "register_operand" ""))
   (use (match_operand 3 "memory_operand" ""))
   (clobber (scratch))]
  "reload_completed"
  [(parallel
    [(unspec [(match_dup 2) (match_dup 3)
              (const_int 0)] UNSPEC_EXECUTE)
     (set (match_dup 0) (match_dup 1))
     (use (const_int 1))])]
  "")

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (match_operand:BLK 1 "memory_operand" ""))
   (use (match_operand 2 "register_operand" ""))
   (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
   (clobber (scratch))]
  "TARGET_Z10 && reload_completed"
  [(parallel
    [(unspec [(match_dup 2) (const_int 0)
              (label_ref (match_dup 3))] UNSPEC_EXECUTE)
     (set (match_dup 0) (match_dup 1))
     (use (const_int 1))])]
  "operands[3] = gen_label_rtx ();")

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (match_operand:BLK 1 "memory_operand" ""))
   (use (match_operand 2 "register_operand" ""))
   (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
   (clobber (match_operand 3 "register_operand" ""))]
  "reload_completed"
  [(set (match_dup 3) (label_ref (match_dup 4)))
   (parallel
    [(unspec [(match_dup 2) (mem:BLK (match_dup 3))
              (label_ref (match_dup 4))] UNSPEC_EXECUTE)
     (set (match_dup 0) (match_dup 1))
     (use (const_int 1))])]
  "operands[4] = gen_label_rtx ();")

; Move a block of arbitrary length.

(define_expand "cpymem_long"
  [(parallel
    [(clobber (match_dup 2))
     (clobber (match_dup 3))
     (set (match_operand:BLK 0 "memory_operand" "")
          (match_operand:BLK 1 "memory_operand" ""))
     (use (match_operand 2 "general_operand" ""))
     (use (match_dup 3))
     (clobber (reg:CC CC_REGNUM))])]
  ""
{
  machine_mode sreg_mode = TARGET_ZARCH ? DImode : SImode;
  machine_mode dreg_mode = TARGET_ZARCH ? TImode : DImode;
  rtx reg0 = gen_reg_rtx (dreg_mode);
  rtx reg1 = gen_reg_rtx (dreg_mode);
  rtx addr0 = gen_lowpart (Pmode, gen_highpart (sreg_mode, reg0));
  rtx addr1 = gen_lowpart (Pmode, gen_highpart (sreg_mode, reg1));
  rtx len0 = gen_lowpart (Pmode, reg0);
  rtx len1 = gen_lowpart (Pmode, reg1);

  emit_clobber (reg0);
  emit_move_insn (addr0, force_operand (XEXP (operands[0], 0), NULL_RTX));
  emit_move_insn (len0, operands[2]);

  emit_clobber (reg1);
  emit_move_insn (addr1, force_operand (XEXP (operands[1], 0), NULL_RTX));
  emit_move_insn (len1, operands[2]);

  operands[0] = replace_equiv_address_nv (operands[0], addr0);
  operands[1] = replace_equiv_address_nv (operands[1], addr1);
  operands[2] = reg0;
  operands[3] = reg1;
})

(define_insn "*cpymem_long"
  [(clobber (match_operand:<DBL> 0 "register_operand" "=d"))
   (clobber (match_operand:<DBL> 1 "register_operand" "=d"))
   (set (mem:BLK (subreg:P (match_operand:<DBL> 2 "register_operand" "0") 0))
        (mem:BLK (subreg:P (match_operand:<DBL> 3 "register_operand" "1") 0)))
   (use (match_dup 2))
   (use (match_dup 3))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_64BIT || !TARGET_ZARCH"
  "mvcle\t%0,%1,0\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

(define_insn "*cpymem_long_31z"
  [(clobber (match_operand:TI 0 "register_operand" "=d"))
   (clobber (match_operand:TI 1 "register_operand" "=d"))
   (set (mem:BLK (subreg:SI (match_operand:TI 2 "register_operand" "0") 4))
        (mem:BLK (subreg:SI (match_operand:TI 3 "register_operand" "1") 4)))
   (use (match_dup 2))
   (use (match_dup 3))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_64BIT && TARGET_ZARCH"
  "mvcle\t%0,%1,0\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])


;
; Test data class.
;

(define_expand "signbit<mode>2"
  [(set (reg:CCZ CC_REGNUM)
        (unspec:CCZ [(match_operand:FP_ALL 1 "register_operand" "f")
                     (match_dup 2)]
                     UNSPEC_TDC_INSN))
   (set (match_operand:SI 0 "register_operand" "=d")
        (unspec:SI [(reg:CCZ CC_REGNUM)] UNSPEC_CC_TO_INT))]
  "TARGET_HARD_FLOAT"
{
  operands[2] = GEN_INT (S390_TDC_SIGNBIT_SET);
})

(define_expand "isinf<mode>2"
  [(set (reg:CCZ CC_REGNUM)
        (unspec:CCZ [(match_operand:FP_ALL 1 "register_operand" "f")
                     (match_dup 2)]
                     UNSPEC_TDC_INSN))
   (set (match_operand:SI 0 "register_operand" "=d")
        (unspec:SI [(reg:CCZ CC_REGNUM)] UNSPEC_CC_TO_INT))]
  "TARGET_HARD_FLOAT"
{
  operands[2] = GEN_INT (S390_TDC_INFINITY);
})

; This extracts CC into a GPR properly shifted.  The actual IPM
; instruction will be issued by reload.  The constraint of operand 1
; forces reload to use a GPR.  So reload will issue a movcc insn for
; copying CC into a GPR first.
(define_insn_and_split "*cc_to_int"
  [(set (match_operand:SI 0 "nonimmediate_operand"     "=d")
        (unspec:SI [(match_operand 1 "register_operand" "0")]
                   UNSPEC_CC_TO_INT))]
  "operands != NULL"
  "#"
  "reload_completed"
  [(set (match_dup 0) (lshiftrt:SI (match_dup 0) (const_int 28)))])

; This insn is used to generate all variants of the Test Data Class
; instruction, namely tcxb, tcdb, and tceb.  The insn's first operand
; is the register to be tested and the second one is the bit mask
; specifying the required test(s).
;
; tcxb, tcdb, tceb, tdcxt, tdcdt, tdcet
(define_insn "*TDC_insn_<mode>"
  [(set (reg:CCZ CC_REGNUM)
        (unspec:CCZ [(match_operand:FP_ALL 0 "register_operand" "f")
                     (match_operand:SI 1 "const_int_operand")] UNSPEC_TDC_INSN))]
  "TARGET_HARD_FLOAT"
  "t<_d>c<xde><bt>\t%0,%1"
   [(set_attr "op_type" "RXE")
    (set_attr "type"  "fsimp<mode>")])



;
; setmemM instruction pattern(s).
;

(define_expand "setmem<mode>"
  [(set (match_operand:BLK 0 "memory_operand" "")
        (match_operand:QI 2 "general_operand" ""))
   (use (match_operand:GPR 1 "general_operand" ""))
   (match_operand 3 "" "")]
  ""
  "s390_expand_setmem (operands[0], operands[1], operands[2]); DONE;")

; Clear a block that is up to 256 bytes in length.
; The block length is taken as (operands[1] % 256) + 1.

(define_expand "clrmem_short"
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (const_int 0))
     (use (match_operand 1 "nonmemory_operand" ""))
     (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
     (clobber (match_dup 2))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "operands[2] = gen_rtx_SCRATCH (Pmode);")

(define_insn "*clrmem_short"
  [(set (match_operand:BLK 0 "memory_operand" "=Q,Q,Q,Q")
        (const_int 0))
   (use (match_operand 1 "nonmemory_operand" "n,a,a,a"))
   (use (match_operand 2 "immediate_operand" "X,R,X,X"))
   (clobber (match_scratch:P 3 "=X,X,X,&a"))
   (clobber (reg:CC CC_REGNUM))]
  "(GET_MODE (operands[1]) == Pmode || GET_MODE (operands[1]) == VOIDmode)"
  "#"
  [(set_attr "type" "cs")
   (set_attr "cpu_facility" "*,*,z10,cpu_zarch")])

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (const_int 0))
   (use (match_operand 1 "const_int_operand" ""))
   (use (match_operand 2 "immediate_operand" ""))
   (clobber (scratch))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (const_int 0))
     (use (match_dup 1))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[1] = GEN_INT ((INTVAL (operands[1]) & 0xff) + 1);")

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (const_int 0))
   (use (match_operand 1 "register_operand" ""))
   (use (match_operand 2 "memory_operand" ""))
   (clobber (scratch))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(unspec [(match_dup 1) (match_dup 2)
              (const_int 0)] UNSPEC_EXECUTE)
     (set (match_dup 0) (const_int 0))
     (use (const_int 1))
     (clobber (reg:CC CC_REGNUM))])]
  "")

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (const_int 0))
   (use (match_operand 1 "register_operand" ""))
   (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
   (clobber (scratch))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10 && reload_completed"
  [(parallel
    [(unspec [(match_dup 1) (const_int 0)
              (label_ref (match_dup 3))] UNSPEC_EXECUTE)
     (set (match_dup 0) (const_int 0))
     (use (const_int 1))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[3] = gen_label_rtx ();")

(define_split
  [(set (match_operand:BLK 0 "memory_operand" "")
        (const_int 0))
   (use (match_operand 1 "register_operand" ""))
   (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
   (clobber (match_operand 2 "register_operand" ""))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(set (match_dup 2) (label_ref (match_dup 3)))
   (parallel
    [(unspec [(match_dup 1) (mem:BLK (match_dup 2))
              (label_ref (match_dup 3))] UNSPEC_EXECUTE)
     (set (match_dup 0) (const_int 0))
     (use (const_int 1))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[3] = gen_label_rtx ();")

; Initialize a block of arbitrary length with (operands[2] % 256).

(define_expand "setmem_long_<P:mode>"
  [(parallel
    [(clobber (match_dup 1))
     (set (match_operand:BLK 0 "memory_operand" "")
	  (unspec:BLK [(match_operand:P 2 "setmem_operand" "")
		      (match_dup 4)] UNSPEC_REPLICATE_BYTE))
     (use (match_dup 3))
     (clobber (reg:CC CC_REGNUM))])]
  ""
{
  machine_mode sreg_mode = TARGET_ZARCH ? DImode : SImode;
  machine_mode dreg_mode = TARGET_ZARCH ? TImode : DImode;
  rtx reg0 = gen_reg_rtx (dreg_mode);
  rtx reg1 = gen_reg_rtx (dreg_mode);
  rtx addr0 = gen_lowpart (Pmode, gen_highpart (sreg_mode, reg0));
  rtx len0 = gen_lowpart (Pmode, reg0);

  emit_clobber (reg0);
  emit_move_insn (addr0, force_operand (XEXP (operands[0], 0), NULL_RTX));
  emit_move_insn (len0, operands[1]);

  emit_move_insn (reg1, const0_rtx);

  operands[0] = replace_equiv_address_nv (operands[0], addr0);
  operands[1] = reg0;
  operands[3] = reg1;
  operands[4] = gen_lowpart (Pmode, operands[1]);
})

; Patterns for 31 bit + Esa and 64 bit + Zarch.

(define_insn "*setmem_long"
  [(clobber (match_operand:<DBL> 0 "register_operand" "=d"))
   (set (mem:BLK (subreg:P (match_operand:<DBL> 3 "register_operand" "0") 0))
        (unspec:BLK [(match_operand:P 2 "setmem_operand" "Y")
		     (subreg:P (match_dup 3) <modesize>)]
		     UNSPEC_REPLICATE_BYTE))
   (use (match_operand:<DBL> 1 "register_operand" "d"))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_64BIT || !TARGET_ZARCH"
  "mvcle\t%0,%1,%Y2\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

(define_insn "*setmem_long_and"
  [(clobber (match_operand:<DBL> 0 "register_operand" "=d"))
   (set (mem:BLK (subreg:P (match_operand:<DBL> 3 "register_operand" "0") 0))
        (unspec:BLK [(zero_extend:P (match_operand:QI 2 "setmem_operand" "Y"))
		    (subreg:P (match_dup 3) <modesize>)]
		    UNSPEC_REPLICATE_BYTE))
   (use (match_operand:<DBL> 1 "register_operand" "d"))
   (clobber (reg:CC CC_REGNUM))]
  "(TARGET_64BIT || !TARGET_ZARCH)"
  "mvcle\t%0,%1,%Y2\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

; Variants for 31 bit + Zarch, necessary because of the odd in-register offsets
; of the SImode subregs.

(define_insn "*setmem_long_31z"
  [(clobber (match_operand:TI 0 "register_operand" "=d"))
   (set (mem:BLK (subreg:SI (match_operand:TI 3 "register_operand" "0") 4))
        (unspec:BLK [(match_operand:SI 2 "setmem_operand" "Y")
		     (subreg:SI (match_dup 3) 12)] UNSPEC_REPLICATE_BYTE))
   (use (match_operand:TI 1 "register_operand" "d"))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_64BIT && TARGET_ZARCH"
  "mvcle\t%0,%1,%Y2\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

(define_insn "*setmem_long_and_31z"
  [(clobber (match_operand:TI 0 "register_operand" "=d"))
   (set (mem:BLK (subreg:SI (match_operand:TI 3 "register_operand" "0") 4))
        (unspec:BLK [(zero_extend:SI (match_operand:QI 2 "setmem_operand" "Y"))
		    (subreg:SI (match_dup 3) 12)] UNSPEC_REPLICATE_BYTE))
   (use (match_operand:TI 1 "register_operand" "d"))
   (clobber (reg:CC CC_REGNUM))]
  "(!TARGET_64BIT && TARGET_ZARCH)"
  "mvcle\t%0,%1,%Y2\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

;
; cmpmemM instruction pattern(s).
;

(define_expand "cmpmemsi"
  [(set (match_operand:SI 0 "register_operand" "")
        (compare:SI (match_operand:BLK 1 "memory_operand" "")
                    (match_operand:BLK 2 "memory_operand" "") ) )
   (use (match_operand:SI 3 "general_operand" ""))
   (use (match_operand:SI 4 "" ""))]
  ""
{
  if (s390_expand_cmpmem (operands[0], operands[1],
                          operands[2], operands[3]))
    DONE;
  else
    FAIL;
})

; Compare a block that is up to 256 bytes in length.
; The block length is taken as (operands[2] % 256) + 1.

(define_expand "cmpmem_short"
  [(parallel
    [(set (reg:CCU CC_REGNUM)
          (compare:CCU (match_operand:BLK 0 "memory_operand" "")
                       (match_operand:BLK 1 "memory_operand" "")))
     (use (match_operand 2 "nonmemory_operand" ""))
     (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
     (clobber (match_dup 3))])]
  ""
  "operands[3] = gen_rtx_SCRATCH (Pmode);")

(define_insn "*cmpmem_short"
  [(set (reg:CCU CC_REGNUM)
        (compare:CCU (match_operand:BLK 0 "memory_operand" "Q,Q,Q,Q")
                     (match_operand:BLK 1 "memory_operand" "Q,Q,Q,Q")))
   (use (match_operand 2 "nonmemory_operand" "n,a,a,a"))
   (use (match_operand 3 "immediate_operand" "X,R,X,X"))
   (clobber (match_scratch:P 4 "=X,X,X,&a"))]
  "(GET_MODE (operands[2]) == Pmode || GET_MODE (operands[2]) == VOIDmode)"
  "#"
  [(set_attr "type" "cs")
   (set_attr "cpu_facility" "*,*,z10,cpu_zarch")])

(define_split
  [(set (reg:CCU CC_REGNUM)
        (compare:CCU (match_operand:BLK 0 "memory_operand" "")
                     (match_operand:BLK 1 "memory_operand" "")))
   (use (match_operand 2 "const_int_operand" ""))
   (use (match_operand 3 "immediate_operand" ""))
   (clobber (scratch))]
  "reload_completed"
  [(parallel
    [(set (reg:CCU CC_REGNUM) (compare:CCU (match_dup 0) (match_dup 1)))
     (use (match_dup 2))])]
  "operands[2] = GEN_INT ((INTVAL (operands[2]) & 0xff) + 1);")

(define_split
  [(set (reg:CCU CC_REGNUM)
        (compare:CCU (match_operand:BLK 0 "memory_operand" "")
                     (match_operand:BLK 1 "memory_operand" "")))
   (use (match_operand 2 "register_operand" ""))
   (use (match_operand 3 "memory_operand" ""))
   (clobber (scratch))]
  "reload_completed"
  [(parallel
    [(unspec [(match_dup 2) (match_dup 3)
              (const_int 0)] UNSPEC_EXECUTE)
     (set (reg:CCU CC_REGNUM) (compare:CCU (match_dup 0) (match_dup 1)))
     (use (const_int 1))])]
  "")

(define_split
  [(set (reg:CCU CC_REGNUM)
        (compare:CCU (match_operand:BLK 0 "memory_operand" "")
                     (match_operand:BLK 1 "memory_operand" "")))
   (use (match_operand 2 "register_operand" ""))
   (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
   (clobber (scratch))]
  "TARGET_Z10 && reload_completed"
  [(parallel
    [(unspec [(match_dup 2) (const_int 0)
              (label_ref (match_dup 4))] UNSPEC_EXECUTE)
     (set (reg:CCU CC_REGNUM) (compare:CCU (match_dup 0) (match_dup 1)))
     (use (const_int 1))])]
  "operands[4] = gen_label_rtx ();")

(define_split
  [(set (reg:CCU CC_REGNUM)
        (compare:CCU (match_operand:BLK 0 "memory_operand" "")
                     (match_operand:BLK 1 "memory_operand" "")))
   (use (match_operand 2 "register_operand" ""))
   (use (const:BLK (unspec:BLK [(const_int 0)] UNSPEC_INSN)))
   (clobber (match_operand 3 "register_operand" ""))]
  "reload_completed"
  [(set (match_dup 3) (label_ref (match_dup 4)))
   (parallel
    [(unspec [(match_dup 2) (mem:BLK (match_dup 3))
              (label_ref (match_dup 4))] UNSPEC_EXECUTE)
     (set (reg:CCU CC_REGNUM) (compare:CCU (match_dup 0) (match_dup 1)))
     (use (const_int 1))])]
  "operands[4] = gen_label_rtx ();")

; Compare a block of arbitrary length.

(define_expand "cmpmem_long"
  [(parallel
    [(clobber (match_dup 2))
     (clobber (match_dup 3))
     (set (reg:CCU CC_REGNUM)
          (compare:CCU (match_operand:BLK 0 "memory_operand" "")
                       (match_operand:BLK 1 "memory_operand" "")))
     (use (match_operand 2 "general_operand" ""))
     (use (match_dup 3))])]
  ""
{
  machine_mode sreg_mode = TARGET_ZARCH ? DImode : SImode;
  machine_mode dreg_mode = TARGET_ZARCH ? TImode : DImode;
  rtx reg0 = gen_reg_rtx (dreg_mode);
  rtx reg1 = gen_reg_rtx (dreg_mode);
  rtx addr0 = gen_lowpart (Pmode, gen_highpart (sreg_mode, reg0));
  rtx addr1 = gen_lowpart (Pmode, gen_highpart (sreg_mode, reg1));
  rtx len0 = gen_lowpart (Pmode, reg0);
  rtx len1 = gen_lowpart (Pmode, reg1);

  emit_clobber (reg0);
  emit_move_insn (addr0, force_operand (XEXP (operands[0], 0), NULL_RTX));
  emit_move_insn (len0, operands[2]);

  emit_clobber (reg1);
  emit_move_insn (addr1, force_operand (XEXP (operands[1], 0), NULL_RTX));
  emit_move_insn (len1, operands[2]);

  operands[0] = replace_equiv_address_nv (operands[0], addr0);
  operands[1] = replace_equiv_address_nv (operands[1], addr1);
  operands[2] = reg0;
  operands[3] = reg1;
})

(define_insn "*cmpmem_long"
  [(clobber (match_operand:<DBL> 0 "register_operand" "=d"))
   (clobber (match_operand:<DBL> 1 "register_operand" "=d"))
   (set (reg:CCU CC_REGNUM)
        (compare:CCU (mem:BLK (subreg:P (match_operand:<DBL> 2 "register_operand" "0") 0))
                     (mem:BLK (subreg:P (match_operand:<DBL> 3 "register_operand" "1") 0))))
   (use (match_dup 2))
   (use (match_dup 3))]
  "TARGET_64BIT || !TARGET_ZARCH"
  "clcle\t%0,%1,0\;jo\t.-4"
  [(set_attr "length" "8")
   (set_attr "type" "vs")])

(define_insn "*cmpmem_long_31z"
  [(clobber (match_operand:TI 0 "register_operand" "=d"))
   (clobber (match_operand:TI 1 "register_operand" "=d"))
   (set (reg:CCU CC_REGNUM)
        (compare:CCU (mem:BLK (subreg:SI (match_operand:TI 2 "register_operand" "0") 4))
                     (mem:BLK (subreg:SI (match_operand:TI 3 "register_operand" "1") 4))))
   (use (match_dup 2))
   (use (match_dup 3))]
  "!TARGET_64BIT && TARGET_ZARCH"
  "clcle\t%0,%1,0\;jo\t.-4"
  [(set_attr "op_type" "NN")
   (set_attr "type"    "vs")
   (set_attr "length"  "8")])

; Convert CCUmode condition code to integer.
; Result is zero if EQ, positive if LTU, negative if GTU.

(define_insn_and_split "cmpint"
  [(set (match_operand:SI 0 "register_operand" "=d")
        (unspec:SI [(match_operand:CCU 1 "register_operand" "0")]
                   UNSPEC_STRCMPCC_TO_INT))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "#"
  "reload_completed"
  [(set (match_dup 0) (ashift:SI (match_dup 0) (const_int 2)))
   (parallel
    [(set (match_dup 0) (ashiftrt:SI (match_dup 0) (const_int 30)))
     (clobber (reg:CC CC_REGNUM))])])

(define_insn_and_split "*cmpint_cc"
  [(set (reg CC_REGNUM)
        (compare (unspec:SI [(match_operand:CCU 1 "register_operand" "0")]
                            UNSPEC_STRCMPCC_TO_INT)
                 (const_int 0)))
   (set (match_operand:SI 0 "register_operand" "=d")
        (unspec:SI [(match_dup 1)] UNSPEC_STRCMPCC_TO_INT))]
  "s390_match_ccmode (insn, CCSmode)"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (ashift:SI (match_dup 0) (const_int 2)))
   (parallel
    [(set (match_dup 2) (match_dup 3))
     (set (match_dup 0) (ashiftrt:SI (match_dup 0) (const_int 30)))])]
{
  rtx result = gen_rtx_ASHIFTRT (SImode, operands[0], GEN_INT (30));
  operands[2] = SET_DEST (XVECEXP (PATTERN (curr_insn), 0, 0));
  operands[3] = gen_rtx_COMPARE (GET_MODE (operands[2]), result, const0_rtx);
})

(define_insn_and_split "*cmpint_sign"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (sign_extend:DI (unspec:SI [(match_operand:CCU 1 "register_operand" "0")]
                                   UNSPEC_STRCMPCC_TO_INT)))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (ashift:DI (match_dup 0) (const_int 34)))
   (parallel
    [(set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 62)))
     (clobber (reg:CC CC_REGNUM))])])

(define_insn_and_split "*cmpint_sign_cc"
  [(set (reg CC_REGNUM)
        (compare (ashiftrt:DI (ashift:DI (subreg:DI
                   (unspec:SI [(match_operand:CCU 1 "register_operand" "0")]
                              UNSPEC_STRCMPCC_TO_INT) 0)
                   (const_int 32)) (const_int 32))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d")
        (sign_extend:DI (unspec:SI [(match_dup 1)] UNSPEC_STRCMPCC_TO_INT)))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (ashift:DI (match_dup 0) (const_int 34)))
   (parallel
    [(set (match_dup 2) (match_dup 3))
     (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 62)))])]
{
  rtx result = gen_rtx_ASHIFTRT (DImode, operands[0], GEN_INT (62));
  operands[2] = SET_DEST (XVECEXP (PATTERN (curr_insn), 0, 0));
  operands[3] = gen_rtx_COMPARE (GET_MODE (operands[2]), result, const0_rtx);
})


;;
;;- Conversion instructions.
;;

(define_insn "*sethighpartsi"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
	(unspec:SI [(match_operand:BLK 1 "s_operand" "Q,S")
		    (match_operand 2 "const_int_operand" "n,n")] UNSPEC_ICM))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   icm\t%0,%2,%S1
   icmy\t%0,%2,%S1"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_insn "*sethighpartdi_64"
  [(set (match_operand:DI 0 "register_operand" "=d")
	(unspec:DI [(match_operand:BLK 1 "s_operand" "S")
		    (match_operand 2 "const_int_operand" "n")] UNSPEC_ICM))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "icmh\t%0,%2,%S1"
  [(set_attr "op_type" "RSY")
   (set_attr "z10prop" "z10_super")])

(define_insn "*sethighpartdi_31"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
	(unspec:DI [(match_operand:BLK 1 "s_operand" "Q,S")
		    (match_operand 2 "const_int_operand" "n,n")] UNSPEC_ICM))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH"
  "@
   icm\t%0,%2,%S1
   icmy\t%0,%2,%S1"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

;
; extv instruction patterns
;

; FIXME: This expander needs to be converted from DI to GPR as well
; after resolving some issues with it.

(define_expand "extzv"
  [(parallel
    [(set (match_operand:DI 0 "register_operand" "=d")
        (zero_extract:DI
         (match_operand:DI 1 "register_operand" "d")
         (match_operand 2 "const_int_operand" "")   ; size
         (match_operand 3 "const_int_operand" ""))) ; start
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_Z10"
{
  if (! EXTRACT_ARGS_IN_RANGE (INTVAL (operands[2]), INTVAL (operands[3]), 64))
    FAIL;
  /* Starting with zEC12 there is risbgn not clobbering CC.  */
  if (TARGET_ZEC12)
    {
      emit_move_insn (operands[0],
                    gen_rtx_ZERO_EXTRACT (DImode,
                                          operands[1],
                                          operands[2],
                                          operands[3]));
      DONE;
    }
})

(define_insn "*extzv<mode><clobbercc_or_nocc>"
  [(set (match_operand:GPR 0 "register_operand" "=d")
      (zero_extract:GPR
        (match_operand:GPR 1 "register_operand" "d")
        (match_operand 2 "const_int_operand" "")   ; size
        (match_operand 3 "const_int_operand" ""))) ; start
  ]
  "<z10_or_zEC12_cond>
   && EXTRACT_ARGS_IN_RANGE (INTVAL (operands[2]), INTVAL (operands[3]),
			     GET_MODE_BITSIZE (<MODE>mode))"
  "<risbg_n>\t%0,%1,64-%2,128+63,<bitoff_plus>%3+%2" ; dst, src, start, end, shift
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

; 64 bit: (a & -16) | ((b >> 8) & 15)
(define_insn "*extzvdi<clobbercc_or_nocc>_lshiftrt"
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+d")
			 (match_operand 1 "const_int_operand" "")  ; size
			 (match_operand 2 "const_int_operand" "")) ; start
	(lshiftrt:DI (match_operand:DI 3 "register_operand" "d")
		     (match_operand:DI 4 "nonzero_shift_count_operand" "")))]
  "<z10_or_zEC12_cond>
   && EXTRACT_ARGS_IN_RANGE (INTVAL (operands[1]), INTVAL (operands[2]), 64)
   && 64 - UINTVAL (operands[4]) >= UINTVAL (operands[1])"
  "<risbg_n>\t%0,%3,%2,%2+%1-1,128-%2-%1-%4"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

; 32 bit: (a & -16) | ((b >> 8) & 15)
(define_insn "*<risbg_n>_ior_and_sr_ze"
  [(set (match_operand:SI 0 "register_operand" "=d")
	(ior:SI (and:SI
		 (match_operand:SI 1 "register_operand" "0")
		 (match_operand:SI 2 "const_int_operand" ""))
		(subreg:SI
		 (zero_extract:DI
		  (match_operand:DI 3 "register_operand" "d")
		  (match_operand 4 "const_int_operand" "") ; size
		  (match_operand 5 "const_int_operand" "")) ; start
		 4)))]
  "<z10_or_zEC12_cond>
   && EXTRACT_ARGS_IN_RANGE (INTVAL (operands[4]), INTVAL (operands[5]), 64)
   && UINTVAL (operands[2]) == (HOST_WIDE_INT_M1U << UINTVAL (operands[4]))"
  "<risbg_n>\t%0,%3,64-%4,63,%4+%5"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

; ((int)foo >> 10) & 1;
(define_insn "*extract1bitdi<clobbercc_or_nocc>"
  [(set (match_operand:DI 0 "register_operand" "=d")
	(ne:DI (zero_extract:DI
		(match_operand:DI 1 "register_operand" "d")
		(const_int 1)  ; size
		(match_operand 2 "const_int_operand" "")) ; start
	       (const_int 0)))]
  "<z10_or_zEC12_cond>
   && EXTRACT_ARGS_IN_RANGE (1, INTVAL (operands[2]), 64)"
  "<risbg_n>\t%0,%1,64-1,128+63,%2+1" ; dst, src, start, end, shift
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn "*<risbg_n>_and_subregdi_rotr"
  [(set (match_operand:DI 0 "register_operand" "=d")
	(and:DI (subreg:DI
		 (rotate:SINT (match_operand:SINT 1 "register_operand" "d")
			     (match_operand:SINT 2 "const_int_operand" "")) 0)
		(match_operand:DI 3 "contiguous_bitmask_operand" "")))]
  "<z10_or_zEC12_cond>
   && (UINTVAL (operands[3])
       < (HOST_WIDE_INT_1U << (UINTVAL (operands[2]) & 0x3f)))"
  "<risbg_n>\t%0,%1,%s3,128+%e3,<bitoff_plus>%2" ; dst, src, start, end, shift
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn "*<risbg_n>_and_subregdi_rotl"
  [(set (match_operand:DI 0 "register_operand" "=d")
	(and:DI (subreg:DI
		 (rotate:SINT (match_operand:SINT 1 "register_operand" "d")
			     (match_operand:SINT 2 "const_int_operand" "")) 0)
		(match_operand:DI 3 "contiguous_bitmask_operand" "")))]
  "<z10_or_zEC12_cond>
   && !(UINTVAL (operands[3])
	& ((HOST_WIDE_INT_1U << (UINTVAL (operands[2]) & 0x3f)) - 1))"
  "<risbg_n>\t%0,%1,%s3,128+%e3,%2" ; dst, src, start, end, shift
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn "*<risbg_n>_di_and_rot"
  [(set (match_operand:DI 0 "register_operand" "=d")
	(and:DI (rotate:DI (match_operand:DI 1 "register_operand" "d")
			    (match_operand:DI 2 "const_int_operand" ""))
		(match_operand:DI 3 "contiguous_bitmask_operand" "")))]
  "<z10_or_zEC12_cond>"
  "<risbg_n>\t%0,%1,%s3,128+%e3,%2" ; dst, src, start, end, shift
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn_and_split "*pre_z10_extzv<mode>"
  [(set (match_operand:GPR 0 "register_operand" "=d")
	(zero_extract:GPR (match_operand:QI 1 "s_operand" "S")
		          (match_operand 2 "nonzero_shift_count_operand" "")
		          (const_int 0)))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_Z10"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (match_dup 0) (unspec:GPR [(match_dup 1) (match_dup 3)] UNSPEC_ICM))
     (clobber (reg:CC CC_REGNUM))])
   (set (match_dup 0) (lshiftrt:GPR (match_dup 0) (match_dup 2)))]
{
  int bitsize = INTVAL (operands[2]);
  int size = (bitsize - 1) / BITS_PER_UNIT + 1; /* round up */
  unsigned HOST_WIDE_INT mask
    = ((HOST_WIDE_INT_1U << size) - 1) << (GET_MODE_SIZE (SImode) - size);

  operands[1] = adjust_address (operands[1], BLKmode, 0);
  set_mem_size (operands[1], size);
  operands[2] = GEN_INT (<GPR:bitsize> - bitsize);
  operands[3] = GEN_INT (mask);
})

(define_insn_and_split "*pre_z10_extv<mode>"
  [(set (match_operand:GPR 0 "register_operand" "=d")
	(sign_extract:GPR (match_operand:QI 1 "s_operand" "S")
		          (match_operand 2 "nonzero_shift_count_operand" "")
		          (const_int 0)))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "#"
  "&& reload_completed"
  [(parallel
    [(set (match_dup 0) (unspec:GPR [(match_dup 1) (match_dup 3)] UNSPEC_ICM))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_dup 0) (ashiftrt:GPR (match_dup 0) (match_dup 2)))
     (clobber (reg:CC CC_REGNUM))])]
{
  int bitsize = INTVAL (operands[2]);
  int size = (bitsize - 1) / BITS_PER_UNIT + 1; /* round up */
  unsigned HOST_WIDE_INT mask
    = ((HOST_WIDE_INT_1U << size) - 1) << (GET_MODE_SIZE (SImode) - size);

  operands[1] = adjust_address (operands[1], BLKmode, 0);
  set_mem_size (operands[1], size);
  operands[2] = GEN_INT (<GPR:bitsize> - bitsize);
  operands[3] = GEN_INT (mask);
})

;
; insv instruction patterns
;

(define_expand "insv"
  [(set (zero_extract (match_operand 0 "nonimmediate_operand" "")
		      (match_operand 1 "const_int_operand" "")
		      (match_operand 2 "const_int_operand" ""))
	(match_operand 3 "general_operand" ""))]
  ""
{
  if (s390_expand_insv (operands[0], operands[1], operands[2], operands[3]))
    DONE;
  FAIL;
})


; The normal RTL expansion will never generate a zero_extract where
; the location operand isn't word mode.  However, we do this in the
; back-end when generating atomic operations. See s390_two_part_insv.
(define_insn "*insv<mode><clobbercc_or_nocc>"
  [(set (zero_extract:GPR (match_operand:GPR 0 "nonimmediate_operand" "+d")
			  (match_operand 1 "const_int_operand"    "I")  ; size
			  (match_operand 2 "const_int_operand"    "I")) ; pos
	(match_operand:GPR 3 "nonimmediate_operand" "d"))]
  "<z10_or_zEC12_cond>
   && EXTRACT_ARGS_IN_RANGE (INTVAL (operands[1]), INTVAL (operands[2]),
			     GET_MODE_BITSIZE (<MODE>mode))
   && (INTVAL (operands[1]) + INTVAL (operands[2])) <= <bitsize>"
  "<risbg_n>\t%0,%3,<bitoff_plus>%2,<bitoff_plus>%2+%1-1,<bitsize>-%2-%1"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

; and op1 with a mask being 1 for the selected bits and 0 for the rest
; and op3=op0 with a mask being 0 for the selected bits and 1 for the rest
(define_insn "*insv<mode><clobbercc_or_nocc>_noshift"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d,d")
	(ior:GPR (and:GPR (match_operand:GPR 1 "nonimmediate_operand" "d,0")
			  (match_operand:GPR 2 "contiguous_bitmask_operand" ""))
		 (and:GPR (match_operand:GPR 3 "nonimmediate_operand" "0,d")
			  (match_operand:GPR 4 "const_int_operand" ""))))]
  "<z10_or_zEC12_cond> && INTVAL (operands[2]) == ~INTVAL (operands[4])"
  "@
   <risbg_n>\t%0,%1,%<bfstart>2,%<bfend>2,0
   <risbg_n>\t%0,%3,%<bfstart>4,%<bfend>4,0"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn "*insv_z10_noshift_cc"
  [(set (reg CC_REGNUM)
       (compare
	(ior:DI
	 (and:DI (match_operand:DI 1 "nonimmediate_operand" "d,0")
		  (match_operand:DI 2 "contiguous_bitmask_operand" ""))
	 (and:DI (match_operand:DI 3 "nonimmediate_operand" "0,d")
		  (match_operand:DI 4 "const_int_operand" "")))
	(const_int 0)))
   (set (match_operand:DI 0 "nonimmediate_operand" "=d,d")
	(ior:DI (and:DI (match_dup 1) (match_dup 2))
		 (and:DI (match_dup 3) (match_dup 4))))]
  "TARGET_Z10 && s390_match_ccmode (insn, CCSmode)
   && INTVAL (operands[2]) == ~INTVAL (operands[4])"
  "@
   risbg\t%0,%1,%s2,%e2,0
   risbg\t%0,%3,%s4,%e4,0"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn "*insv_z10_noshift_cconly"
  [(set
    (reg CC_REGNUM)
    (compare
     (ior:DI
      (and:DI (match_operand:DI 1 "nonimmediate_operand" "d,0")
	       (match_operand:DI 2 "contiguous_bitmask_operand" ""))
      (and:DI (match_operand:DI 3 "nonimmediate_operand" "0,d")
	       (match_operand:DI 4 "const_int_operand" "")))
     (const_int 0)))
  (clobber (match_scratch:DI 0 "=d,d"))]
  "TARGET_Z10 && s390_match_ccmode (insn, CCSmode)
   && INTVAL (operands[2]) == ~INTVAL (operands[4])"
  "@
   risbg\t%0,%1,%s2,%e2,0
   risbg\t%0,%3,%s4,%e4,0"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

; Implement appending Y on the left of S bits of X
; x = (y << s) | (x & ((1 << s) - 1))
(define_insn "*insv<mode><clobbercc_or_nocc>_appendbitsleft"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d")
	(ior:GPR (and:GPR (match_operand:GPR 1 "nonimmediate_operand" "0")
			  (match_operand:GPR 2 "immediate_operand" ""))
		 (ashift:GPR (match_operand:GPR 3 "nonimmediate_operand" "d")
			     (match_operand:GPR 4 "nonzero_shift_count_operand" ""))))]
  "<z10_or_zEC12_cond>
   && UINTVAL (operands[2]) == (HOST_WIDE_INT_1U << UINTVAL (operands[4])) - 1"
  "<risbg_n>\t%0,%3,<bitoff>,64-%4-1,%4"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

; a = ((i32)a & -16777216) | (((ui32)b) >> 8)
(define_insn "*<risbg_n>_<mode>_ior_and_lshiftrt"
  [(set (match_operand:GPR 0 "register_operand" "=d")
	(ior:GPR (and:GPR
		  (match_operand:GPR 1 "register_operand" "0")
		  (match_operand:GPR 2 "const_int_operand" ""))
		 (lshiftrt:GPR
		  (match_operand:GPR 3 "register_operand" "d")
		  (match_operand:GPR 4 "nonzero_shift_count_operand" ""))))]
  "<z10_or_zEC12_cond> && UINTVAL (operands[2])
   == (HOST_WIDE_INT_M1U
       << (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4])))"
  "<risbg_n>\t%0,%3,<bitoff_plus>%4,63,64-%4"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

; (ui32)(((ui64)x) >> 48) | ((i32)y & -65536);
(define_insn "*<risbg_n>_sidi_ior_and_lshiftrt"
  [(set (match_operand:SI 0 "register_operand" "=d")
	(ior:SI (and:SI
		 (match_operand:SI 1 "register_operand" "0")
		 (match_operand:SI 2 "const_int_operand" ""))
		(subreg:SI
		 (lshiftrt:DI
		  (match_operand:DI 3 "register_operand" "d")
		  (match_operand:DI 4 "nonzero_shift_count_operand" "")) 4)))]
  "<z10_or_zEC12_cond>
   && UINTVAL (operands[2]) == ~(HOST_WIDE_INT_M1U >> UINTVAL (operands[4]))"
  "<risbg_n>\t%0,%3,%4,63,64-%4"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

; (ui32)(((ui64)x) >> 12) & -4
(define_insn "*trunc_sidi_and_subreg_lshrt<clobbercc_or_nocc>"
  [(set (match_operand:SI 0 "register_operand" "=d")
	(and:SI
	 (subreg:SI (lshiftrt:DI
		     (match_operand:DI 1 "register_operand" "d")
		     (match_operand:DI 2 "nonzero_shift_count_operand" "")) 4)
	 (match_operand:SI 3 "contiguous_bitmask_nowrap_operand" "")))]
  "<z10_or_zEC12_cond>"
  "<risbg_n>\t%0,%1,%t3,128+%f3,64-%2"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

; z = (x << c) | (y >> d) with (x << c) and (y >> d) not overlapping after shifting
;  -> z = y >> d; z = (x << c) | (z & ((1 << c) - 1))
;  -> z = y >> d; z = risbg;

(define_split
  [(set (match_operand:GPR 0 "nonimmediate_operand" "")
	(ior:GPR (lshiftrt:GPR (match_operand:GPR 1 "nonimmediate_operand" "")
			       (match_operand:GPR 2 "nonzero_shift_count_operand" ""))
		 (ashift:GPR (match_operand:GPR 3 "nonimmediate_operand" "")
			     (match_operand:GPR 4 "nonzero_shift_count_operand" ""))))]
  "TARGET_ZEC12 && UINTVAL (operands[2]) + UINTVAL (operands[4]) >= <bitsize>"
  [(set (match_dup 6)
	(lshiftrt:GPR (match_dup 1) (match_dup 2)))
   (set (match_dup 0)
	(ior:GPR (and:GPR (match_dup 6) (match_dup 5))
		 (ashift:GPR (match_dup 3) (match_dup 4))))]
{
  operands[5] = GEN_INT ((HOST_WIDE_INT_1U << UINTVAL (operands[4])) - 1);
  if (reg_overlap_mentioned_p (operands[0], operands[3]))
    {
      if (!can_create_pseudo_p ())
	FAIL;
      operands[6] = gen_reg_rtx (<MODE>mode);
    }
  else
    operands[6] = operands[0];
})

(define_split
  [(parallel
    [(set (match_operand:GPR 0 "nonimmediate_operand" "")
	  (ior:GPR (lshiftrt:GPR (match_operand:GPR 1 "nonimmediate_operand" "")
				 (match_operand:GPR 2 "nonzero_shift_count_operand" ""))
		   (ashift:GPR (match_operand:GPR 3 "nonimmediate_operand" "")
			       (match_operand:GPR 4 "nonzero_shift_count_operand" ""))))
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_Z10 && !TARGET_ZEC12 && UINTVAL (operands[2]) + UINTVAL (operands[4]) >= <bitsize>"
  [(set (match_dup 6)
	(lshiftrt:GPR (match_dup 1) (match_dup 2)))
   (parallel
    [(set (match_dup 0)
	  (ior:GPR (and:GPR (match_dup 6) (match_dup 5))
		   (ashift:GPR (match_dup 3) (match_dup 4))))
     (clobber (reg:CC CC_REGNUM))])]
{
  operands[5] = GEN_INT ((HOST_WIDE_INT_1U << UINTVAL (operands[4])) - 1);
  if (reg_overlap_mentioned_p (operands[0], operands[3]))
    {
      if (!can_create_pseudo_p ())
	FAIL;
      operands[6] = gen_reg_rtx (<MODE>mode);
    }
  else
    operands[6] = operands[0];
})

; rosbg, rxsbg
(define_insn "*r<noxa>sbg_<mode>_noshift"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d")
	(IXOR:GPR
	  (and:GPR (match_operand:GPR 1 "nonimmediate_operand" "d")
                   (match_operand:GPR 2 "contiguous_bitmask_operand" ""))
	  (match_operand:GPR 3 "nonimmediate_operand" "0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10"
  "r<noxa>sbg\t%0,%1,%<bfstart>2,%<bfend>2,0"
  [(set_attr "op_type" "RIE")])

; rosbg, rxsbg
(define_insn "*r<noxa>sbg_di_rotl"
  [(set (match_operand:DI 0 "nonimmediate_operand" "=d")
	(IXOR:DI
	  (and:DI
	    (rotate:DI
	      (match_operand:DI 1 "nonimmediate_operand" "d")
              (match_operand:DI 3 "const_int_operand" ""))
            (match_operand:DI 2 "contiguous_bitmask_operand" ""))
	  (match_operand:DI 4 "nonimmediate_operand" "0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10"
  "r<noxa>sbg\t%0,%1,%s2,%e2,%b3"
  [(set_attr "op_type" "RIE")])

; rosbg, rxsbg
(define_insn "*r<noxa>sbg_<mode>_srl_bitmask"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d")
	(IXOR:GPR
	  (and:GPR
	    (lshiftrt:GPR
              (match_operand:GPR 1 "nonimmediate_operand" "d")
              (match_operand:GPR 3 "nonzero_shift_count_operand" ""))
            (match_operand:GPR 2 "contiguous_bitmask_nowrap_operand" ""))
	  (match_operand:GPR 4 "nonimmediate_operand" "0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10
   && s390_extzv_shift_ok (<bitsize>, 64 - INTVAL (operands[3]),
                           INTVAL (operands[2]))"
  {
    operands[3] = GEN_INT (64 - INTVAL (operands[3]));
    return "r<noxa>sbg\t%0,%1,%<bfstart>2,%<bfend>2,%3";
  }
  [(set_attr "op_type" "RIE")])

; rosbg, rxsbg
(define_insn "*r<noxa>sbg_<mode>_sll_bitmask"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d")
	(IXOR:GPR
	  (and:GPR
	    (ashift:GPR
              (match_operand:GPR 1 "nonimmediate_operand" "d")
              (match_operand:GPR 3 "nonzero_shift_count_operand" ""))
            (match_operand:GPR 2 "contiguous_bitmask_nowrap_operand" ""))
	  (match_operand:GPR 4 "nonimmediate_operand" "0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10
   && s390_extzv_shift_ok (<bitsize>, INTVAL (operands[3]),
                           INTVAL (operands[2]))"
  "r<noxa>sbg\t%0,%1,%<bfstart>2,%<bfend>2,%3"
  [(set_attr "op_type" "RIE")])

;; unsigned {int,long} a, b
;; a = a | (b << const_int)
;; a = a ^ (b << const_int)
; rosbg, rxsbg
(define_insn "*r<noxa>sbg_<mode>_sll"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d")
	(IXOR:GPR
	  (ashift:GPR
            (match_operand:GPR 1 "nonimmediate_operand" "d")
            (match_operand:GPR 2 "nonzero_shift_count_operand" ""))
	  (match_operand:GPR 3 "nonimmediate_operand" "0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10"
  {
    operands[3] = GEN_INT (63 - INTVAL (operands[2]));
    return "r<noxa>sbg\t%0,%1,<bitoff>,%3,%2";
  }
  [(set_attr "op_type" "RIE")])

;; unsigned {int,long} a, b
;; a = a | (b >> const_int)
;; a = a ^ (b >> const_int)
; rosbg, rxsbg
(define_insn "*r<noxa>sbg_<mode>_srl"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d")
	(IXOR:GPR
	  (lshiftrt:GPR
            (match_operand:GPR 1 "nonimmediate_operand" "d")
            (match_operand:GPR 2 "nonzero_shift_count_operand" ""))
	  (match_operand:GPR 3 "nonimmediate_operand" "0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10"
  {
    operands[3] = GEN_INT (64 - INTVAL (operands[2]));
    operands[2] = GEN_INT (<bitoff_plus> INTVAL (operands[2]));
    return "r<noxa>sbg\t%0,%1,%2,63,%3";
  }
  [(set_attr "op_type" "RIE")])

; rosbg, rxsbg
(define_insn "*r<noxa>sbg_sidi_srl"
  [(set (match_operand:SI 0 "nonimmediate_operand" "=d")
        (IXOR:SI
          (subreg:SI
            (zero_extract:DI
              (match_operand:DI 1 "nonimmediate_operand" "d")
              (const_int 32)
              (match_operand:DI 2 "immediate_operand" ""))
            4)
          (match_operand:SI 3 "nonimmediate_operand" "0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10"
  {
    operands[2] = GEN_INT (32 + INTVAL (operands[2]));
    return "r<noxa>sbg\t%0,%1,32,63,%2";
  }
  [(set_attr "op_type" "RIE")])

;; These two are generated by combine for s.bf &= val.
;; ??? For bitfields smaller than 32-bits, we wind up with SImode
;; shifts and ands, which results in some truly awful patterns
;; including subregs of operations.  Rather unnecessisarily, IMO.
;; Instead of
;;
;; (set (zero_extract:DI (reg/v:DI 50 [ s ])
;;        (const_int 24 [0x18])
;;        (const_int 0 [0]))
;;    (subreg:DI (and:SI (subreg:SI (lshiftrt:DI (reg/v:DI 50 [ s ])
;;                    (const_int 40 [0x28])) 4)
;;            (reg:SI 4 %r4 [ y+4 ])) 0))
;;
;; we should instead generate
;;
;; (set (zero_extract:DI (reg/v:DI 50 [ s ])
;;        (const_int 24 [0x18])
;;        (const_int 0 [0]))
;;    (and:DI (lshiftrt:DI (reg/v:DI 50 [ s ])
;;                    (const_int 40 [0x28]))
;;            (subreg:DI (reg:SI 4 %r4 [ y+4 ]) 0)))
;;
;; by noticing that we can push down the outer paradoxical subreg
;; into the operation.

(define_insn "*insv_rnsbg_noshift"
  [(set (zero_extract:DI
	  (match_operand:DI 0 "nonimmediate_operand" "+d")
	  (match_operand 1 "const_int_operand" "")
	  (match_operand 2 "const_int_operand" ""))
	(and:DI
	  (match_dup 0)
	  (match_operand:DI 3 "nonimmediate_operand" "d")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10
   && EXTRACT_ARGS_IN_RANGE (INTVAL (operands[1]), INTVAL (operands[2]), 64)
   && INTVAL (operands[1]) + INTVAL (operands[2]) == 64"
  "rnsbg\t%0,%3,%2,63,0"
  [(set_attr "op_type" "RIE")])

(define_insn "*insv_rnsbg_srl"
  [(set (zero_extract:DI
	  (match_operand:DI 0 "nonimmediate_operand" "+d")
	  (match_operand 1 "const_int_operand" "")
	  (match_operand 2 "const_int_operand" ""))
	(and:DI
	  (lshiftrt:DI
	    (match_dup 0)
	    (match_operand 3 "const_int_operand" ""))
	  (match_operand:DI 4 "nonimmediate_operand" "d")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z10
   && EXTRACT_ARGS_IN_RANGE (INTVAL (operands[1]), INTVAL (operands[2]), 64)
   && INTVAL (operands[3]) == 64 - INTVAL (operands[1]) - INTVAL (operands[2])"
  "rnsbg\t%0,%4,%2,%2+%1-1,%3"
  [(set_attr "op_type" "RIE")])

(define_insn "*insv<mode>_mem_reg"
  [(set (zero_extract:W (match_operand:QI 0 "memory_operand" "+Q,S")
			(match_operand 1 "const_int_operand" "n,n")
			(const_int 0))
	(match_operand:W 2 "register_operand" "d,d"))]
  "EXTRACT_ARGS_IN_RANGE (INTVAL (operands[1]), 0, 64)
   && INTVAL (operands[1]) > 0
   && INTVAL (operands[1]) <= GET_MODE_BITSIZE (SImode)
   && INTVAL (operands[1]) % BITS_PER_UNIT == 0"
{
    int size = INTVAL (operands[1]) / BITS_PER_UNIT;

    operands[1] = GEN_INT ((HOST_WIDE_INT_1U << size) - 1);
    return (which_alternative == 0) ? "stcm\t%2,%1,%S0"
				    : "stcmy\t%2,%1,%S0";
}
  [(set_attr "op_type" "RS,RSY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_super,z10_super")])

(define_insn "*insvdi_mem_reghigh"
  [(set (zero_extract:DI (match_operand:QI 0 "memory_operand" "+S")
			 (match_operand 1 "const_int_operand" "n")
			 (const_int 0))
	(lshiftrt:DI (match_operand:DI 2 "register_operand" "d")
		     (const_int 32)))]
  "TARGET_ZARCH
   && EXTRACT_ARGS_IN_RANGE (INTVAL (operands[1]), 0, 64)
   && INTVAL (operands[1]) > 0
   && INTVAL (operands[1]) <= GET_MODE_BITSIZE (SImode)
   && INTVAL (operands[1]) % BITS_PER_UNIT == 0"
{
    int size = INTVAL (operands[1]) / BITS_PER_UNIT;

    operands[1] = GEN_INT ((HOST_WIDE_INT_1U << size) - 1);
    return "stcmh\t%2,%1,%S0";
}
[(set_attr "op_type" "RSY")
 (set_attr "z10prop" "z10_super")])

(define_insn "*insvdi_reg_imm"
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+d")
			 (const_int 16)
			 (match_operand 1 "const_int_operand" "n"))
	(match_operand:DI 2 "const_int_operand" "n"))]
  "TARGET_ZARCH
   && EXTRACT_ARGS_IN_RANGE (16, INTVAL (operands[1]), 64)
   && INTVAL (operands[1]) >= 0
   && INTVAL (operands[1]) < BITS_PER_WORD
   && INTVAL (operands[1]) % 16 == 0"
{
  switch (BITS_PER_WORD - INTVAL (operands[1]))
    {
      case 64: return "iihh\t%0,%x2"; break;
      case 48: return "iihl\t%0,%x2"; break;
      case 32: return "iilh\t%0,%x2"; break;
      case 16: return "iill\t%0,%x2"; break;
      default: gcc_unreachable();
    }
}
  [(set_attr "op_type" "RI")
   (set_attr "z10prop" "z10_super_E1")])

; Update the left-most 32 bit of a DI.
(define_insn "*insv_h_di_reg_extimm"
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+d")
			 (const_int 32)
			 (const_int 0))
	(match_operand:DI 1 "const_int_operand" "n"))]
  "TARGET_EXTIMM"
  "iihf\t%0,%o1"
  [(set_attr "op_type" "RIL")
   (set_attr "z10prop" "z10_fwd_E1")])

; Update the right-most 32 bit of a DI.
(define_insn "*insv_l_di_reg_extimm"
  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+d")
			 (const_int 32)
			 (const_int 32))
	(match_operand:DI 1 "const_int_operand" "n"))]
  "TARGET_EXTIMM"
  "iilf\t%0,%o1"
  [(set_attr "op_type" "RIL")
   (set_attr "z10prop" "z10_fwd_A1")])

;
; extendsidi2 instruction pattern(s).
;

(define_expand "extendsidi2"
  [(set (match_operand:DI 0 "register_operand" "")
        (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "")))]
  ""
{
  if (!TARGET_ZARCH)
    {
      emit_clobber (operands[0]);
      emit_move_insn (gen_highpart (SImode, operands[0]), operands[1]);
      emit_move_insn (gen_lowpart (SImode, operands[0]), const0_rtx);
      emit_insn (gen_ashrdi3 (operands[0], operands[0], GEN_INT (32)));
      DONE;
    }
})

(define_insn "*extendsidi2"
  [(set (match_operand:DI 0 "register_operand" "=d,d,d")
        (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "d,T,b")))]
  "TARGET_ZARCH"
  "@
   lgfr\t%0,%1
   lgf\t%0,%1
   lgfrl\t%0,%1"
  [(set_attr "op_type"      "RRE,RXY,RIL")
   (set_attr "type"         "*,*,larl")
   (set_attr "cpu_facility" "*,*,z10")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,z10_super_E1")
   (set_attr "relative_long" "*,*,yes")])

;
; extend(hi|qi)(si|di)2 instruction pattern(s).
;

(define_expand "extend<HQI:mode><DSI:mode>2"
  [(set (match_operand:DSI 0 "register_operand" "")
        (sign_extend:DSI (match_operand:HQI 1 "nonimmediate_operand" "")))]
  ""
{
  if (<DSI:MODE>mode == DImode && !TARGET_ZARCH)
    {
      rtx tmp = gen_reg_rtx (SImode);
      emit_insn (gen_extend<HQI:mode>si2 (tmp, operands[1]));
      emit_insn (gen_extendsidi2 (operands[0], tmp));
      DONE;
    }
  else if (!TARGET_EXTIMM)
    {
      rtx bitcount = GEN_INT (<DSI:bitsize> - <HQI:bitsize>);

      operands[1] = gen_lowpart (<DSI:MODE>mode, operands[1]);
      emit_insn (gen_ashl<DSI:mode>3 (operands[0], operands[1], bitcount));
      emit_insn (gen_ashr<DSI:mode>3 (operands[0], operands[0], bitcount));
      DONE;
    }
})

;
; extendhidi2 instruction pattern(s).
;

(define_insn "*extendhidi2_extimm"
  [(set (match_operand:DI 0 "register_operand" "=d,d,d")
        (sign_extend:DI (match_operand:HI 1 "general_operand" "d,T,b")))]
  "TARGET_ZARCH && TARGET_EXTIMM"
  "@
   lghr\t%0,%1
   lgh\t%0,%1
   lghrl\t%0,%1"
  [(set_attr "op_type"      "RRE,RXY,RIL")
   (set_attr "type"         "*,*,larl")
   (set_attr "cpu_facility" "extimm,extimm,z10")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,z10_super_E1")
   (set_attr "relative_long" "*,*,yes")])

(define_insn "*extendhidi2"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (sign_extend:DI (match_operand:HI 1 "memory_operand" "T")))]
  "TARGET_ZARCH"
  "lgh\t%0,%1"
  [(set_attr "op_type" "RXY")
   (set_attr "z10prop" "z10_super_E1")])

;
; extendhisi2 instruction pattern(s).
;

(define_insn "*extendhisi2_extimm"
  [(set (match_operand:SI 0 "register_operand" "=d,d,d,d")
        (sign_extend:SI (match_operand:HI 1 "nonimmediate_operand" " d,R,T,b")))]
  "TARGET_EXTIMM"
  "@
   lhr\t%0,%1
   lh\t%0,%1
   lhy\t%0,%1
   lhrl\t%0,%1"
  [(set_attr "op_type"      "RRE,RX,RXY,RIL")
   (set_attr "type"         "*,*,*,larl")
   (set_attr "cpu_facility" "extimm,extimm,extimm,z10")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,z10_super_E1,z10_super_E1")
   (set_attr "relative_long" "*,*,*,yes")])

(define_insn "*extendhisi2"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (sign_extend:SI (match_operand:HI 1 "memory_operand" "R,T")))]
  "!TARGET_EXTIMM"
  "@
   lh\t%0,%1
   lhy\t%0,%1"
  [(set_attr "op_type" "RX,RXY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

;
; extendqi(si|di)2 instruction pattern(s).
;

; lbr, lgbr, lb, lgb
(define_insn "*extendqi<mode>2_extimm"
  [(set (match_operand:GPR 0 "register_operand" "=d,d")
        (sign_extend:GPR (match_operand:QI 1 "nonimmediate_operand" "d,T")))]
  "TARGET_EXTIMM"
  "@
   l<g>br\t%0,%1
   l<g>b\t%0,%1"
  [(set_attr "op_type" "RRE,RXY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

; lb, lgb
(define_insn "*extendqi<mode>2"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (sign_extend:GPR (match_operand:QI 1 "memory_operand" "T")))]
  "!TARGET_EXTIMM && TARGET_LONG_DISPLACEMENT"
  "l<g>b\t%0,%1"
  [(set_attr "op_type" "RXY")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn_and_split "*extendqi<mode>2_short_displ"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (sign_extend:GPR (match_operand:QI 1 "s_operand" "Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_EXTIMM && !TARGET_LONG_DISPLACEMENT"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (match_dup 0) (unspec:GPR [(match_dup 1) (const_int 8)] UNSPEC_ICM))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_dup 0) (ashiftrt:GPR (match_dup 0) (match_dup 2)))
     (clobber (reg:CC CC_REGNUM))])]
{
  operands[1] = adjust_address (operands[1], BLKmode, 0);
  set_mem_size (operands[1], GET_MODE_SIZE (QImode));
  operands[2] = GEN_INT (<GPR:bitsize> - BITS_PER_UNIT);
})

;
; zero_extendsidi2 instruction pattern(s).
;

(define_expand "zero_extendsidi2"
  [(set (match_operand:DI 0 "register_operand" "")
        (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "")))]
  ""
{
  if (!TARGET_ZARCH)
    {
      emit_clobber (operands[0]);
      emit_move_insn (gen_lowpart (SImode, operands[0]), operands[1]);
      emit_move_insn (gen_highpart (SImode, operands[0]), const0_rtx);
      DONE;
    }
})

(define_insn "*zero_extendsidi2"
  [(set (match_operand:DI 0 "register_operand" "=d,d,d")
        (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "d,T,b")))]
  "TARGET_ZARCH"
  "@
   llgfr\t%0,%1
   llgf\t%0,%1
   llgfrl\t%0,%1"
  [(set_attr "op_type"      "RRE,RXY,RIL")
   (set_attr "type"         "*,*,larl")
   (set_attr "cpu_facility" "*,*,z10")
   (set_attr "z10prop" "z10_fwd_E1,z10_fwd_A3,z10_fwd_A3")
   (set_attr "relative_long" "*,*,yes")])

;
; LLGT-type instructions (zero-extend from 31 bit to 64 bit).
;

(define_insn "*llgt_sidi"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (and:DI (subreg:DI (match_operand:SI 1 "memory_operand" "T") 0)
		(const_int 2147483647)))]
  "TARGET_ZARCH"
  "llgt\t%0,%1"
  [(set_attr "op_type"  "RXE")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn_and_split "*llgt_sidi_split"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (and:DI (subreg:DI (match_operand:SI 1 "memory_operand" "T") 0)
		(const_int 2147483647)))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0)
        (and:DI (subreg:DI (match_dup 1) 0)
		(const_int 2147483647)))]
  "")

(define_insn "*llgt_sisi"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (and:SI (match_operand:SI 1 "nonimmediate_operand" "d,T")
		(const_int 2147483647)))]
  "TARGET_ZARCH"
  "@
   llgtr\t%0,%1
   llgt\t%0,%1"
  [(set_attr "op_type"  "RRE,RXE")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_insn "*llgt_didi"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (and:DI (match_operand:DI 1 "nonimmediate_operand" "d,o")
                (const_int 2147483647)))]
  "TARGET_ZARCH"
  "@
   llgtr\t%0,%1
   llgt\t%0,%N1"
  [(set_attr "op_type"  "RRE,RXE")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_split
  [(set (match_operand:DSI 0 "register_operand" "")
        (and:DSI (match_operand:DSI 1 "nonimmediate_operand" "")
                 (const_int 2147483647)))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && reload_completed"
  [(set (match_dup 0)
        (and:DSI (match_dup 1)
                 (const_int 2147483647)))]
  "")

;
; zero_extend(hi|qi)(si|di)2 instruction pattern(s).
;

(define_expand "zero_extend<mode>di2"
  [(set (match_operand:DI 0 "register_operand" "")
        (zero_extend:DI (match_operand:HQI 1 "nonimmediate_operand" "")))]
  ""
{
  if (!TARGET_ZARCH)
    {
      rtx tmp = gen_reg_rtx (SImode);
      emit_insn (gen_zero_extend<mode>si2 (tmp, operands[1]));
      emit_insn (gen_zero_extendsidi2 (operands[0], tmp));
      DONE;
    }
  else if (!TARGET_EXTIMM)
    {
      rtx bitcount = GEN_INT (64 - <HQI:bitsize>);
      operands[1] = gen_lowpart (DImode, operands[1]);
      emit_insn (gen_ashldi3 (operands[0], operands[1], bitcount));
      emit_insn (gen_lshrdi3 (operands[0], operands[0], bitcount));
      DONE;
    }
})

(define_expand "zero_extend<mode>si2"
  [(set (match_operand:SI 0 "register_operand" "")
        (zero_extend:SI (match_operand:HQI 1 "nonimmediate_operand" "")))]
  ""
{
  if (!TARGET_EXTIMM)
    {
      operands[1] = gen_lowpart (SImode, operands[1]);
      emit_insn (gen_andsi3 (operands[0], operands[1],
			     GEN_INT ((1 << <HQI:bitsize>) - 1)));
      DONE;
    }
})

; llhrl, llghrl
(define_insn "*zero_extendhi<mode>2_z10"
  [(set (match_operand:GPR 0 "register_operand" "=d,d,d")
        (zero_extend:GPR (match_operand:HI 1 "nonimmediate_operand" "d,T,b")))]
  "TARGET_Z10"
  "@
   ll<g>hr\t%0,%1
   ll<g>h\t%0,%1
   ll<g>hrl\t%0,%1"
  [(set_attr "op_type"      "RXY,RRE,RIL")
   (set_attr "type"         "*,*,larl")
   (set_attr "cpu_facility" "*,*,z10")
   (set_attr "z10prop" "z10_super_E1,z10_fwd_A3,z10_fwd_A3")
   (set_attr "relative_long" "*,*,yes")])

; llhr, llcr, llghr, llgcr, llh, llc, llgh, llgc
(define_insn "*zero_extend<HQI:mode><GPR:mode>2_extimm"
  [(set (match_operand:GPR 0 "register_operand" "=d,d")
        (zero_extend:GPR (match_operand:HQI 1 "nonimmediate_operand" "d,T")))]
  "TARGET_EXTIMM"
  "@
   ll<g><hc>r\t%0,%1
   ll<g><hc>\t%0,%1"
  [(set_attr "op_type" "RRE,RXY")
   (set_attr "z10prop" "z10_super_E1,z10_fwd_A3")])

; llgh, llgc
(define_insn "*zero_extend<HQI:mode><GPR:mode>2"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (zero_extend:GPR (match_operand:HQI 1 "memory_operand" "T")))]
  "TARGET_ZARCH && !TARGET_EXTIMM"
  "llg<hc>\t%0,%1"
  [(set_attr "op_type" "RXY")
   (set_attr "z10prop" "z10_fwd_A3")])

(define_insn_and_split "*zero_extendhisi2_31"
  [(set (match_operand:SI 0 "register_operand" "=&d")
        (zero_extend:SI (match_operand:HI 1 "s_operand" "S")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (const_int 0))
   (parallel
    [(set (strict_low_part (match_dup 2)) (match_dup 1))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[2] = gen_lowpart (HImode, operands[0]);")

(define_insn_and_split "*zero_extendqisi2_31"
  [(set (match_operand:SI 0 "register_operand" "=&d")
        (zero_extend:SI (match_operand:QI 1 "memory_operand" "T")))]
  "!TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (const_int 0))
   (set (strict_low_part (match_dup 2)) (match_dup 1))]
  "operands[2] = gen_lowpart (QImode, operands[0]);")

;
; zero_extendqihi2 instruction pattern(s).
;

(define_expand "zero_extendqihi2"
  [(set (match_operand:HI 0 "register_operand" "")
        (zero_extend:HI (match_operand:QI 1 "register_operand" "")))]
  "TARGET_ZARCH && !TARGET_EXTIMM"
{
  operands[1] = gen_lowpart (HImode, operands[1]);
  emit_insn (gen_andhi3 (operands[0], operands[1], GEN_INT (0xff)));
  DONE;
})

(define_insn "*zero_extendqihi2_64"
  [(set (match_operand:HI 0 "register_operand" "=d")
        (zero_extend:HI (match_operand:QI 1 "memory_operand" "T")))]
  "TARGET_ZARCH && !TARGET_EXTIMM"
  "llgc\t%0,%1"
  [(set_attr "op_type" "RXY")
   (set_attr "z10prop" "z10_fwd_A3")])

(define_insn_and_split "*zero_extendqihi2_31"
  [(set (match_operand:HI 0 "register_operand" "=&d")
        (zero_extend:HI (match_operand:QI 1 "memory_operand" "T")))]
  "!TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (const_int 0))
   (set (strict_low_part (match_dup 2)) (match_dup 1))]
  "operands[2] = gen_lowpart (QImode, operands[0]);")

;
; fixuns_trunc(dd|td|sf|df|tf)(si|di)2 expander
;

; This is the only entry point for fixuns_trunc.  It multiplexes the
; expansion to either the *_emu expanders below for pre z196 machines
; or emits the default pattern otherwise.
(define_expand "fixuns_trunc<FP:mode><GPR:mode>2"
  [(parallel
    [(set (match_operand:GPR 0 "register_operand" "")
	  (unsigned_fix:GPR (match_operand:FP 1 "register_operand" "")))
     (unspec:GPR [(match_dup 2)] UNSPEC_ROUND)
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_HARD_FLOAT"
{
  if (!TARGET_Z196)
    {
      /* We don't provide emulation for TD|DD->SI.  */
      if (GET_MODE_CLASS (<FP:MODE>mode) == MODE_DECIMAL_FLOAT
	  && <GPR:MODE>mode == SImode)
	FAIL;
      emit_insn (gen_fixuns_trunc<FP:mode><GPR:mode>2_emu (operands[0],
							       operands[1]));
      DONE;
    }

  if (GET_MODE_CLASS (<FP:MODE>mode) == MODE_DECIMAL_FLOAT)
    operands[2] = GEN_INT (DFP_RND_TOWARD_0);
  else
    operands[2] = GEN_INT (BFP_RND_TOWARD_0);
})

; (sf|df|tf)->unsigned (si|di)

; Emulate the unsigned conversion with the signed version for pre z196
; machines.
(define_expand "fixuns_trunc<BFP:mode><GPR:mode>2_emu"
  [(parallel
    [(set (match_operand:GPR 0 "register_operand" "")
	  (unsigned_fix:GPR (match_operand:BFP 1 "register_operand" "")))
     (unspec:GPR [(const_int BFP_RND_TOWARD_0)] UNSPEC_ROUND)
     (clobber (reg:CC CC_REGNUM))])]
  "!TARGET_Z196 && TARGET_HARD_FLOAT"
{
  rtx_code_label *label1 = gen_label_rtx ();
  rtx_code_label *label2 = gen_label_rtx ();
  rtx temp = gen_reg_rtx (<BFP:MODE>mode);
  REAL_VALUE_TYPE cmp, sub;

  operands[1] = force_reg (<BFP:MODE>mode, operands[1]);
  real_2expN (&cmp, <GPR:bitsize> - 1, <BFP:MODE>mode);
  real_2expN (&sub, <GPR:bitsize>, <BFP:MODE>mode);

  emit_cmp_and_jump_insns (operands[1],
			   const_double_from_real_value (cmp, <BFP:MODE>mode),
			   LT, NULL_RTX, VOIDmode, 0, label1);
  emit_insn (gen_sub<BFP:mode>3 (temp, operands[1],
	       const_double_from_real_value (sub, <BFP:MODE>mode)));
  emit_insn (gen_fix_trunc<BFP:mode><GPR:mode>2_bfp (operands[0], temp,
	       GEN_INT (BFP_RND_TOWARD_MINF)));
  emit_jump (label2);

  emit_label (label1);
  emit_insn (gen_fix_trunc<BFP:mode><GPR:mode>2_bfp (operands[0],
							 operands[1],
							 GEN_INT (BFP_RND_TOWARD_0)));
  emit_label (label2);
  DONE;
})

; dd->unsigned di

; Emulate the unsigned conversion with the signed version for pre z196
; machines.
(define_expand "fixuns_truncdddi2_emu"
  [(parallel
    [(set (match_operand:DI 0 "register_operand" "")
	  (unsigned_fix:DI (match_operand:DD 1 "register_operand" "")))
     (unspec:DI [(const_int DFP_RND_TOWARD_0)] UNSPEC_ROUND)
     (clobber (reg:CC CC_REGNUM))])]

  "!TARGET_Z196 && TARGET_HARD_DFP"
{
  rtx_code_label *label1 = gen_label_rtx ();
  rtx_code_label *label2 = gen_label_rtx ();
  rtx temp = gen_reg_rtx (TDmode);
  REAL_VALUE_TYPE cmp, sub;

  decimal_real_from_string (&cmp, "9223372036854775808.0");  /* 2^63 */
  decimal_real_from_string (&sub, "18446744073709551616.0"); /* 2^64 */

  /* 2^63 can't be represented as 64bit DFP number with full precision.  The
     solution is doing the check and the subtraction in TD mode and using a
     TD -> DI convert afterwards.  */
  emit_insn (gen_extendddtd2 (temp, operands[1]));
  temp = force_reg (TDmode, temp);
  emit_cmp_and_jump_insns (temp,
			   const_double_from_real_value (cmp, TDmode),
			   LT, NULL_RTX, VOIDmode, 0, label1);
  emit_insn (gen_subtd3 (temp, temp,
			 const_double_from_real_value (sub, TDmode)));
  emit_insn (gen_fix_trunctddi2_dfp (operands[0], temp,
				     GEN_INT (DFP_RND_TOWARD_MINF)));
  emit_jump (label2);

  emit_label (label1);
  emit_insn (gen_fix_truncdddi2_dfp (operands[0], operands[1],
				     GEN_INT (DFP_RND_TOWARD_0)));
  emit_label (label2);
  DONE;
})

; td->unsigned di

; Emulate the unsigned conversion with the signed version for pre z196
; machines.
(define_expand "fixuns_trunctddi2_emu"
  [(parallel
    [(set (match_operand:DI 0 "register_operand" "")
	  (unsigned_fix:DI (match_operand:TD 1 "register_operand" "")))
     (unspec:DI [(const_int DFP_RND_TOWARD_0)] UNSPEC_ROUND)
     (clobber (reg:CC CC_REGNUM))])]

  "!TARGET_Z196 && TARGET_HARD_DFP"
{
  rtx_code_label *label1 = gen_label_rtx ();
  rtx_code_label *label2 = gen_label_rtx ();
  rtx temp = gen_reg_rtx (TDmode);
  REAL_VALUE_TYPE cmp, sub;

  operands[1] = force_reg (TDmode, operands[1]);
  decimal_real_from_string (&cmp, "9223372036854775808.0");  /* 2^63 */
  decimal_real_from_string (&sub, "18446744073709551616.0"); /* 2^64 */

  emit_cmp_and_jump_insns (operands[1],
			   const_double_from_real_value (cmp, TDmode),
			   LT, NULL_RTX, VOIDmode, 0, label1);
  emit_insn (gen_subtd3 (temp, operands[1],
			 const_double_from_real_value (sub, TDmode)));
  emit_insn (gen_fix_trunctddi2_dfp (operands[0], temp,
				     GEN_INT (DFP_RND_TOWARD_MINF)));
  emit_jump (label2);

  emit_label (label1);
  emit_insn (gen_fix_trunctddi2_dfp (operands[0], operands[1],
				     GEN_INT (DFP_RND_TOWARD_0)));
  emit_label (label2);
  DONE;
})

; Just a dummy to make the code in the first expander a bit easier.
(define_expand "fixuns_trunc<mode>si2_emu"
  [(parallel
    [(set (match_operand:SI 0 "register_operand" "")
	  (unsigned_fix:SI (match_operand:DFP 1 "register_operand" "")))
     (unspec:DI [(const_int DFP_RND_TOWARD_0)] UNSPEC_ROUND)
     (clobber (reg:CC CC_REGNUM))])]

  "!TARGET_Z196 && TARGET_HARD_DFP"
 {
   FAIL;
 })


; fixuns_trunc(tf|df|sf|td|dd)(di|si)2 instruction patterns.

; df -> unsigned di, vxe2: sf -> unsigned si
; clgdbr, clfebr, wclgdb, wclfeb
(define_insn "*fixuns_trunc<VX_CONV_BFP:mode><VX_CONV_INT:mode>2_z13"
  [(set (match_operand:VX_CONV_INT                           0 "register_operand" "=d,v")
	(unsigned_fix:VX_CONV_INT (match_operand:VX_CONV_BFP 1 "register_operand"  "f,v")))
   (unspec:DI [(match_operand:DI                             2 "immediate_operand" "K,K")] UNSPEC_ROUND)
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_VX && TARGET_HARD_FLOAT
   && GET_MODE_SIZE (<VX_CONV_INT:MODE>mode) == GET_MODE_SIZE (<VX_CONV_BFP:MODE>mode)"
  "@
   cl<VX_CONV_INT:gf><VX_CONV_BFP:xde>br\t%0,%h2,%1,0
   wcl<VX_CONV_INT:gf><VX_CONV_BFP:xde>b\t%v0,%v1,0,%h2"
  [(set_attr "op_type" "RRF,VRR")
   (set_attr "type"    "ftoi")])

; (dd|td|sf|df|tf)->unsigned (di|si)
; clfebr, clfdbr, clfxbr, clgebr, clgdbr, clgxbr
;         clfdtr, clfxtr,         clgdtr, clgxtr
(define_insn "*fixuns_trunc<FP:mode><GPR:mode>2_z196"
  [(set (match_operand:GPR                  0 "register_operand" "=d")
	(unsigned_fix:GPR (match_operand:FP 1 "register_operand"  "f")))
   (unspec:GPR [(match_operand:GPR          2 "immediate_operand" "K")] UNSPEC_ROUND)
   (clobber (reg:CC CC_REGNUM))]
   "TARGET_Z196 && TARGET_HARD_FLOAT
    && (!TARGET_VX || <GPR:MODE>mode != DImode || <FP:MODE>mode != DFmode)"
   "cl<GPR:gf><FP:xde><FP:bt>r\t%0,%h2,%1,0"
   [(set_attr "op_type" "RRF")
    (set_attr "type"    "ftoi")])

(define_expand "fix_trunc<DSF:mode><GPR:mode>2"
  [(set (match_operand:GPR 0 "register_operand" "")
        (fix:GPR (match_operand:DSF 1 "register_operand" "")))]
  "TARGET_HARD_FLOAT"
{
  emit_insn (gen_fix_trunc<DSF:mode><GPR:mode>2_bfp (operands[0], operands[1],
             GEN_INT (BFP_RND_TOWARD_0)));
  DONE;
})

; df -> signed di, vxe2: sf -> signed si
; cgdbr, cfebr, wcgdb, wcfeb
(define_insn "*fix_trunc<VX_CONV_BFP:mode><VX_CONV_INT:mode>2_bfp_z13"
  [(set (match_operand:VX_CONV_INT                  0 "register_operand" "=d,v")
        (fix:VX_CONV_INT (match_operand:VX_CONV_BFP 1 "register_operand"  "f,v")))
   (unspec:VX_CONV_INT [(match_operand:VX_CONV_INT  2 "immediate_operand" "K,K")] UNSPEC_ROUND)
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_VX && TARGET_HARD_FLOAT
   && GET_MODE_SIZE (<VX_CONV_INT:MODE>mode) == GET_MODE_SIZE (<VX_CONV_BFP:MODE>mode)"
  "@
   c<VX_CONV_INT:gf><VX_CONV_BFP:xde>br\t%0,%h2,%1
   wc<VX_CONV_INT:gf><VX_CONV_BFP:xde>b\t%v0,%v1,0,%h2"
  [(set_attr "op_type" "RRE,VRR")
   (set_attr "type"    "ftoi")])

; cgxbr, cgdbr, cgebr, cfxbr, cfdbr, cfebr
(define_insn "*fix_trunc<BFP:mode><GPR:mode>2_bfp"
  [(set (match_operand:GPR          0 "register_operand" "=d")
        (fix:GPR (match_operand:BFP 1 "register_operand"  "f")))
   (unspec:GPR [(match_operand:GPR  2 "immediate_operand" "K")] UNSPEC_ROUND)
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_FLOAT
    && (!TARGET_VX || <GPR:MODE>mode != DImode || <BFP:MODE>mode != DFmode)"
  "c<GPR:gf><BFP:xde>br\t%0,%h2,%1"
  [(set_attr "op_type" "RRE")
   (set_attr "type"    "ftoi")])

(define_expand "fix_trunc<BFP:mode><GPR:mode>2_bfp"
  [(parallel
    [(set (match_operand:GPR          0 "register_operand" "=d")
	  (fix:GPR (match_operand:BFP 1 "register_operand"  "f")))
     (unspec:GPR [(match_operand:GPR  2 "immediate_operand" "K")] UNSPEC_ROUND)
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_HARD_FLOAT")
;
; fix_trunc(td|dd)di2 instruction pattern(s).
;

(define_expand "fix_trunc<mode>di2"
  [(set (match_operand:DI 0 "register_operand" "")
        (fix:DI (match_operand:DFP 1 "nonimmediate_operand" "")))]
  "TARGET_ZARCH && TARGET_HARD_DFP"
{
  operands[1] = force_reg (<MODE>mode, operands[1]);
  emit_insn (gen_fix_trunc<mode>di2_dfp (operands[0], operands[1],
      GEN_INT (DFP_RND_TOWARD_0)));
  DONE;
})

; cgxtr, cgdtr
(define_insn "fix_trunc<DFP:mode>di2_dfp"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (fix:DI (match_operand:DFP 1 "register_operand" "f")))
   (unspec:DI [(match_operand:DI 2 "immediate_operand" "K")] UNSPEC_ROUND)
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && TARGET_HARD_DFP"
  "cg<DFP:xde>tr\t%0,%h2,%1"
  [(set_attr "op_type" "RRF")
   (set_attr "type"    "ftoidfp")])


;
; fix_trunctf(si|di)2 instruction pattern(s).
;

(define_expand "fix_trunctf<mode>2"
  [(parallel [(set (match_operand:GPR 0 "register_operand" "")
		   (fix:GPR (match_operand:TF 1 "register_operand" "")))
	      (unspec:GPR [(const_int BFP_RND_TOWARD_0)] UNSPEC_ROUND)
	      (clobber (reg:CC CC_REGNUM))])]
  "TARGET_HARD_FLOAT"
  "")


;
; float(si|di)(tf|df|sf|td|dd)2 instruction pattern(s).
;

; cxgbr, cdgbr, cegbr, cxgtr, cdgtr
(define_insn "floatdi<mode>2"
  [(set (match_operand:FP           0 "register_operand" "=f,v")
        (float:FP (match_operand:DI 1 "register_operand"  "d,v")))]
  "TARGET_ZARCH && TARGET_HARD_FLOAT"
  "@
   c<xde>g<bt>r\t%0,%1
   wcdgb\t%v0,%v1,0,0"
  [(set_attr "op_type"      "RRE,VRR")
   (set_attr "type"         "itof<mode>" )
   (set_attr "cpu_facility" "*,vx")
   (set_attr "enabled"      "*,<DFDI>")])

; cxfbr, cdfbr, cefbr, wcefb
(define_insn "floatsi<mode>2"
  [(set (match_operand:BFP           0 "register_operand" "=f,v")
        (float:BFP (match_operand:SI 1 "register_operand"  "d,v")))]
  "TARGET_HARD_FLOAT"
  "@
   c<xde>fbr\t%0,%1
   wcefb\t%v0,%v1,0,0"
  [(set_attr "op_type"      "RRE,VRR")
   (set_attr "type"         "itof<mode>" )
   (set_attr "cpu_facility" "*,vxe2")
   (set_attr "enabled"      "*,<SFSI>")])

; cxftr, cdftr
(define_insn "floatsi<mode>2"
  [(set (match_operand:DFP 0 "register_operand" "=f")
        (float:DFP (match_operand:SI 1 "register_operand" "d")))]
  "TARGET_Z196 && TARGET_HARD_FLOAT"
  "c<xde>ftr\t%0,0,%1,0"
  [(set_attr "op_type" "RRE")
   (set_attr "type"   "itof<mode>" )])

;
; floatuns(si|di)(tf|df|sf|td|dd)2 instruction pattern(s).
;

(define_insn "*floatuns<VX_CONV_INT:mode><VX_CONV_BFP:mode>2_z13"
  [(set (match_operand:VX_CONV_BFP                             0 "register_operand" "=f,v")
        (unsigned_float:VX_CONV_BFP (match_operand:VX_CONV_INT 1 "register_operand"  "d,v")))]
  "TARGET_VX && TARGET_HARD_FLOAT
   && GET_MODE_SIZE (<VX_CONV_INT:MODE>mode) == GET_MODE_SIZE (<VX_CONV_BFP:MODE>mode)"
  "@
   c<VX_CONV_BFP:xde>l<VX_CONV_INT:gf>br\t%0,0,%1,0
   wc<VX_CONV_BFP:xde>l<VX_CONV_INT:gf>b\t%v0,%v1,0,0"
  [(set_attr "op_type" "RRE,VRR")
   (set_attr "type"    "itofdf")])

; cxlgbr, cdlgbr, celgbr, cxlgtr, cdlgtr
; cxlfbr, cdlfbr, celfbr, cxlftr, cdlftr
(define_insn "*floatuns<GPR:mode><FP:mode>2"
  [(set (match_operand:FP                     0 "register_operand" "=f")
        (unsigned_float:FP (match_operand:GPR 1 "register_operand"  "d")))]
  "TARGET_Z196 && TARGET_HARD_FLOAT
   && (!TARGET_VX || <FP:MODE>mode != DFmode || <GPR:MODE>mode != DImode)"
  "c<FP:xde>l<GPR:gf><FP:bt>r\t%0,0,%1,0"
  [(set_attr "op_type" "RRE")
   (set_attr "type"    "itof<FP:mode>")])

(define_expand "floatuns<GPR:mode><FP:mode>2"
  [(set (match_operand:FP                     0 "register_operand" "")
        (unsigned_float:FP (match_operand:GPR 1 "register_operand" "")))]
  "TARGET_Z196 && TARGET_HARD_FLOAT")

;
; truncdfsf2 instruction pattern(s).
;

(define_insn "truncdfsf2"
  [(set (match_operand:SF                    0 "register_operand" "=f,v")
        (float_truncate:SF (match_operand:DF 1 "register_operand"  "f,v")))]
  "TARGET_HARD_FLOAT"
  "@
   ledbr\t%0,%1
   wledb\t%v0,%v1,0,0" ; IEEE inexact exception not suppressed
                       ; According to BFP rounding mode
  [(set_attr "op_type"      "RRE,VRR")
   (set_attr "type"         "ftruncdf")
   (set_attr "cpu_facility" "*,vx")])

;
; trunctf(df|sf)2 instruction pattern(s).
;

; ldxbr, lexbr
(define_insn "trunctf<mode>2"
  [(set (match_operand:DSF 0 "register_operand" "=f")
        (float_truncate:DSF (match_operand:TF 1 "register_operand" "f")))
   (clobber (match_scratch:TF 2 "=f"))]
  "TARGET_HARD_FLOAT"
  "l<xde>xbr\t%2,%1\;l<xde>r\t%0,%2"
  [(set_attr "length" "6")
   (set_attr "type"   "ftrunctf")])

;
; trunctddd2 and truncddsd2 instruction pattern(s).
;


(define_expand "trunctddd2"
  [(parallel
    [(set (match_operand:DD 0 "register_operand" "")
	  (float_truncate:DD (match_operand:TD 1 "register_operand" "")))
     (unspec:DI [(const_int DFP_RND_CURRENT)] UNSPEC_ROUND)
     (clobber (scratch:TD))])]
  "TARGET_HARD_DFP")

(define_insn "*trunctddd2"
  [(set (match_operand:DD 0 "register_operand" "=f")
	(float_truncate:DD (match_operand:TD 1 "register_operand" "f")))
   (unspec:DI [(match_operand:DI 2 "const_mask_operand" "I")] UNSPEC_ROUND)
   (clobber (match_scratch:TD 3 "=f"))]
  "TARGET_HARD_DFP"
  "ldxtr\t%3,%2,%1,0\;ldr\t%0,%3"
  [(set_attr "length"  "6")
   (set_attr "type"    "ftruncdd")])

(define_insn "truncddsd2"
  [(set (match_operand:SD 0 "register_operand" "=f")
	(float_truncate:SD (match_operand:DD 1 "register_operand" "f")))]
  "TARGET_HARD_DFP"
  "ledtr\t%0,0,%1,0"
  [(set_attr "op_type" "RRF")
   (set_attr "type"    "ftruncsd")])

(define_expand "trunctdsd2"
  [(parallel
    [(set (match_dup 2)
	  (float_truncate:DD (match_operand:TD 1 "register_operand" "")))
     (unspec:DI [(const_int DFP_RND_PREP_FOR_SHORT_PREC)] UNSPEC_ROUND)
     (clobber (match_scratch:TD 3 ""))])
   (set (match_operand:SD 0 "register_operand" "")
	(float_truncate:SD (match_dup 2)))]
  "TARGET_HARD_DFP"
{
  operands[2] = gen_reg_rtx (DDmode);
})

;
; extend(sf|df)(df|tf)2 instruction pattern(s).
;

; wflls
(define_insn "*extendsfdf2_z13"
  [(set (match_operand:DF                  0 "register_operand"     "=f,f,v")
        (float_extend:DF (match_operand:SF 1 "nonimmediate_operand"  "f,R,v")))]
  "TARGET_VX && TARGET_HARD_FLOAT"
  "@
   ldebr\t%0,%1
   ldeb\t%0,%1
   wldeb\t%v0,%v1"
  [(set_attr "op_type" "RRE,RXE,VRR")
   (set_attr "type"    "fsimpdf, floaddf,fsimpdf")])

; ldebr, ldeb, lxdbr, lxdb, lxebr, lxeb
(define_insn "*extend<DSF:mode><BFP:mode>2"
  [(set (match_operand:BFP                   0 "register_operand"     "=f,f")
        (float_extend:BFP (match_operand:DSF 1 "nonimmediate_operand"  "f,R")))]
  "TARGET_HARD_FLOAT
   && GET_MODE_SIZE (<BFP:MODE>mode) > GET_MODE_SIZE (<DSF:MODE>mode)
   && (!TARGET_VX || <BFP:MODE>mode != DFmode || <DSF:MODE>mode != SFmode)"
  "@
   l<BFP:xde><DSF:xde>br\t%0,%1
   l<BFP:xde><DSF:xde>b\t%0,%1"
  [(set_attr "op_type" "RRE,RXE")
   (set_attr "type"    "fsimp<BFP:mode>, fload<BFP:mode>")])

(define_expand "extend<DSF:mode><BFP:mode>2"
  [(set (match_operand:BFP                   0 "register_operand"     "")
        (float_extend:BFP (match_operand:DSF 1 "nonimmediate_operand" "")))]
  "TARGET_HARD_FLOAT
   && GET_MODE_SIZE (<BFP:MODE>mode) > GET_MODE_SIZE (<DSF:MODE>mode)")

;
; extendddtd2 and extendsddd2 instruction pattern(s).
;

(define_insn "extendddtd2"
  [(set (match_operand:TD 0 "register_operand" "=f")
	(float_extend:TD (match_operand:DD 1 "register_operand" "f")))]
  "TARGET_HARD_DFP"
  "lxdtr\t%0,%1,0"
  [(set_attr "op_type" "RRF")
   (set_attr "type"    "fsimptf")])

(define_insn "extendsddd2"
  [(set (match_operand:DD 0 "register_operand" "=f")
	(float_extend:DD (match_operand:SD 1 "register_operand" "f")))]
  "TARGET_HARD_DFP"
  "ldetr\t%0,%1,0"
  [(set_attr "op_type" "RRF")
   (set_attr "type"    "fsimptf")])

(define_expand "extendsdtd2"
  [(set (match_dup 2)
	(float_extend:DD (match_operand:SD 1 "register_operand" "")))
   (set (match_operand:TD 0 "register_operand" "")
	(float_extend:TD (match_dup 2)))]
  "TARGET_HARD_DFP"
{
  operands[2] = gen_reg_rtx (DDmode);
})

; Binary Floating Point - load fp integer

; Expanders for: floor, btrunc, round, ceil, and nearbyint
; For all of them the inexact exceptions are suppressed.

; fiebra, fidbra, fixbra
(define_insn "<FPINT:fpint_name><BFP:mode>2"
  [(set (match_operand:BFP 0 "register_operand" "=f")
	(unspec:BFP [(match_operand:BFP 1 "register_operand" "f")]
		    FPINT))]
  "TARGET_Z196"
  "fi<BFP:xde>bra\t%0,<FPINT:fpint_roundingmode>,%1,4"
  [(set_attr "op_type"   "RRF")
   (set_attr "type"      "fsimp<BFP:mode>")])

; rint is supposed to raise an inexact exception so we can use the
; older instructions.

; fiebr, fidbr, fixbr
(define_insn "rint<BFP:mode>2"
  [(set (match_operand:BFP 0 "register_operand" "=f")
	(unspec:BFP [(match_operand:BFP 1 "register_operand" "f")]
		    UNSPEC_FPINT_RINT))]
  ""
  "fi<BFP:xde>br\t%0,0,%1"
  [(set_attr "op_type"   "RRF")
   (set_attr "type"      "fsimp<BFP:mode>")])


; Decimal Floating Point - load fp integer

; fidtr, fixtr
(define_insn "<FPINT:fpint_name><DFP:mode>2"
  [(set (match_operand:DFP 0 "register_operand" "=f")
	(unspec:DFP [(match_operand:DFP 1 "register_operand" "f")]
		    FPINT))]
  "TARGET_HARD_DFP"
  "fi<DFP:xde>tr\t%0,<FPINT:fpint_roundingmode>,%1,4"
  [(set_attr "op_type"   "RRF")
   (set_attr "type"      "fsimp<DFP:mode>")])

; fidtr, fixtr
(define_insn "rint<DFP:mode>2"
  [(set (match_operand:DFP 0 "register_operand" "=f")
	(unspec:DFP [(match_operand:DFP 1 "register_operand" "f")]
		    UNSPEC_FPINT_RINT))]
  "TARGET_HARD_DFP"
  "fi<DFP:xde>tr\t%0,0,%1,0"
  [(set_attr "op_type"   "RRF")
   (set_attr "type"      "fsimp<DFP:mode>")])

;
; Binary <-> Decimal floating point trunc patterns
;

(define_insn "*trunc<BFP:mode><DFP_ALL:mode>2"
  [(set (reg:DFP_ALL FPR0_REGNUM)
        (float_truncate:DFP_ALL (reg:BFP FPR4_REGNUM)))
   (use (reg:SI GPR0_REGNUM))
   (clobber (reg:CC CC_REGNUM))
   (clobber (reg:SI GPR1_REGNUM))]
  "TARGET_HARD_DFP"
  "pfpo")

(define_insn "*trunc<DFP_ALL:mode><BFP:mode>2"
  [(set (reg:BFP FPR0_REGNUM)
        (float_truncate:BFP (reg:DFP_ALL FPR4_REGNUM)))
   (use (reg:SI GPR0_REGNUM))
   (clobber (reg:CC CC_REGNUM))
   (clobber (reg:SI GPR1_REGNUM))]
  "TARGET_HARD_DFP"
  "pfpo")

(define_expand "trunc<BFP:mode><DFP_ALL:mode>2"
  [(set (reg:BFP FPR4_REGNUM) (match_operand:BFP 1 "nonimmediate_operand" ""))
   (set (reg:SI GPR0_REGNUM) (match_dup 2))
   (parallel
    [(set (reg:DFP_ALL FPR0_REGNUM)
          (float_truncate:DFP_ALL (reg:BFP FPR4_REGNUM)))
     (use (reg:SI GPR0_REGNUM))
     (clobber (reg:CC CC_REGNUM))
     (clobber (reg:SI GPR1_REGNUM))])
   (set (match_operand:DFP_ALL 0 "nonimmediate_operand" "")
        (reg:DFP_ALL FPR0_REGNUM))]
  "TARGET_HARD_DFP
   && GET_MODE_SIZE (<BFP:MODE>mode) > GET_MODE_SIZE (<DFP_ALL:MODE>mode)"
{
  HOST_WIDE_INT flags;

  /* According to IEEE 754 2008 4.3 'Rounding-direction attributes' the
     rounding mode of the target format needs to be used.  */

  flags = (PFPO_CONVERT |
           PFPO_OP_TYPE_<DFP_ALL:MODE> << PFPO_OP0_TYPE_SHIFT |
           PFPO_OP_TYPE_<BFP:MODE> << PFPO_OP1_TYPE_SHIFT |
	   PFPO_RND_MODE_DFP);

  operands[2] = GEN_INT (flags);
})

(define_expand "trunc<DFP_ALL:mode><BFP:mode>2"
  [(set (reg:DFP_ALL FPR4_REGNUM)
        (match_operand:DFP_ALL 1 "nonimmediate_operand" ""))
   (set (reg:SI GPR0_REGNUM) (match_dup 2))
   (parallel
    [(set (reg:BFP FPR0_REGNUM) (float_truncate:BFP (reg:DFP_ALL FPR4_REGNUM)))
     (use (reg:SI GPR0_REGNUM))
     (clobber (reg:CC CC_REGNUM))
     (clobber (reg:SI GPR1_REGNUM))])
   (set (match_operand:BFP 0 "nonimmediate_operand" "") (reg:BFP FPR0_REGNUM))]
  "TARGET_HARD_DFP
   && GET_MODE_SIZE (<DFP_ALL:MODE>mode) >= GET_MODE_SIZE (<BFP:MODE>mode)"
{
  HOST_WIDE_INT flags;

  /* According to IEEE 754 2008 4.3 'Rounding-direction attributes' the
     rounding mode of the target format needs to be used.  */

  flags = (PFPO_CONVERT |
           PFPO_OP_TYPE_<BFP:MODE> << PFPO_OP0_TYPE_SHIFT |
           PFPO_OP_TYPE_<DFP_ALL:MODE> << PFPO_OP1_TYPE_SHIFT |
	   PFPO_RND_MODE_BFP);

  operands[2] = GEN_INT (flags);
})

;
; Binary <-> Decimal floating point extend patterns
;

(define_insn "*extend<BFP:mode><DFP_ALL:mode>2"
  [(set (reg:DFP_ALL FPR0_REGNUM) (float_extend:DFP_ALL (reg:BFP FPR4_REGNUM)))
   (use (reg:SI GPR0_REGNUM))
   (clobber (reg:CC CC_REGNUM))
   (clobber (reg:SI GPR1_REGNUM))]
  "TARGET_HARD_DFP"
  "pfpo")

(define_insn "*extend<DFP_ALL:mode><BFP:mode>2"
  [(set (reg:BFP FPR0_REGNUM) (float_extend:BFP (reg:DFP_ALL FPR4_REGNUM)))
   (use (reg:SI GPR0_REGNUM))
   (clobber (reg:CC CC_REGNUM))
   (clobber (reg:SI GPR1_REGNUM))]
  "TARGET_HARD_DFP"
  "pfpo")

(define_expand "extend<BFP:mode><DFP_ALL:mode>2"
  [(set (reg:BFP FPR4_REGNUM) (match_operand:BFP 1 "nonimmediate_operand" ""))
   (set (reg:SI GPR0_REGNUM) (match_dup 2))
   (parallel
    [(set (reg:DFP_ALL FPR0_REGNUM)
          (float_extend:DFP_ALL (reg:BFP FPR4_REGNUM)))
     (use (reg:SI GPR0_REGNUM))
     (clobber (reg:CC CC_REGNUM))
     (clobber (reg:SI GPR1_REGNUM))])
   (set (match_operand:DFP_ALL 0 "nonimmediate_operand" "")
        (reg:DFP_ALL FPR0_REGNUM))]
  "TARGET_HARD_DFP
   && GET_MODE_SIZE (<BFP:MODE>mode) <= GET_MODE_SIZE (<DFP_ALL:MODE>mode)"
{
  HOST_WIDE_INT flags;

  /* According to IEEE 754 2008 4.3 'Rounding-direction attributes' the
     rounding mode of the target format needs to be used.  */

  flags = (PFPO_CONVERT |
           PFPO_OP_TYPE_<DFP_ALL:MODE> << PFPO_OP0_TYPE_SHIFT |
           PFPO_OP_TYPE_<BFP:MODE> << PFPO_OP1_TYPE_SHIFT |
	   PFPO_RND_MODE_DFP);

  operands[2] = GEN_INT (flags);
})

(define_expand "extend<DFP_ALL:mode><BFP:mode>2"
  [(set (reg:DFP_ALL FPR4_REGNUM)
        (match_operand:DFP_ALL 1 "nonimmediate_operand" ""))
   (set (reg:SI GPR0_REGNUM) (match_dup 2))
   (parallel
    [(set (reg:BFP FPR0_REGNUM) (float_extend:BFP (reg:DFP_ALL FPR4_REGNUM)))
     (use (reg:SI GPR0_REGNUM))
     (clobber (reg:CC CC_REGNUM))
     (clobber (reg:SI GPR1_REGNUM))])
   (set (match_operand:BFP 0 "nonimmediate_operand" "") (reg:BFP FPR0_REGNUM))]
  "TARGET_HARD_DFP
   && GET_MODE_SIZE (<DFP_ALL:MODE>mode) < GET_MODE_SIZE (<BFP:MODE>mode)"
{
  HOST_WIDE_INT flags;

  /* According to IEEE 754 2008 4.3 'Rounding-direction attributes' the
     rounding mode of the target format needs to be used.  */

  flags = (PFPO_CONVERT |
           PFPO_OP_TYPE_<BFP:MODE> << PFPO_OP0_TYPE_SHIFT |
           PFPO_OP_TYPE_<DFP_ALL:MODE> << PFPO_OP1_TYPE_SHIFT |
	   PFPO_RND_MODE_BFP);

  operands[2] = GEN_INT (flags);
})


;;
;; ARITHMETIC OPERATIONS
;;
;  arithmetic operations set the ConditionCode,
;  because of unpredictable Bits in Register for Halfword and Byte
;  the ConditionCode can be set wrong in operations for Halfword and Byte

;;
;;- Add instructions.
;;

;
; addti3 instruction pattern(s).
;

(define_expand "addti3"
  [(parallel
    [(set (match_operand:TI          0 "register_operand"     "")
	  (plus:TI (match_operand:TI 1 "nonimmediate_operand" "")
		   (match_operand:TI 2 "general_operand"      "") ) )
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_ZARCH"
{
  /* For z13 we have vaq which doesn't set CC.  */
  if (TARGET_VX)
    {
      emit_insn (gen_rtx_SET (operands[0],
			      gen_rtx_PLUS (TImode,
                                            copy_to_mode_reg (TImode, operands[1]),
                                            copy_to_mode_reg (TImode, operands[2]))));
      DONE;
    }
})

(define_insn_and_split "*addti3"
  [(set (match_operand:TI          0 "register_operand"    "=&d")
        (plus:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
                 (match_operand:TI 2 "general_operand"      "do") ) )
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (reg:CCL1 CC_REGNUM)
          (compare:CCL1 (plus:DI (match_dup 7) (match_dup 8))
                        (match_dup 7)))
     (set (match_dup 6) (plus:DI (match_dup 7) (match_dup 8)))])
   (parallel
    [(set (match_dup 3) (plus:DI
                          (plus:DI (ltu:DI (reg:CCL1 CC_REGNUM) (const_int 0))
                                   (match_dup 4)) (match_dup 5)))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[3] = operand_subword (operands[0], 0, 0, TImode);
   operands[4] = operand_subword (operands[1], 0, 0, TImode);
   operands[5] = operand_subword (operands[2], 0, 0, TImode);
   operands[6] = operand_subword (operands[0], 1, 0, TImode);
   operands[7] = operand_subword (operands[1], 1, 0, TImode);
   operands[8] = operand_subword (operands[2], 1, 0, TImode);"
  [(set_attr "op_type"  "*")
   (set_attr "cpu_facility" "*")])

;
; adddi3 instruction pattern(s).
;

(define_expand "adddi3"
  [(parallel
    [(set (match_operand:DI 0 "nonimmediate_operand" "")
          (plus:DI (match_operand:DI 1 "nonimmediate_operand" "")
                   (match_operand:DI 2 "general_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")

(define_insn "*adddi3_sign"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (plus:DI (sign_extend:DI (match_operand:SI 2 "general_operand" "d,T"))
                 (match_operand:DI 1 "register_operand" "0,0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "@
   agfr\t%0,%2
   agf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z196prop" "z196_cracked,z196_cracked")])

(define_insn "*adddi3_zero_cc"
  [(set (reg CC_REGNUM)
        (compare (plus:DI (zero_extend:DI (match_operand:SI 2 "general_operand" "d,T"))
                          (match_operand:DI 1 "register_operand" "0,0"))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d,d")
        (plus:DI (zero_extend:DI (match_dup 2)) (match_dup 1)))]
  "s390_match_ccmode (insn, CCLmode) && TARGET_ZARCH"
  "@
   algfr\t%0,%2
   algf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_insn "*adddi3_zero_cconly"
  [(set (reg CC_REGNUM)
        (compare (plus:DI (zero_extend:DI (match_operand:SI 2 "general_operand" "d,T"))
                          (match_operand:DI 1 "register_operand" "0,0"))
                 (const_int 0)))
   (clobber (match_scratch:DI 0 "=d,d"))]
  "s390_match_ccmode (insn, CCLmode) && TARGET_ZARCH"
  "@
   algfr\t%0,%2
   algf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_insn "*adddi3_zero"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (plus:DI (zero_extend:DI (match_operand:SI 2 "general_operand" "d,T"))
                 (match_operand:DI 1 "register_operand" "0,0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "@
   algfr\t%0,%2
   algf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1")])

(define_insn_and_split "*adddi3_31z"
  [(set (match_operand:DI 0 "nonimmediate_operand" "=&d")
        (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0")
                 (match_operand:DI 2 "general_operand" "do") ) )
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (reg:CCL1 CC_REGNUM)
          (compare:CCL1 (plus:SI (match_dup 7) (match_dup 8))
                        (match_dup 7)))
     (set (match_dup 6) (plus:SI (match_dup 7) (match_dup 8)))])
   (parallel
    [(set (match_dup 3) (plus:SI
			  (plus:SI (ltu:SI (reg:CCL1 CC_REGNUM) (const_int 0))
				   (match_dup 4)) (match_dup 5)))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[3] = operand_subword (operands[0], 0, 0, DImode);
   operands[4] = operand_subword (operands[1], 0, 0, DImode);
   operands[5] = operand_subword (operands[2], 0, 0, DImode);
   operands[6] = operand_subword (operands[0], 1, 0, DImode);
   operands[7] = operand_subword (operands[1], 1, 0, DImode);
   operands[8] = operand_subword (operands[2], 1, 0, DImode);")

;
; addsi3 instruction pattern(s).
;

(define_expand "addsi3"
  [(parallel
    [(set (match_operand:SI 0 "nonimmediate_operand" "")
          (plus:SI (match_operand:SI 1 "nonimmediate_operand" "")
                   (match_operand:SI 2 "general_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")

(define_insn "*addsi3_sign"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (plus:SI (sign_extend:SI (match_operand:HI 2 "memory_operand" "R,T"))
                 (match_operand:SI 1 "register_operand" "0,0")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   ah\t%0,%2
   ahy\t%0,%2"
  [(set_attr "op_type"  "RX,RXY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z196prop" "z196_cracked,z196_cracked")])

;
; add(di|si)3 instruction pattern(s).
;

; ark, agrk, ar, ahi, ahik, aghik, alfi, slfi, a, ay, agr, aghi, algfi, slgfi, ag, asi, agsi
(define_insn "*add<mode>3"
  [(set (match_operand:GPR 0 "nonimmediate_operand"           "=d,d,d,d, d, d,d,d,S")
        (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d,0,d, 0, 0,0,0,0")
		  (match_operand:GPR 2 "general_operand"      " d,d,K,K,Op,On,R,T,C") ) )
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   a<g>r\t%0,%2
   a<g>rk\t%0,%1,%2
   a<g>hi\t%0,%h2
   a<g>hik\t%0,%1,%h2
   al<g>fi\t%0,%2
   sl<g>fi\t%0,%n2
   a<g>\t%0,%2
   a<y>\t%0,%2
   a<g>si\t%0,%c2"
  [(set_attr "op_type"  "RR<E>,RRF,RI,RIE,RIL,RIL,RX<Y>,RXY,SIY")
   (set_attr "cpu_facility" "*,z196,*,z196,extimm,extimm,*,longdisp,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,*,z10_super_E1,z10_super_E1,
                        z10_super_E1,z10_super_E1,z10_super_E1")])

; alr, alfi, slfi, al, aly, alrk, alhsik, algr, algfi, slgfi, alg, alsi, algsi, algrk, alghsik
(define_insn "*add<mode>3_carry1_cc"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d, 0, 0,d,0,0,0")
			   (match_operand:GPR 2 "general_operand"      " d,d,Op,On,K,R,T,C"))
                 (match_dup 1)))
   (set (match_operand:GPR 0 "nonimmediate_operand"                    "=d,d, d, d,d,d,d,d")
        (plus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCL1mode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>fi\t%0,%2
   sl<g>fi\t%0,%n2
   al<g>hsik\t%0,%1,%h2
   al<g>\t%0,%2
   al<y>\t%0,%2
   al<g>si\t%0,%c2"
  [(set_attr "op_type"      "RR<E>,RRF,RIL,RIL,RIE,RX<Y>,RXY,SIY")
   (set_attr "cpu_facility" "*,z196,extimm,extimm,z196,*,longdisp,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1,z10_super_E1")])

; alr, al, aly, algr, alg, alrk, algrk
(define_insn "*add<mode>3_carry1_cconly"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d,0,0")
			   (match_operand:GPR 2 "general_operand"       "d,d,R,T"))
                 (match_dup 1)))
   (clobber (match_scratch:GPR 0                                       "=d,d,d,d"))]
  "s390_match_ccmode (insn, CCL1mode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>\t%0,%2
   al<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

; alr, alfi, slfi, al, aly, algr, algfi, slgfi, alg, alsi, algsi, alrk, algrk, alhsik, alghsik
(define_insn "*add<mode>3_carry2_cc"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d, 0, 0,d,0,0,0")
			   (match_operand:GPR 2 "general_operand"      " d,d,Op,On,K,R,T,C"))
                 (match_dup 2)))
   (set (match_operand:GPR 0 "nonimmediate_operand"                    "=d,d, d, d,d,d,d,S")
        (plus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCL1mode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>fi\t%0,%2
   sl<g>fi\t%0,%n2
   al<g>hsik\t%0,%1,%h2
   al<g>\t%0,%2
   al<y>\t%0,%2
   al<g>si\t%0,%c2"
  [(set_attr "op_type"  "RR<E>,RRF,RIL,RIL,RIE,RX<Y>,RXY,SIY")
   (set_attr "cpu_facility" "*,z196,extimm,extimm,z196,*,longdisp,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1,z10_super_E1")])

; alr, al, aly, algr, alg, alrk, algrk
(define_insn "*add<mode>3_carry2_cconly"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d,0,0")
			   (match_operand:GPR 2 "general_operand"       "d,d,R,T"))
                 (match_dup 2)))
   (clobber (match_scratch:GPR 0                                       "=d,d,d,d"))]
  "s390_match_ccmode (insn, CCL1mode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>\t%0,%2
   al<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

; alr, alfi, slfi, al, aly, algr, algfi, slgfi, alg, alsi, algsi, alrk, algrk, alhsik, alghsik
(define_insn "*add<mode>3_cc"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d, 0, 0,d,0,0,0")
			   (match_operand:GPR 2 "general_operand"      " d,d,Op,On,K,R,T,C"))
                 (const_int 0)))
   (set (match_operand:GPR 0 "nonimmediate_operand"                    "=d,d, d, d,d,d,d,S")
        (plus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCLmode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>fi\t%0,%2
   sl<g>fi\t%0,%n2
   al<g>hsik\t%0,%1,%h2
   al<g>\t%0,%2
   al<y>\t%0,%2
   al<g>si\t%0,%c2"
  [(set_attr "op_type"  "RR<E>,RRF,RIL,RIL,RIE,RX<Y>,RXY,SIY")
   (set_attr "cpu_facility" "*,z196,extimm,extimm,z196,*,longdisp,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1,
                        *,z10_super_E1,z10_super_E1,z10_super_E1")])

; alr, al, aly, algr, alg, alrk, algrk
(define_insn "*add<mode>3_cconly"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,d,0,0")
			   (match_operand:GPR 2 "general_operand"       "d,d,R,T"))
                 (const_int 0)))
   (clobber (match_scratch:GPR 0                                       "=d,d,d,d"))]
  "s390_match_ccmode (insn, CCLmode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>\t%0,%2
   al<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

; alr, al, aly, algr, alg, alrk, algrk
(define_insn "*add<mode>3_cconly2"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 1 "nonimmediate_operand"    "%0,d,0,0")
                 (neg:GPR (match_operand:GPR 2 "general_operand" "d,d,R,T"))))
   (clobber (match_scratch:GPR 0                                "=d,d,d,d"))]
  "s390_match_ccmode(insn, CCLmode)"
  "@
   al<g>r\t%0,%2
   al<g>rk\t%0,%1,%2
   al<g>\t%0,%2
   al<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

; ahi, afi, aghi, agfi, asi, agsi
(define_insn "*add<mode>3_imm_cc"
  [(set (reg CC_REGNUM)
        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" " 0, d,0, 0")
			   (match_operand:GPR 2 "const_int_operand"    " K, K,Os,C"))
                 (const_int 0)))
   (set (match_operand:GPR 0 "nonimmediate_operand"                    "=d, d,d, S")
        (plus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCAmode)
   && (CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[2]), 'K', \"K\")
       || (CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[2]), 'O', \"Os\")
           /* Avoid INT32_MIN on 32 bit.  */
           && (!TARGET_ZARCH || INTVAL (operands[2]) != -0x7fffffff - 1)))"
  "@
   a<g>hi\t%0,%h2
   a<g>hik\t%0,%1,%h2
   a<g>fi\t%0,%2
   a<g>si\t%0,%c2"
  [(set_attr "op_type"      "RI,RIE,RIL,SIY")
   (set_attr "cpu_facility" "*,z196,extimm,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

(define_insn "*adddi3_sign"
  [(set (match_operand:DI                          0 "register_operand" "=d")
        (plus:DI (sign_extend:DI (match_operand:HI 2 "memory_operand"    "T"))
		 (match_operand:DI                 1 "register_operand"  "0")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z14"
  "agh\t%0,%2"
  [(set_attr "op_type"  "RXY")])

;
; add(tf|df|sf|td|dd)3 instruction pattern(s).
;

; axbr, adbr, aebr, axb, adb, aeb, adtr, axtr
; FIXME: wfadb does not clobber cc
(define_insn "add<mode>3"
  [(set (match_operand:FP          0 "register_operand"     "=f,f,f,v,v")
        (plus:FP (match_operand:FP 1 "nonimmediate_operand" "%f,0,0,v,v")
		 (match_operand:FP 2 "general_operand"       "f,f,R,v,v")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_FLOAT"
  "@
   a<xde>tr\t%0,%1,%2
   a<xde>br\t%0,%2
   a<xde>b\t%0,%2
   wfadb\t%v0,%v1,%v2
   wfasb\t%v0,%v1,%v2"
  [(set_attr "op_type"      "RRF,RRE,RXE,VRR,VRR")
   (set_attr "type"         "fsimp<mode>")
   (set_attr "cpu_facility" "*,*,*,vx,vxe")
   (set_attr "enabled"      "<nBFP>,<nDFP>,<DSF>,<DF>,<SF>")])

; axbr, adbr, aebr, axb, adb, aeb, adtr, axtr
(define_insn "*add<mode>3_cc"
  [(set (reg CC_REGNUM)
	(compare (plus:FP (match_operand:FP 1 "nonimmediate_operand" "%f,0,0")
			  (match_operand:FP 2 "general_operand"       "f,f,R"))
		 (match_operand:FP 3 "const0_operand" "")))
   (set (match_operand:FP 0 "register_operand" "=f,f,f")
	(plus:FP (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "@
   a<xde>tr\t%0,%1,%2
   a<xde>br\t%0,%2
   a<xde>b\t%0,%2"
  [(set_attr "op_type"  "RRF,RRE,RXE")
   (set_attr "type"     "fsimp<mode>")
   (set_attr "enabled"  "<nBFP>,<nDFP>,<DSF>")])

; axbr, adbr, aebr, axb, adb, aeb, adtr, axtr
(define_insn "*add<mode>3_cconly"
  [(set (reg CC_REGNUM)
	(compare (plus:FP (match_operand:FP 1 "nonimmediate_operand" "%f,0,0")
			  (match_operand:FP 2 "general_operand"       "f,f,R"))
		 (match_operand:FP 3 "const0_operand" "")))
   (clobber (match_scratch:FP 0 "=f,f,f"))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "@
   a<xde>tr\t%0,%1,%2
   a<xde>br\t%0,%2
   a<xde>b\t%0,%2"
  [(set_attr "op_type"  "RRF,RRE,RXE")
   (set_attr "type"     "fsimp<mode>")
   (set_attr "enabled"  "<nBFP>,<nDFP>,<DSF>")])

;
; Pointer add instruction patterns
;

; This will match "*la_64"
(define_expand "addptrdi3"
  [(set (match_operand:DI 0 "register_operand" "")
        (plus:DI (match_operand:DI 1 "register_operand" "")
		 (match_operand:DI 2 "nonmemory_operand" "")))]
  "TARGET_64BIT"
{
  if (GET_CODE (operands[2]) == CONST_INT)
    {
      HOST_WIDE_INT c = INTVAL (operands[2]);

      if (!CONST_OK_FOR_CONSTRAINT_P (c, 'K', "K")
	  && !CONST_OK_FOR_CONSTRAINT_P (c, 'O', "Os"))
        {
	  operands[2] = force_const_mem (DImode, operands[2]);
	  operands[2] = force_reg (DImode, operands[2]);
        }
      else if (!DISP_IN_RANGE (INTVAL (operands[2])))
        operands[2] = force_reg (DImode, operands[2]);
    }
})

; For 31 bit we have to prevent the generated pattern from matching
; normal ADDs since la only does a 31 bit add.  This is supposed to
; match "force_la_31".
(define_expand "addptrsi3"
  [(parallel
    [(set (match_operand:SI 0 "register_operand" "")
	  (plus:SI (match_operand:SI 1 "register_operand" "")
		   (match_operand:SI 2 "nonmemory_operand" "")))
		   (use (const_int 0))])]
  "!TARGET_64BIT"
{
  if (GET_CODE (operands[2]) == CONST_INT)
    {
      HOST_WIDE_INT c = INTVAL (operands[2]);

      if (!CONST_OK_FOR_CONSTRAINT_P (c, 'K', "K")
	  && !CONST_OK_FOR_CONSTRAINT_P (c, 'O', "Os"))
        {
	  operands[2] = force_const_mem (SImode, operands[2]);
	  operands[2] = force_reg (SImode, operands[2]);
        }
      else if (!DISP_IN_RANGE (INTVAL (operands[2])))
        operands[2] = force_reg (SImode, operands[2]);
    }
})

;;
;;- Subtract instructions.
;;

;
; subti3 instruction pattern(s).
;

(define_expand "subti3"
  [(parallel
    [(set (match_operand:TI           0 "register_operand" "")
	  (minus:TI (match_operand:TI 1 "register_operand" "")
		    (match_operand:TI 2 "general_operand"  "") ) )
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_ZARCH"
{
  /* For z13 we have vsq which doesn't set CC.  */
  if (TARGET_VX)
    {
      emit_insn (gen_rtx_SET (operands[0],
			      gen_rtx_MINUS (TImode,
                                            operands[1],
                                            copy_to_mode_reg (TImode, operands[2]))));
      DONE;
    }
})

(define_insn_and_split "*subti3"
  [(set (match_operand:TI           0 "register_operand" "=&d")
        (minus:TI (match_operand:TI 1 "register_operand"   "0")
                  (match_operand:TI 2 "general_operand"   "do") ) )
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (reg:CCL2 CC_REGNUM)
          (compare:CCL2 (minus:DI (match_dup 7) (match_dup 8))
                        (match_dup 7)))
     (set (match_dup 6) (minus:DI (match_dup 7) (match_dup 8)))])
   (parallel
    [(set (match_dup 3) (minus:DI (minus:DI (match_dup 4) (match_dup 5))
                                  (gtu:DI (reg:CCL2 CC_REGNUM) (const_int 0))))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[3] = operand_subword (operands[0], 0, 0, TImode);
   operands[4] = operand_subword (operands[1], 0, 0, TImode);
   operands[5] = operand_subword (operands[2], 0, 0, TImode);
   operands[6] = operand_subword (operands[0], 1, 0, TImode);
   operands[7] = operand_subword (operands[1], 1, 0, TImode);
   operands[8] = operand_subword (operands[2], 1, 0, TImode);"
  [(set_attr "op_type"      "*")
   (set_attr "cpu_facility" "*")])

;
; subdi3 instruction pattern(s).
;

(define_expand "subdi3"
  [(parallel
    [(set (match_operand:DI 0 "register_operand" "")
          (minus:DI (match_operand:DI 1 "register_operand" "")
                    (match_operand:DI 2 "general_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")

(define_insn "*subdi3_sign"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (minus:DI (match_operand:DI 1 "register_operand" "0,0")
                  (sign_extend:DI (match_operand:SI 2 "general_operand" "d,T"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "@
   sgfr\t%0,%2
   sgf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_c,*")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*subdi3_zero_cc"
  [(set (reg CC_REGNUM)
        (compare (minus:DI (match_operand:DI 1 "register_operand" "0,0")
                           (zero_extend:DI (match_operand:SI 2 "general_operand" "d,T")))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d,d")
        (minus:DI (match_dup 1) (zero_extend:DI (match_dup 2))))]
  "s390_match_ccmode (insn, CCLmode) && TARGET_ZARCH"
  "@
   slgfr\t%0,%2
   slgf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_super_c_E1,z10_super_E1")])

(define_insn "*subdi3_zero_cconly"
  [(set (reg CC_REGNUM)
        (compare (minus:DI (match_operand:DI 1 "register_operand" "0,0")
                           (zero_extend:DI (match_operand:SI 2 "general_operand" "d,T")))
                 (const_int 0)))
   (clobber (match_scratch:DI 0 "=d,d"))]
  "s390_match_ccmode (insn, CCLmode) && TARGET_ZARCH"
  "@
   slgfr\t%0,%2
   slgf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_super_c_E1,z10_super_E1")])

(define_insn "*subdi3_zero"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (minus:DI (match_operand:DI 1 "register_operand" "0,0")
                  (zero_extend:DI (match_operand:SI 2 "general_operand" "d,T"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "@
   slgfr\t%0,%2
   slgf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_super_c_E1,z10_super_E1")])

(define_insn_and_split "*subdi3_31z"
  [(set (match_operand:DI 0 "register_operand" "=&d")
        (minus:DI (match_operand:DI 1 "register_operand" "0")
                  (match_operand:DI 2 "general_operand" "do") ) )
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH"
  "#"
  "&& reload_completed"
  [(parallel
    [(set (reg:CCL2 CC_REGNUM)
          (compare:CCL2 (minus:SI (match_dup 7) (match_dup 8))
                        (match_dup 7)))
     (set (match_dup 6) (minus:SI (match_dup 7) (match_dup 8)))])
   (parallel
    [(set (match_dup 3) (minus:SI (minus:SI (match_dup 4) (match_dup 5))
                                  (gtu:SI (reg:CCL2 CC_REGNUM) (const_int 0))))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[3] = operand_subword (operands[0], 0, 0, DImode);
   operands[4] = operand_subword (operands[1], 0, 0, DImode);
   operands[5] = operand_subword (operands[2], 0, 0, DImode);
   operands[6] = operand_subword (operands[0], 1, 0, DImode);
   operands[7] = operand_subword (operands[1], 1, 0, DImode);
   operands[8] = operand_subword (operands[2], 1, 0, DImode);")

;
; subsi3 instruction pattern(s).
;

(define_expand "subsi3"
  [(parallel
    [(set (match_operand:SI 0 "register_operand" "")
          (minus:SI (match_operand:SI 1 "register_operand" "")
                    (match_operand:SI 2 "general_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")

(define_insn "*subsi3_sign"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (minus:SI (match_operand:SI 1 "register_operand" "0,0")
                  (sign_extend:SI (match_operand:HI 2 "memory_operand" "R,T"))))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   sh\t%0,%2
   shy\t%0,%2"
  [(set_attr "op_type"  "RX,RXY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z196prop" "z196_cracked,z196_cracked")])

;
; sub(di|si)3 instruction pattern(s).
;

; sr, s, sy, sgr, sg, srk, sgrk
(define_insn "*sub<mode>3"
  [(set (match_operand:GPR 0 "register_operand"           "=d,d,d,d")
        (minus:GPR (match_operand:GPR 1 "register_operand" "0,d,0,0")
		   (match_operand:GPR 2 "general_operand"  "d,d,R,T") ) )
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   s<g>r\t%0,%2
   s<g>rk\t%0,%1,%2
   s<g>\t%0,%2
   s<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])

; slr, sl, sly, slgr, slg, slrk, slgrk
(define_insn "*sub<mode>3_borrow_cc"
  [(set (reg CC_REGNUM)
        (compare (minus:GPR (match_operand:GPR 1 "register_operand" "0,d,0,0")
			    (match_operand:GPR 2 "general_operand"  "d,d,R,T"))
                 (match_dup 1)))
   (set (match_operand:GPR 0 "register_operand"                    "=d,d,d,d")
        (minus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCL2mode)"
  "@
   sl<g>r\t%0,%2
   sl<g>rk\t%0,%1,%2
   sl<g>\t%0,%2
   sl<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])

; slr, sl, sly, slgr, slg, slrk, slgrk
(define_insn "*sub<mode>3_borrow_cconly"
  [(set (reg CC_REGNUM)
        (compare (minus:GPR (match_operand:GPR 1 "register_operand" "0,d,0,0")
			    (match_operand:GPR 2 "general_operand"  "d,d,R,T"))
                 (match_dup 1)))
   (clobber (match_scratch:GPR 0                                   "=d,d,d,d"))]
  "s390_match_ccmode (insn, CCL2mode)"
  "@
   sl<g>r\t%0,%2
   sl<g>rk\t%0,%1,%2
   sl<g>\t%0,%2
   sl<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])

; slr, sl, sly, slgr, slg, slrk, slgrk
(define_insn "*sub<mode>3_cc"
  [(set (reg CC_REGNUM)
        (compare (minus:GPR (match_operand:GPR 1 "register_operand" "0,d,0,0")
			    (match_operand:GPR 2 "general_operand"  "d,d,R,T"))
                 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand"                    "=d,d,d,d")
        (minus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCLmode)"
  "@
   sl<g>r\t%0,%2
   sl<g>rk\t%0,%1,%2
   sl<g>\t%0,%2
   sl<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])

; slr, sl, sly, slgr, slg, slrk, slgrk
(define_insn "*sub<mode>3_cc2"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 1 "register_operand" "0,d,0,0")
                 (match_operand:GPR 2 "general_operand"  "d,d,R,T")))
   (set (match_operand:GPR 0 "register_operand"         "=d,d,d,d")
        (minus:GPR (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCL3mode)"
  "@
   sl<g>r\t%0,%2
   sl<g>rk\t%0,%1,%2
   sl<g>\t%0,%2
   sl<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])

; slr, sl, sly, slgr, slg, slrk, slgrk
(define_insn "*sub<mode>3_cconly"
  [(set (reg CC_REGNUM)
        (compare (minus:GPR (match_operand:GPR 1 "register_operand" "0,d,0,0")
			    (match_operand:GPR 2 "general_operand"  "d,d,R,T"))
                 (const_int 0)))
   (clobber (match_scratch:GPR 0                                   "=d,d,d,d"))]
  "s390_match_ccmode (insn, CCLmode)"
  "@
   sl<g>r\t%0,%2
   sl<g>rk\t%0,%1,%2
   sl<g>\t%0,%2
   sl<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])


; slr, sl, sly, slgr, slg, slrk, slgrk
(define_insn "*sub<mode>3_cconly2"
  [(set (reg CC_REGNUM)
        (compare (match_operand:GPR 1 "register_operand" "0,d,0,0")
                 (match_operand:GPR 2 "general_operand"  "d,d,R,T")))
   (clobber (match_scratch:GPR 0                        "=d,d,d,d"))]
  "s390_match_ccmode (insn, CCL3mode)"
  "@
   sl<g>r\t%0,%2
   sl<g>rk\t%0,%1,%2
   sl<g>\t%0,%2
   sl<y>\t%0,%2"
  [(set_attr "op_type"  "RR<E>,RRF,RX<Y>,RXY")
   (set_attr "cpu_facility" "*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_c_E1,*,z10_super_E1,z10_super_E1")])

(define_insn "*subdi3_sign"
  [(set (match_operand:DI                           0 "register_operand" "=d")
        (minus:DI (match_operand:DI                 1 "register_operand"  "0")
                  (sign_extend:DI (match_operand:HI 2 "memory_operand"    "T"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z14"
  "sgh\t%0,%2"
  [(set_attr "op_type"  "RXY")])


;
; sub(tf|df|sf|td|dd)3 instruction pattern(s).
;

; FIXME: (clobber (match_scratch:CC 3 "=c,c,c,X,X")) does not work - why?
; sxbr, sdbr, sebr, sdb, seb, sxtr, sdtr
(define_insn "sub<mode>3"
  [(set (match_operand:FP           0 "register_operand" "=f,f,f,v,v")
        (minus:FP (match_operand:FP 1 "register_operand"  "f,0,0,v,v")
		  (match_operand:FP 2 "general_operand"   "f,f,R,v,v")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_FLOAT"
  "@
   s<xde>tr\t%0,%1,%2
   s<xde>br\t%0,%2
   s<xde>b\t%0,%2
   wfsdb\t%v0,%v1,%v2
   wfssb\t%v0,%v1,%v2"
  [(set_attr "op_type"      "RRF,RRE,RXE,VRR,VRR")
   (set_attr "type"         "fsimp<mode>")
   (set_attr "cpu_facility" "*,*,*,vx,vxe")
   (set_attr "enabled"      "<nBFP>,<nDFP>,<DSF>,<DF>,<SF>")])

; sxbr, sdbr, sebr, sdb, seb, sxtr, sdtr
(define_insn "*sub<mode>3_cc"
  [(set (reg CC_REGNUM)
	(compare (minus:FP (match_operand:FP 1 "nonimmediate_operand" "f,0,0")
			   (match_operand:FP 2 "general_operand"      "f,f,R"))
		 (match_operand:FP 3 "const0_operand" "")))
   (set (match_operand:FP 0 "register_operand" "=f,f,f")
	(minus:FP (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "@
   s<xde>tr\t%0,%1,%2
   s<xde>br\t%0,%2
   s<xde>b\t%0,%2"
  [(set_attr "op_type"  "RRF,RRE,RXE")
   (set_attr "type"     "fsimp<mode>")
   (set_attr "enabled"  "<nBFP>,<nDFP>,<DSF>")])

; sxbr, sdbr, sebr, sdb, seb, sxtr, sdtr
(define_insn "*sub<mode>3_cconly"
  [(set (reg CC_REGNUM)
	(compare (minus:FP (match_operand:FP 1 "nonimmediate_operand" "f,0,0")
			   (match_operand:FP 2 "general_operand"      "f,f,R"))
		 (match_operand:FP 3 "const0_operand" "")))
   (clobber (match_scratch:FP 0 "=f,f,f"))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "@
   s<xde>tr\t%0,%1,%2
   s<xde>br\t%0,%2
   s<xde>b\t%0,%2"
  [(set_attr "op_type"  "RRF,RRE,RXE")
   (set_attr "type"     "fsimp<mode>")
   (set_attr "enabled"  "<nBFP>,<nDFP>,<DSF>")])


;;
;;- Conditional add/subtract instructions.
;;

;
; add(di|si)cc instruction pattern(s).
;

; the following 4 patterns are used when the result of an add with
; carry is checked for an overflow condition

; op1 + op2 + c < op1

; alcr, alc, alcgr, alcg
(define_insn "*add<mode>3_alc_carry1_cc"
  [(set (reg CC_REGNUM)
        (compare
          (plus:GPR (plus:GPR (match_operand:GPR 3 "s390_alc_comparison" "")
                              (match_operand:GPR 1 "nonimmediate_operand" "%0,0"))
                    (match_operand:GPR 2 "general_operand" "d,T"))
          (match_dup 1)))
   (set (match_operand:GPR 0 "register_operand" "=d,d")
        (plus:GPR (plus:GPR (match_dup 3) (match_dup 1)) (match_dup 2)))]
  "s390_match_ccmode (insn, CCL1mode)"
  "@
   alc<g>r\t%0,%2
   alc<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z196prop" "z196_alone,z196_alone")])

; alcr, alc, alcgr, alcg
(define_insn "*add<mode>3_alc_carry1_cconly"
  [(set (reg CC_REGNUM)
        (compare
          (plus:GPR (plus:GPR (match_operand:GPR 3 "s390_alc_comparison" "")
                              (match_operand:GPR 1 "nonimmediate_operand" "%0,0"))
                    (match_operand:GPR 2 "general_operand" "d,T"))
          (match_dup 1)))
   (clobber (match_scratch:GPR 0 "=d,d"))]
  "s390_match_ccmode (insn, CCL1mode)"
  "@
   alc<g>r\t%0,%2
   alc<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z196prop" "z196_alone,z196_alone")])

; op1 + op2 + c < op2

; alcr, alc, alcgr, alcg
(define_insn "*add<mode>3_alc_carry2_cc"
  [(set (reg CC_REGNUM)
        (compare
          (plus:GPR (plus:GPR (match_operand:GPR 3 "s390_alc_comparison" "")
                              (match_operand:GPR 1 "nonimmediate_operand" "%0,0"))
                    (match_operand:GPR 2 "general_operand" "d,T"))
          (match_dup 2)))
   (set (match_operand:GPR 0 "register_operand" "=d,d")
        (plus:GPR (plus:GPR (match_dup 3) (match_dup 1)) (match_dup 2)))]
  "s390_match_ccmode (insn, CCL1mode)"
  "@
   alc<g>r\t%0,%2
   alc<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")])

; alcr, alc, alcgr, alcg
(define_insn "*add<mode>3_alc_carry2_cconly"
  [(set (reg CC_REGNUM)
        (compare
          (plus:GPR (plus:GPR (match_operand:GPR 3 "s390_alc_comparison" "")
                              (match_operand:GPR 1 "nonimmediate_operand" "%0,0"))
                    (match_operand:GPR 2 "general_operand" "d,T"))
          (match_dup 2)))
   (clobber (match_scratch:GPR 0 "=d,d"))]
  "s390_match_ccmode (insn, CCL1mode)"
  "@
   alc<g>r\t%0,%2
   alc<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")])

; alcr, alc, alcgr, alcg
(define_insn "*add<mode>3_alc_cc"
  [(set (reg CC_REGNUM)
        (compare
          (plus:GPR (plus:GPR (match_operand:GPR 3 "s390_alc_comparison" "")
                              (match_operand:GPR 1 "nonimmediate_operand" "%0,0"))
                    (match_operand:GPR 2 "general_operand" "d,T"))
          (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d,d")
        (plus:GPR (plus:GPR (match_dup 3) (match_dup 1)) (match_dup 2)))]
  "s390_match_ccmode (insn, CCLmode)"
  "@
   alc<g>r\t%0,%2
   alc<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")])

; alcr, alc, alcgr, alcg
(define_insn "*add<mode>3_alc"
  [(set (match_operand:GPR 0 "register_operand" "=d,d")
        (plus:GPR (plus:GPR (match_operand:GPR 3 "s390_alc_comparison" "")
                            (match_operand:GPR 1 "nonimmediate_operand" "%0,0"))
                  (match_operand:GPR 2 "general_operand" "d,T")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   alc<g>r\t%0,%2
   alc<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")])

; slbr, slb, slbgr, slbg
(define_insn "*sub<mode>3_slb_cc"
  [(set (reg CC_REGNUM)
        (compare
          (minus:GPR (minus:GPR (match_operand:GPR 1 "nonimmediate_operand" "0,0")
                                (match_operand:GPR 2 "general_operand" "d,T"))
                     (match_operand:GPR 3 "s390_slb_comparison" ""))
          (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d,d")
        (minus:GPR (minus:GPR (match_dup 1) (match_dup 2)) (match_dup 3)))]
  "s390_match_ccmode (insn, CCLmode)"
  "@
   slb<g>r\t%0,%2
   slb<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_c,*")])

; slbr, slb, slbgr, slbg
(define_insn "*sub<mode>3_slb"
  [(set (match_operand:GPR 0 "register_operand" "=d,d")
        (minus:GPR (minus:GPR (match_operand:GPR 1 "nonimmediate_operand" "0,0")
                              (match_operand:GPR 2 "general_operand" "d,T"))
                   (match_operand:GPR 3 "s390_slb_comparison" "")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   slb<g>r\t%0,%2
   slb<g>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "z10prop" "z10_c,*")])

(define_expand "add<mode>cc"
  [(match_operand:GPR 0 "register_operand" "")
   (match_operand 1 "comparison_operator" "")
   (match_operand:GPR 2 "register_operand" "")
   (match_operand:GPR 3 "const_int_operand" "")]
  ""
  "if (!s390_expand_addcc (GET_CODE (operands[1]),
			   XEXP (operands[1], 0), XEXP (operands[1], 1),
			   operands[0], operands[2],
			   operands[3])) FAIL; DONE;")

;
; scond instruction pattern(s).
;

(define_insn_and_split "*scond<mode>"
  [(set (match_operand:GPR 0 "register_operand" "=&d")
        (match_operand:GPR 1 "s390_alc_comparison" ""))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (const_int 0))
   (parallel
    [(set (match_dup 0) (plus:GPR (plus:GPR (match_dup 1) (match_dup 0))
                                  (match_dup 0)))
     (clobber (reg:CC CC_REGNUM))])]
  "")

(define_insn_and_split "*scond<mode>_neg"
  [(set (match_operand:GPR 0 "register_operand" "=&d")
        (match_operand:GPR 1 "s390_slb_comparison" ""))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "#"
  "&& reload_completed"
  [(set (match_dup 0) (const_int 0))
   (parallel
    [(set (match_dup 0) (minus:GPR (minus:GPR (match_dup 0) (match_dup 0))
                                   (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_dup 0) (neg:GPR (match_dup 0)))
     (clobber (reg:CC CC_REGNUM))])]
  "")


(define_expand "cstore<mode>4"
  [(set (match_operand:SI 0 "register_operand" "")
        (match_operator:SI 1 "s390_scond_operator"
  	 [(match_operand:GPR 2 "register_operand" "")
          (match_operand:GPR 3 "general_operand" "")]))]
  ""
  "if (!s390_expand_addcc (GET_CODE (operands[1]), operands[2], operands[3],
			   operands[0], const0_rtx, const1_rtx)) FAIL; DONE;")

(define_expand "cstorecc4"
  [(parallel
    [(set (match_operand:SI 0 "register_operand" "")
	  (match_operator:SI 1 "s390_eqne_operator"
           [(match_operand 2 "cc_reg_operand")
	    (match_operand 3 "const0_operand")]))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "machine_mode mode = GET_MODE (operands[2]);
   if (TARGET_Z196)
     {
       rtx cond, ite;

       if (GET_CODE (operands[1]) == NE)
	 cond = gen_rtx_NE (VOIDmode, operands[2], const0_rtx);
       else
	 cond = gen_rtx_EQ (VOIDmode, operands[2], const0_rtx);
       ite = gen_rtx_IF_THEN_ELSE (SImode, cond, const1_rtx, const0_rtx);
       emit_insn (gen_rtx_SET (operands[0], ite));
     }
   else
     {
       if (mode != CCZ1mode)
	 FAIL;
       emit_insn (gen_sne (operands[0], operands[2]));
       if (GET_CODE (operands[1]) == EQ)
	 emit_insn (gen_xorsi3 (operands[0], operands[0], const1_rtx));
     }
   DONE;")

(define_insn_and_split "sne"
  [(set (match_operand:SI 0 "register_operand" "=d")
	(ne:SI (match_operand:CCZ1 1 "register_operand" "0")
	       (const_int 0)))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "#"
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (ashiftrt:SI (match_dup 0) (const_int 28)))
     (clobber (reg:CC CC_REGNUM))])])


;;
;; - Conditional move instructions (introduced with z196)
;;

(define_expand "mov<mode>cc"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "")
	(if_then_else:GPR (match_operand 1 "comparison_operator" "")
			  (match_operand:GPR 2 "loc_operand" "")
			  (match_operand:GPR 3 "loc_operand" "")))]
  "TARGET_Z196"
{
  if (!TARGET_Z13 && CONSTANT_P (operands[2]))
    operands[2] = force_reg (<MODE>mode, operands[2]);

  if (!TARGET_Z13 && CONSTANT_P (operands[3]))
    operands[3] = force_reg (<MODE>mode, operands[3]);

  /* Emit the comparison insn in case we do not already have a comparison result.  */
  if (!s390_comparison (operands[1], VOIDmode))
    operands[1] = s390_emit_compare (GET_CODE (operands[1]),
				     XEXP (operands[1], 0),
				     XEXP (operands[1], 1));
})

;;
;; - We do not have instructions for QImode or HImode but still
;;   enable load on condition/if conversion for them.
(define_expand "mov<mode>cc"
 [(set (match_operand:HQI 0 "nonimmediate_operand" "")
	(if_then_else:HQI (match_operand 1 "comparison_operator" "")
		(match_operand:HQI 2 "loc_operand" "")
		(match_operand:HQI 3 "loc_operand" "")))]
 "TARGET_Z196"
{
  /* Emit the comparison insn in case we do not already have a comparison
     result. */
  if (!s390_comparison (operands[1], VOIDmode))
    operands[1] = s390_emit_compare (GET_CODE (operands[1]),
			      XEXP (operands[1], 0),
			      XEXP (operands[1], 1));

  rtx then = operands[2];
  rtx els = operands[3];

  if ((!TARGET_Z13 && CONSTANT_P (then)) || MEM_P (then))
	then = force_reg (<MODE>mode, then);
  if ((!TARGET_Z13 && CONSTANT_P (els)) || MEM_P (els))
	els = force_reg (<MODE>mode, els);

  if (!CONSTANT_P (then))
    then = simplify_gen_subreg (E_SImode, then, <MODE>mode, 0);
  if (!CONSTANT_P (els))
    els = simplify_gen_subreg (E_SImode, els, <MODE>mode, 0);

  rtx tmp_target = gen_reg_rtx (E_SImode);
  emit_insn (gen_movsicc (tmp_target, operands[1], then, els));
  emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp_target));
  DONE;
})



; locr, loc, stoc, locgr, locg, stocg, lochi, locghi, selr, selgr
(define_insn "*mov<mode>cc"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "=d,d,d,d,d,d,d,S,S")
	(if_then_else:GPR
	  (match_operator 1 "s390_comparison"
	    [(match_operand 2 "cc_reg_operand"      " c,c,c,c,c,c,c,c,c")
	     (match_operand 5 "const_int_operand"   "")])
	  (match_operand:GPR 3 "loc_operand"        " d,0,d,S,0,K,0,d,0")
	  (match_operand:GPR 4 "loc_operand"        " 0,d,d,0,S,0,K,0,d")))]
  "TARGET_Z196"
  "@
   loc<g>r%C1\t%0,%3
   loc<g>r%D1\t%0,%4
   sel<g>r%C1\t%0,%3,%4
   loc<g>%C1\t%0,%3
   loc<g>%D1\t%0,%4
   loc<g>hi%C1\t%0,%h3
   loc<g>hi%D1\t%0,%h4
   stoc<g>%C1\t%3,%0
   stoc<g>%D1\t%4,%0"
  [(set_attr "op_type" "RRF,RRF,RRF,RSY,RSY,RIE,RIE,RSY,RSY")
   (set_attr "cpu_facility" "*,*,arch13,*,*,z13,z13,*,*")])

;;
;;- Multiply instructions.
;;

;
; muldi3 instruction pattern(s).
;

(define_expand "muldi3"
  [(parallel
    [(set (match_operand:DI          0 "register_operand")
	  (mult:DI (match_operand:DI 1 "nonimmediate_operand")
		   (match_operand:DI 2 "general_operand")))
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_ZARCH")

(define_insn "*muldi3_sign"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (mult:DI (sign_extend:DI (match_operand:SI 2 "general_operand" "d,T"))
                 (match_operand:DI 1 "register_operand" "0,0")))]
  "TARGET_ZARCH"
  "@
   msgfr\t%0,%2
   msgf\t%0,%2"
  [(set_attr "op_type"      "RRE,RXY")
   (set_attr "type"         "imuldi")])

(define_insn "*muldi3"
  [(set (match_operand:DI          0 "register_operand"     "=d,d,d,d,d")
	(mult:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d,0,0,0")
		 (match_operand:DI 2 "general_operand"       "d,d,K,T,Os")))
   (clobber (match_scratch:CC      3                        "=X,c,X,X,X"))]
  "TARGET_ZARCH"
  "@
   msgr\t%0,%2
   msgrkc\t%0,%1,%2
   mghi\t%0,%h2
   msg\t%0,%2
   msgfi\t%0,%2"
  [(set_attr "op_type"      "RRE,RRF,RI,RXY,RIL")
   (set_attr "type"         "imuldi")
   (set_attr "cpu_facility" "*,z14,*,*,z10")])

(define_insn "mulditi3"
  [(set (match_operand:TI 0 "register_operand"               "=d,d")
        (mult:TI (sign_extend:TI
		  (match_operand:DI 1 "register_operand"     "%d,0"))
		 (sign_extend:TI
		  (match_operand:DI 2 "nonimmediate_operand" " d,T"))))]
  "TARGET_Z14"
  "@
   mgrk\t%0,%1,%2
   mg\t%0,%2"
  [(set_attr "op_type"  "RRF,RXY")])

; Combine likes op1 and op2 to be swapped sometimes.
(define_insn "mulditi3_2"
  [(set (match_operand:TI 0 "register_operand"               "=d,d")
        (mult:TI (sign_extend:TI
		  (match_operand:DI 1 "nonimmediate_operand" "%d,T"))
		 (sign_extend:TI
		  (match_operand:DI 2 "register_operand"     " d,0"))))]
  "TARGET_Z14"
  "@
   mgrk\t%0,%1,%2
   mg\t%0,%1"
  [(set_attr "op_type"  "RRF,RXY")])

(define_insn "*muldi3_sign"
  [(set (match_operand:DI                          0 "register_operand" "=d")
        (mult:DI (sign_extend:DI (match_operand:HI 2 "memory_operand"    "T"))
                 (match_operand:DI                 1 "register_operand"  "0")))]
  "TARGET_Z14"
  "mgh\t%0,%2"
  [(set_attr "op_type" "RXY")])


;
; mulsi3 instruction pattern(s).
;

(define_expand "mulsi3"
  [(parallel
    [(set (match_operand:SI           0 "register_operand"     "=d,d,d,d,d,d")
	  (mult:SI  (match_operand:SI 1 "nonimmediate_operand" "%0,d,0,0,0,0")
		    (match_operand:SI 2 "general_operand"       "d,d,K,R,T,Os")))
     (clobber (reg:CC CC_REGNUM))])]
  "")

(define_insn "*mulsi3_sign"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (mult:SI (sign_extend:SI (match_operand:HI 2 "memory_operand" "R,T"))
                 (match_operand:SI 1 "register_operand" "0,0")))]
  ""
  "@
   mh\t%0,%2
   mhy\t%0,%2"
  [(set_attr "op_type"      "RX,RXY")
   (set_attr "type"         "imulhi")
   (set_attr "cpu_facility" "*,z10")])

(define_insn "*mulsi3"
  [(set (match_operand:SI           0 "register_operand"     "=d,d,d,d,d,d")
        (mult:SI  (match_operand:SI 1 "nonimmediate_operand" "%0,d,0,0,0,0")
                  (match_operand:SI 2 "general_operand"       "d,d,K,R,T,Os")))
   (clobber (match_scratch:CC       3                        "=X,c,X,X,X,X"))]
  ""
  "@
   msr\t%0,%2
   msrkc\t%0,%1,%2
   mhi\t%0,%h2
   ms\t%0,%2
   msy\t%0,%2
   msfi\t%0,%2"
  [(set_attr "op_type"      "RRE,RRF,RI,RX,RXY,RIL")
   (set_attr "type"         "imulsi,*,imulhi,imulsi,imulsi,imulsi")
   (set_attr "cpu_facility" "*,z14,*,*,longdisp,z10")])

;
; mulsidi3 instruction pattern(s).
;

(define_insn "mulsidi3"
  [(set (match_operand:DI 0 "register_operand" "=d,d,d")
        (mult:DI (sign_extend:DI
	           (match_operand:SI 1 "register_operand" "%0,0,0"))
                 (sign_extend:DI
	           (match_operand:SI 2 "nonimmediate_operand" "d,R,T"))))]
  "!TARGET_ZARCH"
  "@
   mr\t%0,%2
   m\t%0,%2
   mfy\t%0,%2"
  [(set_attr "op_type"      "RR,RX,RXY")
   (set_attr "type"         "imulsi")
   (set_attr "cpu_facility" "*,*,z10")])

;
; umul instruction pattern(s).
;

; mlr, ml, mlgr, mlg
(define_insn "umul<dwh><mode>3"
  [(set (match_operand:DW 0 "register_operand"                   "=d,d")
        (mult:DW (zero_extend:DW
	           (match_operand:<DWH> 1 "register_operand"     "%0,0"))
                 (zero_extend:DW
	           (match_operand:<DWH> 2 "nonimmediate_operand" " d,T"))))]
  ""
  "@
   ml<tg>r\t%0,%2
   ml<tg>\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "type"     "imul<dwh>")])

;
; mul(tf|df|sf|td|dd)3 instruction pattern(s).
;

; mxbr, mdbr, meebr, mxb, mxb, meeb, mdtr, mxtr
(define_insn "mul<mode>3"
  [(set (match_operand:FP          0 "register_operand"     "=f,f,f,v,v")
        (mult:FP (match_operand:FP 1 "nonimmediate_operand" "%f,0,0,v,v")
		 (match_operand:FP 2 "general_operand"       "f,f,R,v,v")))]
  "TARGET_HARD_FLOAT"
  "@
   m<xdee>tr\t%0,%1,%2
   m<xdee>br\t%0,%2
   m<xdee>b\t%0,%2
   wfmdb\t%v0,%v1,%v2
   wfmsb\t%v0,%v1,%v2"
  [(set_attr "op_type"      "RRF,RRE,RXE,VRR,VRR")
   (set_attr "type"         "fmul<mode>")
   (set_attr "cpu_facility" "*,*,*,vx,vxe")
   (set_attr "enabled"      "<nBFP>,<nDFP>,<DSF>,<DF>,<SF>")])

; madbr, maebr, maxb, madb, maeb
(define_insn "fma<mode>4"
  [(set (match_operand:DSF          0 "register_operand"     "=f,f,v,v")
	(fma:DSF (match_operand:DSF 1 "nonimmediate_operand" "%f,f,v,v")
		 (match_operand:DSF 2 "nonimmediate_operand"  "f,R,v,v")
		 (match_operand:DSF 3 "register_operand"      "0,0,v,v")))]
  "TARGET_HARD_FLOAT"
  "@
   ma<xde>br\t%0,%1,%2
   ma<xde>b\t%0,%1,%2
   wfmadb\t%v0,%v1,%v2,%v3
   wfmasb\t%v0,%v1,%v2,%v3"
  [(set_attr "op_type"      "RRE,RXE,VRR,VRR")
   (set_attr "type"         "fmadd<mode>")
   (set_attr "cpu_facility" "*,*,vx,vxe")
   (set_attr "enabled"      "*,*,<DF>,<SF>")])

; msxbr, msdbr, msebr, msxb, msdb, mseb
(define_insn "fms<mode>4"
  [(set (match_operand:DSF                   0 "register_operand"     "=f,f,v,v")
	(fma:DSF (match_operand:DSF          1 "nonimmediate_operand" "%f,f,v,v")
		 (match_operand:DSF          2 "nonimmediate_operand"  "f,R,v,v")
		 (neg:DSF (match_operand:DSF 3 "register_operand"      "0,0,v,v"))))]
  "TARGET_HARD_FLOAT"
  "@
   ms<xde>br\t%0,%1,%2
   ms<xde>b\t%0,%1,%2
   wfmsdb\t%v0,%v1,%v2,%v3
   wfmssb\t%v0,%v1,%v2,%v3"
  [(set_attr "op_type"      "RRE,RXE,VRR,VRR")
   (set_attr "type"         "fmadd<mode>")
   (set_attr "cpu_facility" "*,*,vx,vxe")
   (set_attr "enabled"      "*,*,<DF>,<SF>")])

;;
;;- Divide and modulo instructions.
;;

;
; divmoddi4 instruction pattern(s).
;

(define_expand "divmoddi4"
  [(parallel [(set (match_operand:DI 0 "general_operand" "")
		   (div:DI (match_operand:DI 1 "register_operand" "")
			   (match_operand:DI 2 "general_operand" "")))
	      (set (match_operand:DI 3 "general_operand" "")
		   (mod:DI (match_dup 1) (match_dup 2)))])
   (clobber (match_dup 4))]
  "TARGET_ZARCH"
{
  rtx div_equal, mod_equal;
  rtx_insn *insn;

  div_equal = gen_rtx_DIV (DImode, operands[1], operands[2]);
  mod_equal = gen_rtx_MOD (DImode, operands[1], operands[2]);

  operands[4] = gen_reg_rtx(TImode);
  emit_insn (gen_divmodtidi3 (operands[4], operands[1], operands[2]));

  insn = emit_move_insn (operands[0], gen_lowpart (DImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, div_equal);

  insn = emit_move_insn (operands[3], gen_highpart (DImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, mod_equal);

  DONE;
})

(define_insn "divmodtidi3"
  [(set (match_operand:TI 0 "register_operand" "=d,d")
        (ior:TI
          (ashift:TI
            (zero_extend:TI
              (mod:DI (match_operand:DI 1 "register_operand" "0,0")
                      (match_operand:DI 2 "general_operand" "d,T")))
            (const_int 64))
          (zero_extend:TI (div:DI (match_dup 1) (match_dup 2)))))]
  "TARGET_ZARCH"
  "@
   dsgr\t%0,%2
   dsg\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "type"     "idiv")])

(define_insn "divmodtisi3"
  [(set (match_operand:TI 0 "register_operand" "=d,d")
        (ior:TI
          (ashift:TI
            (zero_extend:TI
              (mod:DI (match_operand:DI 1 "register_operand" "0,0")
                      (sign_extend:DI
                        (match_operand:SI 2 "nonimmediate_operand" "d,T"))))
            (const_int 64))
          (zero_extend:TI
            (div:DI (match_dup 1) (sign_extend:DI (match_dup 2))))))]
  "TARGET_ZARCH"
  "@
   dsgfr\t%0,%2
   dsgf\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "type"     "idiv")])

;
; udivmoddi4 instruction pattern(s).
;

(define_expand "udivmoddi4"
  [(parallel [(set (match_operand:DI 0 "general_operand" "")
		   (udiv:DI (match_operand:DI 1 "general_operand" "")
			    (match_operand:DI 2 "nonimmediate_operand" "")))
	      (set (match_operand:DI 3 "general_operand" "")
		   (umod:DI (match_dup 1) (match_dup 2)))])
   (clobber (match_dup 4))]
  "TARGET_ZARCH"
{
  rtx div_equal, mod_equal, equal;
  rtx_insn *insn;

  div_equal = gen_rtx_UDIV (DImode, operands[1], operands[2]);
  mod_equal = gen_rtx_UMOD (DImode, operands[1], operands[2]);
  equal = gen_rtx_IOR (TImode,
		       gen_rtx_ASHIFT (TImode,
				       gen_rtx_ZERO_EXTEND (TImode, mod_equal),
				       GEN_INT (64)),
		       gen_rtx_ZERO_EXTEND (TImode, div_equal));

  operands[4] = gen_reg_rtx(TImode);
  emit_clobber (operands[4]);
  emit_move_insn (gen_lowpart (DImode, operands[4]), operands[1]);
  emit_move_insn (gen_highpart (DImode, operands[4]), const0_rtx);

  insn = emit_insn (gen_udivmodtidi3 (operands[4], operands[4], operands[2]));
  set_unique_reg_note (insn, REG_EQUAL, equal);

  insn = emit_move_insn (operands[0], gen_lowpart (DImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, div_equal);

  insn = emit_move_insn (operands[3], gen_highpart (DImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, mod_equal);

  DONE;
})

(define_insn "udivmodtidi3"
  [(set (match_operand:TI 0 "register_operand" "=d,d")
        (ior:TI
          (ashift:TI
            (zero_extend:TI
              (truncate:DI
                (umod:TI (match_operand:TI 1 "register_operand" "0,0")
                         (zero_extend:TI
                           (match_operand:DI 2 "nonimmediate_operand" "d,T")))))
            (const_int 64))
          (zero_extend:TI
            (truncate:DI
              (udiv:TI (match_dup 1) (zero_extend:TI (match_dup 2)))))))]
  "TARGET_ZARCH"
  "@
   dlgr\t%0,%2
   dlg\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "type"     "idiv")])

;
; divmodsi4 instruction pattern(s).
;

(define_expand "divmodsi4"
  [(parallel [(set (match_operand:SI 0 "general_operand" "")
		   (div:SI (match_operand:SI 1 "general_operand" "")
			   (match_operand:SI 2 "nonimmediate_operand" "")))
	      (set (match_operand:SI 3 "general_operand" "")
		   (mod:SI (match_dup 1) (match_dup 2)))])
   (clobber (match_dup 4))]
  "!TARGET_ZARCH"
{
  rtx div_equal, mod_equal, equal;
  rtx_insn *insn;

  div_equal = gen_rtx_DIV (SImode, operands[1], operands[2]);
  mod_equal = gen_rtx_MOD (SImode, operands[1], operands[2]);
  equal = gen_rtx_IOR (DImode,
		       gen_rtx_ASHIFT (DImode,
				       gen_rtx_ZERO_EXTEND (DImode, mod_equal),
				       GEN_INT (32)),
		       gen_rtx_ZERO_EXTEND (DImode, div_equal));

  operands[4] = gen_reg_rtx(DImode);
  emit_insn (gen_extendsidi2 (operands[4], operands[1]));

  insn = emit_insn (gen_divmoddisi3 (operands[4], operands[4], operands[2]));
  set_unique_reg_note (insn, REG_EQUAL, equal);

  insn = emit_move_insn (operands[0], gen_lowpart (SImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, div_equal);

  insn = emit_move_insn (operands[3], gen_highpart (SImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, mod_equal);

  DONE;
})

(define_insn "divmoddisi3"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (ior:DI
          (ashift:DI
            (zero_extend:DI
              (truncate:SI
                (mod:DI (match_operand:DI 1 "register_operand" "0,0")
                        (sign_extend:DI
                          (match_operand:SI 2 "nonimmediate_operand" "d,R")))))
            (const_int 32))
          (zero_extend:DI
            (truncate:SI
              (div:DI (match_dup 1) (sign_extend:DI (match_dup 2)))))))]
  "!TARGET_ZARCH"
  "@
   dr\t%0,%2
   d\t%0,%2"
  [(set_attr "op_type"  "RR,RX")
   (set_attr "type"     "idiv")])

;
; udivsi3 and umodsi3 instruction pattern(s).
;

(define_expand "udivmodsi4"
  [(parallel [(set (match_operand:SI 0 "general_operand" "")
		   (udiv:SI (match_operand:SI 1 "general_operand" "")
			    (match_operand:SI 2 "nonimmediate_operand" "")))
	      (set (match_operand:SI 3 "general_operand" "")
		   (umod:SI (match_dup 1) (match_dup 2)))])
   (clobber (match_dup 4))]
  "!TARGET_ZARCH"
{
  rtx div_equal, mod_equal, equal;
  rtx_insn *insn;

  div_equal = gen_rtx_UDIV (SImode, operands[1], operands[2]);
  mod_equal = gen_rtx_UMOD (SImode, operands[1], operands[2]);
  equal = gen_rtx_IOR (DImode,
		       gen_rtx_ASHIFT (DImode,
				       gen_rtx_ZERO_EXTEND (DImode, mod_equal),
				       GEN_INT (32)),
		       gen_rtx_ZERO_EXTEND (DImode, div_equal));

  operands[4] = gen_reg_rtx(DImode);
  emit_clobber (operands[4]);
  emit_move_insn (gen_lowpart (SImode, operands[4]), operands[1]);
  emit_move_insn (gen_highpart (SImode, operands[4]), const0_rtx);

  insn = emit_insn (gen_udivmoddisi3 (operands[4], operands[4], operands[2]));
  set_unique_reg_note (insn, REG_EQUAL, equal);

  insn = emit_move_insn (operands[0], gen_lowpart (SImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, div_equal);

  insn = emit_move_insn (operands[3], gen_highpart (SImode, operands[4]));
  set_unique_reg_note (insn, REG_EQUAL, mod_equal);

  DONE;
})

(define_insn "udivmoddisi3"
  [(set (match_operand:DI 0 "register_operand" "=d,d")
        (ior:DI
          (ashift:DI
            (zero_extend:DI
              (truncate:SI
                (umod:DI (match_operand:DI 1 "register_operand" "0,0")
                         (zero_extend:DI
                           (match_operand:SI 2 "nonimmediate_operand" "d,T")))))
            (const_int 32))
          (zero_extend:DI
            (truncate:SI
              (udiv:DI (match_dup 1) (zero_extend:DI (match_dup 2)))))))]
  "!TARGET_ZARCH"
  "@
   dlr\t%0,%2
   dl\t%0,%2"
  [(set_attr "op_type"  "RRE,RXY")
   (set_attr "type"     "idiv")])

;
; div(df|sf)3 instruction pattern(s).
;

; dxbr, ddbr, debr, dxb, ddb, deb, ddtr, dxtr
(define_insn "div<mode>3"
  [(set (match_operand:FP         0 "register_operand" "=f,f,f,v,v")
        (div:FP (match_operand:FP 1 "register_operand"  "f,0,0,v,v")
		(match_operand:FP 2 "general_operand"   "f,f,R,v,v")))]
  "TARGET_HARD_FLOAT"
  "@
   d<xde>tr\t%0,%1,%2
   d<xde>br\t%0,%2
   d<xde>b\t%0,%2
   wfddb\t%v0,%v1,%v2
   wfdsb\t%v0,%v1,%v2"
  [(set_attr "op_type"      "RRF,RRE,RXE,VRR,VRR")
   (set_attr "type"         "fdiv<mode>")
   (set_attr "cpu_facility" "*,*,*,vx,vxe")
   (set_attr "enabled"      "<nBFP>,<nDFP>,<DSF>,<DF>,<SF>")])


;;
;;- And instructions.
;;

(define_expand "and<mode>3"
  [(set (match_operand:INT 0 "nonimmediate_operand" "")
        (and:INT (match_operand:INT 1 "nonimmediate_operand" "")
                 (match_operand:INT 2 "general_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "s390_expand_logical_operator (AND, <MODE>mode, operands); DONE;")

;
; anddi3 instruction pattern(s).
;

(define_insn "*anddi3_cc"
  [(set (reg CC_REGNUM)
        (compare
	  (and:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d,0,    d")
                  (match_operand:DI 2 "general_operand"      " d,d,T,NxxDw"))
          (const_int 0)))
   (set (match_operand:DI 0 "register_operand"               "=d,d,d,    d")
        (and:DI (match_dup 1) (match_dup 2)))]
  "TARGET_ZARCH && s390_match_ccmode(insn, CCTmode)"
  "@
   ngr\t%0,%2
   ngrk\t%0,%1,%2
   ng\t%0,%2
   risbg\t%0,%1,%s2,128+%e2,0"
  [(set_attr "op_type"  "RRE,RRF,RXY,RIE")
   (set_attr "cpu_facility" "*,z196,*,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

(define_insn "*anddi3_cconly"
  [(set (reg CC_REGNUM)
        (compare
	  (and:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d,0,    d")
                  (match_operand:DI 2 "general_operand"      " d,d,T,NxxDw"))
                 (const_int 0)))
   (clobber (match_scratch:DI 0                              "=d,d,d,    d"))]
  "TARGET_ZARCH
   && s390_match_ccmode(insn, CCTmode)
   /* Do not steal TM patterns.  */
   && s390_single_part (operands[2], DImode, HImode, 0) < 0"
  "@
   ngr\t%0,%2
   ngrk\t%0,%1,%2
   ng\t%0,%2
   risbg\t%0,%1,%s2,128+%e2,0"
  [(set_attr "op_type"  "RRE,RRF,RXY,RIE")
   (set_attr "cpu_facility" "*,z196,*,z10")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super_E1")])

(define_insn "*anddi3"
  [(set (match_operand:DI 0 "nonimmediate_operand"
            "=d,d,    d,    d,    d,    d,    d,    d,d,d,d,    d,   AQ,Q")
        (and:DI
	  (match_operand:DI 1 "nonimmediate_operand"
            "%d,o,    0,    0,    0,    0,    0,    0,0,d,0,    d,    0,0")
          (match_operand:DI 2 "general_operand"
            "M, M,N0HDF,N1HDF,N2HDF,N3HDF,N0SDF,N1SDF,d,d,T,NxxDw,NxQDF,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   #
   #
   nihh\t%0,%j2
   nihl\t%0,%j2
   nilh\t%0,%j2
   nill\t%0,%j2
   nihf\t%0,%m2
   nilf\t%0,%m2
   ngr\t%0,%2
   ngrk\t%0,%1,%2
   ng\t%0,%2
   risbg\t%0,%1,%s2,128+%e2,0
   #
   #"
  [(set_attr "op_type" "RRE,RXE,RI,RI,RI,RI,RIL,RIL,RRE,RRF,RXY,RIE,SI,SS")
   (set_attr "cpu_facility" "*,*,*,*,*,*,extimm,extimm,*,z196,*,z10,*,*")
   (set_attr "z10prop" "*,
                        *,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        *")])

(define_split
  [(set (match_operand:DI 0 "s_operand" "")
        (and:DI (match_dup 0) (match_operand:DI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (and:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (AND, &operands[0], &operands[1]);")

;; These two are what combine generates for (ashift (zero_extract)).
(define_insn "*extzv_<mode>_srl<clobbercc_or_nocc>"
  [(set (match_operand:GPR 0 "register_operand" "=d")
	(and:GPR (lshiftrt:GPR
		   (match_operand:GPR 1 "register_operand" "d")
		   (match_operand:GPR 2 "nonzero_shift_count_operand" ""))
		(match_operand:GPR 3 "contiguous_bitmask_nowrap_operand" "")))]
  "<z10_or_zEC12_cond>
   /* Note that even for the SImode pattern, the rotate is always DImode.  */
   && s390_extzv_shift_ok (<bitsize>, -INTVAL (operands[2]),
			   INTVAL (operands[3]))"
  "<risbg_n>\t%0,%1,%<bfstart>3,128+%<bfend>3,64-%2"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])

(define_insn "*extzv_<mode>_sll<clobbercc_or_nocc>"
  [(set (match_operand:GPR 0 "register_operand" "=d")
	(and:GPR (ashift:GPR
		  (match_operand:GPR 1 "register_operand" "d")
		  (match_operand:GPR 2 "nonzero_shift_count_operand" ""))
		(match_operand:GPR 3 "contiguous_bitmask_nowrap_operand" "")))]
  "<z10_or_zEC12_cond>
   && s390_extzv_shift_ok (<bitsize>, INTVAL (operands[2]),
			   INTVAL (operands[3]))"
  "<risbg_n>\t%0,%1,%<bfstart>3,128+%<bfend>3,%2"
  [(set_attr "op_type" "RIE")
   (set_attr "z10prop" "z10_super_E1")])


;
; andsi3 instruction pattern(s).
;

(define_insn "*andsi3_cc"
  [(set (reg CC_REGNUM)
        (compare
	  (and:SI
	    (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0,    d")
            (match_operand:SI 2 "general_operand"      "Os,d,d,R,T,NxxSq"))
          (const_int 0)))
   (set (match_operand:SI 0 "register_operand"         "=d,d,d,d,d,    d")
        (and:SI (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   nilf\t%0,%o2
   nr\t%0,%2
   nrk\t%0,%1,%2
   n\t%0,%2
   ny\t%0,%2
   risbg\t%0,%1,%t2,128+%f2,0"
  [(set_attr "op_type"  "RIL,RR,RRF,RX,RXY,RIE")
   (set_attr "cpu_facility" "*,*,z196,*,longdisp,z10")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,
			z10_super_E1,z10_super_E1,z10_super_E1")])

(define_insn "*andsi3_cconly"
  [(set (reg CC_REGNUM)
        (compare
	  (and:SI
	    (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0,    d")
            (match_operand:SI 2 "general_operand"      "Os,d,d,R,T,NxxSq"))
          (const_int 0)))
   (clobber (match_scratch:SI 0                        "=d,d,d,d,d,    d"))]
  "s390_match_ccmode(insn, CCTmode)
   /* Do not steal TM patterns.  */
   && s390_single_part (operands[2], SImode, HImode, 0) < 0"
  "@
   nilf\t%0,%o2
   nr\t%0,%2
   nrk\t%0,%1,%2
   n\t%0,%2
   ny\t%0,%2
   risbg\t%0,%1,%t2,128+%f2,0"
  [(set_attr "op_type"  "RIL,RR,RRF,RX,RXY,RIE")
   (set_attr "cpu_facility" "*,*,z196,*,longdisp,z10")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1,z10_super_E1")])

(define_insn "*andsi3_zarch"
  [(set (match_operand:SI 0 "nonimmediate_operand"
                            "=d,d,    d,    d, d,d,d,d,d,    d,   AQ,Q")
        (and:SI (match_operand:SI 1 "nonimmediate_operand"
			    "%d,o,    0,    0, 0,0,d,0,0,    d,    0,0")
                (match_operand:SI 2 "general_operand"
			    " M,M,N0HSF,N1HSF,Os,d,d,R,T,NxxSw,NxQSF,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   #
   #
   nilh\t%0,%j2
   nill\t%0,%j2
   nilf\t%0,%o2
   nr\t%0,%2
   nrk\t%0,%1,%2
   n\t%0,%2
   ny\t%0,%2
   risbg\t%0,%1,%t2,128+%f2,0
   #
   #"
  [(set_attr "op_type"  "RRE,RXE,RI,RI,RIL,RR,RRF,RX,RXY,RIE,SI,SS")
   (set_attr "cpu_facility" "*,*,*,*,*,*,z196,*,longdisp,z10,*,*")
   (set_attr "z10prop" "*,
                        *,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        *")])

(define_insn "*andsi3_esa"
  [(set (match_operand:SI 0 "nonimmediate_operand"         "=d,d,   AQ,Q")
        (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,    0,0")
                (match_operand:SI 2 "general_operand"      " d,R,NxQSF,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   nr\t%0,%2
   n\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RR,RX,SI,SS")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,*")])


(define_split
  [(set (match_operand:SI 0 "s_operand" "")
        (and:SI (match_dup 0) (match_operand:SI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (and:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (AND, &operands[0], &operands[1]);")

;
; andhi3 instruction pattern(s).
;

(define_insn "*andhi3_zarch"
  [(set (match_operand:HI 0 "nonimmediate_operand"         "=d,d,d,   AQ,Q")
        (and:HI (match_operand:HI 1 "nonimmediate_operand" "%0,d,0,    0,0")
                (match_operand:HI 2 "general_operand"      " d,d,n,NxQHF,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   nr\t%0,%2
   nrk\t%0,%1,%2
   nill\t%0,%x2
   #
   #"
  [(set_attr "op_type"  "RR,RRF,RI,SI,SS")
   (set_attr "cpu_facility" "*,z196,*,*,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,*,*")
])

(define_insn "*andhi3_esa"
  [(set (match_operand:HI 0 "nonimmediate_operand" "=d,AQ,Q")
        (and:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0,0")
                (match_operand:HI 2 "general_operand" "d,NxQHF,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   nr\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RR,SI,SS")
   (set_attr "z10prop" "z10_super_E1,*,*")
])

(define_split
  [(set (match_operand:HI 0 "s_operand" "")
        (and:HI (match_dup 0) (match_operand:HI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (and:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (AND, &operands[0], &operands[1]);")

;
; andqi3 instruction pattern(s).
;

(define_insn "*andqi3_zarch"
  [(set (match_operand:QI 0 "nonimmediate_operand"         "=d,d,d,Q,S,Q")
        (and:QI (match_operand:QI 1 "nonimmediate_operand" "%0,d,0,0,0,0")
                (match_operand:QI 2 "general_operand"      " d,d,n,n,n,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   nr\t%0,%2
   nrk\t%0,%1,%2
   nill\t%0,%b2
   ni\t%S0,%b2
   niy\t%S0,%b2
   #"
  [(set_attr "op_type"  "RR,RRF,RI,SI,SIY,SS")
   (set_attr "cpu_facility" "*,z196,*,*,longdisp,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,z10_super,z10_super,*")])

(define_insn "*andqi3_esa"
  [(set (match_operand:QI 0 "nonimmediate_operand" "=d,Q,Q")
        (and:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
                (match_operand:QI 2 "general_operand" "d,n,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   nr\t%0,%2
   ni\t%S0,%b2
   #"
  [(set_attr "op_type"  "RR,SI,SS")
   (set_attr "z10prop" "z10_super_E1,z10_super,*")])

;
; And with complement
;
; c = ~b & a = (b & a) ^ a

(define_insn_and_split "*andc_split_<mode>"
  [(set (match_operand:GPR 0 "nonimmediate_operand" "")
	(and:GPR (not:GPR (match_operand:GPR 1 "nonimmediate_operand" ""))
		 (match_operand:GPR 2 "general_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ARCH13
   && ! reload_completed
   && (GET_CODE (operands[0]) != MEM
      /* Ensure that s390_logical_operator_ok_p will succeed even
	 on the split xor if (b & a) is stored into a pseudo.  */
       || rtx_equal_p (operands[0], operands[2]))"
  "#"
  "&& 1"
  [
  (parallel
   [(set (match_dup 3) (and:GPR (match_dup 1) (match_dup 2)))
   (clobber (reg:CC CC_REGNUM))])
  (parallel
   [(set (match_dup 0) (xor:GPR (match_dup 3) (match_dup 2)))
   (clobber (reg:CC CC_REGNUM))])]
{
  if (reg_overlap_mentioned_p (operands[0], operands[2]))
    operands[3] = gen_reg_rtx (<MODE>mode);
  else
    operands[3] = operands[0];
})

;
; Block and (NC) patterns.
;

(define_insn "*nc"
  [(set (match_operand:BLK 0 "memory_operand" "=Q")
        (and:BLK (match_dup 0)
                 (match_operand:BLK 1 "memory_operand" "Q")))
   (use (match_operand 2 "const_int_operand" "n"))
   (clobber (reg:CC CC_REGNUM))]
  "INTVAL (operands[2]) >= 1 && INTVAL (operands[2]) <= 256"
  "nc\t%O0(%2,%R0),%S1"
  [(set_attr "op_type" "SS")
   (set_attr "z196prop" "z196_cracked")])

(define_split
  [(set (match_operand 0 "memory_operand" "")
        (and (match_dup 0)
             (match_operand 1 "memory_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed
   && GET_MODE (operands[0]) == GET_MODE (operands[1])
   && GET_MODE_SIZE (GET_MODE (operands[0])) > 0"
  [(parallel
    [(set (match_dup 0) (and:BLK (match_dup 0) (match_dup 1)))
     (use (match_dup 2))
     (clobber (reg:CC CC_REGNUM))])]
{
  operands[2] = GEN_INT (GET_MODE_SIZE (GET_MODE (operands[0])));
  operands[0] = adjust_address (operands[0], BLKmode, 0);
  operands[1] = adjust_address (operands[1], BLKmode, 0);
})

(define_peephole2
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (and:BLK (match_dup 0)
                   (match_operand:BLK 1 "memory_operand" "")))
     (use (match_operand 2 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_operand:BLK 3 "memory_operand" "")
          (and:BLK (match_dup 3)
                   (match_operand:BLK 4 "memory_operand" "")))
     (use (match_operand 5 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_offset_p (operands[0], operands[3], operands[2])
   && s390_offset_p (operands[1], operands[4], operands[2])
   && !s390_overlap_p (operands[0], operands[1],
                       INTVAL (operands[2]) + INTVAL (operands[5]))
   && INTVAL (operands[2]) + INTVAL (operands[5]) <= 256"
  [(parallel
    [(set (match_dup 6) (and:BLK (match_dup 6) (match_dup 7)))
     (use (match_dup 8))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[6] = gen_rtx_MEM (BLKmode, XEXP (operands[0], 0));
   operands[7] = gen_rtx_MEM (BLKmode, XEXP (operands[1], 0));
   operands[8] = GEN_INT (INTVAL (operands[2]) + INTVAL (operands[5]));")


;;
;;- Bit set (inclusive or) instructions.
;;

(define_expand "ior<mode>3"
  [(set (match_operand:INT 0 "nonimmediate_operand" "")
        (ior:INT (match_operand:INT 1 "nonimmediate_operand" "")
                 (match_operand:INT 2 "general_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "s390_expand_logical_operator (IOR, <MODE>mode, operands); DONE;")

;
; iordi3 instruction pattern(s).
;

(define_insn "*iordi3_cc"
  [(set (reg CC_REGNUM)
        (compare (ior:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d,0")
                         (match_operand:DI 2 "general_operand"      " d,d,T"))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand"                      "=d,d,d")
        (ior:DI (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode(insn, CCTmode) && TARGET_ZARCH"
  "@
   ogr\t%0,%2
   ogrk\t%0,%1,%2
   og\t%0,%2"
  [(set_attr "op_type"  "RRE,RRF,RXY")
   (set_attr "cpu_facility" "*,z196,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1")])

(define_insn "*iordi3_cconly"
  [(set (reg CC_REGNUM)
        (compare (ior:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d,0")
                         (match_operand:DI 2 "general_operand"      " d,d,T"))
                 (const_int 0)))
   (clobber (match_scratch:DI 0                                     "=d,d,d"))]
  "s390_match_ccmode(insn, CCTmode) && TARGET_ZARCH"
  "@
   ogr\t%0,%2
   ogrk\t%0,%1,%2
   og\t%0,%2"
  [(set_attr "op_type"  "RRE,RRF,RXY")
   (set_attr "cpu_facility" "*,z196,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1")])

(define_insn "*iordi3"
  [(set (match_operand:DI 0 "nonimmediate_operand"
                               "=d,    d,    d,    d,    d,    d,d,d,d,   AQ,Q")
        (ior:DI (match_operand:DI 1 "nonimmediate_operand"
                            "   %0,    0,    0,    0,    0,    0,0,d,0,    0,0")
                (match_operand:DI 2 "general_operand"
                            "N0HD0,N1HD0,N2HD0,N3HD0,N0SD0,N1SD0,d,d,T,NxQD0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   oihh\t%0,%i2
   oihl\t%0,%i2
   oilh\t%0,%i2
   oill\t%0,%i2
   oihf\t%0,%k2
   oilf\t%0,%k2
   ogr\t%0,%2
   ogrk\t%0,%1,%2
   og\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RI,RI,RI,RI,RIL,RIL,RRE,RRF,RXY,SI,SS")
   (set_attr "cpu_facility" "*,*,*,*,extimm,extimm,*,z196,*,*,*")
   (set_attr "z10prop" "z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        z10_super_E1,
                        *,
                        *")])

(define_split
  [(set (match_operand:DI 0 "s_operand" "")
        (ior:DI (match_dup 0) (match_operand:DI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (ior:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (IOR, &operands[0], &operands[1]);")

;
; iorsi3 instruction pattern(s).
;

(define_insn "*iorsi3_cc"
  [(set (reg CC_REGNUM)
        (compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0")
                         (match_operand:SI 2 "general_operand"      "Os,d,d,R,T"))
                 (const_int 0)))
   (set (match_operand:SI 0 "register_operand"                      "=d,d,d,d,d")
        (ior:SI (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   oilf\t%0,%o2
   or\t%0,%2
   ork\t%0,%1,%2
   o\t%0,%2
   oy\t%0,%2"
  [(set_attr "op_type"  "RIL,RR,RRF,RX,RXY")
   (set_attr "cpu_facility" "*,*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,z10_super_E1,z10_super_E1")])

(define_insn "*iorsi3_cconly"
  [(set (reg CC_REGNUM)
        (compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0")
                         (match_operand:SI 2 "general_operand"      "Os,d,d,R,T"))
                 (const_int 0)))
   (clobber (match_scratch:SI 0                                     "=d,d,d,d,d"))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   oilf\t%0,%o2
   or\t%0,%2
   ork\t%0,%1,%2
   o\t%0,%2
   oy\t%0,%2"
  [(set_attr "op_type"  "RIL,RR,RRF,RX,RXY")
   (set_attr "cpu_facility" "*,*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,z10_super_E1,z10_super_E1")])

(define_insn "*iorsi3_zarch"
  [(set (match_operand:SI 0 "nonimmediate_operand"         "=d,    d, d,d,d,d,d,   AQ,Q")
        (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,    0, 0,0,d,0,0,    0,0")
                (match_operand:SI 2 "general_operand"   "N0HS0,N1HS0,Os,d,d,R,T,NxQS0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   oilh\t%0,%i2
   oill\t%0,%i2
   oilf\t%0,%o2
   or\t%0,%2
   ork\t%0,%1,%2
   o\t%0,%2
   oy\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RI,RI,RIL,RR,RRF,RX,RXY,SI,SS")
   (set_attr "cpu_facility" "*,*,*,*,z196,*,longdisp,*,*")
   (set_attr "z10prop" "z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        z10_super_E1,
                        z10_super_E1,
                        *,
                        *")])

(define_insn "*iorsi3_esa"
  [(set (match_operand:SI 0 "nonimmediate_operand" "=d,d,AQ,Q")
        (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0,0")
                (match_operand:SI 2 "general_operand" "d,R,NxQS0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   or\t%0,%2
   o\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RR,RX,SI,SS")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,*")])

(define_split
  [(set (match_operand:SI 0 "s_operand" "")
        (ior:SI (match_dup 0) (match_operand:SI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (ior:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (IOR, &operands[0], &operands[1]);")

;
; iorhi3 instruction pattern(s).
;

(define_insn "*iorhi3_zarch"
  [(set (match_operand:HI 0 "nonimmediate_operand"         "=d,d,d,   AQ,Q")
        (ior:HI (match_operand:HI 1 "nonimmediate_operand" "%0,d,0,    0,0")
                (match_operand:HI 2 "general_operand"      " d,d,n,NxQH0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   or\t%0,%2
   ork\t%0,%1,%2
   oill\t%0,%x2
   #
   #"
  [(set_attr "op_type"  "RR,RRF,RI,SI,SS")
   (set_attr "cpu_facility" "*,z196,*,*,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,*,*")])

(define_insn "*iorhi3_esa"
  [(set (match_operand:HI 0 "nonimmediate_operand" "=d,AQ,Q")
        (ior:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0,0")
                (match_operand:HI 2 "general_operand" "d,NxQH0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   or\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RR,SI,SS")
   (set_attr "z10prop" "z10_super_E1,*,*")])

(define_split
  [(set (match_operand:HI 0 "s_operand" "")
        (ior:HI (match_dup 0) (match_operand:HI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (ior:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (IOR, &operands[0], &operands[1]);")

;
; iorqi3 instruction pattern(s).
;

(define_insn "*iorqi3_zarch"
  [(set (match_operand:QI 0 "nonimmediate_operand"         "=d,d,d,Q,S,Q")
        (ior:QI (match_operand:QI 1 "nonimmediate_operand" "%0,d,0,0,0,0")
                (match_operand:QI 2 "general_operand"      " d,d,n,n,n,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   or\t%0,%2
   ork\t%0,%1,%2
   oill\t%0,%b2
   oi\t%S0,%b2
   oiy\t%S0,%b2
   #"
  [(set_attr "op_type" "RR,RRF,RI,SI,SIY,SS")
   (set_attr "cpu_facility" "*,z196,*,*,longdisp,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1,
                        z10_super,z10_super,*")])

(define_insn "*iorqi3_esa"
  [(set (match_operand:QI 0 "nonimmediate_operand" "=d,Q,Q")
        (ior:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
                (match_operand:QI 2 "general_operand" "d,n,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   or\t%0,%2
   oi\t%S0,%b2
   #"
  [(set_attr "op_type"  "RR,SI,SS")
   (set_attr "z10prop" "z10_super_E1,z10_super,*")])

;
; And/Or with complement
;

; ncrk, ncgrk, ocrk, ocgrk
(define_insn "*<ANDOR:bitops_name>c<GPR:mode>_cc"
  [(set (reg CC_REGNUM)
	(compare
	 (ANDOR:GPR (not:GPR (match_operand:GPR 1 "register_operand" "d"))
		    (match_operand:GPR 2 "register_operand" "d"))
	 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d")
	(ANDOR:GPR (not:GPR (match_dup 1))
		   (match_dup 2)))]
  "TARGET_ARCH13 && s390_match_ccmode(insn, CCTmode)"
  "<ANDOR:noxa>c<GPR:g>rk\t%0,%2,%1"
  [(set_attr "op_type" "RRF")])

; ncrk, ncgrk, ocrk, ocgrk
(define_insn "*<ANDOR:bitops_name>c<GPR:mode>_cconly"
  [(set (reg CC_REGNUM)
	(compare
	 (ANDOR:GPR (not:GPR (match_operand:GPR 1 "register_operand" "d"))
		    (match_operand:GPR 2 "register_operand" "d"))
	 (const_int 0)))
   (clobber (match_scratch:GPR 0 "=d"))]
  "TARGET_ARCH13 && s390_match_ccmode(insn, CCTmode)"
  "<ANDOR:noxa>c<GPR:g>rk\t%0,%2,%1"
  [(set_attr "op_type" "RRF")])

; ncrk, ncgrk, ocrk, ocgrk
(define_insn "*<ANDOR:bitops_name>c<GPR:mode>"
  [(set (match_operand:GPR 0 "register_operand" "=d")
	(ANDOR:GPR (not:GPR (match_operand:GPR 1 "register_operand" "d"))
		   (match_operand:GPR 2 "register_operand" "d")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ARCH13"
  "<ANDOR:noxa>c<GPR:g>rk\t%0,%2,%1"
  [(set_attr "op_type" "RRF")])

;
;- Nand/Nor instructions.
;

; nnrk, nngrk, nork, nogrk
(define_insn "*n<ANDOR:inv_bitops_name><GPR:mode>_cc"
  [(set (reg CC_REGNUM)
	(compare
	 (ANDOR:GPR (not:GPR (match_operand:GPR 1 "register_operand" "d"))
		    (not:GPR (match_operand:GPR 2 "register_operand" "d")))
	 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d")
	(ANDOR:GPR (not:GPR (match_dup 1))
		   (not:GPR (match_dup 2))))]
  "TARGET_ARCH13 && s390_match_ccmode(insn, CCTmode)"
  "n<ANDOR:inv_no><GPR:g>rk\t%0,%1,%2"
  [(set_attr "op_type" "RRF")])

; nnrk, nngrk, nork, nogrk
(define_insn "*n<ANDOR:inv_bitops_name><mode>_cconly"
  [(set (reg CC_REGNUM)
	(compare
	 (ANDOR:GPR (not:GPR (match_operand:GPR 1 "register_operand" "d"))
		    (not:GPR (match_operand:GPR 2 "register_operand" "d")))
	 (const_int 0)))
   (clobber (match_scratch:GPR 0 "=d"))]
  "TARGET_ARCH13 && s390_match_ccmode(insn, CCTmode)"
  "n<ANDOR:inv_no><GPR:g>rk\t%0,%1,%2"
  [(set_attr "op_type" "RRF")])

; nnrk, nngrk, nork, nogrk
(define_insn "*n<ANDOR:inv_bitops_name><mode>"
  [(set (match_operand:GPR 0 "register_operand" "=d")
	(ANDOR:GPR (not:GPR (match_operand:GPR 1 "register_operand" "d"))
		   (not:GPR (match_operand:GPR 2 "register_operand" "d"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ARCH13"
  "n<ANDOR:inv_no><GPR:g>rk\t%0,%1,%2"
  [(set_attr "op_type" "RRF")])


;
; Block inclusive or (OC) patterns.
;

(define_insn "*oc"
  [(set (match_operand:BLK 0 "memory_operand" "=Q")
        (ior:BLK (match_dup 0)
                 (match_operand:BLK 1 "memory_operand" "Q")))
   (use (match_operand 2 "const_int_operand" "n"))
   (clobber (reg:CC CC_REGNUM))]
  "INTVAL (operands[2]) >= 1 && INTVAL (operands[2]) <= 256"
  "oc\t%O0(%2,%R0),%S1"
  [(set_attr "op_type" "SS")
   (set_attr "z196prop" "z196_cracked")])

(define_split
  [(set (match_operand 0 "memory_operand" "")
        (ior (match_dup 0)
             (match_operand 1 "memory_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed
   && GET_MODE (operands[0]) == GET_MODE (operands[1])
   && GET_MODE_SIZE (GET_MODE (operands[0])) > 0"
  [(parallel
    [(set (match_dup 0) (ior:BLK (match_dup 0) (match_dup 1)))
     (use (match_dup 2))
     (clobber (reg:CC CC_REGNUM))])]
{
  operands[2] = GEN_INT (GET_MODE_SIZE (GET_MODE (operands[0])));
  operands[0] = adjust_address (operands[0], BLKmode, 0);
  operands[1] = adjust_address (operands[1], BLKmode, 0);
})

(define_peephole2
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (ior:BLK (match_dup 0)
                   (match_operand:BLK 1 "memory_operand" "")))
     (use (match_operand 2 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_operand:BLK 3 "memory_operand" "")
          (ior:BLK (match_dup 3)
                   (match_operand:BLK 4 "memory_operand" "")))
     (use (match_operand 5 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_offset_p (operands[0], operands[3], operands[2])
   && s390_offset_p (operands[1], operands[4], operands[2])
   && !s390_overlap_p (operands[0], operands[1],
                       INTVAL (operands[2]) + INTVAL (operands[5]))
   && INTVAL (operands[2]) + INTVAL (operands[5]) <= 256"
  [(parallel
    [(set (match_dup 6) (ior:BLK (match_dup 6) (match_dup 7)))
     (use (match_dup 8))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[6] = gen_rtx_MEM (BLKmode, XEXP (operands[0], 0));
   operands[7] = gen_rtx_MEM (BLKmode, XEXP (operands[1], 0));
   operands[8] = GEN_INT (INTVAL (operands[2]) + INTVAL (operands[5]));")


;;
;;- Xor instructions.
;;

(define_expand "xor<mode>3"
  [(set (match_operand:INT 0 "nonimmediate_operand" "")
        (xor:INT (match_operand:INT 1 "nonimmediate_operand" "")
                 (match_operand:INT 2 "general_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "s390_expand_logical_operator (XOR, <MODE>mode, operands); DONE;")

; Combine replaces (xor (x) (const_int -1)) with (not (x)) when doing
; simplifications.  So its better to have something matching.
(define_split
  [(set (match_operand:INT 0 "nonimmediate_operand" "")
        (not:INT (match_operand:INT 1 "nonimmediate_operand" "")))]
  ""
  [(parallel
    [(set (match_dup 0) (xor:INT (match_dup 1) (match_dup 2)))
     (clobber (reg:CC CC_REGNUM))])]
{
  operands[2] = constm1_rtx;
  if (!s390_logical_operator_ok_p (operands))
    FAIL;
})

;
; xordi3 instruction pattern(s).
;

(define_insn "*xordi3_cc"
  [(set (reg CC_REGNUM)
        (compare (xor:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d,0")
                         (match_operand:DI 2 "general_operand"      " d,d,T"))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand"                      "=d,d,d")
        (xor:DI (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode(insn, CCTmode) && TARGET_ZARCH"
  "@
   xgr\t%0,%2
   xgrk\t%0,%1,%2
   xg\t%0,%2"
  [(set_attr "op_type" "RRE,RRF,RXY")
   (set_attr "cpu_facility" "*,z196,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1")])

(define_insn "*xordi3_cconly"
  [(set (reg CC_REGNUM)
        (compare (xor:DI (match_operand:DI 1 "nonimmediate_operand" "%0,d,0")
                         (match_operand:DI 2 "general_operand"      " d,d,T"))
                 (const_int 0)))
   (clobber (match_scratch:DI 0                                     "=d,d,d"))]
  "s390_match_ccmode(insn, CCTmode) && TARGET_ZARCH"
  "@
   xgr\t%0,%2
   xgrk\t%0,%1,%2
   xg\t%0,%2"
  [(set_attr "op_type" "RRE,RRF,RXY")
   (set_attr "cpu_facility" "*,z196,*")
   (set_attr "z10prop" "z10_super_E1,*,z10_super_E1")])

(define_insn "*xordi3"
  [(set (match_operand:DI 0 "nonimmediate_operand"         "=d,    d,d,d,d,   AQ,Q")
        (xor:DI (match_operand:DI 1 "nonimmediate_operand" "%0,    0,0,d,0,    0,0")
                (match_operand:DI 2 "general_operand"   "N0SD0,N1SD0,d,d,T,NxQD0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH && s390_logical_operator_ok_p (operands)"
  "@
   xihf\t%0,%k2
   xilf\t%0,%k2
   xgr\t%0,%2
   xgrk\t%0,%1,%2
   xg\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RIL,RIL,RRE,RRF,RXY,SI,SS")
   (set_attr "cpu_facility" "extimm,extimm,*,z196,*,*,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,z10_super_E1,
                        *,z10_super_E1,*,*")])

(define_split
  [(set (match_operand:DI 0 "s_operand" "")
        (xor:DI (match_dup 0) (match_operand:DI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (xor:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (XOR, &operands[0], &operands[1]);")

;
; xorsi3 instruction pattern(s).
;

(define_insn "*xorsi3_cc"
  [(set (reg CC_REGNUM)
        (compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0")
                         (match_operand:SI 2 "general_operand"      "Os,d,d,R,T"))
                 (const_int 0)))
   (set (match_operand:SI 0 "register_operand"                      "=d,d,d,d,d")
        (xor:SI (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   xilf\t%0,%o2
   xr\t%0,%2
   xrk\t%0,%1,%2
   x\t%0,%2
   xy\t%0,%2"
  [(set_attr "op_type" "RIL,RR,RRF,RX,RXY")
   (set_attr "cpu_facility" "*,*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1")])

(define_insn "*xorsi3_cconly"
  [(set (reg CC_REGNUM)
        (compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0")
                         (match_operand:SI 2 "general_operand"      "Os,d,d,R,T"))
                 (const_int 0)))
   (clobber (match_scratch:SI 0                                     "=d,d,d,d,d"))]
  "s390_match_ccmode(insn, CCTmode)"
  "@
   xilf\t%0,%o2
   xr\t%0,%2
   xrk\t%0,%1,%2
   x\t%0,%2
   xy\t%0,%2"
  [(set_attr "op_type" "RIL,RR,RRF,RX,RXY")
   (set_attr "cpu_facility" "*,*,z196,*,longdisp")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1")])

(define_insn "*xorsi3"
  [(set (match_operand:SI 0 "nonimmediate_operand"         "=d,d,d,d,d,   AQ,Q")
        (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,d,0,0,    0,0")
                (match_operand:SI 2 "general_operand"      "Os,d,d,R,T,NxQS0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "s390_logical_operator_ok_p (operands)"
  "@
   xilf\t%0,%o2
   xr\t%0,%2
   xrk\t%0,%1,%2
   x\t%0,%2
   xy\t%0,%2
   #
   #"
  [(set_attr "op_type"  "RIL,RR,RRF,RX,RXY,SI,SS")
   (set_attr "cpu_facility" "*,*,z196,*,longdisp,*,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,
                        z10_super_E1,z10_super_E1,*,*")])

(define_split
  [(set (match_operand:SI 0 "s_operand" "")
        (xor:SI (match_dup 0) (match_operand:SI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (xor:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (XOR, &operands[0], &operands[1]);")

;
; xorhi3 instruction pattern(s).
;

(define_insn "*xorhi3"
  [(set (match_operand:HI 0 "nonimmediate_operand"         "=d,d,d,   AQ,Q")
        (xor:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0,d,    0,0")
                (match_operand:HI 2 "general_operand"      "Os,d,d,NxQH0,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "s390_logical_operator_ok_p (operands)"
  "@
   xilf\t%0,%x2
   xr\t%0,%2
   xrk\t%0,%1,%2
   #
   #"
  [(set_attr "op_type"  "RIL,RR,RRF,SI,SS")
   (set_attr "cpu_facility" "*,*,z196,*,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,*,*")])

(define_split
  [(set (match_operand:HI 0 "s_operand" "")
        (xor:HI (match_dup 0) (match_operand:HI 1 "immediate_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed"
  [(parallel
    [(set (match_dup 0) (xor:QI (match_dup 0) (match_dup 1)))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_narrow_logical_operator (XOR, &operands[0], &operands[1]);")

;
; xorqi3 instruction pattern(s).
;

(define_insn "*xorqi3"
  [(set (match_operand:QI 0 "nonimmediate_operand"         "=d,d,d,Q,S,Q")
        (xor:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,d,0,0,0")
                (match_operand:QI 2 "general_operand"      "Os,d,d,n,n,Q")))
   (clobber (reg:CC CC_REGNUM))]
  "s390_logical_operator_ok_p (operands)"
  "@
   xilf\t%0,%b2
   xr\t%0,%2
   xrk\t%0,%1,%2
   xi\t%S0,%b2
   xiy\t%S0,%b2
   #"
  [(set_attr "op_type"  "RIL,RR,RRF,SI,SIY,SS")
   (set_attr "cpu_facility" "*,*,z196,*,longdisp,*")
   (set_attr "z10prop" "z10_super_E1,z10_super_E1,*,z10_super,z10_super,*")])


;
; Block exclusive or (XC) patterns.
;

(define_insn "*xc"
  [(set (match_operand:BLK 0 "memory_operand" "=Q")
        (xor:BLK (match_dup 0)
                 (match_operand:BLK 1 "memory_operand" "Q")))
   (use (match_operand 2 "const_int_operand" "n"))
   (clobber (reg:CC CC_REGNUM))]
  "INTVAL (operands[2]) >= 1 && INTVAL (operands[2]) <= 256"
  "xc\t%O0(%2,%R0),%S1"
  [(set_attr "op_type" "SS")])

(define_split
  [(set (match_operand 0 "memory_operand" "")
        (xor (match_dup 0)
             (match_operand 1 "memory_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "reload_completed
   && GET_MODE (operands[0]) == GET_MODE (operands[1])
   && GET_MODE_SIZE (GET_MODE (operands[0])) > 0"
  [(parallel
    [(set (match_dup 0) (xor:BLK (match_dup 0) (match_dup 1)))
     (use (match_dup 2))
     (clobber (reg:CC CC_REGNUM))])]
{
  operands[2] = GEN_INT (GET_MODE_SIZE (GET_MODE (operands[0])));
  operands[0] = adjust_address (operands[0], BLKmode, 0);
  operands[1] = adjust_address (operands[1], BLKmode, 0);
})

(define_peephole2
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (xor:BLK (match_dup 0)
                   (match_operand:BLK 1 "memory_operand" "")))
     (use (match_operand 2 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_operand:BLK 3 "memory_operand" "")
          (xor:BLK (match_dup 3)
                   (match_operand:BLK 4 "memory_operand" "")))
     (use (match_operand 5 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_offset_p (operands[0], operands[3], operands[2])
   && s390_offset_p (operands[1], operands[4], operands[2])
   && !s390_overlap_p (operands[0], operands[1],
                       INTVAL (operands[2]) + INTVAL (operands[5]))
   && INTVAL (operands[2]) + INTVAL (operands[5]) <= 256"
  [(parallel
    [(set (match_dup 6) (xor:BLK (match_dup 6) (match_dup 7)))
     (use (match_dup 8))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[6] = gen_rtx_MEM (BLKmode, XEXP (operands[0], 0));
   operands[7] = gen_rtx_MEM (BLKmode, XEXP (operands[1], 0));
   operands[8] = GEN_INT (INTVAL (operands[2]) + INTVAL (operands[5]));")

;
; Block xor (XC) patterns with src == dest.
;

(define_insn "*xc_zero"
  [(set (match_operand:BLK 0 "memory_operand" "=Q")
        (const_int 0))
   (use (match_operand 1 "const_int_operand" "n"))
   (clobber (reg:CC CC_REGNUM))]
  "INTVAL (operands[1]) >= 1 && INTVAL (operands[1]) <= 256"
  "xc\t%O0(%1,%R0),%S0"
  [(set_attr "op_type" "SS")
   (set_attr "z196prop" "z196_cracked")])

(define_peephole2
  [(parallel
    [(set (match_operand:BLK 0 "memory_operand" "")
          (const_int 0))
     (use (match_operand 1 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (match_operand:BLK 2 "memory_operand" "")
          (const_int 0))
     (use (match_operand 3 "const_int_operand" ""))
     (clobber (reg:CC CC_REGNUM))])]
  "s390_offset_p (operands[0], operands[2], operands[1])
   && INTVAL (operands[1]) + INTVAL (operands[3]) <= 256"
  [(parallel
    [(set (match_dup 4) (const_int 0))
     (use (match_dup 5))
     (clobber (reg:CC CC_REGNUM))])]
  "operands[4] = gen_rtx_MEM (BLKmode, XEXP (operands[0], 0));
   operands[5] = GEN_INT (INTVAL (operands[1]) + INTVAL (operands[3]));")

;
;- Nxor instructions.
;

; nxrk, nxgrk
(define_insn "*nxor<GPR:mode>_cc"
  [(set (reg CC_REGNUM)
	(compare
	 (not:GPR (xor:GPR (match_operand:GPR 1 "register_operand" "d")
			   (match_operand:GPR 2 "register_operand" "d")))
	 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d")
	(xor:GPR (not:GPR (match_dup 1))
		    (match_dup 2)))]
  "TARGET_ARCH13 && s390_match_ccmode(insn, CCTmode)"
  "nx<GPR:g>rk\t%0,%1,%2"
  [(set_attr "op_type" "RRF")])

; nxrk, nxgrk
(define_insn "*nxor<mode>_cconly"
  [(set (reg CC_REGNUM)
	(compare
	 (not:GPR (xor:GPR (match_operand:GPR 1 "register_operand" "d")
			   (match_operand:GPR 2 "register_operand" "d")))
	 (const_int 0)))
   (clobber (match_scratch:GPR 0 "=d"))]
  "TARGET_ARCH13 && s390_match_ccmode(insn, CCTmode)"
  "nx<GPR:g>rk\t%0,%1,%2"
  [(set_attr "op_type" "RRF")])

; nxrk, nxgrk
(define_insn "*nxor<mode>"
  [(set (match_operand:GPR 0 "register_operand" "=d")
	(not:GPR (xor:GPR (match_operand:GPR 1 "register_operand" "d")
			  (match_operand:GPR 2 "register_operand" "d"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ARCH13"
  "nx<GPR:g>rk\t%0,%1,%2"
  [(set_attr "op_type" "RRF")])

;;
;;- Negate instructions.
;;

;
; neg(di|si)2 instruction pattern(s).
;

(define_expand "neg<mode>2"
  [(parallel
    [(set (match_operand:DSI 0 "register_operand" "=d")
          (neg:DSI (match_operand:DSI 1 "register_operand" "d")))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")

(define_insn "*negdi2_sign_cc"
  [(set (reg CC_REGNUM)
        (compare (neg:DI (ashiftrt:DI (ashift:DI (subreg:DI
                           (match_operand:SI 1 "register_operand" "d") 0)
                           (const_int 32)) (const_int 32)))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d")
        (neg:DI (sign_extend:DI (match_dup 1))))]
  "TARGET_ZARCH && s390_match_ccmode (insn, CCAmode)"
  "lcgfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "z10prop" "z10_c")])

(define_insn "*negdi2_sign"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (neg:DI (sign_extend:DI (match_operand:SI 1 "register_operand" "d"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "lcgfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "z10prop" "z10_c")])

; lcr, lcgr
(define_insn "*neg<mode>2_cc"
  [(set (reg CC_REGNUM)
        (compare (neg:GPR (match_operand:GPR 1 "register_operand" "d"))
                 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d")
        (neg:GPR (match_dup 1)))]
  "s390_match_ccmode (insn, CCAmode)"
  "lc<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_super_c_E1")])

; lcr, lcgr
(define_insn "*neg<mode>2_cconly"
  [(set (reg CC_REGNUM)
        (compare (neg:GPR (match_operand:GPR 1 "register_operand" "d"))
                 (const_int 0)))
   (clobber (match_scratch:GPR 0 "=d"))]
  "s390_match_ccmode (insn, CCAmode)"
  "lc<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_super_c_E1")])

; lcr, lcgr
(define_insn "*neg<mode>2"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (neg:GPR (match_operand:GPR 1 "register_operand" "d")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "lc<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_super_c_E1")])

(define_insn "*negdi2_31"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (neg:DI (match_operand:DI 1 "register_operand" "d")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH"
  "#")

; Split a DImode NEG on 31bit into 2 SImode NEGs

; Doing the twos complement separately on the SImode parts does an
; unwanted +1 on the high part which needs to be subtracted afterwards
; ... unless the +1 on the low part created an overflow.

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (neg:DI (match_operand:DI 1 "register_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH
   && (REGNO (operands[0]) == REGNO (operands[1])
      || s390_split_ok_p (operands[0], operands[1], DImode, 0))
   && reload_completed"
  [(parallel
    [(set (match_dup 2) (neg:SI (match_dup 3)))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (reg:CCAP CC_REGNUM)
          (compare:CCAP (neg:SI (match_dup 5)) (const_int 0)))
     (set (match_dup 4) (neg:SI (match_dup 5)))])
   (set (pc)
        (if_then_else (ne (reg:CCAP CC_REGNUM) (const_int 0))
                      (pc)
                      (label_ref (match_dup 6))))
   (parallel
    [(set (match_dup 2) (plus:SI (match_dup 2) (const_int -1)))
     (clobber (reg:CC CC_REGNUM))])
   (match_dup 6)]
  "operands[2] = operand_subword (operands[0], 0, 0, DImode);
   operands[3] = operand_subword (operands[1], 0, 0, DImode);
   operands[4] = operand_subword (operands[0], 1, 0, DImode);
   operands[5] = operand_subword (operands[1], 1, 0, DImode);
   operands[6] = gen_label_rtx ();")

; Like above but first make a copy of the low part of the src operand
; since it might overlap with the high part of the destination.

(define_split
  [(set (match_operand:DI 0 "register_operand" "")
        (neg:DI (match_operand:DI 1 "register_operand" "")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH
   && s390_split_ok_p (operands[0], operands[1], DImode, 1)
   && reload_completed"
  [; Make a backup of op5 first
   (set (match_dup 4) (match_dup 5))
   ; Setting op2 here might clobber op5
   (parallel
    [(set (match_dup 2) (neg:SI (match_dup 3)))
     (clobber (reg:CC CC_REGNUM))])
   (parallel
    [(set (reg:CCAP CC_REGNUM)
          (compare:CCAP (neg:SI (match_dup 4)) (const_int 0)))
     (set (match_dup 4) (neg:SI (match_dup 4)))])
   (set (pc)
        (if_then_else (ne (reg:CCAP CC_REGNUM) (const_int 0))
                      (pc)
                      (label_ref (match_dup 6))))
   (parallel
    [(set (match_dup 2) (plus:SI (match_dup 2) (const_int -1)))
     (clobber (reg:CC CC_REGNUM))])
   (match_dup 6)]
  "operands[2] = operand_subword (operands[0], 0, 0, DImode);
   operands[3] = operand_subword (operands[1], 0, 0, DImode);
   operands[4] = operand_subword (operands[0], 1, 0, DImode);
   operands[5] = operand_subword (operands[1], 1, 0, DImode);
   operands[6] = gen_label_rtx ();")

;
; neg(df|sf)2 instruction pattern(s).
;

(define_expand "neg<mode>2"
  [(parallel
    [(set (match_operand:BFP          0 "register_operand")
          (neg:BFP (match_operand:BFP 1 "register_operand")))
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_HARD_FLOAT")

; lcxbr, lcdbr, lcebr
(define_insn "*neg<mode>2_cc"
  [(set (reg CC_REGNUM)
        (compare (neg:BFP (match_operand:BFP 1 "register_operand" "f"))
                 (match_operand:BFP 2 "const0_operand" "")))
   (set (match_operand:BFP 0 "register_operand" "=f")
        (neg:BFP (match_dup 1)))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "lc<xde>br\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lcxbr, lcdbr, lcebr
(define_insn "*neg<mode>2_cconly"
  [(set (reg CC_REGNUM)
        (compare (neg:BFP (match_operand:BFP 1 "register_operand" "f"))
                 (match_operand:BFP 2 "const0_operand" "")))
   (clobber (match_scratch:BFP 0 "=f"))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "lc<xde>br\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lcdfr
(define_insn "*neg<mode>2_nocc"
  [(set (match_operand:FP 0 "register_operand"         "=f")
        (neg:FP (match_operand:FP 1 "register_operand" "<fT0>")))]
  "TARGET_DFP"
  "lcdfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lcxbr, lcdbr, lcebr
; FIXME: wflcdb does not clobber cc
; FIXME: Does wflcdb ever match here?
(define_insn "*neg<mode>2"
  [(set (match_operand:BFP          0 "register_operand" "=f,v,v")
        (neg:BFP (match_operand:BFP 1 "register_operand"  "f,v,v")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_FLOAT"
  "@
   lc<xde>br\t%0,%1
   wflcdb\t%0,%1
   wflcsb\t%0,%1"
  [(set_attr "op_type"      "RRE,VRR,VRR")
   (set_attr "cpu_facility" "*,vx,vxe")
   (set_attr "type"         "fsimp<mode>,*,*")
   (set_attr "enabled"      "*,<DF>,<SF>")])


;;
;;- Absolute value instructions.
;;

;
; abs(di|si)2 instruction pattern(s).
;

(define_insn "*absdi2_sign_cc"
  [(set (reg CC_REGNUM)
        (compare (abs:DI (ashiftrt:DI (ashift:DI (subreg:DI
                           (match_operand:SI 1 "register_operand" "d") 0)
                           (const_int 32)) (const_int 32)))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d")
        (abs:DI (sign_extend:DI (match_dup 1))))]
  "TARGET_ZARCH && s390_match_ccmode (insn, CCAmode)"
  "lpgfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "z10prop" "z10_c")])

(define_insn "*absdi2_sign"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (abs:DI (sign_extend:DI (match_operand:SI 1 "register_operand" "d"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "lpgfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "z10prop" "z10_c")])

; lpr, lpgr
(define_insn "*abs<mode>2_cc"
  [(set (reg CC_REGNUM)
        (compare (abs:GPR (match_operand:DI 1 "register_operand" "d"))
                 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d")
        (abs:GPR (match_dup 1)))]
  "s390_match_ccmode (insn, CCAmode)"
  "lp<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_c")])

; lpr, lpgr
(define_insn "*abs<mode>2_cconly"
  [(set (reg CC_REGNUM)
        (compare (abs:GPR (match_operand:GPR 1 "register_operand" "d"))
                 (const_int 0)))
   (clobber (match_scratch:GPR 0 "=d"))]
  "s390_match_ccmode (insn, CCAmode)"
  "lp<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_c")])

; lpr, lpgr
(define_insn "abs<mode>2"
  [(set (match_operand:GPR 0 "register_operand" "=d")
        (abs:GPR (match_operand:GPR 1 "register_operand" "d")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "lp<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_c")])

;
; abs(df|sf)2 instruction pattern(s).
;

(define_expand "abs<mode>2"
  [(parallel
    [(set (match_operand:BFP 0 "register_operand" "=f")
          (abs:BFP (match_operand:BFP 1 "register_operand" "f")))
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_HARD_FLOAT"
  "")

; lpxbr, lpdbr, lpebr
(define_insn "*abs<mode>2_cc"
  [(set (reg CC_REGNUM)
        (compare (abs:BFP (match_operand:BFP 1 "register_operand" "f"))
                 (match_operand:BFP 2 "const0_operand" "")))
   (set (match_operand:BFP 0 "register_operand" "=f")
        (abs:BFP (match_dup 1)))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "lp<xde>br\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lpxbr, lpdbr, lpebr
(define_insn "*abs<mode>2_cconly"
  [(set (reg CC_REGNUM)
        (compare (abs:BFP (match_operand:BFP 1 "register_operand" "f"))
                 (match_operand:BFP 2 "const0_operand" "")))
   (clobber (match_scratch:BFP 0 "=f"))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "lp<xde>br\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lpdfr
(define_insn "*abs<mode>2_nocc"
  [(set (match_operand:FP 0 "register_operand"         "=f")
        (abs:FP (match_operand:FP 1 "register_operand" "<fT0>")))]
  "TARGET_DFP"
  "lpdfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lpxbr, lpdbr, lpebr
; FIXME: wflpdb does not clobber cc
(define_insn "*abs<mode>2"
  [(set (match_operand:BFP          0 "register_operand" "=f,v")
        (abs:BFP (match_operand:BFP 1 "register_operand"  "f,v")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_FLOAT"
  "@
    lp<xde>br\t%0,%1
    wflpdb\t%0,%1"
  [(set_attr "op_type"      "RRE,VRR")
   (set_attr "cpu_facility" "*,vx")
   (set_attr "type"         "fsimp<mode>,*")
   (set_attr "enabled"      "*,<DFDI>")])


;;
;;- Negated absolute value instructions
;;

;
; Integer
;

(define_insn "*negabsdi2_sign_cc"
  [(set (reg CC_REGNUM)
        (compare (neg:DI (abs:DI (ashiftrt:DI (ashift:DI (subreg:DI
                           (match_operand:SI 1 "register_operand" "d") 0)
                           (const_int 32)) (const_int 32))))
                 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d")
        (neg:DI (abs:DI (sign_extend:DI (match_dup 1)))))]
  "TARGET_ZARCH && s390_match_ccmode (insn, CCAmode)"
  "lngfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "z10prop" "z10_c")])

(define_insn "*negabsdi2_sign"
  [(set (match_operand:DI 0 "register_operand" "=d")
	(neg:DI (abs:DI (sign_extend:DI
                          (match_operand:SI 1 "register_operand" "d")))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
  "lngfr\t%0,%1"
  [(set_attr "op_type" "RRE")
   (set_attr "z10prop" "z10_c")])

; lnr, lngr
(define_insn "*negabs<mode>2_cc"
  [(set (reg CC_REGNUM)
        (compare (neg:GPR (abs:GPR (match_operand:GPR 1 "register_operand" "d")))
                 (const_int 0)))
   (set (match_operand:GPR 0 "register_operand" "=d")
        (neg:GPR (abs:GPR (match_dup 1))))]
  "s390_match_ccmode (insn, CCAmode)"
  "ln<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_c")])

; lnr, lngr
(define_insn "*negabs<mode>2_cconly"
  [(set (reg CC_REGNUM)
        (compare (neg:GPR (abs:GPR (match_operand:GPR 1 "register_operand" "d")))
                 (const_int 0)))
   (clobber (match_scratch:GPR 0 "=d"))]
  "s390_match_ccmode (insn, CCAmode)"
  "ln<g>r\t%0,%1"
  [(set_attr "op_type"  "RR<E>")
   (set_attr "z10prop" "z10_c")])

; lnr, lngr
(define_insn "*negabs<mode>2"
  [(set (match_operand:GPR 0 "register_operand" "=d")
	(neg:GPR (abs:GPR (match_operand:GPR 1 "register_operand" "d"))))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "ln<g>r\t%0,%1"
  [(set_attr "op_type" "RR<E>")
   (set_attr "z10prop" "z10_c")])

;
; Floating point
;

; lnxbr, lndbr, lnebr
(define_insn "*negabs<mode>2_cc"
  [(set (reg CC_REGNUM)
        (compare (neg:BFP (abs:BFP (match_operand:BFP 1 "register_operand" "f")))
                 (match_operand:BFP 2 "const0_operand" "")))
   (set (match_operand:BFP 0 "register_operand" "=f")
        (neg:BFP (abs:BFP (match_dup 1))))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "ln<xde>br\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lnxbr, lndbr, lnebr
(define_insn "*negabs<mode>2_cconly"
  [(set (reg CC_REGNUM)
        (compare (neg:BFP (abs:BFP (match_operand:BFP 1 "register_operand" "f")))
                 (match_operand:BFP 2 "const0_operand" "")))
   (clobber (match_scratch:BFP 0 "=f"))]
  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT"
  "ln<xde>br\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lndfr
(define_insn "*negabs<mode>2_nocc"
  [(set (match_operand:FP 0 "register_operand"                  "=f")
        (neg:FP (abs:FP (match_operand:BFP 1 "register_operand" "<fT0>"))))]
  "TARGET_DFP"
  "lndfr\t%0,%1"
  [(set_attr "op_type"  "RRE")
   (set_attr "type"     "fsimp<mode>")])

; lnxbr, lndbr, lnebr
; FIXME: wflndb does not clobber cc
(define_insn "*negabs<mode>2"
  [(set (match_operand:BFP                   0 "register_operand" "=f,v")
        (neg:BFP (abs:BFP (match_operand:BFP 1 "register_operand"  "f,v"))))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_HARD_FLOAT"
  "@
   ln<xde>br\t%0,%1
   wflndb\t%0,%1"
  [(set_attr "op_type"      "RRE,VRR")
   (set_attr "cpu_facility" "*,vx")
   (set_attr "type"         "fsimp<mode>,*")
   (set_attr "enabled"      "*,<DFDI>")])

;;
;;- Square root instructions.
;;

;
; sqrt(df|sf)2 instruction pattern(s).
;

; sqxbr, sqdbr, sqebr, sqdb, sqeb
(define_insn "sqrt<mode>2"
  [(set (match_operand:BFP           0 "register_operand" "=f,f,v")
	(sqrt:BFP (match_operand:BFP 1 "general_operand"   "f,R,v")))]
  "TARGET_HARD_FLOAT"
  "@
   sq<xde>br\t%0,%1
   sq<xde>b\t%0,%1
   wfsqdb\t%v0,%v1"
  [(set_attr "op_type"      "RRE,RXE,VRR")
   (set_attr "type"         "fsqrt<mode>")
   (set_attr "cpu_facility" "*,*,vx")
   (set_attr "enabled"      "*,<DSF>,<DFDI>")])


;;
;;- One complement instructions.
;;

;
; one_cmpl(di|si|hi|qi)2 instruction pattern(s).
;

(define_expand "one_cmpl<mode>2"
  [(parallel
    [(set (match_operand:INT 0 "register_operand" "")
          (xor:INT (match_operand:INT 1 "register_operand" "")
		   (const_int -1)))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")


;;
;; Find leftmost bit instructions.
;;

(define_expand "clzdi2"
  [(set (match_operand:DI 0 "register_operand" "=d")
	(clz:DI (match_operand:DI 1 "register_operand" "d")))]
  "TARGET_EXTIMM && TARGET_ZARCH"
{
  rtx_insn *insn;
  rtx clz_equal;
  rtx wide_reg = gen_reg_rtx (TImode);
  rtx msb = gen_rtx_CONST_INT (DImode, HOST_WIDE_INT_1U << 63);

  clz_equal = gen_rtx_CLZ (DImode, operands[1]);

  emit_insn (gen_clztidi2 (wide_reg, operands[1], msb));

  insn = emit_move_insn (operands[0], gen_highpart (DImode, wide_reg));
  set_unique_reg_note (insn, REG_EQUAL, clz_equal);

  DONE;
})

; CLZ result is in hard reg op0 - this is the high part of the target operand
; The source with the left-most one bit cleared is in hard reg op0 + 1 - the low part
(define_insn "clztidi2"
  [(set (match_operand:TI 0 "register_operand" "=d")
	(ior:TI
	  (ashift:TI (zero_extend:TI (clz:DI (match_operand:DI 1 "register_operand" "d")))
		     (const_int 64))
	  (zero_extend:TI
	   (xor:DI (match_dup 1)
		   (lshiftrt (match_operand:DI 2 "const_int_operand" "")
			     (subreg:SI (clz:DI (match_dup 1)) 4))))))
   (clobber (reg:CC CC_REGNUM))]
  "UINTVAL (operands[2]) == HOST_WIDE_INT_1U << 63
   && TARGET_EXTIMM && TARGET_ZARCH"
  "flogr\t%0,%1"
  [(set_attr "op_type"  "RRE")])


;;
;;- Rotate instructions.
;;

;
; rotl(di|si)3 instruction pattern(s).
;

(define_expand "rotl<mode>3"
  [(set (match_operand:GPR 0 "register_operand" "")
        (rotate:GPR (match_operand:GPR 1 "register_operand" "")
		    (match_operand:SI 2 "nonmemory_operand" "")))]
  ""
  "")

; rll, rllg
(define_insn "*rotl<mode>3<addr_style_op><masked_op>"
  [(set (match_operand:GPR             0 "register_operand"  "=d")
	(rotate:GPR (match_operand:GPR 1 "register_operand"   "d")
		    (match_operand:SI  2 "nonmemory_operand" "an")))]
  ""
  "rll<g>\t%0,%1,<addr_style_op_ops>"
  [(set_attr "op_type"  "RSE")
   (set_attr "atype"    "reg")
   (set_attr "z10prop"  "z10_super_E1")])


;;
;;- Shift instructions.
;;

;
; (ashl|lshr)(di|si)3 instruction pattern(s).
; Left shifts and logical right shifts

(define_expand "<shift><mode>3"
  [(set (match_operand:DSI 0 "register_operand" "")
        (SHIFT:DSI (match_operand:DSI 1 "register_operand" "")
                   (match_operand:SI 2 "nonmemory_operand" "")))]
  ""
  "")

; ESA 64 bit register pair shift with reg or imm shift count
; sldl, srdl
(define_insn "*<shift>di3_31<addr_style_op><masked_op>"
  [(set (match_operand:DI 0 "register_operand"            "=d")
        (SHIFT:DI (match_operand:DI 1 "register_operand"   "0")
                  (match_operand:SI 2 "nonmemory_operand" "an")))]
  "!TARGET_ZARCH"
  "s<lr>dl\t%0,<addr_style_op_ops>"
  [(set_attr "op_type"  "RS")
   (set_attr "atype"    "reg")
   (set_attr "z196prop" "z196_cracked")])


; 64 bit register shift with reg or imm shift count
; sll, srl, sllg, srlg, sllk, srlk
(define_insn "*<shift><mode>3<addr_style_op><masked_op>"
  [(set (match_operand:GPR 0 "register_operand"              "=d, d")
        (SHIFT:GPR (match_operand:GPR 1 "register_operand" "<d0>, d")
                   (match_operand:SI 2 "nonmemory_operand"   "an,an")))]
  ""
  "@
   s<lr>l<g>\t%0,<1><addr_style_op_ops>
   s<lr>l<gk>\t%0,%1,<addr_style_op_ops>"
  [(set_attr "op_type"  "RS<E>,RSY")
   (set_attr "atype"    "reg,reg")
   (set_attr "cpu_facility" "*,z196")
   (set_attr "z10prop"  "z10_super_E1,*")])

;
; ashr(di|si)3 instruction pattern(s).
; Arithmetic right shifts

(define_expand "ashr<mode>3"
  [(parallel
    [(set (match_operand:DSI 0 "register_operand" "")
          (ashiftrt:DSI (match_operand:DSI 1 "register_operand" "")
                        (match_operand:SI 2 "nonmemory_operand" "")))
     (clobber (reg:CC CC_REGNUM))])]
  ""
  "")

; FIXME: The number of alternatives is doubled here to match the fix
; number of 2 in the subst pattern for the (clobber (match_scratch...
; The right fix should be to support match_scratch in the output
; pattern of a define_subst.
(define_insn "*ashrdi3_31<addr_style_op_cc><masked_op_cc><setcc><cconly>"
  [(set (match_operand:DI 0 "register_operand"               "=d, d")
        (ashiftrt:DI (match_operand:DI 1 "register_operand"   "0, 0")
                     (match_operand:SI 2 "nonmemory_operand" "an,an")))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH"
  "@
   srda\t%0,<addr_style_op_cc_ops>
   srda\t%0,<addr_style_op_cc_ops>"
  [(set_attr "op_type" "RS")
   (set_attr "atype"   "reg")])


; sra, srag
(define_insn "*ashr<mode>3<addr_style_op_cc><masked_op_cc><setcc><cconly>"
  [(set (match_operand:GPR 0 "register_operand"                 "=d, d")
        (ashiftrt:GPR (match_operand:GPR 1 "register_operand" "<d0>, d")
                      (match_operand:SI 2 "nonmemory_operand"   "an,an")))
   (clobber (reg:CC CC_REGNUM))]
  ""
  "@
   sra<g>\t%0,<1><addr_style_op_cc_ops>
   sra<gk>\t%0,%1,<addr_style_op_cc_ops>"
  [(set_attr "op_type"  "RS<E>,RSY")
   (set_attr "atype"    "reg")
   (set_attr "cpu_facility" "*,z196")
   (set_attr "z10prop" "z10_super_E1,*")])


;;
;; Branch instruction patterns.
;;

(define_expand "cbranch<mode>4"
  [(set (pc)
        (if_then_else (match_operator 0 "comparison_operator"
        	       [(match_operand:GPR 1 "register_operand" "")
                        (match_operand:GPR 2 "general_operand" "")])
		      (label_ref (match_operand 3 "" ""))
                      (pc)))]
  ""
  "s390_emit_jump (operands[3],
    s390_emit_compare (GET_CODE (operands[0]), operands[1], operands[2]));
   DONE;")

(define_expand "cbranch<mode>4"
  [(set (pc)
        (if_then_else (match_operator 0 "comparison_operator"
        	       [(match_operand:FP 1 "register_operand" "")
                        (match_operand:FP 2 "general_operand" "")])
		      (label_ref (match_operand 3 "" ""))
                      (pc)))]
  "TARGET_HARD_FLOAT"
  "s390_emit_jump (operands[3],
    s390_emit_compare (GET_CODE (operands[0]), operands[1], operands[2]));
   DONE;")

(define_expand "cbranchcc4"
  [(set (pc)
        (if_then_else (match_operator 0 "s390_comparison"
        	       [(match_operand 1 "cc_reg_operand" "")
                        (match_operand 2 "const_int_operand" "")])
		      (label_ref (match_operand 3 "" ""))
                      (pc)))]
  ""
  "")


;;
;;- Conditional jump instructions.
;;

(define_insn "*cjump_64"
  [(set (pc)
        (if_then_else
          (match_operator 1 "s390_comparison" [(reg CC_REGNUM)
					       (match_operand 2 "const_int_operand" "")])
          (label_ref (match_operand 0 "" ""))
          (pc)))]
  ""
{
  if (get_attr_length (insn) == 4)
    return "j%C1\t%l0";
  else
    return "jg%C1\t%l0";
}
  [(set_attr "op_type" "RI")
   (set_attr "type"    "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 4) (const_int 6)))])

(define_insn "*cjump_long"
  [(set (pc)
        (if_then_else
          (match_operator 1 "s390_comparison" [(reg CC_REGNUM) (const_int 0)])
          (match_operand 0 "address_operand" "ZQZR")
          (pc)))]
  "!TARGET_INDIRECT_BRANCH_NOBP_JUMP"
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "b%C1r\t%0";
  else
    return "b%C1\t%a0";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set (attr "mnemonic")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "bcr") (const_string "bc")))
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])

;; A conditional return instruction.
(define_insn "*c<code>"
  [(set (pc)
        (if_then_else
          (match_operator 0 "s390_comparison" [(reg CC_REGNUM) (const_int 0)])
          (ANY_RETURN)
          (pc)))]
  "s390_can_use_<code>_insn ()"
{
  if (TARGET_INDIRECT_BRANCH_NOBP_RET)
    {
      s390_indirect_branch_via_thunk (RETURN_REGNUM,
				      INVALID_REGNUM,
				      operands[0],
				      s390_indirect_branch_type_return);
      return "";
    }
  else
    return "b%C0r\t%%r14";
}
  [(set (attr "op_type")
	(if_then_else (match_test "TARGET_INDIRECT_BRANCH_NOBP_RET")
		      (const_string "RIL")
		      (const_string "RR")))
   (set (attr "mnemonic")
	(if_then_else (match_test "TARGET_INDIRECT_BRANCH_NOBP_RET")
		      (const_string "brcl")
		      (const_string "bcr")))
   (set_attr "type"  "jsr")
   (set_attr "atype" "agen")])

;;
;;- Negated conditional jump instructions.
;;

(define_insn "*icjump_64"
  [(set (pc)
        (if_then_else
          (match_operator 1 "s390_comparison" [(reg CC_REGNUM) (const_int 0)])
          (pc)
          (label_ref (match_operand 0 "" ""))))]
  ""
{
  if (get_attr_length (insn) == 4)
    return "j%D1\t%l0";
  else
    return "jg%D1\t%l0";
}
  [(set_attr "op_type" "RI")
   (set_attr "type"    "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 4) (const_int 6)))])

(define_insn "*icjump_long"
  [(set (pc)
        (if_then_else
          (match_operator 1 "s390_comparison" [(reg CC_REGNUM) (const_int 0)])
          (pc)
          (match_operand 0 "address_operand" "ZQZR")))]
  "!TARGET_INDIRECT_BRANCH_NOBP_JUMP"
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "b%D1r\t%0";
  else
    return "b%D1\t%a0";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set (attr "mnemonic")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "bcr") (const_string "bc")))
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])

;;
;;- Trap instructions.
;;

(define_insn "trap"
  [(trap_if (const_int 1) (const_int 0))]
  ""
  "j\t.+2"
  [(set_attr "op_type" "RI")
   (set_attr "type"  "branch")])

(define_expand "ctrap<mode>4"
  [(trap_if (match_operator 0 "comparison_operator"
             [(match_operand:GPR 1 "register_operand" "")
              (match_operand:GPR 2 "general_operand" "")])
	     (match_operand 3 "const0_operand" ""))]
  ""
  {
    rtx cond = s390_emit_compare (GET_CODE (operands[0]),
                                  operands[1], operands[2]);
    emit_insn (gen_condtrap (cond, XEXP (cond, 0)));
    DONE;
  })

(define_expand "ctrap<mode>4"
  [(trap_if (match_operator 0 "comparison_operator"
             [(match_operand:FP 1 "register_operand" "")
              (match_operand:FP 2 "general_operand" "")])
	     (match_operand 3 "const0_operand" ""))]
  ""
  {
    rtx cond = s390_emit_compare (GET_CODE (operands[0]),
                                  operands[1], operands[2]);
    emit_insn (gen_condtrap (cond, XEXP (cond, 0)));
    DONE;
  })

(define_insn "condtrap"
  [(trap_if (match_operator 0 "s390_comparison"
             [(match_operand 1 "cc_reg_operand" "c")
              (const_int 0)])
	    (const_int 0))]
  ""
  "j%C0\t.+2";
  [(set_attr "op_type" "RI")
   (set_attr "type"  "branch")])

; crt, cgrt, cit, cgit
(define_insn "*cmp_and_trap_signed_int<mode>"
  [(trap_if (match_operator 0 "s390_signed_integer_comparison"
	       [(match_operand:GPR 1 "register_operand"  "d,d")
		(match_operand:GPR 2 "nonmemory_operand" "d,K")])
	    (const_int 0))]
  "TARGET_Z10"
  "@
   c<g>rt%C0\t%1,%2
   c<g>it%C0\t%1,%h2"
  [(set_attr "op_type" "RRF,RIE")
   (set_attr "type"    "branch")
   (set_attr "z10prop" "z10_super_c,z10_super")])

; clrt, clgrt, clfit, clgit, clt, clgt
(define_insn "*cmp_and_trap_unsigned_int<mode>"
  [(trap_if (match_operator 0 "s390_unsigned_integer_comparison"
	       [(match_operand:GPR 1 "register_operand" "d,d,d")
		(match_operand:GPR 2 "general_operand"  "d,D,T")])
	    (const_int 0))]
  "TARGET_Z10"
  "@
   cl<g>rt%C0\t%1,%2
   cl<gf>it%C0\t%1,%x2
   cl<g>t%C0\t%1,%2"
  [(set_attr "op_type"      "RRF,RIE,RSY")
   (set_attr "type"         "branch")
   (set_attr "z10prop"      "z10_super_c,z10_super,*")
   (set_attr "cpu_facility" "z10,z10,zEC12")])

; lat, lgat
(define_insn "*load_and_trap<mode>"
  [(trap_if (eq (match_operand:GPR 0 "memory_operand"  "T")
		(const_int 0))
	    (const_int 0))
   (set (match_operand:GPR 1 "register_operand" "=d")
	(match_dup 0))]
  "TARGET_ZEC12"
  "l<g>at\t%1,%0"
  [(set_attr "op_type" "RXY")])


;;
;;- Loop instructions.
;;
;;  This is all complicated by the fact that since this is a jump insn
;;  we must handle our own output reloads.

;; branch on index

; This splitter will be matched by combine and has to add the 2 moves
; necessary to load the compare and the increment values into a
; register pair as needed by brxle.

(define_insn_and_split "*brx_stage1_<GPR:mode>"
  [(set (pc)
        (if_then_else
	 (match_operator 6 "s390_brx_operator"
	    [(plus:GPR (match_operand:GPR 1 "register_operand" "")
		       (match_operand:GPR 2 "general_operand"  ""))
	     (match_operand:GPR 3 "register_operand" "")])
	 (label_ref (match_operand 0 "" ""))
	 (pc)))
   (set (match_operand:GPR 4 "nonimmediate_operand" "")
        (plus:GPR (match_dup 1) (match_dup 2)))
   (clobber (match_scratch:GPR 5 ""))]
  ""
  "#"
  "!reload_completed && !reload_in_progress"
  [(set (match_dup 7) (match_dup 2)) ; the increment
   (set (match_dup 8) (match_dup 3)) ; the comparison value
   (parallel [(set (pc)
		   (if_then_else
		    (match_op_dup 6
		       [(plus:GPR (match_dup 1) (match_dup 7))
			(match_dup 8)])
		    (label_ref (match_dup 0))
		    (pc)))
	      (set (match_dup 4)
		   (plus:GPR (match_dup 1) (match_dup 7)))
	      (clobber (match_dup 5))
	      (clobber (reg:CC CC_REGNUM))])]
  {
    rtx dreg = gen_reg_rtx (word_mode == DImode ? TImode : DImode);
    operands[7] = gen_lowpart (<GPR:MODE>mode,
			       gen_highpart (word_mode, dreg));
    operands[8] = gen_lowpart (<GPR:MODE>mode,
			       gen_lowpart (word_mode, dreg));
  })

; brxlg, brxhg

(define_insn_and_split "*brxg_64bit"
  [(set (pc)
        (if_then_else
          (match_operator 5 "s390_brx_operator"
	     [(plus:DI (match_operand:DI 1 "register_operand" "d,d,d")
		       (subreg:DI (match_operand:TI 2 "register_operand" "d,d,d") 0))
              (subreg:DI (match_dup 2) 8)])
          (label_ref (match_operand 0 "" ""))
          (pc)))
   (set (match_operand:DI 3 "nonimmediate_operand" "=1,?X,?X")
        (plus:DI (match_dup 1)
		 (subreg:DI (match_dup 2) 0)))
   (clobber (match_scratch:DI 4 "=X,&1,&?d"))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
{
  if (which_alternative != 0)
    return "#";
  else if (get_attr_length (insn) == 6)
    return "brx%E5g\t%1,%2,%l0";
  else
    return "agr\t%1,%2\;cgr\t%1,%M2\;jg%C5\t%l0";
}
  "&& reload_completed
   && (!REG_P (operands[3])
       || !rtx_equal_p (operands[1], operands[3]))"
  [(set (match_dup 4) (match_dup 1))
   (parallel [(set (match_dup 4) (plus:DI (match_dup 4) (subreg:DI (match_dup 2) 0)))
	      (clobber (reg:CC CC_REGNUM))])
   (set (reg:CCS CC_REGNUM) (compare:CCS (match_dup 4) (subreg:DI (match_dup 2) 8)))
   (set (match_dup 3) (match_dup 4))
   (set (pc) (if_then_else (match_op_dup 5 [(reg:CCS CC_REGNUM) (const_int 0)])
			   (label_ref (match_dup 0))
			   (pc)))]
  ""
  [(set_attr "op_type"  "RIE")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 6) (const_int 16)))])

; brxle, brxh

(define_insn_and_split "*brx_64bit"
  [(set (pc)
        (if_then_else
          (match_operator 5 "s390_brx_operator"
	     [(plus:SI (match_operand:SI 1 "register_operand" "d,d,d")
		       (subreg:SI (match_operand:TI 2 "register_operand" "d,d,d") 4))
              (subreg:SI (match_dup 2) 12)])
          (label_ref (match_operand 0 "" ""))
          (pc)))
   (set (match_operand:SI 3 "nonimmediate_operand" "=1,?X,?X")
        (plus:SI (match_dup 1)
		 (subreg:SI (match_dup 2) 4)))
   (clobber (match_scratch:SI 4 "=X,&1,&?d"))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
{
  if (which_alternative != 0)
    return "#";
  else if (get_attr_length (insn) == 6)
    return "brx%C5\t%1,%2,%l0";
  else
    return "ar\t%1,%2\;cr\t%1,%M2\;jg%C5\t%l0";
}
  "&& reload_completed
   && (!REG_P (operands[3])
       || !rtx_equal_p (operands[1], operands[3]))"
  [(set (match_dup 4) (match_dup 1))
   (parallel [(set (match_dup 4) (plus:SI (match_dup 4) (subreg:SI (match_dup 2) 4)))
	      (clobber (reg:CC CC_REGNUM))])
   (set (reg:CCS CC_REGNUM) (compare:CCS (match_dup 4) (subreg:SI (match_dup 2) 12)))
   (set (match_dup 3) (match_dup 4))
   (set (pc) (if_then_else (match_op_dup 5 [(reg:CCS CC_REGNUM) (const_int 0)])
			   (label_ref (match_dup 0))
			   (pc)))]
  ""
  [(set_attr "op_type"  "RSI")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 6) (const_int 14)))])

; brxle, brxh

(define_insn_and_split "*brx_31bit"
  [(set (pc)
        (if_then_else
          (match_operator 5 "s390_brx_operator"
	    [(plus:SI (match_operand:SI 1 "register_operand" "d,d,d")
		      (subreg:SI (match_operand:DI 2 "register_operand" "d,d,d") 0))
	     (subreg:SI (match_dup 2) 4)])
          (label_ref (match_operand 0 "" ""))
          (pc)))
   (set (match_operand:SI 3 "nonimmediate_operand" "=1,?X,?X")
        (plus:SI (match_dup 1)
		 (subreg:SI (match_dup 2) 0)))
   (clobber (match_scratch:SI 4 "=X,&1,&?d"))
   (clobber (reg:CC CC_REGNUM))]
  "!TARGET_ZARCH"
{
  if (which_alternative != 0)
    return "#";
  else if (get_attr_length (insn) == 6)
    return "brx%C5\t%1,%2,%l0";
  else
    return "ar\t%1,%2\;cr\t%1,%M2\;jg%C5\t%l0";
}
  "&& reload_completed
   && (!REG_P (operands[3])
       || !rtx_equal_p (operands[1], operands[3]))"
  [(set (match_dup 4) (match_dup 1))
   (parallel [(set (match_dup 4) (plus:SI (match_dup 4) (subreg:SI (match_dup 2) 0)))
	      (clobber (reg:CC CC_REGNUM))])
   (set (reg:CCS CC_REGNUM) (compare:CCS (match_dup 4) (subreg:SI (match_dup 2) 4)))
   (set (match_dup 3) (match_dup 4))
   (set (pc) (if_then_else (match_op_dup 5 [(reg:CCS CC_REGNUM) (const_int 0)])
			   (label_ref (match_dup 0))
			   (pc)))]
  ""
  [(set_attr "op_type"  "RSI")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 6) (const_int 14)))])


;; branch on count

(define_expand "doloop_end"
  [(use (match_operand 0 "" ""))        ; loop pseudo
   (use (match_operand 1 "" ""))]       ; label
  ""
{
  if (GET_MODE (operands[0]) == SImode)
    emit_jump_insn (gen_doloop_si64 (operands[1], operands[0], operands[0]));
  else if (GET_MODE (operands[0]) == DImode && TARGET_ZARCH)
    emit_jump_insn (gen_doloop_di (operands[1], operands[0], operands[0]));
  else
    FAIL;

  DONE;
})

(define_insn_and_split "doloop_si64"
  [(set (pc)
        (if_then_else
          (ne (match_operand:SI 1 "register_operand" "d,d,d")
              (const_int 1))
          (label_ref (match_operand 0 "" ""))
          (pc)))
   (set (match_operand:SI 2 "nonimmediate_operand" "=1,?X,?X")
        (plus:SI (match_dup 1) (const_int -1)))
   (clobber (match_scratch:SI 3 "=X,&1,&?d"))
   (clobber (reg:CC CC_REGNUM))]
  ""
{
  if (which_alternative != 0)
    return "#";
  else if (get_attr_length (insn) == 4)
    return "brct\t%1,%l0";
  else
    return "ahi\t%1,-1\;jgne\t%l0";
}
  "&& reload_completed
   && (! REG_P (operands[2])
       || ! rtx_equal_p (operands[1], operands[2]))"
  [(set (match_dup 3) (match_dup 1))
   (parallel [(set (reg:CCAN CC_REGNUM)
                   (compare:CCAN (plus:SI (match_dup 3) (const_int -1))
                                 (const_int 0)))
              (set (match_dup 3) (plus:SI (match_dup 3) (const_int -1)))])
   (set (match_dup 2) (match_dup 3))
   (set (pc) (if_then_else (ne (reg:CCAN CC_REGNUM) (const_int 0))
                           (label_ref (match_dup 0))
                           (pc)))]
  ""
  [(set_attr "op_type"  "RI")
   ; Strictly speaking, the z10 properties are valid for brct only, however, it does not
   ; hurt us in the (rare) case of ahi.
   (set_attr "z10prop"  "z10_super_E1")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 4) (const_int 10)))])

(define_insn_and_split "doloop_di"
  [(set (pc)
        (if_then_else
          (ne (match_operand:DI 1 "register_operand" "d,d,d")
              (const_int 1))
          (label_ref (match_operand 0 "" ""))
          (pc)))
   (set (match_operand:DI 2 "nonimmediate_operand" "=1,?X,?X")
        (plus:DI (match_dup 1) (const_int -1)))
   (clobber (match_scratch:DI 3 "=X,&1,&?d"))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ZARCH"
{
  if (which_alternative != 0)
    return "#";
  else if (get_attr_length (insn) == 4)
    return "brctg\t%1,%l0";
  else
    return "aghi\t%1,-1\;jgne\t%l0";
}
  "&& reload_completed
   && (! REG_P (operands[2])
       || ! rtx_equal_p (operands[1], operands[2]))"
  [(set (match_dup 3) (match_dup 1))
   (parallel [(set (reg:CCAN CC_REGNUM)
                   (compare:CCAN (plus:DI (match_dup 3) (const_int -1))
                                 (const_int 0)))
              (set (match_dup 3) (plus:DI (match_dup 3) (const_int -1)))])
   (set (match_dup 2) (match_dup 3))
   (set (pc) (if_then_else (ne (reg:CCAN CC_REGNUM) (const_int 0))
                           (label_ref (match_dup 0))
                           (pc)))]
  ""
  [(set_attr "op_type"  "RI")
   ; Strictly speaking, the z10 properties are valid for brct only, however, it does not
   ; hurt us in the (rare) case of ahi.
   (set_attr "z10prop"  "z10_super_E1")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 4) (const_int 10)))])

;;
;;- Unconditional jump instructions.
;;

;
; jump instruction pattern(s).
;

(define_expand "jump"
  [(match_operand 0 "" "")]
  ""
  "s390_emit_jump (operands[0], NULL_RTX); DONE;")

(define_insn "*jump64"
  [(set (pc) (label_ref (match_operand 0 "" "")))]
  ""
{
  if (get_attr_length (insn) == 4)
    return "j\t%l0";
  else
    return "jg\t%l0";
}
  [(set_attr "op_type" "RI")
   (set_attr "type"  "branch")
   (set (attr "length")
        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
                      (const_int 4) (const_int 6)))])

;
; indirect-jump instruction pattern(s).
;

(define_expand "indirect_jump"
  [(set (pc) (match_operand 0 "nonimmediate_operand" ""))]
  ""
{
  if (address_operand (operands[0], GET_MODE (operands[0])))
    ;
  else if (TARGET_Z14
	   && GET_MODE (operands[0]) == Pmode
	   && memory_operand (operands[0], Pmode))
    ;
  else
    operands[0] = force_reg (Pmode, operands[0]);

  if (TARGET_INDIRECT_BRANCH_NOBP_JUMP_THUNK)
    {
      operands[0] = force_reg (Pmode, operands[0]);
      if (TARGET_CPU_Z10)
	{
	  if (TARGET_64BIT)
	    emit_jump_insn (gen_indirect_jump_via_thunkdi_z10 (operands[0]));
	  else
	    emit_jump_insn (gen_indirect_jump_via_thunksi_z10 (operands[0]));
	}
      else
	{
	  if (TARGET_64BIT)
	    emit_jump_insn (gen_indirect_jump_via_thunkdi (operands[0]));
	  else
	    emit_jump_insn (gen_indirect_jump_via_thunksi (operands[0]));
	}
      DONE;
    }

  if (TARGET_INDIRECT_BRANCH_NOBP_JUMP_INLINE_THUNK)
    {
      operands[0] = force_reg (Pmode, operands[0]);
      rtx label_ref = gen_rtx_LABEL_REF (VOIDmode, gen_label_rtx ());
      if (TARGET_CPU_Z10)
	{
	  if (TARGET_64BIT)
	    emit_jump_insn (gen_indirect_jump_via_inlinethunkdi_z10 (operands[0],
								     label_ref));
	  else
	    emit_jump_insn (gen_indirect_jump_via_inlinethunksi_z10 (operands[0],
								     label_ref));
	}
      else
	{
	  if (TARGET_64BIT)
	    emit_jump_insn (gen_indirect_jump_via_inlinethunkdi (operands[0],
								 label_ref,
								 force_reg (Pmode, label_ref)));
	  else
	    emit_jump_insn (gen_indirect_jump_via_inlinethunksi (operands[0],
								 label_ref,
								 force_reg (Pmode, label_ref)));
	}
      DONE;
    }
})

(define_insn "*indirect_jump"
  [(set (pc)
	(match_operand 0 "address_operand" "ZR"))]
 "!TARGET_INDIRECT_BRANCH_NOBP_JUMP_THUNK"
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "br\t%0";
  else
    return "b\t%a0";
}
 [(set (attr "op_type")
       (if_then_else (match_operand 0 "register_operand" "")
		     (const_string "RR") (const_string "RX")))
  (set (attr "mnemonic")
       (if_then_else (match_operand 0 "register_operand" "")
		     (const_string "br") (const_string "b")))
  (set_attr "type"  "branch")
  (set_attr "atype" "agen")])

(define_insn "indirect_jump_via_thunk<mode>_z10"
  [(set (pc)
	(match_operand:P 0 "register_operand" "a"))]
 "TARGET_INDIRECT_BRANCH_NOBP_JUMP_THUNK
  && TARGET_CPU_Z10"
{
  s390_indirect_branch_via_thunk (REGNO (operands[0]),
				  INVALID_REGNUM,
				  NULL_RTX,
				  s390_indirect_branch_type_jump);
  return "";
}
 [(set_attr "op_type"  "RIL")
  (set_attr "mnemonic" "jg")
  (set_attr "type"  "branch")
  (set_attr "atype" "agen")])

(define_insn "indirect_jump_via_thunk<mode>"
  [(set (pc)
	(match_operand:P 0 "register_operand" " a"))
   (clobber (reg:P INDIRECT_BRANCH_THUNK_REGNUM))]
 "TARGET_INDIRECT_BRANCH_NOBP_JUMP_THUNK
  && !TARGET_CPU_Z10"
{
  s390_indirect_branch_via_thunk (REGNO (operands[0]),
				  INVALID_REGNUM,
				  NULL_RTX,
				  s390_indirect_branch_type_jump);
  return "";
}
 [(set_attr "op_type"  "RIL")
  (set_attr "mnemonic" "jg")
  (set_attr "type"  "branch")
  (set_attr "atype" "agen")])


; The label_ref is wrapped into an if_then_else in order to hide it
; from mark_jump_label.  Without this the label_ref would become the
; ONLY jump target of that jump breaking the control flow graph.
(define_insn "indirect_jump_via_inlinethunk<mode>_z10"
  [(unspec [(if_then_else (match_operand:P 1 "larl_operand" "X")
			  (const_int 0)
			  (const_int 0))
	    (const_int 0)] UNSPEC_EXECUTE_JUMP)
   (set (pc) (match_operand:P 0 "register_operand" "a"))]
  "TARGET_INDIRECT_BRANCH_NOBP_JUMP_INLINE_THUNK
   && TARGET_CPU_Z10"
{
  s390_indirect_branch_via_inline_thunk (operands[1]);
  return "";
}
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "branch")
   (set_attr "length"  "10")])

(define_insn "indirect_jump_via_inlinethunk<mode>"
  [(unspec [(if_then_else (match_operand:P 1 "larl_operand" "X")
			  (const_int 0)
			  (const_int 0))
	    (match_operand:P 2 "register_operand" "a")] UNSPEC_EXECUTE_JUMP)
   (set (pc) (match_operand:P 0 "register_operand" "a"))]
  "TARGET_INDIRECT_BRANCH_NOBP_JUMP_INLINE_THUNK
   && !TARGET_CPU_Z10"
{
  s390_indirect_branch_via_inline_thunk (operands[2]);
  return "";
}
  [(set_attr "op_type" "RX")
   (set_attr "type"    "branch")
   (set_attr "length"  "8")])

; FIXME: LRA does not appear to be able to deal with MEMs being
; checked against address constraints like ZR above.  So make this a
; separate pattern for now.
(define_insn "*indirect2_jump"
  [(set (pc)
	(match_operand 0 "nonimmediate_operand" "a,T"))]
 "!TARGET_INDIRECT_BRANCH_NOBP_JUMP"
 "@
  br\t%0
  bi\t%0"
 [(set_attr "op_type" "RR,RXY")
  (set_attr "type"  "branch")
  (set_attr "atype" "agen")
  (set_attr "cpu_facility" "*,z14")])

;
; casesi instruction pattern(s).
;

(define_expand "casesi_jump"
  [(parallel
    [(set (pc) (match_operand 0 "address_operand"))
     (use (label_ref (match_operand 1 "")))])]
  ""
{
  if (TARGET_INDIRECT_BRANCH_NOBP_JUMP_THUNK)
    {
      operands[0] = force_reg (GET_MODE (operands[0]), operands[0]);

      if (TARGET_CPU_Z10)
	{
	  if (TARGET_64BIT)
	    emit_jump_insn (gen_casesi_jump_via_thunkdi_z10 (operands[0],
							     operands[1]));
	  else
	    emit_jump_insn (gen_casesi_jump_via_thunksi_z10 (operands[0],
							     operands[1]));
	}
      else
	{
	  if (TARGET_64BIT)
	    emit_jump_insn (gen_casesi_jump_via_thunkdi (operands[0],
							 operands[1]));
	  else
	    emit_jump_insn (gen_casesi_jump_via_thunksi (operands[0],
							 operands[1]));
	}
      DONE;
    }

    if (TARGET_INDIRECT_BRANCH_NOBP_JUMP_INLINE_THUNK)
    {
      operands[0] = force_reg (Pmode, operands[0]);
      rtx label_ref = gen_rtx_LABEL_REF (VOIDmode, gen_label_rtx ());
      if (TARGET_CPU_Z10)
	{
	  if (TARGET_64BIT)
	    emit_jump_insn (gen_casesi_jump_via_inlinethunkdi_z10 (operands[0],
								   operands[1],
								   label_ref));
	  else
	    emit_jump_insn (gen_casesi_jump_via_inlinethunksi_z10 (operands[0],
								   operands[1],
								   label_ref));
	}
      else
	{
	  if (TARGET_64BIT)
	    emit_jump_insn (gen_casesi_jump_via_inlinethunkdi (operands[0],
							       operands[1],
							       label_ref,
							       force_reg (Pmode, label_ref)));
	  else
	    emit_jump_insn (gen_casesi_jump_via_inlinethunksi (operands[0],
							       operands[1],
							       label_ref,
							       force_reg (Pmode, label_ref)));
	}
      DONE;
    }
})

(define_insn "*casesi_jump"
 [(set (pc) (match_operand 0 "address_operand" "ZR"))
  (use (label_ref (match_operand 1 "" "")))]
 "!TARGET_INDIRECT_BRANCH_NOBP_JUMP_THUNK"
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "br\t%0";
  else
    return "b\t%a0";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set (attr "mnemonic")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "br") (const_string "b")))
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])

(define_insn "casesi_jump_via_thunk<mode>_z10"
 [(set (pc) (match_operand:P 0 "register_operand" "a"))
  (use (label_ref (match_operand 1 "" "")))]
 "TARGET_INDIRECT_BRANCH_NOBP_JUMP_THUNK
  && TARGET_CPU_Z10"
{
  s390_indirect_branch_via_thunk (REGNO (operands[0]),
				  INVALID_REGNUM,
				  NULL_RTX,
				  s390_indirect_branch_type_jump);
  return "";
}
  [(set_attr "op_type" "RIL")
   (set_attr "mnemonic" "jg")
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])

(define_insn "casesi_jump_via_thunk<mode>"
 [(set (pc) (match_operand:P 0 "register_operand" "a"))
  (use (label_ref (match_operand 1 "" "")))
  (clobber (reg:P INDIRECT_BRANCH_THUNK_REGNUM))]
 "TARGET_INDIRECT_BRANCH_NOBP_JUMP_THUNK
  && !TARGET_CPU_Z10"
{
  s390_indirect_branch_via_thunk (REGNO (operands[0]),
				  INVALID_REGNUM,
				  NULL_RTX,
				  s390_indirect_branch_type_jump);
  return "";
}
  [(set_attr "op_type" "RIL")
   (set_attr "mnemonic" "jg")
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])


; The label_ref is wrapped into an if_then_else in order to hide it
; from mark_jump_label.  Without this the label_ref would become the
; ONLY jump target of that jump breaking the control flow graph.
(define_insn "casesi_jump_via_inlinethunk<mode>_z10"
  [(unspec [(if_then_else (match_operand:P 2 "larl_operand" "X")
			  (const_int 0)
			  (const_int 0))
	    (const_int 0)] UNSPEC_EXECUTE_JUMP)
   (set (pc) (match_operand:P 0 "register_operand" "a"))
   (use (label_ref (match_operand 1 "" "")))]
  "TARGET_INDIRECT_BRANCH_NOBP_JUMP_INLINE_THUNK
   && TARGET_CPU_Z10"
{
  s390_indirect_branch_via_inline_thunk (operands[2]);
  return "";
}
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "cs")
   (set_attr "length"  "10")])

(define_insn "casesi_jump_via_inlinethunk<mode>"
  [(unspec [(if_then_else (match_operand:P 2 "larl_operand" "X")
			  (const_int 0)
			  (const_int 0))
	    (match_operand:P 3 "register_operand" "a")] UNSPEC_EXECUTE_JUMP)
   (set (pc) (match_operand:P 0 "register_operand" "a"))
   (use (label_ref (match_operand 1 "" "")))]
  "TARGET_INDIRECT_BRANCH_NOBP_JUMP_INLINE_THUNK
   && !TARGET_CPU_Z10"
{
  s390_indirect_branch_via_inline_thunk (operands[3]);
  return "";
}
  [(set_attr "op_type" "RX")
   (set_attr "type"    "cs")
   (set_attr "length"  "8")])

(define_expand "casesi"
  [(match_operand:SI 0 "general_operand" "")
   (match_operand:SI 1 "general_operand" "")
   (match_operand:SI 2 "general_operand" "")
   (label_ref (match_operand 3 "" ""))
   (label_ref (match_operand 4 "" ""))]
  ""
{
   rtx index  = gen_reg_rtx (SImode);
   rtx base   = gen_reg_rtx (Pmode);
   rtx target = gen_reg_rtx (Pmode);

   emit_move_insn (index, operands[0]);
   emit_insn (gen_subsi3 (index, index, operands[1]));
   emit_cmp_and_jump_insns (index, operands[2], GTU, NULL_RTX, SImode, 1,
                            operands[4]);

   if (Pmode != SImode)
     index = convert_to_mode (Pmode, index, 1);
   if (GET_CODE (index) != REG)
     index = copy_to_mode_reg (Pmode, index);

   if (TARGET_64BIT)
       emit_insn (gen_ashldi3 (index, index, GEN_INT (3)));
   else
       emit_insn (gen_ashlsi3 (index, index, const2_rtx));

   emit_move_insn (base, gen_rtx_LABEL_REF (Pmode, operands[3]));

   index = gen_const_mem (Pmode, gen_rtx_PLUS (Pmode, base, index));
   emit_move_insn (target, index);

   if (flag_pic)
     target = gen_rtx_PLUS (Pmode, base, target);
   emit_jump_insn (gen_casesi_jump (target, operands[3]));

   DONE;
})


;;
;;- Jump to subroutine.
;;
;;

;
; untyped call instruction pattern(s).
;

;; Call subroutine returning any type.
(define_expand "untyped_call"
  [(parallel [(call (match_operand 0 "" "")
                    (const_int 0))
              (match_operand 1 "" "")
              (match_operand 2 "" "")])]
  ""
{
  int i;

  emit_call_insn (gen_call (operands[0], const0_rtx, const0_rtx));

  for (i = 0; i < XVECLEN (operands[2], 0); i++)
    {
      rtx set = XVECEXP (operands[2], 0, i);
      emit_move_insn (SET_DEST (set), SET_SRC (set));
    }

  /* The optimizer does not know that the call sets the function value
     registers we stored in the result block.  We avoid problems by
     claiming that all hard registers are used and clobbered at this
     point.  */
  emit_insn (gen_blockage ());

  DONE;
})

;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
;; all of memory.  This blocks insns from being moved across this point.

(define_insn "blockage"
  [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
  ""
  ""
  [(set_attr "type"    "none")
   (set_attr "length"  "0")])

;
; sibcall patterns
;

(define_expand "sibcall"
  [(call (match_operand 0 "" "")
	 (match_operand 1 "" ""))]
  ""
{
  s390_emit_call (XEXP (operands[0], 0), NULL_RTX, NULL_RTX, NULL_RTX);
  DONE;
})

(define_insn "*sibcall_br"
  [(call (mem:QI (reg SIBCALL_REGNUM))
         (match_operand 0 "const_int_operand" "n"))]
  "SIBLING_CALL_P (insn)
   && GET_MODE (XEXP (XEXP (PATTERN (insn), 0), 0)) == Pmode"
{
  if (TARGET_INDIRECT_BRANCH_NOBP_CALL)
    {
      gcc_assert (TARGET_CPU_Z10);
      s390_indirect_branch_via_thunk (SIBCALL_REGNUM,
				      INVALID_REGNUM,
				      NULL_RTX,
				      s390_indirect_branch_type_call);
      return "";
    }
  else
    return "br\t%%r1";
}
 [(set (attr "op_type")
       (if_then_else (match_test "TARGET_INDIRECT_BRANCH_NOBP_CALL")
		     (const_string "RIL")
		     (const_string "RR")))
  (set (attr "mnemonic")
       (if_then_else (match_test "TARGET_INDIRECT_BRANCH_NOBP_CALL")
		     (const_string "jg")
		     (const_string "br")))
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])

(define_insn "*sibcall_brc"
  [(call (mem:QI (match_operand 0 "bras_sym_operand" "X"))
         (match_operand 1 "const_int_operand" "n"))]
  "SIBLING_CALL_P (insn) && TARGET_SMALL_EXEC"
  "j\t%0"
  [(set_attr "op_type" "RI")
   (set_attr "type"    "branch")])

(define_insn "*sibcall_brcl"
  [(call (mem:QI (match_operand 0 "bras_sym_operand" "X"))
         (match_operand 1 "const_int_operand" "n"))]
  "SIBLING_CALL_P (insn)"
  "jg\t%0"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "branch")])

;
; sibcall_value patterns
;

(define_expand "sibcall_value"
  [(set (match_operand 0 "" "")
	(call (match_operand 1 "" "")
	      (match_operand 2 "" "")))]
  ""
{
  s390_emit_call (XEXP (operands[1], 0), NULL_RTX, operands[0], NULL_RTX);
  DONE;
})

(define_insn "*sibcall_value_br"
  [(set (match_operand 0 "" "")
	(call (mem:QI (reg SIBCALL_REGNUM))
	      (match_operand 1 "const_int_operand" "n")))]
  "SIBLING_CALL_P (insn)
   && GET_MODE (XEXP (XEXP (XEXP (PATTERN (insn), 1), 0), 0)) == Pmode"
{
  if (TARGET_INDIRECT_BRANCH_NOBP_CALL)
    {
      gcc_assert (TARGET_CPU_Z10);
      s390_indirect_branch_via_thunk (SIBCALL_REGNUM,
				      INVALID_REGNUM,
				      NULL_RTX,
				      s390_indirect_branch_type_call);
      return "";
    }
  else
    return "br\t%%r1";
}
  [(set (attr "op_type")
       (if_then_else (match_test "TARGET_INDIRECT_BRANCH_NOBP_CALL")
		     (const_string "RIL")
		     (const_string "RR")))
   (set (attr "mnemonic")
       (if_then_else (match_test "TARGET_INDIRECT_BRANCH_NOBP_CALL")
		     (const_string "jg")
		     (const_string "br")))
   (set_attr "type"  "branch")
   (set_attr "atype" "agen")])

(define_insn "*sibcall_value_brc"
  [(set (match_operand 0 "" "")
	(call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
	      (match_operand 2 "const_int_operand" "n")))]
  "SIBLING_CALL_P (insn) && TARGET_SMALL_EXEC"
  "j\t%1"
  [(set_attr "op_type" "RI")
   (set_attr "type"    "branch")])

(define_insn "*sibcall_value_brcl"
  [(set (match_operand 0 "" "")
	(call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
	      (match_operand 2 "const_int_operand" "n")))]
  "SIBLING_CALL_P (insn)"
  "jg\t%1"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "branch")])


;
; call instruction pattern(s).
;

(define_expand "call"
  [(call (match_operand 0 "" "")
         (match_operand 1 "" ""))
   (use (match_operand 2 "" ""))]
  ""
{
  s390_emit_call (XEXP (operands[0], 0), NULL_RTX, NULL_RTX,
		  gen_rtx_REG (Pmode, RETURN_REGNUM));
  DONE;
})

(define_insn "*bras"
  [(call (mem:QI (match_operand 0 "bras_sym_operand" "X"))
         (match_operand 1 "const_int_operand" "n"))
   (clobber (match_operand 2 "register_operand" "=r"))]
  "!SIBLING_CALL_P (insn)
   && TARGET_SMALL_EXEC
   && GET_MODE (operands[2]) == Pmode"
  "bras\t%2,%0"
  [(set_attr "op_type" "RI")
   (set_attr "type"    "jsr")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*brasl"
  [(call (mem:QI (match_operand 0 "bras_sym_operand" "X"))
         (match_operand 1 "const_int_operand" "n"))
   (clobber (match_operand 2 "register_operand" "=r"))]
  "!SIBLING_CALL_P (insn)

   && GET_MODE (operands[2]) == Pmode"
  "brasl\t%2,%0"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "jsr")
   (set_attr "z196prop" "z196_cracked")
   (set_attr "relative_long" "yes")])

(define_insn "*basr"
  [(call (mem:QI (match_operand 0 "address_operand" "ZR"))
         (match_operand 1 "const_int_operand" "n"))
   (clobber (match_operand 2 "register_operand" "=r"))]
  "!TARGET_INDIRECT_BRANCH_NOBP_CALL
   && !SIBLING_CALL_P (insn)
   && GET_MODE (operands[2]) == Pmode"
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "basr\t%2,%0";
  else
    return "bas\t%2,%a0";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set (attr "mnemonic")
        (if_then_else (match_operand 0 "register_operand" "")
                      (const_string "basr") (const_string "bas")))
   (set_attr "type"  "jsr")
   (set_attr "atype" "agen")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*basr_via_thunk<mode>_z10"
  [(call (mem:QI (match_operand:P 0 "register_operand" "a"))
         (match_operand 1 "const_int_operand"          "n"))
   (clobber (match_operand:P 2 "register_operand"    "=&r"))]
  "TARGET_INDIRECT_BRANCH_NOBP_CALL
   && TARGET_CPU_Z10
   && !SIBLING_CALL_P (insn)"
{
  s390_indirect_branch_via_thunk (REGNO (operands[0]),
				  REGNO (operands[2]),
				  NULL_RTX,
				  s390_indirect_branch_type_call);
  return "";
}
  [(set_attr "op_type" "RIL")
   (set_attr "mnemonic" "brasl")
   (set_attr "type"  "jsr")
   (set_attr "atype" "agen")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*basr_via_thunk<mode>"
  [(call (mem:QI (match_operand:P 0 "register_operand" "a"))
         (match_operand 1 "const_int_operand"          "n"))
   (clobber (match_operand:P 2 "register_operand"    "=&r"))
   (clobber (reg:P INDIRECT_BRANCH_THUNK_REGNUM))]
  "TARGET_INDIRECT_BRANCH_NOBP_CALL
   && !TARGET_CPU_Z10
   && !SIBLING_CALL_P (insn)"
{
  s390_indirect_branch_via_thunk (REGNO (operands[0]),
				  REGNO (operands[2]),
				  NULL_RTX,
				  s390_indirect_branch_type_call);
  return "";
}
  [(set_attr "op_type" "RIL")
   (set_attr "mnemonic" "brasl")
   (set_attr "type"  "jsr")
   (set_attr "atype" "agen")
   (set_attr "z196prop" "z196_cracked")])

;
; call_value instruction pattern(s).
;

(define_expand "call_value"
  [(set (match_operand 0 "" "")
        (call (match_operand 1 "" "")
              (match_operand 2 "" "")))
   (use (match_operand 3 "" ""))]
  ""
{
  s390_emit_call (XEXP (operands[1], 0), NULL_RTX, operands[0],
		  gen_rtx_REG (Pmode, RETURN_REGNUM));
  DONE;
})

(define_insn "*bras_r"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
              (match_operand:SI 2 "const_int_operand" "n")))
   (clobber (match_operand 3 "register_operand" "=r"))]
  "!SIBLING_CALL_P (insn)
   && TARGET_SMALL_EXEC
   && GET_MODE (operands[3]) == Pmode"
  "bras\t%3,%1"
  [(set_attr "op_type" "RI")
   (set_attr "type"    "jsr")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*brasl_r"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
              (match_operand 2 "const_int_operand" "n")))
   (clobber (match_operand 3 "register_operand" "=r"))]
  "!SIBLING_CALL_P (insn)

   && GET_MODE (operands[3]) == Pmode"
  "brasl\t%3,%1"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "jsr")
   (set_attr "z196prop" "z196_cracked")
   (set_attr "relative_long" "yes")])

(define_insn "*basr_r"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "address_operand" "ZR"))
              (match_operand 2 "const_int_operand" "n")))
   (clobber (match_operand 3 "register_operand" "=r"))]
  "!TARGET_INDIRECT_BRANCH_NOBP_CALL
   && !SIBLING_CALL_P (insn)
   && GET_MODE (operands[3]) == Pmode"
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "basr\t%3,%1";
  else
    return "bas\t%3,%a1";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 1 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set (attr "mnemonic")
        (if_then_else (match_operand 1 "register_operand" "")
                      (const_string "basr") (const_string "bas")))
   (set_attr "type"  "jsr")
   (set_attr "atype" "agen")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*basr_r_via_thunk_z10"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "register_operand" "a"))
              (match_operand 2 "const_int_operand"        "n")))
   (clobber (match_operand 3 "register_operand"         "=&r"))]
  "TARGET_INDIRECT_BRANCH_NOBP_CALL
   && TARGET_CPU_Z10
   && !SIBLING_CALL_P (insn)
   && GET_MODE (operands[3]) == Pmode"
{
  s390_indirect_branch_via_thunk (REGNO (operands[1]),
				  REGNO (operands[3]),
				  NULL_RTX,
				  s390_indirect_branch_type_call);
  return "";
}
  [(set_attr "op_type" "RIL")
   (set_attr "mnemonic" "brasl")
   (set_attr "type"  "jsr")
   (set_attr "atype" "agen")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*basr_r_via_thunk"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "register_operand" "a"))
              (match_operand 2 "const_int_operand"        "n")))
   (clobber (match_operand 3 "register_operand"         "=&r"))
   (clobber (reg:P INDIRECT_BRANCH_THUNK_REGNUM))]
  "TARGET_INDIRECT_BRANCH_NOBP_CALL
   && !TARGET_CPU_Z10
   && !SIBLING_CALL_P (insn)
   && GET_MODE (operands[3]) == Pmode"
{
  s390_indirect_branch_via_thunk (REGNO (operands[1]),
				  REGNO (operands[3]),
				  NULL_RTX,
				  s390_indirect_branch_type_call);
  return "";
}
  [(set_attr "op_type" "RIL")
   (set_attr "mnemonic"  "brasl")
   (set_attr "type"  "jsr")
   (set_attr "atype" "agen")
   (set_attr "z196prop" "z196_cracked")])

;;
;;- Thread-local storage support.
;;

(define_expand "get_thread_pointer<mode>"
  [(set (match_operand:P 0 "nonimmediate_operand" "") (reg:P TP_REGNUM))]
  ""
  "")

(define_expand "set_thread_pointer<mode>"
  [(set (reg:P TP_REGNUM) (match_operand:P 0 "nonimmediate_operand" ""))
   (set (reg:P TP_REGNUM) (unspec_volatile:P [(reg:P TP_REGNUM)] UNSPECV_SET_TP))]
  ""
  "")

(define_insn "*set_tp"
  [(set (reg TP_REGNUM) (unspec_volatile [(reg TP_REGNUM)] UNSPECV_SET_TP))]
  ""
  ""
  [(set_attr "type" "none")
   (set_attr "length" "0")])

(define_insn "*tls_load_64"
  [(set (match_operand:DI 0 "register_operand" "=d")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "T")
                    (match_operand:DI 2 "" "")]
		   UNSPEC_TLS_LOAD))]
  "TARGET_64BIT"
  "lg\t%0,%1%J2"
  [(set_attr "op_type" "RXE")
   (set_attr "z10prop" "z10_fwd_A3")])

(define_insn "*tls_load_31"
  [(set (match_operand:SI 0 "register_operand" "=d,d")
        (unspec:SI [(match_operand:SI 1 "memory_operand" "R,T")
                    (match_operand:SI 2 "" "")]
		   UNSPEC_TLS_LOAD))]
  "!TARGET_64BIT"
  "@
   l\t%0,%1%J2
   ly\t%0,%1%J2"
  [(set_attr "op_type" "RX,RXY")
   (set_attr "type" "load")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "z10prop" "z10_fwd_A3,z10_fwd_A3")])

(define_insn "*bras_tls"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
              (match_operand 2 "const_int_operand" "n")))
   (clobber (match_operand 3 "register_operand" "=r"))
   (use (match_operand 4 "" ""))]
  "!SIBLING_CALL_P (insn)
   && TARGET_SMALL_EXEC
   && GET_MODE (operands[3]) == Pmode"
  "bras\t%3,%1%J4"
  [(set_attr "op_type" "RI")
   (set_attr "type"    "jsr")
   (set_attr "z196prop" "z196_cracked")])

(define_insn "*brasl_tls"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
              (match_operand 2 "const_int_operand" "n")))
   (clobber (match_operand 3 "register_operand" "=r"))
   (use (match_operand 4 "" ""))]
  "!SIBLING_CALL_P (insn)

   && GET_MODE (operands[3]) == Pmode"
  "brasl\t%3,%1%J4"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "jsr")
   (set_attr "z196prop" "z196_cracked")
   (set_attr "relative_long" "yes")])

(define_insn "*basr_tls"
  [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand 1 "address_operand" "ZR"))
              (match_operand 2 "const_int_operand" "n")))
   (clobber (match_operand 3 "register_operand" "=r"))
   (use (match_operand 4 "" ""))]
  "!SIBLING_CALL_P (insn) && GET_MODE (operands[3]) == Pmode"
{
  if (get_attr_op_type (insn) == OP_TYPE_RR)
    return "basr\t%3,%1%J4";
  else
    return "bas\t%3,%a1%J4";
}
  [(set (attr "op_type")
        (if_then_else (match_operand 1 "register_operand" "")
                      (const_string "RR") (const_string "RX")))
   (set_attr "type"  "jsr")
   (set_attr "atype" "agen")
   (set_attr "z196prop" "z196_cracked")])

;;
;;- Atomic operations
;;

;
; memory barrier patterns.
;

(define_expand "mem_thread_fence"
  [(match_operand:SI 0 "const_int_operand")]		;; model
  ""
{
  /* Unless this is a SEQ_CST fence, the s390 memory model is strong
     enough not to require barriers of any kind.  */
  if (is_mm_seq_cst (memmodel_from_int (INTVAL (operands[0]))))
    {
      rtx mem = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
      MEM_VOLATILE_P (mem) = 1;
      emit_insn (gen_mem_thread_fence_1 (mem));
    }
  DONE;
})

; Although bcr is superscalar on Z10, this variant will never
; become part of an execution group.
; With z196 we can make use of the fast-BCR-serialization facility.
; This allows for a slightly faster sync which is sufficient for our
; purposes.
(define_insn "mem_thread_fence_1"
  [(set (match_operand:BLK 0 "" "")
	(unspec:BLK [(match_dup 0)] UNSPEC_MB))]
  ""
{
  if (TARGET_Z196)
    return "bcr\t14,0";
  else
    return "bcr\t15,0";
}
  [(set_attr "op_type" "RR")
   (set_attr "mnemonic" "bcr_flush")
   (set_attr "z196prop" "z196_alone")])

;
; atomic load/store operations
;

; Atomic loads need not examine the memory model at all.
(define_expand "atomic_load<mode>"
  [(match_operand:DINT 0 "register_operand")	;; output
   (match_operand:DINT 1 "memory_operand")	;; memory
   (match_operand:SI 2 "const_int_operand")]	;; model
  ""
{
  if (MEM_ALIGN (operands[1]) < GET_MODE_BITSIZE (GET_MODE (operands[1])))
    FAIL;

  if (<MODE>mode == TImode)
    emit_insn (gen_atomic_loadti_1 (operands[0], operands[1]));
  else if (<MODE>mode == DImode && !TARGET_ZARCH)
    emit_insn (gen_atomic_loaddi_1 (operands[0], operands[1]));
  else
    emit_move_insn (operands[0], operands[1]);
  DONE;
})

; Different from movdi_31 in that we want no splitters.
(define_insn "atomic_loaddi_1"
  [(set (match_operand:DI 0 "register_operand" "=d,d,!*f,!*f")
	(unspec:DI [(match_operand:DI 1 "memory_operand" "Q,S,R,T")]
		   UNSPEC_MOVA))]
  "!TARGET_ZARCH"
  "@
   lm\t%0,%M0,%S1
   lmy\t%0,%M0,%S1
   ld\t%0,%1
   ldy\t%0,%1"
  [(set_attr "op_type" "RS,RSY,RS,RSY")
   (set_attr "cpu_facility" "*,longdisp,*,longdisp")
   (set_attr "type" "lm,lm,floaddf,floaddf")])

(define_insn "atomic_loadti_1"
  [(set (match_operand:TI 0 "register_operand" "=r")
	(unspec:TI [(match_operand:TI 1 "memory_operand" "T")]
		   UNSPEC_MOVA))]
  "TARGET_ZARCH"
  "lpq\t%0,%1"
  [(set_attr "op_type" "RXY")
   (set_attr "type" "other")])

; Atomic stores must(?) enforce sequential consistency.
(define_expand "atomic_store<mode>"
  [(match_operand:DINT 0 "memory_operand")	;; memory
   (match_operand:DINT 1 "register_operand")	;; input
   (match_operand:SI 2 "const_int_operand")]	;; model
  ""
{
  enum memmodel model = memmodel_from_int (INTVAL (operands[2]));

  if (MEM_ALIGN (operands[0]) < GET_MODE_BITSIZE (GET_MODE (operands[0])))
    FAIL;

  if (<MODE>mode == TImode)
    emit_insn (gen_atomic_storeti_1 (operands[0], operands[1]));
  else if (<MODE>mode == DImode && !TARGET_ZARCH)
    emit_insn (gen_atomic_storedi_1 (operands[0], operands[1]));
  else
    emit_move_insn (operands[0], operands[1]);
  if (is_mm_seq_cst (model))
    emit_insn (gen_mem_thread_fence (operands[2]));
  DONE;
})

; Different from movdi_31 in that we want no splitters.
(define_insn "atomic_storedi_1"
  [(set (match_operand:DI 0 "memory_operand" "=Q,S,R,T")
	(unspec:DI [(match_operand:DI 1 "register_operand" "d,d,!*f,!*f")]
		   UNSPEC_MOVA))]
  "!TARGET_ZARCH"
  "@
   stm\t%1,%N1,%S0
   stmy\t%1,%N1,%S0
   std %1,%0
   stdy %1,%0"
  [(set_attr "op_type" "RS,RSY,RS,RSY")
   (set_attr "cpu_facility" "*,longdisp,*,longdisp")
   (set_attr "type" "stm,stm,fstoredf,fstoredf")])

(define_insn "atomic_storeti_1"
  [(set (match_operand:TI 0 "memory_operand" "=T")
	(unspec:TI [(match_operand:TI 1 "register_operand" "r")]
		   UNSPEC_MOVA))]
  "TARGET_ZARCH"
  "stpq\t%1,%0"
  [(set_attr "op_type" "RXY")
   (set_attr "type" "other")])

;
; compare and swap patterns.
;

(define_expand "atomic_compare_and_swap<mode>"
  [(match_operand:SI 0 "register_operand")	;; bool success output
   (match_operand:DINT 1 "nonimmediate_operand");; oldval output
   (match_operand:DINT 2 "s_operand")		;; memory
   (match_operand:DINT 3 "general_operand")	;; expected intput
   (match_operand:DINT 4 "general_operand")	;; newval intput
   (match_operand:SI 5 "const_int_operand")	;; is_weak
   (match_operand:SI 6 "const_int_operand")	;; success model
   (match_operand:SI 7 "const_int_operand")]	;; failure model
  ""
{
  if (GET_MODE_BITSIZE (<MODE>mode) >= 16
      && GET_MODE_BITSIZE (<MODE>mode) > MEM_ALIGN (operands[2]))
    FAIL;

  s390_expand_cs (<MODE>mode, operands[0], operands[1], operands[2],
		  operands[3], operands[4], INTVAL (operands[5]));
  DONE;})

(define_expand "atomic_compare_and_swap<mode>_internal"
  [(parallel
     [(set (match_operand:DGPR 0 "register_operand")
	   (match_operand:DGPR 1 "s_operand"))
      (set (match_dup 1)
	   (unspec_volatile:DGPR
	     [(match_dup 1)
	      (match_operand:DGPR 2 "register_operand")
	      (match_operand:DGPR 3 "register_operand")]
	     UNSPECV_CAS))
      (set (match_operand 4 "cc_reg_operand")
	   (match_dup 5))])]
  "GET_MODE (operands[4]) == CCZmode
   || GET_MODE (operands[4]) == CCZ1mode"
{
  operands[5]
    = gen_rtx_COMPARE (GET_MODE (operands[4]), operands[1], operands[2]);
})

; cdsg, csg
(define_insn "*atomic_compare_and_swap<mode>_1"
  [(set (match_operand:TDI 0 "register_operand" "=r")
	(match_operand:TDI 1 "nonsym_memory_operand" "+S"))
   (set (match_dup 1)
	(unspec_volatile:TDI
	  [(match_dup 1)
	   (match_operand:TDI 2 "register_operand" "0")
	   (match_operand:TDI 3 "register_operand" "r")]
	  UNSPECV_CAS))
   (set (reg CC_REGNUM)
	(compare (match_dup 1) (match_dup 2)))]
  "TARGET_ZARCH
   && s390_match_ccmode (insn, CCZ1mode)"
  "c<td>sg\t%0,%3,%S1"
  [(set_attr "op_type" "RSY")
   (set_attr "type"   "sem")])

; cds, cdsy
(define_insn "*atomic_compare_and_swapdi_2"
  [(set (match_operand:DI 0 "register_operand" "=r,r")
	(match_operand:DI 1 "nonsym_memory_operand" "+Q,S"))
   (set (match_dup 1)
	(unspec_volatile:DI
	  [(match_dup 1)
	   (match_operand:DI 2 "register_operand" "0,0")
	   (match_operand:DI 3 "register_operand" "r,r")]
	  UNSPECV_CAS))
   (set (reg CC_REGNUM)
	(compare (match_dup 1) (match_dup 2)))]
  "!TARGET_ZARCH
   && s390_match_ccmode (insn, CCZ1mode)"
  "@
   cds\t%0,%3,%S1
   cdsy\t%0,%3,%S1"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "type" "sem")])

; cs, csy
(define_insn "*atomic_compare_and_swapsi_3"
  [(set (match_operand:SI 0 "register_operand" "=r,r")
	(match_operand:SI 1 "nonsym_memory_operand" "+Q,S"))
   (set (match_dup 1)
	(unspec_volatile:SI
	  [(match_dup 1)
	   (match_operand:SI 2 "register_operand" "0,0")
	   (match_operand:SI 3 "register_operand" "r,r")]
	  UNSPECV_CAS))
   (set (reg CC_REGNUM)
	(compare (match_dup 1) (match_dup 2)))]
  "s390_match_ccmode (insn, CCZ1mode)"
  "@
   cs\t%0,%3,%S1
   csy\t%0,%3,%S1"
  [(set_attr "op_type" "RS,RSY")
   (set_attr "cpu_facility" "*,longdisp")
   (set_attr "type"   "sem")])

;
; Other atomic instruction patterns.
;

; z196 load and add, xor, or and and instructions

(define_expand "atomic_fetch_<atomic><mode>"
  [(match_operand:GPR 0 "register_operand")		;; val out
   (ATOMIC_Z196:GPR
     (match_operand:GPR 1 "memory_operand")		;; memory
     (match_operand:GPR 2 "register_operand"))		;; val in
   (match_operand:SI 3 "const_int_operand")]		;; model
  "TARGET_Z196"
{
  if (MEM_ALIGN (operands[1]) < GET_MODE_BITSIZE (GET_MODE (operands[1])))
    FAIL;

  emit_insn (gen_atomic_fetch_<atomic><mode>_iaf
	     (operands[0], operands[1], operands[2]));
  DONE;
})

; lan, lang, lao, laog, lax, laxg, laa, laag
(define_insn "atomic_fetch_<atomic><mode>_iaf"
  [(set (match_operand:GPR 0 "register_operand" "=d")
	(match_operand:GPR 1 "memory_operand" "+S"))
   (set (match_dup 1)
	(unspec_volatile:GPR
	 [(ATOMIC_Z196:GPR (match_dup 1)
			   (match_operand:GPR 2 "general_operand" "d"))]
	 UNSPECV_ATOMIC_OP))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z196"
  "la<noxa><g>\t%0,%2,%1"
  [(set_attr "op_type" "RSY")
   (set_attr "type" "sem")])

;; For SImode and larger, the optabs.c code will do just fine in
;; expanding a compare-and-swap loop.  For QI/HImode, we can do
;; better by expanding our own loop.

(define_expand "atomic_<atomic><mode>"
  [(ATOMIC:HQI
     (match_operand:HQI 0 "memory_operand")		;; memory
     (match_operand:HQI 1 "general_operand"))		;; val in
   (match_operand:SI 2 "const_int_operand")]		;; model
  ""
{
  s390_expand_atomic (<MODE>mode, <CODE>, NULL_RTX, operands[0],
		       operands[1], false);
  DONE;
})

(define_expand "atomic_fetch_<atomic><mode>"
  [(match_operand:HQI 0 "register_operand")		;; val out
   (ATOMIC:HQI
     (match_operand:HQI 1 "memory_operand")		;; memory
     (match_operand:HQI 2 "general_operand"))		;; val in
   (match_operand:SI 3 "const_int_operand")]		;; model
  ""
{
  s390_expand_atomic (<MODE>mode, <CODE>, operands[0], operands[1],
		      operands[2], false);
  DONE;
})

(define_expand "atomic_<atomic>_fetch<mode>"
  [(match_operand:HQI 0 "register_operand")		;; val out
   (ATOMIC:HQI
     (match_operand:HQI 1 "memory_operand")		;; memory
     (match_operand:HQI 2 "general_operand"))		;; val in
   (match_operand:SI 3 "const_int_operand")]		;; model
  ""
{
  s390_expand_atomic (<MODE>mode, <CODE>, operands[0], operands[1],
		      operands[2], true);
  DONE;
})

;; Pattern to implement atomic_exchange with a compare-and-swap loop.  The code
;; generated by the middleend is not good.
(define_expand "atomic_exchange<mode>"
  [(match_operand:DINT 0 "register_operand")		;; val out
   (match_operand:DINT 1 "s_operand")			;; memory
   (match_operand:DINT 2 "general_operand")		;; val in
   (match_operand:SI 3 "const_int_operand")]		;; model
  ""
{
  if (<MODE>mode != QImode
      && MEM_ALIGN (operands[1]) < GET_MODE_BITSIZE (<MODE>mode))
    FAIL;
  if (<MODE>mode == HImode || <MODE>mode == QImode)
    s390_expand_atomic (<MODE>mode, SET, operands[0], operands[1], operands[2],
			false);
  else if (<MODE>mode == SImode || TARGET_ZARCH)
    s390_expand_atomic_exchange_tdsi (operands[0], operands[1], operands[2]);
  else
    FAIL;
  DONE;
})

;;
;;- Miscellaneous instructions.
;;

;
; allocate stack instruction pattern(s).
;

(define_expand "allocate_stack"
  [(match_operand 0 "general_operand" "")
   (match_operand 1 "general_operand" "")]
 "TARGET_BACKCHAIN"
{
  rtx temp = gen_reg_rtx (Pmode);

  emit_move_insn (temp, s390_back_chain_rtx ());
  anti_adjust_stack (operands[1]);
  emit_move_insn (s390_back_chain_rtx (), temp);

  emit_move_insn (operands[0], virtual_stack_dynamic_rtx);
  DONE;
})


;
; setjmp instruction pattern.
;

(define_expand "builtin_setjmp_receiver"
  [(match_operand 0 "" "")]
  "flag_pic"
{
  emit_insn (s390_load_got ());
  emit_use (pic_offset_table_rtx);
  DONE;
})

;; These patterns say how to save and restore the stack pointer.  We need not
;; save the stack pointer at function level since we are careful to
;; preserve the backchain.  At block level, we have to restore the backchain
;; when we restore the stack pointer.
;;
;; For nonlocal gotos, we must save both the stack pointer and its
;; backchain and restore both.  Note that in the nonlocal case, the
;; save area is a memory location.

(define_expand "save_stack_function"
  [(match_operand 0 "general_operand" "")
   (match_operand 1 "general_operand" "")]
  ""
  "DONE;")

(define_expand "restore_stack_function"
  [(match_operand 0 "general_operand" "")
   (match_operand 1 "general_operand" "")]
  ""
  "DONE;")

(define_expand "restore_stack_block"
  [(match_operand 0 "register_operand" "")
   (match_operand 1 "register_operand" "")]
  "TARGET_BACKCHAIN"
{
  rtx temp = gen_reg_rtx (Pmode);

  emit_move_insn (temp, s390_back_chain_rtx ());
  emit_move_insn (operands[0], operands[1]);
  emit_move_insn (s390_back_chain_rtx (), temp);

  DONE;
})

(define_expand "save_stack_nonlocal"
  [(match_operand 0 "memory_operand" "")
   (match_operand 1 "register_operand" "")]
  ""
{
  rtx base = gen_rtx_REG (Pmode, BASE_REGNUM);

  /* Copy the backchain to the first word, sp to the second and the
     literal pool base to the third.  */

  rtx save_bc = adjust_address (operands[0], Pmode, 0);
  rtx save_sp = adjust_address (operands[0], Pmode, GET_MODE_SIZE (Pmode));
  rtx save_bp = adjust_address (operands[0], Pmode, 2 * GET_MODE_SIZE (Pmode));

  if (TARGET_BACKCHAIN)
    emit_move_insn (save_bc, force_reg (Pmode, s390_back_chain_rtx ()));

  emit_move_insn (save_sp, operands[1]);
  emit_move_insn (save_bp, base);

  DONE;
})

(define_expand "restore_stack_nonlocal"
  [(match_operand 0 "register_operand" "")
   (match_operand 1 "memory_operand" "")]
  ""
{
  rtx base = gen_rtx_REG (Pmode, BASE_REGNUM);
  rtx temp = NULL_RTX;

  /* Restore the backchain from the first word, sp from the second and the
     literal pool base from the third.  */

  rtx save_bc = adjust_address (operands[1], Pmode, 0);
  rtx save_sp = adjust_address (operands[1], Pmode, GET_MODE_SIZE (Pmode));
  rtx save_bp = adjust_address (operands[1], Pmode, 2 * GET_MODE_SIZE (Pmode));

  if (TARGET_BACKCHAIN)
    temp = force_reg (Pmode, save_bc);

  emit_move_insn (base, save_bp);
  emit_move_insn (operands[0], save_sp);

  if (temp)
    emit_move_insn (s390_back_chain_rtx (), temp);

  emit_use (base);
  DONE;
})

(define_expand "exception_receiver"
  [(const_int 0)]
  ""
{
  s390_set_has_landing_pad_p (true);
  DONE;
})

;
; nop instruction pattern(s).
;

(define_insn "nop"
  [(const_int 0)]
  ""
  "nopr\t%%r0"
  [(set_attr "op_type" "RR")])

; non-branch NOPs required for optimizing compare-and-branch patterns
; on z10

(define_insn "nop_lr0"
  [(unspec_volatile [(const_int 0)] UNSPECV_NOP_LR_0)]
  ""
  "lr\t0,0"
  [(set_attr "op_type" "RR")
   (set_attr "z10prop"  "z10_fr_E1")])

(define_insn "nop_lr1"
  [(unspec_volatile [(const_int 0)] UNSPECV_NOP_LR_1)]
  ""
  "lr\t1,1"
  [(set_attr "op_type" "RR")])

;;- Undeletable nops (used for hotpatching)

(define_insn "nop_2_byte"
  [(unspec_volatile [(const_int 0)] UNSPECV_NOP_2_BYTE)]
  ""
  "nopr\t%%r0"
  [(set_attr "op_type" "RR")])

(define_insn "nop_4_byte"
  [(unspec_volatile [(const_int 0)] UNSPECV_NOP_4_BYTE)]
  ""
  "nop\t0"
  [(set_attr "op_type" "RX")])

(define_insn "nop_6_byte"
  [(unspec_volatile [(const_int 0)] UNSPECV_NOP_6_BYTE)]
  ""
  "brcl\t0, 0"
  [(set_attr "op_type" "RIL")
   (set_attr "relative_long" "yes")])


;
; Special literal pool access instruction pattern(s).
;

(define_insn "*pool_entry"
  [(unspec_volatile [(match_operand 0 "consttable_operand" "X")]
                    UNSPECV_POOL_ENTRY)]
  ""
{
  machine_mode mode = GET_MODE (PATTERN (insn));
  unsigned int align = GET_MODE_BITSIZE (mode);
  s390_output_pool_entry (operands[0], mode, align);
  return "";
}
  [(set (attr "length")
        (symbol_ref "GET_MODE_SIZE (GET_MODE (PATTERN (insn)))"))])

(define_insn "pool_align"
  [(unspec_volatile [(match_operand 0 "const_int_operand" "n")]
                    UNSPECV_POOL_ALIGN)]
  ""
  ".align\t%0"
  [(set (attr "length") (symbol_ref "INTVAL (operands[0])"))])

(define_insn "pool_section_start"
  [(unspec_volatile [(const_int 1)] UNSPECV_POOL_SECTION)]
  ""
{
  switch_to_section (targetm.asm_out.function_rodata_section
		 (current_function_decl));
  return "";
}
  [(set_attr "length" "0")])

(define_insn "pool_section_end"
  [(unspec_volatile [(const_int 0)] UNSPECV_POOL_SECTION)]
  ""
{
  switch_to_section (current_function_section ());
  return "";
}
  [(set_attr "length" "0")])

(define_insn "main_base_64"
  [(set (match_operand 0 "register_operand" "=a")
        (unspec [(label_ref (match_operand 1 "" ""))] UNSPEC_MAIN_BASE))]
  "GET_MODE (operands[0]) == Pmode"
  "larl\t%0,%1"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "larl")
   (set_attr "z10prop" "z10_fwd_A1")
   (set_attr "relative_long" "yes")])

(define_insn "main_pool"
  [(set (match_operand 0 "register_operand" "=a")
        (unspec_volatile [(const_int 0)] UNSPECV_MAIN_POOL))]
  "GET_MODE (operands[0]) == Pmode"
{
  gcc_unreachable ();
}
  [(set (attr "type")
        (const_string "larl"))])

(define_insn "reload_base_64"
  [(set (match_operand 0 "register_operand" "=a")
        (unspec [(label_ref (match_operand 1 "" ""))] UNSPEC_RELOAD_BASE))]
  "GET_MODE (operands[0]) == Pmode"
  "larl\t%0,%1"
  [(set_attr "op_type" "RIL")
   (set_attr "type"    "larl")
   (set_attr "z10prop" "z10_fwd_A1")])

(define_insn "pool"
  [(unspec_volatile [(match_operand 0 "const_int_operand" "n")] UNSPECV_POOL)]
  ""
{
  gcc_unreachable ();
}
  [(set (attr "length") (symbol_ref "INTVAL (operands[0])"))])

;;
;; Insns related to generating the function prologue and epilogue.
;;


(define_expand "prologue"
  [(use (const_int 0))]
  ""
  "s390_emit_prologue (); DONE;")

(define_expand "epilogue"
  [(use (const_int 1))]
  ""
  "s390_emit_epilogue (false); DONE;")

(define_expand "sibcall_epilogue"
  [(use (const_int 0))]
  ""
  "s390_emit_epilogue (true); DONE;")

;; A direct return instruction, without using an epilogue.
(define_insn "<code>"
  [(ANY_RETURN)]
  "s390_can_use_<code>_insn ()"
{
  if (TARGET_INDIRECT_BRANCH_NOBP_RET)
    {
      /* The target is always r14 so there is no clobber
	 of r1 needed for pre z10 targets.  */
      s390_indirect_branch_via_thunk (RETURN_REGNUM,
				      INVALID_REGNUM,
				      NULL_RTX,
				      s390_indirect_branch_type_return);
      return "";
    }
  else
    return "br\t%%r14";
}
  [(set (attr "op_type")
	(if_then_else (match_test "TARGET_INDIRECT_BRANCH_NOBP_RET")
		      (const_string "RIL")
		      (const_string "RR")))
   (set (attr "mnemonic")
	(if_then_else (match_test "TARGET_INDIRECT_BRANCH_NOBP_RET")
		      (const_string "jg")
		      (const_string "br")))
   (set_attr "type"    "jsr")
   (set_attr "atype"   "agen")])


(define_expand "return_use"
  [(parallel
    [(return)
     (use (match_operand 0 "register_operand" "a"))])]
  ""
{
  if (!TARGET_CPU_Z10
      && TARGET_INDIRECT_BRANCH_NOBP_RET_OPTION)
    {
      if (TARGET_64BIT)
        emit_jump_insn (gen_returndi_prez10 (operands[0]));
      else
        emit_jump_insn (gen_returnsi_prez10 (operands[0]));
      DONE;
    }
})

(define_insn "*return<mode>"
  [(return)
   (use (match_operand:P 0 "register_operand" "a"))]
  "TARGET_CPU_Z10 || !TARGET_INDIRECT_BRANCH_NOBP_RET_OPTION"
{
  if (TARGET_INDIRECT_BRANCH_NOBP_RET)
    {
      s390_indirect_branch_via_thunk (REGNO (operands[0]),
                                      INVALID_REGNUM,
                                      NULL_RTX,
                                      s390_indirect_branch_type_return);
      return "";
    }
  else
    return "br\t%0";
}
  [(set (attr "op_type")
       (if_then_else (match_test "TARGET_INDIRECT_BRANCH_NOBP_RET")
                     (const_string "RIL")
                     (const_string "RR")))
   (set (attr "mnemonic")
       (if_then_else (match_test "TARGET_INDIRECT_BRANCH_NOBP_RET")
                     (const_string "jg")
                     (const_string "br")))
   (set_attr "type"    "jsr")
   (set_attr "atype"   "agen")])

(define_insn "return<mode>_prez10"
  [(return)
   (use (match_operand:P 0 "register_operand" "a"))
   (clobber (reg:P INDIRECT_BRANCH_THUNK_REGNUM))]
  "!TARGET_CPU_Z10 && TARGET_INDIRECT_BRANCH_NOBP_RET_OPTION"
{
  if (TARGET_INDIRECT_BRANCH_NOBP_RET)
    {
      s390_indirect_branch_via_thunk (REGNO (operands[0]),
                                      INVALID_REGNUM,
                                      NULL_RTX,
                                      s390_indirect_branch_type_return);
      return "";
    }
  else
    return "br\t%0";
}
  [(set (attr "op_type")
       (if_then_else (match_test "TARGET_INDIRECT_BRANCH_NOBP_RET")
                     (const_string "RIL")
                     (const_string "RR")))
   (set (attr "mnemonic")
       (if_then_else (match_test "TARGET_INDIRECT_BRANCH_NOBP_RET")
                     (const_string "jg")
                     (const_string "br")))
   (set_attr "type"    "jsr")
   (set_attr "atype"   "agen")])


;; Instruction definition to extend a 31-bit pointer into a 64-bit
;; pointer. This is used for compatibility.

(define_expand "ptr_extend"
  [(set (match_operand:DI 0 "register_operand" "=r")
        (match_operand:SI 1 "register_operand" "r"))]
  "TARGET_64BIT"
{
  emit_insn (gen_anddi3 (operands[0],
			 gen_lowpart (DImode, operands[1]),
			 GEN_INT (0x7fffffff)));
  DONE;
})

;; Instruction definition to expand eh_return macro to support
;; swapping in special linkage return addresses.

(define_expand "eh_return"
  [(use (match_operand 0 "register_operand" ""))]
  "TARGET_TPF"
{
  s390_emit_tpf_eh_return (operands[0]);
  DONE;
})

;
; Stack Protector Patterns
;

(define_expand "stack_protect_set"
  [(set (match_operand 0 "memory_operand" "")
	(match_operand 1 "memory_operand" ""))]
  ""
{
#ifdef TARGET_THREAD_SSP_OFFSET
  operands[1]
    = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, s390_get_thread_pointer (),
                                        GEN_INT (TARGET_THREAD_SSP_OFFSET)));
#endif
  if (TARGET_64BIT)
    emit_insn (gen_stack_protect_setdi (operands[0], operands[1]));
  else
    emit_insn (gen_stack_protect_setsi (operands[0], operands[1]));

  DONE;
})

(define_insn "stack_protect_set<mode>"
  [(set (match_operand:DSI 0 "memory_operand" "=Q")
        (unspec:DSI [(match_operand:DSI 1 "memory_operand" "Q")] UNSPEC_SP_SET))]
  ""
  "mvc\t%O0(%G0,%R0),%S1"
  [(set_attr "op_type" "SS")])

(define_expand "stack_protect_test"
  [(set (reg:CC CC_REGNUM)
	(compare (match_operand 0 "memory_operand" "")
		 (match_operand 1 "memory_operand" "")))
   (match_operand 2 "" "")]
  ""
{
  rtx cc_reg, test;
#ifdef TARGET_THREAD_SSP_OFFSET
  operands[1]
    = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, s390_get_thread_pointer (),
                                        GEN_INT (TARGET_THREAD_SSP_OFFSET)));
#endif
  if (TARGET_64BIT)
    emit_insn (gen_stack_protect_testdi (operands[0], operands[1]));
  else
    emit_insn (gen_stack_protect_testsi (operands[0], operands[1]));

  cc_reg = gen_rtx_REG (CCZmode, CC_REGNUM);
  test = gen_rtx_EQ (VOIDmode, cc_reg, const0_rtx);
  emit_jump_insn (gen_cbranchcc4 (test, cc_reg, const0_rtx, operands[2]));
  DONE;
})

(define_insn "stack_protect_test<mode>"
  [(set (reg:CCZ CC_REGNUM)
        (unspec:CCZ [(match_operand:DSI 0 "memory_operand" "Q")
		     (match_operand:DSI 1 "memory_operand" "Q")] UNSPEC_SP_TEST))]
  ""
  "clc\t%O0(%G0,%R0),%S1"
  [(set_attr "op_type" "SS")])

; This is used in s390_emit_prologue in order to prevent insns
; adjusting the stack pointer to be moved over insns writing stack
; slots using a copy of the stack pointer in a different register.
(define_insn "stack_tie"
  [(set (match_operand:BLK 0 "memory_operand" "+m")
        (unspec:BLK [(match_dup 0)] UNSPEC_TIE))]
  ""
  ""
  [(set_attr "length" "0")])


(define_insn "stack_restore_from_fpr"
  [(set (reg:DI STACK_REGNUM)
	(match_operand:DI 0 "register_operand" "f"))
   (clobber (mem:BLK (scratch)))]
  "TARGET_Z10"
  "lgdr\t%%r15,%0"
  [(set_attr "op_type"  "RRE")])

;
; Data prefetch patterns
;

(define_insn "prefetch"
  [(prefetch (match_operand 0    "address_operand"   "ZT,X")
	     (match_operand:SI 1 "const_int_operand" " n,n")
	     (match_operand:SI 2 "const_int_operand" " n,n"))]
  "TARGET_Z10"
{
  switch (which_alternative)
    {
      case 0:
        return INTVAL (operands[1]) == 1 ? "pfd\t2,%a0" : "pfd\t1,%a0";
      case 1:
        if (larl_operand (operands[0], Pmode))
	  return INTVAL (operands[1]) == 1 ? "pfdrl\t2,%a0" : "pfdrl\t1,%a0";
	  /* fallthrough */
      default:

        /* This might be reached for symbolic operands with an odd
           addend.  We simply omit the prefetch for such rare cases.  */

        return "";
     }
}
  [(set_attr "type" "load,larl")
   (set_attr "op_type" "RXY,RIL")
   (set_attr "z10prop" "z10_super")
   (set_attr "z196prop" "z196_alone")
   (set_attr "relative_long" "yes")])


;
; Byte swap instructions
;

; FIXME: There is also mvcin but we cannot use it since src and target
; may overlap.
; lrvr, lrv, strv, lrvgr, lrvg, strvg
(define_insn "bswap<mode>2"
  [(set (match_operand:GPR 0            "nonimmediate_operand" "=d,d,T")
	(bswap:GPR (match_operand:GPR 1 "nonimmediate_operand" " d,T,d")))]
  ""
  "@
   lrv<g>r\t%0,%1
   lrv<g>\t%0,%1
   strv<g>\t%1,%0"
  [(set_attr "type" "*,load,store")
   (set_attr "op_type" "RRE,RXY,RXY")
   (set_attr "z10prop" "z10_super")])

(define_insn "bswaphi2"
  [(set (match_operand:HI 0           "nonimmediate_operand" "=d,d,T")
	(bswap:HI (match_operand:HI 1 "nonimmediate_operand" " d,T,d")))]
  ""
  "@
   #
   lrvh\t%0,%1
   strvh\t%1,%0"
  [(set_attr "type" "*,load,store")
   (set_attr "op_type" "RRE,RXY,RXY")
   (set_attr "z10prop" "z10_super")])

(define_split
  [(set (match_operand:HI 0           "register_operand" "")
	(bswap:HI (match_operand:HI 1 "register_operand" "")))]
  ""
  [(set (match_dup 2) (bswap:SI (match_dup 3)))
   (set (match_dup 2) (lshiftrt:SI (match_dup 2) (const_int 16)))]
{
  operands[2] = simplify_gen_subreg (SImode, operands[0], HImode, 0);
  operands[3] = simplify_gen_subreg (SImode, operands[1], HImode, 0);
})


;
; Population count instruction
;

(define_insn "*popcountdi_arch13_cc"
  [(set (reg CC_REGNUM)
	(compare (popcount:DI (match_operand:DI 1 "register_operand" "d"))
		 (const_int 0)))
   (set (match_operand:DI 0 "register_operand" "=d")
	(match_dup 1))]
  "TARGET_ARCH13 && s390_match_ccmode (insn, CCTmode)"
  "popcnt\t%0,%1,8"
  [(set_attr "op_type" "RRF")])

(define_insn "*popcountdi_arch13_cconly"
  [(set (reg CC_REGNUM)
	(compare (popcount:DI (match_operand:DI 1 "register_operand" "d"))
		 (const_int 0)))
   (clobber (match_scratch:DI 0 "=d"))]
  "TARGET_ARCH13 && s390_match_ccmode(insn, CCTmode)"
  "popcnt\t%0,%1,8"
  [(set_attr "op_type" "RRF")])

(define_insn "*popcountdi_arch13"
  [(set (match_operand:DI 0 "register_operand" "=d")
	(popcount:DI (match_operand:DI 1 "register_operand" "d")))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_ARCH13"
  "popcnt\t%0,%1,8"
  [(set_attr "op_type" "RRF")])

; The pre-arch13 popcount instruction counts the bits of op1 in 8 byte
; portions and stores the result in the corresponding bytes in op0.
(define_insn "*popcount<mode>_z196"
  [(set (match_operand:INT 0 "register_operand" "=d")
	(unspec:INT [(match_operand:INT 1 "register_operand" "d")] UNSPEC_POPCNT))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z196"
  "popcnt\t%0,%1"
  [(set_attr "op_type" "RRE")])

(define_expand "popcountdi2_z196"
  [; popcnt op0, op1
   (parallel [(set (match_operand:DI 0 "register_operand" "")
		   (unspec:DI [(match_operand:DI 1 "register_operand")]
			      UNSPEC_POPCNT))
	      (clobber (reg:CC CC_REGNUM))])
   ; sllg op2, op0, 32
   (set (match_dup 2) (ashift:DI (match_dup 0) (const_int 32)))
   ; agr op0, op2
   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (match_dup 2)))
	      (clobber (reg:CC CC_REGNUM))])
   ; sllg op2, op0, 16
   (set (match_dup 2)
	(ashift:DI (match_dup 0) (const_int 16)))
   ; agr op0, op2
   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (match_dup 2)))
	      (clobber (reg:CC CC_REGNUM))])
   ; sllg op2, op0, 8
   (set (match_dup 2) (ashift:DI (match_dup 0) (const_int 8)))
   ; agr op0, op2
   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (match_dup 2)))
	      (clobber (reg:CC CC_REGNUM))])
   ; srlg op0, op0, 56
   (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 56)))]
  "TARGET_Z196"
  "operands[2] = gen_reg_rtx (DImode);")

(define_expand "popcountdi2"
  [(parallel
    [(set (match_operand:DI 0 "register_operand" "")
	  (popcount:DI (match_operand:DI 1 "register_operand")))
     (clobber (reg:CC CC_REGNUM))])]
  "TARGET_Z196"
{
  if (!TARGET_ARCH13)
    {
      emit_insn (gen_popcountdi2_z196 (operands[0], operands[1]));
      DONE;
    }
 })

(define_expand "popcountsi2_z196"
  [; popcnt op0, op1
   (parallel [(set (match_operand:SI 0 "register_operand" "")
		   (unspec:SI [(match_operand:SI 1 "register_operand")]
			      UNSPEC_POPCNT))
	      (clobber (reg:CC CC_REGNUM))])
   ; sllk op2, op0, 16
   (set (match_dup 2)
	(ashift:SI (match_dup 0) (const_int 16)))
   ; ar op0, op2
   (parallel [(set (match_dup 0) (plus:SI (match_dup 0) (match_dup 2)))
	      (clobber (reg:CC CC_REGNUM))])
   ; sllk op2, op0, 8
   (set (match_dup 2) (ashift:SI (match_dup 0) (const_int 8)))
   ; ar op0, op2
   (parallel [(set (match_dup 0) (plus:SI (match_dup 0) (match_dup 2)))
	      (clobber (reg:CC CC_REGNUM))])
   ; srl op0, op0, 24
   (set (match_dup 0) (lshiftrt:SI (match_dup 0) (const_int 24)))]
  "TARGET_Z196"
  "operands[2] = gen_reg_rtx (SImode);")

; popcount always counts on the full 64 bit. With the z196 version
; counting bits per byte we just ignore the upper 4 bytes.  With the
; arch13 version we have to zero out the upper 32 bits first.
(define_expand "popcountsi2"
  [(set (match_dup 2)
	(zero_extend:DI (match_operand:SI 1 "register_operand")))
   (parallel [(set (match_dup 3) (popcount:DI (match_dup 2)))
	      (clobber (reg:CC CC_REGNUM))])
   (set (match_operand:SI 0 "register_operand")
	(subreg:SI (match_dup 3) 4))]
  "TARGET_Z196"
{
  if (!TARGET_ARCH13)
    {
      emit_insn (gen_popcountsi2_z196 (operands[0], operands[1]));
      DONE;
    }
  else
    {
      operands[2] = gen_reg_rtx (DImode);
      operands[3] = gen_reg_rtx (DImode);
    }
})

(define_expand "popcounthi2_z196"
  [; popcnt op0, op1
   (parallel [(set (match_operand:HI 0 "register_operand" "")
		   (unspec:HI [(match_operand:HI 1 "register_operand")]
			      UNSPEC_POPCNT))
	      (clobber (reg:CC CC_REGNUM))])
   ; sllk op2, op0, 8
   (set (match_dup 2)
	(ashift:SI (match_dup 0) (const_int 8)))
   ; ar op0, op2
   (parallel [(set (match_dup 0) (plus:SI (match_dup 0) (match_dup 2)))
	      (clobber (reg:CC CC_REGNUM))])
   ; srl op0, op0, 8
   (set (match_dup 0) (lshiftrt:HI (match_dup 0) (const_int 8)))]
  "TARGET_Z196"
  "operands[2] = gen_reg_rtx (SImode);")

(define_expand "popcounthi2"
  [(set (match_dup 2)
	(zero_extend:DI (match_operand:HI 1 "register_operand")))
   (parallel [(set (match_dup 3) (popcount:DI (match_dup 2)))
	      (clobber (reg:CC CC_REGNUM))])
   (set (match_operand:HI 0 "register_operand")
	(subreg:HI (match_dup 3) 6))]
  "TARGET_Z196"
{
  if (!TARGET_ARCH13)
    {
      emit_insn (gen_popcounthi2_z196 (operands[0], operands[1]));
      DONE;
    }
  else
    {
      operands[2] = gen_reg_rtx (DImode);
      operands[3] = gen_reg_rtx (DImode);
    }
})

; For popcount on a single byte the old z196 style popcount
; instruction is ideal.  Since it anyway does a byte-wise popcount we
; just use it instead of zero extending the QImode input to DImode and
; using the arch13 popcount variant.
(define_expand "popcountqi2"
  [; popcnt op0, op1
   (parallel [(set (match_operand:QI 0 "register_operand" "")
		   (unspec:QI [(match_operand:QI 1 "register_operand")]
			      UNSPEC_POPCNT))
	      (clobber (reg:CC CC_REGNUM))])]
  "TARGET_Z196"
  "")

;;
;;- Copy sign instructions
;;

(define_insn "copysign<mode>3"
  [(set (match_operand:FP 0 "register_operand" "=f")
      (unspec:FP [(match_operand:FP 1 "register_operand" "<fT0>")
                  (match_operand:FP 2 "register_operand" "f")]
                  UNSPEC_COPYSIGN))]
  "TARGET_Z196"
  "cpsdr\t%0,%2,%1"
  [(set_attr "op_type"  "RRF")
   (set_attr "type"     "fsimp<mode>")])


;;
;;- Transactional execution instructions
;;

; This splitter helps combine to make use of CC directly when
; comparing the integer result of a tbegin builtin with a constant.
; The unspec is already removed by canonicalize_comparison. So this
; splitters only job is to turn the PARALLEL into separate insns
; again.  Unfortunately this only works with the very first cc/int
; compare since combine is not able to deal with data flow across
; basic block boundaries.

; It needs to be an insn pattern as well since combine does not apply
; the splitter directly.  Combine would only use it if it actually
; would reduce the number of instructions.
(define_insn_and_split "*ccraw_to_int"
  [(set (pc)
	(if_then_else
	 (match_operator 0 "s390_eqne_operator"
			 [(reg:CCRAW CC_REGNUM)
			  (match_operand 1 "const_int_operand" "")])
	 (label_ref (match_operand 2 "" ""))
	 (pc)))
   (set (match_operand:SI 3 "register_operand" "=d")
	(unspec:SI [(reg:CCRAW CC_REGNUM)] UNSPEC_CC_TO_INT))]
  ""
  "#"
  ""
  [(set (match_dup 3)
	(unspec:SI [(reg:CCRAW CC_REGNUM)] UNSPEC_CC_TO_INT))
   (set (pc)
	(if_then_else (match_op_dup 0 [(reg:CCRAW CC_REGNUM) (match_dup 1)])
		      (label_ref (match_dup 2))
		      (pc)))]
  "")

; Non-constrained transaction begin

(define_expand "tbegin"
  [(match_operand:SI 0 "register_operand" "")
   (match_operand:BLK 1 "memory_operand" "")]
  "TARGET_HTM"
{
  s390_expand_tbegin (operands[0], operands[1], NULL_RTX, true);
  DONE;
})

(define_expand "tbegin_nofloat"
  [(match_operand:SI 0 "register_operand" "")
   (match_operand:BLK 1 "memory_operand" "")]
  "TARGET_HTM"
{
  s390_expand_tbegin (operands[0], operands[1], NULL_RTX, false);
  DONE;
})

(define_expand "tbegin_retry"
  [(match_operand:SI 0 "register_operand" "")
   (match_operand:BLK 1 "memory_operand" "")
   (match_operand:SI 2 "general_operand" "")]
  "TARGET_HTM"
{
  s390_expand_tbegin (operands[0], operands[1], operands[2], true);
  DONE;
})

(define_expand "tbegin_retry_nofloat"
  [(match_operand:SI 0 "register_operand" "")
   (match_operand:BLK 1 "memory_operand" "")
   (match_operand:SI 2 "general_operand" "")]
  "TARGET_HTM"
{
  s390_expand_tbegin (operands[0], operands[1], operands[2], false);
  DONE;
})

; Clobber VRs since they don't get restored
(define_insn "tbegin_1_z13"
  [(set (reg:CCRAW CC_REGNUM)
	(unspec_volatile:CCRAW [(match_operand 0 "const_int_operand" "D")]
			       UNSPECV_TBEGIN))
   (set (match_operand:BLK 1 "memory_operand" "=Q")
	(unspec_volatile:BLK [(match_dup 0)] UNSPECV_TBEGIN_TDB))
   (clobber (reg:TI 16)) (clobber (reg:TI 38))
   (clobber (reg:TI 17)) (clobber (reg:TI 39))
   (clobber (reg:TI 18)) (clobber (reg:TI 40))
   (clobber (reg:TI 19)) (clobber (reg:TI 41))
   (clobber (reg:TI 20)) (clobber (reg:TI 42))
   (clobber (reg:TI 21)) (clobber (reg:TI 43))
   (clobber (reg:TI 22)) (clobber (reg:TI 44))
   (clobber (reg:TI 23)) (clobber (reg:TI 45))
   (clobber (reg:TI 24)) (clobber (reg:TI 46))
   (clobber (reg:TI 25)) (clobber (reg:TI 47))
   (clobber (reg:TI 26)) (clobber (reg:TI 48))
   (clobber (reg:TI 27)) (clobber (reg:TI 49))
   (clobber (reg:TI 28)) (clobber (reg:TI 50))
   (clobber (reg:TI 29)) (clobber (reg:TI 51))
   (clobber (reg:TI 30)) (clobber (reg:TI 52))
   (clobber (reg:TI 31)) (clobber (reg:TI 53))]
; CONST_OK_FOR_CONSTRAINT_P does not work with D constraint since D is
; not supposed to be used for immediates (see genpreds.c).
  "TARGET_VX && INTVAL (operands[0]) >= 0 && INTVAL (operands[0]) <= 0xffff"
  "tbegin\t%1,%x0"
  [(set_attr "op_type" "SIL")])

(define_insn "tbegin_1"
  [(set (reg:CCRAW CC_REGNUM)
	(unspec_volatile:CCRAW [(match_operand 0 "const_int_operand" "D")]
			       UNSPECV_TBEGIN))
   (set (match_operand:BLK 1 "memory_operand" "=Q")
	(unspec_volatile:BLK [(match_dup 0)] UNSPECV_TBEGIN_TDB))
   (clobber (reg:DF 16))
   (clobber (reg:DF 17))
   (clobber (reg:DF 18))
   (clobber (reg:DF 19))
   (clobber (reg:DF 20))
   (clobber (reg:DF 21))
   (clobber (reg:DF 22))
   (clobber (reg:DF 23))
   (clobber (reg:DF 24))
   (clobber (reg:DF 25))
   (clobber (reg:DF 26))
   (clobber (reg:DF 27))
   (clobber (reg:DF 28))
   (clobber (reg:DF 29))
   (clobber (reg:DF 30))
   (clobber (reg:DF 31))]
; CONST_OK_FOR_CONSTRAINT_P does not work with D constraint since D is
; not supposed to be used for immediates (see genpreds.c).
  "TARGET_HTM && INTVAL (operands[0]) >= 0 && INTVAL (operands[0]) <= 0xffff"
  "tbegin\t%1,%x0"
  [(set_attr "op_type" "SIL")])

; Same as above but without the FPR clobbers
(define_insn "tbegin_nofloat_1"
  [(set (reg:CCRAW CC_REGNUM)
	(unspec_volatile:CCRAW [(match_operand 0 "const_int_operand" "D")]
			       UNSPECV_TBEGIN))
   (set (match_operand:BLK 1 "memory_operand" "=Q")
	(unspec_volatile:BLK [(match_dup 0)] UNSPECV_TBEGIN_TDB))]
  "TARGET_HTM && INTVAL (operands[0]) >= 0 && INTVAL (operands[0]) <= 0xffff"
  "tbegin\t%1,%x0"
  [(set_attr "op_type" "SIL")])


; Constrained transaction begin

(define_expand "tbeginc"
  [(set (reg:CCRAW CC_REGNUM)
	(unspec_volatile:CCRAW [(const_int TBEGINC_MASK)]
			       UNSPECV_TBEGINC))]
  "TARGET_HTM"
  "")

(define_insn "*tbeginc_1"
  [(set (reg:CCRAW CC_REGNUM)
	(unspec_volatile:CCRAW [(match_operand 0 "const_int_operand" " D")]
			       UNSPECV_TBEGINC))]
  "TARGET_HTM && INTVAL (operands[0]) >= 0 && INTVAL (operands[0]) <= 0xffff"
  "tbeginc\t0,%x0"
  [(set_attr "op_type" "SIL")])

; Transaction end

(define_expand "tend"
  [(set (reg:CCRAW CC_REGNUM)
	(unspec_volatile:CCRAW [(const_int 0)] UNSPECV_TEND))
   (set (match_operand:SI 0 "register_operand" "")
	(unspec:SI [(reg:CCRAW CC_REGNUM)] UNSPEC_CC_TO_INT))]
  "TARGET_HTM"
  "")

(define_insn "*tend_1"
  [(set (reg:CCRAW CC_REGNUM)
	(unspec_volatile:CCRAW [(const_int 0)] UNSPECV_TEND))]
  "TARGET_HTM"
  "tend"
  [(set_attr "op_type" "S")])

; Transaction abort

(define_expand "tabort"
  [(unspec_volatile [(match_operand:SI 0 "nonmemory_operand" "")]
		    UNSPECV_TABORT)]
  "TARGET_HTM && operands != NULL"
{
  if (CONST_INT_P (operands[0])
      && INTVAL (operands[0]) >= 0 && INTVAL (operands[0]) <= 255)
    {
      error ("invalid transaction abort code: %wd;  values in range 0 "
	     "through 255 are reserved", INTVAL (operands[0]));
      FAIL;
    }
})

(define_insn "*tabort_1"
  [(unspec_volatile [(match_operand:SI 0 "nonmemory_operand" "aJ")]
		    UNSPECV_TABORT)]
  "TARGET_HTM && operands != NULL"
  "tabort\t%Y0"
  [(set_attr "op_type" "S")])

(define_insn "*tabort_1_plus"
  [(unspec_volatile [(plus:SI (match_operand:SI 0 "register_operand"  "a")
			      (match_operand:SI 1 "const_int_operand" "J"))]
		    UNSPECV_TABORT)]
  "TARGET_HTM && operands != NULL
   && CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[1]), 'J', \"J\")"
  "tabort\t%1(%0)"
  [(set_attr "op_type" "S")])

; Transaction extract nesting depth

(define_insn "etnd"
  [(set (match_operand:SI 0 "register_operand" "=d")
	(unspec_volatile:SI [(const_int 0)] UNSPECV_ETND))]
  "TARGET_HTM"
  "etnd\t%0"
  [(set_attr "op_type" "RRE")])

; Non-transactional store

(define_insn "ntstg"
  [(set (match_operand:DI 0 "memory_operand" "=T")
	(unspec_volatile:DI [(match_operand:DI 1 "register_operand" "d")]
			    UNSPECV_NTSTG))]
  "TARGET_HTM"
  "ntstg\t%1,%0"
  [(set_attr "op_type" "RXY")])

; Transaction perform processor assist

(define_expand "tx_assist"
  [(unspec_volatile [(match_operand:SI 0 "register_operand" "")
		     (reg:SI GPR0_REGNUM)
		     (const_int PPA_TX_ABORT)]
		    UNSPECV_PPA)]
  "TARGET_HTM"
  "")

(define_insn "*ppa"
  [(unspec_volatile [(match_operand:SI 0 "register_operand" "d")
		     (match_operand:SI 1 "register_operand" "d")
		     (match_operand 2 "const_int_operand" "I")]
		    UNSPECV_PPA)]
  "(TARGET_ZEC12 || TARGET_HTM) && INTVAL (operands[2]) < 16"
  "ppa\t%0,%1,%2"
  [(set_attr "op_type" "RRF")])


; Set and get floating point control register

(define_insn "sfpc"
  [(unspec_volatile [(match_operand:SI 0 "register_operand" "d")]
		    UNSPECV_SFPC)]
  "TARGET_HARD_FLOAT"
  "sfpc\t%0")

(define_insn "efpc"
  [(set (match_operand:SI 0 "register_operand" "=d")
	(unspec_volatile:SI [(const_int 0)] UNSPECV_EFPC))]
  "TARGET_HARD_FLOAT"
  "efpc\t%0")


; Load count to block boundary

(define_insn "lcbb"
  [(set (match_operand:SI             0 "register_operand"  "=d")
	(unspec:SI [(match_operand    1 "address_operand" "ZR")
		    (match_operand:SI 2 "immediate_operand"  "C")] UNSPEC_LCBB))
   (clobber (reg:CC CC_REGNUM))]
  "TARGET_Z13"
  "lcbb\t%0,%a1,%b2"
  [(set_attr "op_type" "VRX")])

; Handle -fsplit-stack.

(define_expand "split_stack_prologue"
  [(const_int 0)]
  ""
{
  s390_expand_split_stack_prologue ();
  DONE;
})

;; If there are operand 0 bytes available on the stack, jump to
;; operand 1.

(define_expand "split_stack_space_check"
  [(set (pc) (if_then_else
	      (ltu (minus (reg 15)
			  (match_operand 0 "register_operand"))
		   (unspec [(const_int 0)] UNSPEC_STACK_CHECK))
	      (label_ref (match_operand 1))
	      (pc)))]
  ""
{
  /* Offset from thread pointer to __private_ss.  */
  int psso = TARGET_64BIT ? 0x38 : 0x20;
  rtx tp = s390_get_thread_pointer ();
  rtx guard = gen_rtx_MEM (Pmode, plus_constant (Pmode, tp, psso));
  rtx reg = gen_reg_rtx (Pmode);
  rtx cc;
  if (TARGET_64BIT)
    emit_insn (gen_subdi3 (reg, stack_pointer_rtx, operands[0]));
  else
    emit_insn (gen_subsi3 (reg, stack_pointer_rtx, operands[0]));
  cc = s390_emit_compare (GT, reg, guard);
  s390_emit_jump (operands[1], cc);

  DONE;
})

;; __morestack parameter block for split stack prologue.  Parameters are:
;; parameter block label, label to be called by __morestack, frame size,
;; stack parameter size.

(define_insn "split_stack_data"
  [(unspec_volatile [(match_operand 0 "" "X")
		     (match_operand 1 "" "X")
		     (match_operand 2 "const_int_operand" "X")
		     (match_operand 3 "const_int_operand" "X")]
		    UNSPECV_SPLIT_STACK_DATA)]
  ""
{
  switch_to_section (targetm.asm_out.function_rodata_section
		 (current_function_decl));

  if (TARGET_64BIT)
    output_asm_insn (".align\t8", operands);
  else
    output_asm_insn (".align\t4", operands);
  (*targetm.asm_out.internal_label) (asm_out_file, "L",
				     CODE_LABEL_NUMBER (operands[0]));
  if (TARGET_64BIT)
    {
      output_asm_insn (".quad\t%2", operands);
      output_asm_insn (".quad\t%3", operands);
      output_asm_insn (".quad\t%1-%0", operands);
    }
  else
    {
      output_asm_insn (".long\t%2", operands);
      output_asm_insn (".long\t%3", operands);
      output_asm_insn (".long\t%1-%0", operands);
    }

  switch_to_section (current_function_section ());
  return "";
}
  [(set_attr "length" "0")])


;; A jg with minimal fuss for use in split stack prologue.

(define_expand "split_stack_call"
  [(match_operand 0 "bras_sym_operand" "X")
   (match_operand 1 "" "")]
  ""
{
  if (TARGET_64BIT)
    emit_jump_insn (gen_split_stack_call_di (operands[0], operands[1]));
  else
    emit_jump_insn (gen_split_stack_call_si (operands[0], operands[1]));
  DONE;
})

(define_insn "split_stack_call_<mode>"
  [(set (pc) (label_ref (match_operand 1 "" "")))
   (set (reg:P 1) (unspec_volatile [(match_operand 0 "bras_sym_operand" "X")
				    (reg:P 1)]
				   UNSPECV_SPLIT_STACK_CALL))]
  ""
  "jg\t%0"
  [(set_attr "op_type" "RIL")
   (set_attr "type"  "branch")])

;; Also a conditional one.

(define_expand "split_stack_cond_call"
  [(match_operand 0 "bras_sym_operand" "X")
   (match_operand 1 "" "")
   (match_operand 2 "" "")]
  ""
{
  if (TARGET_64BIT)
    emit_jump_insn (gen_split_stack_cond_call_di (operands[0], operands[1], operands[2]));
  else
    emit_jump_insn (gen_split_stack_cond_call_si (operands[0], operands[1], operands[2]));
  DONE;
})

(define_insn "split_stack_cond_call_<mode>"
  [(set (pc)
	(if_then_else
	  (match_operand 1 "" "")
	  (label_ref (match_operand 2 "" ""))
	  (pc)))
   (set (reg:P 1) (unspec_volatile [(match_operand 0 "bras_sym_operand" "X")
				    (reg:P 1)]
				   UNSPECV_SPLIT_STACK_CALL))]
  ""
  "jg%C1\t%0"
  [(set_attr "op_type" "RIL")
   (set_attr "type"  "branch")])

(define_insn "osc_break"
  [(unspec_volatile [(const_int 0)] UNSPECV_OSC_BREAK)]
  ""
  "bcr\t7,%%r0"
  [(set_attr "op_type" "RR")])

(define_expand "speculation_barrier"
  [(unspec_volatile [(reg:SI GPR0_REGNUM)
		     (reg:SI GPR0_REGNUM)
		     (const_int PPA_OOO_BARRIER)]
		    UNSPECV_PPA)]
  "TARGET_ZEC12"
  "")