[thirdparty/gcc.git] / gcc / config / arm / sync.md

;; Machine description for ARM processor synchronization primitives.
;; Copyright (C) 2010-2014 Free Software Foundation, Inc.
;; Written by Marcus Shawcroft (marcus.shawcroft@arm.com)
;; 64bit Atomics by Dave Gilbert (david.gilbert@linaro.org)
;;
;; This file is part of GCC.
;;
;; GCC is free software; you can redistribute it and/or modify it
;; under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 3, or (at your option)
;; any later version.
;;
;; GCC is distributed in the hope that it will be useful, but
;; WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
;; General Public License for more details.
;;
;; You should have received a copy of the GNU General Public License
;; along with GCC; see the file COPYING3.  If not see
;; <http://www.gnu.org/licenses/>.  */

(define_mode_attr sync_predtab
  [(QI "TARGET_HAVE_LDREXBH && TARGET_HAVE_MEMORY_BARRIER")
   (HI "TARGET_HAVE_LDREXBH && TARGET_HAVE_MEMORY_BARRIER")
   (SI "TARGET_HAVE_LDREX && TARGET_HAVE_MEMORY_BARRIER")
   (DI "TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN
	&& TARGET_HAVE_MEMORY_BARRIER")])

(define_code_iterator syncop [plus minus ior xor and])

(define_code_attr sync_optab
  [(ior "or") (xor "xor") (and "and") (plus "add") (minus "sub")])

(define_mode_attr sync_sfx
  [(QI "b") (HI "h") (SI "") (DI "d")])

(define_expand "memory_barrier"
  [(set (match_dup 0)
	(unspec:BLK [(match_dup 0)] UNSPEC_MEMORY_BARRIER))]
  "TARGET_HAVE_MEMORY_BARRIER"
{
  operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
  MEM_VOLATILE_P (operands[0]) = 1;
})

(define_insn "*memory_barrier"
  [(set (match_operand:BLK 0 "" "")
	(unspec:BLK [(match_dup 0)] UNSPEC_MEMORY_BARRIER))]
  "TARGET_HAVE_MEMORY_BARRIER"
  {
    if (TARGET_HAVE_DMB)
      {
	/* Note we issue a system level barrier. We should consider issuing
	   a inner shareabilty zone barrier here instead, ie. "DMB ISH".  */
	/* ??? Differentiate based on SEQ_CST vs less strict?  */
	return "dmb\tsy";
      }

    if (TARGET_HAVE_DMB_MCR)
      return "mcr\tp15, 0, r0, c7, c10, 5";

    gcc_unreachable ();
  }
  [(set_attr "length" "4")
   (set_attr "conds" "unconditional")
   (set_attr "predicable" "no")])

(define_insn "atomic_load<mode>"
  [(set (match_operand:QHSI 0 "register_operand" "=r")
    (unspec_volatile:QHSI
      [(match_operand:QHSI 1 "arm_sync_memory_operand" "Q")
       (match_operand:SI 2 "const_int_operand")]		;; model
      VUNSPEC_LDA))]
  "TARGET_HAVE_LDACQ"
  {
    enum memmodel model = (enum memmodel) INTVAL (operands[2]);
    if (model == MEMMODEL_RELAXED
        || model == MEMMODEL_CONSUME
        || model == MEMMODEL_RELEASE)
      return \"ldr<sync_sfx>\\t%0, %1\";
    else
      return \"lda<sync_sfx>\\t%0, %1\";
  }
)

(define_insn "atomic_store<mode>"
  [(set (match_operand:QHSI 0 "memory_operand" "=Q")
    (unspec_volatile:QHSI
      [(match_operand:QHSI 1 "general_operand" "r")
       (match_operand:SI 2 "const_int_operand")]		;; model
      VUNSPEC_STL))]
  "TARGET_HAVE_LDACQ"
  {
    enum memmodel model = (enum memmodel) INTVAL (operands[2]);
    if (model == MEMMODEL_RELAXED
        || model == MEMMODEL_CONSUME
        || model == MEMMODEL_ACQUIRE)
      return \"str<sync_sfx>\t%1, %0\";
    else
      return \"stl<sync_sfx>\t%1, %0\";
  }
)

;; Note that ldrd and vldr are *not* guaranteed to be single-copy atomic,
;; even for a 64-bit aligned address.  Instead we use a ldrexd unparied
;; with a store.
(define_expand "atomic_loaddi"
  [(match_operand:DI 0 "s_register_operand")		;; val out
   (match_operand:DI 1 "mem_noofs_operand")		;; memory
   (match_operand:SI 2 "const_int_operand")]		;; model
  "TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN"
{
  enum memmodel model = (enum memmodel) INTVAL (operands[2]);
  expand_mem_thread_fence (model);
  emit_insn (gen_atomic_loaddi_1 (operands[0], operands[1]));
  if (model == MEMMODEL_SEQ_CST)
    expand_mem_thread_fence (model);
  DONE;
})

(define_insn "atomic_loaddi_1"
  [(set (match_operand:DI 0 "s_register_operand" "=r")
	(unspec:DI [(match_operand:DI 1 "mem_noofs_operand" "Ua")]
		   UNSPEC_LL))]
  "TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN"
  "ldrexd%?\t%0, %H0, %C1"
  [(set_attr "predicable" "yes")
   (set_attr "predicable_short_it" "no")])

(define_expand "atomic_compare_and_swap<mode>"
  [(match_operand:SI 0 "s_register_operand" "")		;; bool out
   (match_operand:QHSD 1 "s_register_operand" "")	;; val out
   (match_operand:QHSD 2 "mem_noofs_operand" "")	;; memory
   (match_operand:QHSD 3 "general_operand" "")		;; expected
   (match_operand:QHSD 4 "s_register_operand" "")	;; desired
   (match_operand:SI 5 "const_int_operand")		;; is_weak
   (match_operand:SI 6 "const_int_operand")		;; mod_s
   (match_operand:SI 7 "const_int_operand")]		;; mod_f
  "<sync_predtab>"
{
  arm_expand_compare_and_swap (operands);
  DONE;
})

(define_insn_and_split "atomic_compare_and_swap<mode>_1"
  [(set (reg:CC_Z CC_REGNUM)					;; bool out
	(unspec_volatile:CC_Z [(const_int 0)] VUNSPEC_ATOMIC_CAS))
   (set (match_operand:SI 0 "s_register_operand" "=&r")		;; val out
	(zero_extend:SI
	  (match_operand:NARROW 1 "mem_noofs_operand" "+Ua")))	;; memory
   (set (match_dup 1)
	(unspec_volatile:NARROW
	  [(match_operand:SI 2 "arm_add_operand" "rIL")		;; expected
	   (match_operand:NARROW 3 "s_register_operand" "r")	;; desired
	   (match_operand:SI 4 "const_int_operand")		;; is_weak
	   (match_operand:SI 5 "const_int_operand")		;; mod_s
	   (match_operand:SI 6 "const_int_operand")]		;; mod_f
	  VUNSPEC_ATOMIC_CAS))
   (clobber (match_scratch:SI 7 "=&r"))]
  "<sync_predtab>"
  "#"
  "&& reload_completed"
  [(const_int 0)]
  {
    arm_split_compare_and_swap (operands);
    DONE;
  })

(define_mode_attr cas_cmp_operand
  [(SI "arm_add_operand") (DI "cmpdi_operand")])
(define_mode_attr cas_cmp_str
  [(SI "rIL") (DI "rDi")])

(define_insn_and_split "atomic_compare_and_swap<mode>_1"
  [(set (reg:CC_Z CC_REGNUM)					;; bool out
	(unspec_volatile:CC_Z [(const_int 0)] VUNSPEC_ATOMIC_CAS))
   (set (match_operand:SIDI 0 "s_register_operand" "=&r")	;; val out
	(match_operand:SIDI 1 "mem_noofs_operand" "+Ua"))	;; memory
   (set (match_dup 1)
	(unspec_volatile:SIDI
	  [(match_operand:SIDI 2 "<cas_cmp_operand>" "<cas_cmp_str>") ;; expect
	   (match_operand:SIDI 3 "s_register_operand" "r")	;; desired
	   (match_operand:SI 4 "const_int_operand")		;; is_weak
	   (match_operand:SI 5 "const_int_operand")		;; mod_s
	   (match_operand:SI 6 "const_int_operand")]		;; mod_f
	  VUNSPEC_ATOMIC_CAS))
   (clobber (match_scratch:SI 7 "=&r"))]
  "<sync_predtab>"
  "#"
  "&& reload_completed"
  [(const_int 0)]
  {
    arm_split_compare_and_swap (operands);
    DONE;
  })

(define_insn_and_split "atomic_exchange<mode>"
  [(set (match_operand:QHSD 0 "s_register_operand" "=&r")	;; output
	(match_operand:QHSD 1 "mem_noofs_operand" "+Ua"))	;; memory
   (set (match_dup 1)
	(unspec_volatile:QHSD
	  [(match_operand:QHSD 2 "s_register_operand" "r")	;; input
	   (match_operand:SI 3 "const_int_operand" "")]		;; model
	  VUNSPEC_ATOMIC_XCHG))
   (clobber (reg:CC CC_REGNUM))
   (clobber (match_scratch:SI 4 "=&r"))]
  "<sync_predtab>"
  "#"
  "&& reload_completed"
  [(const_int 0)]
  {
    arm_split_atomic_op (SET, operands[0], NULL, operands[1],
			 operands[2], operands[3], operands[4]);
    DONE;
  })

(define_mode_attr atomic_op_operand
  [(QI "reg_or_int_operand")
   (HI "reg_or_int_operand")
   (SI "reg_or_int_operand")
   (DI "s_register_operand")])

(define_mode_attr atomic_op_str
  [(QI "rn") (HI "rn") (SI "rn") (DI "r")])

(define_insn_and_split "atomic_<sync_optab><mode>"
  [(set (match_operand:QHSD 0 "mem_noofs_operand" "+Ua")
	(unspec_volatile:QHSD
	  [(syncop:QHSD (match_dup 0)
	     (match_operand:QHSD 1 "<atomic_op_operand>" "<atomic_op_str>"))
	   (match_operand:SI 2 "const_int_operand")]		;; model
	  VUNSPEC_ATOMIC_OP))
   (clobber (reg:CC CC_REGNUM))
   (clobber (match_scratch:QHSD 3 "=&r"))
   (clobber (match_scratch:SI 4 "=&r"))]
  "<sync_predtab>"
  "#"
  "&& reload_completed"
  [(const_int 0)]
  {
    arm_split_atomic_op (<CODE>, NULL, operands[3], operands[0],
			 operands[1], operands[2], operands[4]);
    DONE;
  })

(define_insn_and_split "atomic_nand<mode>"
  [(set (match_operand:QHSD 0 "mem_noofs_operand" "+Ua")
	(unspec_volatile:QHSD
	  [(not:QHSD
	     (and:QHSD (match_dup 0)
	       (match_operand:QHSD 1 "<atomic_op_operand>" "<atomic_op_str>")))
	   (match_operand:SI 2 "const_int_operand")]		;; model
	  VUNSPEC_ATOMIC_OP))
   (clobber (reg:CC CC_REGNUM))
   (clobber (match_scratch:QHSD 3 "=&r"))
   (clobber (match_scratch:SI 4 "=&r"))]
  "<sync_predtab>"
  "#"
  "&& reload_completed"
  [(const_int 0)]
  {
    arm_split_atomic_op (NOT, NULL, operands[3], operands[0],
			 operands[1], operands[2], operands[4]);
    DONE;
  })

(define_insn_and_split "atomic_fetch_<sync_optab><mode>"
  [(set (match_operand:QHSD 0 "s_register_operand" "=&r")
	(match_operand:QHSD 1 "mem_noofs_operand" "+Ua"))
   (set (match_dup 1)
	(unspec_volatile:QHSD
	  [(syncop:QHSD (match_dup 1)
	     (match_operand:QHSD 2 "<atomic_op_operand>" "<atomic_op_str>"))
	   (match_operand:SI 3 "const_int_operand")]		;; model
	  VUNSPEC_ATOMIC_OP))
   (clobber (reg:CC CC_REGNUM))
   (clobber (match_scratch:QHSD 4 "=&r"))
   (clobber (match_scratch:SI 5 "=&r"))]
  "<sync_predtab>"
  "#"
  "&& reload_completed"
  [(const_int 0)]
  {
    arm_split_atomic_op (<CODE>, operands[0], operands[4], operands[1],
			 operands[2], operands[3], operands[5]);
    DONE;
  })

(define_insn_and_split "atomic_fetch_nand<mode>"
  [(set (match_operand:QHSD 0 "s_register_operand" "=&r")
	(match_operand:QHSD 1 "mem_noofs_operand" "+Ua"))
   (set (match_dup 1)
	(unspec_volatile:QHSD
	  [(not:QHSD
	     (and:QHSD (match_dup 1)
	       (match_operand:QHSD 2 "<atomic_op_operand>" "<atomic_op_str>")))
	   (match_operand:SI 3 "const_int_operand")]		;; model
	  VUNSPEC_ATOMIC_OP))
   (clobber (reg:CC CC_REGNUM))
   (clobber (match_scratch:QHSD 4 "=&r"))
   (clobber (match_scratch:SI 5 "=&r"))]
  "<sync_predtab>"
  "#"
  "&& reload_completed"
  [(const_int 0)]
  {
    arm_split_atomic_op (NOT, operands[0], operands[4], operands[1],
			 operands[2], operands[3], operands[5]);
    DONE;
  })

(define_insn_and_split "atomic_<sync_optab>_fetch<mode>"
  [(set (match_operand:QHSD 0 "s_register_operand" "=&r")
	(syncop:QHSD
	  (match_operand:QHSD 1 "mem_noofs_operand" "+Ua")
	  (match_operand:QHSD 2 "<atomic_op_operand>" "<atomic_op_str>")))
   (set (match_dup 1)
	(unspec_volatile:QHSD
	  [(match_dup 1) (match_dup 2)
	   (match_operand:SI 3 "const_int_operand")]		;; model
	  VUNSPEC_ATOMIC_OP))
   (clobber (reg:CC CC_REGNUM))
   (clobber (match_scratch:SI 4 "=&r"))]
  "<sync_predtab>"
  "#"
  "&& reload_completed"
  [(const_int 0)]
  {
    arm_split_atomic_op (<CODE>, NULL, operands[0], operands[1],
			 operands[2], operands[3], operands[4]);
    DONE;
  })

(define_insn_and_split "atomic_nand_fetch<mode>"
  [(set (match_operand:QHSD 0 "s_register_operand" "=&r")
	(not:QHSD
	  (and:QHSD
	    (match_operand:QHSD 1 "mem_noofs_operand" "+Ua")
	    (match_operand:QHSD 2 "<atomic_op_operand>" "<atomic_op_str>"))))
   (set (match_dup 1)
	(unspec_volatile:QHSD
	  [(match_dup 1) (match_dup 2)
	   (match_operand:SI 3 "const_int_operand")]		;; model
	  VUNSPEC_ATOMIC_OP))
   (clobber (reg:CC CC_REGNUM))
   (clobber (match_scratch:SI 4 "=&r"))]
  "<sync_predtab>"
  "#"
  "&& reload_completed"
  [(const_int 0)]
  {
    arm_split_atomic_op (NOT, NULL, operands[0], operands[1],
			 operands[2], operands[3], operands[4]);
    DONE;
  })

(define_insn "arm_load_exclusive<mode>"
  [(set (match_operand:SI 0 "s_register_operand" "=r")
        (zero_extend:SI
	  (unspec_volatile:NARROW
	    [(match_operand:NARROW 1 "mem_noofs_operand" "Ua")]
	    VUNSPEC_LL)))]
  "TARGET_HAVE_LDREXBH"
  "ldrex<sync_sfx>%?\t%0, %C1"
  [(set_attr "predicable" "yes")
   (set_attr "predicable_short_it" "no")])

(define_insn "arm_load_acquire_exclusive<mode>"
  [(set (match_operand:SI 0 "s_register_operand" "=r")
        (zero_extend:SI
	  (unspec_volatile:NARROW
	    [(match_operand:NARROW 1 "mem_noofs_operand" "Ua")]
	    VUNSPEC_LAX)))]
  "TARGET_HAVE_LDACQ"
  "ldaex<sync_sfx>%?\\t%0, %C1"
  [(set_attr "predicable" "yes")
   (set_attr "predicable_short_it" "no")])

(define_insn "arm_load_exclusivesi"
  [(set (match_operand:SI 0 "s_register_operand" "=r")
	(unspec_volatile:SI
	  [(match_operand:SI 1 "mem_noofs_operand" "Ua")]
	  VUNSPEC_LL))]
  "TARGET_HAVE_LDREX"
  "ldrex%?\t%0, %C1"
  [(set_attr "predicable" "yes")
   (set_attr "predicable_short_it" "no")])

(define_insn "arm_load_acquire_exclusivesi"
  [(set (match_operand:SI 0 "s_register_operand" "=r")
	(unspec_volatile:SI
	  [(match_operand:SI 1 "mem_noofs_operand" "Ua")]
	  VUNSPEC_LAX))]
  "TARGET_HAVE_LDACQ"
  "ldaex%?\t%0, %C1"
  [(set_attr "predicable" "yes")
   (set_attr "predicable_short_it" "no")])

(define_insn "arm_load_exclusivedi"
  [(set (match_operand:DI 0 "s_register_operand" "=r")
	(unspec_volatile:DI
	  [(match_operand:DI 1 "mem_noofs_operand" "Ua")]
	  VUNSPEC_LL))]
  "TARGET_HAVE_LDREXD"
  "ldrexd%?\t%0, %H0, %C1"
  [(set_attr "predicable" "yes")
   (set_attr "predicable_short_it" "no")])

(define_insn "arm_load_acquire_exclusivedi"
  [(set (match_operand:DI 0 "s_register_operand" "=r")
	(unspec_volatile:DI
	  [(match_operand:DI 1 "mem_noofs_operand" "Ua")]
	  VUNSPEC_LAX))]
  "TARGET_HAVE_LDACQ && ARM_DOUBLEWORD_ALIGN"
  "ldaexd%?\t%0, %H0, %C1"
  [(set_attr "predicable" "yes")
   (set_attr "predicable_short_it" "no")])

(define_insn "arm_store_exclusive<mode>"
  [(set (match_operand:SI 0 "s_register_operand" "=&r")
	(unspec_volatile:SI [(const_int 0)] VUNSPEC_SC))
   (set (match_operand:QHSD 1 "mem_noofs_operand" "=Ua")
	(unspec_volatile:QHSD
	  [(match_operand:QHSD 2 "s_register_operand" "r")]
	  VUNSPEC_SC))]
  "<sync_predtab>"
  {
    if (<MODE>mode == DImode)
      {
	rtx value = operands[2];
	/* The restrictions on target registers in ARM mode are that the two
	   registers are consecutive and the first one is even; Thumb is
	   actually more flexible, but DI should give us this anyway.
	   Note that the 1st register always gets the lowest word in memory.  */
	gcc_assert ((REGNO (value) & 1) == 0 || TARGET_THUMB2);
	operands[3] = gen_rtx_REG (SImode, REGNO (value) + 1);
	return "strexd%?\t%0, %2, %3, %C1";
      }
    return "strex<sync_sfx>%?\t%0, %2, %C1";
  }
  [(set_attr "predicable" "yes")
   (set_attr "predicable_short_it" "no")])

(define_insn "arm_store_release_exclusivedi"
  [(set (match_operand:SI 0 "s_register_operand" "=&r")
	(unspec_volatile:SI [(const_int 0)] VUNSPEC_SLX))
   (set (match_operand:DI 1 "mem_noofs_operand" "=Ua")
	(unspec_volatile:DI
	  [(match_operand:DI 2 "s_register_operand" "r")]
	  VUNSPEC_SLX))]
  "TARGET_HAVE_LDACQ && ARM_DOUBLEWORD_ALIGN"
  {
    rtx value = operands[2];
    /* See comment in arm_store_exclusive<mode> above.  */
    gcc_assert ((REGNO (value) & 1) == 0 || TARGET_THUMB2);
    operands[3] = gen_rtx_REG (SImode, REGNO (value) + 1);
    return "stlexd%?\t%0, %2, %3, %C1";
  }
  [(set_attr "predicable" "yes")
   (set_attr "predicable_short_it" "no")])

(define_insn "arm_store_release_exclusive<mode>"
  [(set (match_operand:SI 0 "s_register_operand" "=&r")
	(unspec_volatile:SI [(const_int 0)] VUNSPEC_SLX))
   (set (match_operand:QHSI 1 "mem_noofs_operand" "=Ua")
	(unspec_volatile:QHSI
	  [(match_operand:QHSI 2 "s_register_operand" "r")]
	  VUNSPEC_SLX))]
  "TARGET_HAVE_LDACQ"
  "stlex<sync_sfx>%?\t%0, %2, %C1"
  [(set_attr "predicable" "yes")
   (set_attr "predicable_short_it" "no")])
Commit	Line	Data
029e79eb	1	;; Machine description for ARM processor synchronization primitives.
23a5b65a	2	;; Copyright (C) 2010-2014 Free Software Foundation, Inc.
029e79eb	3	;; Written by Marcus Shawcroft (marcus.shawcroft@arm.com)
cfe52743	4	;; 64bit Atomics by Dave Gilbert (david.gilbert@linaro.org)
029e79eb MS	5	;;
	6	;; This file is part of GCC.
	7	;;
	8	;; GCC is free software; you can redistribute it and/or modify it
	9	;; under the terms of the GNU General Public License as published by
	10	;; the Free Software Foundation; either version 3, or (at your option)
	11	;; any later version.
	12	;;
	13	;; GCC is distributed in the hope that it will be useful, but
	14	;; WITHOUT ANY WARRANTY; without even the implied warranty of
	15	;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	;; General Public License for more details.
	17	;;
	18	;; You should have received a copy of the GNU General Public License
	19	;; along with GCC; see the file COPYING3. If not see
	20	;; <http://www.gnu.org/licenses/>. */
	21
18f0fe6b RH	22	(define_mode_attr sync_predtab
	23	[(QI "TARGET_HAVE_LDREXBH && TARGET_HAVE_MEMORY_BARRIER")
	24	(HI "TARGET_HAVE_LDREXBH && TARGET_HAVE_MEMORY_BARRIER")
	25	(SI "TARGET_HAVE_LDREX && TARGET_HAVE_MEMORY_BARRIER")
	26	(DI "TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN
	27	&& TARGET_HAVE_MEMORY_BARRIER")])
	28
	29	(define_code_iterator syncop [plus minus ior xor and])
	30
	31	(define_code_attr sync_optab
e4eb230d	32	[(ior "or") (xor "xor") (and "and") (plus "add") (minus "sub")])
18f0fe6b RH	33
	34	(define_mode_attr sync_sfx
	35	[(QI "b") (HI "h") (SI "") (DI "d")])
029e79eb MS	36
	37	(define_expand "memory_barrier"
	38	[(set (match_dup 0)
	39	(unspec:BLK [(match_dup 0)] UNSPEC_MEMORY_BARRIER))]
	40	"TARGET_HAVE_MEMORY_BARRIER"
	41	{
	42	operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
	43	MEM_VOLATILE_P (operands[0]) = 1;
	44	})
	45
18f0fe6b RH	46	(define_insn "*memory_barrier"
	47	[(set (match_operand:BLK 0 "" "")
	48	(unspec:BLK [(match_dup 0)] UNSPEC_MEMORY_BARRIER))]
	49	"TARGET_HAVE_MEMORY_BARRIER"
029e79eb	50	{
18f0fe6b RH	51	if (TARGET_HAVE_DMB)
	52	{
	53	/* Note we issue a system level barrier. We should consider issuing
	54	a inner shareabilty zone barrier here instead, ie. "DMB ISH". */
	55	/* ??? Differentiate based on SEQ_CST vs less strict? */
	56	return "dmb\tsy";
	57	}
029e79eb	58
18f0fe6b RH	59	if (TARGET_HAVE_DMB_MCR)
18f0fe6b RH	60	return "mcr\tp15, 0, r0, c7, c10, 5";
b7b79b54	61
18f0fe6b RH	62	gcc_unreachable ();
	63	}
	64	[(set_attr "length" "4")
	65	(set_attr "conds" "unconditional")
	66	(set_attr "predicable" "no")])
029e79eb	67
5ad29f12 KT	68	(define_insn "atomic_load<mode>"
	69	[(set (match_operand:QHSI 0 "register_operand" "=r")
	70	(unspec_volatile:QHSI
	71	[(match_operand:QHSI 1 "arm_sync_memory_operand" "Q")
	72	(match_operand:SI 2 "const_int_operand")] ;; model
	73	VUNSPEC_LDA))]
	74	"TARGET_HAVE_LDACQ"
	75	{
	76	enum memmodel model = (enum memmodel) INTVAL (operands[2]);
	77	if (model == MEMMODEL_RELAXED
	78	\|\| model == MEMMODEL_CONSUME
	79	\|\| model == MEMMODEL_RELEASE)
	80	return \"ldr<sync_sfx>\\t%0, %1\";
	81	else
	82	return \"lda<sync_sfx>\\t%0, %1\";
	83	}
	84	)
	85
	86	(define_insn "atomic_store<mode>"
	87	[(set (match_operand:QHSI 0 "memory_operand" "=Q")
	88	(unspec_volatile:QHSI
	89	[(match_operand:QHSI 1 "general_operand" "r")
	90	(match_operand:SI 2 "const_int_operand")] ;; model
	91	VUNSPEC_STL))]
	92	"TARGET_HAVE_LDACQ"
	93	{
	94	enum memmodel model = (enum memmodel) INTVAL (operands[2]);
	95	if (model == MEMMODEL_RELAXED
	96	\|\| model == MEMMODEL_CONSUME
	97	\|\| model == MEMMODEL_ACQUIRE)
	98	return \"str<sync_sfx>\t%1, %0\";
	99	else
	100	return \"stl<sync_sfx>\t%1, %0\";
	101	}
	102	)
	103
a7b8107f RH	104	;; Note that ldrd and vldr are not guaranteed to be single-copy atomic,
	105	;; even for a 64-bit aligned address. Instead we use a ldrexd unparied
	106	;; with a store.
	107	(define_expand "atomic_loaddi"
	108	[(match_operand:DI 0 "s_register_operand") ;; val out
	109	(match_operand:DI 1 "mem_noofs_operand") ;; memory
	110	(match_operand:SI 2 "const_int_operand")] ;; model
	111	"TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN"
	112	{
	113	enum memmodel model = (enum memmodel) INTVAL (operands[2]);
	114	expand_mem_thread_fence (model);
	115	emit_insn (gen_atomic_loaddi_1 (operands[0], operands[1]));
	116	if (model == MEMMODEL_SEQ_CST)
	117	expand_mem_thread_fence (model);
	118	DONE;
	119	})
	120
	121	(define_insn "atomic_loaddi_1"
	122	[(set (match_operand:DI 0 "s_register_operand" "=r")
	123	(unspec:DI [(match_operand:DI 1 "mem_noofs_operand" "Ua")]
	124	UNSPEC_LL))]
	125	"TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN"
	126	"ldrexd%?\t%0, %H0, %C1"
12b4e7ef KT	127	[(set_attr "predicable" "yes")
12b4e7ef KT	128	(set_attr "predicable_short_it" "no")])
a7b8107f	129
18f0fe6b RH	130	(define_expand "atomic_compare_and_swap<mode>"
	131	[(match_operand:SI 0 "s_register_operand" "") ;; bool out
	132	(match_operand:QHSD 1 "s_register_operand" "") ;; val out
	133	(match_operand:QHSD 2 "mem_noofs_operand" "") ;; memory
	134	(match_operand:QHSD 3 "general_operand" "") ;; expected
	135	(match_operand:QHSD 4 "s_register_operand" "") ;; desired
	136	(match_operand:SI 5 "const_int_operand") ;; is_weak
	137	(match_operand:SI 6 "const_int_operand") ;; mod_s
	138	(match_operand:SI 7 "const_int_operand")] ;; mod_f
cfe52743	139	"<sync_predtab>"
18f0fe6b RH	140	{
	141	arm_expand_compare_and_swap (operands);
	142	DONE;
	143	})
029e79eb	144
18f0fe6b RH	145	(define_insn_and_split "atomic_compare_and_swap<mode>_1"
	146	[(set (reg:CC_Z CC_REGNUM) ;; bool out
	147	(unspec_volatile:CC_Z [(const_int 0)] VUNSPEC_ATOMIC_CAS))
	148	(set (match_operand:SI 0 "s_register_operand" "=&r") ;; val out
	149	(zero_extend:SI
	150	(match_operand:NARROW 1 "mem_noofs_operand" "+Ua"))) ;; memory
	151	(set (match_dup 1)
	152	(unspec_volatile:NARROW
	153	[(match_operand:SI 2 "arm_add_operand" "rIL") ;; expected
	154	(match_operand:NARROW 3 "s_register_operand" "r") ;; desired
	155	(match_operand:SI 4 "const_int_operand") ;; is_weak
	156	(match_operand:SI 5 "const_int_operand") ;; mod_s
	157	(match_operand:SI 6 "const_int_operand")] ;; mod_f
	158	VUNSPEC_ATOMIC_CAS))
	159	(clobber (match_scratch:SI 7 "=&r"))]
cfe52743	160	"<sync_predtab>"
18f0fe6b RH	161	"#"
	162	"&& reload_completed"
	163	[(const_int 0)]
029e79eb	164	{
18f0fe6b	165	arm_split_compare_and_swap (operands);
029e79eb MS	166	DONE;
	167	})
	168
18f0fe6b RH	169	(define_mode_attr cas_cmp_operand
	170	[(SI "arm_add_operand") (DI "cmpdi_operand")])
	171	(define_mode_attr cas_cmp_str
	172	[(SI "rIL") (DI "rDi")])
029e79eb	173
18f0fe6b RH	174	(define_insn_and_split "atomic_compare_and_swap<mode>_1"
	175	[(set (reg:CC_Z CC_REGNUM) ;; bool out
	176	(unspec_volatile:CC_Z [(const_int 0)] VUNSPEC_ATOMIC_CAS))
	177	(set (match_operand:SIDI 0 "s_register_operand" "=&r") ;; val out
	178	(match_operand:SIDI 1 "mem_noofs_operand" "+Ua")) ;; memory
	179	(set (match_dup 1)
	180	(unspec_volatile:SIDI
	181	[(match_operand:SIDI 2 "<cas_cmp_operand>" "<cas_cmp_str>") ;; expect
	182	(match_operand:SIDI 3 "s_register_operand" "r") ;; desired
	183	(match_operand:SI 4 "const_int_operand") ;; is_weak
	184	(match_operand:SI 5 "const_int_operand") ;; mod_s
	185	(match_operand:SI 6 "const_int_operand")] ;; mod_f
	186	VUNSPEC_ATOMIC_CAS))
	187	(clobber (match_scratch:SI 7 "=&r"))]
cfe52743	188	"<sync_predtab>"
18f0fe6b RH	189	"#"
	190	"&& reload_completed"
	191	[(const_int 0)]
029e79eb	192	{
18f0fe6b	193	arm_split_compare_and_swap (operands);
029e79eb MS	194	DONE;
	195	})
	196
18f0fe6b RH	197	(define_insn_and_split "atomic_exchange<mode>"
	198	[(set (match_operand:QHSD 0 "s_register_operand" "=&r") ;; output
	199	(match_operand:QHSD 1 "mem_noofs_operand" "+Ua")) ;; memory
	200	(set (match_dup 1)
	201	(unspec_volatile:QHSD
	202	[(match_operand:QHSD 2 "s_register_operand" "r") ;; input
	203	(match_operand:SI 3 "const_int_operand" "")] ;; model
	204	VUNSPEC_ATOMIC_XCHG))
	205	(clobber (reg:CC CC_REGNUM))
	206	(clobber (match_scratch:SI 4 "=&r"))]
cfe52743	207	"<sync_predtab>"
18f0fe6b RH	208	"#"
	209	"&& reload_completed"
	210	[(const_int 0)]
029e79eb	211	{
18f0fe6b RH	212	arm_split_atomic_op (SET, operands[0], NULL, operands[1],
18f0fe6b RH	213	operands[2], operands[3], operands[4]);
029e79eb MS	214	DONE;
	215	})
	216
18f0fe6b RH	217	(define_mode_attr atomic_op_operand
	218	[(QI "reg_or_int_operand")
	219	(HI "reg_or_int_operand")
	220	(SI "reg_or_int_operand")
	221	(DI "s_register_operand")])
029e79eb	222
18f0fe6b RH	223	(define_mode_attr atomic_op_str
18f0fe6b RH	224	[(QI "rn") (HI "rn") (SI "rn") (DI "r")])
029e79eb	225
18f0fe6b RH	226	(define_insn_and_split "atomic_<sync_optab><mode>"
	227	[(set (match_operand:QHSD 0 "mem_noofs_operand" "+Ua")
	228	(unspec_volatile:QHSD
	229	[(syncop:QHSD (match_dup 0)
	230	(match_operand:QHSD 1 "<atomic_op_operand>" "<atomic_op_str>"))
	231	(match_operand:SI 2 "const_int_operand")] ;; model
	232	VUNSPEC_ATOMIC_OP))
029e79eb	233	(clobber (reg:CC CC_REGNUM))
18f0fe6b RH	234	(clobber (match_scratch:QHSD 3 "=&r"))
18f0fe6b RH	235	(clobber (match_scratch:SI 4 "=&r"))]
cfe52743	236	"<sync_predtab>"
18f0fe6b RH	237	"#"
	238	"&& reload_completed"
	239	[(const_int 0)]
029e79eb	240	{
18f0fe6b RH	241	arm_split_atomic_op (<CODE>, NULL, operands[3], operands[0],
	242	operands[1], operands[2], operands[4]);
	243	DONE;
	244	})
029e79eb	245
18f0fe6b RH	246	(define_insn_and_split "atomic_nand<mode>"
	247	[(set (match_operand:QHSD 0 "mem_noofs_operand" "+Ua")
	248	(unspec_volatile:QHSD
	249	[(not:QHSD
	250	(and:QHSD (match_dup 0)
	251	(match_operand:QHSD 1 "<atomic_op_operand>" "<atomic_op_str>")))
	252	(match_operand:SI 2 "const_int_operand")] ;; model
	253	VUNSPEC_ATOMIC_OP))
029e79eb	254	(clobber (reg:CC CC_REGNUM))
18f0fe6b RH	255	(clobber (match_scratch:QHSD 3 "=&r"))
18f0fe6b RH	256	(clobber (match_scratch:SI 4 "=&r"))]
cfe52743	257	"<sync_predtab>"
18f0fe6b RH	258	"#"
	259	"&& reload_completed"
	260	[(const_int 0)]
029e79eb	261	{
18f0fe6b RH	262	arm_split_atomic_op (NOT, NULL, operands[3], operands[0],
	263	operands[1], operands[2], operands[4]);
	264	DONE;
	265	})
029e79eb	266
18f0fe6b RH	267	(define_insn_and_split "atomic_fetch_<sync_optab><mode>"
	268	[(set (match_operand:QHSD 0 "s_register_operand" "=&r")
	269	(match_operand:QHSD 1 "mem_noofs_operand" "+Ua"))
029e79eb	270	(set (match_dup 1)
18f0fe6b RH	271	(unspec_volatile:QHSD
	272	[(syncop:QHSD (match_dup 1)
	273	(match_operand:QHSD 2 "<atomic_op_operand>" "<atomic_op_str>"))
	274	(match_operand:SI 3 "const_int_operand")] ;; model
	275	VUNSPEC_ATOMIC_OP))
029e79eb	276	(clobber (reg:CC CC_REGNUM))
18f0fe6b RH	277	(clobber (match_scratch:QHSD 4 "=&r"))
18f0fe6b RH	278	(clobber (match_scratch:SI 5 "=&r"))]
cfe52743	279	"<sync_predtab>"
18f0fe6b RH	280	"#"
	281	"&& reload_completed"
	282	[(const_int 0)]
029e79eb	283	{
18f0fe6b RH	284	arm_split_atomic_op (<CODE>, operands[0], operands[4], operands[1],
	285	operands[2], operands[3], operands[5]);
	286	DONE;
	287	})
029e79eb	288
18f0fe6b RH	289	(define_insn_and_split "atomic_fetch_nand<mode>"
	290	[(set (match_operand:QHSD 0 "s_register_operand" "=&r")
	291	(match_operand:QHSD 1 "mem_noofs_operand" "+Ua"))
029e79eb	292	(set (match_dup 1)
18f0fe6b RH	293	(unspec_volatile:QHSD
	294	[(not:QHSD
	295	(and:QHSD (match_dup 1)
	296	(match_operand:QHSD 2 "<atomic_op_operand>" "<atomic_op_str>")))
	297	(match_operand:SI 3 "const_int_operand")] ;; model
	298	VUNSPEC_ATOMIC_OP))
029e79eb	299	(clobber (reg:CC CC_REGNUM))
18f0fe6b RH	300	(clobber (match_scratch:QHSD 4 "=&r"))
18f0fe6b RH	301	(clobber (match_scratch:SI 5 "=&r"))]
cfe52743	302	"<sync_predtab>"
18f0fe6b RH	303	"#"
	304	"&& reload_completed"
	305	[(const_int 0)]
029e79eb	306	{
18f0fe6b RH	307	arm_split_atomic_op (NOT, operands[0], operands[4], operands[1],
	308	operands[2], operands[3], operands[5]);
	309	DONE;
	310	})
029e79eb	311
18f0fe6b RH	312	(define_insn_and_split "atomic_<sync_optab>_fetch<mode>"
	313	[(set (match_operand:QHSD 0 "s_register_operand" "=&r")
	314	(syncop:QHSD
	315	(match_operand:QHSD 1 "mem_noofs_operand" "+Ua")
	316	(match_operand:QHSD 2 "<atomic_op_operand>" "<atomic_op_str>")))
029e79eb	317	(set (match_dup 1)
18f0fe6b RH	318	(unspec_volatile:QHSD
	319	[(match_dup 1) (match_dup 2)
	320	(match_operand:SI 3 "const_int_operand")] ;; model
	321	VUNSPEC_ATOMIC_OP))
029e79eb	322	(clobber (reg:CC CC_REGNUM))
18f0fe6b	323	(clobber (match_scratch:SI 4 "=&r"))]
cfe52743	324	"<sync_predtab>"
18f0fe6b RH	325	"#"
	326	"&& reload_completed"
	327	[(const_int 0)]
029e79eb	328	{
18f0fe6b RH	329	arm_split_atomic_op (<CODE>, NULL, operands[0], operands[1],
	330	operands[2], operands[3], operands[4]);
	331	DONE;
	332	})
029e79eb	333
18f0fe6b RH	334	(define_insn_and_split "atomic_nand_fetch<mode>"
	335	[(set (match_operand:QHSD 0 "s_register_operand" "=&r")
	336	(not:QHSD
	337	(and:QHSD
	338	(match_operand:QHSD 1 "mem_noofs_operand" "+Ua")
	339	(match_operand:QHSD 2 "<atomic_op_operand>" "<atomic_op_str>"))))
029e79eb	340	(set (match_dup 1)
18f0fe6b RH	341	(unspec_volatile:QHSD
	342	[(match_dup 1) (match_dup 2)
	343	(match_operand:SI 3 "const_int_operand")] ;; model
	344	VUNSPEC_ATOMIC_OP))
029e79eb	345	(clobber (reg:CC CC_REGNUM))
18f0fe6b	346	(clobber (match_scratch:SI 4 "=&r"))]
cfe52743	347	"<sync_predtab>"
18f0fe6b RH	348	"#"
	349	"&& reload_completed"
	350	[(const_int 0)]
029e79eb	351	{
18f0fe6b RH	352	arm_split_atomic_op (NOT, NULL, operands[0], operands[1],
	353	operands[2], operands[3], operands[4]);
	354	DONE;
	355	})
029e79eb	356
18f0fe6b RH	357	(define_insn "arm_load_exclusive<mode>"
	358	[(set (match_operand:SI 0 "s_register_operand" "=r")
	359	(zero_extend:SI
	360	(unspec_volatile:NARROW
	361	[(match_operand:NARROW 1 "mem_noofs_operand" "Ua")]
	362	VUNSPEC_LL)))]
	363	"TARGET_HAVE_LDREXBH"
	364	"ldrex<sync_sfx>%?\t%0, %C1"
12b4e7ef KT	365	[(set_attr "predicable" "yes")
12b4e7ef KT	366	(set_attr "predicable_short_it" "no")])
029e79eb	367
5ad29f12 KT	368	(define_insn "arm_load_acquire_exclusive<mode>"
	369	[(set (match_operand:SI 0 "s_register_operand" "=r")
	370	(zero_extend:SI
	371	(unspec_volatile:NARROW
	372	[(match_operand:NARROW 1 "mem_noofs_operand" "Ua")]
	373	VUNSPEC_LAX)))]
	374	"TARGET_HAVE_LDACQ"
	375	"ldaex<sync_sfx>%?\\t%0, %C1"
12b4e7ef KT	376	[(set_attr "predicable" "yes")
12b4e7ef KT	377	(set_attr "predicable_short_it" "no")])
5ad29f12	378
18f0fe6b RH	379	(define_insn "arm_load_exclusivesi"
	380	[(set (match_operand:SI 0 "s_register_operand" "=r")
	381	(unspec_volatile:SI
	382	[(match_operand:SI 1 "mem_noofs_operand" "Ua")]
	383	VUNSPEC_LL))]
	384	"TARGET_HAVE_LDREX"
	385	"ldrex%?\t%0, %C1"
12b4e7ef KT	386	[(set_attr "predicable" "yes")
12b4e7ef KT	387	(set_attr "predicable_short_it" "no")])
029e79eb	388
5ad29f12 KT	389	(define_insn "arm_load_acquire_exclusivesi"
	390	[(set (match_operand:SI 0 "s_register_operand" "=r")
	391	(unspec_volatile:SI
	392	[(match_operand:SI 1 "mem_noofs_operand" "Ua")]
	393	VUNSPEC_LAX))]
	394	"TARGET_HAVE_LDACQ"
	395	"ldaex%?\t%0, %C1"
12b4e7ef KT	396	[(set_attr "predicable" "yes")
12b4e7ef KT	397	(set_attr "predicable_short_it" "no")])
5ad29f12	398
18f0fe6b RH	399	(define_insn "arm_load_exclusivedi"
	400	[(set (match_operand:DI 0 "s_register_operand" "=r")
	401	(unspec_volatile:DI
	402	[(match_operand:DI 1 "mem_noofs_operand" "Ua")]
	403	VUNSPEC_LL))]
	404	"TARGET_HAVE_LDREXD"
a7b8107f	405	"ldrexd%?\t%0, %H0, %C1"
12b4e7ef KT	406	[(set_attr "predicable" "yes")
12b4e7ef KT	407	(set_attr "predicable_short_it" "no")])
029e79eb	408
5ad29f12 KT	409	(define_insn "arm_load_acquire_exclusivedi"
	410	[(set (match_operand:DI 0 "s_register_operand" "=r")
	411	(unspec_volatile:DI
	412	[(match_operand:DI 1 "mem_noofs_operand" "Ua")]
	413	VUNSPEC_LAX))]
	414	"TARGET_HAVE_LDACQ && ARM_DOUBLEWORD_ALIGN"
	415	"ldaexd%?\t%0, %H0, %C1"
12b4e7ef KT	416	[(set_attr "predicable" "yes")
12b4e7ef KT	417	(set_attr "predicable_short_it" "no")])
5ad29f12	418
18f0fe6b	419	(define_insn "arm_store_exclusive<mode>"
029e79eb	420	[(set (match_operand:SI 0 "s_register_operand" "=&r")
18f0fe6b RH	421	(unspec_volatile:SI [(const_int 0)] VUNSPEC_SC))
	422	(set (match_operand:QHSD 1 "mem_noofs_operand" "=Ua")
	423	(unspec_volatile:QHSD
	424	[(match_operand:QHSD 2 "s_register_operand" "r")]
	425	VUNSPEC_SC))]
cfe52743	426	"<sync_predtab>"
029e79eb	427	{
18f0fe6b RH	428	if (<MODE>mode == DImode)
	429	{
	430	rtx value = operands[2];
	431	/* The restrictions on target registers in ARM mode are that the two
	432	registers are consecutive and the first one is even; Thumb is
	433	actually more flexible, but DI should give us this anyway.
	434	Note that the 1st register always gets the lowest word in memory. */
	435	gcc_assert ((REGNO (value) & 1) == 0 \|\| TARGET_THUMB2);
	436	operands[3] = gen_rtx_REG (SImode, REGNO (value) + 1);
	437	return "strexd%?\t%0, %2, %3, %C1";
	438	}
	439	return "strex<sync_sfx>%?\t%0, %2, %C1";
029e79eb	440	}
12b4e7ef KT	441	[(set_attr "predicable" "yes")
12b4e7ef KT	442	(set_attr "predicable_short_it" "no")])
5ad29f12 KT	443
	444	(define_insn "arm_store_release_exclusivedi"
	445	[(set (match_operand:SI 0 "s_register_operand" "=&r")
	446	(unspec_volatile:SI [(const_int 0)] VUNSPEC_SLX))
	447	(set (match_operand:DI 1 "mem_noofs_operand" "=Ua")
	448	(unspec_volatile:DI
	449	[(match_operand:DI 2 "s_register_operand" "r")]
	450	VUNSPEC_SLX))]
	451	"TARGET_HAVE_LDACQ && ARM_DOUBLEWORD_ALIGN"
	452	{
	453	rtx value = operands[2];
	454	/* See comment in arm_store_exclusive<mode> above. */
	455	gcc_assert ((REGNO (value) & 1) == 0 \|\| TARGET_THUMB2);
	456	operands[3] = gen_rtx_REG (SImode, REGNO (value) + 1);
	457	return "stlexd%?\t%0, %2, %3, %C1";
	458	}
12b4e7ef KT	459	[(set_attr "predicable" "yes")
12b4e7ef KT	460	(set_attr "predicable_short_it" "no")])
5ad29f12 KT	461
	462	(define_insn "arm_store_release_exclusive<mode>"
	463	[(set (match_operand:SI 0 "s_register_operand" "=&r")
	464	(unspec_volatile:SI [(const_int 0)] VUNSPEC_SLX))
	465	(set (match_operand:QHSI 1 "mem_noofs_operand" "=Ua")
	466	(unspec_volatile:QHSI
	467	[(match_operand:QHSI 2 "s_register_operand" "r")]
	468	VUNSPEC_SLX))]
	469	"TARGET_HAVE_LDACQ"
	470	"stlex<sync_sfx>%?\t%0, %2, %C1"
12b4e7ef KT	471	[(set_attr "predicable" "yes")
12b4e7ef KT	472	(set_attr "predicable_short_it" "no")])