[thirdparty/gcc.git] / libatomic / config / arm / exch_n.c

/* Copyright (C) 2012-2019 Free Software Foundation, Inc.
   Contributed by Richard Henderson <rth@redhat.com>.

   This file is part of the GNU Atomic Library (libatomic).

   Libatomic is free software; you can redistribute it and/or modify it
   under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3 of the License, or
   (at your option) any later version.

   Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
   more details.

   Under Section 7 of GPL version 3, you are granted additional
   permissions described in the GCC Runtime Library Exception, version
   3.1, as published by the Free Software Foundation.

   You should have received a copy of the GNU General Public License and
   a copy of the GCC Runtime Library Exception along with this program;
   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
   <http://www.gnu.org/licenses/>.  */

#include <libatomic_i.h>
#include <arm-config.h>


/* When using STREX to implement sub-word exchange, we can do much better
   than the compiler by using the APSR.GE and APSR.C flags.  */

#if !DONE && __ARM_FEATURE_SIMD32 && HAVE_STREX && !HAVE_STREXBH && N == 2
UTYPE
SIZE(libat_exchange) (UTYPE *mptr, UTYPE newval, int smodel)
{
  UWORD t1, t2;
  UTYPE oldval;

  __atomic_thread_fence (__ATOMIC_SEQ_CST);

  /* In the N=2 case, there are only two cases for MPTR: mptr % 4 == {0,2}.
     Rather than computing a variable shift for this, we can store the one
     bit of misalignment in the carry flag, and use conditional constant
     shifts instead.  This saves a register.  */
#ifdef __ARMEB__
# define HI	"cc"				/* iff value is in high half */
# define LO	"cs"				/* iff value is in low half */
#else
# define HI	"cs"
# define LO	"cc"
#endif

  asm volatile (
		"lsrs	%[t2],%[ptr],#2\n"	/* carry = mptr & 2 */
	"	bic	%[ptr],%[ptr],#3\n"	/* align mptr */
	"	itt	"HI"\n"
	"	lsl"HI"	%[t1],%[t1],#16\n"	/* shift mask into place */
	"	lsl"HI"	%[new],%[new],#16\n"	/* shift newval into place */
	"	uadd16	%[t1],%[t1],%[t1]\n"	/* copy mask into APSR.GE */
	"0:	ldrex	%[t2],[%[ptr]]\n"
	"	ite	"LO"\n"
	"	uxth"LO" %[old],%[t2]\n"	/* return old value */
	"	uxth"HI" %[old],%[t2], ror #16\n"
	"	sel	%[t1],%[new],%[t2]\n"	/* merge newval */
	"	strex	%[t2],%[t1],[%[ptr]]\n"
	"	tst	%[t2],%[t2]\n"		/* dont clobber carry */
	"	bne	0b"
	: [old] "=&r"(oldval), [t1] "=&r"(t1), [t2] "=&r"(t2),
	  [ptr] "+r"(mptr), [new] "+r"(newval)
	: "1"(0xffff)
	: "memory");

  __atomic_thread_fence (__ATOMIC_SEQ_CST);

  return oldval;
}

#define DONE 1
#endif /* !HAVE_STREXBH && N == 2 */


#if !DONE && __ARM_FEATURE_SIMD32 && HAVE_STREX && !HAVE_STREXBH && N == 1
UTYPE
SIZE(libat_exchange) (UTYPE *mptr, UTYPE newval, int smodel)
{
  UWORD *wptr, woldval, wnewval, shift, mask, t1, t2;

  __atomic_thread_fence (__ATOMIC_SEQ_CST);

  wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE);
  shift = (((uintptr_t)mptr % WORDSIZE) * CHAR_BIT) ^ INVERT_MASK_1;
  mask = MASK_1 << shift;
  wnewval = newval << shift;

  asm volatile (
		"uadd8	%[t1],%[t1],%[t1]\n"	/* move mask to APSR.GE */
	"0:	ldrex	%[old],[%[wptr]]\n"
	"	sel	%[t1],%[new],%[old]\n"	/* merge newval */
	"	strex	%[t2],%[t1],[%[wptr]]\n"
	"	cmp	%[t2],#0\n"
	"	bne	0b"
	: [old] "=&r"(woldval), [t1] "=&r"(t1), [t2] "=&r"(t2)
	: [new] "r"(wnewval), [wptr] "r"(wptr), "1"(mask)
	: "memory");

  __atomic_thread_fence (__ATOMIC_SEQ_CST);

  return woldval >> shift;
}

#define DONE 1
#endif /* !HAVE_STREXBH && N == 1 */

#include "../../exch_n.c"
Commit	Line	Data
fbd26352	1	/* Copyright (C) 2012-2019 Free Software Foundation, Inc.
6db1d2ca	2	Contributed by Richard Henderson <rth@redhat.com>.
	3
	4	This file is part of the GNU Atomic Library (libatomic).
	5
	6	Libatomic is free software; you can redistribute it and/or modify it
	7	under the terms of the GNU General Public License as published by
	8	the Free Software Foundation; either version 3 of the License, or
	9	(at your option) any later version.
	10
	11	Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
	12	WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
	13	FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	14	more details.
	15
	16	Under Section 7 of GPL version 3, you are granted additional
	17	permissions described in the GCC Runtime Library Exception, version
	18	3.1, as published by the Free Software Foundation.
	19
	20	You should have received a copy of the GNU General Public License and
	21	a copy of the GCC Runtime Library Exception along with this program;
	22	see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
	23	<http://www.gnu.org/licenses/>. */
	24
	25	#include <libatomic_i.h>
	26	#include <arm-config.h>
	27
	28
	29	/* When using STREX to implement sub-word exchange, we can do much better
	30	than the compiler by using the APSR.GE and APSR.C flags. */
	31
6c0d8bb0	32	#if !DONE && __ARM_FEATURE_SIMD32 && HAVE_STREX && !HAVE_STREXBH && N == 2
6db1d2ca	33	UTYPE
	34	SIZE(libat_exchange) (UTYPE *mptr, UTYPE newval, int smodel)
	35	{
	36	UWORD t1, t2;
	37	UTYPE oldval;
	38
	39	__atomic_thread_fence (__ATOMIC_SEQ_CST);
	40
	41	/* In the N=2 case, there are only two cases for MPTR: mptr % 4 == {0,2}.
	42	Rather than computing a variable shift for this, we can store the one
	43	bit of misalignment in the carry flag, and use conditional constant
	44	shifts instead. This saves a register. */
	45	#ifdef __ARMEB__
	46	# define HI "cc" /* iff value is in high half */
	47	# define LO "cs" /* iff value is in low half */
	48	#else
	49	# define HI "cs"
	50	# define LO "cc"
	51	#endif
	52
	53	asm volatile (
	54	"lsrs %[t2],%[ptr],#2\n" /* carry = mptr & 2 */
	55	" bic %[ptr],%[ptr],#3\n" /* align mptr */
	56	" itt "HI"\n"
	57	" lsl"HI" %[t1],%[t1],#16\n" /* shift mask into place */
	58	" lsl"HI" %[new],%[new],#16\n" /* shift newval into place */
	59	" uadd16 %[t1],%[t1],%[t1]\n" /* copy mask into APSR.GE */
	60	"0: ldrex %[t2],[%[ptr]]\n"
	61	" ite "LO"\n"
	62	" uxth"LO" %[old],%[t2]\n" /* return old value */
	63	" uxth"HI" %[old],%[t2], ror #16\n"
	64	" sel %[t1],%[new],%[t2]\n" /* merge newval */
	65	" strex %[t2],%[t1],[%[ptr]]\n"
	66	" tst %[t2],%[t2]\n" /* dont clobber carry */
	67	" bne 0b"
	68	: [old] "=&r"(oldval), [t1] "=&r"(t1), [t2] "=&r"(t2),
	69	[ptr] "+r"(mptr), [new] "+r"(newval)
	70	: "1"(0xffff)
	71	: "memory");
	72
	73	__atomic_thread_fence (__ATOMIC_SEQ_CST);
	74
	75	return oldval;
	76	}
	77
	78	#define DONE 1
	79	#endif /* !HAVE_STREXBH && N == 2 */
	80
	81
6c0d8bb0	82	#if !DONE && __ARM_FEATURE_SIMD32 && HAVE_STREX && !HAVE_STREXBH && N == 1
6db1d2ca	83	UTYPE
	84	SIZE(libat_exchange) (UTYPE *mptr, UTYPE newval, int smodel)
	85	{
	86	UWORD *wptr, woldval, wnewval, shift, mask, t1, t2;
	87
	88	__atomic_thread_fence (__ATOMIC_SEQ_CST);
	89
	90	wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE);
	91	shift = (((uintptr_t)mptr % WORDSIZE) * CHAR_BIT) ^ INVERT_MASK_1;
	92	mask = MASK_1 << shift;
	93	wnewval = newval << shift;
	94
	95	asm volatile (
	96	"uadd8 %[t1],%[t1],%[t1]\n" /* move mask to APSR.GE */
	97	"0: ldrex %[old],[%[wptr]]\n"
	98	" sel %[t1],%[new],%[old]\n" /* merge newval */
	99	" strex %[t2],%[t1],[%[wptr]]\n"
	100	" cmp %[t2],#0\n"
	101	" bne 0b"
	102	: [old] "=&r"(woldval), [t1] "=&r"(t1), [t2] "=&r"(t2)
	103	: [new] "r"(wnewval), [wptr] "r"(wptr), "1"(mask)
	104	: "memory");
	105
	106	__atomic_thread_fence (__ATOMIC_SEQ_CST);
	107
	108	return woldval >> shift;
	109	}
	110
	111	#define DONE 1
	112	#endif /* !HAVE_STREXBH && N == 1 */
	113
	114	#include "../../exch_n.c"