[thirdparty/gcc.git] / libgcc / config / nds32 / linux-atomic.c

/* Linux-specific atomic operations for NDS32 Linux.
   Copyright (C) 2012-2020 Free Software Foundation, Inc.

This file is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 3, or (at your option) any
later version.

This file is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
General Public License for more details.

Under Section 7 of GPL version 3, you are granted additional
permissions described in the GCC Runtime Library Exception, version
3.1, as published by the Free Software Foundation.

You should have received a copy of the GNU General Public License and
a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
<http://www.gnu.org/licenses/>.  */

/* We implement byte, short and int versions of each atomic operation
   using the kernel helper defined below.  There is no support for
   64-bit operations yet.  */

/* This function copy form NDS32 Linux-kernal. */
static inline int
__kernel_cmpxchg (int oldval, int newval, int *mem)
{
  int temp1, temp2, temp3, offset;

  asm volatile ("msync\tall\n"
		"movi\t%0, #0\n"
		"1:\n"
		"\tllw\t%1, [%4+%0]\n"
		"\tsub\t%3, %1, %6\n"
		"\tcmovz\t%2, %5, %3\n"
		"\tcmovn\t%2, %1, %3\n"
		"\tscw\t%2, [%4+%0]\n"
		"\tbeqz\t%2, 1b\n"
		: "=&r" (offset), "=&r" (temp3), "=&r" (temp2), "=&r" (temp1)
		: "r" (mem), "r" (newval), "r" (oldval) : "memory");

  return temp1;
}

#define HIDDEN __attribute__ ((visibility ("hidden")))

#ifdef __NDS32_EL__
#define INVERT_MASK_1 0
#define INVERT_MASK_2 0
#else
#define INVERT_MASK_1 24
#define INVERT_MASK_2 16
#endif

#define MASK_1 0xffu
#define MASK_2 0xffffu

#define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP)				\
  int HIDDEN								\
  __sync_fetch_and_##OP##_4 (int *ptr, int val)				\
  {									\
    int failure, tmp;							\
									\
    do {								\
      tmp = __atomic_load_n (ptr, __ATOMIC_SEQ_CST);			\
      failure = __kernel_cmpxchg (tmp, PFX_OP (tmp INF_OP val), ptr);	\
    } while (failure != 0);						\
									\
    return tmp;								\
  }

FETCH_AND_OP_WORD (add,   , +)
FETCH_AND_OP_WORD (sub,   , -)
FETCH_AND_OP_WORD (or,    , |)
FETCH_AND_OP_WORD (and,   , &)
FETCH_AND_OP_WORD (xor,   , ^)
FETCH_AND_OP_WORD (nand, ~, &)

#define NAME_oldval(OP, WIDTH) __sync_fetch_and_##OP##_##WIDTH
#define NAME_newval(OP, WIDTH) __sync_##OP##_and_fetch_##WIDTH

/* Implement both __sync_<op>_and_fetch and __sync_fetch_and_<op> for
   subword-sized quantities.  */

#define SUBWORD_SYNC_OP(OP, PFX_OP, INF_OP, TYPE, WIDTH, RETURN)	\
  TYPE HIDDEN								\
  NAME##_##RETURN (OP, WIDTH) (TYPE *ptr, TYPE val)			\
  {									\
    int *wordptr = (int *) ((unsigned long) ptr & ~3);			\
    unsigned int mask, shift, oldval, newval;				\
    int failure;							\
									\
    shift = (((unsigned long) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH;	\
    mask = MASK_##WIDTH << shift;					\
									\
    do {								\
      oldval = __atomic_load_n (wordptr, __ATOMIC_SEQ_CST);		\
      newval = ((PFX_OP (((oldval & mask) >> shift)			\
			 INF_OP (unsigned int) val)) << shift) & mask;	\
      newval |= oldval & ~mask;						\
      failure = __kernel_cmpxchg (oldval, newval, wordptr);		\
    } while (failure != 0);						\
									\
    return (RETURN & mask) >> shift;					\
  }


SUBWORD_SYNC_OP (add,   , +, unsigned short, 2, oldval)
SUBWORD_SYNC_OP (sub,   , -, unsigned short, 2, oldval)
SUBWORD_SYNC_OP (or,    , |, unsigned short, 2, oldval)
SUBWORD_SYNC_OP (and,   , &, unsigned short, 2, oldval)
SUBWORD_SYNC_OP (xor,   , ^, unsigned short, 2, oldval)
SUBWORD_SYNC_OP (nand, ~, &, unsigned short, 2, oldval)

SUBWORD_SYNC_OP (add,   , +, unsigned char, 1, oldval)
SUBWORD_SYNC_OP (sub,   , -, unsigned char, 1, oldval)
SUBWORD_SYNC_OP (or,    , |, unsigned char, 1, oldval)
SUBWORD_SYNC_OP (and,   , &, unsigned char, 1, oldval)
SUBWORD_SYNC_OP (xor,   , ^, unsigned char, 1, oldval)
SUBWORD_SYNC_OP (nand, ~, &, unsigned char, 1, oldval)

#define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP)				\
  int HIDDEN								\
  __sync_##OP##_and_fetch_4 (int *ptr, int val)				\
  {									\
    int tmp, failure;							\
									\
    do {								\
      tmp = __atomic_load_n (ptr, __ATOMIC_SEQ_CST);			\
      failure = __kernel_cmpxchg (tmp, PFX_OP (tmp INF_OP val), ptr);	\
    } while (failure != 0);						\
									\
    return PFX_OP (tmp INF_OP val);					\
  }

OP_AND_FETCH_WORD (add,   , +)
OP_AND_FETCH_WORD (sub,   , -)
OP_AND_FETCH_WORD (or,    , |)
OP_AND_FETCH_WORD (and,   , &)
OP_AND_FETCH_WORD (xor,   , ^)
OP_AND_FETCH_WORD (nand, ~, &)

SUBWORD_SYNC_OP (add,   , +, unsigned short, 2, newval)
SUBWORD_SYNC_OP (sub,   , -, unsigned short, 2, newval)
SUBWORD_SYNC_OP (or,    , |, unsigned short, 2, newval)
SUBWORD_SYNC_OP (and,   , &, unsigned short, 2, newval)
SUBWORD_SYNC_OP (xor,   , ^, unsigned short, 2, newval)
SUBWORD_SYNC_OP (nand, ~, &, unsigned short, 2, newval)

SUBWORD_SYNC_OP (add,   , +, unsigned char, 1, newval)
SUBWORD_SYNC_OP (sub,   , -, unsigned char, 1, newval)
SUBWORD_SYNC_OP (or,    , |, unsigned char, 1, newval)
SUBWORD_SYNC_OP (and,   , &, unsigned char, 1, newval)
SUBWORD_SYNC_OP (xor,   , ^, unsigned char, 1, newval)
SUBWORD_SYNC_OP (nand, ~, &, unsigned char, 1, newval)

int HIDDEN
__sync_val_compare_and_swap_4 (int *ptr, int oldval, int newval)
{
  int actual_oldval, fail;

  while (1)
    {
      actual_oldval = __atomic_load_n (ptr, __ATOMIC_SEQ_CST);

      if (oldval != actual_oldval)
	return actual_oldval;

      fail = __kernel_cmpxchg (actual_oldval, newval, ptr);

      if (!fail)
	return oldval;
    }
}

#define SUBWORD_VAL_CAS(TYPE, WIDTH)					\
  TYPE HIDDEN								\
  __sync_val_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval,		\
				       TYPE newval)			\
  {									\
    int *wordptr = (int *)((unsigned long) ptr & ~3), fail;		\
    unsigned int mask, shift, actual_oldval, actual_newval;		\
									\
    shift = (((unsigned long) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH;	\
    mask = MASK_##WIDTH << shift;					\
									\
    while (1)								\
      {									\
	actual_oldval = __atomic_load_n (wordptr, __ATOMIC_SEQ_CST); 	\
									\
	if (((actual_oldval & mask) >> shift) != (unsigned int) oldval)	\
	  return (actual_oldval & mask) >> shift;			\
									\
	actual_newval = (actual_oldval & ~mask)				\
			| (((unsigned int) newval << shift) & mask);	\
									\
	fail = __kernel_cmpxchg (actual_oldval, actual_newval,		\
				 wordptr);				\
									\
	if (!fail)							\
	  return oldval;						\
      }									\
  }

SUBWORD_VAL_CAS (unsigned short, 2)
SUBWORD_VAL_CAS (unsigned char,  1)

typedef unsigned char bool;

bool HIDDEN
__sync_bool_compare_and_swap_4 (int *ptr, int oldval, int newval)
{
  int failure = __kernel_cmpxchg (oldval, newval, ptr);
  return (failure == 0);
}

#define SUBWORD_BOOL_CAS(TYPE, WIDTH)					\
  bool HIDDEN								\
  __sync_bool_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval,		\
					TYPE newval)			\
  {									\
    TYPE actual_oldval							\
      = __sync_val_compare_and_swap_##WIDTH (ptr, oldval, newval);	\
    return (oldval == actual_oldval);					\
  }

SUBWORD_BOOL_CAS (unsigned short, 2)
SUBWORD_BOOL_CAS (unsigned char,  1)

int HIDDEN
__sync_lock_test_and_set_4 (int *ptr, int val)
{
  int failure, oldval;

  do {
    oldval = __atomic_load_n (ptr, __ATOMIC_SEQ_CST);
    failure = __kernel_cmpxchg (oldval, val, ptr);
  } while (failure != 0);

  return oldval;
}

#define SUBWORD_TEST_AND_SET(TYPE, WIDTH)				\
  TYPE HIDDEN								\
  __sync_lock_test_and_set_##WIDTH (TYPE *ptr, TYPE val)		\
  {									\
    int failure;							\
    unsigned int oldval, newval, shift, mask;				\
    int *wordptr = (int *) ((unsigned long) ptr & ~3);			\
									\
    shift = (((unsigned long) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH;	\
    mask = MASK_##WIDTH << shift;					\
									\
    do {								\
      oldval = __atomic_load_n (wordptr, __ATOMIC_SEQ_CST);		\
      newval = (oldval & ~mask)						\
	       | (((unsigned int) val << shift) & mask);		\
      failure = __kernel_cmpxchg (oldval, newval, wordptr);		\
    } while (failure != 0);						\
									\
    return (oldval & mask) >> shift;					\
  }

SUBWORD_TEST_AND_SET (unsigned short, 2)
SUBWORD_TEST_AND_SET (unsigned char,  1)

#define SYNC_LOCK_RELEASE(TYPE, WIDTH)					\
  void HIDDEN								\
  __sync_lock_release_##WIDTH (TYPE *ptr)				\
  {									\
    /* All writes before this point must be seen before we release	\
       the lock itself.  */						\
    __builtin_nds32_msync_all ();					\
    *ptr = 0;								\
  }

SYNC_LOCK_RELEASE (int,   4)
SYNC_LOCK_RELEASE (short, 2)
SYNC_LOCK_RELEASE (char,  1)
Commit	Line	Data
cf3cd43d	1	/* Linux-specific atomic operations for NDS32 Linux.
8d9254fc	2	Copyright (C) 2012-2020 Free Software Foundation, Inc.
cf3cd43d CJW	3
	4	This file is free software; you can redistribute it and/or modify it
	5	under the terms of the GNU General Public License as published by the
	6	Free Software Foundation; either version 3, or (at your option) any
	7	later version.
	8
	9	This file is distributed in the hope that it will be useful, but
	10	WITHOUT ANY WARRANTY; without even the implied warranty of
	11	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	12	General Public License for more details.
	13
	14	Under Section 7 of GPL version 3, you are granted additional
	15	permissions described in the GCC Runtime Library Exception, version
	16	3.1, as published by the Free Software Foundation.
	17
	18	You should have received a copy of the GNU General Public License and
	19	a copy of the GCC Runtime Library Exception along with this program;
	20	see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
	21	<http://www.gnu.org/licenses/>. */
	22
	23	/* We implement byte, short and int versions of each atomic operation
	24	using the kernel helper defined below. There is no support for
	25	64-bit operations yet. */
	26
	27	/* This function copy form NDS32 Linux-kernal. */
	28	static inline int
	29	__kernel_cmpxchg (int oldval, int newval, int *mem)
	30	{
	31	int temp1, temp2, temp3, offset;
	32
	33	asm volatile ("msync\tall\n"
	34	"movi\t%0, #0\n"
	35	"1:\n"
	36	"\tllw\t%1, [%4+%0]\n"
	37	"\tsub\t%3, %1, %6\n"
	38	"\tcmovz\t%2, %5, %3\n"
	39	"\tcmovn\t%2, %1, %3\n"
	40	"\tscw\t%2, [%4+%0]\n"
	41	"\tbeqz\t%2, 1b\n"
	42	: "=&r" (offset), "=&r" (temp3), "=&r" (temp2), "=&r" (temp1)
	43	: "r" (mem), "r" (newval), "r" (oldval) : "memory");
	44
	45	return temp1;
	46	}
	47
	48	#define HIDDEN __attribute__ ((visibility ("hidden")))
	49
	50	#ifdef __NDS32_EL__
	51	#define INVERT_MASK_1 0
	52	#define INVERT_MASK_2 0
	53	#else
	54	#define INVERT_MASK_1 24
	55	#define INVERT_MASK_2 16
	56	#endif
	57
	58	#define MASK_1 0xffu
	59	#define MASK_2 0xffffu
	60
	61	#define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP) \
	62	int HIDDEN \
	63	__sync_fetch_and_##OP##_4 (int *ptr, int val) \
	64	{ \
	65	int failure, tmp; \
	66	\
67	do { \
68	tmp = __atomic_load_n (ptr, __ATOMIC_SEQ_CST); \
69	failure = __kernel_cmpxchg (tmp, PFX_OP (tmp INF_OP val), ptr); \
70	} while (failure != 0); \
71	\
72	return tmp; \
73	}
74
75	FETCH_AND_OP_WORD (add, , +)
76	FETCH_AND_OP_WORD (sub, , -)
77	FETCH_AND_OP_WORD (or, , \|)
78	FETCH_AND_OP_WORD (and, , &)
79	FETCH_AND_OP_WORD (xor, , ^)
80	FETCH_AND_OP_WORD (nand, ~, &)
81
82	#define NAME_oldval(OP, WIDTH) __sync_fetch_and_##OP##_##WIDTH
83	#define NAME_newval(OP, WIDTH) __sync_##OP##_and_fetch_##WIDTH
84
85	/* Implement both __sync_<op>_and_fetch and __sync_fetch_and_<op> for
86	subword-sized quantities. */
87
88	#define SUBWORD_SYNC_OP(OP, PFX_OP, INF_OP, TYPE, WIDTH, RETURN) \
89	TYPE HIDDEN \
90	NAME##_##RETURN (OP, WIDTH) (TYPE *ptr, TYPE val) \
91	{ \
92	int wordptr = (int ) ((unsigned long) ptr & ~3); \
93	unsigned int mask, shift, oldval, newval; \
94	int failure; \
95	\
96	shift = (((unsigned long) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \
97	mask = MASK_##WIDTH << shift; \
98	\
99	do { \
100	oldval = __atomic_load_n (wordptr, __ATOMIC_SEQ_CST); \
101	newval = ((PFX_OP (((oldval & mask) >> shift) \
102	INF_OP (unsigned int) val)) << shift) & mask; \
103	newval \|= oldval & ~mask; \
104	failure = __kernel_cmpxchg (oldval, newval, wordptr); \
105	} while (failure != 0); \
106	\
107	return (RETURN & mask) >> shift; \
108	}
109
110
111	SUBWORD_SYNC_OP (add, , +, unsigned short, 2, oldval)
112	SUBWORD_SYNC_OP (sub, , -, unsigned short, 2, oldval)
113	SUBWORD_SYNC_OP (or, , \|, unsigned short, 2, oldval)
114	SUBWORD_SYNC_OP (and, , &, unsigned short, 2, oldval)
115	SUBWORD_SYNC_OP (xor, , ^, unsigned short, 2, oldval)
116	SUBWORD_SYNC_OP (nand, ~, &, unsigned short, 2, oldval)
117
118	SUBWORD_SYNC_OP (add, , +, unsigned char, 1, oldval)
119	SUBWORD_SYNC_OP (sub, , -, unsigned char, 1, oldval)
120	SUBWORD_SYNC_OP (or, , \|, unsigned char, 1, oldval)
121	SUBWORD_SYNC_OP (and, , &, unsigned char, 1, oldval)
122	SUBWORD_SYNC_OP (xor, , ^, unsigned char, 1, oldval)
123	SUBWORD_SYNC_OP (nand, ~, &, unsigned char, 1, oldval)
124
125	#define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP) \
126	int HIDDEN \
127	__sync_##OP##_and_fetch_4 (int *ptr, int val) \
128	{ \
129	int tmp, failure; \
130	\
131	do { \
132	tmp = __atomic_load_n (ptr, __ATOMIC_SEQ_CST); \
133	failure = __kernel_cmpxchg (tmp, PFX_OP (tmp INF_OP val), ptr); \
134	} while (failure != 0); \
135	\
136	return PFX_OP (tmp INF_OP val); \
137	}
138
139	OP_AND_FETCH_WORD (add, , +)
140	OP_AND_FETCH_WORD (sub, , -)
141	OP_AND_FETCH_WORD (or, , \|)
142	OP_AND_FETCH_WORD (and, , &)
143	OP_AND_FETCH_WORD (xor, , ^)
144	OP_AND_FETCH_WORD (nand, ~, &)
145
146	SUBWORD_SYNC_OP (add, , +, unsigned short, 2, newval)
147	SUBWORD_SYNC_OP (sub, , -, unsigned short, 2, newval)
148	SUBWORD_SYNC_OP (or, , \|, unsigned short, 2, newval)
149	SUBWORD_SYNC_OP (and, , &, unsigned short, 2, newval)
150	SUBWORD_SYNC_OP (xor, , ^, unsigned short, 2, newval)
151	SUBWORD_SYNC_OP (nand, ~, &, unsigned short, 2, newval)
152
153	SUBWORD_SYNC_OP (add, , +, unsigned char, 1, newval)
154	SUBWORD_SYNC_OP (sub, , -, unsigned char, 1, newval)
155	SUBWORD_SYNC_OP (or, , \|, unsigned char, 1, newval)
156	SUBWORD_SYNC_OP (and, , &, unsigned char, 1, newval)
157	SUBWORD_SYNC_OP (xor, , ^, unsigned char, 1, newval)
158	SUBWORD_SYNC_OP (nand, ~, &, unsigned char, 1, newval)
159
160	int HIDDEN
161	__sync_val_compare_and_swap_4 (int *ptr, int oldval, int newval)
162	{
163	int actual_oldval, fail;
164
165	while (1)
166	{
167	actual_oldval = __atomic_load_n (ptr, __ATOMIC_SEQ_CST);
168
169	if (oldval != actual_oldval)
170	return actual_oldval;
171
172	fail = __kernel_cmpxchg (actual_oldval, newval, ptr);
173
174	if (!fail)
175	return oldval;
176	}
177	}
178
179	#define SUBWORD_VAL_CAS(TYPE, WIDTH) \
180	TYPE HIDDEN \
181	__sync_val_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval, \
182	TYPE newval) \
183	{ \
184	int wordptr = (int )((unsigned long) ptr & ~3), fail; \
185	unsigned int mask, shift, actual_oldval, actual_newval; \
186	\
187	shift = (((unsigned long) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \
188	mask = MASK_##WIDTH << shift; \
189	\
190	while (1) \
191	{ \
192	actual_oldval = __atomic_load_n (wordptr, __ATOMIC_SEQ_CST); \
193	\
194	if (((actual_oldval & mask) >> shift) != (unsigned int) oldval) \
195	return (actual_oldval & mask) >> shift; \
196	\
197	actual_newval = (actual_oldval & ~mask) \
198	\| (((unsigned int) newval << shift) & mask); \
199	\
200	fail = __kernel_cmpxchg (actual_oldval, actual_newval, \
201	wordptr); \
202	\
203	if (!fail) \
204	return oldval; \
205	} \
206	}
207
208	SUBWORD_VAL_CAS (unsigned short, 2)
209	SUBWORD_VAL_CAS (unsigned char, 1)
210
211	typedef unsigned char bool;
212
213	bool HIDDEN
214	__sync_bool_compare_and_swap_4 (int *ptr, int oldval, int newval)
215	{
216	int failure = __kernel_cmpxchg (oldval, newval, ptr);
217	return (failure == 0);
218	}
219
220	#define SUBWORD_BOOL_CAS(TYPE, WIDTH) \
221	bool HIDDEN \
222	__sync_bool_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval, \
223	TYPE newval) \
224	{ \
225	TYPE actual_oldval \
226	= __sync_val_compare_and_swap_##WIDTH (ptr, oldval, newval); \
227	return (oldval == actual_oldval); \
228	}
229
230	SUBWORD_BOOL_CAS (unsigned short, 2)
231	SUBWORD_BOOL_CAS (unsigned char, 1)
232
233	int HIDDEN
234	__sync_lock_test_and_set_4 (int *ptr, int val)
235	{
236	int failure, oldval;
237
238	do {
239	oldval = __atomic_load_n (ptr, __ATOMIC_SEQ_CST);
240	failure = __kernel_cmpxchg (oldval, val, ptr);
241	} while (failure != 0);
242
243	return oldval;
244	}
245
246	#define SUBWORD_TEST_AND_SET(TYPE, WIDTH) \
247	TYPE HIDDEN \
248	__sync_lock_test_and_set_##WIDTH (TYPE *ptr, TYPE val) \
249	{ \
250	int failure; \
251	unsigned int oldval, newval, shift, mask; \
252	int wordptr = (int ) ((unsigned long) ptr & ~3); \
253	\
254	shift = (((unsigned long) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \
255	mask = MASK_##WIDTH << shift; \
256	\
257	do { \
258	oldval = __atomic_load_n (wordptr, __ATOMIC_SEQ_CST); \
259	newval = (oldval & ~mask) \
260	\| (((unsigned int) val << shift) & mask); \
261	failure = __kernel_cmpxchg (oldval, newval, wordptr); \
262	} while (failure != 0); \
263	\
264	return (oldval & mask) >> shift; \
265	}
266
267	SUBWORD_TEST_AND_SET (unsigned short, 2)
268	SUBWORD_TEST_AND_SET (unsigned char, 1)
269
270	#define SYNC_LOCK_RELEASE(TYPE, WIDTH) \
271	void HIDDEN \
272	__sync_lock_release_##WIDTH (TYPE *ptr) \
273	{ \
274	/* All writes before this point must be seen before we release \
275	the lock itself. */ \
276	__builtin_nds32_msync_all (); \
277	*ptr = 0; \
278	}
279
280	SYNC_LOCK_RELEASE (int, 4)
281	SYNC_LOCK_RELEASE (short, 2)
282	SYNC_LOCK_RELEASE (char, 1)