[thirdparty/gcc.git] / libatomic / config / x86 / host-config.h

/* Copyright (C) 2012-2023 Free Software Foundation, Inc.
   Contributed by Richard Henderson <rth@redhat.com>.

   This file is part of the GNU Atomic Library (libatomic).

   Libatomic is free software; you can redistribute it and/or modify it
   under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3 of the License, or
   (at your option) any later version.

   Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
   more details.

   Under Section 7 of GPL version 3, you are granted additional
   permissions described in the GCC Runtime Library Exception, version
   3.1, as published by the Free Software Foundation.

   You should have received a copy of the GNU General Public License and
   a copy of the GCC Runtime Library Exception along with this program;
   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
   <http://www.gnu.org/licenses/>.  */

#if HAVE_IFUNC
#include <cpuid.h>

#ifdef __x86_64__
# define FEAT1_REGISTER ecx
#else
# define FEAT1_REGISTER edx
#endif

/* Value of the CPUID feature register FEAT1_REGISTER for the cmpxchg
   bit for IFUNC_COND1 below.  */
extern unsigned int __libat_feat1 HIDDEN;

/* Initialize libat_feat1 and return its value.  */
unsigned int __libat_feat1_init (void) HIDDEN;

/* Return the value of the relevant feature register for the relevant
   cmpxchg bit, or 0 if there is no CPUID support.  */
static inline unsigned int
__attribute__ ((const))
load_feat1 (void)
{
  /* See the store in __libat_feat1_init.  */
  unsigned int feat1 = __atomic_load_n (&__libat_feat1, __ATOMIC_RELAXED);
  if (feat1 == 0)
    /* Assume that initialization has not happened yet.  This may get
       called repeatedly if the CPU does not have any feature bits at
       all.  */
    feat1 = __libat_feat1_init ();
  return feat1;
}

#ifdef __x86_64__
# define IFUNC_COND_1	((load_feat1 () & (bit_AVX | bit_CMPXCHG16B)) \
			 == (bit_AVX | bit_CMPXCHG16B))
# define IFUNC_COND_2	(load_feat1 () & bit_CMPXCHG16B)
#else
# define IFUNC_COND_1	(load_feat1 () & bit_CMPXCHG8B)
#endif

#ifdef __x86_64__
# define IFUNC_NCOND(N) (2 * (N == 16))
#else
# define IFUNC_NCOND(N) (N == 8)
#endif

#ifdef __x86_64__
# undef MAYBE_HAVE_ATOMIC_CAS_16
# define MAYBE_HAVE_ATOMIC_CAS_16	IFUNC_COND_2
# undef MAYBE_HAVE_ATOMIC_EXCHANGE_16
# define MAYBE_HAVE_ATOMIC_EXCHANGE_16	IFUNC_COND_2
# undef MAYBE_HAVE_ATOMIC_LDST_16
# define MAYBE_HAVE_ATOMIC_LDST_16	IFUNC_COND_2
/* Since load and store are implemented with CAS, they are not fast.  */
# undef FAST_ATOMIC_LDST_16
# define FAST_ATOMIC_LDST_16		0
# if IFUNC_ALT != 0
#  undef HAVE_ATOMIC_CAS_16
#  define HAVE_ATOMIC_CAS_16 1
# endif
# if IFUNC_ALT == 1
#  undef HAVE_ATOMIC_LDST_16
#  define HAVE_ATOMIC_LDST_16 1
# endif
#else
# undef MAYBE_HAVE_ATOMIC_CAS_8
# define MAYBE_HAVE_ATOMIC_CAS_8	IFUNC_COND_1
# undef MAYBE_HAVE_ATOMIC_EXCHANGE_8
# define MAYBE_HAVE_ATOMIC_EXCHANGE_8	IFUNC_COND_1
# undef MAYBE_HAVE_ATOMIC_LDST_8
# define MAYBE_HAVE_ATOMIC_LDST_8	IFUNC_COND_1
# if IFUNC_ALT == 1
#  undef HAVE_ATOMIC_CAS_8
#  define HAVE_ATOMIC_CAS_8 1
# endif
#endif

#if defined(__x86_64__) && N == 16 && IFUNC_ALT != 0
static inline bool
atomic_compare_exchange_n (UTYPE *mptr, UTYPE *eptr, UTYPE newval,
                           bool weak_p UNUSED, int sm UNUSED, int fm UNUSED)
{
  UTYPE cmpval = *eptr;
  UTYPE oldval = __sync_val_compare_and_swap_16 (mptr, cmpval, newval);
  if (oldval == cmpval)
    return true;
  *eptr = oldval;
  return false;
}
# define atomic_compare_exchange_n atomic_compare_exchange_n
#endif /* Have CAS 16 */

#if defined(__x86_64__) && N == 16 && IFUNC_ALT == 1
#define __atomic_load_n(ptr, model) \
  (sizeof (*ptr) == 16 ? atomic_load_n (ptr, model) \
		       : (__atomic_load_n) (ptr, model))
#define __atomic_store_n(ptr, val, model) \
  (sizeof (*ptr) == 16 ? atomic_store_n (ptr, val, model) \
		       : (__atomic_store_n) (ptr, val, model))

static inline UTYPE
atomic_load_n (UTYPE *ptr, int model UNUSED)
{
  UTYPE ret;
  __asm__ ("vmovdqa\t{%1, %0|%0, %1}" : "=x" (ret) : "m" (*ptr));
  return ret;
}

static inline void
atomic_store_n (UTYPE *ptr, UTYPE val, int model UNUSED)
{
  __asm__ ("vmovdqa\t{%1, %0|%0, %1}\n\tmfence" : "=m" (*ptr) : "x" (val));
}
#endif

#endif /* HAVE_IFUNC */

#include_next <host-config.h>
Commit	Line	Data
83ffe9cd	1	/* Copyright (C) 2012-2023 Free Software Foundation, Inc.
48310492 RH	2	Contributed by Richard Henderson <rth@redhat.com>.
	3
	4	This file is part of the GNU Atomic Library (libatomic).
	5
	6	Libatomic is free software; you can redistribute it and/or modify it
	7	under the terms of the GNU General Public License as published by
	8	the Free Software Foundation; either version 3 of the License, or
	9	(at your option) any later version.
	10
	11	Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
	12	WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
	13	FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	14	more details.
	15
	16	Under Section 7 of GPL version 3, you are granted additional
	17	permissions described in the GCC Runtime Library Exception, version
	18	3.1, as published by the Free Software Foundation.
	19
	20	You should have received a copy of the GNU General Public License and
	21	a copy of the GCC Runtime Library Exception along with this program;
	22	see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
	23	<http://www.gnu.org/licenses/>. */
	24
	25	#if HAVE_IFUNC
	26	#include <cpuid.h>
	27
cb3c90cc FW	28	#ifdef __x86_64__
	29	# define FEAT1_REGISTER ecx
	30	#else
	31	# define FEAT1_REGISTER edx
	32	#endif
	33
	34	/* Value of the CPUID feature register FEAT1_REGISTER for the cmpxchg
	35	bit for IFUNC_COND1 below. */
	36	extern unsigned int __libat_feat1 HIDDEN;
	37
	38	/* Initialize libat_feat1 and return its value. */
	39	unsigned int __libat_feat1_init (void) HIDDEN;
	40
	41	/* Return the value of the relevant feature register for the relevant
	42	cmpxchg bit, or 0 if there is no CPUID support. */
	43	static inline unsigned int
	44	__attribute__ ((const))
	45	load_feat1 (void)
	46	{
	47	/* See the store in __libat_feat1_init. */
	48	unsigned int feat1 = __atomic_load_n (&__libat_feat1, __ATOMIC_RELAXED);
	49	if (feat1 == 0)
	50	/* Assume that initialization has not happened yet. This may get
	51	called repeatedly if the CPU does not have any feature bits at
	52	all. */
	53	feat1 = __libat_feat1_init ();
	54	return feat1;
	55	}
48310492 RH	56
48310492 RH	57	#ifdef __x86_64__
1d47c051 JJ	58	# define IFUNC_COND_1 ((load_feat1 () & (bit_AVX \| bit_CMPXCHG16B)) \
	59	== (bit_AVX \| bit_CMPXCHG16B))
	60	# define IFUNC_COND_2 (load_feat1 () & bit_CMPXCHG16B)
48310492	61	#else
cb3c90cc	62	# define IFUNC_COND_1 (load_feat1 () & bit_CMPXCHG8B)
48310492 RH	63	#endif
	64
	65	#ifdef __x86_64__
1d47c051	66	# define IFUNC_NCOND(N) (2 * (N == 16))
48310492 RH	67	#else
	68	# define IFUNC_NCOND(N) (N == 8)
	69	#endif
	70
	71	#ifdef __x86_64__
	72	# undef MAYBE_HAVE_ATOMIC_CAS_16
1d47c051	73	# define MAYBE_HAVE_ATOMIC_CAS_16 IFUNC_COND_2
48310492	74	# undef MAYBE_HAVE_ATOMIC_EXCHANGE_16
1d47c051	75	# define MAYBE_HAVE_ATOMIC_EXCHANGE_16 IFUNC_COND_2
48310492	76	# undef MAYBE_HAVE_ATOMIC_LDST_16
1d47c051	77	# define MAYBE_HAVE_ATOMIC_LDST_16 IFUNC_COND_2
969a32ce TR	78	/* Since load and store are implemented with CAS, they are not fast. */
	79	# undef FAST_ATOMIC_LDST_16
	80	# define FAST_ATOMIC_LDST_16 0
1d47c051	81	# if IFUNC_ALT != 0
48310492 RH	82	# undef HAVE_ATOMIC_CAS_16
	83	# define HAVE_ATOMIC_CAS_16 1
	84	# endif
1d47c051 JJ	85	# if IFUNC_ALT == 1
	86	# undef HAVE_ATOMIC_LDST_16
	87	# define HAVE_ATOMIC_LDST_16 1
	88	# endif
48310492 RH	89	#else
	90	# undef MAYBE_HAVE_ATOMIC_CAS_8
	91	# define MAYBE_HAVE_ATOMIC_CAS_8 IFUNC_COND_1
	92	# undef MAYBE_HAVE_ATOMIC_EXCHANGE_8
	93	# define MAYBE_HAVE_ATOMIC_EXCHANGE_8 IFUNC_COND_1
	94	# undef MAYBE_HAVE_ATOMIC_LDST_8
	95	# define MAYBE_HAVE_ATOMIC_LDST_8 IFUNC_COND_1
	96	# if IFUNC_ALT == 1
	97	# undef HAVE_ATOMIC_CAS_8
	98	# define HAVE_ATOMIC_CAS_8 1
	99	# endif
	100	#endif
	101
1d47c051	102	#if defined(__x86_64__) && N == 16 && IFUNC_ALT != 0
969a32ce TR	103	static inline bool
	104	atomic_compare_exchange_n (UTYPE mptr, UTYPE eptr, UTYPE newval,
	105	bool weak_p UNUSED, int sm UNUSED, int fm UNUSED)
	106	{
	107	UTYPE cmpval = *eptr;
	108	UTYPE oldval = __sync_val_compare_and_swap_16 (mptr, cmpval, newval);
	109	if (oldval == cmpval)
	110	return true;
	111	*eptr = oldval;
	112	return false;
	113	}
	114	# define atomic_compare_exchange_n atomic_compare_exchange_n
	115	#endif /* Have CAS 16 */
	116
1d47c051 JJ	117	#if defined(__x86_64__) && N == 16 && IFUNC_ALT == 1
	118	#define __atomic_load_n(ptr, model) \
	119	(sizeof (*ptr) == 16 ? atomic_load_n (ptr, model) \
	120	: (__atomic_load_n) (ptr, model))
	121	#define __atomic_store_n(ptr, val, model) \
	122	(sizeof (*ptr) == 16 ? atomic_store_n (ptr, val, model) \
	123	: (__atomic_store_n) (ptr, val, model))
	124
	125	static inline UTYPE
	126	atomic_load_n (UTYPE *ptr, int model UNUSED)
	127	{
	128	UTYPE ret;
	129	__asm__ ("vmovdqa\t{%1, %0\|%0, %1}" : "=x" (ret) : "m" (*ptr));
	130	return ret;
	131	}
	132
	133	static inline void
	134	atomic_store_n (UTYPE *ptr, UTYPE val, int model UNUSED)
	135	{
	136	__asm__ ("vmovdqa\t{%1, %0\|%0, %1}\n\tmfence" : "=m" (*ptr) : "x" (val));
	137	}
	138	#endif
	139
48310492 RH	140	#endif /* HAVE_IFUNC */
	141
	142	#include_next <host-config.h>