[thirdparty/gcc.git] / libsanitizer / sanitizer_common / sanitizer_atomic_msvc.h

//===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
// Not intended for direct inclusion. Include sanitizer_atomic.h.
//
//===----------------------------------------------------------------------===//

#ifndef SANITIZER_ATOMIC_MSVC_H
#define SANITIZER_ATOMIC_MSVC_H

extern "C" void _ReadWriteBarrier();
#pragma intrinsic(_ReadWriteBarrier)
extern "C" void _mm_mfence();
#pragma intrinsic(_mm_mfence)
extern "C" void _mm_pause();
#pragma intrinsic(_mm_pause)
extern "C" char _InterlockedExchange8(char volatile *Addend, char Value);
#pragma intrinsic(_InterlockedExchange8)
extern "C" short _InterlockedExchange16(short volatile *Addend, short Value);
#pragma intrinsic(_InterlockedExchange16)
extern "C" long _InterlockedExchange(long volatile *Addend, long Value);
#pragma intrinsic(_InterlockedExchange)
extern "C" long _InterlockedExchangeAdd(long volatile *Addend, long Value);
#pragma intrinsic(_InterlockedExchangeAdd)
extern "C" char _InterlockedCompareExchange8(char volatile *Destination,
                                             char Exchange, char Comparand);
#pragma intrinsic(_InterlockedCompareExchange8)
extern "C" short _InterlockedCompareExchange16(short volatile *Destination,
                                               short Exchange, short Comparand);
#pragma intrinsic(_InterlockedCompareExchange16)
extern "C" long long _InterlockedCompareExchange64(
    long long volatile *Destination, long long Exchange, long long Comparand);
#pragma intrinsic(_InterlockedCompareExchange64)
extern "C" void *_InterlockedCompareExchangePointer(
    void *volatile *Destination,
    void *Exchange, void *Comparand);
#pragma intrinsic(_InterlockedCompareExchangePointer)
extern "C" long __cdecl _InterlockedCompareExchange(long volatile *Destination,
                                                    long Exchange,
                                                    long Comparand);
#pragma intrinsic(_InterlockedCompareExchange)

#ifdef _WIN64
extern "C" long long _InterlockedExchangeAdd64(long long volatile *Addend,
                                               long long Value);
#pragma intrinsic(_InterlockedExchangeAdd64)
#endif

namespace __sanitizer {

INLINE void atomic_signal_fence(memory_order) {
  _ReadWriteBarrier();
}

INLINE void atomic_thread_fence(memory_order) {
  _mm_mfence();
}

INLINE void proc_yield(int cnt) {
  for (int i = 0; i < cnt; i++)
    _mm_pause();
}

template<typename T>
INLINE typename T::Type atomic_load(
    const volatile T *a, memory_order mo) {
  DCHECK(mo & (memory_order_relaxed | memory_order_consume
      | memory_order_acquire | memory_order_seq_cst));
  DCHECK(!((uptr)a % sizeof(*a)));
  typename T::Type v;
  // FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
  if (mo == memory_order_relaxed) {
    v = a->val_dont_use;
  } else {
    atomic_signal_fence(memory_order_seq_cst);
    v = a->val_dont_use;
    atomic_signal_fence(memory_order_seq_cst);
  }
  return v;
}

template<typename T>
INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
  DCHECK(mo & (memory_order_relaxed | memory_order_release
      | memory_order_seq_cst));
  DCHECK(!((uptr)a % sizeof(*a)));
  // FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
  if (mo == memory_order_relaxed) {
    a->val_dont_use = v;
  } else {
    atomic_signal_fence(memory_order_seq_cst);
    a->val_dont_use = v;
    atomic_signal_fence(memory_order_seq_cst);
  }
  if (mo == memory_order_seq_cst)
    atomic_thread_fence(memory_order_seq_cst);
}

INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
    u32 v, memory_order mo) {
  (void)mo;
  DCHECK(!((uptr)a % sizeof(*a)));
  return (u32)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
                                      (long)v);
}

INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
    uptr v, memory_order mo) {
  (void)mo;
  DCHECK(!((uptr)a % sizeof(*a)));
#ifdef _WIN64
  return (uptr)_InterlockedExchangeAdd64((volatile long long *)&a->val_dont_use,
                                         (long long)v);
#else
  return (uptr)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
                                       (long)v);
#endif
}

INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
    u32 v, memory_order mo) {
  (void)mo;
  DCHECK(!((uptr)a % sizeof(*a)));
  return (u32)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
                                      -(long)v);
}

INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
    uptr v, memory_order mo) {
  (void)mo;
  DCHECK(!((uptr)a % sizeof(*a)));
#ifdef _WIN64
  return (uptr)_InterlockedExchangeAdd64((volatile long long *)&a->val_dont_use,
                                         -(long long)v);
#else
  return (uptr)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
                                       -(long)v);
#endif
}

INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
    u8 v, memory_order mo) {
  (void)mo;
  DCHECK(!((uptr)a % sizeof(*a)));
  return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
}

INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
    u16 v, memory_order mo) {
  (void)mo;
  DCHECK(!((uptr)a % sizeof(*a)));
  return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
}

INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
    u32 v, memory_order mo) {
  (void)mo;
  DCHECK(!((uptr)a % sizeof(*a)));
  return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
}

INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
                                           u8 *cmp,
                                           u8 xchgv,
                                           memory_order mo) {
  (void)mo;
  DCHECK(!((uptr)a % sizeof(*a)));
  u8 cmpv = *cmp;
#ifdef _WIN64
  u8 prev = (u8)_InterlockedCompareExchange8(
      (volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv);
#else
  u8 prev;
  __asm {
    mov al, cmpv
    mov ecx, a
    mov dl, xchgv
    lock cmpxchg [ecx], dl
    mov prev, al
  }
#endif
  if (prev == cmpv)
    return true;
  *cmp = prev;
  return false;
}

INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
                                           uptr *cmp,
                                           uptr xchg,
                                           memory_order mo) {
  uptr cmpv = *cmp;
  uptr prev = (uptr)_InterlockedCompareExchangePointer(
      (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
  if (prev == cmpv)
    return true;
  *cmp = prev;
  return false;
}

INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
                                           u16 *cmp,
                                           u16 xchg,
                                           memory_order mo) {
  u16 cmpv = *cmp;
  u16 prev = (u16)_InterlockedCompareExchange16(
      (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
  if (prev == cmpv)
    return true;
  *cmp = prev;
  return false;
}

INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
                                           u32 *cmp,
                                           u32 xchg,
                                           memory_order mo) {
  u32 cmpv = *cmp;
  u32 prev = (u32)_InterlockedCompareExchange(
      (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
  if (prev == cmpv)
    return true;
  *cmp = prev;
  return false;
}

INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
                                           u64 *cmp,
                                           u64 xchg,
                                           memory_order mo) {
  u64 cmpv = *cmp;
  u64 prev = (u64)_InterlockedCompareExchange64(
      (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
  if (prev == cmpv)
    return true;
  *cmp = prev;
  return false;
}

template<typename T>
INLINE bool atomic_compare_exchange_weak(volatile T *a,
                                         typename T::Type *cmp,
                                         typename T::Type xchg,
                                         memory_order mo) {
  return atomic_compare_exchange_strong(a, cmp, xchg, mo);
}

}  // namespace __sanitizer

#endif  // SANITIZER_ATOMIC_CLANG_H
Commit	Line	Data
f35db108 WM	1	//===-- sanitizer_atomic_msvc.h ---------------------------------- C++ --===//
f35db108 WM	2	//
b667dd70 ML	3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	4	// See https://llvm.org/LICENSE.txt for license information.
	5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
f35db108 WM	6	//
	7	//===----------------------------------------------------------------------===//
	8	//
	9	// This file is a part of ThreadSanitizer/AddressSanitizer runtime.
	10	// Not intended for direct inclusion. Include sanitizer_atomic.h.
	11	//
	12	//===----------------------------------------------------------------------===//
	13
	14	#ifndef SANITIZER_ATOMIC_MSVC_H
	15	#define SANITIZER_ATOMIC_MSVC_H
	16
	17	extern "C" void _ReadWriteBarrier();
	18	#pragma intrinsic(_ReadWriteBarrier)
	19	extern "C" void _mm_mfence();
	20	#pragma intrinsic(_mm_mfence)
	21	extern "C" void _mm_pause();
	22	#pragma intrinsic(_mm_pause)
3ca75cd5	23	extern "C" char _InterlockedExchange8(char volatile *Addend, char Value);
696d846a	24	#pragma intrinsic(_InterlockedExchange8)
3ca75cd5	25	extern "C" short _InterlockedExchange16(short volatile *Addend, short Value);
696d846a	26	#pragma intrinsic(_InterlockedExchange16)
3ca75cd5	27	extern "C" long _InterlockedExchange(long volatile *Addend, long Value);
696d846a	28	#pragma intrinsic(_InterlockedExchange)
3ca75cd5	29	extern "C" long _InterlockedExchangeAdd(long volatile *Addend, long Value);
f35db108	30	#pragma intrinsic(_InterlockedExchangeAdd)
3ca75cd5 ML	31	extern "C" char _InterlockedCompareExchange8(char volatile *Destination,
3ca75cd5 ML	32	char Exchange, char Comparand);
10189819	33	#pragma intrinsic(_InterlockedCompareExchange8)
3ca75cd5 ML	34	extern "C" short _InterlockedCompareExchange16(short volatile *Destination,
3ca75cd5 ML	35	short Exchange, short Comparand);
dee5ea7a	36	#pragma intrinsic(_InterlockedCompareExchange16)
3ca75cd5 ML	37	extern "C" long long _InterlockedCompareExchange64(
3ca75cd5 ML	38	long long volatile *Destination, long long Exchange, long long Comparand);
dee5ea7a	39	#pragma intrinsic(_InterlockedCompareExchange64)
e9772e16	40	extern "C" void *_InterlockedCompareExchangePointer(
f35db108 WM	41	void volatile Destination,
f35db108 WM	42	void Exchange, void Comparand);
e9772e16	43	#pragma intrinsic(_InterlockedCompareExchangePointer)
3ca75cd5 ML	44	extern "C" long __cdecl _InterlockedCompareExchange(long volatile *Destination,
	45	long Exchange,
	46	long Comparand);
e9772e16 KS	47	#pragma intrinsic(_InterlockedCompareExchange)
e9772e16 KS	48
866e32ad	49	#ifdef _WIN64
3ca75cd5 ML	50	extern "C" long long _InterlockedExchangeAdd64(long long volatile *Addend,
3ca75cd5 ML	51	long long Value);
866e32ad	52	#pragma intrinsic(_InterlockedExchangeAdd64)
e9772e16	53	#endif
f35db108 WM	54
	55	namespace __sanitizer {
	56
	57	INLINE void atomic_signal_fence(memory_order) {
	58	_ReadWriteBarrier();
	59	}
	60
	61	INLINE void atomic_thread_fence(memory_order) {
	62	_mm_mfence();
	63	}
	64
	65	INLINE void proc_yield(int cnt) {
	66	for (int i = 0; i < cnt; i++)
	67	_mm_pause();
	68	}
	69
	70	template<typename T>
	71	INLINE typename T::Type atomic_load(
	72	const volatile T *a, memory_order mo) {
	73	DCHECK(mo & (memory_order_relaxed \| memory_order_consume
	74	\| memory_order_acquire \| memory_order_seq_cst));
	75	DCHECK(!((uptr)a % sizeof(*a)));
	76	typename T::Type v;
2660d12d	77	// FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
f35db108 WM	78	if (mo == memory_order_relaxed) {
	79	v = a->val_dont_use;
	80	} else {
	81	atomic_signal_fence(memory_order_seq_cst);
	82	v = a->val_dont_use;
	83	atomic_signal_fence(memory_order_seq_cst);
	84	}
	85	return v;
	86	}
	87
	88	template<typename T>
	89	INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
	90	DCHECK(mo & (memory_order_relaxed \| memory_order_release
	91	\| memory_order_seq_cst));
	92	DCHECK(!((uptr)a % sizeof(*a)));
2660d12d	93	// FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
f35db108 WM	94	if (mo == memory_order_relaxed) {
	95	a->val_dont_use = v;
	96	} else {
	97	atomic_signal_fence(memory_order_seq_cst);
	98	a->val_dont_use = v;
	99	atomic_signal_fence(memory_order_seq_cst);
	100	}
	101	if (mo == memory_order_seq_cst)
	102	atomic_thread_fence(memory_order_seq_cst);
	103	}
	104
	105	INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
	106	u32 v, memory_order mo) {
	107	(void)mo;
	108	DCHECK(!((uptr)a % sizeof(*a)));
3ca75cd5 ML	109	return (u32)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
3ca75cd5 ML	110	(long)v);
f35db108 WM	111	}
f35db108 WM	112
dee5ea7a KS	113	INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
	114	uptr v, memory_order mo) {
	115	(void)mo;
	116	DCHECK(!((uptr)a % sizeof(*a)));
	117	#ifdef _WIN64
3ca75cd5 ML	118	return (uptr)_InterlockedExchangeAdd64((volatile long long *)&a->val_dont_use,
3ca75cd5 ML	119	(long long)v);
dee5ea7a	120	#else
3ca75cd5 ML	121	return (uptr)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
3ca75cd5 ML	122	(long)v);
dee5ea7a KS	123	#endif
	124	}
	125
	126	INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
	127	u32 v, memory_order mo) {
	128	(void)mo;
	129	DCHECK(!((uptr)a % sizeof(*a)));
3ca75cd5 ML	130	return (u32)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
3ca75cd5 ML	131	-(long)v);
dee5ea7a KS	132	}
	133
	134	INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
	135	uptr v, memory_order mo) {
	136	(void)mo;
	137	DCHECK(!((uptr)a % sizeof(*a)));
	138	#ifdef _WIN64
3ca75cd5 ML	139	return (uptr)_InterlockedExchangeAdd64((volatile long long *)&a->val_dont_use,
3ca75cd5 ML	140	-(long long)v);
dee5ea7a	141	#else
3ca75cd5 ML	142	return (uptr)_InterlockedExchangeAdd((volatile long *)&a->val_dont_use,
3ca75cd5 ML	143	-(long)v);
dee5ea7a KS	144	#endif
	145	}
	146
f35db108 WM	147	INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
	148	u8 v, memory_order mo) {
	149	(void)mo;
	150	DCHECK(!((uptr)a % sizeof(*a)));
696d846a	151	return (u8)_InterlockedExchange8((volatile char*)&a->val_dont_use, v);
f35db108 WM	152	}
	153
	154	INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
	155	u16 v, memory_order mo) {
	156	(void)mo;
	157	DCHECK(!((uptr)a % sizeof(*a)));
696d846a MO	158	return (u16)_InterlockedExchange16((volatile short*)&a->val_dont_use, v);
	159	}
	160
	161	INLINE u32 atomic_exchange(volatile atomic_uint32_t *a,
	162	u32 v, memory_order mo) {
	163	(void)mo;
	164	DCHECK(!((uptr)a % sizeof(*a)));
	165	return (u32)_InterlockedExchange((volatile long*)&a->val_dont_use, v);
f35db108 WM	166	}
f35db108 WM	167
ef1b3fda KS	168	INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
	169	u8 *cmp,
	170	u8 xchgv,
	171	memory_order mo) {
	172	(void)mo;
	173	DCHECK(!((uptr)a % sizeof(*a)));
	174	u8 cmpv = *cmp;
10189819 MO	175	#ifdef _WIN64
	176	u8 prev = (u8)_InterlockedCompareExchange8(
	177	(volatile char*)&a->val_dont_use, (char)xchgv, (char)cmpv);
	178	#else
ef1b3fda KS	179	u8 prev;
	180	__asm {
	181	mov al, cmpv
	182	mov ecx, a
	183	mov dl, xchgv
	184	lock cmpxchg [ecx], dl
	185	mov prev, al
	186	}
10189819	187	#endif
ef1b3fda KS	188	if (prev == cmpv)
	189	return true;
	190	*cmp = prev;
	191	return false;
	192	}
	193
f35db108 WM	194	INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
	195	uptr *cmp,
	196	uptr xchg,
	197	memory_order mo) {
	198	uptr cmpv = *cmp;
e9772e16	199	uptr prev = (uptr)_InterlockedCompareExchangePointer(
f35db108 WM	200	(voidvolatile)&a->val_dont_use, (void)xchg, (void)cmpv);
	201	if (prev == cmpv)
	202	return true;
	203	*cmp = prev;
	204	return false;
	205	}
	206
dee5ea7a KS	207	INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
	208	u16 *cmp,
	209	u16 xchg,
	210	memory_order mo) {
	211	u16 cmpv = *cmp;
	212	u16 prev = (u16)_InterlockedCompareExchange16(
	213	(volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
	214	if (prev == cmpv)
	215	return true;
	216	*cmp = prev;
	217	return false;
	218	}
	219
	220	INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
	221	u32 *cmp,
	222	u32 xchg,
	223	memory_order mo) {
	224	u32 cmpv = *cmp;
	225	u32 prev = (u32)_InterlockedCompareExchange(
	226	(volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
	227	if (prev == cmpv)
	228	return true;
	229	*cmp = prev;
	230	return false;
	231	}
	232
	233	INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
	234	u64 *cmp,
	235	u64 xchg,
	236	memory_order mo) {
	237	u64 cmpv = *cmp;
	238	u64 prev = (u64)_InterlockedCompareExchange64(
	239	(volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
	240	if (prev == cmpv)
	241	return true;
	242	*cmp = prev;
	243	return false;
	244	}
	245
f35db108 WM	246	template<typename T>
f35db108 WM	247	INLINE bool atomic_compare_exchange_weak(volatile T *a,
ef1b3fda KS	248	typename T::Type *cmp,
	249	typename T::Type xchg,
	250	memory_order mo) {
f35db108 WM	251	return atomic_compare_exchange_strong(a, cmp, xchg, mo);
	252	}
	253
	254	} // namespace __sanitizer
	255
	256	#endif // SANITIZER_ATOMIC_CLANG_H