*/
#ifdef VM_ARM_64
-# include "vm_atomic_arm64_begin.h"
-#endif
+#include "vm_basic_asm_arm64.h"
+
+#define ARM_64_ATOMIC_16_OPV_PRIVATE(atm,modval,op) ({ \
+ uint32 _failed; \
+ uint16 _sample; \
+ __asm__ __volatile__ ( \
+ "1: ldxrh %w0, [%2] \n\t" \
+ #op " %w0, %w0, %w3 \n\t" \
+ " stxrh %w1, %w0, [%2] \n\t" \
+ " cbnz %w1, 1b \n\t" \
+ : "=&r" (_sample), \
+ "=&r" (_failed) \
+ : "r" (atm), \
+ "r" (modval) \
+ : "memory"); \
+})
+
+#define ARM_64_ATOMIC_16_OPO_PRIVATE(atm,modval,op) ({ \
+ uint32 _failed; \
+ uint16 _newval; \
+ uint16 _oldval; \
+ __asm__ __volatile__ ( \
+ "1: ldxrh %w0, [%3] \n\t" \
+ #op " %w1, %w0, %w4 \n\t" \
+ " stxrh %w2, %w1, [%3] \n\t" \
+ " cbnz %w2, 1b \n\t" \
+ : "=&r" (_oldval), \
+ "=&r" (_newval), \
+ "=&r" (_failed) \
+ : "r" (atm), \
+ "r" (modval) \
+ : "memory"); \
+ _oldval; \
+})
+
+#define ARM_64_ATOMIC_32_OPV_PRIVATE(atm,modval,op) ({ \
+ uint32 _failed; \
+ uint32 _sample; \
+ __asm__ __volatile__ ( \
+ "1: ldxr %w0, [%2] \n\t" \
+ #op " %w0, %w0, %w3 \n\t" \
+ " stxr %w1, %w0, [%2] \n\t" \
+ " cbnz %w1, 1b \n\t" \
+ : "=&r" (_sample), \
+ "=&r" (_failed) \
+ : "r" (atm), \
+ "r" (modval) \
+ : "memory"); \
+})
+
+#define ARM_64_ATOMIC_32_OPO_PRIVATE(atm,modval,op) ({ \
+ uint32 _failed; \
+ uint32 _newval; \
+ uint32 _oldval; \
+ __asm__ __volatile__ ( \
+ "1: ldxr %w0, [%3] \n\t" \
+ #op " %w1, %w0, %w4 \n\t" \
+ " stxr %w2, %w1, [%3] \n\t" \
+ " cbnz %w2, 1b \n\t" \
+ : "=&r" (_oldval), \
+ "=&r" (_newval), \
+ "=&r" (_failed) \
+ : "r" (atm), \
+ "r" (modval) \
+ : "memory"); \
+ _oldval; \
+})
+
+#define ARM_64_ATOMIC_64_OPV_PRIVATE(atm,modval,op) ({ \
+ uint32 _failed; \
+ uint64 _sample; \
+ __asm__ __volatile__ ( \
+ "1: ldxr %x0, [%2] \n\t" \
+ #op " %x0, %x0, %x3 \n\t" \
+ " stxr %w1, %x0, [%2] \n\t" \
+ " cbnz %w1, 1b \n\t" \
+ : "=&r" (_sample), \
+ "=&r" (_failed) \
+ : "r" (atm), \
+ "r" (modval) \
+ : "memory"); \
+})
+
+#define ARM_64_ATOMIC_64_OPO_PRIVATE(atm,modval,op) ({ \
+ uint32 _failed; \
+ uint64 _newval; \
+ uint64 _oldval; \
+ __asm__ __volatile__ ( \
+ "1: ldxr %x0, [%3] \n\t" \
+ #op " %x1, %x0, %x4 \n\t" \
+ " stxr %w2, %x1, [%3] \n\t" \
+ " cbnz %w2, 1b \n\t" \
+ : "=&r" (_oldval), \
+ "=&r" (_newval), \
+ "=&r" (_failed) \
+ : "r" (atm), \
+ "r" (modval) \
+ : "memory"); \
+ _oldval; \
+})
+
+#define ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(atm,modval,op) ({ \
+ DMB(); \
+ ARM_64_ATOMIC_16_OPV_PRIVATE(atm,modval,op); \
+ DMB(); \
+})
+
+#define ARM_64_ATOMIC_16_OPO_PRIVATE_FENCED(atm,modval,op) ({ \
+ uint16 _oldval_fenced; \
+ DMB(); \
+ _oldval_fenced = ARM_64_ATOMIC_16_OPO_PRIVATE(atm,modval,op); \
+ DMB(); \
+ _oldval_fenced; \
+})
+
+#define ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(atm,modval,op) ({ \
+ DMB(); \
+ ARM_64_ATOMIC_32_OPV_PRIVATE(atm,modval,op); \
+ DMB(); \
+})
+
+#define ARM_64_ATOMIC_32_OPO_PRIVATE_FENCED(atm,modval,op) ({ \
+ uint32 _oldval_fenced; \
+ DMB(); \
+ _oldval_fenced = ARM_64_ATOMIC_32_OPO_PRIVATE(atm,modval,op); \
+ DMB(); \
+ _oldval_fenced; \
+})
+
+#define ARM_64_ATOMIC_64_OPV_PRIVATE_FENCED(atm,modval,op) ({ \
+ DMB(); \
+ ARM_64_ATOMIC_64_OPV_PRIVATE(atm,modval,op); \
+ DMB(); \
+})
+
+#define ARM_64_ATOMIC_64_OPO_PRIVATE_FENCED(atm,modval,op) ({ \
+ uint64 _oldval_fenced; \
+ DMB(); \
+ _oldval_fenced = ARM_64_ATOMIC_64_OPO_PRIVATE(atm,modval,op); \
+ DMB(); \
+ _oldval_fenced; \
+})
+
+#endif /* VM_ARM_64 */
/* Basic atomic types: 16, 32 and 64 bits */
Bool oldVal);
#endif
-/* The ARM32 team is expected to provide an implementation real soon now. */
-#if defined(VM_ARM_32)
-extern Bool AtomicUndefined(void);
-#endif
-
typedef struct Atomic_Bool {
volatile Bool value;
} Atomic_Bool;
-/* This should be enforced on all architectures not just ARM... */
-#if defined(VM_ARM_ANY)
-MY_ASSERTS(AtomicBoolSize,
- ASSERT_ON_COMPILE(sizeof (Atomic_Bool) == sizeof (uint8));
-)
+/* The ARM team can come along and add the code real soon now */
+#if defined(VM_ARM_32) || defined(VM_ARM_64)
+extern Bool AtomicUndefinedOnARM(void);
+MY_ASSERTS(ARM64_BOOL_SIZE,
+ ASSERT_ON_COMPILE(sizeof(Atomic_Bool) == sizeof(uint8));
+ )
#endif
-#if defined VMKERNEL || defined VMM
+#if defined VMKERNEL || defined VMM || defined VM_ARM_64
/*
*-----------------------------------------------------------------------------
*
uint8 val;
#if defined(VM_ARM_64)
- val = _VMATOM_X(RIFEQW, 8, TRUE, ptr, oldVal, newVal);
+ register uint32 failed;
+
+ __asm__ __volatile__ (
+ " dmb sy \n\t"
+ "1: ldxrb %w0, [%2] \n\t"
+ " cmp %w0, %w3, UXTB \n\t"
+ " b.ne 2f \n\t"
+ " stxrb %w1, %w4, [%2] \n\t"
+ " cbnz %w1, 1b \n\t"
+ "2: clrex \n\t"
+ " dmb sy \n\t"
+ : "=&r" (val), "=&r" (failed)
+ : "r" (ptr), "r" (oldVal), "r" (newVal)
+ : "cc", "memory");
#else
__asm__ __volatile__("lock; cmpxchgb %b2, %1"
: "=a" (val),
"0" (oldVal)
: "cc");
-#endif
+#endif /* defined(VM_ARM_64) */
return val;
}
#endif
uint64 hi;
};
} res, _old = { oldVal }, _new = { newVal };
- uint32 failed;
-
+ int failed;
LDST_LDST_MEM_BARRIER();
__asm__ __volatile__ (
"1: ldxp %x0, %x1, [%3] \n\t"
: "=&r" (res.lo),
"=&r" (res.hi),
"=&r" (failed)
- : "r" (&ptr->value),
+ : "r" (ptr),
"r" (_old.lo),
"r" (_old.hi),
"r" (_new.lo),
"r" (_new.hi)
: "cc", "memory");
LDST_LDST_MEM_BARRIER();
-
return res.raw;
#endif
}
Bool val;
#if defined(__GNUC__) && defined(VM_ARM_32)
- val = AtomicUndefined();
-#elif defined(VM_ARM_64)
- val = _VMATOM_X(R, 8, &var->value);
+ val = AtomicUndefinedOnARM();
+#elif defined(__GNUC__) && defined(VM_ARM_64)
+ __asm__ __volatile__ (
+ "ldrb %w0, [%1]"
+ : "=r" (val)
+ : "r" (&var->value)
+ );
+
#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
__asm__ __volatile__(
"movb %1, %0"
Bool val) // IN:
{
#if defined(__GNUC__) && defined(VM_ARM_32)
- return AtomicUndefined();
-#elif defined(VM_ARM_64)
- return _VMATOM_X(RW, 8, TRUE, &var->value, val);
+ return AtomicUndefinedOnARM();
+#elif defined(__GNUC__) && defined(VM_ARM_64)
+ register Bool retVal;
+ register uint32 failed;
+
+ __asm__ __volatile__(
+ " dmb sy \n\t"
+ "1: ldxrb %w0, [%2] \n\t"
+ " stxrb %w1, %w3, [%2] \n\t"
+ " cbnz %w1, 1b \n\t"
+ " dmb sy \n\t"
+ : "=&r" (retVal), "=&r" (failed)
+ : "r" (&var->value), "r" (val)
+ : "memory"
+ );
+
+ return retVal;
+
#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
__asm__ __volatile__(
"xchgb %0, %1"
Bool val) // IN:
{
#if defined(__GNUC__) && defined(VM_ARM_32)
- AtomicUndefined();
-#elif defined(VM_ARM_64)
- _VMATOM_X(W, 8, &var->value, val);
+ AtomicUndefinedOnARM();
+#elif defined(__GNUC__) && defined(VM_ARM_64)
+ Atomic_ReadWriteBool(var, val);
#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
__asm__ __volatile__(
"movb %1, %0"
Bool newVal) // IN:
{
#if defined(__GNUC__) && defined(VM_ARM_32)
- return AtomicUndefined();
-#elif defined(VM_ARM_64)
- return _VMATOM_X(RIFEQW, 8, TRUE, &var->value, oldVal, newVal);
+ return AtomicUndefinedOnARM();
+#elif defined(__GNUC__) && defined(VM_ARM_64)
+ return (Bool)CMPXCHG1B((volatile uint8 *)&var->value, oldVal, newVal);
#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
Bool val;
{
uint32 value;
-#if defined(VMM) || defined(VM_ARM_64)
+#if defined(VMM)
ASSERT(((uintptr_t)var % 4) == 0);
#endif
: "r" (&var->value)
);
#elif defined(VM_ARM_64)
- value = _VMATOM_X(R, 32, &var->value);
+ __asm__ __volatile__ (
+ "ldr %w0, [%1]"
+ : "=r" (value)
+ : "r" (&var->value)
+ );
#else
__asm__ __volatile__(
"mov %1, %0"
return retVal;
#elif defined(VM_ARM_64)
- return _VMATOM_X(RW, 32, TRUE, &var->value, val);
+ register uint32 retVal;
+ register uint32 failed;
+
+ __asm__ __volatile__(
+ " dmb sy \n\t"
+ "1: ldxr %w0, [%2] \n\t"
+ " stxr %w1, %w3, [%2] \n\t"
+ " cbnz %w1, 1b \n\t"
+ " dmb sy \n\t"
+ : "=&r" (retVal), "=&r" (failed)
+ : "r" (&var->value), "r" (val)
+ : "memory"
+ );
+
+ return retVal;
#else /* VM_X86_ANY */
/* Checked against the Intel manual and GCC --walken */
__asm__ __volatile__(
#if defined(__GNUC__)
#if defined(VM_ARM_64)
- _VMATOM_X(W, 32, &var->value, val);
+ /*
+ * Just like x86, this only works for aligned addresses - and that's fine.
+ * Clearing the exclusive monitor is not required. The local monitor is
+ * cleared on any exception return, and the global monitor (as per B2.10.2,
+ * ARM DDI 0487A.k) is cleared by a successful write.
+ */
+ __asm__ __volatile__ (
+ "str %w0, [%1]"
+ :
+ : "r" (val), "r" (&var->value)
+ : "memory"
+ );
#elif defined(VM_ARM_32)
/*
* Best left this way due to the intricacies of exclusive load/store
return retVal;
#elif defined(VM_ARM_64)
- return _VMATOM_X(RIFEQW, 32, TRUE, &var->value, oldVal, newVal);
+ register uint32 retVal;
+ register uint32 failed;
+
+ __asm__ __volatile__ (
+ " dmb sy \n\t"
+ "1: ldxr %w0, [%2] \n\t"
+ " cmp %w0, %w3 \n\t"
+ " b.ne 2f \n\t"
+ " stxr %w1, %w4, [%2] \n\t"
+ " cbnz %w1, 1b \n\t"
+ "2: clrex \n\t"
+ " dmb sy \n\t"
+ : "=&r" (retVal), "=&r" (failed)
+ : "r" (&var->value), "r" (oldVal), "r" (newVal)
+ : "cc", "memory");
+
+ return retVal;
#else /* VM_X86_ANY */
uint32 val;
return retVal;
#elif defined(VM_ARM_64)
- return _VMATOM_X(RIFEQW, 64, TRUE, &var->value, oldVal, newVal);
+ register uint64 retVal;
+ register uint32 failed;
+
+ __asm__ __volatile__ (
+ " dmb sy \n\t"
+ "1: ldxr %x0, [%2] \n\t"
+ " cmp %x0, %x3 \n\t"
+ " b.ne 2f \n\t"
+ " stxr %w1, %x4, [%2] \n\t"
+ " cbnz %w1, 1b \n\t"
+ "2: clrex \n\t"
+ " dmb sy \n\t"
+ : "=&r" (retVal), "=&r" (failed)
+ : "r" (&var->value), "r" (oldVal), "r" (newVal)
+ : "cc", "memory");
+
+ return retVal;
#else /* VM_X86_64 */
uint64 val;
dmb();
#elif defined(VM_ARM_64)
- _VMATOM_X(OP, 32, TRUE, &var->value, and, val);
+ ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, and);
#else /* VM_X86_ANY */
/* Checked against the Intel manual and GCC --walken */
__asm__ __volatile__(
dmb();
#elif defined(VM_ARM_64)
- _VMATOM_X(OP, 32, TRUE, &var->value, orr, val);
+ ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, orr);
#else /* VM_X86_ANY */
/* Checked against the Intel manual and GCC --walken */
__asm__ __volatile__(
dmb();
#elif defined(VM_ARM_64)
- _VMATOM_X(OP, 32, TRUE, &var->value, eor, val);
+ ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, eor);
#else /* VM_X86_ANY */
/* Checked against the Intel manual and GCC --walken */
__asm__ __volatile__(
{
#if defined(__GNUC__)
#if defined(VM_ARM_64)
- _VMATOM_X(OP, 64, TRUE, &var->value, eor, val);
+ ARM_64_ATOMIC_64_OPV_PRIVATE_FENCED(var, val, eor);
#else /* VM_X86_64 */
/* Checked against the AMD manual and GCC --hpreg */
__asm__ __volatile__(
dmb();
#elif defined(VM_ARM_64)
- _VMATOM_X(OP, 32, TRUE, &var->value, add, val);
+ ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, add);
#else /* VM_X86_ANY */
/* Checked against the Intel manual and GCC --walken */
__asm__ __volatile__(
dmb();
#elif defined(VM_ARM_64)
- _VMATOM_X(OP, 32, TRUE, &var->value, sub, val);
+ ARM_64_ATOMIC_32_OPV_PRIVATE_FENCED(var, val, sub);
#else /* VM_X86_ANY */
/* Checked against the Intel manual and GCC --walken */
__asm__ __volatile__(
return retVal;
#elif defined(VM_ARM_64)
- return _VMATOM_X(ROP, 32, TRUE, &var->value, add, val);
+ return ARM_64_ATOMIC_32_OPO_PRIVATE_FENCED(var, val, add);
#else /* VM_X86_ANY */
/* Checked against the Intel manual and GCC --walken */
__asm__ __volatile__(
{
#if defined(__GNUC__)
#if defined(VM_ARM_ANY)
- return Atomic_ReadIfEqualWrite64(var, *oldVal, *newVal) == *oldVal;
+ return (Atomic_ReadIfEqualWrite64(var, *oldVal, *newVal) == *oldVal);
#else /* VM_X86_ANY */
Bool equal;
{
#if defined(__GNUC__)
#if defined(VM_ARM_ANY)
- return Atomic_ReadIfEqualWrite(var, oldVal, newVal) == oldVal;
+ return (Atomic_ReadIfEqualWrite(var, oldVal, newVal) == oldVal);
#else /* VM_X86_ANY */
Bool equal;
uint32 dummy;
static INLINE uint64
Atomic_Read64(Atomic_uint64 const *var) // IN
{
-#if defined VMM || defined VM_ARM_64
- ASSERT((uintptr_t)var % 8 == 0);
-#endif
-
#if defined(__GNUC__) && defined(__x86_64__)
uint64 value;
+
+#ifdef VMM
+ ASSERT((uintptr_t)var % 8 == 0);
+#endif
/*
* Use asm to ensure we emit a single load.
*/
);
return value;
-#elif defined(VM_ARM_64)
- return _VMATOM_X(R, 64, &var->value);
-#endif
+#elif defined(__GNUC__) && defined(VM_ARM_64)
+ uint64 value;
+
+ __asm__ __volatile__(
+ "ldr %0, [%1]"
+ : "=r" (value)
+ : "r" (&var->value));
+
+ return value;
+#endif /* defined(__GNUC__) && defined(VM_ARM_64) */
}
uint64 val) // IN
{
#if defined(VM_ARM_64)
- return _VMATOM_X(ROP, 64, TRUE, &var->value, add, val);
+ return ARM_64_ATOMIC_64_OPO_PRIVATE_FENCED(var, val, add);
#elif defined(__x86_64__)
#if defined(__GNUC__)
Atomic_ReadAdd64(var, val); /* Return value is unused. */
#elif defined(__GNUC__)
#if defined(VM_ARM_64)
- _VMATOM_X(OP, 64, TRUE, &var->value, add, val);
+ ARM_64_ATOMIC_64_OPV_PRIVATE_FENCED(var, val, add);
#else /* defined(VM_X86_64) */
/* Checked against the AMD manual and GCC --hpreg */
__asm__ __volatile__(
Atomic_Sub64(Atomic_uint64 *var, // IN/OUT
uint64 val) // IN
{
-#if !defined(VM_64BIT)
+#if !defined(__x86_64__)
Atomic_ReadSub64(var, val); /* Return value is unused. */
#elif defined(__GNUC__)
#if defined(VM_ARM_64)
- _VMATOM_X(OP, 64, TRUE, &var->value, sub, val);
+ ARM_64_ATOMIC_64_OPV_PRIVATE_FENCED(var, val, sub);
#else /* VM_X86_64 */
/* Checked against the AMD manual and GCC --hpreg */
__asm__ __volatile__(
static INLINE void
Atomic_Inc64(Atomic_uint64 *var) // IN/OUT
{
-#if defined(VM_ARM_64)
- Atomic_Add64(var, 1);
-#elif !defined(__x86_64__)
+#if !defined(__x86_64__)
Atomic_ReadInc64(var); /* Return value is unused. */
#elif defined(__GNUC__)
/* Checked against the AMD manual and GCC --hpreg */
static INLINE void
Atomic_Dec64(Atomic_uint64 *var) // IN/OUT
{
-#if defined(VM_ARM_64)
- Atomic_Sub64(var, 1);
-#elif !defined(__x86_64__)
+#if !defined(__x86_64__)
Atomic_ReadDec64(var); /* Return value is unused. */
#elif defined(__GNUC__)
/* Checked against the AMD manual and GCC --hpreg */
#else
#error No compiler defined for Atomic_ReadWrite64
#endif
-#elif defined(VM_ARM_64)
- return _VMATOM_X(RW, 64, TRUE, &var->value, val);
#else
uint64 oldVal;
Atomic_Write64(Atomic_uint64 *var, // OUT
uint64 val) // IN
{
+#if defined(__x86_64__)
+#if defined(__GNUC__)
+
#if defined(VMM) || defined(VM_ARM_64)
ASSERT((uintptr_t)var % 8 == 0);
#endif
-
-#if defined(__x86_64__)
-#if defined(__GNUC__)
/*
* There is no move instruction for 64-bit immediate to memory, so unless
* the immediate value fits in 32-bit (i.e. can be sign-extended), GCC
#else
#error No compiler defined for Atomic_Write64
#endif
-#elif defined(VM_ARM_64)
- _VMATOM_X(W, 64, &var->value, val);
+#elif defined(VM_ARM_64) && defined(__GNUC__)
+ /*
+ * Just like x86, this only works for aligned addresses - and that's fine.
+ * Clearing the exclusive monitor is not required. The local monitor is
+ * cleared on any exception return, and the global monitor (as per B2.10.2,
+ * ARM DDI 0487A.k) is cleared by a successful write.
+ */
+ __asm__ __volatile__ (
+ "str %x0, [%1]"
+ :
+ : "r" (val), "r" (&var->value)
+ : "memory"
+ );
#else
(void)Atomic_ReadWrite64(var, val);
#endif
#else
#error No compiler defined for Atomic_Or64
#endif
-#elif defined(VM_ARM_64)
- _VMATOM_X(OP, 64, TRUE, &var->value, orr, val);
#else // __x86_64__
uint64 oldVal;
uint64 newVal;
#else
#error No compiler defined for Atomic_And64
#endif
-#elif defined(VM_ARM_64)
- _VMATOM_X(OP, 64, TRUE, &var->value, and, val);
#else // __x86_64__
uint64 oldVal;
uint64 newVal;
{
uint16 value;
-#if defined VMM || defined VM_ARM_64
+#ifdef VMM
ASSERT((uintptr_t)var % 2 == 0);
#endif
#if defined(__GNUC__)
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__))
__asm__ __volatile__(
"movw %1, %0"
: "r" (&var->value)
);
#elif defined(VM_ARM_64)
- value = _VMATOM_X(R, 16, &var->value);
+ __asm__ __volatile__ (
+ "ldrh %w0, [%1]"
+ : "=r" (value)
+ : "r" (&var->value)
+ );
+
#else
#error No 16-bits atomics.
#endif
uint16 val) // IN:
{
#if defined(__GNUC__)
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__))
__asm__ __volatile__(
"xchgw %0, %1"
: "=r" (val),
return retVal;
#elif defined(VM_ARM_64)
- return _VMATOM_X(RW, 16, TRUE, &var->value, val);
+ register uint16 retVal;
+ register uint16 failed;
+
+ __asm__ __volatile__(
+ " dmb sy \n\t"
+ "1: ldxrh %w0, [%2] \n\t"
+ " stxrh %w1, %w3, [%2] \n\t"
+ " cbnz %w1, 1b \n\t"
+ " dmb sy \n\t"
+ : "=&r" (retVal), "=&r" (failed)
+ : "r" (&var->value), "r" (val)
+ : "memory"
+ );
+
+ return retVal;
#else
#error No 16-bits atomics.
#endif
#endif
#if defined(__GNUC__)
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__))
+
__asm__ __volatile__(
"movw %1, %0"
: "=m" (var->value)
: "r" (val)
);
#elif defined(VM_ARM_64)
- _VMATOM_X(W, 16, &var->value, val);
+ /*
+ * Just like x86, this only works for aligned addresses - and that's fine.
+ * Clearing the exclusive monitor is not required. The local monitor is
+ * cleared on any exception return, and the global monitor (as per B2.10.2,
+ * ARM DDI 0487A.k) is cleared by a successful write.
+ */
+ __asm__ __volatile__ (
+ "strh %w0, [%1]"
+ :
+ : "r" (val), "r" (&var->value)
+ : "memory"
+ );
#elif defined(VM_ARM_32)
/*
* Best left this way due to the intricacies of exclusive load/store
uint16 newVal) // IN
{
#if defined(__GNUC__)
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__))
uint16 val;
__asm__ __volatile__(
return retVal;
#elif defined(VM_ARM_64)
- return _VMATOM_X(RIFEQW, 16, TRUE, &var->value, oldVal, newVal);
+ register uint16 retVal;
+ register uint16 failed;
+
+ __asm__ __volatile__ (
+ " dmb sy \n\t"
+ "1: ldxrh %w0, [%2] \n\t"
+ " cmp %w0, %w3 \n\t"
+ " b.ne 2f \n\t"
+ " stxrh %w1, %w4, [%2] \n\t"
+ " cbnz %w1, 1b \n\t"
+ "2: clrex \n\t"
+ " dmb sy \n\t"
+ : "=&r" (retVal), "=&r" (failed)
+ : "r" (&var->value), "r" (oldVal), "r" (newVal)
+ : "cc", "memory");
+
+ return retVal;
#else
#error No 16-bits atomics.
#endif
uint16 val) // IN
{
#if defined(__GNUC__)
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__))
__asm__ __volatile__(
"lock; andw %1, %0"
: "+m" (var->value)
dmb();
#elif defined(VM_ARM_64)
- _VMATOM_X(OP, 16, TRUE, &var->value, and, val);
+ ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, and);
#else
#error No 16-bits atomics.
#endif
uint16 val) // IN
{
#if defined(__GNUC__)
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__))
__asm__ __volatile__(
"lock; orw %1, %0"
: "+m" (var->value)
dmb();
#elif defined(VM_ARM_64)
- _VMATOM_X(OP, 16, TRUE, &var->value, orr, val);
+ ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, orr);
#else
#error No 16-bits atomics.
#endif
#endif
}
-
/*
*-----------------------------------------------------------------------------
*
uint16 val) // IN
{
#if defined(__GNUC__)
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__))
__asm__ __volatile__(
"lock; xorw %1, %0"
: "+m" (var->value)
dmb();
#elif defined(VM_ARM_64)
- _VMATOM_X(OP, 16, TRUE, &var->value, eor, val);
+ ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, eor);
#else
#error No 16-bits atomics.
#endif
uint16 val) // IN
{
#if defined(__GNUC__)
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__))
__asm__ __volatile__(
"lock; addw %1, %0"
: "+m" (var->value)
dmb();
#elif defined(VM_ARM_64)
- _VMATOM_X(OP, 16, TRUE, &var->value, add, val);
+ ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, add);
#else
#error No 16-bits atomics.
#endif
uint16 val) // IN
{
#if defined(__GNUC__)
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__))
__asm__ __volatile__(
"lock; subw %1, %0"
: "+m" (var->value)
dmb();
#elif defined(VM_ARM_64)
- _VMATOM_X(OP, 16, TRUE, &var->value, sub, val);
+ ARM_64_ATOMIC_16_OPV_PRIVATE_FENCED(var, val, sub);
#else
#error No 16-bits atomics.
#endif
Atomic_Inc16(Atomic_uint16 *var) // IN/OUT
{
#if defined(__GNUC__)
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__))
__asm__ __volatile__(
"lock; incw %0"
: "+m" (var->value)
Atomic_Dec16(Atomic_uint16 *var) // IN/OUT
{
#if defined(__GNUC__)
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__))
__asm__ __volatile__(
"lock; decw %0"
: "+m" (var->value)
uint16 val) // IN:
{
#if defined(__GNUC__)
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__))
__asm__ __volatile__(
"lock; xaddw %0, %1"
: "=r" (val),
return retVal;
#elif defined(VM_ARM_64)
- return _VMATOM_X(ROP, 16, TRUE, &var->value, add, val);
+ return ARM_64_ATOMIC_16_OPO_PRIVATE_FENCED(var, val, add);
#else
#error No 16-bits atomics.
#endif
#endif /* ATOMIC_COMPILER_BARRIER */
#endif // Should we define Atomic_MFence?
-#ifdef VM_ARM_64
-# include "vm_atomic_arm64_end.h"
-#endif
-
#if defined(__cplusplus)
} // extern "C"
#endif
projects / thirdparty / open-vm-tools.git / commitdiff
