#ifdef _POSIX_PRIORITY_SCHEDULING
#include <sched.h>
#endif
+#include <haproxy/atomic.h>
#include <haproxy/api.h>
#define __decl_rwlock(lock)
#define __decl_aligned_rwlock(lock)
-#define HA_ATOMIC_CAS(val, old, new) \
- ({ \
- typeof(val) _v = (val); \
- typeof(old) _o = (old); \
- (*_v == *_o) ? ((*_v = (new)), 1) : ((*_o = *_v), 0); \
- })
-
-/* warning, n is a pointer to the double value for dwcas */
-#define HA_ATOMIC_DWCAS(val, o, n) \
- ({ \
- long *_v = (long*)(val); \
- long *_o = (long*)(o); \
- long *_n = (long*)(n); \
- long _v0 = _v[0], _v1 = _v[1]; \
- (_v0 == _o[0] && _v1 == _o[1]) ? \
- (_v[0] = _n[0], _v[1] = _n[1], 1) : \
- (_o[0] = _v0, _o[1] = _v1, 0); \
- })
-
-#define HA_ATOMIC_ADD(val, i) ({*(val) += (i);})
-#define HA_ATOMIC_SUB(val, i) ({*(val) -= (i);})
-#define HA_ATOMIC_XADD(val, i) \
- ({ \
- typeof((val)) __p_xadd = (val); \
- typeof(*(val)) __old_xadd = *__p_xadd; \
- *__p_xadd += i; \
- __old_xadd; \
- })
-#define HA_ATOMIC_AND(val, flags) ({*(val) &= (flags);})
-#define HA_ATOMIC_OR(val, flags) ({*(val) |= (flags);})
-#define HA_ATOMIC_XCHG(val, new) \
- ({ \
- typeof(*(val)) __old_xchg = *(val); \
- *(val) = new; \
- __old_xchg; \
- })
-#define HA_ATOMIC_BTS(val, bit) \
- ({ \
- typeof((val)) __p_bts = (val); \
- typeof(*__p_bts) __b_bts = (1UL << (bit)); \
- typeof(*__p_bts) __t_bts = *__p_bts & __b_bts; \
- if (!__t_bts) \
- *__p_bts |= __b_bts; \
- __t_bts; \
- })
-#define HA_ATOMIC_BTR(val, bit) \
- ({ \
- typeof((val)) __p_btr = (val); \
- typeof(*__p_btr) __b_btr = (1UL << (bit)); \
- typeof(*__p_btr) __t_btr = *__p_btr & __b_btr; \
- if (__t_btr) \
- *__p_btr &= ~__b_btr; \
- __t_btr; \
- })
-#define HA_ATOMIC_LOAD(val) *(val)
-#define HA_ATOMIC_STORE(val, new) ({*(val) = new;})
-#define HA_ATOMIC_UPDATE_MAX(val, new) \
- ({ \
- typeof(val) __val = (val); \
- typeof(*(val)) __new_max = (new); \
- \
- if (*__val < __new_max) \
- *__val = __new_max; \
- *__val; \
- })
-
-#define HA_ATOMIC_UPDATE_MIN(val, new) \
- ({ \
- typeof(val) __val = (val); \
- typeof(*(val)) __new_min = (new); \
- \
- if (*__val > __new_min) \
- *__val = __new_min; \
- *__val; \
- })
-
-#define HA_BARRIER() do { } while (0)
-
#define HA_SPIN_INIT(l) do { /* do nothing */ } while(0)
#define HA_SPIN_DESTROY(l) do { /* do nothing */ } while(0)
#define HA_SPIN_LOCK(lbl, l) do { /* do nothing */ } while(0)
raise(sig);
}
-static inline void __ha_barrier_atomic_load(void)
-{
-}
-
-static inline void __ha_barrier_atomic_store(void)
-{
-}
-
-static inline void __ha_barrier_atomic_full(void)
-{
-}
-
-static inline void __ha_barrier_load(void)
-{
-}
-
-static inline void __ha_barrier_store(void)
-{
-}
-
-static inline void __ha_barrier_full(void)
-{
-}
-
static inline void thread_harmless_now()
{
}
HA_RWLOCK_T (lock) __attribute__((aligned(64))); \
INITCALL1(STG_LOCK, ha_rwlock_init, &(lock))
-/* TODO: thread: For now, we rely on GCC builtins but it could be a good idea to
- * have a header file regrouping all functions dealing with threads. */
-
-#if defined(__GNUC__) && (__GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ < 7) && !defined(__clang__)
-/* gcc < 4.7 */
-
-#define HA_ATOMIC_ADD(val, i) __sync_add_and_fetch(val, i)
-#define HA_ATOMIC_SUB(val, i) __sync_sub_and_fetch(val, i)
-#define HA_ATOMIC_XADD(val, i) __sync_fetch_and_add(val, i)
-#define HA_ATOMIC_AND(val, flags) __sync_and_and_fetch(val, flags)
-#define HA_ATOMIC_OR(val, flags) __sync_or_and_fetch(val, flags)
-
-/* the CAS is a bit complicated. The older API doesn't support returning the
- * value and the swap's result at the same time. So here we take what looks
- * like the safest route, consisting in using the boolean version guaranteeing
- * that the operation was performed or not, and we snoop a previous value. If
- * the compare succeeds, we return. If it fails, we return the previous value,
- * but only if it differs from the expected one. If it's the same it's a race
- * thus we try again to avoid confusing a possibly sensitive caller.
- */
-#define HA_ATOMIC_CAS(val, old, new) \
- ({ \
- typeof((val)) __val_cas = (val); \
- typeof((old)) __oldp_cas = (old); \
- typeof(*(old)) __oldv_cas; \
- typeof((new)) __new_cas = (new); \
- int __ret_cas; \
- do { \
- __oldv_cas = *__val_cas; \
- __ret_cas = __sync_bool_compare_and_swap(__val_cas, *__oldp_cas, __new_cas); \
- } while (!__ret_cas && *__oldp_cas == __oldv_cas); \
- if (!__ret_cas) \
- *__oldp_cas = __oldv_cas; \
- __ret_cas; \
- })
-
-/* warning, n is a pointer to the double value for dwcas */
-#define HA_ATOMIC_DWCAS(val, o, n) __ha_cas_dw(val, o, n)
-
-#define HA_ATOMIC_XCHG(val, new) \
- ({ \
- typeof((val)) __val_xchg = (val); \
- typeof(*(val)) __old_xchg; \
- typeof((new)) __new_xchg = (new); \
- do { __old_xchg = *__val_xchg; \
- } while (!__sync_bool_compare_and_swap(__val_xchg, __old_xchg, __new_xchg)); \
- __old_xchg; \
- })
-
-#define HA_ATOMIC_BTS(val, bit) \
- ({ \
- typeof(*(val)) __b_bts = (1UL << (bit)); \
- __sync_fetch_and_or((val), __b_bts) & __b_bts; \
- })
-
-#define HA_ATOMIC_BTR(val, bit) \
- ({ \
- typeof(*(val)) __b_btr = (1UL << (bit)); \
- __sync_fetch_and_and((val), ~__b_btr) & __b_btr; \
- })
-
-#define HA_ATOMIC_LOAD(val) \
- ({ \
- typeof(*(val)) ret; \
- __sync_synchronize(); \
- ret = *(volatile typeof(val))val; \
- __sync_synchronize(); \
- ret; \
- })
-
-#define HA_ATOMIC_STORE(val, new) \
- ({ \
- typeof((val)) __val_store = (val); \
- typeof(*(val)) __old_store; \
- typeof((new)) __new_store = (new); \
- do { __old_store = *__val_store; \
- } while (!__sync_bool_compare_and_swap(__val_store, __old_store, __new_store)); \
- })
-#else
-/* gcc >= 4.7 */
-#define HA_ATOMIC_CAS(val, old, new) __atomic_compare_exchange_n(val, old, new, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
-/* warning, n is a pointer to the double value for dwcas */
-#define HA_ATOMIC_DWCAS(val, o, n) __ha_cas_dw(val, o, n)
-#define HA_ATOMIC_ADD(val, i) __atomic_add_fetch(val, i, __ATOMIC_SEQ_CST)
-#define HA_ATOMIC_XADD(val, i) __atomic_fetch_add(val, i, __ATOMIC_SEQ_CST)
-#define HA_ATOMIC_SUB(val, i) __atomic_sub_fetch(val, i, __ATOMIC_SEQ_CST)
-#define HA_ATOMIC_AND(val, flags) __atomic_and_fetch(val, flags, __ATOMIC_SEQ_CST)
-#define HA_ATOMIC_OR(val, flags) __atomic_or_fetch(val, flags, __ATOMIC_SEQ_CST)
-#define HA_ATOMIC_BTS(val, bit) \
- ({ \
- typeof(*(val)) __b_bts = (1UL << (bit)); \
- __sync_fetch_and_or((val), __b_bts) & __b_bts; \
- })
-
-#define HA_ATOMIC_BTR(val, bit) \
- ({ \
- typeof(*(val)) __b_btr = (1UL << (bit)); \
- __sync_fetch_and_and((val), ~__b_btr) & __b_btr; \
- })
-
-#define HA_ATOMIC_XCHG(val, new) __atomic_exchange_n(val, new, __ATOMIC_SEQ_CST)
-#define HA_ATOMIC_STORE(val, new) __atomic_store_n(val, new, __ATOMIC_SEQ_CST)
-#define HA_ATOMIC_LOAD(val) __atomic_load_n(val, __ATOMIC_SEQ_CST)
-
-/* Variants that don't generate any memory barrier.
- * If you're unsure how to deal with barriers, just use the HA_ATOMIC_* version,
- * that will always generate correct code.
- * Usually it's fine to use those when updating data that have no dependency,
- * ie updating a counter. Otherwise a barrier is required.
- */
-#define _HA_ATOMIC_CAS(val, old, new) __atomic_compare_exchange_n(val, old, new, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)
-/* warning, n is a pointer to the double value for dwcas */
-#define _HA_ATOMIC_DWCAS(val, o, n) __ha_cas_dw(val, o, n)
-#define _HA_ATOMIC_ADD(val, i) __atomic_add_fetch(val, i, __ATOMIC_RELAXED)
-#define _HA_ATOMIC_XADD(val, i) __atomic_fetch_add(val, i, __ATOMIC_RELAXED)
-#define _HA_ATOMIC_SUB(val, i) __atomic_sub_fetch(val, i, __ATOMIC_RELAXED)
-#define _HA_ATOMIC_AND(val, flags) __atomic_and_fetch(val, flags, __ATOMIC_RELAXED)
-#define _HA_ATOMIC_OR(val, flags) __atomic_or_fetch(val, flags, __ATOMIC_RELAXED)
-#define _HA_ATOMIC_XCHG(val, new) __atomic_exchange_n(val, new, __ATOMIC_RELAXED)
-#define _HA_ATOMIC_STORE(val, new) __atomic_store_n(val, new, __ATOMIC_RELAXED)
-#define _HA_ATOMIC_LOAD(val) __atomic_load_n(val, __ATOMIC_RELAXED)
-
-#endif /* gcc >= 4.7 */
-
-#define HA_ATOMIC_UPDATE_MAX(val, new) \
- ({ \
- typeof(val) __val = (val); \
- typeof(*(val)) __old_max = *__val; \
- typeof(*(val)) __new_max = (new); \
- \
- while (__old_max < __new_max && \
- !HA_ATOMIC_CAS(__val, &__old_max, __new_max)); \
- *__val; \
- })
-#define HA_ATOMIC_UPDATE_MIN(val, new) \
- ({ \
- typeof(val) __val = (val); \
- typeof(*(val)) __old_min = *__val; \
- typeof(*(val)) __new_min = (new); \
- \
- while (__old_min > __new_min && \
- !HA_ATOMIC_CAS(__val, &__old_min, __new_min)); \
- *__val; \
- })
-
-#define HA_BARRIER() pl_barrier()
-
void thread_harmless_till_end();
void thread_isolate();
void thread_release();
#endif /* DEBUG_THREAD */
-#ifdef __x86_64__
-
-static __inline int
-__ha_cas_dw(void *target, void *compare, const void *set)
-{
- char ret;
-
- __asm __volatile("lock cmpxchg16b %0; setz %3"
- : "+m" (*(void **)target),
- "=a" (((void **)compare)[0]),
- "=d" (((void **)compare)[1]),
- "=q" (ret)
- : "a" (((void **)compare)[0]),
- "d" (((void **)compare)[1]),
- "b" (((const void **)set)[0]),
- "c" (((const void **)set)[1])
- : "memory", "cc");
- return (ret);
-}
-
-/* Use __ha_barrier_atomic* when you're trying to protect data that are
- * are modified using HA_ATOMIC* (except HA_ATOMIC_STORE)
- */
-static __inline void
-__ha_barrier_atomic_load(void)
-{
- __asm __volatile("" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_atomic_store(void)
-{
- __asm __volatile("" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_atomic_full(void)
-{
- __asm __volatile("" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_load(void)
-{
- __asm __volatile("lfence" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_store(void)
-{
- __asm __volatile("sfence" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_full(void)
-{
- __asm __volatile("mfence" ::: "memory");
-}
-
-#elif defined(__arm__) && (defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__))
-
-/* Use __ha_barrier_atomic* when you're trying to protect data that are
- * are modified using HA_ATOMIC* (except HA_ATOMIC_STORE)
- */
-static __inline void
-__ha_barrier_atomic_load(void)
-{
- __asm __volatile("dmb" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_atomic_store(void)
-{
- __asm __volatile("dsb" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_atomic_full(void)
-{
- __asm __volatile("dmb" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_load(void)
-{
- __asm __volatile("dmb" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_store(void)
-{
- __asm __volatile("dsb" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_full(void)
-{
- __asm __volatile("dmb" ::: "memory");
-}
-
-static __inline int __ha_cas_dw(void *target, void *compare, const void *set)
-{
- uint64_t previous;
- int tmp;
-
- __asm __volatile("1:"
- "ldrexd %0, [%4];"
- "cmp %Q0, %Q2;"
- "ittt eq;"
- "cmpeq %R0, %R2;"
- "strexdeq %1, %3, [%4];"
- "cmpeq %1, #1;"
- "beq 1b;"
- : "=&r" (previous), "=&r" (tmp)
- : "r" (*(uint64_t *)compare), "r" (*(uint64_t *)set), "r" (target)
- : "memory", "cc");
- tmp = (previous == *(uint64_t *)compare);
- *(uint64_t *)compare = previous;
- return (tmp);
-}
-
-#elif defined (__aarch64__)
-
-/* Use __ha_barrier_atomic* when you're trying to protect data that are
- * are modified using HA_ATOMIC* (except HA_ATOMIC_STORE)
- */
-static __inline void
-__ha_barrier_atomic_load(void)
-{
- __asm __volatile("dmb ishld" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_atomic_store(void)
-{
- __asm __volatile("dmb ishst" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_atomic_full(void)
-{
- __asm __volatile("dmb ish" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_load(void)
-{
- __asm __volatile("dmb ishld" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_store(void)
-{
- __asm __volatile("dmb ishst" ::: "memory");
-}
-
-static __inline void
-__ha_barrier_full(void)
-{
- __asm __volatile("dmb ish" ::: "memory");
-}
-
-static __inline int __ha_cas_dw(void *target, void *compare, void *set)
-{
- void *value[2];
- uint64_t tmp1, tmp2;
-
- __asm__ __volatile__("1:"
- "ldxp %0, %1, [%4];"
- "mov %2, %0;"
- "mov %3, %1;"
- "eor %0, %0, %5;"
- "eor %1, %1, %6;"
- "orr %1, %0, %1;"
- "mov %w0, #0;"
- "cbnz %1, 2f;"
- "stxp %w0, %7, %8, [%4];"
- "cbnz %w0, 1b;"
- "mov %w0, #1;"
- "2:"
- : "=&r" (tmp1), "=&r" (tmp2), "=&r" (value[0]), "=&r" (value[1])
- : "r" (target), "r" (((void **)(compare))[0]), "r" (((void **)(compare))[1]), "r" (((void **)(set))[0]), "r" (((void **)(set))[1])
- : "cc", "memory");
-
- memcpy(compare, &value, sizeof(value));
- return (tmp1);
-}
-
-#else
-#define __ha_barrier_atomic_load __sync_synchronize
-#define __ha_barrier_atomic_store __sync_synchronize
-#define __ha_barrier_atomic_full __sync_synchronize
-#define __ha_barrier_load __sync_synchronize
-#define __ha_barrier_store __sync_synchronize
-#define __ha_barrier_full __sync_synchronize
-#endif
void ha_spin_init(HA_SPINLOCK_T *l);
void ha_rwlock_init(HA_RWLOCK_T *l);
extern int thread_cpus_enabled_at_boot;
-static inline void __ha_compiler_barrier(void)
-{
- __asm __volatile("" ::: "memory");
-}
-
int parse_nbthread(const char *arg, char **err);
int thread_get_default_count();
-#ifndef _HA_ATOMIC_CAS
-#define _HA_ATOMIC_CAS HA_ATOMIC_CAS
-#endif /* !_HA_ATOMIC_CAS */
-
-#ifndef _HA_ATOMIC_DWCAS
-#define _HA_ATOMIC_DWCAS HA_ATOMIC_DWCAS
-#endif /* !_HA_ATOMIC_CAS */
-
-#ifndef _HA_ATOMIC_ADD
-#define _HA_ATOMIC_ADD HA_ATOMIC_ADD
-#endif /* !_HA_ATOMIC_ADD */
-
-#ifndef _HA_ATOMIC_XADD
-#define _HA_ATOMIC_XADD HA_ATOMIC_XADD
-#endif /* !_HA_ATOMIC_SUB */
-
-#ifndef _HA_ATOMIC_SUB
-#define _HA_ATOMIC_SUB HA_ATOMIC_SUB
-#endif /* !_HA_ATOMIC_SUB */
-
-#ifndef _HA_ATOMIC_AND
-#define _HA_ATOMIC_AND HA_ATOMIC_AND
-#endif /* !_HA_ATOMIC_AND */
-
-#ifndef _HA_ATOMIC_OR
-#define _HA_ATOMIC_OR HA_ATOMIC_OR
-#endif /* !_HA_ATOMIC_OR */
-
-#ifndef _HA_ATOMIC_XCHG
-#define _HA_ATOMIC_XCHG HA_ATOMIC_XCHG
-#endif /* !_HA_ATOMIC_XCHG */
-
-#ifndef _HA_ATOMIC_STORE
-#define _HA_ATOMIC_STORE HA_ATOMIC_STORE
-#endif /* !_HA_ATOMIC_STORE */
-
-#ifndef _HA_ATOMIC_LOAD
-#define _HA_ATOMIC_LOAD HA_ATOMIC_LOAD
-#endif /* !_HA_ATOMIC_LOAD */
#endif /* _COMMON_HATHREADS_H */
--- /dev/null
+/*
+ * include/haproxy/atomic.h
+ * Macros and inline functions for thread-safe atomic operations.
+ *
+ * Copyright (C) 2017 Christopher Faulet - cfaulet@haproxy.com
+ * Copyright (C) 2020 Willy Tarreau - w@1wt.eu
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation, version 2.1
+ * exclusively.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef _HAPROXY_ATOMIC_H
+#define _HAPROXY_ATOMIC_H
+
+/* A few notes for the macros and functions here:
+ * - this file is painful to edit, most operations exist in 3 variants,
+ * no-thread, threads with gcc<4.7, threads with gcc>=4.7. Be careful when
+ * modifying it not to break any of them.
+ *
+ * - macros named HA_ATOMIC_* are or use in the general case, they contain the
+ * required memory barriers to guarantee sequential consistency
+ *
+ * - macros named _HA_ATOMIC_* are the same but without the memory barriers,
+ * so they may only be used if followed by other HA_ATOMIC_* or within a
+ * sequence of _HA_ATOMIC_* terminated by a store barrier, or when there is
+ * no data dependency (e.g. updating a counter). Not all of them are
+ * implemented, in which case fallbacks to the safe ones are provided. In
+ * case of doubt, don't use them and use the generic ones instead.
+ *
+ * - the __ha_atomic_* barriers are for use around _HA_ATOMIC_* operations.
+ * Some architectures make them useless and they will automatically be
+ * dropped in such a case. Don't use them outside of this use case.
+ *
+ * - in general, the more underscores you find in front of a function or macro
+ * name, the riskier it is to use. Barriers are among them because validating
+ * their usage is not trivial at all and it's often safer to fall back to
+ * more generic behaviors.
+ *
+ * There is also a compiler barrier (__ha_compiler_barrier) which is eliminated
+ * when threads are disabled. We currently don't have a permanent compiler
+ * barrier to prevent the compiler from reordering signal-sensitive code for
+ * example.
+ */
+
+
+#ifndef USE_THREAD
+
+/* Threads are DISABLED, atomic ops are also not used. Note that these MUST
+ * NOT be used for inter-process synchronization nor signal-safe variable
+ * manipulations which might occur without threads, as they are not atomic.
+ */
+
+#define HA_ATOMIC_LOAD(val) *(val)
+#define HA_ATOMIC_STORE(val, new) ({*(val) = new;})
+
+#define HA_ATOMIC_XCHG(val, new) \
+ ({ \
+ typeof(*(val)) __old_xchg = *(val); \
+ *(val) = new; \
+ __old_xchg; \
+ })
+
+#define HA_ATOMIC_AND(val, flags) ({*(val) &= (flags);})
+#define HA_ATOMIC_OR(val, flags) ({*(val) |= (flags);})
+#define HA_ATOMIC_ADD(val, i) ({*(val) += (i);})
+#define HA_ATOMIC_SUB(val, i) ({*(val) -= (i);})
+#define HA_ATOMIC_XADD(val, i) \
+ ({ \
+ typeof((val)) __p_xadd = (val); \
+ typeof(*(val)) __old_xadd = *__p_xadd; \
+ *__p_xadd += i; \
+ __old_xadd; \
+ })
+
+#define HA_ATOMIC_BTS(val, bit) \
+ ({ \
+ typeof((val)) __p_bts = (val); \
+ typeof(*__p_bts) __b_bts = (1UL << (bit)); \
+ typeof(*__p_bts) __t_bts = *__p_bts & __b_bts; \
+ if (!__t_bts) \
+ *__p_bts |= __b_bts; \
+ __t_bts; \
+ })
+
+#define HA_ATOMIC_BTR(val, bit) \
+ ({ \
+ typeof((val)) __p_btr = (val); \
+ typeof(*__p_btr) __b_btr = (1UL << (bit)); \
+ typeof(*__p_btr) __t_btr = *__p_btr & __b_btr; \
+ if (__t_btr) \
+ *__p_btr &= ~__b_btr; \
+ __t_btr; \
+ })
+
+#define HA_ATOMIC_CAS(val, old, new) \
+ ({ \
+ typeof(val) _v = (val); \
+ typeof(old) _o = (old); \
+ (*_v == *_o) ? ((*_v = (new)), 1) : ((*_o = *_v), 0); \
+ })
+
+/* warning, n is a pointer to the double value for dwcas */
+#define HA_ATOMIC_DWCAS(val, o, n) \
+ ({ \
+ long *_v = (long*)(val); \
+ long *_o = (long*)(o); \
+ long *_n = (long*)(n); \
+ long _v0 = _v[0], _v1 = _v[1]; \
+ (_v0 == _o[0] && _v1 == _o[1]) ? \
+ (_v[0] = _n[0], _v[1] = _n[1], 1) : \
+ (_o[0] = _v0, _o[1] = _v1, 0); \
+ })
+
+#define HA_ATOMIC_UPDATE_MAX(val, new) \
+ ({ \
+ typeof(val) __val = (val); \
+ typeof(*(val)) __new_max = (new); \
+ \
+ if (*__val < __new_max) \
+ *__val = __new_max; \
+ *__val; \
+ })
+
+#define HA_ATOMIC_UPDATE_MIN(val, new) \
+ ({ \
+ typeof(val) __val = (val); \
+ typeof(*(val)) __new_min = (new); \
+ \
+ if (*__val > __new_min) \
+ *__val = __new_min; \
+ *__val; \
+ })
+
+/* various barriers */
+#define __ha_barrier_atomic_load() do { } while (0)
+#define __ha_barrier_atomic_store() do { } while (0)
+#define __ha_barrier_atomic_full() do { } while (0)
+#define __ha_barrier_load() do { } while (0)
+#define __ha_barrier_store() do { } while (0)
+#define __ha_barrier_full() do { } while (0)
+#define __ha_compiler_barrier() do { } while (0)
+
+#else /* !USE_THREAD */
+
+/* Threads are ENABLED, all atomic ops are made thread-safe. By extension they
+ * can also be used for inter-process synchronization but one must verify that
+ * the code still builds with threads disabled.
+ */
+
+#if defined(__GNUC__) && (__GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ < 7) && !defined(__clang__)
+/* gcc < 4.7 */
+
+#define HA_ATOMIC_LOAD(val) \
+ ({ \
+ typeof(*(val)) ret; \
+ __sync_synchronize(); \
+ ret = *(volatile typeof(val))val; \
+ __sync_synchronize(); \
+ ret; \
+ })
+
+#define HA_ATOMIC_STORE(val, new) \
+ ({ \
+ typeof((val)) __val_store = (val); \
+ typeof(*(val)) __old_store; \
+ typeof((new)) __new_store = (new); \
+ do { __old_store = *__val_store; \
+ } while (!__sync_bool_compare_and_swap(__val_store, __old_store, __new_store)); \
+ })
+
+#define HA_ATOMIC_XCHG(val, new) \
+ ({ \
+ typeof((val)) __val_xchg = (val); \
+ typeof(*(val)) __old_xchg; \
+ typeof((new)) __new_xchg = (new); \
+ do { __old_xchg = *__val_xchg; \
+ } while (!__sync_bool_compare_and_swap(__val_xchg, __old_xchg, __new_xchg)); \
+ __old_xchg; \
+ })
+
+#define HA_ATOMIC_AND(val, flags) __sync_and_and_fetch(val, flags)
+#define HA_ATOMIC_OR(val, flags) __sync_or_and_fetch(val, flags)
+#define HA_ATOMIC_ADD(val, i) __sync_add_and_fetch(val, i)
+#define HA_ATOMIC_SUB(val, i) __sync_sub_and_fetch(val, i)
+#define HA_ATOMIC_XADD(val, i) __sync_fetch_and_add(val, i)
+
+#define HA_ATOMIC_BTS(val, bit) \
+ ({ \
+ typeof(*(val)) __b_bts = (1UL << (bit)); \
+ __sync_fetch_and_or((val), __b_bts) & __b_bts; \
+ })
+
+#define HA_ATOMIC_BTR(val, bit) \
+ ({ \
+ typeof(*(val)) __b_btr = (1UL << (bit)); \
+ __sync_fetch_and_and((val), ~__b_btr) & __b_btr; \
+ })
+
+/* the CAS is a bit complicated. The older API doesn't support returning the
+ * value and the swap's result at the same time. So here we take what looks
+ * like the safest route, consisting in using the boolean version guaranteeing
+ * that the operation was performed or not, and we snoop a previous value. If
+ * the compare succeeds, we return. If it fails, we return the previous value,
+ * but only if it differs from the expected one. If it's the same it's a race
+ * thus we try again to avoid confusing a possibly sensitive caller.
+ */
+#define HA_ATOMIC_CAS(val, old, new) \
+ ({ \
+ typeof((val)) __val_cas = (val); \
+ typeof((old)) __oldp_cas = (old); \
+ typeof(*(old)) __oldv_cas; \
+ typeof((new)) __new_cas = (new); \
+ int __ret_cas; \
+ do { \
+ __oldv_cas = *__val_cas; \
+ __ret_cas = __sync_bool_compare_and_swap(__val_cas, *__oldp_cas, __new_cas); \
+ } while (!__ret_cas && *__oldp_cas == __oldv_cas); \
+ if (!__ret_cas) \
+ *__oldp_cas = __oldv_cas; \
+ __ret_cas; \
+ })
+
+/* warning, n is a pointer to the double value for dwcas */
+#define HA_ATOMIC_DWCAS(val, o, n) __ha_cas_dw(val, o, n)
+
+#define HA_ATOMIC_UPDATE_MAX(val, new) \
+ ({ \
+ typeof(val) __val = (val); \
+ typeof(*(val)) __old_max = *__val; \
+ typeof(*(val)) __new_max = (new); \
+ \
+ while (__old_max < __new_max && \
+ !HA_ATOMIC_CAS(__val, &__old_max, __new_max)); \
+ *__val; \
+ })
+
+#define HA_ATOMIC_UPDATE_MIN(val, new) \
+ ({ \
+ typeof(val) __val = (val); \
+ typeof(*(val)) __old_min = *__val; \
+ typeof(*(val)) __new_min = (new); \
+ \
+ while (__old_min > __new_min && \
+ !HA_ATOMIC_CAS(__val, &__old_min, __new_min)); \
+ *__val; \
+ })
+
+#else /* gcc */
+
+/* gcc >= 4.7 or clang */
+
+#define HA_ATOMIC_STORE(val, new) __atomic_store_n(val, new, __ATOMIC_SEQ_CST)
+#define HA_ATOMIC_LOAD(val) __atomic_load_n(val, __ATOMIC_SEQ_CST)
+#define HA_ATOMIC_XCHG(val, new) __atomic_exchange_n(val, new, __ATOMIC_SEQ_CST)
+
+#define HA_ATOMIC_AND(val, flags) __atomic_and_fetch(val, flags, __ATOMIC_SEQ_CST)
+#define HA_ATOMIC_OR(val, flags) __atomic_or_fetch(val, flags, __ATOMIC_SEQ_CST)
+#define HA_ATOMIC_ADD(val, i) __atomic_add_fetch(val, i, __ATOMIC_SEQ_CST)
+#define HA_ATOMIC_SUB(val, i) __atomic_sub_fetch(val, i, __ATOMIC_SEQ_CST)
+#define HA_ATOMIC_XADD(val, i) __atomic_fetch_add(val, i, __ATOMIC_SEQ_CST)
+
+#define HA_ATOMIC_BTS(val, bit) \
+ ({ \
+ typeof(*(val)) __b_bts = (1UL << (bit)); \
+ __sync_fetch_and_or((val), __b_bts) & __b_bts; \
+ })
+
+#define HA_ATOMIC_BTR(val, bit) \
+ ({ \
+ typeof(*(val)) __b_btr = (1UL << (bit)); \
+ __sync_fetch_and_and((val), ~__b_btr) & __b_btr; \
+ })
+
+#define HA_ATOMIC_CAS(val, old, new) __atomic_compare_exchange_n(val, old, new, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
+
+/* warning, n is a pointer to the double value for dwcas */
+#define HA_ATOMIC_DWCAS(val, o, n) __ha_cas_dw(val, o, n)
+
+#define HA_ATOMIC_UPDATE_MAX(val, new) \
+ ({ \
+ typeof(val) __val = (val); \
+ typeof(*(val)) __old_max = *__val; \
+ typeof(*(val)) __new_max = (new); \
+ \
+ while (__old_max < __new_max && \
+ !HA_ATOMIC_CAS(__val, &__old_max, __new_max)); \
+ *__val; \
+ })
+
+#define HA_ATOMIC_UPDATE_MIN(val, new) \
+ ({ \
+ typeof(val) __val = (val); \
+ typeof(*(val)) __old_min = *__val; \
+ typeof(*(val)) __new_min = (new); \
+ \
+ while (__old_min > __new_min && \
+ !HA_ATOMIC_CAS(__val, &__old_min, __new_min)); \
+ *__val; \
+ })
+
+/* Modern compilers provide variants that don't generate any memory barrier.
+ * If you're unsure how to deal with barriers, just use the HA_ATOMIC_* version,
+ * that will always generate correct code.
+ * Usually it's fine to use those when updating data that have no dependency,
+ * ie updating a counter. Otherwise a barrier is required.
+ */
+
+#define _HA_ATOMIC_LOAD(val) __atomic_load_n(val, __ATOMIC_RELAXED)
+#define _HA_ATOMIC_STORE(val, new) __atomic_store_n(val, new, __ATOMIC_RELAXED)
+#define _HA_ATOMIC_XCHG(val, new) __atomic_exchange_n(val, new, __ATOMIC_RELAXED)
+#define _HA_ATOMIC_AND(val, flags) __atomic_and_fetch(val, flags, __ATOMIC_RELAXED)
+#define _HA_ATOMIC_OR(val, flags) __atomic_or_fetch(val, flags, __ATOMIC_RELAXED)
+#define _HA_ATOMIC_ADD(val, i) __atomic_add_fetch(val, i, __ATOMIC_RELAXED)
+#define _HA_ATOMIC_SUB(val, i) __atomic_sub_fetch(val, i, __ATOMIC_RELAXED)
+#define _HA_ATOMIC_XADD(val, i) __atomic_fetch_add(val, i, __ATOMIC_RELAXED)
+#define _HA_ATOMIC_CAS(val, old, new) __atomic_compare_exchange_n(val, old, new, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)
+/* warning, n is a pointer to the double value for dwcas */
+#define _HA_ATOMIC_DWCAS(val, o, n) __ha_cas_dw(val, o, n)
+
+#endif /* gcc >= 4.7 */
+
+/* Here come a few architecture-specific double-word CAS and barrier
+ * implementations.
+ */
+
+#ifdef __x86_64__
+
+static __inline void
+__ha_barrier_load(void)
+{
+ __asm __volatile("lfence" ::: "memory");
+}
+
+static __inline void
+__ha_barrier_store(void)
+{
+ __asm __volatile("sfence" ::: "memory");
+}
+
+static __inline void
+__ha_barrier_full(void)
+{
+ __asm __volatile("mfence" ::: "memory");
+}
+
+/* Use __ha_barrier_atomic* when you're trying to protect data that are
+ * are modified using _HA_ATOMIC*
+ */
+static __inline void
+__ha_barrier_atomic_load(void)
+{
+ __asm __volatile("" ::: "memory");
+}
+
+static __inline void
+__ha_barrier_atomic_store(void)
+{
+ __asm __volatile("" ::: "memory");
+}
+
+static __inline void
+__ha_barrier_atomic_full(void)
+{
+ __asm __volatile("" ::: "memory");
+}
+
+static __inline int
+__ha_cas_dw(void *target, void *compare, const void *set)
+{
+ char ret;
+
+ __asm __volatile("lock cmpxchg16b %0; setz %3"
+ : "+m" (*(void **)target),
+ "=a" (((void **)compare)[0]),
+ "=d" (((void **)compare)[1]),
+ "=q" (ret)
+ : "a" (((void **)compare)[0]),
+ "d" (((void **)compare)[1]),
+ "b" (((const void **)set)[0]),
+ "c" (((const void **)set)[1])
+ : "memory", "cc");
+ return (ret);
+}
+
+#elif defined(__arm__) && (defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__))
+
+static __inline void
+__ha_barrier_load(void)
+{
+ __asm __volatile("dmb" ::: "memory");
+}
+
+static __inline void
+__ha_barrier_store(void)
+{
+ __asm __volatile("dsb" ::: "memory");
+}
+
+static __inline void
+__ha_barrier_full(void)
+{
+ __asm __volatile("dmb" ::: "memory");
+}
+
+/* Use __ha_barrier_atomic* when you're trying to protect data that are
+ * are modified using _HA_ATOMIC*
+ */
+static __inline void
+__ha_barrier_atomic_load(void)
+{
+ __asm __volatile("dmb" ::: "memory");
+}
+
+static __inline void
+__ha_barrier_atomic_store(void)
+{
+ __asm __volatile("dsb" ::: "memory");
+}
+
+static __inline void
+__ha_barrier_atomic_full(void)
+{
+ __asm __volatile("dmb" ::: "memory");
+}
+
+static __inline int __ha_cas_dw(void *target, void *compare, const void *set)
+{
+ uint64_t previous;
+ int tmp;
+
+ __asm __volatile("1:"
+ "ldrexd %0, [%4];"
+ "cmp %Q0, %Q2;"
+ "ittt eq;"
+ "cmpeq %R0, %R2;"
+ "strexdeq %1, %3, [%4];"
+ "cmpeq %1, #1;"
+ "beq 1b;"
+ : "=&r" (previous), "=&r" (tmp)
+ : "r" (*(uint64_t *)compare), "r" (*(uint64_t *)set), "r" (target)
+ : "memory", "cc");
+ tmp = (previous == *(uint64_t *)compare);
+ *(uint64_t *)compare = previous;
+ return (tmp);
+}
+
+#elif defined (__aarch64__)
+
+static __inline void
+__ha_barrier_load(void)
+{
+ __asm __volatile("dmb ishld" ::: "memory");
+}
+
+static __inline void
+__ha_barrier_store(void)
+{
+ __asm __volatile("dmb ishst" ::: "memory");
+}
+
+static __inline void
+__ha_barrier_full(void)
+{
+ __asm __volatile("dmb ish" ::: "memory");
+}
+
+/* Use __ha_barrier_atomic* when you're trying to protect data that are
+ * are modified using _HA_ATOMIC*
+ */
+static __inline void
+__ha_barrier_atomic_load(void)
+{
+ __asm __volatile("dmb ishld" ::: "memory");
+}
+
+static __inline void
+__ha_barrier_atomic_store(void)
+{
+ __asm __volatile("dmb ishst" ::: "memory");
+}
+
+static __inline void
+__ha_barrier_atomic_full(void)
+{
+ __asm __volatile("dmb ish" ::: "memory");
+}
+
+static __inline int __ha_cas_dw(void *target, void *compare, void *set)
+{
+ void *value[2];
+ uint64_t tmp1, tmp2;
+
+ __asm__ __volatile__("1:"
+ "ldxp %0, %1, [%4];"
+ "mov %2, %0;"
+ "mov %3, %1;"
+ "eor %0, %0, %5;"
+ "eor %1, %1, %6;"
+ "orr %1, %0, %1;"
+ "mov %w0, #0;"
+ "cbnz %1, 2f;"
+ "stxp %w0, %7, %8, [%4];"
+ "cbnz %w0, 1b;"
+ "mov %w0, #1;"
+ "2:"
+ : "=&r" (tmp1), "=&r" (tmp2), "=&r" (value[0]), "=&r" (value[1])
+ : "r" (target), "r" (((void **)(compare))[0]), "r" (((void **)(compare))[1]), "r" (((void **)(set))[0]), "r" (((void **)(set))[1])
+ : "cc", "memory");
+
+ memcpy(compare, &value, sizeof(value));
+ return (tmp1);
+}
+
+#else /* unknown / unhandled architecture, fall back to generic barriers */
+
+#define __ha_barrier_atomic_load __sync_synchronize
+#define __ha_barrier_atomic_store __sync_synchronize
+#define __ha_barrier_atomic_full __sync_synchronize
+#define __ha_barrier_load __sync_synchronize
+#define __ha_barrier_store __sync_synchronize
+#define __ha_barrier_full __sync_synchronize
+/* Note: there is no generic DWCAS */
+
+#endif /* end of arch-specific barrier/dwcas */
+
+static inline void __ha_compiler_barrier(void)
+{
+ __asm __volatile("" ::: "memory");
+}
+
+#endif /* USE_THREAD */
+
+
+/* fallbacks to remap all undefined _HA_ATOMIC_* on to their safe equivalent */
+#ifndef _HA_ATOMIC_CAS
+#define _HA_ATOMIC_CAS HA_ATOMIC_CAS
+#endif /* !_HA_ATOMIC_CAS */
+
+#ifndef _HA_ATOMIC_DWCAS
+#define _HA_ATOMIC_DWCAS HA_ATOMIC_DWCAS
+#endif /* !_HA_ATOMIC_CAS */
+
+#ifndef _HA_ATOMIC_ADD
+#define _HA_ATOMIC_ADD HA_ATOMIC_ADD
+#endif /* !_HA_ATOMIC_ADD */
+
+#ifndef _HA_ATOMIC_XADD
+#define _HA_ATOMIC_XADD HA_ATOMIC_XADD
+#endif /* !_HA_ATOMIC_SUB */
+
+#ifndef _HA_ATOMIC_SUB
+#define _HA_ATOMIC_SUB HA_ATOMIC_SUB
+#endif /* !_HA_ATOMIC_SUB */
+
+#ifndef _HA_ATOMIC_AND
+#define _HA_ATOMIC_AND HA_ATOMIC_AND
+#endif /* !_HA_ATOMIC_AND */
+
+#ifndef _HA_ATOMIC_OR
+#define _HA_ATOMIC_OR HA_ATOMIC_OR
+#endif /* !_HA_ATOMIC_OR */
+
+#ifndef _HA_ATOMIC_XCHG
+#define _HA_ATOMIC_XCHG HA_ATOMIC_XCHG
+#endif /* !_HA_ATOMIC_XCHG */
+
+#ifndef _HA_ATOMIC_STORE
+#define _HA_ATOMIC_STORE HA_ATOMIC_STORE
+#endif /* !_HA_ATOMIC_STORE */
+
+#ifndef _HA_ATOMIC_LOAD
+#define _HA_ATOMIC_LOAD HA_ATOMIC_LOAD
+#endif /* !_HA_ATOMIC_LOAD */
+
+#endif /* _HAPROXY_ATOMIC_H */
projects / thirdparty / haproxy.git / commitdiff
