From 64b1fb5f03266f0ef4eef3ad2b7d97170bb05b78 Mon Sep 17 00:00:00 2001
From: Nathan Bossart <nathan@postgresql.org>
Date: Mon, 18 Dec 2023 10:53:32 -0600
Subject: [PATCH] Optimize pg_atomic_exchange_u32 and pg_atomic_exchange_u64.

Presently, all platforms implement atomic exchanges by performing
an atomic compare-and-swap in a loop until it succeeds.  This can
be especially expensive when there is contention on the atomic
variable.  This commit optimizes atomic exchanges on many platforms
by using compiler intrinsics, which should compile into something
much less expensive than a compare-and-swap loop.  Since these
intrinsics have been available for some time, the inline assembly
implementations are omitted.

Suggested-by: Andres Freund
Reviewed-by: Andres Freund
Discussion: https://postgr.es/m/20231129212905.GA1258737%40nathanxps13
---
 src/include/port/atomics/generic-gcc.h    | 34 +++++++++++++++++++++++
 src/include/port/atomics/generic-msvc.h   | 18 ++++++++++++
 src/include/port/atomics/generic-sunpro.h | 14 ++++++++++
 3 files changed, 66 insertions(+)

diff --git a/src/include/port/atomics/generic-gcc.h b/src/include/port/atomics/generic-gcc.h
index da04e9f0dc3..de0e34cb60a 100644
--- a/src/include/port/atomics/generic-gcc.h
+++ b/src/include/port/atomics/generic-gcc.h
@@ -176,6 +176,23 @@ pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
 }
 #endif
 
+/*
+ * __sync_lock_test_and_set() only supports setting the value to 1 on some
+ * platforms, so we only provide an __atomic implementation for
+ * pg_atomic_exchange.
+ *
+ * We assume the availability of 32-bit __atomic_compare_exchange_n() implies
+ * the availability of 32-bit __atomic_exchange_n().
+ */
+#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS)
+#define PG_HAVE_ATOMIC_EXCHANGE_U32
+static inline uint32
+pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval)
+{
+	return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST);
+}
+#endif
+
 /* if we have 32-bit __sync_val_compare_and_swap, assume we have these too: */
 
 #if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
@@ -243,6 +260,23 @@ pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
 }
 #endif
 
+/*
+ * __sync_lock_test_and_set() only supports setting the value to 1 on some
+ * platforms, so we only provide an __atomic implementation for
+ * pg_atomic_exchange.
+ *
+ * We assume the availability of 64-bit __atomic_compare_exchange_n() implies
+ * the availability of 64-bit __atomic_exchange_n().
+ */
+#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS)
+#define PG_HAVE_ATOMIC_EXCHANGE_U64
+static inline uint64
+pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval)
+{
+	return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST);
+}
+#endif
+
 /* if we have 64-bit __sync_val_compare_and_swap, assume we have these too: */
 
 #if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
diff --git a/src/include/port/atomics/generic-msvc.h b/src/include/port/atomics/generic-msvc.h
index 8835f4ceea8..7566d6f9377 100644
--- a/src/include/port/atomics/generic-msvc.h
+++ b/src/include/port/atomics/generic-msvc.h
@@ -58,6 +58,13 @@ pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
 	return ret;
 }
 
+#define PG_HAVE_ATOMIC_EXCHANGE_U32
+static inline uint32
+pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval)
+{
+	return InterlockedExchange(&ptr->value, newval);
+}
+
 #define PG_HAVE_ATOMIC_FETCH_ADD_U32
 static inline uint32
 pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
@@ -88,6 +95,16 @@ pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
 
 /* Only implemented on 64bit builds */
 #ifdef _WIN64
+
+#pragma intrinsic(_InterlockedExchange64)
+
+#define PG_HAVE_ATOMIC_EXCHANGE_U64
+static inline uint64
+pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval)
+{
+	return _InterlockedExchange64(&ptr->value, newval);
+}
+
 #pragma intrinsic(_InterlockedExchangeAdd64)
 
 #define PG_HAVE_ATOMIC_FETCH_ADD_U64
@@ -96,6 +113,7 @@ pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
 {
 	return _InterlockedExchangeAdd64(&ptr->value, add_);
 }
+
 #endif /* _WIN64 */
 
 #endif /* HAVE_ATOMICS */
diff --git a/src/include/port/atomics/generic-sunpro.h b/src/include/port/atomics/generic-sunpro.h
index 30f7d8b5362..5d41b73de06 100644
--- a/src/include/port/atomics/generic-sunpro.h
+++ b/src/include/port/atomics/generic-sunpro.h
@@ -87,6 +87,13 @@ pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
 	return ret;
 }
 
+#define PG_HAVE_ATOMIC_EXCHANGE_U32
+static inline uint32
+pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval)
+{
+	return atomic_swap_32(&ptr->value, newval);
+}
+
 #define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64
 static inline bool
 pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
@@ -101,6 +108,13 @@ pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
 	return ret;
 }
 
+#define PG_HAVE_ATOMIC_EXCHANGE_U64
+static inline uint64
+pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval)
+{
+	return atomic_swap_64(&ptr->value, newval);
+}
+
 #endif /* HAVE_ATOMIC_H */
 
 #endif /* defined(HAVE_ATOMICS) */
-- 
2.39.5