Improve the spinloop pause / yield hint

Unfortunately, C still lacks a standard function for pause (x86,
sparc) or yeild (arm) instructions, for use in spin lock or CAS loops.
BIND has its own based on vendor intrinsics or inline asm.

Previously, it was buried in the `isc_rwlock` implementation. This
commit renames `isc_rwlock_pause()` to `isc_pause()` and moves
it into <isc/pause.h>.

This commit also fixes the configure script so that it detects ARM
yield support on systems that identify as `aarch*` instead of `arm*`.

On 64-bit ARM systems we now use the ISB (instruction synchronization
barrier) instruction in preference to yield. The ISB instruction
pauses the CPU for longer, several nanoseconds, which is more like the
x86 pause instruction. There are more details in a Rust pull request,
which also refers to MySQL making the same change:
https://github.com/rust-lang/rust/pull/84725

diff --git a/CHANGES b/CHANGES
index 6a9eee18f5a61a943e442d781e5245256a521e75..26cdfad32467c6e441a8bd9a36fafab2f74382ee 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,6 @@
+6097.  [port]          Improve support for yield / pause instructions in spin
+                       loops on AArch64 platforms. [GL !7469]
+
 6096.  [bug]           Fix RPZ reference counting error on shutdown in
                        dns__rpz_timer_cb(). [GL #3866]
 

diff --git a/configure.ac b/configure.ac
index c6106cd5cd9e1c19b14dd6f6724c2c8e90ba4236..4aa1761b26df6ddd8183f2bd169b0fa4b4e52f12 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -359,8 +359,8 @@ AC_C_VOLATILE
 # Check for yield support on ARM processors
 #
 AS_CASE([$host],
-       [arm*],
-       [AC_MSG_CHECKING([for yield instruction support])
+       [arm*|aarch64*],
+       [AC_MSG_CHECKING([for ARM yield instruction support])
         AC_COMPILE_IFELSE(
             [AC_LANG_PROGRAM([[]],
                             [[__asm__ __volatile__ ("yield")]])],

diff --git a/lib/isc/Makefile.am b/lib/isc/Makefile.am
index 8201e71d6a4f4dba9ebe849d9b0ffa3a29a2a9ed..b532587cf2d439ae7c6b3841c3f7a390a7ccfe30 100644 (file)
--- a/lib/isc/Makefile.am
+++ b/lib/isc/Makefile.am
@@ -67,6 +67,7 @@ libisc_la_HEADERS =                   \
        include/isc/once.h              \
        include/isc/os.h                \
        include/isc/parseint.h          \
+       include/isc/pause.h             \
        include/isc/portset.h           \
        include/isc/print.h             \
        include/isc/quota.h             \

diff --git a/lib/isc/include/isc/pause.h b/lib/isc/include/isc/pause.h
new file mode 100644 (file)
index 0000000..8d7fa57
--- /dev/null
+++ b/lib/isc/include/isc/pause.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+#if defined(__x86_64__)
+#include <immintrin.h>
+#define isc_pause() _mm_pause()
+#elif defined(__i386__)
+#define isc_pause() __asm__ __volatile__("rep; nop")
+#elif defined(__ia64__)
+#define isc_pause() __asm__ __volatile__("hint @pause")
+#elif defined(__aarch64__)
+#define isc_pause() __asm__ __volatile__("isb")
+#elif defined(__arm__) && HAVE_ARM_YIELD
+#define isc_pause() __asm__ __volatile__("yield")
+#elif defined(sun) && (defined(__sparc) || defined(__sparc__))
+#include <synch.h>
+#define isc_pause() smt_pause()
+#elif (defined(__sparc) || defined(__sparc__)) && HAVE_SPARC_PAUSE
+#define isc_pause() __asm__ __volatile__("pause")
+#elif defined(__ppc__) || defined(_ARCH_PPC) || defined(_ARCH_PWR) || \
+       defined(_ARCH_PWR2) || defined(_POWER)
+#define isc_pause() __asm__ volatile("or 27,27,27")
+#else
+#define isc_pause() sched_yield()
+#endif
+
+#define isc_pause_n(iters)                                     \
+       for (size_t __pause = 0; __pause < iters; __pause++) { \
+               isc_pause();                                   \
+       }

diff --git a/lib/isc/rwlock.c b/lib/isc/rwlock.c
index d00ec7655fbcc04a50c0e0bb16c65f1e392b808a..62bdbf1a625fc1543866c14d1dcb99c4391336c1 100644 (file)
--- a/lib/isc/rwlock.c
+++ b/lib/isc/rwlock.c
@@ -17,12 +17,9 @@
 #include <stdbool.h>
 #include <stddef.h>
 
-#if defined(sun) && (defined(__sparc) || defined(__sparc__))
-#include <synch.h> /* for smt_pause(3c) */
-#endif /* if defined(sun) && (defined(__sparc) || defined(__sparc__)) */
-
 #include <isc/atomic.h>
 #include <isc/magic.h>
+#include <isc/pause.h>
 #include <isc/print.h>
 #include <isc/rwlock.h>
 #include <isc/util.h>
@@ -42,29 +39,6 @@
 #define RWLOCK_MAX_ADAPTIVE_COUNT 100
 #endif /* ifndef RWLOCK_MAX_ADAPTIVE_COUNT */
 
-#if defined(_MSC_VER)
-#include <intrin.h>
-#define isc_rwlock_pause() YieldProcessor()
-#elif defined(__x86_64__)
-#include <immintrin.h>
-#define isc_rwlock_pause() _mm_pause()
-#elif defined(__i386__)
-#define isc_rwlock_pause() __asm__ __volatile__("rep; nop")
-#elif defined(__ia64__)
-#define isc_rwlock_pause() __asm__ __volatile__("hint @pause")
-#elif defined(__arm__) && HAVE_ARM_YIELD
-#define isc_rwlock_pause() __asm__ __volatile__("yield")
-#elif defined(sun) && (defined(__sparc) || defined(__sparc__))
-#define isc_rwlock_pause() smt_pause()
-#elif (defined(__sparc) || defined(__sparc__)) && HAVE_SPARC_PAUSE
-#define isc_rwlock_pause() __asm__ __volatile__("pause")
-#elif defined(__ppc__) || defined(_ARCH_PPC) || defined(_ARCH_PWR) || \
-       defined(_ARCH_PWR2) || defined(_POWER)
-#define isc_rwlock_pause() __asm__ volatile("or 27,27,27")
-#else /* if defined(_MSC_VER) */
-#define isc_rwlock_pause()
-#endif /* if defined(_MSC_VER) */
-
 #ifdef ISC_RWLOCK_TRACE
 #include <stdio.h> /* Required for fprintf/stderr. */
 
@@ -331,7 +305,7 @@ isc__rwlock_lock(isc__rwlock_t *rwl, isc_rwlocktype_t type) {
                        rwlock_lock(rwl, type);
                        break;
                }
-               isc_rwlock_pause();
+               isc_pause();
        } while (isc_rwlock_trylock(rwl, type) != ISC_R_SUCCESS);
 
        atomic_fetch_add_release(&rwl->spins, (cnt - spins) / 8);