[master] adaptive RW locks

3440.	[performance]	Implement adaptive read-write locks, reducing the
			overhead of locks that are only held briefly.
			[RT #37329]

diff --git a/CHANGES b/CHANGES
index 81a037d3cc7c0b3e8fa22ca680987d3cadcdee47..0325dce907fea6729039597d15d2d4e0040a4027 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,7 @@
+3440.  [performance]   Implement adaptive read-write locks, reducing the
+                       overhead of locks that are only held briefly.
+                       [RT #37329]
+
 4339.  [test]          Use "mdig" to test pipelined queries. [RT #41929]
 
 4338.  [bug]           Reimplement change 4324 as it wasn't properly doing

diff --git a/configure b/configure
index 5483289f0fae7bdf8fd1722adea2b06f9b7c1d31..7ce158192d45613a3ed6c077ba404a9b1bb8210f 100755 (executable)
--- a/configure
+++ b/configure
@@ -712,6 +712,7 @@ DNSTAPOBJS
 DNSTAPSRCS
 DNSTAP
 PROTOC_C
+ISC_PLATFORM_BUSYWAITNOP
 ISC_ARCH_DIR
 ISC_PLATFORM_USEMACASM
 ISC_PLATFORM_USESTDASM
@@ -19964,6 +19965,115 @@ $as_echo "#define HAVE_BUILTIN_EXPECT 1" >>confdefs.h
 
 fi
 
+#
+# CPU relax (for spin locks)
+#
+if $use_threads
+then
+       case "$host" in
+       i[3456]86-*)
+               # x86_32
+               { $as_echo "$as_me:${as_lineno-$LINENO}: checking if asm(\"rep; nop\"); works" >&5
+$as_echo_n "checking if asm(\"rep; nop\"); works... " >&6; }
+               cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main ()
+{
+asm("rep; nop");
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+                ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"rep; nop\")"
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+               ;;
+       x86_64-*|amd64-*)
+               # x86_64
+               { $as_echo "$as_me:${as_lineno-$LINENO}: checking if asm(\"rep; nop\"); works" >&5
+$as_echo_n "checking if asm(\"rep; nop\"); works... " >&6; }
+               cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main ()
+{
+asm("rep; nop");
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+                ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"rep; nop\")"
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+               ;;
+       ia64-*)
+               # ia64
+               { $as_echo "$as_me:${as_lineno-$LINENO}: checking if asm(\"hint @pause\"); works" >&5
+$as_echo_n "checking if asm(\"hint @pause\"); works... " >&6; }
+               cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main ()
+{
+asm("hint @pause");
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+                ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"hint @pause\")"
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+               ;;
+       sparc-*)
+               # sparc
+               { $as_echo "$as_me:${as_lineno-$LINENO}: checking if cpu_relax(); or __cpu_relax(); works" >&5
+$as_echo_n "checking if cpu_relax(); or __cpu_relax(); works... " >&6; }
+               ac_fn_c_check_func "$LINENO" "cpu_relax" "ac_cv_func_cpu_relax"
+if test "x$ac_cv_func_cpu_relax" = xyes; then :
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+                ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP cpu_relax()"
+else
+  ac_fn_c_check_func "$LINENO" "__cpu_relax" "ac_cv_func___cpu_relax"
+if test "x$ac_cv_func___cpu_relax" = xyes; then :
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+                  ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP __cpu_relax()"
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+fi
+
+               ;;
+       esac
+fi
+
+
+
 #
 # Activate "rrset-order fixed" or not?
 #

diff --git a/configure.in b/configure.in
index 506a49e58a8d6043d8b3f2e16b291566332053b1..e6a8b3e8645408bab2588a4946c8f9ddb32d0a53 100644 (file)
--- a/configure.in
+++ b/configure.in
@@ -3910,6 +3910,58 @@ if test "$have_builtin_expect" = "yes"; then
         AC_DEFINE(HAVE_BUILTIN_EXPECT, 1, [Define to 1 if the compiler supports __builtin_expect.])
 fi
 
+#
+# CPU relax (for spin locks)
+#
+if $use_threads
+then
+       case "$host" in
+       [i[3456]86-*])
+               # x86_32
+               AC_MSG_CHECKING([if asm("rep; nop"); works])
+               AC_TRY_COMPILE(,[asm("rep; nop");],
+               [AC_MSG_RESULT(yes)
+                ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"rep; nop\")"],
+               [AC_MSG_RESULT(no)],
+               [AC_MSG_RESULT([cross compile, assume yes])
+                ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"rep; nop\")"])
+               ;;
+       x86_64-*|amd64-*)
+               # x86_64
+               AC_MSG_CHECKING([if asm("rep; nop"); works])
+               AC_TRY_COMPILE(,[asm("rep; nop");],
+               [AC_MSG_RESULT(yes)
+                ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"rep; nop\")"],
+               [AC_MSG_RESULT(no)],
+               [AC_MSG_RESULT([cross compile, assume yes])
+                ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"rep; nop\")"])
+               ;;
+       ia64-*)
+               # ia64
+               AC_MSG_CHECKING([if asm("hint @pause"); works])
+               AC_TRY_COMPILE(,[asm("hint @pause");],
+               [AC_MSG_RESULT(yes)
+                ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"hint @pause\")"],
+               [AC_MSG_RESULT(no)],
+               [AC_MSG_RESULT([cross compile, assume yes])
+                ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP asm(\"hint @pause\")"])
+               ;;
+       sparc-*)
+               # sparc
+               AC_MSG_CHECKING([if cpu_relax(); or __cpu_relax(); works])
+               AC_CHECK_FUNC(cpu_relax,
+               [AC_MSG_RESULT(yes)
+                ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP cpu_relax()"],
+                [AC_CHECK_FUNC(__cpu_relax,
+                 [AC_MSG_RESULT(yes)
+                  ISC_PLATFORM_BUSYWAITNOP="#define ISC_PLATFORM_BUSYWAITNOP __cpu_relax()"],
+                  [AC_MSG_RESULT(no)])])
+               ;;
+       esac
+fi
+
+AC_SUBST(ISC_PLATFORM_BUSYWAITNOP)
+
 #
 # Activate "rrset-order fixed" or not?
 #

diff --git a/lib/isc/include/isc/platform.h.in b/lib/isc/include/isc/platform.h.in
index 75cdec33a34f40f4696e4ee3576220081badbd49..2b451409e65f0341850cd92efa089f686feb39d6 100644 (file)
--- a/lib/isc/include/isc/platform.h.in
+++ b/lib/isc/include/isc/platform.h.in
@@ -320,6 +320,11 @@
  */
 @ISC_PLATFORM_USESTDASM@
 
+/*
+ * Define with the busy wait nop asm or function call.
+ */
+@ISC_PLATFORM_BUSYWAITNOP@
+
 /*
  * Define if the platform has <strings.h>.
  */

diff --git a/lib/isc/include/isc/rwlock.h b/lib/isc/include/isc/rwlock.h
index 28052cdd7f8b4b7a114bbcfd0d965f000cfe97c3..9186c8afc610a797086010df556e3ffdda7bf391 100644 (file)
--- a/lib/isc/include/isc/rwlock.h
+++ b/lib/isc/include/isc/rwlock.h
@@ -44,6 +44,7 @@ struct isc_rwlock {
        /* Unlocked. */
        unsigned int            magic;
        isc_mutex_t             lock;
+       isc_int32_t             spins;
 
 #if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG)
        /*

diff --git a/lib/isc/rwlock.c b/lib/isc/rwlock.c
index 461d6e295a779a697e5c584ae685cf9977f62062..6e290088654fa97e3b738588721a5bd58ac1aa2e 100644 (file)
--- a/lib/isc/rwlock.c
+++ b/lib/isc/rwlock.c
@@ -44,6 +44,15 @@
 #define RWLOCK_DEFAULT_WRITE_QUOTA 4
 #endif
 
+#ifndef RWLOCK_MAX_ADAPTIVE_COUNT
+#define RWLOCK_MAX_ADAPTIVE_COUNT 100
+#endif
+
+#if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG)
+static isc_result_t
+isc__rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type);
+#endif
+
 #ifdef ISC_RWLOCK_TRACE
 #include <stdio.h>             /* Required for fprintf/stderr. */
 #include <isc/thread.h>                /* Required for isc_thread_self(). */
@@ -85,6 +94,7 @@ isc_rwlock_init(isc_rwlock_t *rwl, unsigned int read_quota,
         */
        rwl->magic = 0;
 
+       rwl->spins = 0;
 #if defined(ISC_PLATFORM_HAVEXADD) && defined(ISC_PLATFORM_HAVECMPXCHG)
        rwl->write_requests = 0;
        rwl->write_completions = 0;
@@ -238,8 +248,8 @@ isc_rwlock_destroy(isc_rwlock_t *rwl) {
 #define WRITER_ACTIVE  0x1
 #define READER_INCR    0x2
 
-isc_result_t
-isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
+static isc_result_t
+isc__rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
        isc_int32_t cntflag;
 
        REQUIRE(VALID_RWLOCK(rwl));
@@ -348,6 +358,30 @@ isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
        return (ISC_R_SUCCESS);
 }
 
+isc_result_t
+isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
+       isc_int32_t cnt = 0;
+       isc_int32_t max_cnt = rwl->spins * 2 + 10;
+       isc_result_t result = ISC_R_SUCCESS;
+
+       if (max_cnt > RWLOCK_MAX_ADAPTIVE_COUNT)
+               max_cnt = RWLOCK_MAX_ADAPTIVE_COUNT;
+
+       do {
+               if (cnt++ >= max_cnt) {
+                       result = isc__rwlock_lock(rwl, type);
+                       break;
+               }
+#ifdef ISC_PLATFORM_BUSYWAITNOP
+               ISC_PLATFORM_BUSYWAITNOP;
+#endif
+       } while (isc_rwlock_trylock(rwl, type) != ISC_R_SUCCESS);
+
+       rwl->spins += (cnt - rwl->spins) / 8;
+
+       return (result);
+}
+
 isc_result_t
 isc_rwlock_trylock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
        isc_int32_t cntflag;
@@ -606,7 +640,26 @@ doit(isc_rwlock_t *rwl, isc_rwlocktype_t type, isc_boolean_t nonblock) {
 
 isc_result_t
 isc_rwlock_lock(isc_rwlock_t *rwl, isc_rwlocktype_t type) {
-       return (doit(rwl, type, ISC_FALSE));
+       isc_int32_t cnt = 0;
+       isc_int32_t max_cnt = rwl->spins * 2 + 10;
+       isc_result_t result = ISC_R_SUCCESS;
+
+       if (max_cnt > RWLOCK_MAX_ADAPTIVE_COUNT)
+               max_cnt = RWLOCK_MAX_ADAPTIVE_COUNT;
+
+       do {
+               if (cnt++ >= max_cnt) {
+                       result = doit(rwl, type, ISC_FALSE);
+                       break;
+               }
+#ifdef ISC_PLATFORM_BUSYWAITNOP
+               ISC_PLATFORM_BUSYWAITNOP;
+#endif
+       } while (doit(rwl, type, ISC_TRUE) != ISC_R_SUCCESS);
+
+       rwl->spins += (cnt - rwl->spins) / 8;
+
+       return (ISC_R_SUCCESS);
 }
 
 isc_result_t

diff --git a/lib/isc/win32/include/isc/platform.h.in b/lib/isc/win32/include/isc/platform.h.in
index bbd35aca93f053336a2f35c1992f18de88eeac01..12a2b4f78c081e8ef30e832386f285e838656f98 100644 (file)
--- a/lib/isc/win32/include/isc/platform.h.in
+++ b/lib/isc/win32/include/isc/platform.h.in
@@ -109,6 +109,11 @@
  */
 @ISC_PLATFORM_HAVECMPXCHG@
 
+/*
+ * Define with the busy wait nop asm or function call.
+ */
+@ISC_PLATFORM_BUSYWAITNOP@
+
 /*
  * If the strcasestr() operation is not available on this platform,
  * ISC_PLATFORM_NEEDSTRCASESTR will be defined.

diff --git a/win32utils/Configure b/win32utils/Configure
index 28899ef20f510ca76542372d7a0013c0b8f4ba4c..79ae31e31e9bb877a9cfd5f95ee60d82d2b270a4 100644 (file)
--- a/win32utils/Configure
+++ b/win32utils/Configure
@@ -383,7 +383,8 @@ my @substdefh = ("AES_CC",
 
 my %configdefp;
 
-my @substdefp = ("ISC_PLATFORM_HAVEATOMICSTORE",
+my @substdefp = ("ISC_PLATFORM_BUSYWAITNOP",
+                 "ISC_PLATFORM_HAVEATOMICSTORE",
                  "ISC_PLATFORM_HAVEATOMICSTOREQ",
                  "ISC_PLATFORM_HAVECMPXCHG",
                  "ISC_PLATFORM_HAVEXADD",
@@ -693,11 +694,13 @@ if (($want_win32 eq "yes") && ($want_x64 eq "yes")) {
     $configvar{"BUILD_PLATFORM"} = "Win32";
     $configvar{"MACHINE"} = "/machine:X86";
     $configvar{"BUILD_MACHINE"} = "/machine:X86";
+    $configdefp{"ISC_PLATFORM_BUSYWAITNOP"} = "__asm { rep nop }";
 } elsif ($want_x64 eq "yes") {
     $configvar{"PLATFORM"} = "x64";
     $configvar{"BUILD_PLATFORM"} = "x64";
     $configvar{"MACHINE"} = "/machine:X64";
     $configvar{"BUILD_MACHINE"} = "/machine:X64";
+    $configdefp{"ISC_PLATFORM_BUSYWAITNOP"} = "__asm { rep nop }";
 }
 
 # get the version information