Bucketed statistics.

Even though statistics are lockless they still use atomics which
might cause contention. Split stats counters into buckets, sharded
by an artificial thread identifier, to increase throughput.

diff --git a/lib/isc/stats.c b/lib/isc/stats.c
index 7a4f38c9b89f8e35962ca41a9163d9874f41c35c..d44b8895f97c384b417242dbcd6b3fe1a31ccb0b 100644 (file)
--- a/lib/isc/stats.c
+++ b/lib/isc/stats.c
@@ -23,11 +23,14 @@
 #include <isc/print.h>
 #include <isc/refcount.h>
 #include <isc/stats.h>
+#include <isc/thread.h>
 #include <isc/util.h>
 
 #define ISC_STATS_MAGIC                        ISC_MAGIC('S', 't', 'a', 't')
 #define ISC_STATS_VALID(x)             ISC_MAGIC_VALID(x, ISC_STATS_MAGIC)
 
+#define STATS_BUCKETS 64
+
 #if defined(_WIN32) && !defined(_WIN64)
        typedef atomic_int_fast32_t isc__atomic_statcounter_t;
 #else
@@ -42,6 +45,17 @@ struct isc_stats {
        isc__atomic_statcounter_t       *counters;
 };
 
+ISC_THREAD_LOCAL int isc__stats_thread_v = -1;
+static atomic_uint_fast32_t isc__stats_thread_n = 0;
+
+static int
+threadhash() {
+       if (isc__stats_thread_v < 0) {
+               isc__stats_thread_v = atomic_fetch_add_relaxed(&isc__stats_thread_n, 1) % STATS_BUCKETS;
+       }
+       return (isc__stats_thread_v);
+}
+
 static isc_result_t
 create_stats(isc_mem_t *mctx, int ncounters, isc_stats_t **statsp) {
        isc_stats_t *stats;
@@ -50,7 +64,7 @@ create_stats(isc_mem_t *mctx, int ncounters, isc_stats_t **statsp) {
        REQUIRE(statsp != NULL && *statsp == NULL);
 
        stats = isc_mem_get(mctx, sizeof(*stats));
-       counters_alloc_size = sizeof(isc__atomic_statcounter_t) * ncounters;
+       counters_alloc_size = sizeof(isc__atomic_statcounter_t) * ncounters * STATS_BUCKETS;
        stats->counters = isc_mem_get(mctx, counters_alloc_size);
        isc_refcount_init(&stats->references, 1);
        memset(stats->counters, 0, counters_alloc_size);
@@ -85,7 +99,8 @@ isc_stats_detach(isc_stats_t **statsp) {
                isc_refcount_destroy(&stats->references);
                isc_mem_put(stats->mctx, stats->counters,
                            sizeof(isc__atomic_statcounter_t) *
-                               stats->ncounters);
+                               stats->ncounters *
+                               STATS_BUCKETS);
                isc_mem_putanddetach(&stats->mctx, stats, sizeof(*stats));
        }
 }
@@ -108,8 +123,8 @@ void
 isc_stats_increment(isc_stats_t *stats, isc_statscounter_t counter) {
        REQUIRE(ISC_STATS_VALID(stats));
        REQUIRE(counter < stats->ncounters);
-
-       atomic_fetch_add_explicit(&stats->counters[counter], 1,
+       int idx = threadhash() * stats->ncounters + counter;
+       atomic_fetch_add_explicit(&stats->counters[idx], 1,
                                  memory_order_relaxed);
 }
 
@@ -118,7 +133,8 @@ isc_stats_decrement(isc_stats_t *stats, isc_statscounter_t counter) {
        REQUIRE(ISC_STATS_VALID(stats));
        REQUIRE(counter < stats->ncounters);
 
-       atomic_fetch_sub_explicit(&stats->counters[counter], 1,
+       int idx = threadhash() * stats->ncounters + counter;
+       atomic_fetch_sub_explicit(&stats->counters[idx], 1,
                                  memory_order_relaxed);
 }
 
@@ -131,8 +147,13 @@ isc_stats_dump(isc_stats_t *stats, isc_stats_dumper_t dump_fn,
        REQUIRE(ISC_STATS_VALID(stats));
 
        for (i = 0; i < stats->ncounters; i++) {
-               uint32_t counter = atomic_load_explicit(&stats->counters[i],
+               uint32_t counter = 0;
+               int b;
+               for (b = 0; b < STATS_BUCKETS; b++) {
+                       int idx = stats->ncounters * b + i;
+                       counter += atomic_load_explicit(&stats->counters[idx],
                                                        memory_order_relaxed);
+               }
                if ((options & ISC_STATSDUMP_VERBOSE) == 0 && counter == 0) {
                        continue;
                }
@@ -144,11 +165,17 @@ void
 isc_stats_set(isc_stats_t *stats, uint64_t val,
              isc_statscounter_t counter)
 {
+       int i;
+
        REQUIRE(ISC_STATS_VALID(stats));
        REQUIRE(counter < stats->ncounters);
-
        atomic_store_explicit(&stats->counters[counter], val,
                              memory_order_relaxed);
+       for (i = 1; i < STATS_BUCKETS; i++) {
+               int idx = stats->ncounters * i + counter;
+               atomic_store_explicit(&stats->counters[idx], val,
+                                     memory_order_relaxed);
+       }
 }
 
 void isc_stats_update_if_greater(isc_stats_t *stats,
@@ -158,9 +185,9 @@ void isc_stats_update_if_greater(isc_stats_t *stats,
        REQUIRE(ISC_STATS_VALID(stats));
        REQUIRE(counter < stats->ncounters);
 
-       isc_statscounter_t curr_value =
-               atomic_load_relaxed(&stats->counters[counter]);
+       isc_statscounter_t curr_value;
        do {
+               curr_value = atomic_load_relaxed(&stats->counters[counter]);
                if (curr_value >= value) {
                        break;
                }
@@ -172,9 +199,15 @@ void isc_stats_update_if_greater(isc_stats_t *stats,
 isc_statscounter_t
 isc_stats_get_counter(isc_stats_t *stats, isc_statscounter_t counter)
 {
+       uint32_t value = 0;
+       int i;
        REQUIRE(ISC_STATS_VALID(stats));
        REQUIRE(counter < stats->ncounters);
 
-       return (atomic_load_explicit(&stats->counters[counter],
-                                   memory_order_relaxed));
+       for (i = 0; i < STATS_BUCKETS; i++) {
+               int idx = i * stats->ncounters + counter;
+               value += atomic_load_explicit(&stats->counters[idx],
+                                             memory_order_relaxed);
+       }
+       return (value);
 }