From: Witold Kręcicki <wpk@isc.org>
Date: Thu, 30 Jan 2020 21:34:09 +0000 (+0100)
Subject: Bucketed statistics.
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=fe24b48e9f3d6209a9a939f6e5bd99b04a898dfd;p=thirdparty%2Fbind9.git

Bucketed statistics.

Even though statistics are lockless they still use atomics which
might cause contention. Split stats counters into buckets, sharded
by an artificial thread identifier, to increase throughput.
---

diff --git a/lib/isc/stats.c b/lib/isc/stats.c
index 7a4f38c9b89..d44b8895f97 100644
--- a/lib/isc/stats.c
+++ b/lib/isc/stats.c
@@ -23,11 +23,14 @@
 #include <isc/print.h>
 #include <isc/refcount.h>
 #include <isc/stats.h>
+#include <isc/thread.h>
 #include <isc/util.h>
 
 #define ISC_STATS_MAGIC			ISC_MAGIC('S', 't', 'a', 't')
 #define ISC_STATS_VALID(x)		ISC_MAGIC_VALID(x, ISC_STATS_MAGIC)
 
+#define STATS_BUCKETS 64
+
 #if defined(_WIN32) && !defined(_WIN64)
 	typedef atomic_int_fast32_t isc__atomic_statcounter_t;
 #else
@@ -42,6 +45,17 @@ struct isc_stats {
 	isc__atomic_statcounter_t	*counters;
 };
 
+ISC_THREAD_LOCAL int isc__stats_thread_v = -1;
+static atomic_uint_fast32_t isc__stats_thread_n = 0;
+
+static int
+threadhash() {
+	if (isc__stats_thread_v < 0) {
+		isc__stats_thread_v = atomic_fetch_add_relaxed(&isc__stats_thread_n, 1) % STATS_BUCKETS;
+	}
+	return (isc__stats_thread_v);
+}
+
 static isc_result_t
 create_stats(isc_mem_t *mctx, int ncounters, isc_stats_t **statsp) {
 	isc_stats_t *stats;
@@ -50,7 +64,7 @@ create_stats(isc_mem_t *mctx, int ncounters, isc_stats_t **statsp) {
 	REQUIRE(statsp != NULL && *statsp == NULL);
 
 	stats = isc_mem_get(mctx, sizeof(*stats));
-	counters_alloc_size = sizeof(isc__atomic_statcounter_t) * ncounters;
+	counters_alloc_size = sizeof(isc__atomic_statcounter_t) * ncounters * STATS_BUCKETS;
 	stats->counters = isc_mem_get(mctx, counters_alloc_size);
 	isc_refcount_init(&stats->references, 1);
 	memset(stats->counters, 0, counters_alloc_size);
@@ -85,7 +99,8 @@ isc_stats_detach(isc_stats_t **statsp) {
 		isc_refcount_destroy(&stats->references);
 		isc_mem_put(stats->mctx, stats->counters,
 			    sizeof(isc__atomic_statcounter_t) *
-				stats->ncounters);
+				stats->ncounters *
+				STATS_BUCKETS);
 		isc_mem_putanddetach(&stats->mctx, stats, sizeof(*stats));
 	}
 }
@@ -108,8 +123,8 @@ void
 isc_stats_increment(isc_stats_t *stats, isc_statscounter_t counter) {
 	REQUIRE(ISC_STATS_VALID(stats));
 	REQUIRE(counter < stats->ncounters);
-
-	atomic_fetch_add_explicit(&stats->counters[counter], 1,
+	int idx = threadhash() * stats->ncounters + counter;
+	atomic_fetch_add_explicit(&stats->counters[idx], 1,
 				  memory_order_relaxed);
 }
 
@@ -118,7 +133,8 @@ isc_stats_decrement(isc_stats_t *stats, isc_statscounter_t counter) {
 	REQUIRE(ISC_STATS_VALID(stats));
 	REQUIRE(counter < stats->ncounters);
 
-	atomic_fetch_sub_explicit(&stats->counters[counter], 1,
+	int idx = threadhash() * stats->ncounters + counter;
+	atomic_fetch_sub_explicit(&stats->counters[idx], 1,
 				  memory_order_relaxed);
 }
 
@@ -131,8 +147,13 @@ isc_stats_dump(isc_stats_t *stats, isc_stats_dumper_t dump_fn,
 	REQUIRE(ISC_STATS_VALID(stats));
 
 	for (i = 0; i < stats->ncounters; i++) {
-		uint32_t counter = atomic_load_explicit(&stats->counters[i],
+		uint32_t counter = 0;
+		int b;
+		for (b = 0; b < STATS_BUCKETS; b++) {
+			int idx = stats->ncounters * b + i;
+			counter += atomic_load_explicit(&stats->counters[idx],
 							memory_order_relaxed);
+		}
 		if ((options & ISC_STATSDUMP_VERBOSE) == 0 && counter == 0) {
 			continue;
 		}
@@ -144,11 +165,17 @@ void
 isc_stats_set(isc_stats_t *stats, uint64_t val,
 	      isc_statscounter_t counter)
 {
+	int i;
+
 	REQUIRE(ISC_STATS_VALID(stats));
 	REQUIRE(counter < stats->ncounters);
-
 	atomic_store_explicit(&stats->counters[counter], val,
 			      memory_order_relaxed);
+	for (i = 1; i < STATS_BUCKETS; i++) {
+		int idx = stats->ncounters * i + counter;
+		atomic_store_explicit(&stats->counters[idx], val,
+				      memory_order_relaxed);
+	}
 }
 
 void isc_stats_update_if_greater(isc_stats_t *stats,
@@ -158,9 +185,9 @@ void isc_stats_update_if_greater(isc_stats_t *stats,
 	REQUIRE(ISC_STATS_VALID(stats));
 	REQUIRE(counter < stats->ncounters);
 
-	isc_statscounter_t curr_value =
-		atomic_load_relaxed(&stats->counters[counter]);
+	isc_statscounter_t curr_value;
 	do {
+		curr_value = atomic_load_relaxed(&stats->counters[counter]);
 		if (curr_value >= value) {
 			break;
 		}
@@ -172,9 +199,15 @@ void isc_stats_update_if_greater(isc_stats_t *stats,
 isc_statscounter_t
 isc_stats_get_counter(isc_stats_t *stats, isc_statscounter_t counter)
 {
+	uint32_t value = 0;
+	int i;
 	REQUIRE(ISC_STATS_VALID(stats));
 	REQUIRE(counter < stats->ncounters);
 
-	return (atomic_load_explicit(&stats->counters[counter],
-				    memory_order_relaxed));
+	for (i = 0; i < STATS_BUCKETS; i++) {
+		int idx = i * stats->ncounters + counter;
+		value += atomic_load_explicit(&stats->counters[idx],
+					      memory_order_relaxed);
+	}
+	return (value);
 }