]> git.ipfire.org Git - thirdparty/bind9.git/commitdiff
Reduce false sharing in dns_qpcache
authorOndřej Surý <ondrej@isc.org>
Mon, 3 Feb 2025 12:36:27 +0000 (13:36 +0100)
committerOndřej Surý <ondrej@isc.org>
Tue, 4 Feb 2025 20:37:46 +0000 (21:37 +0100)
Instead of having many node_lock_count * sizeof(<member>) arrays, pack
all the members into a qpcache_bucket_t struct that is cacheline aligned
and have a single array of those.

Additionaly, make both the head and the tail of isc_queue_t padded, not
just the head, to prevent false sharing of the lock-free structure with
the lock that follows it.

lib/dns/qpcache.c
lib/isc/include/isc/queue.h

index 560812a6d17c06ef1e4868604a49132fb2db4c2d..74bdba0223b13084ea18070766ab19d4fc3af163 100644 (file)
@@ -14,6 +14,7 @@
 /*! \file */
 
 #include <inttypes.h>
+#include <stdalign.h>
 #include <stdbool.h>
 #include <sys/mman.h>
 
@@ -21,8 +22,6 @@
 #include <isc/async.h>
 #include <isc/atomic.h>
 #include <isc/file.h>
-#include <isc/hash.h>
-#include <isc/hashmap.h>
 #include <isc/heap.h>
 #include <isc/hex.h>
 #include <isc/log.h>
@@ -217,6 +216,41 @@ struct qpcnode {
        isc_queue_node_t deadlink;
 };
 
+/*%
+ * One bucket structure will be created for each loop, and
+ * nodes in the database will evenly distributed among buckets
+ * to reduce contention between threads.
+ */
+typedef struct qpcache_bucket {
+       /*%
+        * Temporary storage for stale cache nodes and dynamically
+        * deleted nodes that await being cleaned up.
+        */
+       isc_queue_t deadnodes;
+
+       /* Per-bucket lock. */
+       isc_rwlock_t lock;
+
+       /*
+        * Linked list used to implement LRU cache cleaning.
+        */
+       dns_slabheaderlist_t lru;
+
+       /*
+        * The heap is used for TTL based expiry.  Note that qpcache->hmctx
+        * is the memory context to use for heap memory; this differs from
+        * the main database memory context, which is qpcache->common.mctx.
+        */
+       isc_heap_t *heap;
+
+       /* Padding to prevent false sharing between locks. */
+       uint8_t __padding[ISC_OS_CACHELINE_SIZE -
+                         (sizeof(dns_slabheaderlist_t) + sizeof(isc_heap_t *) +
+                          sizeof(isc_rwlock_t)) %
+                                 ISC_OS_CACHELINE_SIZE];
+
+} qpcache_bucket_t;
+
 typedef struct qpcache qpcache_t;
 struct qpcache {
        /* Unlocked. */
@@ -245,10 +279,6 @@ struct qpcache {
         */
        isc_refcount_t references;
 
-       /* Locks for individual tree nodes */
-       unsigned int node_lock_count;
-       isc_rwlock_t *node_locks;
-
        dns_stats_t *rrsetstats;
        isc_stats_t *cachestats;
 
@@ -262,13 +292,6 @@ struct qpcache {
         */
        uint32_t serve_stale_refresh;
 
-       /*
-        * This is an array of linked lists used to implement the LRU cache.
-        * There will be node_lock_count linked lists here.  Nodes in bucket 1
-        * will be placed on the linked list lru[1].
-        */
-       dns_slabheaderlist_t *lru;
-
        /*
         * Start point % node_lock_count for next LRU cleanup.
         */
@@ -280,24 +303,14 @@ struct qpcache {
         */
        _Atomic(isc_stdtime_t) last_used;
 
-       /*%
-        * Temporary storage for stale cache nodes and dynamically deleted
-        * nodes that await being cleaned up.
-        */
-       isc_queue_t *deadnodes;
-
-       /*
-        * Heaps.  These are used for TTL based expiry in a cache,
-        * or for zone resigning in a zone DB.  hmctx is the memory
-        * context to use for the heap (which differs from the main
-        * database memory context in the case of a cache).
-        */
-       isc_mem_t *hmctx;
-       isc_heap_t **heaps;
-
        /* Locked by tree_lock. */
        dns_qp_t *tree;
        dns_qp_t *nsec;
+
+       isc_mem_t *hmctx; /* Memory context for the heaps */
+
+       size_t buckets_count;
+       qpcache_bucket_t buckets[]; /* attribute((counted_by(buckets_count))) */
 };
 
 #ifdef DNS_DB_NODETRACE
@@ -542,9 +555,11 @@ update_header(qpcache_t *qpdb, dns_slabheader_t *header, isc_stdtime_t now) {
        /* To be checked: can we really assume this? XXXMLG */
        INSIST(ISC_LINK_LINKED(header, link));
 
-       ISC_LIST_UNLINK(qpdb->lru[HEADERNODE(header)->locknum], header, link);
+       ISC_LIST_UNLINK(qpdb->buckets[HEADERNODE(header)->locknum].lru, header,
+                       link);
        header->last_used = now;
-       ISC_LIST_PREPEND(qpdb->lru[HEADERNODE(header)->locknum], header, link);
+       ISC_LIST_PREPEND(qpdb->buckets[HEADERNODE(header)->locknum].lru, header,
+                        link);
 }
 
 /*
@@ -773,7 +788,7 @@ qpcnode_release(qpcache_t *qpdb, qpcnode_t *node, isc_rwlocktype_t *nlocktypep,
                 * erefs (but NOT references!), upgrade the node lock,
                 * decrement erefs again, and see if it's still zero.
                 */
-               isc_rwlock_t *nlock = &qpdb->node_locks[node->locknum];
+               isc_rwlock_t *nlock = &qpdb->buckets[node->locknum].lock;
                qpcnode_erefs_increment(qpdb, node, *nlocktypep,
                                        *tlocktypep DNS__DB_FLARG_PASS);
                NODE_FORCEUPGRADE(nlock, nlocktypep);
@@ -833,8 +848,9 @@ qpcnode_release(qpcache_t *qpdb, qpcnode_t *node, isc_rwlocktype_t *nlocktypep,
                                *tlocktypep DNS__DB_FLARG_PASS);
 
                isc_queue_node_init(&node->deadlink);
-               if (!isc_queue_enqueue_entry(&qpdb->deadnodes[node->locknum],
-                                            node, deadlink))
+               if (!isc_queue_enqueue_entry(
+                           &qpdb->buckets[node->locknum].deadnodes, node,
+                           deadlink))
                {
                        /* Queue was empty, trigger new cleaning */
                        isc_loop_t *loop = isc_loop_get(qpdb->loopmgr,
@@ -1161,7 +1177,8 @@ setup_delegation(qpc_search_t *search, dns_dbnode_t **nodep,
        }
        if (rdataset != NULL) {
                isc_rwlocktype_t nlocktype = isc_rwlocktype_none;
-               isc_rwlock_t *nlock = &search->qpdb->node_locks[node->locknum];
+               isc_rwlock_t *nlock =
+                       &search->qpdb->buckets[node->locknum].lock;
                NODE_RDLOCK(nlock, &nlocktype);
                bindrdataset(search->qpdb, node, search->zonecut_header,
                             search->now, nlocktype, tlocktype,
@@ -1299,7 +1316,7 @@ check_zonecut(qpcnode_t *node, void *arg DNS__DB_FLARG) {
 
        REQUIRE(search->zonecut == NULL);
 
-       nlock = &search->qpdb->node_locks[node->locknum];
+       nlock = &search->qpdb->buckets[node->locknum].lock;
        NODE_RDLOCK(nlock, &nlocktype);
 
        /*
@@ -1372,7 +1389,7 @@ find_deepest_zonecut(qpc_search_t *search, qpcnode_t *node,
                isc_rwlocktype_t nlocktype = isc_rwlocktype_none;
 
                dns_qpchain_node(&search->chain, i, NULL, (void **)&node, NULL);
-               nlock = &qpdb->node_locks[node->locknum];
+               nlock = &qpdb->buckets[node->locknum].lock;
 
                NODE_RDLOCK(nlock, &nlocktype);
 
@@ -1522,7 +1539,7 @@ find_coveringnsec(qpc_search_t *search, const dns_name_t *name,
        }
        dns_name_copy(&node->name, fname);
 
-       nlock = &search->qpdb->node_locks[node->locknum];
+       nlock = &search->qpdb->buckets[node->locknum].lock;
        NODE_RDLOCK(nlock, &nlocktype);
        for (header = node->data; header != NULL; header = header_next) {
                header_next = header->next;
@@ -1700,7 +1717,7 @@ qpcache_find(dns_db_t *db, const dns_name_t *name, dns_dbversion_t *version,
         * We now go looking for rdata...
         */
 
-       nlock = &search.qpdb->node_locks[node->locknum];
+       nlock = &search.qpdb->buckets[node->locknum].lock;
        NODE_RDLOCK(nlock, &nlocktype);
 
        /*
@@ -2004,7 +2021,7 @@ tree_exit:
        if (search.need_cleanup) {
                node = search.zonecut;
                INSIST(node != NULL);
-               nlock = &search.qpdb->node_locks[node->locknum];
+               nlock = &search.qpdb->buckets[node->locknum].lock;
 
                NODE_RDLOCK(nlock, &nlocktype);
                qpcnode_release(search.qpdb, node, &nlocktype, &tlocktype,
@@ -2088,7 +2105,7 @@ qpcache_findzonecut(dns_db_t *db, const dns_name_t *name, unsigned int options,
         * We now go looking for an NS rdataset at the node.
         */
 
-       nlock = &search.qpdb->node_locks[node->locknum];
+       nlock = &search.qpdb->buckets[node->locknum].lock;
        NODE_RDLOCK(nlock, &nlocktype);
 
        for (header = node->data; header != NULL; header = header_next) {
@@ -2210,7 +2227,7 @@ qpcache_findrdataset(dns_db_t *db, dns_dbnode_t *node, dns_dbversion_t *version,
                now = isc_stdtime_now();
        }
 
-       nlock = &qpdb->node_locks[qpnode->locknum];
+       nlock = &qpdb->buckets[qpnode->locknum].lock;
        NODE_RDLOCK(nlock, &nlocktype);
 
        matchtype = DNS_TYPEPAIR_VALUE(type, covers);
@@ -2355,7 +2372,7 @@ expiredata(dns_db_t *db, dns_dbnode_t *node, void *data) {
        dns_slabheader_t *header = data;
        isc_rwlocktype_t nlocktype = isc_rwlocktype_none;
        isc_rwlocktype_t tlocktype = isc_rwlocktype_none;
-       isc_rwlock_t *nlock = &qpdb->node_locks[qpnode->locknum];
+       isc_rwlock_t *nlock = &qpdb->buckets[qpnode->locknum].lock;
 
        NODE_WRLOCK(nlock, &nlocktype);
        expireheader(header, &nlocktype, &tlocktype,
@@ -2381,10 +2398,10 @@ expire_lru_headers(qpcache_t *qpdb, unsigned int locknum,
        dns_slabheader_t *header = NULL;
        size_t purged = 0;
 
-       for (header = ISC_LIST_TAIL(qpdb->lru[locknum]);
+       for (header = ISC_LIST_TAIL(qpdb->buckets[locknum].lru);
             header != NULL && header->last_used <= qpdb->last_used &&
             purged <= purgesize;
-            header = ISC_LIST_TAIL(qpdb->lru[locknum]))
+            header = ISC_LIST_TAIL(qpdb->buckets[locknum].lru))
        {
                size_t header_size = rdataset_size(header);
 
@@ -2395,7 +2412,7 @@ expire_lru_headers(qpcache_t *qpdb, unsigned int locknum,
                 * referenced any more (so unlinking is safe) since the
                 * TTL will be reset to 0.
                 */
-               ISC_LIST_UNLINK(qpdb->lru[locknum], header, link);
+               ISC_LIST_UNLINK(qpdb->buckets[locknum].lru, header, link);
                expireheader(header, nlocktypep, tlocktypep,
                             dns_expire_lru DNS__DB_FLARG_PASS);
                purged += header_size;
@@ -2417,7 +2434,7 @@ expire_lru_headers(qpcache_t *qpdb, unsigned int locknum,
 static void
 overmem(qpcache_t *qpdb, dns_slabheader_t *newheader,
        isc_rwlocktype_t *tlocktypep DNS__DB_FLARG) {
-       uint32_t locknum_start = qpdb->lru_sweep++ % qpdb->node_lock_count;
+       uint32_t locknum_start = qpdb->lru_sweep++ % qpdb->buckets_count;
        uint32_t locknum = locknum_start;
        size_t purgesize, purged = 0;
        isc_stdtime_t min_last_used = 0;
@@ -2437,7 +2454,7 @@ overmem(qpcache_t *qpdb, dns_slabheader_t *newheader,
 again:
        do {
                isc_rwlocktype_t nlocktype = isc_rwlocktype_none;
-               isc_rwlock_t *nlock = &qpdb->node_locks[locknum];
+               isc_rwlock_t *nlock = &qpdb->buckets[locknum].lock;
                NODE_WRLOCK(nlock, &nlocktype);
 
                purged += expire_lru_headers(
@@ -2448,14 +2465,15 @@ again:
                 * Work out the oldest remaining last_used values of the list
                 * tails as we walk across the array of lru lists.
                 */
-               dns_slabheader_t *header = ISC_LIST_TAIL(qpdb->lru[locknum]);
+               dns_slabheader_t *header =
+                       ISC_LIST_TAIL(qpdb->buckets[locknum].lru);
                if (header != NULL &&
                    (min_last_used == 0 || header->last_used < min_last_used))
                {
                        min_last_used = header->last_used;
                }
                NODE_UNLOCK(nlock, &nlocktype);
-               locknum = (locknum + 1) % qpdb->node_lock_count;
+               locknum = (locknum + 1) % qpdb->buckets_count;
        } while (locknum != locknum_start && purged <= purgesize);
 
        /*
@@ -2527,40 +2545,15 @@ qpcache__destroy(qpcache_t *qpdb) {
        if (dns_name_dynamic(&qpdb->common.origin)) {
                dns_name_free(&qpdb->common.origin, qpdb->common.mctx);
        }
-       for (i = 0; i < qpdb->node_lock_count; i++) {
-               NODE_DESTROYLOCK(&qpdb->node_locks[i]);
-       }
+       for (i = 0; i < qpdb->buckets_count; i++) {
+               NODE_DESTROYLOCK(&qpdb->buckets[i].lock);
 
-       /*
-        * Clean up LRU / re-signing order lists.
-        */
-       if (qpdb->lru != NULL) {
-               for (i = 0; i < qpdb->node_lock_count; i++) {
-                       INSIST(ISC_LIST_EMPTY(qpdb->lru[i]));
-               }
-               isc_mem_cput(qpdb->common.mctx, qpdb->lru,
-                            qpdb->node_lock_count,
-                            sizeof(dns_slabheaderlist_t));
-       }
-       /*
-        * Clean up dead node buckets.
-        */
-       for (i = 0; i < qpdb->node_lock_count; i++) {
-               INSIST(isc_queue_empty(&qpdb->deadnodes[i]));
-               isc_queue_destroy(&qpdb->deadnodes[i]);
-       }
-       isc_mem_cput(qpdb->common.mctx, qpdb->deadnodes, qpdb->node_lock_count,
-                    sizeof(qpdb->deadnodes[0]));
+               INSIST(ISC_LIST_EMPTY(qpdb->buckets[i].lru));
 
-       /*
-        * Clean up heap objects.
-        */
-       if (qpdb->heaps != NULL) {
-               for (i = 0; i < qpdb->node_lock_count; i++) {
-                       isc_heap_destroy(&qpdb->heaps[i]);
-               }
-               isc_mem_cput(qpdb->hmctx, qpdb->heaps, qpdb->node_lock_count,
-                            sizeof(isc_heap_t *));
+               INSIST(isc_queue_empty(&qpdb->buckets[i].deadnodes));
+               isc_queue_destroy(&qpdb->buckets[i].deadnodes);
+
+               isc_heap_destroy(&qpdb->buckets[i].heap);
        }
 
        if (qpdb->rrsetstats != NULL) {
@@ -2570,8 +2563,6 @@ qpcache__destroy(qpcache_t *qpdb) {
                isc_stats_detach(&qpdb->cachestats);
        }
 
-       isc_mem_cput(qpdb->common.mctx, qpdb->node_locks, qpdb->node_lock_count,
-                    sizeof(qpdb->node_locks[0]));
        TREE_DESTROYLOCK(&qpdb->tree_lock);
        isc_refcount_destroy(&qpdb->references);
        isc_refcount_destroy(&qpdb->common.references);
@@ -2581,7 +2572,9 @@ qpcache__destroy(qpcache_t *qpdb) {
        qpdb->common.impmagic = 0;
        isc_mem_detach(&qpdb->hmctx);
 
-       isc_mem_putanddetach(&qpdb->common.mctx, qpdb, sizeof(*qpdb));
+       isc_mem_putanddetach(&qpdb->common.mctx, qpdb,
+                            sizeof(*qpdb) + qpdb->buckets_count *
+                                                    sizeof(qpdb->buckets[0]));
 }
 
 static void
@@ -2603,18 +2596,19 @@ cleanup_deadnodes(void *arg) {
        uint16_t locknum = isc_tid();
        isc_rwlocktype_t tlocktype = isc_rwlocktype_none;
        isc_rwlocktype_t nlocktype = isc_rwlocktype_none;
-       isc_rwlock_t *nlock = &qpdb->node_locks[locknum];
+       isc_rwlock_t *nlock = &qpdb->buckets[locknum].lock;
        qpcnode_t *qpnode = NULL, *qpnext = NULL;
        isc_queue_t deadnodes;
 
-       INSIST(locknum < qpdb->node_lock_count);
+       INSIST(locknum < qpdb->buckets_count);
 
        isc_queue_init(&deadnodes);
 
        TREE_WRLOCK(&qpdb->tree_lock, &tlocktype);
        NODE_WRLOCK(nlock, &nlocktype);
 
-       RUNTIME_CHECK(isc_queue_splice(&deadnodes, &qpdb->deadnodes[locknum]));
+       RUNTIME_CHECK(isc_queue_splice(&deadnodes,
+                                      &qpdb->buckets[locknum].deadnodes));
        isc_queue_for_each_entry_safe(&deadnodes, qpnode, qpnext, deadlink) {
                qpcnode_release(qpdb, qpnode, &nlocktype, &tlocktype, false);
        }
@@ -2636,7 +2630,7 @@ static void
 reactivate_node(qpcache_t *qpdb, qpcnode_t *node,
                isc_rwlocktype_t tlocktype ISC_ATTR_UNUSED DNS__DB_FLARG) {
        isc_rwlocktype_t nlocktype = isc_rwlocktype_none;
-       isc_rwlock_t *nlock = &qpdb->node_locks[node->locknum];
+       isc_rwlock_t *nlock = &qpdb->buckets[node->locknum].lock;
 
        NODE_RDLOCK(nlock, &nlocktype);
        qpcnode_acquire(qpdb, node, nlocktype, tlocktype DNS__DB_FLARG_PASS);
@@ -2649,11 +2643,9 @@ new_qpcnode(qpcache_t *qpdb, const dns_name_t *name) {
        *newdata = (qpcnode_t){
                .name = DNS_NAME_INITEMPTY,
                .references = ISC_REFCOUNT_INITIALIZER(1),
-               .locknum = isc_random_uniform(qpdb->node_lock_count),
+               .locknum = isc_random_uniform(qpdb->buckets_count),
        };
 
-       INSIST(newdata->locknum < qpdb->node_lock_count);
-
        isc_mem_attach(qpdb->common.mctx, &newdata->mctx);
        dns_name_dupwithoffsets(name, newdata->mctx, &newdata->name);
 
@@ -2728,7 +2720,7 @@ qpcache_detachnode(dns_db_t *db, dns_dbnode_t **nodep DNS__DB_FLARG) {
 
        node = (qpcnode_t *)(*nodep);
        *nodep = NULL;
-       nlock = &qpdb->node_locks[node->locknum];
+       nlock = &qpdb->buckets[node->locknum].lock;
 
        /*
         * We can't destroy qpcache while holding a nodelock, so
@@ -3022,11 +3014,13 @@ find_header:
                        }
                        if (header->last_used != now) {
                                ISC_LIST_UNLINK(
-                                       qpdb->lru[HEADERNODE(header)->locknum],
+                                       qpdb->buckets[HEADERNODE(header)->locknum]
+                                               .lru,
                                        header, link);
                                header->last_used = now;
                                ISC_LIST_PREPEND(
-                                       qpdb->lru[HEADERNODE(header)->locknum],
+                                       qpdb->buckets[HEADERNODE(header)->locknum]
+                                               .lru,
                                        header, link);
                        }
                        if (header->noqname == NULL &&
@@ -3084,11 +3078,13 @@ find_header:
                        }
                        if (header->last_used != now) {
                                ISC_LIST_UNLINK(
-                                       qpdb->lru[HEADERNODE(header)->locknum],
+                                       qpdb->buckets[HEADERNODE(header)->locknum]
+                                               .lru,
                                        header, link);
                                header->last_used = now;
                                ISC_LIST_PREPEND(
-                                       qpdb->lru[HEADERNODE(header)->locknum],
+                                       qpdb->buckets[HEADERNODE(header)->locknum]
+                                               .lru,
                                        header, link);
                        }
                        if (header->noqname == NULL &&
@@ -3117,15 +3113,14 @@ find_header:
                        idx = HEADERNODE(newheader)->locknum;
                        if (ZEROTTL(newheader)) {
                                newheader->last_used = qpdb->last_used + 1;
-                               ISC_LIST_APPEND(qpdb->lru[idx], newheader,
-                                               link);
+                               ISC_LIST_APPEND(qpdb->buckets[idx].lru,
+                                               newheader, link);
                        } else {
-                               ISC_LIST_PREPEND(qpdb->lru[idx], newheader,
-                                                link);
+                               ISC_LIST_PREPEND(qpdb->buckets[idx].lru,
+                                                newheader, link);
                        }
-                       INSIST(qpdb->heaps != NULL);
-                       isc_heap_insert(qpdb->heaps[idx], newheader);
-                       newheader->heap = qpdb->heaps[idx];
+                       isc_heap_insert(qpdb->buckets[idx].heap, newheader);
+                       newheader->heap = qpdb->buckets[idx].heap;
 
                        /*
                         * There are no other references to 'header' when
@@ -3142,16 +3137,15 @@ find_header:
                        dns_slabheader_destroy(&header);
                } else {
                        idx = HEADERNODE(newheader)->locknum;
-                       INSIST(qpdb->heaps != NULL);
-                       isc_heap_insert(qpdb->heaps[idx], newheader);
-                       newheader->heap = qpdb->heaps[idx];
+                       isc_heap_insert(qpdb->buckets[idx].heap, newheader);
+                       newheader->heap = qpdb->buckets[idx].heap;
                        if (ZEROTTL(newheader)) {
                                newheader->last_used = qpdb->last_used + 1;
-                               ISC_LIST_APPEND(qpdb->lru[idx], newheader,
-                                               link);
+                               ISC_LIST_APPEND(qpdb->buckets[idx].lru,
+                                               newheader, link);
                        } else {
-                               ISC_LIST_PREPEND(qpdb->lru[idx], newheader,
-                                                link);
+                               ISC_LIST_PREPEND(qpdb->buckets[idx].lru,
+                                                newheader, link);
                        }
                        if (topheader_prev != NULL) {
                                topheader_prev->next = newheader;
@@ -3181,12 +3175,14 @@ find_header:
                }
 
                idx = HEADERNODE(newheader)->locknum;
-               isc_heap_insert(qpdb->heaps[idx], newheader);
-               newheader->heap = qpdb->heaps[idx];
+               isc_heap_insert(qpdb->buckets[idx].heap, newheader);
+               newheader->heap = qpdb->buckets[idx].heap;
                if (ZEROTTL(newheader)) {
-                       ISC_LIST_APPEND(qpdb->lru[idx], newheader, link);
+                       ISC_LIST_APPEND(qpdb->buckets[idx].lru, newheader,
+                                       link);
                } else {
-                       ISC_LIST_PREPEND(qpdb->lru[idx], newheader, link);
+                       ISC_LIST_PREPEND(qpdb->buckets[idx].lru, newheader,
+                                        link);
                }
 
                if (topheader != NULL) {
@@ -3470,7 +3466,7 @@ qpcache_addrdataset(dns_db_t *db, dns_dbnode_t *node, dns_dbversion_t *version,
                overmem(qpdb, newheader, &tlocktype DNS__DB_FLARG_PASS);
        }
 
-       nlock = &qpdb->node_locks[qpnode->locknum];
+       nlock = &qpdb->buckets[qpnode->locknum].lock;
 
        NODE_WRLOCK(nlock, &nlocktype);
 
@@ -3557,7 +3553,7 @@ qpcache_deleterdataset(dns_db_t *db, dns_dbnode_t *node,
        setttl(newheader, 0);
        atomic_init(&newheader->attributes, DNS_SLABHEADERATTR_NONEXISTENT);
 
-       nlock = &qpdb->node_locks[qpnode->locknum];
+       nlock = &qpdb->buckets[qpnode->locknum].lock;
        NODE_WRLOCK(nlock, &nlocktype);
        result = add(qpdb, qpnode, NULL, newheader, DNS_DBADD_FORCE, false,
                     NULL, 0, nlocktype,
@@ -3596,7 +3592,7 @@ locknode(dns_db_t *db, dns_dbnode_t *node, isc_rwlocktype_t type) {
        qpcache_t *qpdb = (qpcache_t *)db;
        qpcnode_t *qpnode = (qpcnode_t *)node;
 
-       RWLOCK(&qpdb->node_locks[qpnode->locknum], type);
+       RWLOCK(&qpdb->buckets[qpnode->locknum].lock, type);
 }
 
 static void
@@ -3604,7 +3600,7 @@ unlocknode(dns_db_t *db, dns_dbnode_t *node, isc_rwlocktype_t type) {
        qpcache_t *qpdb = (qpcache_t *)db;
        qpcnode_t *qpnode = (qpcnode_t *)node;
 
-       RWUNLOCK(&qpdb->node_locks[qpnode->locknum], type);
+       RWUNLOCK(&qpdb->buckets[qpnode->locknum].lock, type);
 }
 
 isc_result_t
@@ -3616,12 +3612,15 @@ dns__qpcache_create(isc_mem_t *mctx, const dns_name_t *origin,
        isc_mem_t *hmctx = mctx;
        isc_loop_t *loop = isc_loop();
        int i;
+       isc_loopmgr_t *loopmgr = isc_loop_getloopmgr(loop);
+       size_t nloops = isc_loopmgr_nloops(loopmgr);
 
        /* This database implementation only supports cache semantics */
        REQUIRE(type == dns_dbtype_cache);
        REQUIRE(loop != NULL);
 
-       qpdb = isc_mem_get(mctx, sizeof(*qpdb));
+       qpdb = isc_mem_get(mctx,
+                          sizeof(*qpdb) + nloops * sizeof(qpdb->buckets[0]));
        *qpdb = (qpcache_t){
                .common.methods = &qpdb_cachemethods,
                .common.origin = DNS_NAME_INITEMPTY,
@@ -3630,6 +3629,7 @@ dns__qpcache_create(isc_mem_t *mctx, const dns_name_t *origin,
                .common.references = 1,
                .loopmgr = isc_loop_getloopmgr(loop),
                .references = 1,
+               .buckets_count = nloops,
        };
 
        /*
@@ -3642,38 +3642,19 @@ dns__qpcache_create(isc_mem_t *mctx, const dns_name_t *origin,
        isc_rwlock_init(&qpdb->lock);
        TREE_INITLOCK(&qpdb->tree_lock);
 
-       qpdb->node_lock_count = isc_loopmgr_nloops(qpdb->loopmgr);
-       qpdb->node_locks = isc_mem_cget(mctx, qpdb->node_lock_count,
-                                       sizeof(qpdb->node_locks[0]));
+       qpdb->buckets_count = isc_loopmgr_nloops(qpdb->loopmgr);
 
        dns_rdatasetstats_create(mctx, &qpdb->rrsetstats);
-       qpdb->lru = isc_mem_cget(mctx, qpdb->node_lock_count,
-                                sizeof(dns_slabheaderlist_t));
-       for (i = 0; i < (int)qpdb->node_lock_count; i++) {
-               ISC_LIST_INIT(qpdb->lru[i]);
-       }
+       for (i = 0; i < (int)qpdb->buckets_count; i++) {
+               ISC_LIST_INIT(qpdb->buckets[i].lru);
 
-       /*
-        * Create the heaps.
-        */
-       qpdb->heaps = isc_mem_cget(hmctx, qpdb->node_lock_count,
-                                  sizeof(isc_heap_t *));
-       for (i = 0; i < (int)qpdb->node_lock_count; i++) {
+               qpdb->buckets[i].heap = NULL;
                isc_heap_create(hmctx, ttl_sooner, set_index, 0,
-                               &qpdb->heaps[i]);
-       }
+                               &qpdb->buckets[i].heap);
 
-       /*
-        * Create deadnode lists.
-        */
-       qpdb->deadnodes = isc_mem_cget(mctx, qpdb->node_lock_count,
-                                      sizeof(qpdb->deadnodes[0]));
-       for (i = 0; i < (int)(qpdb->node_lock_count); i++) {
-               isc_queue_init(&qpdb->deadnodes[i]);
-       }
+               isc_queue_init(&qpdb->buckets[i].deadnodes);
 
-       for (i = 0; i < (int)(qpdb->node_lock_count); i++) {
-               NODE_INITLOCK(&qpdb->node_locks[i]);
+               NODE_INITLOCK(&qpdb->buckets[i].lock);
        }
 
        /*
@@ -3756,7 +3737,7 @@ rdatasetiter_first(dns_rdatasetiter_t *it DNS__DB_FLARG) {
        qpcnode_t *qpnode = (qpcnode_t *)iterator->common.node;
        dns_slabheader_t *header = NULL, *top_next = NULL;
        isc_rwlocktype_t nlocktype = isc_rwlocktype_none;
-       isc_rwlock_t *nlock = &qpdb->node_locks[qpnode->locknum];
+       isc_rwlock_t *nlock = &qpdb->buckets[qpnode->locknum].lock;
 
        NODE_RDLOCK(nlock, &nlocktype);
 
@@ -3802,7 +3783,7 @@ rdatasetiter_next(dns_rdatasetiter_t *it DNS__DB_FLARG) {
        dns_typepair_t type, negtype;
        dns_rdatatype_t rdtype, covers;
        isc_rwlocktype_t nlocktype = isc_rwlocktype_none;
-       isc_rwlock_t *nlock = &qpdb->node_locks[qpnode->locknum];
+       isc_rwlock_t *nlock = &qpdb->buckets[qpnode->locknum].lock;
        bool expiredok = EXPIREDOK(iterator);
 
        header = iterator->current;
@@ -3890,7 +3871,7 @@ rdatasetiter_current(dns_rdatasetiter_t *it,
        qpcnode_t *qpnode = (qpcnode_t *)iterator->common.node;
        dns_slabheader_t *header = NULL;
        isc_rwlocktype_t nlocktype = isc_rwlocktype_none;
-       isc_rwlock_t *nlock = &qpdb->node_locks[qpnode->locknum];
+       isc_rwlock_t *nlock = &qpdb->buckets[qpnode->locknum].lock;
 
        header = iterator->current;
        REQUIRE(header != NULL);
@@ -3934,7 +3915,7 @@ dereference_iter_node(qpc_dbit_t *qpdbiter DNS__DB_FLARG) {
 
        REQUIRE(tlocktype != isc_rwlocktype_write);
 
-       nlock = &qpdb->node_locks[node->locknum];
+       nlock = &qpdb->buckets[node->locknum].lock;
        NODE_RDLOCK(nlock, &nlocktype);
        qpcnode_release(qpdb, node, &nlocktype, &qpdbiter->tree_locked,
                        false DNS__DB_FLARG_PASS);
@@ -4252,7 +4233,7 @@ deletedata(dns_db_t *db ISC_ATTR_UNUSED, dns_dbnode_t *node ISC_ATTR_UNUSED,
 
        if (ISC_LINK_LINKED(header, link)) {
                int idx = HEADERNODE(header)->locknum;
-               ISC_LIST_UNLINK(qpdb->lru[idx], header, link);
+               ISC_LIST_UNLINK(qpdb->buckets[idx].lru, header, link);
        }
 
        if (header->noqname != NULL) {
@@ -4270,7 +4251,7 @@ static void
 expire_ttl_headers(qpcache_t *qpdb, unsigned int locknum,
                   isc_rwlocktype_t *nlocktypep, isc_rwlocktype_t *tlocktypep,
                   isc_stdtime_t now, bool cache_is_overmem DNS__DB_FLARG) {
-       isc_heap_t *heap = qpdb->heaps[locknum];
+       isc_heap_t *heap = qpdb->buckets[locknum].heap;
 
        for (size_t i = 0; i < DNS_QPDB_EXPIRE_TTL_COUNT; i++) {
                dns_slabheader_t *header = isc_heap_element(heap, 1);
index d34f551262da0a9be05ac17fd4093a9a3c988939..a2abe24c15b50981935ee17ec08276a849e7d052 100644 (file)
@@ -20,9 +20,11 @@ STATIC_ASSERT(sizeof(struct __cds_wfcq_head) <= ISC_OS_CACHELINE_SIZE,
 
 typedef struct isc_queue {
        struct __cds_wfcq_head head;
-       uint8_t                __padding[ISC_OS_CACHELINE_SIZE -
-                                sizeof(struct __cds_wfcq_head)];
+       uint8_t                __padding_head[ISC_OS_CACHELINE_SIZE -
+                                     sizeof(struct __cds_wfcq_head)];
        struct cds_wfcq_tail   tail;
+       uint8_t                __padding_tail[ISC_OS_CACHELINE_SIZE -
+                                     sizeof(struct __cds_wfcq_head)];
 } isc_queue_t;
 
 typedef struct cds_wfcq_node isc_queue_node_t;