From: Maria Matejka Date: Thu, 6 Jun 2024 15:45:35 +0000 (+0200) Subject: Netindex: allow u32 -> netindex without locking X-Git-Tag: v3.0.0~177 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9e88fc5b6b9703772fa7f21451d4984913ec48e7;p=thirdparty%2Fbird.git Netindex: allow u32 -> netindex without locking --- diff --git a/lib/netindex.c b/lib/netindex.c index 2b87ed081..8f6363499 100644 --- a/lib/netindex.c +++ b/lib/netindex.c @@ -9,6 +9,9 @@ #include "lib/birdlib.h" #include "lib/netindex_private.h" +#define NETINDEX_INIT_BLOCK_SIZE 128 +struct netindex netindex_in_progress; + #define NETINDEX_KEY(n) (n)->hash, (n)->addr #define NETINDEX_NEXT(n) (n)->next #define NETINDEX_EQ(h,n,i,o) ((h == i) && net_equal(n,o)) @@ -37,10 +40,13 @@ void netindex_hash_consistency_check(struct netindex_hash_private *nh) { uint count = 0; + struct netindex * _Atomic *block = atomic_load_explicit(&nh->block, memory_order_relaxed); + u32 block_size = atomic_load_explicit(&nh->block_size, memory_order_relaxed); HASH_WALK(nh->hash, next, i) { ASSERT_DIE(count < nh->hash.count); - ASSERT_DIE(nh->block[i->index] == i); + ASSERT_DIE(i->index < block_size); + ASSERT_DIE(atomic_load_explicit(&block[i->index], memory_order_relaxed) == i); count++; } HASH_WALK_END; @@ -67,8 +73,10 @@ netindex_hash_new(pool *sp, event_list *cleanup_target, u8 type) nh->slab = net_addr_length[type] ? sl_new(nh->pool, sizeof (struct netindex) + net_addr_length[type]) : NULL; HASH_INIT(nh->hash, nh->pool, NETINDEX_ORDER); - nh->block_size = 128; - nh->block = mb_allocz(nh->pool, nh->block_size * sizeof (struct netindex *)); + atomic_store_explicit(&nh->block_size, NETINDEX_INIT_BLOCK_SIZE, memory_order_release); + atomic_store_explicit(&nh->block, + mb_allocz(nh->pool, NETINDEX_INIT_BLOCK_SIZE * sizeof *nh->block), + memory_order_release); hmap_init(&nh->id_map, nh->pool, 128); @@ -91,27 +99,61 @@ netindex_hash_cleanup(void *_nh) uint kept = 0; - for (uint i = 0; i < nh->block_size; i++) + uint bs = atomic_load_explicit(&nh->block_size, memory_order_relaxed); + struct netindex * _Atomic *block = atomic_load_explicit(&nh->block, memory_order_relaxed); + + for (uint i = 0; i < bs; i++) { - struct netindex *ni = nh->block[i]; + struct netindex *ni = atomic_load_explicit(&block[i], memory_order_acquire); if (!ni) continue; + /* We may use the acquired netindex pointer as we are + * the only process which deletes them */ ASSERT_DIE(i == ni->index); - if (lfuc_finished(&ni->uc)) + /* Check finished */ + if (!lfuc_finished(&ni->uc)) { - HASH_REMOVE2(nh->hash, NETINDEX, nh->pool, ni); - hmap_clear(&nh->id_map, ni->index); - nh->block[i] = NULL; - - if (nh->slab) - sl_free(ni); - else - mb_free(ni); + kept++; + continue; } - else + + /* Looks finished, try removing temporarily */ + ASSERT_DIE(ni == atomic_exchange_explicit(&block[i], &netindex_in_progress, memory_order_acq_rel)); + + u32 block_epoch = nh->block_epoch; + UNLOCK_DOMAIN(attrs, dom); + synchronize_rcu(); + LOCK_DOMAIN(attrs, dom); + if (block_epoch != nh->block_epoch) + { + /* Somebody reallocated the block inbetween, use the new one */ + block = atomic_load_explicit(&nh->block, memory_order_relaxed); + bs = atomic_load_explicit(&nh->block_size, memory_order_relaxed); + } + + /* Now no reader can possibly still have the old pointer, + * unless somebody found it inbetween and ref'd it. */ + if (!lfuc_finished(&ni->uc)) + { + /* Collision, return the netindex to the block. */ + ASSERT_DIE(&netindex_in_progress == atomic_exchange_explicit(&block[i], ni, memory_order_acq_rel)); kept++; + continue; + } + + /* Now the netindex is definitely obsolete, set block to NULL */ + ASSERT_DIE(&netindex_in_progress == atomic_exchange_explicit(&block[i], NULL, memory_order_acq_rel)); + + /* And free it from other structures */ + HASH_REMOVE2(nh->hash, NETINDEX, nh->pool, ni); + hmap_clear(&nh->id_map, ni->index); + + if (nh->slab) + sl_free(ni); + else + mb_free(ni); } EXPENSIVE_CHECK(netindex_hash_consistency_check(nh)); @@ -178,12 +220,6 @@ net_find_index_fragile(struct netindex_hash_private *hp, const net_addr *n) return HASH_FIND(hp->hash, NETINDEX, h, n); } -static struct netindex * -net_find_index_locked(struct netindex_hash_private *hp, const net_addr *n) -{ - return net_lock_revive_unlock(hp, net_find_index_fragile(hp, n)); -} - static struct netindex * net_new_index_locked(struct netindex_hash_private *hp, const net_addr *n) { @@ -203,20 +239,31 @@ net_new_index_locked(struct netindex_hash_private *hp, const net_addr *n) net_copy(ni->addr, n); HASH_INSERT2(hp->hash, NETINDEX, hp->pool, ni); - while (hp->block_size <= i) + + struct netindex * _Atomic *block = atomic_load_explicit(&hp->block, memory_order_relaxed); + u32 bs = atomic_load_explicit(&hp->block_size, memory_order_relaxed); + u32 nbs = bs; + while (nbs <= i) + nbs *= 2; + + if (nbs > bs) { - u32 bs = hp->block_size; - struct netindex **nb = mb_alloc(hp->pool, bs * 2 * sizeof *nb); - memcpy(nb, hp->block, bs * sizeof *nb); - memset(&nb[bs], 0, bs * sizeof *nb); + struct netindex * _Atomic *nb = mb_alloc(hp->pool, bs * 2 * sizeof *nb); + memcpy(nb, block, bs * sizeof *nb); + memset(&nb[bs], 0, (nbs - bs) * sizeof *nb); + + ASSERT_DIE(block == atomic_exchange_explicit(&hp->block, nb, memory_order_acq_rel)); + ASSERT_DIE(bs == atomic_exchange_explicit(&hp->block_size, nbs, memory_order_acq_rel)); + synchronize_rcu(); - mb_free(hp->block); - hp->block = nb; + mb_free(block); + block = nb; - hp->block_size *= 2; + hp->block_epoch++; } - hp->block[i] = ni; + ASSERT_DIE(i < nbs); + atomic_store_explicit(&block[i], ni, memory_order_release); return net_lock_revive_unlock(hp, ni); } @@ -242,23 +289,47 @@ struct netindex * net_find_index(netindex_hash *h, const net_addr *n) { NH_LOCK(h, hp); - return net_find_index_locked(hp, n); + struct netindex *ni = net_find_index_fragile(hp, n); + return (ni == &netindex_in_progress) ? NULL : net_lock_revive_unlock(hp, ni); } struct netindex * net_get_index(netindex_hash *h, const net_addr *n) { - NH_LOCK(h, hp); - return - net_find_index_locked(hp, n) ?: - net_new_index_locked(hp, n); + while (1) + { + NH_LOCK(h, hp); + struct netindex *ni = net_find_index_fragile(hp, n); + if (ni == &netindex_in_progress) + continue; + + if (ni) + return net_lock_revive_unlock(hp, ni); + else + return net_new_index_locked(hp, n); + } } struct netindex * net_resolve_index(netindex_hash *h, u32 i) { - NH_LOCK(h, hp); + RCU_ANCHOR(u); - struct netindex *ni = hp->block[i]; - return net_lock_revive_unlock(hp, ni); + struct netindex * _Atomic *block = atomic_load_explicit(&h->block, memory_order_relaxed); + u32 bs = atomic_load_explicit(&h->block_size, memory_order_relaxed); + + if (i >= bs) + return NULL; + + struct netindex *ni = atomic_load_explicit(&block[i], memory_order_acquire); + if (ni == NULL) + return NULL; + + if (ni == &netindex_in_progress) + RCU_RETRY(u); + + lfuc_lock_revive(&ni->uc); + net_unlock_index(h, ni); + + return ni; } diff --git a/lib/netindex_private.h b/lib/netindex_private.h index 745870d8e..a9a930486 100644 --- a/lib/netindex_private.h +++ b/lib/netindex_private.h @@ -16,6 +16,8 @@ event_list *cleanup_list; /* Cleanup event list */ \ event cleanup_event; /* Cleanup event */ \ u8 net_type; /* Which NET_* is stored */ \ + uint _Atomic block_size; /* How big block is */ \ + struct netindex * _Atomic * _Atomic block; /* u32 to netindex */ \ struct netindex_hash_private { struct { NETINDEX_HASH_PUBLIC; }; @@ -23,9 +25,8 @@ struct netindex_hash_private { pool *pool; slab *slab; HASH(struct netindex) hash; - uint block_size; - struct netindex **block; struct hmap id_map; + u32 block_epoch; event *deleted_event; event_list *deleted_target; }; @@ -35,6 +36,8 @@ typedef union netindex_hash { struct netindex_hash_private priv; } netindex_hash; +extern struct netindex netindex_in_progress; + LOBJ_UNLOCK_CLEANUP(netindex_hash, attrs); #define NH_LOCK(h, hp) LOBJ_LOCK(h, hp, netindex_hash, attrs)