bpf: Select bpf_local_storage_map_bucket based on bpf_local_storage

A later bpf_local_storage refactor will acquire all locks before
performing any update. To simplified the number of locks needed to take
in bpf_local_storage_map_update(), determine the bucket based on the
local_storage an selem belongs to instead of the selem pointer.

Currently, when a new selem needs to be created to replace the old selem
in bpf_local_storage_map_update(), locks of both buckets need to be
acquired to prevent racing. This can be simplified if the two selem
belongs to the same bucket so that only one bucket needs to be locked.
Therefore, instead of hashing selem, hashing the local_storage pointer
the selem belongs.

Performance wise, this is slightly better as update now requires locking
one bucket. It should not change the level of contention on one bucket
as the pointers to local storages of selems in a map are just as unique
as pointers to selems.

Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Amery Hung <ameryhung@gmail.com>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://patch.msgid.link/20260205222916.1788211-2-ameryhung@gmail.com

diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h
index 66432248cd810ec8b64104035fefd01c1b07af4a..2638487425b8934592a29cb9d9be857c8d2fa416 100644 (file)
--- a/include/linux/bpf_local_storage.h
+++ b/include/linux/bpf_local_storage.h
@@ -179,6 +179,7 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
 void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now);
 
 void bpf_selem_link_map(struct bpf_local_storage_map *smap,
+                       struct bpf_local_storage *local_storage,
                        struct bpf_local_storage_elem *selem);
 
 struct bpf_local_storage_elem *

diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c
index e2fe6c32822b89aae9bbd422a79b000d865b9348..91b28f4e31303f3cff4de8467de2e0eb300e3d15 100644 (file)
--- a/kernel/bpf/bpf_local_storage.c
+++ b/kernel/bpf/bpf_local_storage.c
@@ -19,9 +19,9 @@
 
 static struct bpf_local_storage_map_bucket *
 select_bucket(struct bpf_local_storage_map *smap,
-             struct bpf_local_storage_elem *selem)
+             struct bpf_local_storage *local_storage)
 {
-       return &smap->buckets[hash_ptr(selem, smap->bucket_log)];
+       return &smap->buckets[hash_ptr(local_storage, smap->bucket_log)];
 }
 
 static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size)
@@ -349,6 +349,7 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
 
 static void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)
 {
+       struct bpf_local_storage *local_storage;
        struct bpf_local_storage_map *smap;
        struct bpf_local_storage_map_bucket *b;
        unsigned long flags;
@@ -357,8 +358,10 @@ static void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)
                /* selem has already be unlinked from smap */
                return;
 
+       local_storage = rcu_dereference_check(selem->local_storage,
+                                             bpf_rcu_lock_held());
        smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());
-       b = select_bucket(smap, selem);
+       b = select_bucket(smap, local_storage);
        raw_spin_lock_irqsave(&b->lock, flags);
        if (likely(selem_linked_to_map(selem)))
                hlist_del_init_rcu(&selem->map_node);
@@ -366,11 +369,13 @@ static void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)
 }
 
 void bpf_selem_link_map(struct bpf_local_storage_map *smap,
+                       struct bpf_local_storage *local_storage,
                        struct bpf_local_storage_elem *selem)
 {
-       struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem);
+       struct bpf_local_storage_map_bucket *b;
        unsigned long flags;
 
+       b = select_bucket(smap, local_storage);
        raw_spin_lock_irqsave(&b->lock, flags);
        hlist_add_head_rcu(&selem->map_node, &b->list);
        raw_spin_unlock_irqrestore(&b->lock, flags);
@@ -448,7 +453,7 @@ int bpf_local_storage_alloc(void *owner,
        storage->use_kmalloc_nolock = smap->use_kmalloc_nolock;
 
        bpf_selem_link_storage_nolock(storage, first_selem);
-       bpf_selem_link_map(smap, first_selem);
+       bpf_selem_link_map(smap, storage, first_selem);
 
        owner_storage_ptr =
                (struct bpf_local_storage **)owner_storage(smap, owner);
@@ -576,7 +581,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
 
        alloc_selem = NULL;
        /* First, link the new selem to the map */
-       bpf_selem_link_map(smap, selem);
+       bpf_selem_link_map(smap, local_storage, selem);
 
        /* Second, link (and publish) the new selem to local_storage */
        bpf_selem_link_storage_nolock(local_storage, selem);

diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
index de111818f3a037a86bf43aa3248bd5d355fb37a7..e36273e4fcbdb0db6d54c5ef788ee8b12cd07062 100644 (file)
--- a/net/core/bpf_sk_storage.c
+++ b/net/core/bpf_sk_storage.c
@@ -191,7 +191,7 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
                }
 
                if (new_sk_storage) {
-                       bpf_selem_link_map(smap, copy_selem);
+                       bpf_selem_link_map(smap, new_sk_storage, copy_selem);
                        bpf_selem_link_storage_nolock(new_sk_storage, copy_selem);
                } else {
                        ret = bpf_local_storage_alloc(newsk, smap, copy_selem, GFP_ATOMIC);