bpf: Optimize word-sized keys for resizable hashtable

Specialize the lookup/update/delete paths for keys whose size matches
sizeof(long) (4 bytes on 32-bit, 8 bytes on 64-bit). A static-const
rhashtable_params lets the compiler inline a custom XOR-fold hashfn and
a single-word equality cmpfn, eliminating the indirect jhash dispatch.
The same hashfn and cmpfn are installed into rhashtable's stored params
at rhashtable_init time, so the rehash worker, slow-path inserts, and
rhashtable_next_key() all agree with the inlined fast paths.

The seq_file BPF iterator uses rhashtable_walk_* and is unaffected.

Signed-off-by: Mykyta Yatsenko <yatsenko@meta.com>
Link: https://lore.kernel.org/r/20260605-rhash-v7-7-5b8e05f8630d@meta.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 7b9408b8320c9bc5702bdb190f1e6a70fb7a130e..b4366cad3cfa5dc90b857ee7a09dbff2d2a1148a 100644 (file)
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -2763,6 +2763,31 @@ static inline void *rhtab_elem_value(struct rhtab_elem *l, u32 key_size)
        return l->data + round_up(key_size, 8);
 }
 
+/* Specialize hash function and objcmp for long sized key */
+static __always_inline int rhtab_key_cmp_long(struct rhashtable_compare_arg *arg,
+                                             const void *ptr)
+{
+       const unsigned long key1 = *(const unsigned long *)arg->key;
+       const struct rhtab_elem *key2 = ptr;
+
+       return key1 != *(const unsigned long *)key2->data;
+}
+
+static __always_inline u32 rhtab_hashfn_long(const void *data, u32 len, u32 seed)
+{
+       u64 k = *(const unsigned long *)data;
+
+       return (u32)(k ^ (k >> 32)) ^ seed;
+}
+
+static const struct rhashtable_params rhtab_params_long = {
+       .head_offset = offsetof(struct rhtab_elem, node),
+       .key_offset  = offsetof(struct rhtab_elem, data),
+       .key_len     = sizeof(long),
+       .hashfn      = rhtab_hashfn_long,
+       .obj_cmpfn   = rhtab_key_cmp_long,
+};
+
 static struct bpf_map *rhtab_map_alloc(union bpf_attr *attr)
 {
        struct rhashtable_params params;
@@ -2788,6 +2813,11 @@ static struct bpf_map *rhtab_map_alloc(union bpf_attr *attr)
        params.nelem_hint = (u32)attr->map_extra;
        params.automatic_shrinking = true;
 
+       if (rhtab->map.key_size == sizeof(long)) {
+               params.hashfn = rhtab_hashfn_long;
+               params.obj_cmpfn = rhtab_key_cmp_long;
+       }
+
        err = rhashtable_init(&rhtab->ht, &params);
        if (err)
                goto free_rhtab;
@@ -2878,6 +2908,9 @@ static void *rhtab_lookup_elem(struct bpf_map *map, void *key)
        /* Hold RCU lock in case sleepable program calls via gen_lookup */
        guard(rcu)();
 
+       if (map->key_size == sizeof(long))
+               return rhashtable_lookup_likely(&rhtab->ht, key, rhtab_params_long);
+
        return rhashtable_lookup_likely(&rhtab->ht, key, rhtab_params);
 }
 
@@ -2912,7 +2945,12 @@ static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, v
         * raw tracepoints, which we don't have in rhashtable.
         */
        bpf_disable_instrumentation();
-       err = rhashtable_remove_fast(&rhtab->ht, &elem->node, rhtab_params);
+
+       if (rhtab->map.key_size == sizeof(long))
+               err = rhashtable_remove_fast(&rhtab->ht, &elem->node, rhtab_params_long);
+       else
+               err = rhashtable_remove_fast(&rhtab->ht, &elem->node, rhtab_params);
+
        bpf_enable_instrumentation();
 
        if (err)
@@ -3030,7 +3068,12 @@ static long rhtab_map_update_elem(struct bpf_map *map, void *key, void *value, u
 
        /* Prevent deadlock for NMI programs attempting to take bucket lock */
        bpf_disable_instrumentation();
-       tmp = rhashtable_lookup_get_insert_fast(&rhtab->ht, &elem->node, rhtab_params);
+
+       if (map->key_size == sizeof(long))
+               tmp = rhashtable_lookup_get_insert_fast(&rhtab->ht, &elem->node, rhtab_params_long);
+       else
+               tmp = rhashtable_lookup_get_insert_fast(&rhtab->ht, &elem->node, rhtab_params);
+
        bpf_enable_instrumentation();
 
        if (tmp) {