bpf: Add "bool swap_uptrs" arg to bpf_local_storage_update() and bpf_selem_alloc()

In a later patch, the task local storage will only accept uptr
from the syscall update_elem and will not accept uptr from
the bpf prog. The reason is the bpf prog does not have a way
to provide a valid user space address.

bpf_local_storage_update() and bpf_selem_alloc() are used by
both bpf prog bpf_task_storage_get(BPF_LOCAL_STORAGE_GET_F_CREATE)
and bpf syscall update_elem. "bool swap_uptrs" arg is added
to bpf_local_storage_update() and bpf_selem_alloc() to tell if
it is called by the bpf prog or by the bpf syscall. When
swap_uptrs==true, it is called by the syscall.

The arg is named (swap_)uptrs because the later patch will swap
the uptrs between the newly allocated selem and the user space
provided map_value. It will make error handling easier in case
map->ops->map_update_elem() fails and the caller can decide
if it needs to unpin the uptr in the user space provided
map_value or the bpf_local_storage_update() has already
taken the uptr ownership and will take care of unpinning it also.

Only swap_uptrs==false is passed now. The logic to handle
the true case will be added in a later patch.

Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20241023234759.860539-4-martin.lau@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h
index dcddb0aef7d8cb3ddb30f7d0206056840bd1b2d4..0c7216c065d5411fef47cc1a8fede86ab8fd4f17 100644 (file)
--- a/include/linux/bpf_local_storage.h
+++ b/include/linux/bpf_local_storage.h
@@ -181,7 +181,7 @@ void bpf_selem_link_map(struct bpf_local_storage_map *smap,
 
 struct bpf_local_storage_elem *
 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
-               bool charge_mem, gfp_t gfp_flags);
+               bool charge_mem, bool swap_uptrs, gfp_t gfp_flags);
 
 void bpf_selem_free(struct bpf_local_storage_elem *selem,
                    struct bpf_local_storage_map *smap,
@@ -195,7 +195,7 @@ bpf_local_storage_alloc(void *owner,
 
 struct bpf_local_storage_data *
 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
-                        void *value, u64 map_flags, gfp_t gfp_flags);
+                        void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags);
 
 u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map);
 

diff --git a/kernel/bpf/bpf_cgrp_storage.c b/kernel/bpf/bpf_cgrp_storage.c
index 28efd0a3f2200c98d9f5ac9a6db918c7ddea6bc7..20f05de92e9c3d34ecc6fbda2c95ad0180df0c2f 100644 (file)
--- a/kernel/bpf/bpf_cgrp_storage.c
+++ b/kernel/bpf/bpf_cgrp_storage.c
@@ -107,7 +107,7 @@ static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key,
 
        bpf_cgrp_storage_lock();
        sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
-                                        value, map_flags, GFP_ATOMIC);
+                                        value, map_flags, false, GFP_ATOMIC);
        bpf_cgrp_storage_unlock();
        cgroup_put(cgroup);
        return PTR_ERR_OR_ZERO(sdata);
@@ -181,7 +181,7 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
        if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
            (flags & BPF_LOCAL_STORAGE_GET_F_CREATE))
                sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
-                                                value, BPF_NOEXIST, gfp_flags);
+                                                value, BPF_NOEXIST, false, gfp_flags);
 
 unlock:
        bpf_cgrp_storage_unlock();

diff --git a/kernel/bpf/bpf_inode_storage.c b/kernel/bpf/bpf_inode_storage.c
index 29da6d3838f678f6a79c6362ba4bdb2a08725728..44ccebc745e53f5f6983ae3cc013cacd4612cbf6 100644 (file)
--- a/kernel/bpf/bpf_inode_storage.c
+++ b/kernel/bpf/bpf_inode_storage.c
@@ -100,7 +100,7 @@ static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
 
        sdata = bpf_local_storage_update(file_inode(fd_file(f)),
                                         (struct bpf_local_storage_map *)map,
-                                        value, map_flags, GFP_ATOMIC);
+                                        value, map_flags, false, GFP_ATOMIC);
        return PTR_ERR_OR_ZERO(sdata);
 }
 
@@ -154,7 +154,7 @@ BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
        if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
                sdata = bpf_local_storage_update(
                        inode, (struct bpf_local_storage_map *)map, value,
-                       BPF_NOEXIST, gfp_flags);
+                       BPF_NOEXIST, false, gfp_flags);
                return IS_ERR(sdata) ? (unsigned long)NULL :
                                             (unsigned long)sdata->data;
        }

diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c
index c938dea5ddbf3a7900ee25eeedc076e2bd69c0b5..1cf772cb26eb03c6d27f2dc2aa2de46c7884d9bf 100644 (file)
--- a/kernel/bpf/bpf_local_storage.c
+++ b/kernel/bpf/bpf_local_storage.c
@@ -73,7 +73,7 @@ static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem)
 
 struct bpf_local_storage_elem *
 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,
-               void *value, bool charge_mem, gfp_t gfp_flags)
+               void *value, bool charge_mem, bool swap_uptrs, gfp_t gfp_flags)
 {
        struct bpf_local_storage_elem *selem;
 
@@ -524,7 +524,7 @@ uncharge:
  */
 struct bpf_local_storage_data *
 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
-                        void *value, u64 map_flags, gfp_t gfp_flags)
+                        void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags)
 {
        struct bpf_local_storage_data *old_sdata = NULL;
        struct bpf_local_storage_elem *alloc_selem, *selem = NULL;
@@ -550,7 +550,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
                if (err)
                        return ERR_PTR(err);
 
-               selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);
+               selem = bpf_selem_alloc(smap, owner, value, true, swap_uptrs, gfp_flags);
                if (!selem)
                        return ERR_PTR(-ENOMEM);
 
@@ -584,7 +584,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
        /* A lookup has just been done before and concluded a new selem is
         * needed. The chance of an unnecessary alloc is unlikely.
         */
-       alloc_selem = selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);
+       alloc_selem = selem = bpf_selem_alloc(smap, owner, value, true, swap_uptrs, gfp_flags);
        if (!alloc_selem)
                return ERR_PTR(-ENOMEM);
 

diff --git a/kernel/bpf/bpf_task_storage.c b/kernel/bpf/bpf_task_storage.c
index adf6dfe0ba68a4fe34c9a6a8a408b2c5631caba7..45dc3ca334d37a6d4516aba845b5e14b4d5a5936 100644 (file)
--- a/kernel/bpf/bpf_task_storage.c
+++ b/kernel/bpf/bpf_task_storage.c
@@ -147,7 +147,7 @@ static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key,
        bpf_task_storage_lock();
        sdata = bpf_local_storage_update(
                task, (struct bpf_local_storage_map *)map, value, map_flags,
-               GFP_ATOMIC);
+               false, GFP_ATOMIC);
        bpf_task_storage_unlock();
 
        err = PTR_ERR_OR_ZERO(sdata);
@@ -219,7 +219,7 @@ static void *__bpf_task_storage_get(struct bpf_map *map,
            (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy) {
                sdata = bpf_local_storage_update(
                        task, (struct bpf_local_storage_map *)map, value,
-                       BPF_NOEXIST, gfp_flags);
+                       BPF_NOEXIST, false, gfp_flags);
                return IS_ERR(sdata) ? NULL : sdata->data;
        }
 

diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
index bc01b3aa6b0fad4de6986fd854ad05bdc48960fa..2f4ed83a75ae6c59bebe6ab21e17d5d8e1e4febe 100644 (file)
--- a/net/core/bpf_sk_storage.c
+++ b/net/core/bpf_sk_storage.c
@@ -106,7 +106,7 @@ static long bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,
        if (sock) {
                sdata = bpf_local_storage_update(
                        sock->sk, (struct bpf_local_storage_map *)map, value,
-                       map_flags, GFP_ATOMIC);
+                       map_flags, false, GFP_ATOMIC);
                sockfd_put(sock);
                return PTR_ERR_OR_ZERO(sdata);
        }
@@ -137,7 +137,7 @@ bpf_sk_storage_clone_elem(struct sock *newsk,
 {
        struct bpf_local_storage_elem *copy_selem;
 
-       copy_selem = bpf_selem_alloc(smap, newsk, NULL, true, GFP_ATOMIC);
+       copy_selem = bpf_selem_alloc(smap, newsk, NULL, true, false, GFP_ATOMIC);
        if (!copy_selem)
                return NULL;
 
@@ -243,7 +243,7 @@ BPF_CALL_5(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
            refcount_inc_not_zero(&sk->sk_refcnt)) {
                sdata = bpf_local_storage_update(
                        sk, (struct bpf_local_storage_map *)map, value,
-                       BPF_NOEXIST, gfp_flags);
+                       BPF_NOEXIST, false, gfp_flags);
                /* sk must be a fullsock (guaranteed by verifier),
                 * so sock_gen_put() is unnecessary.
                 */