io_uring/net: move charging socket out of zc io_uring

Currently, io_uring's io_sg_from_iter() duplicates the part of
__zerocopy_sg_from_iter() charging pages to the socket. It'd be too easy
to miss while changing it in net/, the chunk is not the most
straightforward for outside users and full of internal implementation
details. io_uring is not a good place to keep it, deduplicate it by
moving out of the callback into __zerocopy_sg_from_iter().

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index f4cda3fbdb75ab3add0f57eb2444dc65db054d78..9c29bdd5596d89a5e5d8a4b6b01435a8c9d6b765 100644 (file)
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -1703,6 +1703,9 @@ int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
                            struct sk_buff *skb, struct iov_iter *from,
                            size_t length);
 
+int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
+                               struct iov_iter *from, size_t length);
+
 static inline int skb_zerocopy_iter_dgram(struct sk_buff *skb,
                                          struct msghdr *msg, int len)
 {

diff --git a/include/linux/socket.h b/include/linux/socket.h
index 89d16b90370bd412c30a755bd1324f5f8c72aad1..2a1ff91d1914e0dd817d3b610f83591badedb78c 100644 (file)
--- a/include/linux/socket.h
+++ b/include/linux/socket.h
@@ -76,7 +76,7 @@ struct msghdr {
        __kernel_size_t msg_controllen; /* ancillary data buffer length */
        struct kiocb    *msg_iocb;      /* ptr to iocb for async requests */
        struct ubuf_info *msg_ubuf;
-       int (*sg_from_iter)(struct sock *sk, struct sk_buff *skb,
+       int (*sg_from_iter)(struct sk_buff *skb,
                            struct iov_iter *from, size_t length);
 };
 

diff --git a/io_uring/net.c b/io_uring/net.c
index 7c98c4d5094633cb948bb761d2345e65cfb380f2..84a7602bcef10256e91d48b2491f5e4ea87d6331 100644 (file)
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -1265,14 +1265,14 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
        return io_sendmsg_prep_setup(req, req->opcode == IORING_OP_SENDMSG_ZC);
 }
 
-static int io_sg_from_iter_iovec(struct sock *sk, struct sk_buff *skb,
+static int io_sg_from_iter_iovec(struct sk_buff *skb,
                                 struct iov_iter *from, size_t length)
 {
        skb_zcopy_downgrade_managed(skb);
-       return __zerocopy_sg_from_iter(NULL, sk, skb, from, length);
+       return zerocopy_fill_skb_from_iter(skb, from, length);
 }
 
-static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb,
+static int io_sg_from_iter(struct sk_buff *skb,
                           struct iov_iter *from, size_t length)
 {
        struct skb_shared_info *shinfo = skb_shinfo(skb);
@@ -1285,7 +1285,7 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb,
        if (!frag)
                shinfo->flags |= SKBFL_MANAGED_FRAG_REFS;
        else if (unlikely(!skb_zcopy_managed(skb)))
-               return __zerocopy_sg_from_iter(NULL, sk, skb, from, length);
+               return zerocopy_fill_skb_from_iter(skb, from, length);
 
        bi.bi_size = min(from->count, length);
        bi.bi_bvec_done = from->iov_offset;
@@ -1312,14 +1312,6 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb,
        skb->data_len += copied;
        skb->len += copied;
        skb->truesize += truesize;
-
-       if (sk && sk->sk_type == SOCK_STREAM) {
-               sk_wmem_queued_add(sk, truesize);
-               if (!skb_zcopy_pure(skb))
-                       sk_mem_charge(sk, truesize);
-       } else {
-               refcount_add(truesize, &skb->sk->sk_wmem_alloc);
-       }
        return ret;
 }
 

diff --git a/net/core/datagram.c b/net/core/datagram.c
index b0dccefd4a09eee64c2f672c88a9224e2a4c2500..16507b7cc4fb09274f535d80d653b717ab5ba73e 100644 (file)
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -610,8 +610,8 @@ fault:
 }
 EXPORT_SYMBOL(skb_copy_datagram_from_iter);
 
-static int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
-                                       struct iov_iter *from, size_t length)
+int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
+                               struct iov_iter *from, size_t length)
 {
        int frag = skb_shinfo(skb)->nr_frags;
 
@@ -687,11 +687,11 @@ int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
        int ret;
 
        if (msg && msg->msg_ubuf && msg->sg_from_iter)
-               return msg->sg_from_iter(sk, skb, from, length);
+               ret = msg->sg_from_iter(skb, from, length);
+       else
+               ret = zerocopy_fill_skb_from_iter(skb, from, length);
 
-       ret = zerocopy_fill_skb_from_iter(skb, from, length);
        truesize = skb->truesize - orig_size;
-
        if (sk && sk->sk_type == SOCK_STREAM) {
                sk_wmem_queued_add(sk, truesize);
                if (!skb_zcopy_pure(skb))