mptcp: introduce mptcp-level backlog

We are soon using it for incoming data processing.
MPTCP can't leverage the sk_backlog, as the latter is processed
before the release callback, and such callback for MPTCP releases
and re-acquire the socket spinlock, breaking the sk_backlog processing
assumption.

Add a skb backlog list inside the mptcp sock struct, and implement
basic helper to transfer packet to and purge such list.

Packets in the backlog are memory accounted and still use the incoming
subflow receive memory, to allow back-pressure. The backlog size is
implicitly bounded to the sum of subflows rcvbuf.

When a subflow is closed, references from the backlog to such sock
are removed.

No packet is currently added to the backlog, so no functional changes
intended here.

Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Reviewed-by: Mat Martineau <martineau@kernel.org>
Signed-off-by: Matthieu Baerts (NGI0) <matttbe@kernel.org>
Link: https://patch.msgid.link/20251121-net-next-mptcp-memcg-backlog-imp-v1-13-1f34b6c1e0b1@kernel.org
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/net/mptcp/mptcp_diag.c b/net/mptcp/mptcp_diag.c
index ac974299de71cdf85cba7d848d14a241f5ff4dc8..136c2d05c0eeb8f7050344a63396396d8718206c 100644 (file)
--- a/net/mptcp/mptcp_diag.c
+++ b/net/mptcp/mptcp_diag.c
@@ -195,7 +195,8 @@ static void mptcp_diag_get_info(struct sock *sk, struct inet_diag_msg *r,
        struct mptcp_sock *msk = mptcp_sk(sk);
        struct mptcp_info *info = _info;
 
-       r->idiag_rqueue = sk_rmem_alloc_get(sk);
+       r->idiag_rqueue = sk_rmem_alloc_get(sk) +
+                         READ_ONCE(mptcp_sk(sk)->backlog_len);
        r->idiag_wqueue = sk_wmem_alloc_get(sk);
 
        if (inet_sk_state_load(sk) == TCP_LISTEN) {

diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c
index f5526855a2e5ab4f56f30f84e17124bf89347975..dfed036e05918adcbe216fbc3c65b71a178c9e44 100644 (file)
--- a/net/mptcp/protocol.c
+++ b/net/mptcp/protocol.c
@@ -659,6 +659,39 @@ static void mptcp_dss_corruption(struct mptcp_sock *msk, struct sock *ssk)
        }
 }
 
+static void __mptcp_add_backlog(struct sock *sk,
+                               struct mptcp_subflow_context *subflow,
+                               struct sk_buff *skb)
+{
+       struct mptcp_sock *msk = mptcp_sk(sk);
+       struct sk_buff *tail = NULL;
+       bool fragstolen;
+       int delta;
+
+       if (unlikely(sk->sk_state == TCP_CLOSE)) {
+               kfree_skb_reason(skb, SKB_DROP_REASON_SOCKET_CLOSE);
+               return;
+       }
+
+       /* Try to coalesce with the last skb in our backlog */
+       if (!list_empty(&msk->backlog_list))
+               tail = list_last_entry(&msk->backlog_list, struct sk_buff, list);
+
+       if (tail && MPTCP_SKB_CB(skb)->map_seq == MPTCP_SKB_CB(tail)->end_seq &&
+           skb->sk == tail->sk &&
+           __mptcp_try_coalesce(sk, tail, skb, &fragstolen, &delta)) {
+               skb->truesize -= delta;
+               kfree_skb_partial(skb, fragstolen);
+               __mptcp_subflow_lend_fwdmem(subflow, delta);
+               WRITE_ONCE(msk->backlog_len, msk->backlog_len + delta);
+               return;
+       }
+
+       list_add_tail(&skb->list, &msk->backlog_list);
+       mptcp_subflow_lend_fwdmem(subflow, skb);
+       WRITE_ONCE(msk->backlog_len, msk->backlog_len + skb->truesize);
+}
+
 static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk,
                                           struct sock *ssk)
 {
@@ -705,8 +738,13 @@ static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk,
                        size_t len = skb->len - offset;
 
                        mptcp_init_skb(ssk, skb, offset, len);
-                       mptcp_subflow_lend_fwdmem(subflow, skb);
-                       ret = __mptcp_move_skb(sk, skb) || ret;
+
+                       if (true) {
+                               mptcp_subflow_lend_fwdmem(subflow, skb);
+                               ret |= __mptcp_move_skb(sk, skb);
+                       } else {
+                               __mptcp_add_backlog(sk, subflow, skb);
+                       }
                        seq += len;
 
                        if (unlikely(map_remaining < len)) {
@@ -2531,6 +2569,9 @@ out:
 void mptcp_close_ssk(struct sock *sk, struct sock *ssk,
                     struct mptcp_subflow_context *subflow)
 {
+       struct mptcp_sock *msk = mptcp_sk(sk);
+       struct sk_buff *skb;
+
        /* The first subflow can already be closed and still in the list */
        if (subflow->close_event_done)
                return;
@@ -2540,6 +2581,17 @@ void mptcp_close_ssk(struct sock *sk, struct sock *ssk,
        if (sk->sk_state == TCP_ESTABLISHED)
                mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL);
 
+       /* Remove any reference from the backlog to this ssk; backlog skbs consume
+        * space in the msk receive queue, no need to touch sk->sk_rmem_alloc
+        */
+       list_for_each_entry(skb, &msk->backlog_list, list) {
+               if (skb->sk != ssk)
+                       continue;
+
+               atomic_sub(skb->truesize, &skb->sk->sk_rmem_alloc);
+               skb->sk = NULL;
+       }
+
        /* subflow aborted before reaching the fully_established status
         * attempt the creation of the next subflow
         */
@@ -2769,12 +2821,31 @@ static void mptcp_mp_fail_no_response(struct mptcp_sock *msk)
        unlock_sock_fast(ssk, slow);
 }
 
+static void mptcp_backlog_purge(struct sock *sk)
+{
+       struct mptcp_sock *msk = mptcp_sk(sk);
+       struct sk_buff *tmp, *skb;
+       LIST_HEAD(backlog);
+
+       mptcp_data_lock(sk);
+       list_splice_init(&msk->backlog_list, &backlog);
+       msk->backlog_len = 0;
+       mptcp_data_unlock(sk);
+
+       list_for_each_entry_safe(skb, tmp, &backlog, list) {
+               mptcp_borrow_fwdmem(sk, skb);
+               kfree_skb_reason(skb, SKB_DROP_REASON_SOCKET_CLOSE);
+       }
+       sk_mem_reclaim(sk);
+}
+
 static void mptcp_do_fastclose(struct sock *sk)
 {
        struct mptcp_subflow_context *subflow, *tmp;
        struct mptcp_sock *msk = mptcp_sk(sk);
 
        mptcp_set_state(sk, TCP_CLOSE);
+       mptcp_backlog_purge(sk);
 
        /* Explicitly send the fastclose reset as need */
        if (__mptcp_check_fallback(msk))
@@ -2853,11 +2924,13 @@ static void __mptcp_init_sock(struct sock *sk)
        INIT_LIST_HEAD(&msk->conn_list);
        INIT_LIST_HEAD(&msk->join_list);
        INIT_LIST_HEAD(&msk->rtx_queue);
+       INIT_LIST_HEAD(&msk->backlog_list);
        INIT_WORK(&msk->work, mptcp_worker);
        msk->out_of_order_queue = RB_ROOT;
        msk->first_pending = NULL;
        msk->timer_ival = TCP_RTO_MIN;
        msk->scaling_ratio = TCP_DEFAULT_SCALING_RATIO;
+       msk->backlog_len = 0;
 
        WRITE_ONCE(msk->first, NULL);
        inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss;
@@ -3234,6 +3307,7 @@ static void mptcp_destroy_common(struct mptcp_sock *msk)
        struct sock *sk = (struct sock *)msk;
 
        __mptcp_clear_xmit(sk);
+       mptcp_backlog_purge(sk);
 
        /* join list will be eventually flushed (with rst) at sock lock release time */
        mptcp_for_each_subflow_safe(msk, subflow, tmp)

diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index 5e2749d92a495ca9d9dd412bd8a15ea3ca9120b8..fe0dca4122f2edc4a07efada59f0d1cc824e27fa 100644 (file)
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -357,6 +357,9 @@ struct mptcp_sock {
                                         * allow_infinite_fallback and
                                         * allow_join
                                         */
+
+       struct list_head backlog_list;  /* protected by the data lock */
+       u32             backlog_len;
 };
 
 #define mptcp_data_lock(sk) spin_lock_bh(&(sk)->sk_lock.slock)
@@ -407,6 +410,7 @@ static inline int mptcp_space_from_win(const struct sock *sk, int win)
 static inline int __mptcp_space(const struct sock *sk)
 {
        return mptcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) -
+                                   READ_ONCE(mptcp_sk(sk)->backlog_len) -
                                    sk_rmem_alloc_get(sk));
 }
 
@@ -655,23 +659,32 @@ static inline void mptcp_borrow_fwdmem(struct sock *sk, struct sk_buff *skb)
 {
        struct sock *ssk = skb->sk;
 
-       /* The subflow just lend the skb fwd memory, and we know that the skb
-        * is only accounted on the incoming subflow rcvbuf.
+       /* The subflow just lend the skb fwd memory; if the subflow meanwhile
+        * closed, mptcp_close_ssk() already released the ssk rcv memory.
         */
        DEBUG_NET_WARN_ON_ONCE(skb->destructor);
-       skb->sk = NULL;
        sk_forward_alloc_add(sk, skb->truesize);
+       if (!ssk)
+               return;
+
        atomic_sub(skb->truesize, &ssk->sk_rmem_alloc);
+       skb->sk = NULL;
+}
+
+static inline void
+__mptcp_subflow_lend_fwdmem(struct mptcp_subflow_context *subflow, int size)
+{
+       int frag = (subflow->lent_mem_frag + size) & (PAGE_SIZE - 1);
+
+       subflow->lent_mem_frag = frag;
 }
 
 static inline void
 mptcp_subflow_lend_fwdmem(struct mptcp_subflow_context *subflow,
                          struct sk_buff *skb)
 {
-       int frag = (subflow->lent_mem_frag + skb->truesize) & (PAGE_SIZE - 1);
-
+       __mptcp_subflow_lend_fwdmem(subflow, skb->truesize);
        skb->destructor = NULL;
-       subflow->lent_mem_frag = frag;
 }
 
 static inline u64