bpf: sk_msg, fix socket data_ready events

[ Upstream commit 552de91068828daef50a227a665068cf8dde835e ]

When a skb verdict program is in-use and either another BPF program
redirects to that socket or the new SK_PASS support is used the
data_ready callback does not wake up application. Instead because
the stream parser/verdict is using the sk data_ready callback we wake
up the stream parser/verdict block.

Fix this by adding a helper to check if the stream parser block is
enabled on the sk and if so call the saved pointer which is the
upper layers wake up function.

This fixes application stalls observed when an application is waiting
for data in a blocking read().

Fixes: d829e9c4112b ("tls: convert to generic sk_msg interface")
Signed-off-by: John Fastabend <john.fastabend@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h
index 2a11e9d91dfa808b8ba4efe7bcf511179389741b..36bd8586308122ce71ba7c292a3113444061e593 100644 (file)
--- a/include/linux/skmsg.h
+++ b/include/linux/skmsg.h
@@ -416,6 +416,14 @@ static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock)
                sk_psock_drop(sk, psock);
 }
 
+static inline void sk_psock_data_ready(struct sock *sk, struct sk_psock *psock)
+{
+       if (psock->parser.enabled)
+               psock->parser.saved_data_ready(sk);
+       else
+               sk->sk_data_ready(sk);
+}
+
 static inline void psock_set_prog(struct bpf_prog **pprog,
                                  struct bpf_prog *prog)
 {

diff --git a/net/core/skmsg.c b/net/core/skmsg.c
index fc7d59f1c57caa0201580d70b03f6ae3352d3060..54d854807630691e021d8c9f92c2152dba5da14e 100644 (file)
--- a/net/core/skmsg.c
+++ b/net/core/skmsg.c
@@ -406,7 +406,7 @@ static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb)
        msg->skb = skb;
 
        sk_psock_queue_msg(psock, msg);
-       sk->sk_data_ready(sk);
+       sk_psock_data_ready(sk, psock);
        return copied;
 }
 
@@ -739,7 +739,7 @@ static int sk_psock_strp_parse(struct strparser *strp, struct sk_buff *skb)
 }
 
 /* Called with socket lock held. */
-static void sk_psock_data_ready(struct sock *sk)
+static void sk_psock_strp_data_ready(struct sock *sk)
 {
        struct sk_psock *psock;
 
@@ -787,7 +787,7 @@ void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
                return;
 
        parser->saved_data_ready = sk->sk_data_ready;
-       sk->sk_data_ready = sk_psock_data_ready;
+       sk->sk_data_ready = sk_psock_strp_data_ready;
        sk->sk_write_space = sk_psock_write_space;
        parser->enabled = true;
 }

diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c
index 3b45fe530f91e2e1aa697888e11a78cf7e9d211e..2d6bd7a59b2cd5f027cbe59168aae944691d7cfe 100644 (file)
--- a/net/ipv4/tcp_bpf.c
+++ b/net/ipv4/tcp_bpf.c
@@ -198,7 +198,7 @@ static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock,
                msg->sg.start = i;
                msg->sg.size -= apply_bytes;
                sk_psock_queue_msg(psock, tmp);
-               sk->sk_data_ready(sk);
+               sk_psock_data_ready(sk, psock);
        } else {
                sk_msg_free(sk, tmp);
                kfree(tmp);