bpf: Introduce SK_BPF_BYPASS_PROT_MEM.

If a socket has sk->sk_bypass_prot_mem flagged, the socket opts out
of the global protocol memory accounting.

This is easily controlled by net.core.bypass_prot_mem sysctl, but it
lacks flexibility.

Let's support flagging (and clearing) sk->sk_bypass_prot_mem via
bpf_setsockopt() at the BPF_CGROUP_INET_SOCK_CREATE hook.

  int val = 1;

  bpf_setsockopt(ctx, SOL_SOCKET, SK_BPF_BYPASS_PROT_MEM,
                 &val, sizeof(val));

As with net.core.bypass_prot_mem, this is inherited to child sockets,
and BPF always takes precedence over sysctl at socket(2) and accept(2).

SK_BPF_BYPASS_PROT_MEM is only supported at BPF_CGROUP_INET_SOCK_CREATE
and not supported on other hooks for some reasons:

  1. UDP charges memory under sk->sk_receive_queue.lock instead
     of lock_sock()

  2. Modifying the flag after skb is charged to sk requires such
     adjustment during bpf_setsockopt() and complicates the logic
     unnecessarily

We can support other hooks later if a real use case justifies that.

Most changes are inline and hard to trace, but a microbenchmark on
__sk_mem_raise_allocated() during neper/tcp_stream showed that more
samples completed faster with sk->sk_bypass_prot_mem == 1.  This will
be more visible under tcp_mem pressure (but it's not a fair comparison).

  # bpftrace -e 'kprobe:__sk_mem_raise_allocated { @start[tid] = nsecs; }
    kretprobe:__sk_mem_raise_allocated /@start[tid]/
    { @end[tid] = nsecs - @start[tid]; @times = hist(@end[tid]); delete(@start[tid]); }'
  # tcp_stream -6 -F 1000 -N -T 256

Without bpf prog:

  [128, 256)          3846 |                                                    |
  [256, 512)       1505326 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
  [512, 1K)        1371006 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@     |
  [1K, 2K)          198207 |@@@@@@                                              |
  [2K, 4K)           31199 |@                                                   |

With bpf prog in the next patch:
  (must be attached before tcp_stream)
  # bpftool prog load sk_bypass_prot_mem.bpf.o /sys/fs/bpf/test type cgroup/sock_create
  # bpftool cgroup attach /sys/fs/cgroup/test cgroup_inet_sock_create pinned /sys/fs/bpf/test

  [128, 256)          6413 |                                                    |
  [256, 512)       1868425 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
  [512, 1K)        1101697 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@                      |
  [1K, 2K)          117031 |@@@@                                                |
  [2K, 4K)           11773 |                                                    |

Signed-off-by: Kuniyuki Iwashima <kuniyu@google.com>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Acked-by: Roman Gushchin <roman.gushchin@linux.dev>
Link: https://patch.msgid.link/20251014235604.3057003-6-kuniyu@google.com

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 6829936d33f58e0b6ef4d65b6f66e59210434d3e..6eb75ad900b133b6c2bcfd00b065b40feef749db 100644 (file)
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -7200,6 +7200,8 @@ enum {
        TCP_BPF_SYN_MAC         = 1007, /* Copy the MAC, IP[46], and TCP header */
        TCP_BPF_SOCK_OPS_CB_FLAGS = 1008, /* Get or Set TCP sock ops flags */
        SK_BPF_CB_FLAGS         = 1009, /* Get or set sock ops flags in socket */
+       SK_BPF_BYPASS_PROT_MEM  = 1010, /* Get or Set sk->sk_bypass_prot_mem */
+
 };
 
 enum {

diff --git a/net/core/filter.c b/net/core/filter.c
index ed3f0e5360595afb482b3d9273e2c197f966c1ff..16105f52927daa47cdeec1852072659b408a8a8f 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -5733,9 +5733,37 @@ static const struct bpf_func_proto bpf_sock_addr_getsockopt_proto = {
        .arg5_type      = ARG_CONST_SIZE,
 };
 
+static int sk_bpf_set_get_bypass_prot_mem(struct sock *sk,
+                                         char *optval, int optlen,
+                                         bool getopt)
+{
+       int val;
+
+       if (optlen != sizeof(int))
+               return -EINVAL;
+
+       if (!sk_has_account(sk))
+               return -EOPNOTSUPP;
+
+       if (getopt) {
+               *(int *)optval = sk->sk_bypass_prot_mem;
+               return 0;
+       }
+
+       val = *(int *)optval;
+       if (val < 0 || val > 1)
+               return -EINVAL;
+
+       sk->sk_bypass_prot_mem = val;
+       return 0;
+}
+
 BPF_CALL_5(bpf_sock_create_setsockopt, struct sock *, sk, int, level,
           int, optname, char *, optval, int, optlen)
 {
+       if (level == SOL_SOCKET && optname == SK_BPF_BYPASS_PROT_MEM)
+               return sk_bpf_set_get_bypass_prot_mem(sk, optval, optlen, false);
+
        return __bpf_setsockopt(sk, level, optname, optval, optlen);
 }
 
@@ -5753,6 +5781,15 @@ static const struct bpf_func_proto bpf_sock_create_setsockopt_proto = {
 BPF_CALL_5(bpf_sock_create_getsockopt, struct sock *, sk, int, level,
           int, optname, char *, optval, int, optlen)
 {
+       if (level == SOL_SOCKET && optname == SK_BPF_BYPASS_PROT_MEM) {
+               int err = sk_bpf_set_get_bypass_prot_mem(sk, optval, optlen, true);
+
+               if (err)
+                       memset(optval, 0, optlen);
+
+               return err;
+       }
+
        return __bpf_getsockopt(sk, level, optname, optval, optlen);
 }
 

diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 6829936d33f58e0b6ef4d65b6f66e59210434d3e..9b17d937edf73698d2407eaf5beb9970a6f21e9d 100644 (file)
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -7200,6 +7200,7 @@ enum {
        TCP_BPF_SYN_MAC         = 1007, /* Copy the MAC, IP[46], and TCP header */
        TCP_BPF_SOCK_OPS_CB_FLAGS = 1008, /* Get or Set TCP sock ops flags */
        SK_BPF_CB_FLAGS         = 1009, /* Get or set sock ops flags in socket */
+       SK_BPF_BYPASS_PROT_MEM  = 1010, /* Get or Set sk->sk_bypass_prot_mem */
 };
 
 enum {