seg6: Use nested-BH locking for seg6_bpf_srh_states.

The access to seg6_bpf_srh_states is protected by disabling preemption.
Based on the code, the entry point is input_action_end_bpf() and
every other function (the bpf helper functions bpf_lwt_seg6_*()), that
is accessing seg6_bpf_srh_states, should be called from within
input_action_end_bpf().

input_action_end_bpf() accesses seg6_bpf_srh_states first at the top of
the function and then disables preemption. This looks wrong because if
preemption needs to be disabled as part of the locking mechanism then
the variable shouldn't be accessed beforehand.

Looking at how it is used via test_lwt_seg6local.sh then
input_action_end_bpf() is always invoked from softirq context. If this
is always the case then the preempt_disable() statement is superfluous.
If this is not always invoked from softirq then disabling only
preemption is not sufficient.

Replace the preempt_disable() statement with nested-BH locking. This is
not an equivalent replacement as it assumes that the invocation of
input_action_end_bpf() always occurs in softirq context and thus the
preempt_disable() is superfluous.
Add a local_lock_t the data structure and use local_lock_nested_bh() for
locking. Add lockdep_assert_held() to ensure the lock is held while the
per-CPU variable is referenced in the helper functions.

Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: David Ahern <dsahern@kernel.org>
Cc: Hao Luo <haoluo@google.com>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: John Fastabend <john.fastabend@gmail.com>
Cc: KP Singh <kpsingh@kernel.org>
Cc: Martin KaFai Lau <martin.lau@linux.dev>
Cc: Song Liu <song@kernel.org>
Cc: Stanislav Fomichev <sdf@google.com>
Cc: Yonghong Song <yonghong.song@linux.dev>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Link: https://patch.msgid.link/20240620132727.660738-13-bigeasy@linutronix.de
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/net/seg6_local.h b/include/net/seg6_local.h
index 3fab9dec2ec45d4290cf61ebfd5af07d4321ade0..888c1ce6f5272d29378f9d67e2c8a65ccab7ec7c 100644 (file)
--- a/include/net/seg6_local.h
+++ b/include/net/seg6_local.h
@@ -19,6 +19,7 @@ extern int seg6_lookup_nexthop(struct sk_buff *skb, struct in6_addr *nhaddr,
 extern bool seg6_bpf_has_valid_srh(struct sk_buff *skb);
 
 struct seg6_bpf_srh_state {
+       local_lock_t bh_lock;
        struct ipv6_sr_hdr *srh;
        u16 hdrlen;
        bool valid;

diff --git a/net/core/filter.c b/net/core/filter.c
index b077e7467946739649e047ba1d788a2f5c34b8a8..c4f5db324f0426cfccba0490d6a4f6f4858dbed8 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -6455,6 +6455,7 @@ BPF_CALL_4(bpf_lwt_seg6_store_bytes, struct sk_buff *, skb, u32, offset,
        void *srh_tlvs, *srh_end, *ptr;
        int srhoff = 0;
 
+       lockdep_assert_held(&srh_state->bh_lock);
        if (srh == NULL)
                return -EINVAL;
 
@@ -6511,6 +6512,7 @@ BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb,
        int hdroff = 0;
        int err;
 
+       lockdep_assert_held(&srh_state->bh_lock);
        switch (action) {
        case SEG6_LOCAL_ACTION_END_X:
                if (!seg6_bpf_has_valid_srh(skb))
@@ -6587,6 +6589,7 @@ BPF_CALL_3(bpf_lwt_seg6_adjust_srh, struct sk_buff *, skb, u32, offset,
        int srhoff = 0;
        int ret;
 
+       lockdep_assert_held(&srh_state->bh_lock);
        if (unlikely(srh == NULL))
                return -EINVAL;
 

diff --git a/net/ipv6/seg6_local.c b/net/ipv6/seg6_local.c
index c434940131b1d0e1621e541636b2067d4177887c..c74705ead9849ff312fd499a6b91eea9cbd36df1 100644 (file)
--- a/net/ipv6/seg6_local.c
+++ b/net/ipv6/seg6_local.c
@@ -1380,7 +1380,9 @@ drop:
        return err;
 }
 
-DEFINE_PER_CPU(struct seg6_bpf_srh_state, seg6_bpf_srh_states);
+DEFINE_PER_CPU(struct seg6_bpf_srh_state, seg6_bpf_srh_states) = {
+       .bh_lock        = INIT_LOCAL_LOCK(bh_lock),
+};
 
 bool seg6_bpf_has_valid_srh(struct sk_buff *skb)
 {
@@ -1388,6 +1390,7 @@ bool seg6_bpf_has_valid_srh(struct sk_buff *skb)
                this_cpu_ptr(&seg6_bpf_srh_states);
        struct ipv6_sr_hdr *srh = srh_state->srh;
 
+       lockdep_assert_held(&srh_state->bh_lock);
        if (unlikely(srh == NULL))
                return false;
 
@@ -1408,8 +1411,7 @@ bool seg6_bpf_has_valid_srh(struct sk_buff *skb)
 static int input_action_end_bpf(struct sk_buff *skb,
                                struct seg6_local_lwt *slwt)
 {
-       struct seg6_bpf_srh_state *srh_state =
-               this_cpu_ptr(&seg6_bpf_srh_states);
+       struct seg6_bpf_srh_state *srh_state;
        struct ipv6_sr_hdr *srh;
        int ret;
 
@@ -1420,10 +1422,14 @@ static int input_action_end_bpf(struct sk_buff *skb,
        }
        advance_nextseg(srh, &ipv6_hdr(skb)->daddr);
 
-       /* preempt_disable is needed to protect the per-CPU buffer srh_state,
-        * which is also accessed by the bpf_lwt_seg6_* helpers
+       /* The access to the per-CPU buffer srh_state is protected by running
+        * always in softirq context (with disabled BH). On PREEMPT_RT the
+        * required locking is provided by the following local_lock_nested_bh()
+        * statement. It is also accessed by the bpf_lwt_seg6_* helpers via
+        * bpf_prog_run_save_cb().
         */
-       preempt_disable();
+       local_lock_nested_bh(&seg6_bpf_srh_states.bh_lock);
+       srh_state = this_cpu_ptr(&seg6_bpf_srh_states);
        srh_state->srh = srh;
        srh_state->hdrlen = srh->hdrlen << 3;
        srh_state->valid = true;
@@ -1446,15 +1452,15 @@ static int input_action_end_bpf(struct sk_buff *skb,
 
        if (srh_state->srh && !seg6_bpf_has_valid_srh(skb))
                goto drop;
+       local_unlock_nested_bh(&seg6_bpf_srh_states.bh_lock);
 
-       preempt_enable();
        if (ret != BPF_REDIRECT)
                seg6_lookup_nexthop(skb, NULL, 0);
 
        return dst_input(skb);
 
 drop:
-       preempt_enable();
+       local_unlock_nested_bh(&seg6_bpf_srh_states.bh_lock);
        kfree_skb(skb);
        return -EINVAL;
 }