+++ /dev/null
-From stable+bounces-119651-greg=kroah.com@vger.kernel.org Wed Feb 26 10:40:21 2025
-From: "Ricardo Cañuelo Navarro" <rcn@igalia.com>
-Date: Wed, 26 Feb 2025 10:39:05 +0100
-Subject: net: Reference bpf_redirect_info via task_struct on PREEMPT_RT.
-To: stable@vger.kernel.org, Sebastian Andrzej Siewior <bigeasy@linutronix.de>
-Cc: revest@google.com, kernel-dev@igalia.com, "Andrii Nakryiko" <andrii@kernel.org>, "Eduard Zingerman" <eddyz87@gmail.com>, "Hao Luo" <haoluo@google.com>, "Jiri Olsa" <jolsa@kernel.org>, "John Fastabend" <john.fastabend@gmail.com>, "KP Singh" <kpsingh@kernel.org>, "Martin KaFai Lau" <martin.lau@linux.dev>, "Song Liu" <song@kernel.org>, "Stanislav Fomichev" <sdf@google.com>, "Yonghong Song" <yonghong.song@linux.dev>, "Alexei Starovoitov" <ast@kernel.org>, "Jesper Dangaard Brouer" <hawk@kernel.org>, "Toke Høiland-Jørgensen" <toke@redhat.com>, "Jakub Kicinski" <kuba@kernel.org>
-Message-ID: <20250226-20250204-kasan-slab-use-after-free-read-in-dev_map_enqueue__submit-v3-1-360efec441ba@igalia.com>
-
-From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
-
-[ Upstream commit 401cb7dae8130fd34eb84648e02ab4c506df7d5e ]
-
-The XDP redirect process is two staged:
-- bpf_prog_run_xdp() is invoked to run a eBPF program which inspects the
- packet and makes decisions. While doing that, the per-CPU variable
- bpf_redirect_info is used.
-
-- Afterwards xdp_do_redirect() is invoked and accesses bpf_redirect_info
- and it may also access other per-CPU variables like xskmap_flush_list.
-
-At the very end of the NAPI callback, xdp_do_flush() is invoked which
-does not access bpf_redirect_info but will touch the individual per-CPU
-lists.
-
-The per-CPU variables are only used in the NAPI callback hence disabling
-bottom halves is the only protection mechanism. Users from preemptible
-context (like cpu_map_kthread_run()) explicitly disable bottom halves
-for protections reasons.
-Without locking in local_bh_disable() on PREEMPT_RT this data structure
-requires explicit locking.
-
-PREEMPT_RT has forced-threaded interrupts enabled and every
-NAPI-callback runs in a thread. If each thread has its own data
-structure then locking can be avoided.
-
-Create a struct bpf_net_context which contains struct bpf_redirect_info.
-Define the variable on stack, use bpf_net_ctx_set() to save a pointer to
-it, bpf_net_ctx_clear() removes it again.
-The bpf_net_ctx_set() may nest. For instance a function can be used from
-within NET_RX_SOFTIRQ/ net_rx_action which uses bpf_net_ctx_set() and
-NET_TX_SOFTIRQ which does not. Therefore only the first invocations
-updates the pointer.
-Use bpf_net_ctx_get_ri() as a wrapper to retrieve the current struct
-bpf_redirect_info. The returned data structure is zero initialized to
-ensure nothing is leaked from stack. This is done on first usage of the
-struct. bpf_net_ctx_set() sets bpf_redirect_info::kern_flags to 0 to
-note that initialisation is required. First invocation of
-bpf_net_ctx_get_ri() will memset() the data structure and update
-bpf_redirect_info::kern_flags.
-bpf_redirect_info::nh is excluded from memset because it is only used
-once BPF_F_NEIGH is set which also sets the nh member. The kern_flags is
-moved past nh to exclude it from memset.
-
-The pointer to bpf_net_context is saved task's task_struct. Using
-always the bpf_net_context approach has the advantage that there is
-almost zero differences between PREEMPT_RT and non-PREEMPT_RT builds.
-
-Cc: Andrii Nakryiko <andrii@kernel.org>
-Cc: Eduard Zingerman <eddyz87@gmail.com>
-Cc: Hao Luo <haoluo@google.com>
-Cc: Jiri Olsa <jolsa@kernel.org>
-Cc: John Fastabend <john.fastabend@gmail.com>
-Cc: KP Singh <kpsingh@kernel.org>
-Cc: Martin KaFai Lau <martin.lau@linux.dev>
-Cc: Song Liu <song@kernel.org>
-Cc: Stanislav Fomichev <sdf@google.com>
-Cc: Yonghong Song <yonghong.song@linux.dev>
-Acked-by: Alexei Starovoitov <ast@kernel.org>
-Acked-by: Jesper Dangaard Brouer <hawk@kernel.org>
-Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
-Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
-Link: https://patch.msgid.link/20240620132727.660738-15-bigeasy@linutronix.de
-Signed-off-by: Jakub Kicinski <kuba@kernel.org>
-[rcn: the backport addresses the differences in
-net/core/dev.c:napi_threaded_poll(), busy_poll_stop(), napi_busy_loop()
-and net_rx_action() between upstream and stable. This allows the patch
-to be applied without bringing additional dependencies, such as
-dad6b9770263 ("net: Allow to use SMP threads for backlog NAPI."). These
-changes allow applying the patch on stable without bringing the whole
-related series.
-The rest of the changes are made to adapt context lines and are
-unrelated to the purpose of the patch.]
-Signed-off-by: Ricardo Cañuelo Navarro <rcn@igalia.com>
-Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
----
- include/linux/filter.h | 56 ++++++++++++++++++++++++++++++++++++++++---------
- include/linux/sched.h | 3 ++
- kernel/bpf/cpumap.c | 3 ++
- kernel/bpf/devmap.c | 9 +++++++
- kernel/fork.c | 1
- net/bpf/test_run.c | 11 ++++++++-
- net/core/dev.c | 28 +++++++++++++++++++++++-
- net/core/filter.c | 44 ++++++++++----------------------------
- net/core/lwt_bpf.c | 3 ++
- 9 files changed, 113 insertions(+), 45 deletions(-)
-
---- a/include/linux/filter.h
-+++ b/include/linux/filter.h
-@@ -654,21 +654,59 @@ struct bpf_nh_params {
- };
- };
-
-+/* flags for bpf_redirect_info kern_flags */
-+#define BPF_RI_F_RF_NO_DIRECT BIT(0) /* no napi_direct on return_frame */
-+#define BPF_RI_F_RI_INIT BIT(1)
-+
- struct bpf_redirect_info {
- u64 tgt_index;
- void *tgt_value;
- struct bpf_map *map;
- u32 flags;
-- u32 kern_flags;
- u32 map_id;
- enum bpf_map_type map_type;
- struct bpf_nh_params nh;
-+ u32 kern_flags;
- };
-
--DECLARE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info);
-+struct bpf_net_context {
-+ struct bpf_redirect_info ri;
-+};
-
--/* flags for bpf_redirect_info kern_flags */
--#define BPF_RI_F_RF_NO_DIRECT BIT(0) /* no napi_direct on return_frame */
-+static inline struct bpf_net_context *bpf_net_ctx_set(struct bpf_net_context *bpf_net_ctx)
-+{
-+ struct task_struct *tsk = current;
-+
-+ if (tsk->bpf_net_context != NULL)
-+ return NULL;
-+ bpf_net_ctx->ri.kern_flags = 0;
-+
-+ tsk->bpf_net_context = bpf_net_ctx;
-+ return bpf_net_ctx;
-+}
-+
-+static inline void bpf_net_ctx_clear(struct bpf_net_context *bpf_net_ctx)
-+{
-+ if (bpf_net_ctx)
-+ current->bpf_net_context = NULL;
-+}
-+
-+static inline struct bpf_net_context *bpf_net_ctx_get(void)
-+{
-+ return current->bpf_net_context;
-+}
-+
-+static inline struct bpf_redirect_info *bpf_net_ctx_get_ri(void)
-+{
-+ struct bpf_net_context *bpf_net_ctx = bpf_net_ctx_get();
-+
-+ if (!(bpf_net_ctx->ri.kern_flags & BPF_RI_F_RI_INIT)) {
-+ memset(&bpf_net_ctx->ri, 0, offsetof(struct bpf_net_context, ri.nh));
-+ bpf_net_ctx->ri.kern_flags |= BPF_RI_F_RI_INIT;
-+ }
-+
-+ return &bpf_net_ctx->ri;
-+}
-
- /* Compute the linear packet data range [data, data_end) which
- * will be accessed by various program types (cls_bpf, act_bpf,
-@@ -929,25 +967,23 @@ struct bpf_prog *bpf_patch_insn_single(s
- const struct bpf_insn *patch, u32 len);
- int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt);
-
--void bpf_clear_redirect_map(struct bpf_map *map);
--
- static inline bool xdp_return_frame_no_direct(void)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
-
- return ri->kern_flags & BPF_RI_F_RF_NO_DIRECT;
- }
-
- static inline void xdp_set_return_frame_no_direct(void)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
-
- ri->kern_flags |= BPF_RI_F_RF_NO_DIRECT;
- }
-
- static inline void xdp_clear_return_frame_no_direct(void)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
-
- ri->kern_flags &= ~BPF_RI_F_RF_NO_DIRECT;
- }
-@@ -1503,7 +1539,7 @@ static __always_inline long __bpf_xdp_re
- u64 flags, const u64 flag_mask,
- void *lookup_elem(struct bpf_map *map, u32 key))
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
- const u64 action_mask = XDP_ABORTED | XDP_DROP | XDP_PASS | XDP_TX;
-
- /* Lower bits of the flags are used as return code on lookup failure */
---- a/include/linux/sched.h
-+++ b/include/linux/sched.h
-@@ -45,6 +45,7 @@ struct bio_list;
- struct blk_plug;
- struct bpf_local_storage;
- struct bpf_run_ctx;
-+struct bpf_net_context;
- struct capture_control;
- struct cfs_rq;
- struct fs_struct;
-@@ -1494,6 +1495,8 @@ struct task_struct {
- /* Used for BPF run context */
- struct bpf_run_ctx *bpf_ctx;
- #endif
-+ /* Used by BPF for per-TASK xdp storage */
-+ struct bpf_net_context *bpf_net_context;
-
- #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
- unsigned long lowest_stack;
---- a/kernel/bpf/cpumap.c
-+++ b/kernel/bpf/cpumap.c
-@@ -239,12 +239,14 @@ static int cpu_map_bpf_prog_run(struct b
- int xdp_n, struct xdp_cpumap_stats *stats,
- struct list_head *list)
- {
-+ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
- int nframes;
-
- if (!rcpu->prog)
- return xdp_n;
-
- rcu_read_lock_bh();
-+ bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
-
- nframes = cpu_map_bpf_prog_run_xdp(rcpu, frames, xdp_n, stats);
-
-@@ -254,6 +256,7 @@ static int cpu_map_bpf_prog_run(struct b
- if (unlikely(!list_empty(list)))
- cpu_map_bpf_prog_run_skb(rcpu, list, stats);
-
-+ bpf_net_ctx_clear(bpf_net_ctx);
- rcu_read_unlock_bh(); /* resched point, may call do_softirq() */
-
- return nframes;
---- a/kernel/bpf/devmap.c
-+++ b/kernel/bpf/devmap.c
-@@ -196,7 +196,14 @@ static void dev_map_free(struct bpf_map
- list_del_rcu(&dtab->list);
- spin_unlock(&dev_map_lock);
-
-- bpf_clear_redirect_map(map);
-+ /* bpf_redirect_info->map is assigned in __bpf_xdp_redirect_map()
-+ * during NAPI callback and cleared after the XDP redirect. There is no
-+ * explicit RCU read section which protects bpf_redirect_info->map but
-+ * local_bh_disable() also marks the beginning an RCU section. This
-+ * makes the complete softirq callback RCU protected. Thus after
-+ * following synchronize_rcu() there no bpf_redirect_info->map == map
-+ * assignment.
-+ */
- synchronize_rcu();
-
- /* Make sure prior __dev_map_entry_free() have completed. */
---- a/kernel/fork.c
-+++ b/kernel/fork.c
-@@ -2466,6 +2466,7 @@ __latent_entropy struct task_struct *cop
- RCU_INIT_POINTER(p->bpf_storage, NULL);
- p->bpf_ctx = NULL;
- #endif
-+ p->bpf_net_context = NULL;
-
- /* Perform scheduler related setup. Assign this task to a CPU. */
- retval = sched_fork(clone_flags, p);
---- a/net/bpf/test_run.c
-+++ b/net/bpf/test_run.c
-@@ -282,9 +282,10 @@ static int xdp_recv_frames(struct xdp_fr
- static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog,
- u32 repeat)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
- int err = 0, act, ret, i, nframes = 0, batch_sz;
- struct xdp_frame **frames = xdp->frames;
-+ struct bpf_redirect_info *ri;
- struct xdp_page_head *head;
- struct xdp_frame *frm;
- bool redirect = false;
-@@ -294,6 +295,8 @@ static int xdp_test_run_batch(struct xdp
- batch_sz = min_t(u32, repeat, xdp->batch_size);
-
- local_bh_disable();
-+ bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
-+ ri = bpf_net_ctx_get_ri();
- xdp_set_return_frame_no_direct();
-
- for (i = 0; i < batch_sz; i++) {
-@@ -358,6 +361,7 @@ out:
- }
-
- xdp_clear_return_frame_no_direct();
-+ bpf_net_ctx_clear(bpf_net_ctx);
- local_bh_enable();
- return err;
- }
-@@ -393,6 +397,7 @@ static int bpf_test_run_xdp_live(struct
- static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
- u32 *retval, u32 *time, bool xdp)
- {
-+ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
- struct bpf_prog_array_item item = {.prog = prog};
- struct bpf_run_ctx *old_ctx;
- struct bpf_cg_run_ctx run_ctx;
-@@ -418,10 +423,14 @@ static int bpf_test_run(struct bpf_prog
- do {
- run_ctx.prog_item = &item;
- local_bh_disable();
-+ bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
-+
- if (xdp)
- *retval = bpf_prog_run_xdp(prog, ctx);
- else
- *retval = bpf_prog_run(prog, ctx);
-+
-+ bpf_net_ctx_clear(bpf_net_ctx);
- local_bh_enable();
- } while (bpf_test_timer_continue(&t, 1, repeat, &ret, time));
- bpf_reset_run_ctx(old_ctx);
---- a/net/core/dev.c
-+++ b/net/core/dev.c
-@@ -4050,10 +4050,13 @@ sch_handle_ingress(struct sk_buff *skb,
- struct net_device *orig_dev, bool *another)
- {
- struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress);
-+ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
- int sch_ret;
-
- if (!entry)
- return skb;
-+
-+ bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
- if (*pt_prev) {
- *ret = deliver_skb(skb, *pt_prev, orig_dev);
- *pt_prev = NULL;
-@@ -4082,10 +4085,12 @@ ingress_verdict:
- break;
- }
- *ret = NET_RX_SUCCESS;
-+ bpf_net_ctx_clear(bpf_net_ctx);
- return NULL;
- case TC_ACT_SHOT:
- kfree_skb_reason(skb, SKB_DROP_REASON_TC_INGRESS);
- *ret = NET_RX_DROP;
-+ bpf_net_ctx_clear(bpf_net_ctx);
- return NULL;
- /* used by tc_run */
- case TC_ACT_STOLEN:
-@@ -4095,8 +4100,10 @@ ingress_verdict:
- fallthrough;
- case TC_ACT_CONSUMED:
- *ret = NET_RX_SUCCESS;
-+ bpf_net_ctx_clear(bpf_net_ctx);
- return NULL;
- }
-+ bpf_net_ctx_clear(bpf_net_ctx);
-
- return skb;
- }
-@@ -4105,11 +4112,14 @@ static __always_inline struct sk_buff *
- sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
- {
- struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress);
-+ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
- int sch_ret;
-
- if (!entry)
- return skb;
-
-+ bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
-+
- /* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was
- * already set by the caller.
- */
-@@ -4125,10 +4135,12 @@ egress_verdict:
- /* No need to push/pop skb's mac_header here on egress! */
- skb_do_redirect(skb);
- *ret = NET_XMIT_SUCCESS;
-+ bpf_net_ctx_clear(bpf_net_ctx);
- return NULL;
- case TC_ACT_SHOT:
- kfree_skb_reason(skb, SKB_DROP_REASON_TC_EGRESS);
- *ret = NET_XMIT_DROP;
-+ bpf_net_ctx_clear(bpf_net_ctx);
- return NULL;
- /* used by tc_run */
- case TC_ACT_STOLEN:
-@@ -4138,8 +4150,10 @@ egress_verdict:
- fallthrough;
- case TC_ACT_CONSUMED:
- *ret = NET_XMIT_SUCCESS;
-+ bpf_net_ctx_clear(bpf_net_ctx);
- return NULL;
- }
-+ bpf_net_ctx_clear(bpf_net_ctx);
-
- return skb;
- }
-@@ -6242,6 +6256,7 @@ static void __busy_poll_stop(struct napi
- static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll,
- u16 budget)
- {
-+ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
- bool skip_schedule = false;
- unsigned long timeout;
- int rc;
-@@ -6259,6 +6274,7 @@ static void busy_poll_stop(struct napi_s
- clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
-
- local_bh_disable();
-+ bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
-
- if (prefer_busy_poll) {
- napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
-@@ -6281,6 +6297,7 @@ static void busy_poll_stop(struct napi_s
- netpoll_poll_unlock(have_poll_lock);
- if (rc == budget)
- __busy_poll_stop(napi, skip_schedule);
-+ bpf_net_ctx_clear(bpf_net_ctx);
- local_bh_enable();
- }
-
-@@ -6290,6 +6307,7 @@ void napi_busy_loop(unsigned int napi_id
- {
- unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
- int (*napi_poll)(struct napi_struct *napi, int budget);
-+ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
- void *have_poll_lock = NULL;
- struct napi_struct *napi;
-
-@@ -6308,6 +6326,7 @@ restart:
- int work = 0;
-
- local_bh_disable();
-+ bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
- if (!napi_poll) {
- unsigned long val = READ_ONCE(napi->state);
-
-@@ -6337,6 +6356,7 @@ count:
- if (work > 0)
- __NET_ADD_STATS(dev_net(napi->dev),
- LINUX_MIB_BUSYPOLLRXPACKETS, work);
-+ bpf_net_ctx_clear(bpf_net_ctx);
- local_bh_enable();
-
- if (!loop_end || loop_end(loop_end_arg, start_time))
-@@ -6725,6 +6745,7 @@ static void skb_defer_free_flush(struct
-
- static int napi_threaded_poll(void *data)
- {
-+ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
- struct napi_struct *napi = data;
- struct softnet_data *sd;
- void *have;
-@@ -6736,6 +6757,7 @@ static int napi_threaded_poll(void *data
- bool repoll = false;
-
- local_bh_disable();
-+ bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
- sd = this_cpu_ptr(&softnet_data);
- sd->in_napi_threaded_poll = true;
-
-@@ -6751,6 +6773,7 @@ static int napi_threaded_poll(void *data
- net_rps_action_and_irq_enable(sd);
- }
- skb_defer_free_flush(sd);
-+ bpf_net_ctx_clear(bpf_net_ctx);
- local_bh_enable();
-
- if (!repoll)
-@@ -6768,10 +6791,12 @@ static __latent_entropy void net_rx_acti
- struct softnet_data *sd = this_cpu_ptr(&softnet_data);
- unsigned long time_limit = jiffies +
- usecs_to_jiffies(READ_ONCE(netdev_budget_usecs));
-+ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
- int budget = READ_ONCE(netdev_budget);
- LIST_HEAD(list);
- LIST_HEAD(repoll);
-
-+ bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
- start:
- sd->in_net_rx_action = true;
- local_irq_disable();
-@@ -6824,7 +6849,8 @@ start:
- sd->in_net_rx_action = false;
-
- net_rps_action_and_irq_enable(sd);
--end:;
-+end:
-+ bpf_net_ctx_clear(bpf_net_ctx);
- }
-
- struct netdev_adjacent {
---- a/net/core/filter.c
-+++ b/net/core/filter.c
-@@ -2471,12 +2471,9 @@ static const struct bpf_func_proto bpf_c
- .arg3_type = ARG_ANYTHING,
- };
-
--DEFINE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info);
--EXPORT_PER_CPU_SYMBOL_GPL(bpf_redirect_info);
--
- int skb_do_redirect(struct sk_buff *skb)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
- struct net *net = dev_net(skb->dev);
- struct net_device *dev;
- u32 flags = ri->flags;
-@@ -2512,7 +2509,7 @@ out_drop:
-
- BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
-
- if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL)))
- return TC_ACT_SHOT;
-@@ -2533,7 +2530,7 @@ static const struct bpf_func_proto bpf_r
-
- BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
-
- if (unlikely(flags))
- return TC_ACT_SHOT;
-@@ -2555,7 +2552,7 @@ static const struct bpf_func_proto bpf_r
- BPF_CALL_4(bpf_redirect_neigh, u32, ifindex, struct bpf_redir_neigh *, params,
- int, plen, u64, flags)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
-
- if (unlikely((plen && plen < sizeof(*params)) || flags))
- return TC_ACT_SHOT;
-@@ -4297,30 +4294,13 @@ void xdp_do_flush(void)
- }
- EXPORT_SYMBOL_GPL(xdp_do_flush);
-
--void bpf_clear_redirect_map(struct bpf_map *map)
--{
-- struct bpf_redirect_info *ri;
-- int cpu;
--
-- for_each_possible_cpu(cpu) {
-- ri = per_cpu_ptr(&bpf_redirect_info, cpu);
-- /* Avoid polluting remote cacheline due to writes if
-- * not needed. Once we pass this test, we need the
-- * cmpxchg() to make sure it hasn't been changed in
-- * the meantime by remote CPU.
-- */
-- if (unlikely(READ_ONCE(ri->map) == map))
-- cmpxchg(&ri->map, map, NULL);
-- }
--}
--
- DEFINE_STATIC_KEY_FALSE(bpf_master_redirect_enabled_key);
- EXPORT_SYMBOL_GPL(bpf_master_redirect_enabled_key);
-
- u32 xdp_master_redirect(struct xdp_buff *xdp)
- {
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
- struct net_device *master, *slave;
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-
- master = netdev_master_upper_dev_get_rcu(xdp->rxq->dev);
- slave = master->netdev_ops->ndo_xdp_get_xmit_slave(master, xdp);
-@@ -4392,7 +4372,7 @@ static __always_inline int __xdp_do_redi
- map = READ_ONCE(ri->map);
-
- /* The map pointer is cleared when the map is being torn
-- * down by bpf_clear_redirect_map()
-+ * down by dev_map_free()
- */
- if (unlikely(!map)) {
- err = -ENOENT;
-@@ -4437,7 +4417,7 @@ err:
- int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp,
- struct bpf_prog *xdp_prog)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
- enum bpf_map_type map_type = ri->map_type;
-
- if (map_type == BPF_MAP_TYPE_XSKMAP)
-@@ -4451,7 +4431,7 @@ EXPORT_SYMBOL_GPL(xdp_do_redirect);
- int xdp_do_redirect_frame(struct net_device *dev, struct xdp_buff *xdp,
- struct xdp_frame *xdpf, struct bpf_prog *xdp_prog)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
- enum bpf_map_type map_type = ri->map_type;
-
- if (map_type == BPF_MAP_TYPE_XSKMAP)
-@@ -4468,7 +4448,7 @@ static int xdp_do_generic_redirect_map(s
- enum bpf_map_type map_type, u32 map_id,
- u32 flags)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
- struct bpf_map *map;
- int err;
-
-@@ -4480,7 +4460,7 @@ static int xdp_do_generic_redirect_map(s
- map = READ_ONCE(ri->map);
-
- /* The map pointer is cleared when the map is being torn
-- * down by bpf_clear_redirect_map()
-+ * down by dev_map_free()
- */
- if (unlikely(!map)) {
- err = -ENOENT;
-@@ -4522,7 +4502,7 @@ err:
- int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
- struct xdp_buff *xdp, struct bpf_prog *xdp_prog)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
- enum bpf_map_type map_type = ri->map_type;
- void *fwd = ri->tgt_value;
- u32 map_id = ri->map_id;
-@@ -4558,7 +4538,7 @@ err:
-
- BPF_CALL_2(bpf_xdp_redirect, u32, ifindex, u64, flags)
- {
-- struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-+ struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
-
- if (unlikely(flags))
- return XDP_ABORTED;
---- a/net/core/lwt_bpf.c
-+++ b/net/core/lwt_bpf.c
-@@ -38,6 +38,7 @@ static inline struct bpf_lwt *bpf_lwt_lw
- static int run_lwt_bpf(struct sk_buff *skb, struct bpf_lwt_prog *lwt,
- struct dst_entry *dst, bool can_redirect)
- {
-+ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
- int ret;
-
- /* Migration disable and BH disable are needed to protect per-cpu
-@@ -45,6 +46,7 @@ static int run_lwt_bpf(struct sk_buff *s
- */
- migrate_disable();
- local_bh_disable();
-+ bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
- bpf_compute_data_pointers(skb);
- ret = bpf_prog_run_save_cb(lwt->prog, skb);
-
-@@ -77,6 +79,7 @@ static int run_lwt_bpf(struct sk_buff *s
- break;
- }
-
-+ bpf_net_ctx_clear(bpf_net_ctx);
- local_bh_enable();
- migrate_enable();
-
projects / thirdparty / kernel / stable-queue.git / commitdiff
