[thirdparty/kernel/stable.git] / net / core / dev.h

/* SPDX-License-Identifier: GPL-2.0-or-later */
#ifndef _NET_CORE_DEV_H
#define _NET_CORE_DEV_H

#include <linux/cleanup.h>
#include <linux/types.h>
#include <linux/rwsem.h>
#include <linux/netdevice.h>
#include <net/netdev_lock.h>

struct net;
struct netlink_ext_ack;
struct cpumask;

/* Random bits of netdevice that don't need to be exposed */
#define FLOW_LIMIT_HISTORY	(1 << 7)  /* must be ^2 and !overflow buckets */
struct sd_flow_limit {
	struct rcu_head		rcu;
	unsigned int		count;
	u8			log_buckets;
	unsigned int		history_head;
	u16			history[FLOW_LIMIT_HISTORY];
	u8			buckets[];
};

extern int netdev_flow_limit_table_len;

struct napi_struct *
netdev_napi_by_id_lock(struct net *net, unsigned int napi_id);
struct net_device *dev_get_by_napi_id(unsigned int napi_id);

struct net_device *netdev_get_by_index_lock(struct net *net, int ifindex);
struct net_device *__netdev_put_lock(struct net_device *dev, struct net *net);
struct net_device *
netdev_xa_find_lock(struct net *net, struct net_device *dev,
		    unsigned long *index);

DEFINE_FREE(netdev_unlock, struct net_device *, if (_T) netdev_unlock(_T));

#define for_each_netdev_lock_scoped(net, var_name, ifindex)		\
	for (struct net_device *var_name __free(netdev_unlock) = NULL;	\
	     (var_name = netdev_xa_find_lock(net, var_name, &ifindex)); \
	     ifindex++)

struct net_device *
netdev_get_by_index_lock_ops_compat(struct net *net, int ifindex);
struct net_device *
netdev_xa_find_lock_ops_compat(struct net *net, struct net_device *dev,
			       unsigned long *index);

DEFINE_FREE(netdev_unlock_ops_compat, struct net_device *,
	    if (_T) netdev_unlock_ops_compat(_T));

#define for_each_netdev_lock_ops_compat_scoped(net, var_name, ifindex)	\
	for (struct net_device *var_name __free(netdev_unlock_ops_compat) = NULL; \
	     (var_name = netdev_xa_find_lock_ops_compat(net, var_name,	\
							&ifindex));	\
	     ifindex++)

#ifdef CONFIG_PROC_FS
int __init dev_proc_init(void);
#else
#define dev_proc_init() 0
#endif

void linkwatch_init_dev(struct net_device *dev);
void linkwatch_run_queue(void);

void dev_addr_flush(struct net_device *dev);
int dev_addr_init(struct net_device *dev);
void dev_addr_check(struct net_device *dev);

#if IS_ENABLED(CONFIG_NET_SHAPER)
void net_shaper_flush_netdev(struct net_device *dev);
void net_shaper_set_real_num_tx_queues(struct net_device *dev,
				       unsigned int txq);
#else
static inline void net_shaper_flush_netdev(struct net_device *dev) {}
static inline void net_shaper_set_real_num_tx_queues(struct net_device *dev,
						     unsigned int txq) {}
#endif

/* sysctls not referred to from outside net/core/ */
extern int		netdev_unregister_timeout_secs;
extern int		weight_p;
extern int		dev_weight_rx_bias;
extern int		dev_weight_tx_bias;

extern struct rw_semaphore dev_addr_sem;

/* rtnl helpers */
extern struct list_head net_todo_list;
void netdev_run_todo(void);

/* netdev management, shared between various uAPI entry points */
struct netdev_name_node {
	struct hlist_node hlist;
	struct list_head list;
	struct net_device *dev;
	const char *name;
	struct rcu_head rcu;
};

int netdev_get_name(struct net *net, char *name, int ifindex);
int netif_change_name(struct net_device *dev, const char *newname);
int dev_change_name(struct net_device *dev, const char *newname);

#define netdev_for_each_altname(dev, namenode)				\
	list_for_each_entry((namenode), &(dev)->name_node->list, list)
#define netdev_for_each_altname_safe(dev, namenode, next)		\
	list_for_each_entry_safe((namenode), (next), &(dev)->name_node->list, \
				 list)

int netdev_name_node_alt_create(struct net_device *dev, const char *name);
int netdev_name_node_alt_destroy(struct net_device *dev, const char *name);

int dev_validate_mtu(struct net_device *dev, int mtu,
		     struct netlink_ext_ack *extack);
int netif_set_mtu_ext(struct net_device *dev, int new_mtu,
		      struct netlink_ext_ack *extack);

int dev_get_phys_port_id(struct net_device *dev,
			 struct netdev_phys_item_id *ppid);
int dev_get_phys_port_name(struct net_device *dev,
			   char *name, size_t len);

int netif_change_proto_down(struct net_device *dev, bool proto_down);
int dev_change_proto_down(struct net_device *dev, bool proto_down);
void netdev_change_proto_down_reason_locked(struct net_device *dev,
					    unsigned long mask, u32 value);

typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf);
int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
		      int fd, int expected_fd, u32 flags);

int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len);
int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len);
void netif_set_group(struct net_device *dev, int new_group);
void dev_set_group(struct net_device *dev, int new_group);
int netif_change_carrier(struct net_device *dev, bool new_carrier);
int dev_change_carrier(struct net_device *dev, bool new_carrier);

void __dev_set_rx_mode(struct net_device *dev);

void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
			unsigned int gchanges, u32 portid,
			const struct nlmsghdr *nlh);

void unregister_netdevice_many_notify(struct list_head *head,
				      u32 portid, const struct nlmsghdr *nlh);

static inline void netif_set_up(struct net_device *dev, bool value)
{
	if (value)
		dev->flags |= IFF_UP;
	else
		dev->flags &= ~IFF_UP;

	if (!netdev_need_ops_lock(dev))
		netdev_lock(dev);
	dev->up = value;
	if (!netdev_need_ops_lock(dev))
		netdev_unlock(dev);
}

static inline void netif_set_gso_max_size(struct net_device *dev,
					  unsigned int size)
{
	/* dev->gso_max_size is read locklessly from sk_setup_caps() */
	WRITE_ONCE(dev->gso_max_size, size);
	if (size <= GSO_LEGACY_MAX_SIZE)
		WRITE_ONCE(dev->gso_ipv4_max_size, size);
}

static inline void netif_set_gso_max_segs(struct net_device *dev,
					  unsigned int segs)
{
	/* dev->gso_max_segs is read locklessly from sk_setup_caps() */
	WRITE_ONCE(dev->gso_max_segs, segs);
}

static inline void netif_set_gro_max_size(struct net_device *dev,
					  unsigned int size)
{
	/* This pairs with the READ_ONCE() in skb_gro_receive() */
	WRITE_ONCE(dev->gro_max_size, size);
	if (size <= GRO_LEGACY_MAX_SIZE)
		WRITE_ONCE(dev->gro_ipv4_max_size, size);
}

static inline void netif_set_gso_ipv4_max_size(struct net_device *dev,
					       unsigned int size)
{
	/* dev->gso_ipv4_max_size is read locklessly from sk_setup_caps() */
	WRITE_ONCE(dev->gso_ipv4_max_size, size);
}

static inline void netif_set_gro_ipv4_max_size(struct net_device *dev,
					       unsigned int size)
{
	/* This pairs with the READ_ONCE() in skb_gro_receive() */
	WRITE_ONCE(dev->gro_ipv4_max_size, size);
}

/**
 * napi_get_defer_hard_irqs - get the NAPI's defer_hard_irqs
 * @n: napi struct to get the defer_hard_irqs field from
 *
 * Return: the per-NAPI value of the defar_hard_irqs field.
 */
static inline u32 napi_get_defer_hard_irqs(const struct napi_struct *n)
{
	return READ_ONCE(n->defer_hard_irqs);
}

/**
 * napi_set_defer_hard_irqs - set the defer_hard_irqs for a napi
 * @n: napi_struct to set the defer_hard_irqs field
 * @defer: the value the field should be set to
 */
static inline void napi_set_defer_hard_irqs(struct napi_struct *n, u32 defer)
{
	WRITE_ONCE(n->defer_hard_irqs, defer);
}

/**
 * netdev_set_defer_hard_irqs - set defer_hard_irqs for all NAPIs of a netdev
 * @netdev: the net_device for which all NAPIs will have defer_hard_irqs set
 * @defer: the defer_hard_irqs value to set
 */
static inline void netdev_set_defer_hard_irqs(struct net_device *netdev,
					      u32 defer)
{
	unsigned int count = max(netdev->num_rx_queues,
				 netdev->num_tx_queues);
	struct napi_struct *napi;
	int i;

	WRITE_ONCE(netdev->napi_defer_hard_irqs, defer);
	list_for_each_entry(napi, &netdev->napi_list, dev_list)
		napi_set_defer_hard_irqs(napi, defer);

	for (i = 0; i < count; i++)
		netdev->napi_config[i].defer_hard_irqs = defer;
}

/**
 * napi_get_gro_flush_timeout - get the gro_flush_timeout
 * @n: napi struct to get the gro_flush_timeout from
 *
 * Return: the per-NAPI value of the gro_flush_timeout field.
 */
static inline unsigned long
napi_get_gro_flush_timeout(const struct napi_struct *n)
{
	return READ_ONCE(n->gro_flush_timeout);
}

/**
 * napi_set_gro_flush_timeout - set the gro_flush_timeout for a napi
 * @n: napi struct to set the gro_flush_timeout
 * @timeout: timeout value to set
 *
 * napi_set_gro_flush_timeout sets the per-NAPI gro_flush_timeout
 */
static inline void napi_set_gro_flush_timeout(struct napi_struct *n,
					      unsigned long timeout)
{
	WRITE_ONCE(n->gro_flush_timeout, timeout);
}

/**
 * netdev_set_gro_flush_timeout - set gro_flush_timeout of a netdev's NAPIs
 * @netdev: the net_device for which all NAPIs will have gro_flush_timeout set
 * @timeout: the timeout value to set
 */
static inline void netdev_set_gro_flush_timeout(struct net_device *netdev,
						unsigned long timeout)
{
	unsigned int count = max(netdev->num_rx_queues,
				 netdev->num_tx_queues);
	struct napi_struct *napi;
	int i;

	WRITE_ONCE(netdev->gro_flush_timeout, timeout);
	list_for_each_entry(napi, &netdev->napi_list, dev_list)
		napi_set_gro_flush_timeout(napi, timeout);

	for (i = 0; i < count; i++)
		netdev->napi_config[i].gro_flush_timeout = timeout;
}

/**
 * napi_get_irq_suspend_timeout - get the irq_suspend_timeout
 * @n: napi struct to get the irq_suspend_timeout from
 *
 * Return: the per-NAPI value of the irq_suspend_timeout field.
 */
static inline unsigned long
napi_get_irq_suspend_timeout(const struct napi_struct *n)
{
	return READ_ONCE(n->irq_suspend_timeout);
}

/**
 * napi_set_irq_suspend_timeout - set the irq_suspend_timeout for a napi
 * @n: napi struct to set the irq_suspend_timeout
 * @timeout: timeout value to set
 *
 * napi_set_irq_suspend_timeout sets the per-NAPI irq_suspend_timeout
 */
static inline void napi_set_irq_suspend_timeout(struct napi_struct *n,
						unsigned long timeout)
{
	WRITE_ONCE(n->irq_suspend_timeout, timeout);
}

int rps_cpumask_housekeeping(struct cpumask *mask);

#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
void xdp_do_check_flushed(struct napi_struct *napi);
#else
static inline void xdp_do_check_flushed(struct napi_struct *napi) { }
#endif

/* Best effort check that NAPI is not idle (can't be scheduled to run) */
static inline void napi_assert_will_not_race(const struct napi_struct *napi)
{
	/* uninitialized instance, can't race */
	if (!napi->poll_list.next)
		return;

	/* SCHED bit is set on disabled instances */
	WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state));
	WARN_ON(READ_ONCE(napi->list_owner) != -1);
}

void kick_defer_list_purge(struct softnet_data *sd, unsigned int cpu);

#define XMIT_RECURSION_LIMIT	8

#ifndef CONFIG_PREEMPT_RT
static inline bool dev_xmit_recursion(void)
{
	return unlikely(__this_cpu_read(softnet_data.xmit.recursion) >
			XMIT_RECURSION_LIMIT);
}

static inline void dev_xmit_recursion_inc(void)
{
	__this_cpu_inc(softnet_data.xmit.recursion);
}

static inline void dev_xmit_recursion_dec(void)
{
	__this_cpu_dec(softnet_data.xmit.recursion);
}
#else
static inline bool dev_xmit_recursion(void)
{
	return unlikely(current->net_xmit.recursion > XMIT_RECURSION_LIMIT);
}

static inline void dev_xmit_recursion_inc(void)
{
	current->net_xmit.recursion++;
}

static inline void dev_xmit_recursion_dec(void)
{
	current->net_xmit.recursion--;
}
#endif

int dev_set_hwtstamp_phylib(struct net_device *dev,
			    struct kernel_hwtstamp_config *cfg,
			    struct netlink_ext_ack *extack);
int dev_get_hwtstamp_phylib(struct net_device *dev,
			    struct kernel_hwtstamp_config *cfg);
int net_hwtstamp_validate(const struct kernel_hwtstamp_config *cfg);

#endif
Commit	Line	Data
6264f58c JK	1	/* SPDX-License-Identifier: GPL-2.0-or-later */
	2	#ifndef _NET_CORE_DEV_H
	3	#define _NET_CORE_DEV_H
	4
2628f495	5	#include <linux/cleanup.h>
6264f58c	6	#include <linux/types.h>
c7d52737	7	#include <linux/rwsem.h>
b9495b56	8	#include <linux/netdevice.h>
8ef890df	9	#include <net/netdev_lock.h>
6264f58c JK	10
6264f58c JK	11	struct net;
6264f58c	12	struct netlink_ext_ack;
370ca718	13	struct cpumask;
6264f58c JK	14
	15	/* Random bits of netdevice that don't need to be exposed */
	16	#define FLOW_LIMIT_HISTORY (1 << 7) /* must be ^2 and !overflow buckets */
	17	struct sd_flow_limit {
0a7de4a8	18	struct rcu_head rcu;
7b6f0a85	19	unsigned int count;
c3025e94	20	u8 log_buckets;
6264f58c JK	21	unsigned int history_head;
	22	u16 history[FLOW_LIMIT_HISTORY];
	23	u8 buckets[];
	24	};
	25
	26	extern int netdev_flow_limit_table_len;
	27
2628f495 JK	28	struct napi_struct *
2628f495 JK	29	netdev_napi_by_id_lock(struct net *net, unsigned int napi_id);
21520e74	30	struct net_device *dev_get_by_napi_id(unsigned int napi_id);
d1cacd74	31
2628f495	32	struct net_device netdev_get_by_index_lock(struct net net, int ifindex);
a82dc19d	33	struct net_device __netdev_put_lock(struct net_device dev, struct net *net);
2628f495 JK	34	struct net_device *
	35	netdev_xa_find_lock(struct net net, struct net_device dev,
	36	unsigned long *index);
	37
	38	DEFINE_FREE(netdev_unlock, struct net_device *, if (_T) netdev_unlock(_T));
	39
	40	#define for_each_netdev_lock_scoped(net, var_name, ifindex) \
	41	for (struct net_device *var_name __free(netdev_unlock) = NULL; \
	42	(var_name = netdev_xa_find_lock(net, var_name, &ifindex)); \
	43	ifindex++)
	44
4ec9031c JK	45	struct net_device *
	46	netdev_get_by_index_lock_ops_compat(struct net *net, int ifindex);
	47	struct net_device *
	48	netdev_xa_find_lock_ops_compat(struct net net, struct net_device dev,
	49	unsigned long *index);
	50
	51	DEFINE_FREE(netdev_unlock_ops_compat, struct net_device *,
	52	if (_T) netdev_unlock_ops_compat(_T));
	53
	54	#define for_each_netdev_lock_ops_compat_scoped(net, var_name, ifindex) \
	55	for (struct net_device *var_name __free(netdev_unlock_ops_compat) = NULL; \
	56	(var_name = netdev_xa_find_lock_ops_compat(net, var_name, \
	57	&ifindex)); \
	58	ifindex++)
	59
6264f58c JK	60	#ifdef CONFIG_PROC_FS
	61	int __init dev_proc_init(void);
	62	#else
	63	#define dev_proc_init() 0
	64	#endif
	65
	66	void linkwatch_init_dev(struct net_device *dev);
6264f58c JK	67	void linkwatch_run_queue(void);
	68
	69	void dev_addr_flush(struct net_device *dev);
	70	int dev_addr_init(struct net_device *dev);
	71	void dev_addr_check(struct net_device *dev);
	72
4b623f9f PA	73	#if IS_ENABLED(CONFIG_NET_SHAPER)
4b623f9f PA	74	void net_shaper_flush_netdev(struct net_device *dev);
ff7d4deb PA	75	void net_shaper_set_real_num_tx_queues(struct net_device *dev,
ff7d4deb PA	76	unsigned int txq);
4b623f9f PA	77	#else
4b623f9f PA	78	static inline void net_shaper_flush_netdev(struct net_device *dev) {}
ff7d4deb PA	79	static inline void net_shaper_set_real_num_tx_queues(struct net_device *dev,
ff7d4deb PA	80	unsigned int txq) {}
4b623f9f PA	81	#endif
4b623f9f PA	82
6264f58c	83	/* sysctls not referred to from outside net/core/ */
6264f58c JK	84	extern int netdev_unregister_timeout_secs;
	85	extern int weight_p;
	86	extern int dev_weight_rx_bias;
	87	extern int dev_weight_tx_bias;
	88
8033d2ae SF	89	extern struct rw_semaphore dev_addr_sem;
8033d2ae SF	90
6264f58c JK	91	/* rtnl helpers */
	92	extern struct list_head net_todo_list;
	93	void netdev_run_todo(void);
	94
	95	/* netdev management, shared between various uAPI entry points */
	96	struct netdev_name_node {
	97	struct hlist_node hlist;
	98	struct list_head list;
	99	struct net_device *dev;
	100	const char *name;
723de3eb	101	struct rcu_head rcu;
6264f58c JK	102	};
	103
	104	int netdev_get_name(struct net net, char name, int ifindex);
7e4d784f	105	int netif_change_name(struct net_device dev, const char newname);
6264f58c JK	106	int dev_change_name(struct net_device dev, const char newname);
6264f58c JK	107
7663d522 JK	108	#define netdev_for_each_altname(dev, namenode) \
7663d522 JK	109	list_for_each_entry((namenode), &(dev)->name_node->list, list)
d09486a0 JK	110	#define netdev_for_each_altname_safe(dev, namenode, next) \
	111	list_for_each_entry_safe((namenode), (next), &(dev)->name_node->list, \
	112	list)
7663d522	113
6264f58c JK	114	int netdev_name_node_alt_create(struct net_device dev, const char name);
	115	int netdev_name_node_alt_destroy(struct net_device dev, const char name);
	116
	117	int dev_validate_mtu(struct net_device *dev, int mtu,
	118	struct netlink_ext_ack *extack);
7e4d784f SF	119	int netif_set_mtu_ext(struct net_device *dev, int new_mtu,
7e4d784f SF	120	struct netlink_ext_ack *extack);
6264f58c JK	121
	122	int dev_get_phys_port_id(struct net_device *dev,
	123	struct netdev_phys_item_id *ppid);
	124	int dev_get_phys_port_name(struct net_device *dev,
	125	char *name, size_t len);
	126
7e4d784f	127	int netif_change_proto_down(struct net_device *dev, bool proto_down);
6264f58c	128	int dev_change_proto_down(struct net_device *dev, bool proto_down);
7e4d784f SF	129	void netdev_change_proto_down_reason_locked(struct net_device *dev,
7e4d784f SF	130	unsigned long mask, u32 value);
6264f58c JK	131
	132	typedef int (bpf_op_t)(struct net_device dev, struct netdev_bpf *bpf);
	133	int dev_change_xdp_fd(struct net_device dev, struct netlink_ext_ack extack,
	134	int fd, int expected_fd, u32 flags);
	135
7e4d784f	136	int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len);
6264f58c	137	int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len);
7e4d784f	138	void netif_set_group(struct net_device *dev, int new_group);
6264f58c	139	void dev_set_group(struct net_device *dev, int new_group);
7e4d784f	140	int netif_change_carrier(struct net_device *dev, bool new_carrier);
6264f58c JK	141	int dev_change_carrier(struct net_device *dev, bool new_carrier);
	142
	143	void __dev_set_rx_mode(struct net_device *dev);
	144
1d997f10 HL	145	void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
	146	unsigned int gchanges, u32 portid,
	147	const struct nlmsghdr *nlh);
	148
77f4aa9a HL	149	void unregister_netdevice_many_notify(struct list_head *head,
	150	u32 portid, const struct nlmsghdr *nlh);
	151
5112457f JK	152	static inline void netif_set_up(struct net_device *dev, bool value)
	153	{
	154	if (value)
	155	dev->flags \|= IFF_UP;
	156	else
	157	dev->flags &= ~IFF_UP;
	158
d4c22ec6 SF	159	if (!netdev_need_ops_lock(dev))
d4c22ec6 SF	160	netdev_lock(dev);
5112457f	161	dev->up = value;
d4c22ec6 SF	162	if (!netdev_need_ops_lock(dev))
d4c22ec6 SF	163	netdev_unlock(dev);
5112457f JK	164	}
5112457f JK	165
744d49da JK	166	static inline void netif_set_gso_max_size(struct net_device *dev,
	167	unsigned int size)
	168	{
	169	/* dev->gso_max_size is read locklessly from sk_setup_caps() */
	170	WRITE_ONCE(dev->gso_max_size, size);
9eefedd5 XL	171	if (size <= GSO_LEGACY_MAX_SIZE)
9eefedd5 XL	172	WRITE_ONCE(dev->gso_ipv4_max_size, size);
744d49da JK	173	}
	174
	175	static inline void netif_set_gso_max_segs(struct net_device *dev,
	176	unsigned int segs)
	177	{
	178	/* dev->gso_max_segs is read locklessly from sk_setup_caps() */
	179	WRITE_ONCE(dev->gso_max_segs, segs);
	180	}
	181
	182	static inline void netif_set_gro_max_size(struct net_device *dev,
	183	unsigned int size)
	184	{
	185	/* This pairs with the READ_ONCE() in skb_gro_receive() */
	186	WRITE_ONCE(dev->gro_max_size, size);
9eefedd5 XL	187	if (size <= GRO_LEGACY_MAX_SIZE)
	188	WRITE_ONCE(dev->gro_ipv4_max_size, size);
	189	}
	190
	191	static inline void netif_set_gso_ipv4_max_size(struct net_device *dev,
	192	unsigned int size)
	193	{
	194	/* dev->gso_ipv4_max_size is read locklessly from sk_setup_caps() */
	195	WRITE_ONCE(dev->gso_ipv4_max_size, size);
	196	}
	197
	198	static inline void netif_set_gro_ipv4_max_size(struct net_device *dev,
	199	unsigned int size)
	200	{
	201	/* This pairs with the READ_ONCE() in skb_gro_receive() */
	202	WRITE_ONCE(dev->gro_ipv4_max_size, size);
744d49da JK	203	}
744d49da JK	204
f15e3b3d JD	205	/**
	206	* napi_get_defer_hard_irqs - get the NAPI's defer_hard_irqs
	207	* @n: napi struct to get the defer_hard_irqs field from
	208	*
	209	* Return: the per-NAPI value of the defar_hard_irqs field.
	210	*/
	211	static inline u32 napi_get_defer_hard_irqs(const struct napi_struct *n)
	212	{
	213	return READ_ONCE(n->defer_hard_irqs);
	214	}
	215
	216	/**
	217	* napi_set_defer_hard_irqs - set the defer_hard_irqs for a napi
	218	* @n: napi_struct to set the defer_hard_irqs field
	219	* @defer: the value the field should be set to
	220	*/
	221	static inline void napi_set_defer_hard_irqs(struct napi_struct *n, u32 defer)
	222	{
	223	WRITE_ONCE(n->defer_hard_irqs, defer);
	224	}
	225
	226	/**
	227	* netdev_set_defer_hard_irqs - set defer_hard_irqs for all NAPIs of a netdev
	228	* @netdev: the net_device for which all NAPIs will have defer_hard_irqs set
	229	* @defer: the defer_hard_irqs value to set
	230	*/
	231	static inline void netdev_set_defer_hard_irqs(struct net_device *netdev,
	232	u32 defer)
	233	{
86e25f40 JD	234	unsigned int count = max(netdev->num_rx_queues,
86e25f40 JD	235	netdev->num_tx_queues);
f15e3b3d	236	struct napi_struct *napi;
86e25f40	237	int i;
f15e3b3d JD	238
	239	WRITE_ONCE(netdev->napi_defer_hard_irqs, defer);
	240	list_for_each_entry(napi, &netdev->napi_list, dev_list)
	241	napi_set_defer_hard_irqs(napi, defer);
86e25f40 JD	242
	243	for (i = 0; i < count; i++)
	244	netdev->napi_config[i].defer_hard_irqs = defer;
f15e3b3d JD	245	}
f15e3b3d JD	246
acb8d4ed JD	247	/**
	248	* napi_get_gro_flush_timeout - get the gro_flush_timeout
	249	* @n: napi struct to get the gro_flush_timeout from
	250	*
	251	* Return: the per-NAPI value of the gro_flush_timeout field.
	252	*/
	253	static inline unsigned long
	254	napi_get_gro_flush_timeout(const struct napi_struct *n)
	255	{
	256	return READ_ONCE(n->gro_flush_timeout);
	257	}
	258
	259	/**
	260	* napi_set_gro_flush_timeout - set the gro_flush_timeout for a napi
	261	* @n: napi struct to set the gro_flush_timeout
	262	* @timeout: timeout value to set
	263	*
	264	* napi_set_gro_flush_timeout sets the per-NAPI gro_flush_timeout
	265	*/
	266	static inline void napi_set_gro_flush_timeout(struct napi_struct *n,
	267	unsigned long timeout)
	268	{
	269	WRITE_ONCE(n->gro_flush_timeout, timeout);
	270	}
	271
	272	/**
	273	* netdev_set_gro_flush_timeout - set gro_flush_timeout of a netdev's NAPIs
	274	* @netdev: the net_device for which all NAPIs will have gro_flush_timeout set
	275	* @timeout: the timeout value to set
	276	*/
	277	static inline void netdev_set_gro_flush_timeout(struct net_device *netdev,
	278	unsigned long timeout)
	279	{
86e25f40 JD	280	unsigned int count = max(netdev->num_rx_queues,
86e25f40 JD	281	netdev->num_tx_queues);
acb8d4ed	282	struct napi_struct *napi;
86e25f40	283	int i;
acb8d4ed JD	284
	285	WRITE_ONCE(netdev->gro_flush_timeout, timeout);
	286	list_for_each_entry(napi, &netdev->napi_list, dev_list)
	287	napi_set_gro_flush_timeout(napi, timeout);
86e25f40 JD	288
	289	for (i = 0; i < count; i++)
	290	netdev->napi_config[i].gro_flush_timeout = timeout;
acb8d4ed JD	291	}
acb8d4ed JD	292
5dc51ec8 MK	293	/**
	294	* napi_get_irq_suspend_timeout - get the irq_suspend_timeout
	295	* @n: napi struct to get the irq_suspend_timeout from
	296	*
	297	* Return: the per-NAPI value of the irq_suspend_timeout field.
	298	*/
	299	static inline unsigned long
	300	napi_get_irq_suspend_timeout(const struct napi_struct *n)
	301	{
	302	return READ_ONCE(n->irq_suspend_timeout);
	303	}
	304
	305	/**
	306	* napi_set_irq_suspend_timeout - set the irq_suspend_timeout for a napi
	307	* @n: napi struct to set the irq_suspend_timeout
	308	* @timeout: timeout value to set
	309	*
	310	* napi_set_irq_suspend_timeout sets the per-NAPI irq_suspend_timeout
	311	*/
	312	static inline void napi_set_irq_suspend_timeout(struct napi_struct *n,
	313	unsigned long timeout)
	314	{
	315	WRITE_ONCE(n->irq_suspend_timeout, timeout);
	316	}
	317
370ca718	318	int rps_cpumask_housekeeping(struct cpumask *mask);
9a675ba5 SAS	319
	320	#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
	321	void xdp_do_check_flushed(struct napi_struct *napi);
	322	#else
	323	static inline void xdp_do_check_flushed(struct napi_struct *napi) { }
	324	#endif
27f91aaf	325
c1e00bc4 JK	326	/* Best effort check that NAPI is not idle (can't be scheduled to run) */
	327	static inline void napi_assert_will_not_race(const struct napi_struct *napi)
	328	{
	329	/* uninitialized instance, can't race */
	330	if (!napi->poll_list.next)
	331	return;
	332
	333	/* SCHED bit is set on disabled instances */
	334	WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state));
	335	WARN_ON(READ_ONCE(napi->list_owner) != -1);
	336	}
	337
b9495b56 ED	338	void kick_defer_list_purge(struct softnet_data *sd, unsigned int cpu);
b9495b56 ED	339
2fe50a4d	340	#define XMIT_RECURSION_LIMIT 8
ecefbc09 SAS	341
ecefbc09 SAS	342	#ifndef CONFIG_PREEMPT_RT
2fe50a4d ED	343	static inline bool dev_xmit_recursion(void)
	344	{
	345	return unlikely(__this_cpu_read(softnet_data.xmit.recursion) >
	346	XMIT_RECURSION_LIMIT);
	347	}
	348
	349	static inline void dev_xmit_recursion_inc(void)
	350	{
	351	__this_cpu_inc(softnet_data.xmit.recursion);
	352	}
	353
	354	static inline void dev_xmit_recursion_dec(void)
	355	{
	356	__this_cpu_dec(softnet_data.xmit.recursion);
	357	}
ecefbc09 SAS	358	#else
	359	static inline bool dev_xmit_recursion(void)
	360	{
	361	return unlikely(current->net_xmit.recursion > XMIT_RECURSION_LIMIT);
	362	}
	363
	364	static inline void dev_xmit_recursion_inc(void)
	365	{
	366	current->net_xmit.recursion++;
	367	}
	368
	369	static inline void dev_xmit_recursion_dec(void)
	370	{
	371	current->net_xmit.recursion--;
	372	}
	373	#endif
2fe50a4d	374
efb45930 KM	375	int dev_set_hwtstamp_phylib(struct net_device *dev,
	376	struct kernel_hwtstamp_config *cfg,
	377	struct netlink_ext_ack *extack);
5e51e50e KM	378	int dev_get_hwtstamp_phylib(struct net_device *dev,
5e51e50e KM	379	struct kernel_hwtstamp_config *cfg);
b18fe47c	380	int net_hwtstamp_validate(const struct kernel_hwtstamp_config *cfg);
efb45930	381
6264f58c	382	#endif