[thirdparty/kernel/linux.git] / net / core / dst_cache.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/core/dst_cache.c - dst entry cache
 *
 * Copyright (c) 2016 Paolo Abeni <pabeni@redhat.com>
 */

#include <linux/kernel.h>
#include <linux/percpu.h>
#include <net/dst_cache.h>
#include <net/route.h>
#if IS_ENABLED(CONFIG_IPV6)
#include <net/ip6_fib.h>
#endif
#include <uapi/linux/in.h>

struct dst_cache_pcpu {
	unsigned long refresh_ts;
	struct dst_entry *dst;
	u32 cookie;
	union {
		struct in_addr in_saddr;
		struct in6_addr in6_saddr;
	};
};

static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
				      struct dst_entry *dst, u32 cookie)
{
	dst_release(dst_cache->dst);
	if (dst)
		dst_hold(dst);

	dst_cache->cookie = cookie;
	dst_cache->dst = dst;
}

static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache,
					       struct dst_cache_pcpu *idst)
{
	struct dst_entry *dst;

	dst = idst->dst;
	if (!dst)
		goto fail;

	/* the cache already hold a dst reference; it can't go away */
	dst_hold(dst);

	if (unlikely(!time_after(idst->refresh_ts,
				 READ_ONCE(dst_cache->reset_ts)) ||
		     (dst->obsolete && !dst->ops->check(dst, idst->cookie)))) {
		dst_cache_per_cpu_dst_set(idst, NULL, 0);
		dst_release(dst);
		goto fail;
	}
	return dst;

fail:
	idst->refresh_ts = jiffies;
	return NULL;
}

struct dst_entry *dst_cache_get(struct dst_cache *dst_cache)
{
	if (!dst_cache->cache)
		return NULL;

	return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache));
}
EXPORT_SYMBOL_GPL(dst_cache_get);

struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr)
{
	struct dst_cache_pcpu *idst;
	struct dst_entry *dst;

	if (!dst_cache->cache)
		return NULL;

	idst = this_cpu_ptr(dst_cache->cache);
	dst = dst_cache_per_cpu_get(dst_cache, idst);
	if (!dst)
		return NULL;

	*saddr = idst->in_saddr.s_addr;
	return dst_rtable(dst);
}
EXPORT_SYMBOL_GPL(dst_cache_get_ip4);

void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
		       __be32 saddr)
{
	struct dst_cache_pcpu *idst;

	if (!dst_cache->cache)
		return;

	idst = this_cpu_ptr(dst_cache->cache);
	dst_cache_per_cpu_dst_set(idst, dst, 0);
	idst->in_saddr.s_addr = saddr;
}
EXPORT_SYMBOL_GPL(dst_cache_set_ip4);

#if IS_ENABLED(CONFIG_IPV6)
void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst,
		       const struct in6_addr *saddr)
{
	struct dst_cache_pcpu *idst;

	if (!dst_cache->cache)
		return;

	idst = this_cpu_ptr(dst_cache->cache);
	dst_cache_per_cpu_dst_set(idst, dst,
				  rt6_get_cookie(dst_rt6_info(dst)));
	idst->in6_saddr = *saddr;
}
EXPORT_SYMBOL_GPL(dst_cache_set_ip6);

struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache,
				    struct in6_addr *saddr)
{
	struct dst_cache_pcpu *idst;
	struct dst_entry *dst;

	if (!dst_cache->cache)
		return NULL;

	idst = this_cpu_ptr(dst_cache->cache);
	dst = dst_cache_per_cpu_get(dst_cache, idst);
	if (!dst)
		return NULL;

	*saddr = idst->in6_saddr;
	return dst;
}
EXPORT_SYMBOL_GPL(dst_cache_get_ip6);
#endif

int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp)
{
	dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu,
					    gfp | __GFP_ZERO);
	if (!dst_cache->cache)
		return -ENOMEM;

	dst_cache_reset(dst_cache);
	return 0;
}
EXPORT_SYMBOL_GPL(dst_cache_init);

void dst_cache_destroy(struct dst_cache *dst_cache)
{
	int i;

	if (!dst_cache->cache)
		return;

	for_each_possible_cpu(i)
		dst_release(per_cpu_ptr(dst_cache->cache, i)->dst);

	free_percpu(dst_cache->cache);
}
EXPORT_SYMBOL_GPL(dst_cache_destroy);

void dst_cache_reset_now(struct dst_cache *dst_cache)
{
	int i;

	if (!dst_cache->cache)
		return;

	dst_cache_reset(dst_cache);
	for_each_possible_cpu(i) {
		struct dst_cache_pcpu *idst = per_cpu_ptr(dst_cache->cache, i);
		struct dst_entry *dst = idst->dst;

		idst->cookie = 0;
		idst->dst = NULL;
		dst_release(dst);
	}
}
EXPORT_SYMBOL_GPL(dst_cache_reset_now);
Commit	Line	Data
2874c5fd	1	// SPDX-License-Identifier: GPL-2.0-or-later
911362c7 PA	2	/*
	3	* net/core/dst_cache.c - dst entry cache
	4	*
	5	* Copyright (c) 2016 Paolo Abeni <pabeni@redhat.com>
911362c7 PA	6	*/
	7
	8	#include <linux/kernel.h>
	9	#include <linux/percpu.h>
	10	#include <net/dst_cache.h>
	11	#include <net/route.h>
	12	#if IS_ENABLED(CONFIG_IPV6)
	13	#include <net/ip6_fib.h>
	14	#endif
	15	#include <uapi/linux/in.h>
	16
	17	struct dst_cache_pcpu {
	18	unsigned long refresh_ts;
	19	struct dst_entry *dst;
	20	u32 cookie;
	21	union {
	22	struct in_addr in_saddr;
	23	struct in6_addr in6_saddr;
	24	};
	25	};
	26
b73f96fc WF	27	static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
b73f96fc WF	28	struct dst_entry *dst, u32 cookie)
911362c7 PA	29	{
	30	dst_release(dst_cache->dst);
	31	if (dst)
	32	dst_hold(dst);
	33
	34	dst_cache->cookie = cookie;
	35	dst_cache->dst = dst;
	36	}
	37
b73f96fc WF	38	static struct dst_entry dst_cache_per_cpu_get(struct dst_cache dst_cache,
b73f96fc WF	39	struct dst_cache_pcpu *idst)
911362c7 PA	40	{
	41	struct dst_entry *dst;
	42
	43	dst = idst->dst;
	44	if (!dst)
	45	goto fail;
	46
	47	/* the cache already hold a dst reference; it can't go away */
	48	dst_hold(dst);
	49
3b09b2bd ED	50	if (unlikely(!time_after(idst->refresh_ts,
3b09b2bd ED	51	READ_ONCE(dst_cache->reset_ts)) \|\|
911362c7 PA	52	(dst->obsolete && !dst->ops->check(dst, idst->cookie)))) {
	53	dst_cache_per_cpu_dst_set(idst, NULL, 0);
	54	dst_release(dst);
	55	goto fail;
	56	}
	57	return dst;
	58
	59	fail:
	60	idst->refresh_ts = jiffies;
	61	return NULL;
	62	}
	63
	64	struct dst_entry dst_cache_get(struct dst_cache dst_cache)
	65	{
	66	if (!dst_cache->cache)
	67	return NULL;
	68
	69	return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache));
	70	}
	71	EXPORT_SYMBOL_GPL(dst_cache_get);
	72
	73	struct rtable dst_cache_get_ip4(struct dst_cache dst_cache, __be32 *saddr)
	74	{
	75	struct dst_cache_pcpu *idst;
	76	struct dst_entry *dst;
	77
	78	if (!dst_cache->cache)
	79	return NULL;
	80
	81	idst = this_cpu_ptr(dst_cache->cache);
	82	dst = dst_cache_per_cpu_get(dst_cache, idst);
	83	if (!dst)
	84	return NULL;
	85
	86	*saddr = idst->in_saddr.s_addr;
05d6d492	87	return dst_rtable(dst);
911362c7 PA	88	}
	89	EXPORT_SYMBOL_GPL(dst_cache_get_ip4);
	90
	91	void dst_cache_set_ip4(struct dst_cache dst_cache, struct dst_entry dst,
	92	__be32 saddr)
	93	{
	94	struct dst_cache_pcpu *idst;
	95
	96	if (!dst_cache->cache)
	97	return;
	98
	99	idst = this_cpu_ptr(dst_cache->cache);
	100	dst_cache_per_cpu_dst_set(idst, dst, 0);
	101	idst->in_saddr.s_addr = saddr;
	102	}
	103	EXPORT_SYMBOL_GPL(dst_cache_set_ip4);
	104
	105	#if IS_ENABLED(CONFIG_IPV6)
	106	void dst_cache_set_ip6(struct dst_cache dst_cache, struct dst_entry dst,
4c1342d9	107	const struct in6_addr *saddr)
911362c7 PA	108	{
	109	struct dst_cache_pcpu *idst;
	110
	111	if (!dst_cache->cache)
	112	return;
	113
	114	idst = this_cpu_ptr(dst_cache->cache);
e2d09e5a	115	dst_cache_per_cpu_dst_set(idst, dst,
e8dfd42c	116	rt6_get_cookie(dst_rt6_info(dst)));
4c1342d9	117	idst->in6_saddr = *saddr;
911362c7 PA	118	}
	119	EXPORT_SYMBOL_GPL(dst_cache_set_ip6);
	120
	121	struct dst_entry dst_cache_get_ip6(struct dst_cache dst_cache,
	122	struct in6_addr *saddr)
	123	{
	124	struct dst_cache_pcpu *idst;
	125	struct dst_entry *dst;
	126
	127	if (!dst_cache->cache)
	128	return NULL;
	129
	130	idst = this_cpu_ptr(dst_cache->cache);
	131	dst = dst_cache_per_cpu_get(dst_cache, idst);
	132	if (!dst)
	133	return NULL;
	134
	135	*saddr = idst->in6_saddr;
	136	return dst;
	137	}
	138	EXPORT_SYMBOL_GPL(dst_cache_get_ip6);
	139	#endif
	140
	141	int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp)
	142	{
	143	dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu,
	144	gfp \| __GFP_ZERO);
	145	if (!dst_cache->cache)
	146	return -ENOMEM;
	147
	148	dst_cache_reset(dst_cache);
	149	return 0;
	150	}
	151	EXPORT_SYMBOL_GPL(dst_cache_init);
	152
	153	void dst_cache_destroy(struct dst_cache *dst_cache)
	154	{
	155	int i;
	156
	157	if (!dst_cache->cache)
	158	return;
	159
	160	for_each_possible_cpu(i)
	161	dst_release(per_cpu_ptr(dst_cache->cache, i)->dst);
	162
	163	free_percpu(dst_cache->cache);
	164	}
	165	EXPORT_SYMBOL_GPL(dst_cache_destroy);
20ae1d6a JD	166
	167	void dst_cache_reset_now(struct dst_cache *dst_cache)
	168	{
	169	int i;
	170
	171	if (!dst_cache->cache)
	172	return;
	173
3b09b2bd	174	dst_cache_reset(dst_cache);
20ae1d6a JD	175	for_each_possible_cpu(i) {
	176	struct dst_cache_pcpu *idst = per_cpu_ptr(dst_cache->cache, i);
	177	struct dst_entry *dst = idst->dst;
	178
	179	idst->cookie = 0;
	180	idst->dst = NULL;
	181	dst_release(dst);
	182	}
	183	}
	184	EXPORT_SYMBOL_GPL(dst_cache_reset_now);