1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
7 * IPv4 Forwarding Information Base: semantics.
9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
12 #include <linux/uaccess.h>
13 #include <linux/bitops.h>
14 #include <linux/types.h>
15 #include <linux/kernel.h>
16 #include <linux/jiffies.h>
18 #include <linux/string.h>
19 #include <linux/socket.h>
20 #include <linux/sockios.h>
21 #include <linux/errno.h>
23 #include <linux/inet.h>
24 #include <linux/inetdevice.h>
25 #include <linux/netdevice.h>
26 #include <linux/if_arp.h>
27 #include <linux/proc_fs.h>
28 #include <linux/skbuff.h>
29 #include <linux/init.h>
30 #include <linux/slab.h>
31 #include <linux/netlink.h>
32 #include <linux/hash.h>
33 #include <linux/nospec.h>
36 #include <net/inet_dscp.h>
38 #include <net/protocol.h>
39 #include <net/route.h>
42 #include <net/ip_fib.h>
43 #include <net/ip6_fib.h>
44 #include <net/nexthop.h>
45 #include <net/netlink.h>
47 #include <net/lwtunnel.h>
48 #include <net/fib_notifier.h>
49 #include <net/addrconf.h>
51 #include "fib_lookup.h"
53 static DEFINE_SPINLOCK(fib_info_lock
);
54 static struct hlist_head
*fib_info_hash
;
55 static struct hlist_head
*fib_info_laddrhash
;
56 static unsigned int fib_info_hash_size
;
57 static unsigned int fib_info_hash_bits
;
58 static unsigned int fib_info_cnt
;
60 #define DEVINDEX_HASHBITS 8
61 #define DEVINDEX_HASHSIZE (1U << DEVINDEX_HASHBITS)
62 static struct hlist_head fib_info_devhash
[DEVINDEX_HASHSIZE
];
64 /* for_nexthops and change_nexthops only used when nexthop object
65 * is not set in a fib_info. The logic within can reference fib_nh.
67 #ifdef CONFIG_IP_ROUTE_MULTIPATH
69 #define for_nexthops(fi) { \
70 int nhsel; const struct fib_nh *nh; \
71 for (nhsel = 0, nh = (fi)->fib_nh; \
72 nhsel < fib_info_num_path((fi)); \
75 #define change_nexthops(fi) { \
76 int nhsel; struct fib_nh *nexthop_nh; \
77 for (nhsel = 0, nexthop_nh = (struct fib_nh *)((fi)->fib_nh); \
78 nhsel < fib_info_num_path((fi)); \
79 nexthop_nh++, nhsel++)
81 #else /* CONFIG_IP_ROUTE_MULTIPATH */
83 /* Hope, that gcc will optimize it to get rid of dummy loop */
85 #define for_nexthops(fi) { \
86 int nhsel; const struct fib_nh *nh = (fi)->fib_nh; \
87 for (nhsel = 0; nhsel < 1; nhsel++)
89 #define change_nexthops(fi) { \
91 struct fib_nh *nexthop_nh = (struct fib_nh *)((fi)->fib_nh); \
92 for (nhsel = 0; nhsel < 1; nhsel++)
94 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
96 #define endfor_nexthops(fi) }
99 const struct fib_prop fib_props
[RTN_MAX
+ 1] = {
102 .scope
= RT_SCOPE_NOWHERE
,
106 .scope
= RT_SCOPE_UNIVERSE
,
110 .scope
= RT_SCOPE_HOST
,
114 .scope
= RT_SCOPE_LINK
,
118 .scope
= RT_SCOPE_LINK
,
122 .scope
= RT_SCOPE_UNIVERSE
,
126 .scope
= RT_SCOPE_UNIVERSE
,
128 [RTN_UNREACHABLE
] = {
129 .error
= -EHOSTUNREACH
,
130 .scope
= RT_SCOPE_UNIVERSE
,
134 .scope
= RT_SCOPE_UNIVERSE
,
138 .scope
= RT_SCOPE_UNIVERSE
,
142 .scope
= RT_SCOPE_NOWHERE
,
146 .scope
= RT_SCOPE_NOWHERE
,
150 static void rt_fibinfo_free(struct rtable __rcu
**rtp
)
152 struct rtable
*rt
= rcu_dereference_protected(*rtp
, 1);
157 /* Not even needed : RCU_INIT_POINTER(*rtp, NULL);
158 * because we waited an RCU grace period before calling
159 * free_fib_info_rcu()
162 dst_dev_put(&rt
->dst
);
163 dst_release_immediate(&rt
->dst
);
166 static void free_nh_exceptions(struct fib_nh_common
*nhc
)
168 struct fnhe_hash_bucket
*hash
;
171 hash
= rcu_dereference_protected(nhc
->nhc_exceptions
, 1);
174 for (i
= 0; i
< FNHE_HASH_SIZE
; i
++) {
175 struct fib_nh_exception
*fnhe
;
177 fnhe
= rcu_dereference_protected(hash
[i
].chain
, 1);
179 struct fib_nh_exception
*next
;
181 next
= rcu_dereference_protected(fnhe
->fnhe_next
, 1);
183 rt_fibinfo_free(&fnhe
->fnhe_rth_input
);
184 rt_fibinfo_free(&fnhe
->fnhe_rth_output
);
194 static void rt_fibinfo_free_cpus(struct rtable __rcu
* __percpu
*rtp
)
201 for_each_possible_cpu(cpu
) {
204 rt
= rcu_dereference_protected(*per_cpu_ptr(rtp
, cpu
), 1);
206 dst_dev_put(&rt
->dst
);
207 dst_release_immediate(&rt
->dst
);
213 void fib_nh_common_release(struct fib_nh_common
*nhc
)
215 netdev_put(nhc
->nhc_dev
, &nhc
->nhc_dev_tracker
);
216 lwtstate_put(nhc
->nhc_lwtstate
);
217 rt_fibinfo_free_cpus(nhc
->nhc_pcpu_rth_output
);
218 rt_fibinfo_free(&nhc
->nhc_rth_input
);
219 free_nh_exceptions(nhc
);
221 EXPORT_SYMBOL_GPL(fib_nh_common_release
);
223 void fib_nh_release(struct net
*net
, struct fib_nh
*fib_nh
)
225 #ifdef CONFIG_IP_ROUTE_CLASSID
226 if (fib_nh
->nh_tclassid
)
227 atomic_dec(&net
->ipv4
.fib_num_tclassid_users
);
229 fib_nh_common_release(&fib_nh
->nh_common
);
232 /* Release a nexthop info record */
233 static void free_fib_info_rcu(struct rcu_head
*head
)
235 struct fib_info
*fi
= container_of(head
, struct fib_info
, rcu
);
240 change_nexthops(fi
) {
241 fib_nh_release(fi
->fib_net
, nexthop_nh
);
242 } endfor_nexthops(fi
);
245 ip_fib_metrics_put(fi
->fib_metrics
);
250 void free_fib_info(struct fib_info
*fi
)
252 if (fi
->fib_dead
== 0) {
253 pr_warn("Freeing alive fib_info %p\n", fi
);
257 call_rcu(&fi
->rcu
, free_fib_info_rcu
);
259 EXPORT_SYMBOL_GPL(free_fib_info
);
261 void fib_release_info(struct fib_info
*fi
)
263 spin_lock_bh(&fib_info_lock
);
264 if (fi
&& refcount_dec_and_test(&fi
->fib_treeref
)) {
265 hlist_del(&fi
->fib_hash
);
267 /* Paired with READ_ONCE() in fib_create_info(). */
268 WRITE_ONCE(fib_info_cnt
, fib_info_cnt
- 1);
271 hlist_del(&fi
->fib_lhash
);
273 list_del(&fi
->nh_list
);
275 change_nexthops(fi
) {
276 if (!nexthop_nh
->fib_nh_dev
)
278 hlist_del(&nexthop_nh
->nh_hash
);
279 } endfor_nexthops(fi
)
281 /* Paired with READ_ONCE() from fib_table_lookup() */
282 WRITE_ONCE(fi
->fib_dead
, 1);
285 spin_unlock_bh(&fib_info_lock
);
288 static inline int nh_comp(struct fib_info
*fi
, struct fib_info
*ofi
)
290 const struct fib_nh
*onh
;
292 if (fi
->nh
|| ofi
->nh
)
293 return nexthop_cmp(fi
->nh
, ofi
->nh
) ? 0 : -1;
295 if (ofi
->fib_nhs
== 0)
299 onh
= fib_info_nh(ofi
, nhsel
);
301 if (nh
->fib_nh_oif
!= onh
->fib_nh_oif
||
302 nh
->fib_nh_gw_family
!= onh
->fib_nh_gw_family
||
303 nh
->fib_nh_scope
!= onh
->fib_nh_scope
||
304 #ifdef CONFIG_IP_ROUTE_MULTIPATH
305 nh
->fib_nh_weight
!= onh
->fib_nh_weight
||
307 #ifdef CONFIG_IP_ROUTE_CLASSID
308 nh
->nh_tclassid
!= onh
->nh_tclassid
||
310 lwtunnel_cmp_encap(nh
->fib_nh_lws
, onh
->fib_nh_lws
) ||
311 ((nh
->fib_nh_flags
^ onh
->fib_nh_flags
) & ~RTNH_COMPARE_MASK
))
314 if (nh
->fib_nh_gw_family
== AF_INET
&&
315 nh
->fib_nh_gw4
!= onh
->fib_nh_gw4
)
318 if (nh
->fib_nh_gw_family
== AF_INET6
&&
319 ipv6_addr_cmp(&nh
->fib_nh_gw6
, &onh
->fib_nh_gw6
))
321 } endfor_nexthops(fi
);
325 static inline unsigned int fib_devindex_hashfn(unsigned int val
)
327 return hash_32(val
, DEVINDEX_HASHBITS
);
330 static struct hlist_head
*
331 fib_info_devhash_bucket(const struct net_device
*dev
)
333 u32 val
= net_hash_mix(dev_net(dev
)) ^ dev
->ifindex
;
335 return &fib_info_devhash
[fib_devindex_hashfn(val
)];
338 static unsigned int fib_info_hashfn_1(int init_val
, u8 protocol
, u8 scope
,
339 u32 prefsrc
, u32 priority
)
341 unsigned int val
= init_val
;
343 val
^= (protocol
<< 8) | scope
;
350 static unsigned int fib_info_hashfn_result(unsigned int val
)
352 unsigned int mask
= (fib_info_hash_size
- 1);
354 return (val
^ (val
>> 7) ^ (val
>> 12)) & mask
;
357 static inline unsigned int fib_info_hashfn(struct fib_info
*fi
)
361 val
= fib_info_hashfn_1(fi
->fib_nhs
, fi
->fib_protocol
,
362 fi
->fib_scope
, (__force u32
)fi
->fib_prefsrc
,
366 val
^= fib_devindex_hashfn(fi
->nh
->id
);
369 val
^= fib_devindex_hashfn(nh
->fib_nh_oif
);
370 } endfor_nexthops(fi
)
373 return fib_info_hashfn_result(val
);
376 /* no metrics, only nexthop id */
377 static struct fib_info
*fib_find_info_nh(struct net
*net
,
378 const struct fib_config
*cfg
)
380 struct hlist_head
*head
;
384 hash
= fib_info_hashfn_1(fib_devindex_hashfn(cfg
->fc_nh_id
),
385 cfg
->fc_protocol
, cfg
->fc_scope
,
386 (__force u32
)cfg
->fc_prefsrc
,
388 hash
= fib_info_hashfn_result(hash
);
389 head
= &fib_info_hash
[hash
];
391 hlist_for_each_entry(fi
, head
, fib_hash
) {
392 if (!net_eq(fi
->fib_net
, net
))
394 if (!fi
->nh
|| fi
->nh
->id
!= cfg
->fc_nh_id
)
396 if (cfg
->fc_protocol
== fi
->fib_protocol
&&
397 cfg
->fc_scope
== fi
->fib_scope
&&
398 cfg
->fc_prefsrc
== fi
->fib_prefsrc
&&
399 cfg
->fc_priority
== fi
->fib_priority
&&
400 cfg
->fc_type
== fi
->fib_type
&&
401 cfg
->fc_table
== fi
->fib_tb_id
&&
402 !((cfg
->fc_flags
^ fi
->fib_flags
) & ~RTNH_COMPARE_MASK
))
409 static struct fib_info
*fib_find_info(struct fib_info
*nfi
)
411 struct hlist_head
*head
;
415 hash
= fib_info_hashfn(nfi
);
416 head
= &fib_info_hash
[hash
];
418 hlist_for_each_entry(fi
, head
, fib_hash
) {
419 if (!net_eq(fi
->fib_net
, nfi
->fib_net
))
421 if (fi
->fib_nhs
!= nfi
->fib_nhs
)
423 if (nfi
->fib_protocol
== fi
->fib_protocol
&&
424 nfi
->fib_scope
== fi
->fib_scope
&&
425 nfi
->fib_prefsrc
== fi
->fib_prefsrc
&&
426 nfi
->fib_priority
== fi
->fib_priority
&&
427 nfi
->fib_type
== fi
->fib_type
&&
428 nfi
->fib_tb_id
== fi
->fib_tb_id
&&
429 memcmp(nfi
->fib_metrics
, fi
->fib_metrics
,
430 sizeof(u32
) * RTAX_MAX
) == 0 &&
431 !((nfi
->fib_flags
^ fi
->fib_flags
) & ~RTNH_COMPARE_MASK
) &&
432 nh_comp(fi
, nfi
) == 0)
439 /* Check, that the gateway is already configured.
440 * Used only by redirect accept routine.
442 int ip_fib_check_default(__be32 gw
, struct net_device
*dev
)
444 struct hlist_head
*head
;
447 spin_lock(&fib_info_lock
);
449 head
= fib_info_devhash_bucket(dev
);
451 hlist_for_each_entry(nh
, head
, nh_hash
) {
452 if (nh
->fib_nh_dev
== dev
&&
453 nh
->fib_nh_gw4
== gw
&&
454 !(nh
->fib_nh_flags
& RTNH_F_DEAD
)) {
455 spin_unlock(&fib_info_lock
);
460 spin_unlock(&fib_info_lock
);
465 size_t fib_nlmsg_size(struct fib_info
*fi
)
467 size_t payload
= NLMSG_ALIGN(sizeof(struct rtmsg
))
468 + nla_total_size(4) /* RTA_TABLE */
469 + nla_total_size(4) /* RTA_DST */
470 + nla_total_size(4) /* RTA_PRIORITY */
471 + nla_total_size(4) /* RTA_PREFSRC */
472 + nla_total_size(TCP_CA_NAME_MAX
); /* RTAX_CC_ALGO */
473 unsigned int nhs
= fib_info_num_path(fi
);
475 /* space for nested metrics */
476 payload
+= nla_total_size((RTAX_MAX
* nla_total_size(4)));
479 payload
+= nla_total_size(4); /* RTA_NH_ID */
482 size_t nh_encapsize
= 0;
483 /* Also handles the special case nhs == 1 */
485 /* each nexthop is packed in an attribute */
486 size_t nhsize
= nla_total_size(sizeof(struct rtnexthop
));
489 /* may contain flow and gateway attribute */
490 nhsize
+= 2 * nla_total_size(4);
492 /* grab encap info */
493 for (i
= 0; i
< fib_info_num_path(fi
); i
++) {
494 struct fib_nh_common
*nhc
= fib_info_nhc(fi
, i
);
496 if (nhc
->nhc_lwtstate
) {
498 nh_encapsize
+= lwtunnel_get_encap_size(
501 nh_encapsize
+= nla_total_size(2);
505 /* all nexthops are packed in a nested attribute */
506 payload
+= nla_total_size((nhs
* nhsize
) + nh_encapsize
);
513 void rtmsg_fib(int event
, __be32 key
, struct fib_alias
*fa
,
514 int dst_len
, u32 tb_id
, const struct nl_info
*info
,
515 unsigned int nlm_flags
)
517 struct fib_rt_info fri
;
519 u32 seq
= info
->nlh
? info
->nlh
->nlmsg_seq
: 0;
522 skb
= nlmsg_new(fib_nlmsg_size(fa
->fa_info
), GFP_KERNEL
);
526 fri
.fi
= fa
->fa_info
;
529 fri
.dst_len
= dst_len
;
530 fri
.dscp
= fa
->fa_dscp
;
531 fri
.type
= fa
->fa_type
;
532 fri
.offload
= READ_ONCE(fa
->offload
);
533 fri
.trap
= READ_ONCE(fa
->trap
);
534 fri
.offload_failed
= READ_ONCE(fa
->offload_failed
);
535 err
= fib_dump_info(skb
, info
->portid
, seq
, event
, &fri
, nlm_flags
);
537 /* -EMSGSIZE implies BUG in fib_nlmsg_size() */
538 WARN_ON(err
== -EMSGSIZE
);
542 rtnl_notify(skb
, info
->nl_net
, info
->portid
, RTNLGRP_IPV4_ROUTE
,
543 info
->nlh
, GFP_KERNEL
);
547 rtnl_set_sk_err(info
->nl_net
, RTNLGRP_IPV4_ROUTE
, err
);
550 static int fib_detect_death(struct fib_info
*fi
, int order
,
551 struct fib_info
**last_resort
, int *last_idx
,
554 const struct fib_nh_common
*nhc
= fib_info_nhc(fi
, 0);
556 int state
= NUD_NONE
;
558 if (likely(nhc
->nhc_gw_family
== AF_INET
))
559 n
= neigh_lookup(&arp_tbl
, &nhc
->nhc_gw
.ipv4
, nhc
->nhc_dev
);
560 else if (nhc
->nhc_gw_family
== AF_INET6
)
561 n
= neigh_lookup(ipv6_stub
->nd_tbl
, &nhc
->nhc_gw
.ipv6
,
567 state
= READ_ONCE(n
->nud_state
);
572 if (state
== NUD_REACHABLE
)
574 if ((state
& NUD_VALID
) && order
!= dflt
)
576 if ((state
& NUD_VALID
) ||
577 (*last_idx
< 0 && order
> dflt
&& state
!= NUD_INCOMPLETE
)) {
584 int fib_nh_common_init(struct net
*net
, struct fib_nh_common
*nhc
,
585 struct nlattr
*encap
, u16 encap_type
,
586 void *cfg
, gfp_t gfp_flags
,
587 struct netlink_ext_ack
*extack
)
591 nhc
->nhc_pcpu_rth_output
= alloc_percpu_gfp(struct rtable __rcu
*,
593 if (!nhc
->nhc_pcpu_rth_output
)
597 struct lwtunnel_state
*lwtstate
;
599 if (encap_type
== LWTUNNEL_ENCAP_NONE
) {
600 NL_SET_ERR_MSG(extack
, "LWT encap type not specified");
604 err
= lwtunnel_build_state(net
, encap_type
, encap
,
605 nhc
->nhc_family
, cfg
, &lwtstate
,
610 nhc
->nhc_lwtstate
= lwtstate_get(lwtstate
);
616 rt_fibinfo_free_cpus(nhc
->nhc_pcpu_rth_output
);
617 nhc
->nhc_pcpu_rth_output
= NULL
;
620 EXPORT_SYMBOL_GPL(fib_nh_common_init
);
622 int fib_nh_init(struct net
*net
, struct fib_nh
*nh
,
623 struct fib_config
*cfg
, int nh_weight
,
624 struct netlink_ext_ack
*extack
)
628 nh
->fib_nh_family
= AF_INET
;
630 err
= fib_nh_common_init(net
, &nh
->nh_common
, cfg
->fc_encap
,
631 cfg
->fc_encap_type
, cfg
, GFP_KERNEL
, extack
);
635 nh
->fib_nh_oif
= cfg
->fc_oif
;
636 nh
->fib_nh_gw_family
= cfg
->fc_gw_family
;
637 if (cfg
->fc_gw_family
== AF_INET
)
638 nh
->fib_nh_gw4
= cfg
->fc_gw4
;
639 else if (cfg
->fc_gw_family
== AF_INET6
)
640 nh
->fib_nh_gw6
= cfg
->fc_gw6
;
642 nh
->fib_nh_flags
= cfg
->fc_flags
;
644 #ifdef CONFIG_IP_ROUTE_CLASSID
645 nh
->nh_tclassid
= cfg
->fc_flow
;
647 atomic_inc(&net
->ipv4
.fib_num_tclassid_users
);
649 #ifdef CONFIG_IP_ROUTE_MULTIPATH
650 nh
->fib_nh_weight
= nh_weight
;
655 #ifdef CONFIG_IP_ROUTE_MULTIPATH
657 static int fib_count_nexthops(struct rtnexthop
*rtnh
, int remaining
,
658 struct netlink_ext_ack
*extack
)
662 while (rtnh_ok(rtnh
, remaining
)) {
664 rtnh
= rtnh_next(rtnh
, &remaining
);
667 /* leftover implies invalid nexthop configuration, discard it */
669 NL_SET_ERR_MSG(extack
,
670 "Invalid nexthop configuration - extra data after nexthops");
677 static int fib_gw_from_attr(__be32
*gw
, struct nlattr
*nla
,
678 struct netlink_ext_ack
*extack
)
680 if (nla_len(nla
) < sizeof(*gw
)) {
681 NL_SET_ERR_MSG(extack
, "Invalid IPv4 address in RTA_GATEWAY");
685 *gw
= nla_get_in_addr(nla
);
690 /* only called when fib_nh is integrated into fib_info */
691 static int fib_get_nhs(struct fib_info
*fi
, struct rtnexthop
*rtnh
,
692 int remaining
, struct fib_config
*cfg
,
693 struct netlink_ext_ack
*extack
)
695 struct net
*net
= fi
->fib_net
;
696 struct fib_config fib_cfg
;
700 change_nexthops(fi
) {
703 memset(&fib_cfg
, 0, sizeof(fib_cfg
));
705 if (!rtnh_ok(rtnh
, remaining
)) {
706 NL_SET_ERR_MSG(extack
,
707 "Invalid nexthop configuration - extra data after nexthop");
711 if (rtnh
->rtnh_flags
& (RTNH_F_DEAD
| RTNH_F_LINKDOWN
)) {
712 NL_SET_ERR_MSG(extack
,
713 "Invalid flags for nexthop - can not contain DEAD or LINKDOWN");
717 fib_cfg
.fc_flags
= (cfg
->fc_flags
& ~0xFF) | rtnh
->rtnh_flags
;
718 fib_cfg
.fc_oif
= rtnh
->rtnh_ifindex
;
720 attrlen
= rtnh_attrlen(rtnh
);
722 struct nlattr
*nla
, *nlav
, *attrs
= rtnh_attrs(rtnh
);
724 nla
= nla_find(attrs
, attrlen
, RTA_GATEWAY
);
725 nlav
= nla_find(attrs
, attrlen
, RTA_VIA
);
727 NL_SET_ERR_MSG(extack
,
728 "Nexthop configuration can not contain both GATEWAY and VIA");
732 ret
= fib_gw_from_attr(&fib_cfg
.fc_gw4
, nla
,
738 fib_cfg
.fc_gw_family
= AF_INET
;
740 ret
= fib_gw_from_via(&fib_cfg
, nlav
, extack
);
745 nla
= nla_find(attrs
, attrlen
, RTA_FLOW
);
747 if (nla_len(nla
) < sizeof(u32
)) {
748 NL_SET_ERR_MSG(extack
, "Invalid RTA_FLOW");
751 fib_cfg
.fc_flow
= nla_get_u32(nla
);
754 fib_cfg
.fc_encap
= nla_find(attrs
, attrlen
, RTA_ENCAP
);
755 /* RTA_ENCAP_TYPE length checked in
756 * lwtunnel_valid_encap_type_attr
758 nla
= nla_find(attrs
, attrlen
, RTA_ENCAP_TYPE
);
760 fib_cfg
.fc_encap_type
= nla_get_u16(nla
);
763 ret
= fib_nh_init(net
, nexthop_nh
, &fib_cfg
,
764 rtnh
->rtnh_hops
+ 1, extack
);
768 rtnh
= rtnh_next(rtnh
, &remaining
);
769 } endfor_nexthops(fi
);
772 nh
= fib_info_nh(fi
, 0);
773 if (cfg
->fc_oif
&& nh
->fib_nh_oif
!= cfg
->fc_oif
) {
774 NL_SET_ERR_MSG(extack
,
775 "Nexthop device index does not match RTA_OIF");
778 if (cfg
->fc_gw_family
) {
779 if (cfg
->fc_gw_family
!= nh
->fib_nh_gw_family
||
780 (cfg
->fc_gw_family
== AF_INET
&&
781 nh
->fib_nh_gw4
!= cfg
->fc_gw4
) ||
782 (cfg
->fc_gw_family
== AF_INET6
&&
783 ipv6_addr_cmp(&nh
->fib_nh_gw6
, &cfg
->fc_gw6
))) {
784 NL_SET_ERR_MSG(extack
,
785 "Nexthop gateway does not match RTA_GATEWAY or RTA_VIA");
789 #ifdef CONFIG_IP_ROUTE_CLASSID
790 if (cfg
->fc_flow
&& nh
->nh_tclassid
!= cfg
->fc_flow
) {
791 NL_SET_ERR_MSG(extack
,
792 "Nexthop class id does not match RTA_FLOW");
801 /* only called when fib_nh is integrated into fib_info */
802 static void fib_rebalance(struct fib_info
*fi
)
807 if (fib_info_num_path(fi
) < 2)
812 if (nh
->fib_nh_flags
& RTNH_F_DEAD
)
815 if (ip_ignore_linkdown(nh
->fib_nh_dev
) &&
816 nh
->fib_nh_flags
& RTNH_F_LINKDOWN
)
819 total
+= nh
->fib_nh_weight
;
820 } endfor_nexthops(fi
);
823 change_nexthops(fi
) {
826 if (nexthop_nh
->fib_nh_flags
& RTNH_F_DEAD
) {
828 } else if (ip_ignore_linkdown(nexthop_nh
->fib_nh_dev
) &&
829 nexthop_nh
->fib_nh_flags
& RTNH_F_LINKDOWN
) {
832 w
+= nexthop_nh
->fib_nh_weight
;
833 upper_bound
= DIV_ROUND_CLOSEST_ULL((u64
)w
<< 31,
837 atomic_set(&nexthop_nh
->fib_nh_upper_bound
, upper_bound
);
838 } endfor_nexthops(fi
);
840 #else /* CONFIG_IP_ROUTE_MULTIPATH */
842 static int fib_get_nhs(struct fib_info
*fi
, struct rtnexthop
*rtnh
,
843 int remaining
, struct fib_config
*cfg
,
844 struct netlink_ext_ack
*extack
)
846 NL_SET_ERR_MSG(extack
, "Multipath support not enabled in kernel");
851 #define fib_rebalance(fi) do { } while (0)
853 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
855 static int fib_encap_match(struct net
*net
, u16 encap_type
,
856 struct nlattr
*encap
,
857 const struct fib_nh
*nh
,
858 const struct fib_config
*cfg
,
859 struct netlink_ext_ack
*extack
)
861 struct lwtunnel_state
*lwtstate
;
864 if (encap_type
== LWTUNNEL_ENCAP_NONE
)
867 ret
= lwtunnel_build_state(net
, encap_type
, encap
, AF_INET
,
868 cfg
, &lwtstate
, extack
);
870 result
= lwtunnel_cmp_encap(lwtstate
, nh
->fib_nh_lws
);
871 lwtstate_free(lwtstate
);
877 int fib_nh_match(struct net
*net
, struct fib_config
*cfg
, struct fib_info
*fi
,
878 struct netlink_ext_ack
*extack
)
880 #ifdef CONFIG_IP_ROUTE_MULTIPATH
881 struct rtnexthop
*rtnh
;
885 if (cfg
->fc_priority
&& cfg
->fc_priority
!= fi
->fib_priority
)
889 if (fi
->nh
&& cfg
->fc_nh_id
== fi
->nh
->id
)
895 if (cfg
->fc_oif
|| cfg
->fc_gw_family
|| cfg
->fc_mp
)
900 if (cfg
->fc_oif
|| cfg
->fc_gw_family
) {
903 nh
= fib_info_nh(fi
, 0);
905 if (fib_encap_match(net
, cfg
->fc_encap_type
,
906 cfg
->fc_encap
, nh
, cfg
, extack
))
909 #ifdef CONFIG_IP_ROUTE_CLASSID
911 cfg
->fc_flow
!= nh
->nh_tclassid
)
914 if ((cfg
->fc_oif
&& cfg
->fc_oif
!= nh
->fib_nh_oif
) ||
915 (cfg
->fc_gw_family
&&
916 cfg
->fc_gw_family
!= nh
->fib_nh_gw_family
))
919 if (cfg
->fc_gw_family
== AF_INET
&&
920 cfg
->fc_gw4
!= nh
->fib_nh_gw4
)
923 if (cfg
->fc_gw_family
== AF_INET6
&&
924 ipv6_addr_cmp(&cfg
->fc_gw6
, &nh
->fib_nh_gw6
))
930 #ifdef CONFIG_IP_ROUTE_MULTIPATH
935 remaining
= cfg
->fc_mp_len
;
940 if (!rtnh_ok(rtnh
, remaining
))
943 if (rtnh
->rtnh_ifindex
&& rtnh
->rtnh_ifindex
!= nh
->fib_nh_oif
)
946 attrlen
= rtnh_attrlen(rtnh
);
948 struct nlattr
*nla
, *nlav
, *attrs
= rtnh_attrs(rtnh
);
951 nla
= nla_find(attrs
, attrlen
, RTA_GATEWAY
);
952 nlav
= nla_find(attrs
, attrlen
, RTA_VIA
);
954 NL_SET_ERR_MSG(extack
,
955 "Nexthop configuration can not contain both GATEWAY and VIA");
962 err
= fib_gw_from_attr(&gw
, nla
, extack
);
966 if (nh
->fib_nh_gw_family
!= AF_INET
||
967 gw
!= nh
->fib_nh_gw4
)
970 struct fib_config cfg2
;
972 err
= fib_gw_from_via(&cfg2
, nlav
, extack
);
976 switch (nh
->fib_nh_gw_family
) {
978 if (cfg2
.fc_gw_family
!= AF_INET
||
979 cfg2
.fc_gw4
!= nh
->fib_nh_gw4
)
983 if (cfg2
.fc_gw_family
!= AF_INET6
||
984 ipv6_addr_cmp(&cfg2
.fc_gw6
,
991 #ifdef CONFIG_IP_ROUTE_CLASSID
992 nla
= nla_find(attrs
, attrlen
, RTA_FLOW
);
994 if (nla_len(nla
) < sizeof(u32
)) {
995 NL_SET_ERR_MSG(extack
, "Invalid RTA_FLOW");
998 if (nla_get_u32(nla
) != nh
->nh_tclassid
)
1004 rtnh
= rtnh_next(rtnh
, &remaining
);
1005 } endfor_nexthops(fi
);
1010 bool fib_metrics_match(struct fib_config
*cfg
, struct fib_info
*fi
)
1018 nla_for_each_attr(nla
, cfg
->fc_mx
, cfg
->fc_mx_len
, remaining
) {
1019 int type
= nla_type(nla
);
1024 if (type
> RTAX_MAX
)
1027 type
= array_index_nospec(type
, RTAX_MAX
+ 1);
1028 if (type
== RTAX_CC_ALGO
) {
1029 char tmp
[TCP_CA_NAME_MAX
];
1030 bool ecn_ca
= false;
1032 nla_strscpy(tmp
, nla
, sizeof(tmp
));
1033 val
= tcp_ca_get_key_by_name(fi
->fib_net
, tmp
, &ecn_ca
);
1035 if (nla_len(nla
) != sizeof(u32
))
1037 val
= nla_get_u32(nla
);
1040 fi_val
= fi
->fib_metrics
->metrics
[type
- 1];
1041 if (type
== RTAX_FEATURES
)
1042 fi_val
&= ~DST_FEATURE_ECN_CA
;
1051 static int fib_check_nh_v6_gw(struct net
*net
, struct fib_nh
*nh
,
1052 u32 table
, struct netlink_ext_ack
*extack
)
1054 struct fib6_config cfg
= {
1056 .fc_flags
= nh
->fib_nh_flags
| RTF_GATEWAY
,
1057 .fc_ifindex
= nh
->fib_nh_oif
,
1058 .fc_gateway
= nh
->fib_nh_gw6
,
1060 struct fib6_nh fib6_nh
= {};
1063 err
= ipv6_stub
->fib6_nh_init(net
, &fib6_nh
, &cfg
, GFP_KERNEL
, extack
);
1065 nh
->fib_nh_dev
= fib6_nh
.fib_nh_dev
;
1066 netdev_hold(nh
->fib_nh_dev
, &nh
->fib_nh_dev_tracker
,
1068 nh
->fib_nh_oif
= nh
->fib_nh_dev
->ifindex
;
1069 nh
->fib_nh_scope
= RT_SCOPE_LINK
;
1071 ipv6_stub
->fib6_nh_release(&fib6_nh
);
1081 * Semantics of nexthop is very messy by historical reasons.
1082 * We have to take into account, that:
1083 * a) gateway can be actually local interface address,
1084 * so that gatewayed route is direct.
1085 * b) gateway must be on-link address, possibly
1086 * described not by an ifaddr, but also by a direct route.
1087 * c) If both gateway and interface are specified, they should not
1089 * d) If we use tunnel routes, gateway could be not on-link.
1091 * Attempt to reconcile all of these (alas, self-contradictory) conditions
1092 * results in pretty ugly and hairy code with obscure logic.
1094 * I chose to generalized it instead, so that the size
1095 * of code does not increase practically, but it becomes
1096 * much more general.
1097 * Every prefix is assigned a "scope" value: "host" is local address,
1098 * "link" is direct route,
1099 * [ ... "site" ... "interior" ... ]
1100 * and "universe" is true gateway route with global meaning.
1102 * Every prefix refers to a set of "nexthop"s (gw, oif),
1103 * where gw must have narrower scope. This recursion stops
1104 * when gw has LOCAL scope or if "nexthop" is declared ONLINK,
1105 * which means that gw is forced to be on link.
1107 * Code is still hairy, but now it is apparently logically
1108 * consistent and very flexible. F.e. as by-product it allows
1109 * to co-exists in peace independent exterior and interior
1110 * routing processes.
1112 * Normally it looks as following.
1114 * {universe prefix} -> (gw, oif) [scope link]
1116 * |-> {link prefix} -> (gw, oif) [scope local]
1118 * |-> {local prefix} (terminal node)
1120 static int fib_check_nh_v4_gw(struct net
*net
, struct fib_nh
*nh
, u32 table
,
1121 u8 scope
, struct netlink_ext_ack
*extack
)
1123 struct net_device
*dev
;
1124 struct fib_result res
;
1127 if (nh
->fib_nh_flags
& RTNH_F_ONLINK
) {
1128 unsigned int addr_type
;
1130 if (scope
>= RT_SCOPE_LINK
) {
1131 NL_SET_ERR_MSG(extack
, "Nexthop has invalid scope");
1134 dev
= __dev_get_by_index(net
, nh
->fib_nh_oif
);
1136 NL_SET_ERR_MSG(extack
, "Nexthop device required for onlink");
1139 if (!(dev
->flags
& IFF_UP
)) {
1140 NL_SET_ERR_MSG(extack
, "Nexthop device is not up");
1143 addr_type
= inet_addr_type_dev_table(net
, dev
, nh
->fib_nh_gw4
);
1144 if (addr_type
!= RTN_UNICAST
) {
1145 NL_SET_ERR_MSG(extack
, "Nexthop has invalid gateway");
1148 if (!netif_carrier_ok(dev
))
1149 nh
->fib_nh_flags
|= RTNH_F_LINKDOWN
;
1150 nh
->fib_nh_dev
= dev
;
1151 netdev_hold(dev
, &nh
->fib_nh_dev_tracker
, GFP_ATOMIC
);
1152 nh
->fib_nh_scope
= RT_SCOPE_LINK
;
1157 struct fib_table
*tbl
= NULL
;
1158 struct flowi4 fl4
= {
1159 .daddr
= nh
->fib_nh_gw4
,
1160 .flowi4_scope
= scope
+ 1,
1161 .flowi4_oif
= nh
->fib_nh_oif
,
1162 .flowi4_iif
= LOOPBACK_IFINDEX
,
1165 /* It is not necessary, but requires a bit of thinking */
1166 if (fl4
.flowi4_scope
< RT_SCOPE_LINK
)
1167 fl4
.flowi4_scope
= RT_SCOPE_LINK
;
1169 if (table
&& table
!= RT_TABLE_MAIN
)
1170 tbl
= fib_get_table(net
, table
);
1173 err
= fib_table_lookup(tbl
, &fl4
, &res
,
1174 FIB_LOOKUP_IGNORE_LINKSTATE
|
1177 /* on error or if no table given do full lookup. This
1178 * is needed for example when nexthops are in the local
1179 * table rather than the given table
1182 err
= fib_lookup(net
, &fl4
, &res
,
1183 FIB_LOOKUP_IGNORE_LINKSTATE
);
1187 NL_SET_ERR_MSG(extack
, "Nexthop has invalid gateway");
1193 if (res
.type
!= RTN_UNICAST
&& res
.type
!= RTN_LOCAL
) {
1194 NL_SET_ERR_MSG(extack
, "Nexthop has invalid gateway");
1197 nh
->fib_nh_scope
= res
.scope
;
1198 nh
->fib_nh_oif
= FIB_RES_OIF(res
);
1199 nh
->fib_nh_dev
= dev
= FIB_RES_DEV(res
);
1201 NL_SET_ERR_MSG(extack
,
1202 "No egress device for nexthop gateway");
1205 netdev_hold(dev
, &nh
->fib_nh_dev_tracker
, GFP_ATOMIC
);
1206 if (!netif_carrier_ok(dev
))
1207 nh
->fib_nh_flags
|= RTNH_F_LINKDOWN
;
1208 err
= (dev
->flags
& IFF_UP
) ? 0 : -ENETDOWN
;
1214 static int fib_check_nh_nongw(struct net
*net
, struct fib_nh
*nh
,
1215 struct netlink_ext_ack
*extack
)
1217 struct in_device
*in_dev
;
1220 if (nh
->fib_nh_flags
& (RTNH_F_PERVASIVE
| RTNH_F_ONLINK
)) {
1221 NL_SET_ERR_MSG(extack
,
1222 "Invalid flags for nexthop - PERVASIVE and ONLINK can not be set");
1229 in_dev
= inetdev_by_index(net
, nh
->fib_nh_oif
);
1233 if (!(in_dev
->dev
->flags
& IFF_UP
)) {
1234 NL_SET_ERR_MSG(extack
, "Device for nexthop is not up");
1238 nh
->fib_nh_dev
= in_dev
->dev
;
1239 netdev_hold(nh
->fib_nh_dev
, &nh
->fib_nh_dev_tracker
, GFP_ATOMIC
);
1240 nh
->fib_nh_scope
= RT_SCOPE_HOST
;
1241 if (!netif_carrier_ok(nh
->fib_nh_dev
))
1242 nh
->fib_nh_flags
|= RTNH_F_LINKDOWN
;
1249 int fib_check_nh(struct net
*net
, struct fib_nh
*nh
, u32 table
, u8 scope
,
1250 struct netlink_ext_ack
*extack
)
1254 if (nh
->fib_nh_gw_family
== AF_INET
)
1255 err
= fib_check_nh_v4_gw(net
, nh
, table
, scope
, extack
);
1256 else if (nh
->fib_nh_gw_family
== AF_INET6
)
1257 err
= fib_check_nh_v6_gw(net
, nh
, table
, extack
);
1259 err
= fib_check_nh_nongw(net
, nh
, extack
);
1264 static struct hlist_head
*
1265 fib_info_laddrhash_bucket(const struct net
*net
, __be32 val
)
1267 u32 slot
= hash_32(net_hash_mix(net
) ^ (__force u32
)val
,
1268 fib_info_hash_bits
);
1270 return &fib_info_laddrhash
[slot
];
1273 static void fib_info_hash_move(struct hlist_head
*new_info_hash
,
1274 struct hlist_head
*new_laddrhash
,
1275 unsigned int new_size
)
1277 struct hlist_head
*old_info_hash
, *old_laddrhash
;
1278 unsigned int old_size
= fib_info_hash_size
;
1281 spin_lock_bh(&fib_info_lock
);
1282 old_info_hash
= fib_info_hash
;
1283 old_laddrhash
= fib_info_laddrhash
;
1284 fib_info_hash_size
= new_size
;
1285 fib_info_hash_bits
= ilog2(new_size
);
1287 for (i
= 0; i
< old_size
; i
++) {
1288 struct hlist_head
*head
= &fib_info_hash
[i
];
1289 struct hlist_node
*n
;
1290 struct fib_info
*fi
;
1292 hlist_for_each_entry_safe(fi
, n
, head
, fib_hash
) {
1293 struct hlist_head
*dest
;
1294 unsigned int new_hash
;
1296 new_hash
= fib_info_hashfn(fi
);
1297 dest
= &new_info_hash
[new_hash
];
1298 hlist_add_head(&fi
->fib_hash
, dest
);
1301 fib_info_hash
= new_info_hash
;
1303 fib_info_laddrhash
= new_laddrhash
;
1304 for (i
= 0; i
< old_size
; i
++) {
1305 struct hlist_head
*lhead
= &old_laddrhash
[i
];
1306 struct hlist_node
*n
;
1307 struct fib_info
*fi
;
1309 hlist_for_each_entry_safe(fi
, n
, lhead
, fib_lhash
) {
1310 struct hlist_head
*ldest
;
1312 ldest
= fib_info_laddrhash_bucket(fi
->fib_net
,
1314 hlist_add_head(&fi
->fib_lhash
, ldest
);
1318 spin_unlock_bh(&fib_info_lock
);
1320 kvfree(old_info_hash
);
1321 kvfree(old_laddrhash
);
1324 __be32
fib_info_update_nhc_saddr(struct net
*net
, struct fib_nh_common
*nhc
,
1325 unsigned char scope
)
1330 if (nhc
->nhc_family
!= AF_INET
)
1331 return inet_select_addr(nhc
->nhc_dev
, 0, scope
);
1333 nh
= container_of(nhc
, struct fib_nh
, nh_common
);
1334 saddr
= inet_select_addr(nh
->fib_nh_dev
, nh
->fib_nh_gw4
, scope
);
1336 WRITE_ONCE(nh
->nh_saddr
, saddr
);
1337 WRITE_ONCE(nh
->nh_saddr_genid
, atomic_read(&net
->ipv4
.dev_addr_genid
));
1342 __be32
fib_result_prefsrc(struct net
*net
, struct fib_result
*res
)
1344 struct fib_nh_common
*nhc
= res
->nhc
;
1346 if (res
->fi
->fib_prefsrc
)
1347 return res
->fi
->fib_prefsrc
;
1349 if (nhc
->nhc_family
== AF_INET
) {
1352 nh
= container_of(nhc
, struct fib_nh
, nh_common
);
1353 if (READ_ONCE(nh
->nh_saddr_genid
) ==
1354 atomic_read(&net
->ipv4
.dev_addr_genid
))
1355 return READ_ONCE(nh
->nh_saddr
);
1358 return fib_info_update_nhc_saddr(net
, nhc
, res
->fi
->fib_scope
);
1361 static bool fib_valid_prefsrc(struct fib_config
*cfg
, __be32 fib_prefsrc
)
1363 if (cfg
->fc_type
!= RTN_LOCAL
|| !cfg
->fc_dst
||
1364 fib_prefsrc
!= cfg
->fc_dst
) {
1365 u32 tb_id
= cfg
->fc_table
;
1368 if (tb_id
== RT_TABLE_MAIN
)
1369 tb_id
= RT_TABLE_LOCAL
;
1371 rc
= inet_addr_type_table(cfg
->fc_nlinfo
.nl_net
,
1372 fib_prefsrc
, tb_id
);
1374 if (rc
!= RTN_LOCAL
&& tb_id
!= RT_TABLE_LOCAL
) {
1375 rc
= inet_addr_type_table(cfg
->fc_nlinfo
.nl_net
,
1376 fib_prefsrc
, RT_TABLE_LOCAL
);
1379 if (rc
!= RTN_LOCAL
)
1385 struct fib_info
*fib_create_info(struct fib_config
*cfg
,
1386 struct netlink_ext_ack
*extack
)
1389 struct fib_info
*fi
= NULL
;
1390 struct nexthop
*nh
= NULL
;
1391 struct fib_info
*ofi
;
1393 struct net
*net
= cfg
->fc_nlinfo
.nl_net
;
1395 if (cfg
->fc_type
> RTN_MAX
)
1398 /* Fast check to catch the most weird cases */
1399 if (fib_props
[cfg
->fc_type
].scope
> cfg
->fc_scope
) {
1400 NL_SET_ERR_MSG(extack
, "Invalid scope");
1404 if (cfg
->fc_flags
& (RTNH_F_DEAD
| RTNH_F_LINKDOWN
)) {
1405 NL_SET_ERR_MSG(extack
,
1406 "Invalid rtm_flags - can not contain DEAD or LINKDOWN");
1410 if (cfg
->fc_nh_id
) {
1412 fi
= fib_find_info_nh(net
, cfg
);
1414 refcount_inc(&fi
->fib_treeref
);
1419 nh
= nexthop_find_by_id(net
, cfg
->fc_nh_id
);
1421 NL_SET_ERR_MSG(extack
, "Nexthop id does not exist");
1427 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1429 nhs
= fib_count_nexthops(cfg
->fc_mp
, cfg
->fc_mp_len
, extack
);
1437 /* Paired with WRITE_ONCE() in fib_release_info() */
1438 if (READ_ONCE(fib_info_cnt
) >= fib_info_hash_size
) {
1439 unsigned int new_size
= fib_info_hash_size
<< 1;
1440 struct hlist_head
*new_info_hash
;
1441 struct hlist_head
*new_laddrhash
;
1446 bytes
= (size_t)new_size
* sizeof(struct hlist_head
*);
1447 new_info_hash
= kvzalloc(bytes
, GFP_KERNEL
);
1448 new_laddrhash
= kvzalloc(bytes
, GFP_KERNEL
);
1449 if (!new_info_hash
|| !new_laddrhash
) {
1450 kvfree(new_info_hash
);
1451 kvfree(new_laddrhash
);
1453 fib_info_hash_move(new_info_hash
, new_laddrhash
, new_size
);
1455 if (!fib_info_hash_size
)
1459 fi
= kzalloc(struct_size(fi
, fib_nh
, nhs
), GFP_KERNEL
);
1462 fi
->fib_metrics
= ip_fib_metrics_init(fi
->fib_net
, cfg
->fc_mx
,
1463 cfg
->fc_mx_len
, extack
);
1464 if (IS_ERR(fi
->fib_metrics
)) {
1465 err
= PTR_ERR(fi
->fib_metrics
);
1467 return ERR_PTR(err
);
1471 fi
->fib_protocol
= cfg
->fc_protocol
;
1472 fi
->fib_scope
= cfg
->fc_scope
;
1473 fi
->fib_flags
= cfg
->fc_flags
;
1474 fi
->fib_priority
= cfg
->fc_priority
;
1475 fi
->fib_prefsrc
= cfg
->fc_prefsrc
;
1476 fi
->fib_type
= cfg
->fc_type
;
1477 fi
->fib_tb_id
= cfg
->fc_table
;
1481 if (!nexthop_get(nh
)) {
1482 NL_SET_ERR_MSG(extack
, "Nexthop has been deleted");
1489 change_nexthops(fi
) {
1490 nexthop_nh
->nh_parent
= fi
;
1491 } endfor_nexthops(fi
)
1494 err
= fib_get_nhs(fi
, cfg
->fc_mp
, cfg
->fc_mp_len
, cfg
,
1497 err
= fib_nh_init(net
, fi
->fib_nh
, cfg
, 1, extack
);
1503 if (fib_props
[cfg
->fc_type
].error
) {
1504 if (cfg
->fc_gw_family
|| cfg
->fc_oif
|| cfg
->fc_mp
) {
1505 NL_SET_ERR_MSG(extack
,
1506 "Gateway, device and multipath can not be specified for this route type");
1511 switch (cfg
->fc_type
) {
1519 NL_SET_ERR_MSG(extack
, "Invalid route type");
1524 if (cfg
->fc_scope
> RT_SCOPE_HOST
) {
1525 NL_SET_ERR_MSG(extack
, "Invalid scope");
1530 err
= fib_check_nexthop(fi
->nh
, cfg
->fc_scope
, extack
);
1533 } else if (cfg
->fc_scope
== RT_SCOPE_HOST
) {
1534 struct fib_nh
*nh
= fi
->fib_nh
;
1536 /* Local address is added. */
1538 NL_SET_ERR_MSG(extack
,
1539 "Route with host scope can not have multiple nexthops");
1542 if (nh
->fib_nh_gw_family
) {
1543 NL_SET_ERR_MSG(extack
,
1544 "Route with host scope can not have a gateway");
1547 nh
->fib_nh_scope
= RT_SCOPE_NOWHERE
;
1548 nh
->fib_nh_dev
= dev_get_by_index(net
, nh
->fib_nh_oif
);
1550 if (!nh
->fib_nh_dev
)
1552 netdev_tracker_alloc(nh
->fib_nh_dev
, &nh
->fib_nh_dev_tracker
,
1557 change_nexthops(fi
) {
1558 err
= fib_check_nh(cfg
->fc_nlinfo
.nl_net
, nexthop_nh
,
1559 cfg
->fc_table
, cfg
->fc_scope
,
1563 if (nexthop_nh
->fib_nh_flags
& RTNH_F_LINKDOWN
)
1565 } endfor_nexthops(fi
)
1566 if (linkdown
== fi
->fib_nhs
)
1567 fi
->fib_flags
|= RTNH_F_LINKDOWN
;
1570 if (fi
->fib_prefsrc
&& !fib_valid_prefsrc(cfg
, fi
->fib_prefsrc
)) {
1571 NL_SET_ERR_MSG(extack
, "Invalid prefsrc address");
1576 change_nexthops(fi
) {
1577 fib_info_update_nhc_saddr(net
, &nexthop_nh
->nh_common
,
1579 if (nexthop_nh
->fib_nh_gw_family
== AF_INET6
)
1580 fi
->fib_nh_is_v6
= true;
1581 } endfor_nexthops(fi
)
1587 ofi
= fib_find_info(fi
);
1589 /* fib_table_lookup() should not see @fi yet. */
1592 refcount_inc(&ofi
->fib_treeref
);
1596 refcount_set(&fi
->fib_treeref
, 1);
1597 refcount_set(&fi
->fib_clntref
, 1);
1598 spin_lock_bh(&fib_info_lock
);
1600 hlist_add_head(&fi
->fib_hash
,
1601 &fib_info_hash
[fib_info_hashfn(fi
)]);
1602 if (fi
->fib_prefsrc
) {
1603 struct hlist_head
*head
;
1605 head
= fib_info_laddrhash_bucket(net
, fi
->fib_prefsrc
);
1606 hlist_add_head(&fi
->fib_lhash
, head
);
1609 list_add(&fi
->nh_list
, &nh
->fi_list
);
1611 change_nexthops(fi
) {
1612 struct hlist_head
*head
;
1614 if (!nexthop_nh
->fib_nh_dev
)
1616 head
= fib_info_devhash_bucket(nexthop_nh
->fib_nh_dev
);
1617 hlist_add_head(&nexthop_nh
->nh_hash
, head
);
1618 } endfor_nexthops(fi
)
1620 spin_unlock_bh(&fib_info_lock
);
1628 /* fib_table_lookup() should not see @fi yet. */
1633 return ERR_PTR(err
);
1636 int fib_nexthop_info(struct sk_buff
*skb
, const struct fib_nh_common
*nhc
,
1637 u8 rt_family
, unsigned char *flags
, bool skip_oif
)
1639 if (nhc
->nhc_flags
& RTNH_F_DEAD
)
1640 *flags
|= RTNH_F_DEAD
;
1642 if (nhc
->nhc_flags
& RTNH_F_LINKDOWN
) {
1643 *flags
|= RTNH_F_LINKDOWN
;
1646 switch (nhc
->nhc_family
) {
1648 if (ip_ignore_linkdown(nhc
->nhc_dev
))
1649 *flags
|= RTNH_F_DEAD
;
1652 if (ip6_ignore_linkdown(nhc
->nhc_dev
))
1653 *flags
|= RTNH_F_DEAD
;
1659 switch (nhc
->nhc_gw_family
) {
1661 if (nla_put_in_addr(skb
, RTA_GATEWAY
, nhc
->nhc_gw
.ipv4
))
1662 goto nla_put_failure
;
1665 /* if gateway family does not match nexthop family
1666 * gateway is encoded as RTA_VIA
1668 if (rt_family
!= nhc
->nhc_gw_family
) {
1669 int alen
= sizeof(struct in6_addr
);
1673 nla
= nla_reserve(skb
, RTA_VIA
, alen
+ 2);
1675 goto nla_put_failure
;
1677 via
= nla_data(nla
);
1678 via
->rtvia_family
= AF_INET6
;
1679 memcpy(via
->rtvia_addr
, &nhc
->nhc_gw
.ipv6
, alen
);
1680 } else if (nla_put_in6_addr(skb
, RTA_GATEWAY
,
1681 &nhc
->nhc_gw
.ipv6
) < 0) {
1682 goto nla_put_failure
;
1687 *flags
|= (nhc
->nhc_flags
&
1688 (RTNH_F_ONLINK
| RTNH_F_OFFLOAD
| RTNH_F_TRAP
));
1690 if (!skip_oif
&& nhc
->nhc_dev
&&
1691 nla_put_u32(skb
, RTA_OIF
, nhc
->nhc_dev
->ifindex
))
1692 goto nla_put_failure
;
1694 if (nhc
->nhc_lwtstate
&&
1695 lwtunnel_fill_encap(skb
, nhc
->nhc_lwtstate
,
1696 RTA_ENCAP
, RTA_ENCAP_TYPE
) < 0)
1697 goto nla_put_failure
;
1704 EXPORT_SYMBOL_GPL(fib_nexthop_info
);
1706 #if IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) || IS_ENABLED(CONFIG_IPV6)
1707 int fib_add_nexthop(struct sk_buff
*skb
, const struct fib_nh_common
*nhc
,
1708 int nh_weight
, u8 rt_family
, u32 nh_tclassid
)
1710 const struct net_device
*dev
= nhc
->nhc_dev
;
1711 struct rtnexthop
*rtnh
;
1712 unsigned char flags
= 0;
1714 rtnh
= nla_reserve_nohdr(skb
, sizeof(*rtnh
));
1716 goto nla_put_failure
;
1718 rtnh
->rtnh_hops
= nh_weight
- 1;
1719 rtnh
->rtnh_ifindex
= dev
? dev
->ifindex
: 0;
1721 if (fib_nexthop_info(skb
, nhc
, rt_family
, &flags
, true) < 0)
1722 goto nla_put_failure
;
1724 rtnh
->rtnh_flags
= flags
;
1726 if (nh_tclassid
&& nla_put_u32(skb
, RTA_FLOW
, nh_tclassid
))
1727 goto nla_put_failure
;
1729 /* length of rtnetlink header + attributes */
1730 rtnh
->rtnh_len
= nlmsg_get_pos(skb
) - (void *)rtnh
;
1737 EXPORT_SYMBOL_GPL(fib_add_nexthop
);
1740 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1741 static int fib_add_multipath(struct sk_buff
*skb
, struct fib_info
*fi
)
1745 mp
= nla_nest_start_noflag(skb
, RTA_MULTIPATH
);
1747 goto nla_put_failure
;
1749 if (unlikely(fi
->nh
)) {
1750 if (nexthop_mpath_fill_node(skb
, fi
->nh
, AF_INET
) < 0)
1751 goto nla_put_failure
;
1756 u32 nh_tclassid
= 0;
1757 #ifdef CONFIG_IP_ROUTE_CLASSID
1758 nh_tclassid
= nh
->nh_tclassid
;
1760 if (fib_add_nexthop(skb
, &nh
->nh_common
, nh
->fib_nh_weight
,
1761 AF_INET
, nh_tclassid
) < 0)
1762 goto nla_put_failure
;
1763 } endfor_nexthops(fi
);
1766 nla_nest_end(skb
, mp
);
1774 static int fib_add_multipath(struct sk_buff
*skb
, struct fib_info
*fi
)
1780 int fib_dump_info(struct sk_buff
*skb
, u32 portid
, u32 seq
, int event
,
1781 const struct fib_rt_info
*fri
, unsigned int flags
)
1783 unsigned int nhs
= fib_info_num_path(fri
->fi
);
1784 struct fib_info
*fi
= fri
->fi
;
1785 u32 tb_id
= fri
->tb_id
;
1786 struct nlmsghdr
*nlh
;
1789 nlh
= nlmsg_put(skb
, portid
, seq
, event
, sizeof(*rtm
), flags
);
1793 rtm
= nlmsg_data(nlh
);
1794 rtm
->rtm_family
= AF_INET
;
1795 rtm
->rtm_dst_len
= fri
->dst_len
;
1796 rtm
->rtm_src_len
= 0;
1797 rtm
->rtm_tos
= inet_dscp_to_dsfield(fri
->dscp
);
1799 rtm
->rtm_table
= tb_id
;
1801 rtm
->rtm_table
= RT_TABLE_COMPAT
;
1802 if (nla_put_u32(skb
, RTA_TABLE
, tb_id
))
1803 goto nla_put_failure
;
1804 rtm
->rtm_type
= fri
->type
;
1805 rtm
->rtm_flags
= fi
->fib_flags
;
1806 rtm
->rtm_scope
= fi
->fib_scope
;
1807 rtm
->rtm_protocol
= fi
->fib_protocol
;
1809 if (rtm
->rtm_dst_len
&&
1810 nla_put_in_addr(skb
, RTA_DST
, fri
->dst
))
1811 goto nla_put_failure
;
1812 if (fi
->fib_priority
&&
1813 nla_put_u32(skb
, RTA_PRIORITY
, fi
->fib_priority
))
1814 goto nla_put_failure
;
1815 if (rtnetlink_put_metrics(skb
, fi
->fib_metrics
->metrics
) < 0)
1816 goto nla_put_failure
;
1818 if (fi
->fib_prefsrc
&&
1819 nla_put_in_addr(skb
, RTA_PREFSRC
, fi
->fib_prefsrc
))
1820 goto nla_put_failure
;
1823 if (nla_put_u32(skb
, RTA_NH_ID
, fi
->nh
->id
))
1824 goto nla_put_failure
;
1825 if (nexthop_is_blackhole(fi
->nh
))
1826 rtm
->rtm_type
= RTN_BLACKHOLE
;
1827 if (!READ_ONCE(fi
->fib_net
->ipv4
.sysctl_nexthop_compat_mode
))
1832 const struct fib_nh_common
*nhc
= fib_info_nhc(fi
, 0);
1833 unsigned char flags
= 0;
1835 if (fib_nexthop_info(skb
, nhc
, AF_INET
, &flags
, false) < 0)
1836 goto nla_put_failure
;
1838 rtm
->rtm_flags
= flags
;
1839 #ifdef CONFIG_IP_ROUTE_CLASSID
1840 if (nhc
->nhc_family
== AF_INET
) {
1843 nh
= container_of(nhc
, struct fib_nh
, nh_common
);
1844 if (nh
->nh_tclassid
&&
1845 nla_put_u32(skb
, RTA_FLOW
, nh
->nh_tclassid
))
1846 goto nla_put_failure
;
1850 if (fib_add_multipath(skb
, fi
) < 0)
1851 goto nla_put_failure
;
1856 rtm
->rtm_flags
|= RTM_F_OFFLOAD
;
1858 rtm
->rtm_flags
|= RTM_F_TRAP
;
1859 if (fri
->offload_failed
)
1860 rtm
->rtm_flags
|= RTM_F_OFFLOAD_FAILED
;
1862 nlmsg_end(skb
, nlh
);
1866 nlmsg_cancel(skb
, nlh
);
1872 * - local address disappeared -> we must delete all the entries
1874 * - device went down -> we must shutdown all nexthops going via it.
1876 int fib_sync_down_addr(struct net_device
*dev
, __be32 local
)
1878 int tb_id
= l3mdev_fib_table(dev
) ? : RT_TABLE_MAIN
;
1879 struct net
*net
= dev_net(dev
);
1880 struct hlist_head
*head
;
1881 struct fib_info
*fi
;
1884 if (!fib_info_laddrhash
|| local
== 0)
1887 head
= fib_info_laddrhash_bucket(net
, local
);
1888 hlist_for_each_entry(fi
, head
, fib_lhash
) {
1889 if (!net_eq(fi
->fib_net
, net
) ||
1890 fi
->fib_tb_id
!= tb_id
)
1892 if (fi
->fib_prefsrc
== local
) {
1893 fi
->fib_flags
|= RTNH_F_DEAD
;
1894 fi
->pfsrc_removed
= true;
1901 static int call_fib_nh_notifiers(struct fib_nh
*nh
,
1902 enum fib_event_type event_type
)
1904 bool ignore_link_down
= ip_ignore_linkdown(nh
->fib_nh_dev
);
1905 struct fib_nh_notifier_info info
= {
1909 switch (event_type
) {
1910 case FIB_EVENT_NH_ADD
:
1911 if (nh
->fib_nh_flags
& RTNH_F_DEAD
)
1913 if (ignore_link_down
&& nh
->fib_nh_flags
& RTNH_F_LINKDOWN
)
1915 return call_fib4_notifiers(dev_net(nh
->fib_nh_dev
), event_type
,
1917 case FIB_EVENT_NH_DEL
:
1918 if ((ignore_link_down
&& nh
->fib_nh_flags
& RTNH_F_LINKDOWN
) ||
1919 (nh
->fib_nh_flags
& RTNH_F_DEAD
))
1920 return call_fib4_notifiers(dev_net(nh
->fib_nh_dev
),
1921 event_type
, &info
.info
);
1930 /* Update the PMTU of exceptions when:
1931 * - the new MTU of the first hop becomes smaller than the PMTU
1932 * - the old MTU was the same as the PMTU, and it limited discovery of
1933 * larger MTUs on the path. With that limit raised, we can now
1934 * discover larger MTUs
1935 * A special case is locked exceptions, for which the PMTU is smaller
1936 * than the minimal accepted PMTU:
1937 * - if the new MTU is greater than the PMTU, don't make any change
1938 * - otherwise, unlock and set PMTU
1940 void fib_nhc_update_mtu(struct fib_nh_common
*nhc
, u32
new, u32 orig
)
1942 struct fnhe_hash_bucket
*bucket
;
1945 bucket
= rcu_dereference_protected(nhc
->nhc_exceptions
, 1);
1949 for (i
= 0; i
< FNHE_HASH_SIZE
; i
++) {
1950 struct fib_nh_exception
*fnhe
;
1952 for (fnhe
= rcu_dereference_protected(bucket
[i
].chain
, 1);
1954 fnhe
= rcu_dereference_protected(fnhe
->fnhe_next
, 1)) {
1955 if (fnhe
->fnhe_mtu_locked
) {
1956 if (new <= fnhe
->fnhe_pmtu
) {
1957 fnhe
->fnhe_pmtu
= new;
1958 fnhe
->fnhe_mtu_locked
= false;
1960 } else if (new < fnhe
->fnhe_pmtu
||
1961 orig
== fnhe
->fnhe_pmtu
) {
1962 fnhe
->fnhe_pmtu
= new;
1968 void fib_sync_mtu(struct net_device
*dev
, u32 orig_mtu
)
1970 struct hlist_head
*head
= fib_info_devhash_bucket(dev
);
1973 hlist_for_each_entry(nh
, head
, nh_hash
) {
1974 if (nh
->fib_nh_dev
== dev
)
1975 fib_nhc_update_mtu(&nh
->nh_common
, dev
->mtu
, orig_mtu
);
1979 /* Event force Flags Description
1980 * NETDEV_CHANGE 0 LINKDOWN Carrier OFF, not for scope host
1981 * NETDEV_DOWN 0 LINKDOWN|DEAD Link down, not for scope host
1982 * NETDEV_DOWN 1 LINKDOWN|DEAD Last address removed
1983 * NETDEV_UNREGISTER 1 LINKDOWN|DEAD Device removed
1985 * only used when fib_nh is built into fib_info
1987 int fib_sync_down_dev(struct net_device
*dev
, unsigned long event
, bool force
)
1989 struct hlist_head
*head
= fib_info_devhash_bucket(dev
);
1990 struct fib_info
*prev_fi
= NULL
;
1991 int scope
= RT_SCOPE_NOWHERE
;
1998 hlist_for_each_entry(nh
, head
, nh_hash
) {
1999 struct fib_info
*fi
= nh
->nh_parent
;
2002 BUG_ON(!fi
->fib_nhs
);
2003 if (nh
->fib_nh_dev
!= dev
|| fi
== prev_fi
)
2007 change_nexthops(fi
) {
2008 if (nexthop_nh
->fib_nh_flags
& RTNH_F_DEAD
)
2010 else if (nexthop_nh
->fib_nh_dev
== dev
&&
2011 nexthop_nh
->fib_nh_scope
!= scope
) {
2014 case NETDEV_UNREGISTER
:
2015 nexthop_nh
->fib_nh_flags
|= RTNH_F_DEAD
;
2018 nexthop_nh
->fib_nh_flags
|= RTNH_F_LINKDOWN
;
2021 call_fib_nh_notifiers(nexthop_nh
,
2025 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2026 if (event
== NETDEV_UNREGISTER
&&
2027 nexthop_nh
->fib_nh_dev
== dev
) {
2032 } endfor_nexthops(fi
)
2033 if (dead
== fi
->fib_nhs
) {
2036 case NETDEV_UNREGISTER
:
2037 fi
->fib_flags
|= RTNH_F_DEAD
;
2040 fi
->fib_flags
|= RTNH_F_LINKDOWN
;
2052 /* Must be invoked inside of an RCU protected region. */
2053 static void fib_select_default(const struct flowi4
*flp
, struct fib_result
*res
)
2055 struct fib_info
*fi
= NULL
, *last_resort
= NULL
;
2056 struct hlist_head
*fa_head
= res
->fa_head
;
2057 struct fib_table
*tb
= res
->table
;
2058 u8 slen
= 32 - res
->prefixlen
;
2059 int order
= -1, last_idx
= -1;
2060 struct fib_alias
*fa
, *fa1
= NULL
;
2061 u32 last_prio
= res
->fi
->fib_priority
;
2062 dscp_t last_dscp
= 0;
2064 hlist_for_each_entry_rcu(fa
, fa_head
, fa_list
) {
2065 struct fib_info
*next_fi
= fa
->fa_info
;
2066 struct fib_nh_common
*nhc
;
2068 if (fa
->fa_slen
!= slen
)
2071 fa
->fa_dscp
!= inet_dsfield_to_dscp(flp
->flowi4_tos
))
2073 if (fa
->tb_id
!= tb
->tb_id
)
2075 if (next_fi
->fib_priority
> last_prio
&&
2076 fa
->fa_dscp
== last_dscp
) {
2081 if (next_fi
->fib_flags
& RTNH_F_DEAD
)
2083 last_dscp
= fa
->fa_dscp
;
2084 last_prio
= next_fi
->fib_priority
;
2086 if (next_fi
->fib_scope
!= res
->scope
||
2087 fa
->fa_type
!= RTN_UNICAST
)
2090 nhc
= fib_info_nhc(next_fi
, 0);
2091 if (!nhc
->nhc_gw_family
|| nhc
->nhc_scope
!= RT_SCOPE_LINK
)
2094 fib_alias_accessed(fa
);
2097 if (next_fi
!= res
->fi
)
2100 } else if (!fib_detect_death(fi
, order
, &last_resort
,
2101 &last_idx
, fa1
->fa_default
)) {
2102 fib_result_assign(res
, fi
);
2103 fa1
->fa_default
= order
;
2110 if (order
<= 0 || !fi
) {
2112 fa1
->fa_default
= -1;
2116 if (!fib_detect_death(fi
, order
, &last_resort
, &last_idx
,
2118 fib_result_assign(res
, fi
);
2119 fa1
->fa_default
= order
;
2124 fib_result_assign(res
, last_resort
);
2125 fa1
->fa_default
= last_idx
;
2131 * Dead device goes up. We wake up dead nexthops.
2132 * It takes sense only on multipath routes.
2134 * only used when fib_nh is built into fib_info
2136 int fib_sync_up(struct net_device
*dev
, unsigned char nh_flags
)
2138 struct fib_info
*prev_fi
;
2139 struct hlist_head
*head
;
2143 if (!(dev
->flags
& IFF_UP
))
2146 if (nh_flags
& RTNH_F_DEAD
) {
2147 unsigned int flags
= dev_get_flags(dev
);
2149 if (flags
& (IFF_RUNNING
| IFF_LOWER_UP
))
2150 nh_flags
|= RTNH_F_LINKDOWN
;
2154 head
= fib_info_devhash_bucket(dev
);
2157 hlist_for_each_entry(nh
, head
, nh_hash
) {
2158 struct fib_info
*fi
= nh
->nh_parent
;
2161 BUG_ON(!fi
->fib_nhs
);
2162 if (nh
->fib_nh_dev
!= dev
|| fi
== prev_fi
)
2167 change_nexthops(fi
) {
2168 if (!(nexthop_nh
->fib_nh_flags
& nh_flags
)) {
2172 if (!nexthop_nh
->fib_nh_dev
||
2173 !(nexthop_nh
->fib_nh_dev
->flags
& IFF_UP
))
2175 if (nexthop_nh
->fib_nh_dev
!= dev
||
2176 !__in_dev_get_rtnl(dev
))
2179 nexthop_nh
->fib_nh_flags
&= ~nh_flags
;
2180 call_fib_nh_notifiers(nexthop_nh
, FIB_EVENT_NH_ADD
);
2181 } endfor_nexthops(fi
)
2184 fi
->fib_flags
&= ~nh_flags
;
2194 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2195 static bool fib_good_nh(const struct fib_nh
*nh
)
2197 int state
= NUD_REACHABLE
;
2199 if (nh
->fib_nh_scope
== RT_SCOPE_LINK
) {
2200 struct neighbour
*n
;
2204 if (likely(nh
->fib_nh_gw_family
== AF_INET
))
2205 n
= __ipv4_neigh_lookup_noref(nh
->fib_nh_dev
,
2206 (__force u32
)nh
->fib_nh_gw4
);
2207 else if (nh
->fib_nh_gw_family
== AF_INET6
)
2208 n
= __ipv6_neigh_lookup_noref_stub(nh
->fib_nh_dev
,
2213 state
= READ_ONCE(n
->nud_state
);
2218 return !!(state
& NUD_VALID
);
2221 void fib_select_multipath(struct fib_result
*res
, int hash
)
2223 struct fib_info
*fi
= res
->fi
;
2224 struct net
*net
= fi
->fib_net
;
2227 if (unlikely(res
->fi
->nh
)) {
2228 nexthop_path_fib_result(res
, hash
);
2232 change_nexthops(fi
) {
2233 if (READ_ONCE(net
->ipv4
.sysctl_fib_multipath_use_neigh
)) {
2234 if (!fib_good_nh(nexthop_nh
))
2237 res
->nh_sel
= nhsel
;
2238 res
->nhc
= &nexthop_nh
->nh_common
;
2243 if (hash
> atomic_read(&nexthop_nh
->fib_nh_upper_bound
))
2246 res
->nh_sel
= nhsel
;
2247 res
->nhc
= &nexthop_nh
->nh_common
;
2249 } endfor_nexthops(fi
);
2253 void fib_select_path(struct net
*net
, struct fib_result
*res
,
2254 struct flowi4
*fl4
, const struct sk_buff
*skb
)
2256 if (fl4
->flowi4_oif
)
2259 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2260 if (fib_info_num_path(res
->fi
) > 1) {
2261 int h
= fib_multipath_hash(net
, fl4
, skb
, NULL
);
2263 fib_select_multipath(res
, h
);
2267 if (!res
->prefixlen
&&
2268 res
->table
->tb_num_default
> 1 &&
2269 res
->type
== RTN_UNICAST
)
2270 fib_select_default(fl4
, res
);
2274 fl4
->saddr
= fib_result_prefsrc(net
, res
);