1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kernel.h>
3 #include <linux/skbuff.h>
4 #include <linux/export.h>
6 #include <linux/ipv6.h>
7 #include <linux/if_vlan.h>
9 #include <net/dst_metadata.h>
15 #include <linux/igmp.h>
16 #include <linux/icmp.h>
17 #include <linux/sctp.h>
18 #include <linux/dccp.h>
19 #include <linux/if_tunnel.h>
20 #include <linux/if_pppox.h>
21 #include <linux/ppp_defs.h>
22 #include <linux/stddef.h>
23 #include <linux/if_ether.h>
24 #include <linux/mpls.h>
25 #include <linux/tcp.h>
26 #include <net/flow_dissector.h>
27 #include <scsi/fc/fc_fcoe.h>
28 #include <uapi/linux/batadv_packet.h>
29 #include <linux/bpf.h>
30 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
31 #include <net/netfilter/nf_conntrack_core.h>
32 #include <net/netfilter/nf_conntrack_labels.h>
35 static DEFINE_MUTEX(flow_dissector_mutex
);
37 static void dissector_set_key(struct flow_dissector
*flow_dissector
,
38 enum flow_dissector_key_id key_id
)
40 flow_dissector
->used_keys
|= (1 << key_id
);
43 void skb_flow_dissector_init(struct flow_dissector
*flow_dissector
,
44 const struct flow_dissector_key
*key
,
45 unsigned int key_count
)
49 memset(flow_dissector
, 0, sizeof(*flow_dissector
));
51 for (i
= 0; i
< key_count
; i
++, key
++) {
52 /* User should make sure that every key target offset is withing
53 * boundaries of unsigned short.
55 BUG_ON(key
->offset
> USHRT_MAX
);
56 BUG_ON(dissector_uses_key(flow_dissector
,
59 dissector_set_key(flow_dissector
, key
->key_id
);
60 flow_dissector
->offset
[key
->key_id
] = key
->offset
;
63 /* Ensure that the dissector always includes control and basic key.
64 * That way we are able to avoid handling lack of these in fast path.
66 BUG_ON(!dissector_uses_key(flow_dissector
,
67 FLOW_DISSECTOR_KEY_CONTROL
));
68 BUG_ON(!dissector_uses_key(flow_dissector
,
69 FLOW_DISSECTOR_KEY_BASIC
));
71 EXPORT_SYMBOL(skb_flow_dissector_init
);
73 int skb_flow_dissector_prog_query(const union bpf_attr
*attr
,
74 union bpf_attr __user
*uattr
)
76 __u32 __user
*prog_ids
= u64_to_user_ptr(attr
->query
.prog_ids
);
77 u32 prog_id
, prog_cnt
= 0, flags
= 0;
78 struct bpf_prog
*attached
;
81 if (attr
->query
.query_flags
)
84 net
= get_net_ns_by_fd(attr
->query
.target_fd
);
89 attached
= rcu_dereference(net
->flow_dissector_prog
);
92 prog_id
= attached
->aux
->id
;
98 if (copy_to_user(&uattr
->query
.attach_flags
, &flags
, sizeof(flags
)))
100 if (copy_to_user(&uattr
->query
.prog_cnt
, &prog_cnt
, sizeof(prog_cnt
)))
103 if (!attr
->query
.prog_cnt
|| !prog_ids
|| !prog_cnt
)
106 if (copy_to_user(prog_ids
, &prog_id
, sizeof(u32
)))
112 int skb_flow_dissector_bpf_prog_attach(const union bpf_attr
*attr
,
113 struct bpf_prog
*prog
)
115 struct bpf_prog
*attached
;
119 net
= current
->nsproxy
->net_ns
;
120 mutex_lock(&flow_dissector_mutex
);
122 if (net
== &init_net
) {
123 /* BPF flow dissector in the root namespace overrides
124 * any per-net-namespace one. When attaching to root,
125 * make sure we don't have any BPF program attached
126 * to the non-root namespaces.
133 if (rcu_access_pointer(ns
->flow_dissector_prog
)) {
139 /* Make sure root flow dissector is not attached
140 * when attaching to the non-root namespace.
142 if (rcu_access_pointer(init_net
.flow_dissector_prog
)) {
148 attached
= rcu_dereference_protected(net
->flow_dissector_prog
,
149 lockdep_is_held(&flow_dissector_mutex
));
150 if (attached
== prog
) {
151 /* The same program cannot be attached twice */
155 rcu_assign_pointer(net
->flow_dissector_prog
, prog
);
157 bpf_prog_put(attached
);
159 mutex_unlock(&flow_dissector_mutex
);
163 static int flow_dissector_bpf_prog_detach(struct net
*net
)
165 struct bpf_prog
*attached
;
167 mutex_lock(&flow_dissector_mutex
);
168 attached
= rcu_dereference_protected(net
->flow_dissector_prog
,
169 lockdep_is_held(&flow_dissector_mutex
));
171 mutex_unlock(&flow_dissector_mutex
);
174 RCU_INIT_POINTER(net
->flow_dissector_prog
, NULL
);
175 bpf_prog_put(attached
);
176 mutex_unlock(&flow_dissector_mutex
);
180 int skb_flow_dissector_bpf_prog_detach(const union bpf_attr
*attr
)
182 return flow_dissector_bpf_prog_detach(current
->nsproxy
->net_ns
);
185 static void __net_exit
flow_dissector_pernet_pre_exit(struct net
*net
)
187 /* We're not racing with attach/detach because there are no
188 * references to netns left when pre_exit gets called.
190 if (rcu_access_pointer(net
->flow_dissector_prog
))
191 flow_dissector_bpf_prog_detach(net
);
194 static struct pernet_operations flow_dissector_pernet_ops __net_initdata
= {
195 .pre_exit
= flow_dissector_pernet_pre_exit
,
199 * __skb_flow_get_ports - extract the upper layer ports and return them
200 * @skb: sk_buff to extract the ports from
201 * @thoff: transport header offset
202 * @ip_proto: protocol for which to get port offset
203 * @data: raw buffer pointer to the packet, if NULL use skb->data
204 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
206 * The function will try to retrieve the ports at offset thoff + poff where poff
207 * is the protocol port offset returned from proto_ports_offset
209 __be32
__skb_flow_get_ports(const struct sk_buff
*skb
, int thoff
, u8 ip_proto
,
210 void *data
, int hlen
)
212 int poff
= proto_ports_offset(ip_proto
);
216 hlen
= skb_headlen(skb
);
220 __be32
*ports
, _ports
;
222 ports
= __skb_header_pointer(skb
, thoff
+ poff
,
223 sizeof(_ports
), data
, hlen
, &_ports
);
230 EXPORT_SYMBOL(__skb_flow_get_ports
);
232 static bool icmp_has_id(u8 type
)
238 case ICMP_TIMESTAMPREPLY
:
239 case ICMPV6_ECHO_REQUEST
:
240 case ICMPV6_ECHO_REPLY
:
248 * skb_flow_get_icmp_tci - extract ICMP(6) Type, Code and Identifier fields
249 * @skb: sk_buff to extract from
250 * @key_icmp: struct flow_dissector_key_icmp to fill
251 * @data: raw buffer pointer to the packet
252 * @thoff: offset to extract at
253 * @hlen: packet header length
255 void skb_flow_get_icmp_tci(const struct sk_buff
*skb
,
256 struct flow_dissector_key_icmp
*key_icmp
,
257 void *data
, int thoff
, int hlen
)
259 struct icmphdr
*ih
, _ih
;
261 ih
= __skb_header_pointer(skb
, thoff
, sizeof(_ih
), data
, hlen
, &_ih
);
265 key_icmp
->type
= ih
->type
;
266 key_icmp
->code
= ih
->code
;
268 /* As we use 0 to signal that the Id field is not present,
269 * avoid confusion with packets without such field
271 if (icmp_has_id(ih
->type
))
272 key_icmp
->id
= ih
->un
.echo
.id
? : 1;
276 EXPORT_SYMBOL(skb_flow_get_icmp_tci
);
278 /* If FLOW_DISSECTOR_KEY_ICMP is set, dissect an ICMP packet
279 * using skb_flow_get_icmp_tci().
281 static void __skb_flow_dissect_icmp(const struct sk_buff
*skb
,
282 struct flow_dissector
*flow_dissector
,
283 void *target_container
,
284 void *data
, int thoff
, int hlen
)
286 struct flow_dissector_key_icmp
*key_icmp
;
288 if (!dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_ICMP
))
291 key_icmp
= skb_flow_dissector_target(flow_dissector
,
292 FLOW_DISSECTOR_KEY_ICMP
,
295 skb_flow_get_icmp_tci(skb
, key_icmp
, data
, thoff
, hlen
);
298 void skb_flow_dissect_meta(const struct sk_buff
*skb
,
299 struct flow_dissector
*flow_dissector
,
300 void *target_container
)
302 struct flow_dissector_key_meta
*meta
;
304 if (!dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_META
))
307 meta
= skb_flow_dissector_target(flow_dissector
,
308 FLOW_DISSECTOR_KEY_META
,
310 meta
->ingress_ifindex
= skb
->skb_iif
;
312 EXPORT_SYMBOL(skb_flow_dissect_meta
);
315 skb_flow_dissect_set_enc_addr_type(enum flow_dissector_key_id type
,
316 struct flow_dissector
*flow_dissector
,
317 void *target_container
)
319 struct flow_dissector_key_control
*ctrl
;
321 if (!dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_ENC_CONTROL
))
324 ctrl
= skb_flow_dissector_target(flow_dissector
,
325 FLOW_DISSECTOR_KEY_ENC_CONTROL
,
327 ctrl
->addr_type
= type
;
331 skb_flow_dissect_ct(const struct sk_buff
*skb
,
332 struct flow_dissector
*flow_dissector
,
333 void *target_container
,
337 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
338 struct flow_dissector_key_ct
*key
;
339 enum ip_conntrack_info ctinfo
;
340 struct nf_conn_labels
*cl
;
343 if (!dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_CT
))
346 ct
= nf_ct_get(skb
, &ctinfo
);
350 key
= skb_flow_dissector_target(flow_dissector
,
351 FLOW_DISSECTOR_KEY_CT
,
354 if (ctinfo
< mapsize
)
355 key
->ct_state
= ctinfo_map
[ctinfo
];
356 #if IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)
357 key
->ct_zone
= ct
->zone
.id
;
359 #if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
360 key
->ct_mark
= ct
->mark
;
363 cl
= nf_ct_labels_find(ct
);
365 memcpy(key
->ct_labels
, cl
->bits
, sizeof(key
->ct_labels
));
366 #endif /* CONFIG_NF_CONNTRACK */
368 EXPORT_SYMBOL(skb_flow_dissect_ct
);
371 skb_flow_dissect_tunnel_info(const struct sk_buff
*skb
,
372 struct flow_dissector
*flow_dissector
,
373 void *target_container
)
375 struct ip_tunnel_info
*info
;
376 struct ip_tunnel_key
*key
;
378 /* A quick check to see if there might be something to do. */
379 if (!dissector_uses_key(flow_dissector
,
380 FLOW_DISSECTOR_KEY_ENC_KEYID
) &&
381 !dissector_uses_key(flow_dissector
,
382 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
) &&
383 !dissector_uses_key(flow_dissector
,
384 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS
) &&
385 !dissector_uses_key(flow_dissector
,
386 FLOW_DISSECTOR_KEY_ENC_CONTROL
) &&
387 !dissector_uses_key(flow_dissector
,
388 FLOW_DISSECTOR_KEY_ENC_PORTS
) &&
389 !dissector_uses_key(flow_dissector
,
390 FLOW_DISSECTOR_KEY_ENC_IP
) &&
391 !dissector_uses_key(flow_dissector
,
392 FLOW_DISSECTOR_KEY_ENC_OPTS
))
395 info
= skb_tunnel_info(skb
);
401 switch (ip_tunnel_info_af(info
)) {
403 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV4_ADDRS
,
406 if (dissector_uses_key(flow_dissector
,
407 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
)) {
408 struct flow_dissector_key_ipv4_addrs
*ipv4
;
410 ipv4
= skb_flow_dissector_target(flow_dissector
,
411 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
,
413 ipv4
->src
= key
->u
.ipv4
.src
;
414 ipv4
->dst
= key
->u
.ipv4
.dst
;
418 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV6_ADDRS
,
421 if (dissector_uses_key(flow_dissector
,
422 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS
)) {
423 struct flow_dissector_key_ipv6_addrs
*ipv6
;
425 ipv6
= skb_flow_dissector_target(flow_dissector
,
426 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS
,
428 ipv6
->src
= key
->u
.ipv6
.src
;
429 ipv6
->dst
= key
->u
.ipv6
.dst
;
434 if (dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_ENC_KEYID
)) {
435 struct flow_dissector_key_keyid
*keyid
;
437 keyid
= skb_flow_dissector_target(flow_dissector
,
438 FLOW_DISSECTOR_KEY_ENC_KEYID
,
440 keyid
->keyid
= tunnel_id_to_key32(key
->tun_id
);
443 if (dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_ENC_PORTS
)) {
444 struct flow_dissector_key_ports
*tp
;
446 tp
= skb_flow_dissector_target(flow_dissector
,
447 FLOW_DISSECTOR_KEY_ENC_PORTS
,
449 tp
->src
= key
->tp_src
;
450 tp
->dst
= key
->tp_dst
;
453 if (dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_ENC_IP
)) {
454 struct flow_dissector_key_ip
*ip
;
456 ip
= skb_flow_dissector_target(flow_dissector
,
457 FLOW_DISSECTOR_KEY_ENC_IP
,
463 if (dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_ENC_OPTS
)) {
464 struct flow_dissector_key_enc_opts
*enc_opt
;
466 enc_opt
= skb_flow_dissector_target(flow_dissector
,
467 FLOW_DISSECTOR_KEY_ENC_OPTS
,
470 if (info
->options_len
) {
471 enc_opt
->len
= info
->options_len
;
472 ip_tunnel_info_opts_get(enc_opt
->data
, info
);
473 enc_opt
->dst_opt_type
= info
->key
.tun_flags
&
474 TUNNEL_OPTIONS_PRESENT
;
478 EXPORT_SYMBOL(skb_flow_dissect_tunnel_info
);
480 static enum flow_dissect_ret
481 __skb_flow_dissect_mpls(const struct sk_buff
*skb
,
482 struct flow_dissector
*flow_dissector
,
483 void *target_container
, void *data
, int nhoff
, int hlen
)
485 struct flow_dissector_key_keyid
*key_keyid
;
486 struct mpls_label
*hdr
, _hdr
[2];
489 if (!dissector_uses_key(flow_dissector
,
490 FLOW_DISSECTOR_KEY_MPLS_ENTROPY
) &&
491 !dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_MPLS
))
492 return FLOW_DISSECT_RET_OUT_GOOD
;
494 hdr
= __skb_header_pointer(skb
, nhoff
, sizeof(_hdr
), data
,
497 return FLOW_DISSECT_RET_OUT_BAD
;
499 entry
= ntohl(hdr
[0].entry
);
500 label
= (entry
& MPLS_LS_LABEL_MASK
) >> MPLS_LS_LABEL_SHIFT
;
502 if (dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_MPLS
)) {
503 struct flow_dissector_key_mpls
*key_mpls
;
505 key_mpls
= skb_flow_dissector_target(flow_dissector
,
506 FLOW_DISSECTOR_KEY_MPLS
,
508 key_mpls
->mpls_label
= label
;
509 key_mpls
->mpls_ttl
= (entry
& MPLS_LS_TTL_MASK
)
510 >> MPLS_LS_TTL_SHIFT
;
511 key_mpls
->mpls_tc
= (entry
& MPLS_LS_TC_MASK
)
513 key_mpls
->mpls_bos
= (entry
& MPLS_LS_S_MASK
)
517 if (label
== MPLS_LABEL_ENTROPY
) {
518 key_keyid
= skb_flow_dissector_target(flow_dissector
,
519 FLOW_DISSECTOR_KEY_MPLS_ENTROPY
,
521 key_keyid
->keyid
= hdr
[1].entry
& htonl(MPLS_LS_LABEL_MASK
);
523 return FLOW_DISSECT_RET_OUT_GOOD
;
526 static enum flow_dissect_ret
527 __skb_flow_dissect_arp(const struct sk_buff
*skb
,
528 struct flow_dissector
*flow_dissector
,
529 void *target_container
, void *data
, int nhoff
, int hlen
)
531 struct flow_dissector_key_arp
*key_arp
;
533 unsigned char ar_sha
[ETH_ALEN
];
534 unsigned char ar_sip
[4];
535 unsigned char ar_tha
[ETH_ALEN
];
536 unsigned char ar_tip
[4];
537 } *arp_eth
, _arp_eth
;
538 const struct arphdr
*arp
;
541 if (!dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_ARP
))
542 return FLOW_DISSECT_RET_OUT_GOOD
;
544 arp
= __skb_header_pointer(skb
, nhoff
, sizeof(_arp
), data
,
547 return FLOW_DISSECT_RET_OUT_BAD
;
549 if (arp
->ar_hrd
!= htons(ARPHRD_ETHER
) ||
550 arp
->ar_pro
!= htons(ETH_P_IP
) ||
551 arp
->ar_hln
!= ETH_ALEN
||
553 (arp
->ar_op
!= htons(ARPOP_REPLY
) &&
554 arp
->ar_op
!= htons(ARPOP_REQUEST
)))
555 return FLOW_DISSECT_RET_OUT_BAD
;
557 arp_eth
= __skb_header_pointer(skb
, nhoff
+ sizeof(_arp
),
558 sizeof(_arp_eth
), data
,
561 return FLOW_DISSECT_RET_OUT_BAD
;
563 key_arp
= skb_flow_dissector_target(flow_dissector
,
564 FLOW_DISSECTOR_KEY_ARP
,
567 memcpy(&key_arp
->sip
, arp_eth
->ar_sip
, sizeof(key_arp
->sip
));
568 memcpy(&key_arp
->tip
, arp_eth
->ar_tip
, sizeof(key_arp
->tip
));
570 /* Only store the lower byte of the opcode;
571 * this covers ARPOP_REPLY and ARPOP_REQUEST.
573 key_arp
->op
= ntohs(arp
->ar_op
) & 0xff;
575 ether_addr_copy(key_arp
->sha
, arp_eth
->ar_sha
);
576 ether_addr_copy(key_arp
->tha
, arp_eth
->ar_tha
);
578 return FLOW_DISSECT_RET_OUT_GOOD
;
581 static enum flow_dissect_ret
582 __skb_flow_dissect_gre(const struct sk_buff
*skb
,
583 struct flow_dissector_key_control
*key_control
,
584 struct flow_dissector
*flow_dissector
,
585 void *target_container
, void *data
,
586 __be16
*p_proto
, int *p_nhoff
, int *p_hlen
,
589 struct flow_dissector_key_keyid
*key_keyid
;
590 struct gre_base_hdr
*hdr
, _hdr
;
594 hdr
= __skb_header_pointer(skb
, *p_nhoff
, sizeof(_hdr
),
595 data
, *p_hlen
, &_hdr
);
597 return FLOW_DISSECT_RET_OUT_BAD
;
599 /* Only look inside GRE without routing */
600 if (hdr
->flags
& GRE_ROUTING
)
601 return FLOW_DISSECT_RET_OUT_GOOD
;
603 /* Only look inside GRE for version 0 and 1 */
604 gre_ver
= ntohs(hdr
->flags
& GRE_VERSION
);
606 return FLOW_DISSECT_RET_OUT_GOOD
;
608 *p_proto
= hdr
->protocol
;
610 /* Version1 must be PPTP, and check the flags */
611 if (!(*p_proto
== GRE_PROTO_PPP
&& (hdr
->flags
& GRE_KEY
)))
612 return FLOW_DISSECT_RET_OUT_GOOD
;
615 offset
+= sizeof(struct gre_base_hdr
);
617 if (hdr
->flags
& GRE_CSUM
)
618 offset
+= sizeof_field(struct gre_full_hdr
, csum
) +
619 sizeof_field(struct gre_full_hdr
, reserved1
);
621 if (hdr
->flags
& GRE_KEY
) {
625 keyid
= __skb_header_pointer(skb
, *p_nhoff
+ offset
,
627 data
, *p_hlen
, &_keyid
);
629 return FLOW_DISSECT_RET_OUT_BAD
;
631 if (dissector_uses_key(flow_dissector
,
632 FLOW_DISSECTOR_KEY_GRE_KEYID
)) {
633 key_keyid
= skb_flow_dissector_target(flow_dissector
,
634 FLOW_DISSECTOR_KEY_GRE_KEYID
,
637 key_keyid
->keyid
= *keyid
;
639 key_keyid
->keyid
= *keyid
& GRE_PPTP_KEY_MASK
;
641 offset
+= sizeof_field(struct gre_full_hdr
, key
);
644 if (hdr
->flags
& GRE_SEQ
)
645 offset
+= sizeof_field(struct pptp_gre_header
, seq
);
648 if (*p_proto
== htons(ETH_P_TEB
)) {
649 const struct ethhdr
*eth
;
652 eth
= __skb_header_pointer(skb
, *p_nhoff
+ offset
,
654 data
, *p_hlen
, &_eth
);
656 return FLOW_DISSECT_RET_OUT_BAD
;
657 *p_proto
= eth
->h_proto
;
658 offset
+= sizeof(*eth
);
660 /* Cap headers that we access via pointers at the
661 * end of the Ethernet header as our maximum alignment
662 * at that point is only 2 bytes.
665 *p_hlen
= *p_nhoff
+ offset
;
667 } else { /* version 1, must be PPTP */
668 u8 _ppp_hdr
[PPP_HDRLEN
];
671 if (hdr
->flags
& GRE_ACK
)
672 offset
+= sizeof_field(struct pptp_gre_header
, ack
);
674 ppp_hdr
= __skb_header_pointer(skb
, *p_nhoff
+ offset
,
676 data
, *p_hlen
, _ppp_hdr
);
678 return FLOW_DISSECT_RET_OUT_BAD
;
680 switch (PPP_PROTOCOL(ppp_hdr
)) {
682 *p_proto
= htons(ETH_P_IP
);
685 *p_proto
= htons(ETH_P_IPV6
);
688 /* Could probably catch some more like MPLS */
692 offset
+= PPP_HDRLEN
;
696 key_control
->flags
|= FLOW_DIS_ENCAPSULATION
;
697 if (flags
& FLOW_DISSECTOR_F_STOP_AT_ENCAP
)
698 return FLOW_DISSECT_RET_OUT_GOOD
;
700 return FLOW_DISSECT_RET_PROTO_AGAIN
;
704 * __skb_flow_dissect_batadv() - dissect batman-adv header
705 * @skb: sk_buff to with the batman-adv header
706 * @key_control: flow dissectors control key
707 * @data: raw buffer pointer to the packet, if NULL use skb->data
708 * @p_proto: pointer used to update the protocol to process next
709 * @p_nhoff: pointer used to update inner network header offset
710 * @hlen: packet header length
711 * @flags: any combination of FLOW_DISSECTOR_F_*
713 * ETH_P_BATMAN packets are tried to be dissected. Only
714 * &struct batadv_unicast packets are actually processed because they contain an
715 * inner ethernet header and are usually followed by actual network header. This
716 * allows the flow dissector to continue processing the packet.
718 * Return: FLOW_DISSECT_RET_PROTO_AGAIN when &struct batadv_unicast was found,
719 * FLOW_DISSECT_RET_OUT_GOOD when dissector should stop after encapsulation,
720 * otherwise FLOW_DISSECT_RET_OUT_BAD
722 static enum flow_dissect_ret
723 __skb_flow_dissect_batadv(const struct sk_buff
*skb
,
724 struct flow_dissector_key_control
*key_control
,
725 void *data
, __be16
*p_proto
, int *p_nhoff
, int hlen
,
729 struct batadv_unicast_packet batadv_unicast
;
733 hdr
= __skb_header_pointer(skb
, *p_nhoff
, sizeof(_hdr
), data
, hlen
,
736 return FLOW_DISSECT_RET_OUT_BAD
;
738 if (hdr
->batadv_unicast
.version
!= BATADV_COMPAT_VERSION
)
739 return FLOW_DISSECT_RET_OUT_BAD
;
741 if (hdr
->batadv_unicast
.packet_type
!= BATADV_UNICAST
)
742 return FLOW_DISSECT_RET_OUT_BAD
;
744 *p_proto
= hdr
->eth
.h_proto
;
745 *p_nhoff
+= sizeof(*hdr
);
747 key_control
->flags
|= FLOW_DIS_ENCAPSULATION
;
748 if (flags
& FLOW_DISSECTOR_F_STOP_AT_ENCAP
)
749 return FLOW_DISSECT_RET_OUT_GOOD
;
751 return FLOW_DISSECT_RET_PROTO_AGAIN
;
755 __skb_flow_dissect_tcp(const struct sk_buff
*skb
,
756 struct flow_dissector
*flow_dissector
,
757 void *target_container
, void *data
, int thoff
, int hlen
)
759 struct flow_dissector_key_tcp
*key_tcp
;
760 struct tcphdr
*th
, _th
;
762 if (!dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_TCP
))
765 th
= __skb_header_pointer(skb
, thoff
, sizeof(_th
), data
, hlen
, &_th
);
769 if (unlikely(__tcp_hdrlen(th
) < sizeof(_th
)))
772 key_tcp
= skb_flow_dissector_target(flow_dissector
,
773 FLOW_DISSECTOR_KEY_TCP
,
775 key_tcp
->flags
= (*(__be16
*) &tcp_flag_word(th
) & htons(0x0FFF));
779 __skb_flow_dissect_ports(const struct sk_buff
*skb
,
780 struct flow_dissector
*flow_dissector
,
781 void *target_container
, void *data
, int nhoff
,
782 u8 ip_proto
, int hlen
)
784 enum flow_dissector_key_id dissector_ports
= FLOW_DISSECTOR_KEY_MAX
;
785 struct flow_dissector_key_ports
*key_ports
;
787 if (dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_PORTS
))
788 dissector_ports
= FLOW_DISSECTOR_KEY_PORTS
;
789 else if (dissector_uses_key(flow_dissector
,
790 FLOW_DISSECTOR_KEY_PORTS_RANGE
))
791 dissector_ports
= FLOW_DISSECTOR_KEY_PORTS_RANGE
;
793 if (dissector_ports
== FLOW_DISSECTOR_KEY_MAX
)
796 key_ports
= skb_flow_dissector_target(flow_dissector
,
799 key_ports
->ports
= __skb_flow_get_ports(skb
, nhoff
, ip_proto
,
804 __skb_flow_dissect_ipv4(const struct sk_buff
*skb
,
805 struct flow_dissector
*flow_dissector
,
806 void *target_container
, void *data
, const struct iphdr
*iph
)
808 struct flow_dissector_key_ip
*key_ip
;
810 if (!dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_IP
))
813 key_ip
= skb_flow_dissector_target(flow_dissector
,
814 FLOW_DISSECTOR_KEY_IP
,
816 key_ip
->tos
= iph
->tos
;
817 key_ip
->ttl
= iph
->ttl
;
821 __skb_flow_dissect_ipv6(const struct sk_buff
*skb
,
822 struct flow_dissector
*flow_dissector
,
823 void *target_container
, void *data
, const struct ipv6hdr
*iph
)
825 struct flow_dissector_key_ip
*key_ip
;
827 if (!dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_IP
))
830 key_ip
= skb_flow_dissector_target(flow_dissector
,
831 FLOW_DISSECTOR_KEY_IP
,
833 key_ip
->tos
= ipv6_get_dsfield(iph
);
834 key_ip
->ttl
= iph
->hop_limit
;
837 /* Maximum number of protocol headers that can be parsed in
840 #define MAX_FLOW_DISSECT_HDRS 15
842 static bool skb_flow_dissect_allowed(int *num_hdrs
)
846 return (*num_hdrs
<= MAX_FLOW_DISSECT_HDRS
);
849 static void __skb_flow_bpf_to_target(const struct bpf_flow_keys
*flow_keys
,
850 struct flow_dissector
*flow_dissector
,
851 void *target_container
)
853 struct flow_dissector_key_ports
*key_ports
= NULL
;
854 struct flow_dissector_key_control
*key_control
;
855 struct flow_dissector_key_basic
*key_basic
;
856 struct flow_dissector_key_addrs
*key_addrs
;
857 struct flow_dissector_key_tags
*key_tags
;
859 key_control
= skb_flow_dissector_target(flow_dissector
,
860 FLOW_DISSECTOR_KEY_CONTROL
,
862 key_control
->thoff
= flow_keys
->thoff
;
863 if (flow_keys
->is_frag
)
864 key_control
->flags
|= FLOW_DIS_IS_FRAGMENT
;
865 if (flow_keys
->is_first_frag
)
866 key_control
->flags
|= FLOW_DIS_FIRST_FRAG
;
867 if (flow_keys
->is_encap
)
868 key_control
->flags
|= FLOW_DIS_ENCAPSULATION
;
870 key_basic
= skb_flow_dissector_target(flow_dissector
,
871 FLOW_DISSECTOR_KEY_BASIC
,
873 key_basic
->n_proto
= flow_keys
->n_proto
;
874 key_basic
->ip_proto
= flow_keys
->ip_proto
;
876 if (flow_keys
->addr_proto
== ETH_P_IP
&&
877 dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_IPV4_ADDRS
)) {
878 key_addrs
= skb_flow_dissector_target(flow_dissector
,
879 FLOW_DISSECTOR_KEY_IPV4_ADDRS
,
881 key_addrs
->v4addrs
.src
= flow_keys
->ipv4_src
;
882 key_addrs
->v4addrs
.dst
= flow_keys
->ipv4_dst
;
883 key_control
->addr_type
= FLOW_DISSECTOR_KEY_IPV4_ADDRS
;
884 } else if (flow_keys
->addr_proto
== ETH_P_IPV6
&&
885 dissector_uses_key(flow_dissector
,
886 FLOW_DISSECTOR_KEY_IPV6_ADDRS
)) {
887 key_addrs
= skb_flow_dissector_target(flow_dissector
,
888 FLOW_DISSECTOR_KEY_IPV6_ADDRS
,
890 memcpy(&key_addrs
->v6addrs
, &flow_keys
->ipv6_src
,
891 sizeof(key_addrs
->v6addrs
));
892 key_control
->addr_type
= FLOW_DISSECTOR_KEY_IPV6_ADDRS
;
895 if (dissector_uses_key(flow_dissector
, FLOW_DISSECTOR_KEY_PORTS
))
896 key_ports
= skb_flow_dissector_target(flow_dissector
,
897 FLOW_DISSECTOR_KEY_PORTS
,
899 else if (dissector_uses_key(flow_dissector
,
900 FLOW_DISSECTOR_KEY_PORTS_RANGE
))
901 key_ports
= skb_flow_dissector_target(flow_dissector
,
902 FLOW_DISSECTOR_KEY_PORTS_RANGE
,
906 key_ports
->src
= flow_keys
->sport
;
907 key_ports
->dst
= flow_keys
->dport
;
910 if (dissector_uses_key(flow_dissector
,
911 FLOW_DISSECTOR_KEY_FLOW_LABEL
)) {
912 key_tags
= skb_flow_dissector_target(flow_dissector
,
913 FLOW_DISSECTOR_KEY_FLOW_LABEL
,
915 key_tags
->flow_label
= ntohl(flow_keys
->flow_label
);
919 bool bpf_flow_dissect(struct bpf_prog
*prog
, struct bpf_flow_dissector
*ctx
,
920 __be16 proto
, int nhoff
, int hlen
, unsigned int flags
)
922 struct bpf_flow_keys
*flow_keys
= ctx
->flow_keys
;
925 /* Pass parameters to the BPF program */
926 memset(flow_keys
, 0, sizeof(*flow_keys
));
927 flow_keys
->n_proto
= proto
;
928 flow_keys
->nhoff
= nhoff
;
929 flow_keys
->thoff
= flow_keys
->nhoff
;
931 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG
!=
932 (int)FLOW_DISSECTOR_F_PARSE_1ST_FRAG
);
933 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL
!=
934 (int)FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL
);
935 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP
!=
936 (int)FLOW_DISSECTOR_F_STOP_AT_ENCAP
);
937 flow_keys
->flags
= flags
;
939 result
= bpf_prog_run_pin_on_cpu(prog
, ctx
);
941 flow_keys
->nhoff
= clamp_t(u16
, flow_keys
->nhoff
, nhoff
, hlen
);
942 flow_keys
->thoff
= clamp_t(u16
, flow_keys
->thoff
,
943 flow_keys
->nhoff
, hlen
);
945 return result
== BPF_OK
;
949 * __skb_flow_dissect - extract the flow_keys struct and return it
950 * @net: associated network namespace, derived from @skb if NULL
951 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
952 * @flow_dissector: list of keys to dissect
953 * @target_container: target structure to put dissected values into
954 * @data: raw buffer pointer to the packet, if NULL use skb->data
955 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
956 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
957 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
958 * @flags: flags that control the dissection process, e.g.
959 * FLOW_DISSECTOR_F_STOP_AT_ENCAP.
961 * The function will try to retrieve individual keys into target specified
962 * by flow_dissector from either the skbuff or a raw buffer specified by the
965 * Caller must take care of zeroing target container memory.
967 bool __skb_flow_dissect(const struct net
*net
,
968 const struct sk_buff
*skb
,
969 struct flow_dissector
*flow_dissector
,
970 void *target_container
,
971 void *data
, __be16 proto
, int nhoff
, int hlen
,
974 struct flow_dissector_key_control
*key_control
;
975 struct flow_dissector_key_basic
*key_basic
;
976 struct flow_dissector_key_addrs
*key_addrs
;
977 struct flow_dissector_key_tags
*key_tags
;
978 struct flow_dissector_key_vlan
*key_vlan
;
979 struct bpf_prog
*attached
= NULL
;
980 enum flow_dissect_ret fdret
;
981 enum flow_dissector_key_id dissector_vlan
= FLOW_DISSECTOR_KEY_MAX
;
988 proto
= skb_vlan_tag_present(skb
) ?
989 skb
->vlan_proto
: skb
->protocol
;
990 nhoff
= skb_network_offset(skb
);
991 hlen
= skb_headlen(skb
);
992 #if IS_ENABLED(CONFIG_NET_DSA)
993 if (unlikely(skb
->dev
&& netdev_uses_dsa(skb
->dev
) &&
994 proto
== htons(ETH_P_XDSA
))) {
995 const struct dsa_device_ops
*ops
;
998 ops
= skb
->dev
->dsa_ptr
->tag_ops
;
999 if (ops
->flow_dissect
&&
1000 !ops
->flow_dissect(skb
, &proto
, &offset
)) {
1008 /* It is ensured by skb_flow_dissector_init() that control key will
1009 * be always present.
1011 key_control
= skb_flow_dissector_target(flow_dissector
,
1012 FLOW_DISSECTOR_KEY_CONTROL
,
1015 /* It is ensured by skb_flow_dissector_init() that basic key will
1016 * be always present.
1018 key_basic
= skb_flow_dissector_target(flow_dissector
,
1019 FLOW_DISSECTOR_KEY_BASIC
,
1025 net
= dev_net(skb
->dev
);
1027 net
= sock_net(skb
->sk
);
1034 attached
= rcu_dereference(init_net
.flow_dissector_prog
);
1037 attached
= rcu_dereference(net
->flow_dissector_prog
);
1040 struct bpf_flow_keys flow_keys
;
1041 struct bpf_flow_dissector ctx
= {
1042 .flow_keys
= &flow_keys
,
1044 .data_end
= data
+ hlen
,
1046 __be16 n_proto
= proto
;
1050 /* we can't use 'proto' in the skb case
1051 * because it might be set to skb->vlan_proto
1052 * which has been pulled from the data
1054 n_proto
= skb
->protocol
;
1057 ret
= bpf_flow_dissect(attached
, &ctx
, n_proto
, nhoff
,
1059 __skb_flow_bpf_to_target(&flow_keys
, flow_dissector
,
1067 if (dissector_uses_key(flow_dissector
,
1068 FLOW_DISSECTOR_KEY_ETH_ADDRS
)) {
1069 struct ethhdr
*eth
= eth_hdr(skb
);
1070 struct flow_dissector_key_eth_addrs
*key_eth_addrs
;
1072 key_eth_addrs
= skb_flow_dissector_target(flow_dissector
,
1073 FLOW_DISSECTOR_KEY_ETH_ADDRS
,
1075 memcpy(key_eth_addrs
, ð
->h_dest
, sizeof(*key_eth_addrs
));
1079 fdret
= FLOW_DISSECT_RET_CONTINUE
;
1082 case htons(ETH_P_IP
): {
1083 const struct iphdr
*iph
;
1086 iph
= __skb_header_pointer(skb
, nhoff
, sizeof(_iph
), data
, hlen
, &_iph
);
1087 if (!iph
|| iph
->ihl
< 5) {
1088 fdret
= FLOW_DISSECT_RET_OUT_BAD
;
1092 nhoff
+= iph
->ihl
* 4;
1094 ip_proto
= iph
->protocol
;
1096 if (dissector_uses_key(flow_dissector
,
1097 FLOW_DISSECTOR_KEY_IPV4_ADDRS
)) {
1098 key_addrs
= skb_flow_dissector_target(flow_dissector
,
1099 FLOW_DISSECTOR_KEY_IPV4_ADDRS
,
1102 memcpy(&key_addrs
->v4addrs
, &iph
->saddr
,
1103 sizeof(key_addrs
->v4addrs
));
1104 key_control
->addr_type
= FLOW_DISSECTOR_KEY_IPV4_ADDRS
;
1107 if (ip_is_fragment(iph
)) {
1108 key_control
->flags
|= FLOW_DIS_IS_FRAGMENT
;
1110 if (iph
->frag_off
& htons(IP_OFFSET
)) {
1111 fdret
= FLOW_DISSECT_RET_OUT_GOOD
;
1114 key_control
->flags
|= FLOW_DIS_FIRST_FRAG
;
1116 FLOW_DISSECTOR_F_PARSE_1ST_FRAG
)) {
1117 fdret
= FLOW_DISSECT_RET_OUT_GOOD
;
1123 __skb_flow_dissect_ipv4(skb
, flow_dissector
,
1124 target_container
, data
, iph
);
1128 case htons(ETH_P_IPV6
): {
1129 const struct ipv6hdr
*iph
;
1130 struct ipv6hdr _iph
;
1132 iph
= __skb_header_pointer(skb
, nhoff
, sizeof(_iph
), data
, hlen
, &_iph
);
1134 fdret
= FLOW_DISSECT_RET_OUT_BAD
;
1138 ip_proto
= iph
->nexthdr
;
1139 nhoff
+= sizeof(struct ipv6hdr
);
1141 if (dissector_uses_key(flow_dissector
,
1142 FLOW_DISSECTOR_KEY_IPV6_ADDRS
)) {
1143 key_addrs
= skb_flow_dissector_target(flow_dissector
,
1144 FLOW_DISSECTOR_KEY_IPV6_ADDRS
,
1147 memcpy(&key_addrs
->v6addrs
, &iph
->saddr
,
1148 sizeof(key_addrs
->v6addrs
));
1149 key_control
->addr_type
= FLOW_DISSECTOR_KEY_IPV6_ADDRS
;
1152 if ((dissector_uses_key(flow_dissector
,
1153 FLOW_DISSECTOR_KEY_FLOW_LABEL
) ||
1154 (flags
& FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL
)) &&
1155 ip6_flowlabel(iph
)) {
1156 __be32 flow_label
= ip6_flowlabel(iph
);
1158 if (dissector_uses_key(flow_dissector
,
1159 FLOW_DISSECTOR_KEY_FLOW_LABEL
)) {
1160 key_tags
= skb_flow_dissector_target(flow_dissector
,
1161 FLOW_DISSECTOR_KEY_FLOW_LABEL
,
1163 key_tags
->flow_label
= ntohl(flow_label
);
1165 if (flags
& FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL
) {
1166 fdret
= FLOW_DISSECT_RET_OUT_GOOD
;
1171 __skb_flow_dissect_ipv6(skb
, flow_dissector
,
1172 target_container
, data
, iph
);
1176 case htons(ETH_P_8021AD
):
1177 case htons(ETH_P_8021Q
): {
1178 const struct vlan_hdr
*vlan
= NULL
;
1179 struct vlan_hdr _vlan
;
1180 __be16 saved_vlan_tpid
= proto
;
1182 if (dissector_vlan
== FLOW_DISSECTOR_KEY_MAX
&&
1183 skb
&& skb_vlan_tag_present(skb
)) {
1184 proto
= skb
->protocol
;
1186 vlan
= __skb_header_pointer(skb
, nhoff
, sizeof(_vlan
),
1187 data
, hlen
, &_vlan
);
1189 fdret
= FLOW_DISSECT_RET_OUT_BAD
;
1193 proto
= vlan
->h_vlan_encapsulated_proto
;
1194 nhoff
+= sizeof(*vlan
);
1197 if (dissector_vlan
== FLOW_DISSECTOR_KEY_MAX
) {
1198 dissector_vlan
= FLOW_DISSECTOR_KEY_VLAN
;
1199 } else if (dissector_vlan
== FLOW_DISSECTOR_KEY_VLAN
) {
1200 dissector_vlan
= FLOW_DISSECTOR_KEY_CVLAN
;
1202 fdret
= FLOW_DISSECT_RET_PROTO_AGAIN
;
1206 if (dissector_uses_key(flow_dissector
, dissector_vlan
)) {
1207 key_vlan
= skb_flow_dissector_target(flow_dissector
,
1212 key_vlan
->vlan_id
= skb_vlan_tag_get_id(skb
);
1213 key_vlan
->vlan_priority
= skb_vlan_tag_get_prio(skb
);
1215 key_vlan
->vlan_id
= ntohs(vlan
->h_vlan_TCI
) &
1217 key_vlan
->vlan_priority
=
1218 (ntohs(vlan
->h_vlan_TCI
) &
1219 VLAN_PRIO_MASK
) >> VLAN_PRIO_SHIFT
;
1221 key_vlan
->vlan_tpid
= saved_vlan_tpid
;
1224 fdret
= FLOW_DISSECT_RET_PROTO_AGAIN
;
1227 case htons(ETH_P_PPP_SES
): {
1229 struct pppoe_hdr hdr
;
1232 hdr
= __skb_header_pointer(skb
, nhoff
, sizeof(_hdr
), data
, hlen
, &_hdr
);
1234 fdret
= FLOW_DISSECT_RET_OUT_BAD
;
1239 nhoff
+= PPPOE_SES_HLEN
;
1242 proto
= htons(ETH_P_IP
);
1243 fdret
= FLOW_DISSECT_RET_PROTO_AGAIN
;
1245 case htons(PPP_IPV6
):
1246 proto
= htons(ETH_P_IPV6
);
1247 fdret
= FLOW_DISSECT_RET_PROTO_AGAIN
;
1250 fdret
= FLOW_DISSECT_RET_OUT_BAD
;
1255 case htons(ETH_P_TIPC
): {
1256 struct tipc_basic_hdr
*hdr
, _hdr
;
1258 hdr
= __skb_header_pointer(skb
, nhoff
, sizeof(_hdr
),
1261 fdret
= FLOW_DISSECT_RET_OUT_BAD
;
1265 if (dissector_uses_key(flow_dissector
,
1266 FLOW_DISSECTOR_KEY_TIPC
)) {
1267 key_addrs
= skb_flow_dissector_target(flow_dissector
,
1268 FLOW_DISSECTOR_KEY_TIPC
,
1270 key_addrs
->tipckey
.key
= tipc_hdr_rps_key(hdr
);
1271 key_control
->addr_type
= FLOW_DISSECTOR_KEY_TIPC
;
1273 fdret
= FLOW_DISSECT_RET_OUT_GOOD
;
1277 case htons(ETH_P_MPLS_UC
):
1278 case htons(ETH_P_MPLS_MC
):
1279 fdret
= __skb_flow_dissect_mpls(skb
, flow_dissector
,
1280 target_container
, data
,
1283 case htons(ETH_P_FCOE
):
1284 if ((hlen
- nhoff
) < FCOE_HEADER_LEN
) {
1285 fdret
= FLOW_DISSECT_RET_OUT_BAD
;
1289 nhoff
+= FCOE_HEADER_LEN
;
1290 fdret
= FLOW_DISSECT_RET_OUT_GOOD
;
1293 case htons(ETH_P_ARP
):
1294 case htons(ETH_P_RARP
):
1295 fdret
= __skb_flow_dissect_arp(skb
, flow_dissector
,
1296 target_container
, data
,
1300 case htons(ETH_P_BATMAN
):
1301 fdret
= __skb_flow_dissect_batadv(skb
, key_control
, data
,
1302 &proto
, &nhoff
, hlen
, flags
);
1306 fdret
= FLOW_DISSECT_RET_OUT_BAD
;
1310 /* Process result of proto processing */
1312 case FLOW_DISSECT_RET_OUT_GOOD
:
1314 case FLOW_DISSECT_RET_PROTO_AGAIN
:
1315 if (skb_flow_dissect_allowed(&num_hdrs
))
1318 case FLOW_DISSECT_RET_CONTINUE
:
1319 case FLOW_DISSECT_RET_IPPROTO_AGAIN
:
1321 case FLOW_DISSECT_RET_OUT_BAD
:
1327 fdret
= FLOW_DISSECT_RET_CONTINUE
;
1331 fdret
= __skb_flow_dissect_gre(skb
, key_control
, flow_dissector
,
1332 target_container
, data
,
1333 &proto
, &nhoff
, &hlen
, flags
);
1337 case NEXTHDR_ROUTING
:
1338 case NEXTHDR_DEST
: {
1339 u8 _opthdr
[2], *opthdr
;
1341 if (proto
!= htons(ETH_P_IPV6
))
1344 opthdr
= __skb_header_pointer(skb
, nhoff
, sizeof(_opthdr
),
1345 data
, hlen
, &_opthdr
);
1347 fdret
= FLOW_DISSECT_RET_OUT_BAD
;
1351 ip_proto
= opthdr
[0];
1352 nhoff
+= (opthdr
[1] + 1) << 3;
1354 fdret
= FLOW_DISSECT_RET_IPPROTO_AGAIN
;
1357 case NEXTHDR_FRAGMENT
: {
1358 struct frag_hdr _fh
, *fh
;
1360 if (proto
!= htons(ETH_P_IPV6
))
1363 fh
= __skb_header_pointer(skb
, nhoff
, sizeof(_fh
),
1367 fdret
= FLOW_DISSECT_RET_OUT_BAD
;
1371 key_control
->flags
|= FLOW_DIS_IS_FRAGMENT
;
1373 nhoff
+= sizeof(_fh
);
1374 ip_proto
= fh
->nexthdr
;
1376 if (!(fh
->frag_off
& htons(IP6_OFFSET
))) {
1377 key_control
->flags
|= FLOW_DIS_FIRST_FRAG
;
1378 if (flags
& FLOW_DISSECTOR_F_PARSE_1ST_FRAG
) {
1379 fdret
= FLOW_DISSECT_RET_IPPROTO_AGAIN
;
1384 fdret
= FLOW_DISSECT_RET_OUT_GOOD
;
1388 proto
= htons(ETH_P_IP
);
1390 key_control
->flags
|= FLOW_DIS_ENCAPSULATION
;
1391 if (flags
& FLOW_DISSECTOR_F_STOP_AT_ENCAP
) {
1392 fdret
= FLOW_DISSECT_RET_OUT_GOOD
;
1396 fdret
= FLOW_DISSECT_RET_PROTO_AGAIN
;
1400 proto
= htons(ETH_P_IPV6
);
1402 key_control
->flags
|= FLOW_DIS_ENCAPSULATION
;
1403 if (flags
& FLOW_DISSECTOR_F_STOP_AT_ENCAP
) {
1404 fdret
= FLOW_DISSECT_RET_OUT_GOOD
;
1408 fdret
= FLOW_DISSECT_RET_PROTO_AGAIN
;
1413 proto
= htons(ETH_P_MPLS_UC
);
1414 fdret
= FLOW_DISSECT_RET_PROTO_AGAIN
;
1418 __skb_flow_dissect_tcp(skb
, flow_dissector
, target_container
,
1423 case IPPROTO_ICMPV6
:
1424 __skb_flow_dissect_icmp(skb
, flow_dissector
, target_container
,
1432 if (!(key_control
->flags
& FLOW_DIS_IS_FRAGMENT
))
1433 __skb_flow_dissect_ports(skb
, flow_dissector
, target_container
,
1434 data
, nhoff
, ip_proto
, hlen
);
1436 /* Process result of IP proto processing */
1438 case FLOW_DISSECT_RET_PROTO_AGAIN
:
1439 if (skb_flow_dissect_allowed(&num_hdrs
))
1442 case FLOW_DISSECT_RET_IPPROTO_AGAIN
:
1443 if (skb_flow_dissect_allowed(&num_hdrs
))
1444 goto ip_proto_again
;
1446 case FLOW_DISSECT_RET_OUT_GOOD
:
1447 case FLOW_DISSECT_RET_CONTINUE
:
1449 case FLOW_DISSECT_RET_OUT_BAD
:
1458 key_control
->thoff
= min_t(u16
, nhoff
, skb
? skb
->len
: hlen
);
1459 key_basic
->n_proto
= proto
;
1460 key_basic
->ip_proto
= ip_proto
;
1468 EXPORT_SYMBOL(__skb_flow_dissect
);
1470 static siphash_key_t hashrnd __read_mostly
;
1471 static __always_inline
void __flow_hash_secret_init(void)
1473 net_get_random_once(&hashrnd
, sizeof(hashrnd
));
1476 static const void *flow_keys_hash_start(const struct flow_keys
*flow
)
1478 BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET
% SIPHASH_ALIGNMENT
);
1479 return &flow
->FLOW_KEYS_HASH_START_FIELD
;
1482 static inline size_t flow_keys_hash_length(const struct flow_keys
*flow
)
1484 size_t diff
= FLOW_KEYS_HASH_OFFSET
+ sizeof(flow
->addrs
);
1486 BUILD_BUG_ON((sizeof(*flow
) - FLOW_KEYS_HASH_OFFSET
) % sizeof(u32
));
1488 switch (flow
->control
.addr_type
) {
1489 case FLOW_DISSECTOR_KEY_IPV4_ADDRS
:
1490 diff
-= sizeof(flow
->addrs
.v4addrs
);
1492 case FLOW_DISSECTOR_KEY_IPV6_ADDRS
:
1493 diff
-= sizeof(flow
->addrs
.v6addrs
);
1495 case FLOW_DISSECTOR_KEY_TIPC
:
1496 diff
-= sizeof(flow
->addrs
.tipckey
);
1499 return sizeof(*flow
) - diff
;
1502 __be32
flow_get_u32_src(const struct flow_keys
*flow
)
1504 switch (flow
->control
.addr_type
) {
1505 case FLOW_DISSECTOR_KEY_IPV4_ADDRS
:
1506 return flow
->addrs
.v4addrs
.src
;
1507 case FLOW_DISSECTOR_KEY_IPV6_ADDRS
:
1508 return (__force __be32
)ipv6_addr_hash(
1509 &flow
->addrs
.v6addrs
.src
);
1510 case FLOW_DISSECTOR_KEY_TIPC
:
1511 return flow
->addrs
.tipckey
.key
;
1516 EXPORT_SYMBOL(flow_get_u32_src
);
1518 __be32
flow_get_u32_dst(const struct flow_keys
*flow
)
1520 switch (flow
->control
.addr_type
) {
1521 case FLOW_DISSECTOR_KEY_IPV4_ADDRS
:
1522 return flow
->addrs
.v4addrs
.dst
;
1523 case FLOW_DISSECTOR_KEY_IPV6_ADDRS
:
1524 return (__force __be32
)ipv6_addr_hash(
1525 &flow
->addrs
.v6addrs
.dst
);
1530 EXPORT_SYMBOL(flow_get_u32_dst
);
1532 /* Sort the source and destination IP (and the ports if the IP are the same),
1533 * to have consistent hash within the two directions
1535 static inline void __flow_hash_consistentify(struct flow_keys
*keys
)
1539 switch (keys
->control
.addr_type
) {
1540 case FLOW_DISSECTOR_KEY_IPV4_ADDRS
:
1541 addr_diff
= (__force u32
)keys
->addrs
.v4addrs
.dst
-
1542 (__force u32
)keys
->addrs
.v4addrs
.src
;
1543 if ((addr_diff
< 0) ||
1545 ((__force u16
)keys
->ports
.dst
<
1546 (__force u16
)keys
->ports
.src
))) {
1547 swap(keys
->addrs
.v4addrs
.src
, keys
->addrs
.v4addrs
.dst
);
1548 swap(keys
->ports
.src
, keys
->ports
.dst
);
1551 case FLOW_DISSECTOR_KEY_IPV6_ADDRS
:
1552 addr_diff
= memcmp(&keys
->addrs
.v6addrs
.dst
,
1553 &keys
->addrs
.v6addrs
.src
,
1554 sizeof(keys
->addrs
.v6addrs
.dst
));
1555 if ((addr_diff
< 0) ||
1557 ((__force u16
)keys
->ports
.dst
<
1558 (__force u16
)keys
->ports
.src
))) {
1559 for (i
= 0; i
< 4; i
++)
1560 swap(keys
->addrs
.v6addrs
.src
.s6_addr32
[i
],
1561 keys
->addrs
.v6addrs
.dst
.s6_addr32
[i
]);
1562 swap(keys
->ports
.src
, keys
->ports
.dst
);
1568 static inline u32
__flow_hash_from_keys(struct flow_keys
*keys
,
1569 const siphash_key_t
*keyval
)
1573 __flow_hash_consistentify(keys
);
1575 hash
= siphash(flow_keys_hash_start(keys
),
1576 flow_keys_hash_length(keys
), keyval
);
1583 u32
flow_hash_from_keys(struct flow_keys
*keys
)
1585 __flow_hash_secret_init();
1586 return __flow_hash_from_keys(keys
, &hashrnd
);
1588 EXPORT_SYMBOL(flow_hash_from_keys
);
1590 static inline u32
___skb_get_hash(const struct sk_buff
*skb
,
1591 struct flow_keys
*keys
,
1592 const siphash_key_t
*keyval
)
1594 skb_flow_dissect_flow_keys(skb
, keys
,
1595 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL
);
1597 return __flow_hash_from_keys(keys
, keyval
);
1600 struct _flow_keys_digest_data
{
1609 void make_flow_keys_digest(struct flow_keys_digest
*digest
,
1610 const struct flow_keys
*flow
)
1612 struct _flow_keys_digest_data
*data
=
1613 (struct _flow_keys_digest_data
*)digest
;
1615 BUILD_BUG_ON(sizeof(*data
) > sizeof(*digest
));
1617 memset(digest
, 0, sizeof(*digest
));
1619 data
->n_proto
= flow
->basic
.n_proto
;
1620 data
->ip_proto
= flow
->basic
.ip_proto
;
1621 data
->ports
= flow
->ports
.ports
;
1622 data
->src
= flow
->addrs
.v4addrs
.src
;
1623 data
->dst
= flow
->addrs
.v4addrs
.dst
;
1625 EXPORT_SYMBOL(make_flow_keys_digest
);
1627 static struct flow_dissector flow_keys_dissector_symmetric __read_mostly
;
1629 u32
__skb_get_hash_symmetric(const struct sk_buff
*skb
)
1631 struct flow_keys keys
;
1633 __flow_hash_secret_init();
1635 memset(&keys
, 0, sizeof(keys
));
1636 __skb_flow_dissect(NULL
, skb
, &flow_keys_dissector_symmetric
,
1637 &keys
, NULL
, 0, 0, 0,
1638 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL
);
1640 return __flow_hash_from_keys(&keys
, &hashrnd
);
1642 EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric
);
1645 * __skb_get_hash: calculate a flow hash
1646 * @skb: sk_buff to calculate flow hash from
1648 * This function calculates a flow hash based on src/dst addresses
1649 * and src/dst port numbers. Sets hash in skb to non-zero hash value
1650 * on success, zero indicates no valid hash. Also, sets l4_hash in skb
1651 * if hash is a canonical 4-tuple hash over transport ports.
1653 void __skb_get_hash(struct sk_buff
*skb
)
1655 struct flow_keys keys
;
1658 __flow_hash_secret_init();
1660 hash
= ___skb_get_hash(skb
, &keys
, &hashrnd
);
1662 __skb_set_sw_hash(skb
, hash
, flow_keys_have_l4(&keys
));
1664 EXPORT_SYMBOL(__skb_get_hash
);
1666 __u32
skb_get_hash_perturb(const struct sk_buff
*skb
,
1667 const siphash_key_t
*perturb
)
1669 struct flow_keys keys
;
1671 return ___skb_get_hash(skb
, &keys
, perturb
);
1673 EXPORT_SYMBOL(skb_get_hash_perturb
);
1675 u32
__skb_get_poff(const struct sk_buff
*skb
, void *data
,
1676 const struct flow_keys_basic
*keys
, int hlen
)
1678 u32 poff
= keys
->control
.thoff
;
1680 /* skip L4 headers for fragments after the first */
1681 if ((keys
->control
.flags
& FLOW_DIS_IS_FRAGMENT
) &&
1682 !(keys
->control
.flags
& FLOW_DIS_FIRST_FRAG
))
1685 switch (keys
->basic
.ip_proto
) {
1687 /* access doff as u8 to avoid unaligned access */
1691 doff
= __skb_header_pointer(skb
, poff
+ 12, sizeof(_doff
),
1692 data
, hlen
, &_doff
);
1696 poff
+= max_t(u32
, sizeof(struct tcphdr
), (*doff
& 0xF0) >> 2);
1700 case IPPROTO_UDPLITE
:
1701 poff
+= sizeof(struct udphdr
);
1703 /* For the rest, we do not really care about header
1704 * extensions at this point for now.
1707 poff
+= sizeof(struct icmphdr
);
1709 case IPPROTO_ICMPV6
:
1710 poff
+= sizeof(struct icmp6hdr
);
1713 poff
+= sizeof(struct igmphdr
);
1716 poff
+= sizeof(struct dccp_hdr
);
1719 poff
+= sizeof(struct sctphdr
);
1727 * skb_get_poff - get the offset to the payload
1728 * @skb: sk_buff to get the payload offset from
1730 * The function will get the offset to the payload as far as it could
1731 * be dissected. The main user is currently BPF, so that we can dynamically
1732 * truncate packets without needing to push actual payload to the user
1733 * space and can analyze headers only, instead.
1735 u32
skb_get_poff(const struct sk_buff
*skb
)
1737 struct flow_keys_basic keys
;
1739 if (!skb_flow_dissect_flow_keys_basic(NULL
, skb
, &keys
,
1743 return __skb_get_poff(skb
, skb
->data
, &keys
, skb_headlen(skb
));
1746 __u32
__get_hash_from_flowi6(const struct flowi6
*fl6
, struct flow_keys
*keys
)
1748 memset(keys
, 0, sizeof(*keys
));
1750 memcpy(&keys
->addrs
.v6addrs
.src
, &fl6
->saddr
,
1751 sizeof(keys
->addrs
.v6addrs
.src
));
1752 memcpy(&keys
->addrs
.v6addrs
.dst
, &fl6
->daddr
,
1753 sizeof(keys
->addrs
.v6addrs
.dst
));
1754 keys
->control
.addr_type
= FLOW_DISSECTOR_KEY_IPV6_ADDRS
;
1755 keys
->ports
.src
= fl6
->fl6_sport
;
1756 keys
->ports
.dst
= fl6
->fl6_dport
;
1757 keys
->keyid
.keyid
= fl6
->fl6_gre_key
;
1758 keys
->tags
.flow_label
= (__force u32
)flowi6_get_flowlabel(fl6
);
1759 keys
->basic
.ip_proto
= fl6
->flowi6_proto
;
1761 return flow_hash_from_keys(keys
);
1763 EXPORT_SYMBOL(__get_hash_from_flowi6
);
1765 static const struct flow_dissector_key flow_keys_dissector_keys
[] = {
1767 .key_id
= FLOW_DISSECTOR_KEY_CONTROL
,
1768 .offset
= offsetof(struct flow_keys
, control
),
1771 .key_id
= FLOW_DISSECTOR_KEY_BASIC
,
1772 .offset
= offsetof(struct flow_keys
, basic
),
1775 .key_id
= FLOW_DISSECTOR_KEY_IPV4_ADDRS
,
1776 .offset
= offsetof(struct flow_keys
, addrs
.v4addrs
),
1779 .key_id
= FLOW_DISSECTOR_KEY_IPV6_ADDRS
,
1780 .offset
= offsetof(struct flow_keys
, addrs
.v6addrs
),
1783 .key_id
= FLOW_DISSECTOR_KEY_TIPC
,
1784 .offset
= offsetof(struct flow_keys
, addrs
.tipckey
),
1787 .key_id
= FLOW_DISSECTOR_KEY_PORTS
,
1788 .offset
= offsetof(struct flow_keys
, ports
),
1791 .key_id
= FLOW_DISSECTOR_KEY_VLAN
,
1792 .offset
= offsetof(struct flow_keys
, vlan
),
1795 .key_id
= FLOW_DISSECTOR_KEY_FLOW_LABEL
,
1796 .offset
= offsetof(struct flow_keys
, tags
),
1799 .key_id
= FLOW_DISSECTOR_KEY_GRE_KEYID
,
1800 .offset
= offsetof(struct flow_keys
, keyid
),
1804 static const struct flow_dissector_key flow_keys_dissector_symmetric_keys
[] = {
1806 .key_id
= FLOW_DISSECTOR_KEY_CONTROL
,
1807 .offset
= offsetof(struct flow_keys
, control
),
1810 .key_id
= FLOW_DISSECTOR_KEY_BASIC
,
1811 .offset
= offsetof(struct flow_keys
, basic
),
1814 .key_id
= FLOW_DISSECTOR_KEY_IPV4_ADDRS
,
1815 .offset
= offsetof(struct flow_keys
, addrs
.v4addrs
),
1818 .key_id
= FLOW_DISSECTOR_KEY_IPV6_ADDRS
,
1819 .offset
= offsetof(struct flow_keys
, addrs
.v6addrs
),
1822 .key_id
= FLOW_DISSECTOR_KEY_PORTS
,
1823 .offset
= offsetof(struct flow_keys
, ports
),
1827 static const struct flow_dissector_key flow_keys_basic_dissector_keys
[] = {
1829 .key_id
= FLOW_DISSECTOR_KEY_CONTROL
,
1830 .offset
= offsetof(struct flow_keys
, control
),
1833 .key_id
= FLOW_DISSECTOR_KEY_BASIC
,
1834 .offset
= offsetof(struct flow_keys
, basic
),
1838 struct flow_dissector flow_keys_dissector __read_mostly
;
1839 EXPORT_SYMBOL(flow_keys_dissector
);
1841 struct flow_dissector flow_keys_basic_dissector __read_mostly
;
1842 EXPORT_SYMBOL(flow_keys_basic_dissector
);
1844 static int __init
init_default_flow_dissectors(void)
1846 skb_flow_dissector_init(&flow_keys_dissector
,
1847 flow_keys_dissector_keys
,
1848 ARRAY_SIZE(flow_keys_dissector_keys
));
1849 skb_flow_dissector_init(&flow_keys_dissector_symmetric
,
1850 flow_keys_dissector_symmetric_keys
,
1851 ARRAY_SIZE(flow_keys_dissector_symmetric_keys
));
1852 skb_flow_dissector_init(&flow_keys_basic_dissector
,
1853 flow_keys_basic_dissector_keys
,
1854 ARRAY_SIZE(flow_keys_basic_dissector_keys
));
1856 return register_pernet_subsys(&flow_dissector_pernet_ops
);
1858 core_initcall(init_default_flow_dissectors
);