net: add vlan_get_protocol_offset_inline() helper

skb_protocol() is bloated, and forces slow stack canaries in many
fast paths.

Add vlan_get_protocol_offset_inline() which deals with the non-vlan
common cases.

__vlan_get_protocol_offset() is now out of line.

It returns a vlan_type_depth struct to avoid stack canaries in callers.

struct vlan_type_depth {
       __be16 type;
       u16 depth;
};

$ scripts/bloat-o-meter -t vmlinux.old vmlinux.new
add/remove: 0/2 grow/shrink: 0/22 up/down: 0/-6320 (-6320)
Function                                     old     new   delta
vlan_get_protocol_dgram                       61      59      -2
__pfx_skb_protocol                            16       -     -16
__vlan_get_protocol_offset                   307     273     -34
tap_get_user                                1374    1207    -167
ip_md_tunnel_xmit                           1625    1452    -173
tap_sendmsg                                  940     753    -187
netif_skb_features                          1079     866    -213
netem_enqueue                               3017    2800    -217
vlan_parse_protocol                          271      50    -221
tso_start                                    567     344    -223
fq_dequeue                                  1908    1685    -223
skb_network_protocol                         434     205    -229
ip6_tnl_xmit                                2639    2409    -230
br_dev_queue_push_xmit                       474     236    -238
skb_protocol                                 258       -    -258
packet_parse_headers                         621     357    -264
__ip6_tnl_rcv                               1306    1039    -267
skb_csum_hwoffload_help                      515     224    -291
ip_tunnel_xmit                              2635    2339    -296
sch_frag_xmit_hook                          1582    1233    -349
bpf_skb_ecn_set_ce                           868     457    -411
IP6_ECN_decapsulate                         1297     768    -529
ip_tunnel_rcv                               2121    1489    -632
ipip6_rcv                                   2572    1922    -650
Total: Before=24892803, After=24886483, chg -0.03%

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Simon Horman <horms@kernel.org>
Link: https://patch.msgid.link/20260204053023.1622775-1-edumazet@google.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h
index f7f34eb15e068785e464303b60f2f050b3bec0ec..e6272f9c5e42cc8023e8ece45e5c7a6eed5338e0 100644 (file)
--- a/include/linux/if_vlan.h
+++ b/include/linux/if_vlan.h
@@ -594,8 +594,17 @@ static inline int vlan_get_tag(const struct sk_buff *skb, u16 *vlan_tci)
        }
 }
 
+struct vlan_type_depth {
+       __be16 type;
+       u16 depth;
+};
+
+struct vlan_type_depth __vlan_get_protocol_offset(const struct sk_buff *skb,
+                                                 __be16 type,
+                                                 int mac_offset);
+
 /**
- * __vlan_get_protocol_offset() - get protocol EtherType.
+ * vlan_get_protocol_offset_inline() - get protocol EtherType.
  * @skb: skbuff to query
  * @type: first vlan protocol
  * @mac_offset: MAC offset
@@ -604,40 +613,24 @@ static inline int vlan_get_tag(const struct sk_buff *skb, u16 *vlan_tci)
  * Returns: the EtherType of the packet, regardless of whether it is
  * vlan encapsulated (normal or hardware accelerated) or not.
  */
-static inline __be16 __vlan_get_protocol_offset(const struct sk_buff *skb,
-                                               __be16 type,
-                                               int mac_offset,
-                                               int *depth)
+static inline
+__be16 vlan_get_protocol_offset_inline(const struct sk_buff *skb,
+                                      __be16 type,
+                                      int mac_offset,
+                                      int *depth)
 {
-       unsigned int vlan_depth = skb->mac_len, parse_depth = VLAN_MAX_DEPTH;
-
-       /* if type is 802.1Q/AD then the header should already be
-        * present at mac_len - VLAN_HLEN (if mac_len > 0), or at
-        * ETH_HLEN otherwise
-        */
        if (eth_type_vlan(type)) {
-               if (vlan_depth) {
-                       if (WARN_ON(vlan_depth < VLAN_HLEN))
-                               return 0;
-                       vlan_depth -= VLAN_HLEN;
-               } else {
-                       vlan_depth = ETH_HLEN;
-               }
-               do {
-                       struct vlan_hdr vhdr, *vh;
+               struct vlan_type_depth res;
 
-                       vh = skb_header_pointer(skb, mac_offset + vlan_depth,
-                                               sizeof(vhdr), &vhdr);
-                       if (unlikely(!vh || !--parse_depth))
-                               return 0;
+               res = __vlan_get_protocol_offset(skb, type, mac_offset);
 
-                       type = vh->h_vlan_encapsulated_proto;
-                       vlan_depth += VLAN_HLEN;
-               } while (eth_type_vlan(type));
+               if (depth && res.type)
+                       *depth = res.depth;
+               return res.type;
        }
 
        if (depth)
-               *depth = vlan_depth;
+               *depth = skb->mac_len;
 
        return type;
 }
@@ -645,7 +638,7 @@ static inline __be16 __vlan_get_protocol_offset(const struct sk_buff *skb,
 static inline __be16 __vlan_get_protocol(const struct sk_buff *skb, __be16 type,
                                         int *depth)
 {
-       return __vlan_get_protocol_offset(skb, type, 0, depth);
+       return vlan_get_protocol_offset_inline(skb, type, 0, depth);
 }
 
 /**

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 648c20e190387b8c66e2a6ff8e265795f81974c7..45809986dfe17f79297c6bce6a9e8cd107eee603 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -7444,3 +7444,39 @@ void __put_netmem(netmem_ref netmem)
                net_devmem_put_net_iov(netmem_to_net_iov(netmem));
 }
 EXPORT_SYMBOL(__put_netmem);
+
+struct vlan_type_depth __vlan_get_protocol_offset(const struct sk_buff *skb,
+                                                 __be16 type,
+                                                 int mac_offset)
+{
+       unsigned int vlan_depth = skb->mac_len, parse_depth = VLAN_MAX_DEPTH;
+
+       /* if type is 802.1Q/AD then the header should already be
+        * present at mac_len - VLAN_HLEN (if mac_len > 0), or at
+        * ETH_HLEN otherwise
+        */
+       if (vlan_depth) {
+               if (WARN_ON_ONCE(vlan_depth < VLAN_HLEN))
+                       return (struct vlan_type_depth) { 0 };
+               vlan_depth -= VLAN_HLEN;
+       } else {
+               vlan_depth = ETH_HLEN;
+       }
+       do {
+               struct vlan_hdr vhdr, *vh;
+
+               vh = skb_header_pointer(skb, mac_offset + vlan_depth,
+                                       sizeof(vhdr), &vhdr);
+               if (unlikely(!vh || !--parse_depth))
+                       return (struct vlan_type_depth) { 0 };
+
+               type = vh->h_vlan_encapsulated_proto;
+               vlan_depth += VLAN_HLEN;
+       } while (eth_type_vlan(type));
+
+       return (struct vlan_type_depth) {
+               .type = type,
+               .depth = vlan_depth
+       };
+}
+EXPORT_SYMBOL(__vlan_get_protocol_offset);

diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
index 494d628d10a5105a6a32788b4673993f218ec881..a1005359085a8336edc3c95eceaf101025e75489 100644 (file)
--- a/net/packet/af_packet.c
+++ b/net/packet/af_packet.c
@@ -572,8 +572,9 @@ static __be16 vlan_get_protocol_dgram(const struct sk_buff *skb)
        __be16 proto = skb->protocol;
 
        if (unlikely(eth_type_vlan(proto)))
-               proto = __vlan_get_protocol_offset(skb, proto,
-                                                  skb_mac_offset(skb), NULL);
+               proto = vlan_get_protocol_offset_inline(skb, proto,
+                                                       skb_mac_offset(skb),
+                                                       NULL);
 
        return proto;
 }