1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * IPV4 GSO/GRO offload support
4 * Linux INET implementation
6 * TCPv4 GSO/GRO support
9 #include <linux/indirect_call_wrapper.h>
10 #include <linux/skbuff.h>
13 #include <net/protocol.h>
15 static void tcp_gso_tstamp(struct sk_buff
*skb
, unsigned int ts_seq
,
16 unsigned int seq
, unsigned int mss
)
19 if (before(ts_seq
, seq
+ mss
)) {
20 skb_shinfo(skb
)->tx_flags
|= SKBTX_SW_TSTAMP
;
21 skb_shinfo(skb
)->tskey
= ts_seq
;
30 static struct sk_buff
*tcp4_gso_segment(struct sk_buff
*skb
,
31 netdev_features_t features
)
33 if (!(skb_shinfo(skb
)->gso_type
& SKB_GSO_TCPV4
))
34 return ERR_PTR(-EINVAL
);
36 if (!pskb_may_pull(skb
, sizeof(struct tcphdr
)))
37 return ERR_PTR(-EINVAL
);
39 if (unlikely(skb
->ip_summed
!= CHECKSUM_PARTIAL
)) {
40 const struct iphdr
*iph
= ip_hdr(skb
);
41 struct tcphdr
*th
= tcp_hdr(skb
);
43 /* Set up checksum pseudo header, usually expect stack to
44 * have done this already.
48 skb
->ip_summed
= CHECKSUM_PARTIAL
;
49 __tcp_v4_send_check(skb
, iph
->saddr
, iph
->daddr
);
52 return tcp_gso_segment(skb
, features
);
55 struct sk_buff
*tcp_gso_segment(struct sk_buff
*skb
,
56 netdev_features_t features
)
58 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
59 unsigned int sum_truesize
= 0;
66 struct sk_buff
*gso_skb
= skb
;
68 bool ooo_okay
, copy_destructor
;
72 if (thlen
< sizeof(*th
))
75 if (!pskb_may_pull(skb
, thlen
))
78 oldlen
= (u16
)~skb
->len
;
79 __skb_pull(skb
, thlen
);
81 mss
= skb_shinfo(skb
)->gso_size
;
82 if (unlikely(skb
->len
<= mss
))
85 if (skb_gso_ok(skb
, features
| NETIF_F_GSO_ROBUST
)) {
86 /* Packet is from an untrusted source, reset gso_segs. */
88 skb_shinfo(skb
)->gso_segs
= DIV_ROUND_UP(skb
->len
, mss
);
94 copy_destructor
= gso_skb
->destructor
== tcp_wfree
;
95 ooo_okay
= gso_skb
->ooo_okay
;
96 /* All segments but the first should have ooo_okay cleared */
99 segs
= skb_segment(skb
, features
);
103 /* Only first segment might have ooo_okay set */
104 segs
->ooo_okay
= ooo_okay
;
106 /* GSO partial and frag_list segmentation only requires splitting
107 * the frame into an MSS multiple and possibly a remainder, both
108 * cases return a GSO skb. So update the mss now.
110 if (skb_is_gso(segs
))
111 mss
*= skb_shinfo(segs
)->gso_segs
;
113 delta
= htonl(oldlen
+ (thlen
+ mss
));
117 seq
= ntohl(th
->seq
);
119 if (unlikely(skb_shinfo(gso_skb
)->tx_flags
& SKBTX_SW_TSTAMP
))
120 tcp_gso_tstamp(segs
, skb_shinfo(gso_skb
)->tskey
, seq
, mss
);
122 newcheck
= ~csum_fold((__force __wsum
)((__force u32
)th
->check
+
123 (__force u32
)delta
));
126 th
->fin
= th
->psh
= 0;
127 th
->check
= newcheck
;
129 if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
130 gso_reset_checksum(skb
, ~th
->check
);
132 th
->check
= gso_make_checksum(skb
, ~th
->check
);
135 if (copy_destructor
) {
136 skb
->destructor
= gso_skb
->destructor
;
137 skb
->sk
= gso_skb
->sk
;
138 sum_truesize
+= skb
->truesize
;
143 th
->seq
= htonl(seq
);
147 /* Following permits TCP Small Queues to work well with GSO :
148 * The callback to TCP stack will be called at the time last frag
149 * is freed at TX completion, and not right now when gso_skb
150 * is freed by GSO engine
152 if (copy_destructor
) {
155 swap(gso_skb
->sk
, skb
->sk
);
156 swap(gso_skb
->destructor
, skb
->destructor
);
157 sum_truesize
+= skb
->truesize
;
158 delta
= sum_truesize
- gso_skb
->truesize
;
159 /* In some pathological cases, delta can be negative.
160 * We need to either use refcount_add() or refcount_sub_and_test()
162 if (likely(delta
>= 0))
163 refcount_add(delta
, &skb
->sk
->sk_wmem_alloc
);
165 WARN_ON_ONCE(refcount_sub_and_test(-delta
, &skb
->sk
->sk_wmem_alloc
));
168 delta
= htonl(oldlen
+ (skb_tail_pointer(skb
) -
169 skb_transport_header(skb
)) +
171 th
->check
= ~csum_fold((__force __wsum
)((__force u32
)th
->check
+
172 (__force u32
)delta
));
173 if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
174 gso_reset_checksum(skb
, ~th
->check
);
176 th
->check
= gso_make_checksum(skb
, ~th
->check
);
181 struct sk_buff
*tcp_gro_receive(struct list_head
*head
, struct sk_buff
*skb
)
183 struct sk_buff
*pp
= NULL
;
190 unsigned int mss
= 1;
196 off
= skb_gro_offset(skb
);
197 hlen
= off
+ sizeof(*th
);
198 th
= skb_gro_header_fast(skb
, off
);
199 if (skb_gro_header_hard(skb
, hlen
)) {
200 th
= skb_gro_header_slow(skb
, hlen
, off
);
205 thlen
= th
->doff
* 4;
206 if (thlen
< sizeof(*th
))
210 if (skb_gro_header_hard(skb
, hlen
)) {
211 th
= skb_gro_header_slow(skb
, hlen
, off
);
216 skb_gro_pull(skb
, thlen
);
218 len
= skb_gro_len(skb
);
219 flags
= tcp_flag_word(th
);
221 list_for_each_entry(p
, head
, list
) {
222 if (!NAPI_GRO_CB(p
)->same_flow
)
227 if (*(u32
*)&th
->source
^ *(u32
*)&th2
->source
) {
228 NAPI_GRO_CB(p
)->same_flow
= 0;
235 goto out_check_final
;
238 /* Include the IP ID check below from the inner most IP hdr */
239 flush
= NAPI_GRO_CB(p
)->flush
;
240 flush
|= (__force
int)(flags
& TCP_FLAG_CWR
);
241 flush
|= (__force
int)((flags
^ tcp_flag_word(th2
)) &
242 ~(TCP_FLAG_CWR
| TCP_FLAG_FIN
| TCP_FLAG_PSH
));
243 flush
|= (__force
int)(th
->ack_seq
^ th2
->ack_seq
);
244 for (i
= sizeof(*th
); i
< thlen
; i
+= 4)
245 flush
|= *(u32
*)((u8
*)th
+ i
) ^
246 *(u32
*)((u8
*)th2
+ i
);
248 /* When we receive our second frame we can made a decision on if we
249 * continue this flow as an atomic flow with a fixed ID or if we use
250 * an incrementing ID.
252 if (NAPI_GRO_CB(p
)->flush_id
!= 1 ||
253 NAPI_GRO_CB(p
)->count
!= 1 ||
254 !NAPI_GRO_CB(p
)->is_atomic
)
255 flush
|= NAPI_GRO_CB(p
)->flush_id
;
257 NAPI_GRO_CB(p
)->is_atomic
= false;
259 mss
= skb_shinfo(p
)->gso_size
;
261 flush
|= (len
- 1) >= mss
;
262 flush
|= (ntohl(th2
->seq
) + skb_gro_len(p
)) ^ ntohl(th
->seq
);
263 #ifdef CONFIG_TLS_DEVICE
264 flush
|= p
->decrypted
^ skb
->decrypted
;
267 if (flush
|| skb_gro_receive(p
, skb
)) {
269 goto out_check_final
;
272 tcp_flag_word(th2
) |= flags
& (TCP_FLAG_FIN
| TCP_FLAG_PSH
);
276 flush
|= (__force
int)(flags
& (TCP_FLAG_URG
| TCP_FLAG_PSH
|
277 TCP_FLAG_RST
| TCP_FLAG_SYN
|
280 if (p
&& (!NAPI_GRO_CB(skb
)->same_flow
|| flush
))
284 NAPI_GRO_CB(skb
)->flush
|= (flush
!= 0);
289 int tcp_gro_complete(struct sk_buff
*skb
)
291 struct tcphdr
*th
= tcp_hdr(skb
);
293 skb
->csum_start
= (unsigned char *)th
- skb
->head
;
294 skb
->csum_offset
= offsetof(struct tcphdr
, check
);
295 skb
->ip_summed
= CHECKSUM_PARTIAL
;
297 skb_shinfo(skb
)->gso_segs
= NAPI_GRO_CB(skb
)->count
;
300 skb_shinfo(skb
)->gso_type
|= SKB_GSO_TCP_ECN
;
302 if (skb
->encapsulation
)
303 skb
->inner_transport_header
= skb
->transport_header
;
307 EXPORT_SYMBOL(tcp_gro_complete
);
309 INDIRECT_CALLABLE_SCOPE
310 struct sk_buff
*tcp4_gro_receive(struct list_head
*head
, struct sk_buff
*skb
)
312 /* Don't bother verifying checksum if we're going to flush anyway. */
313 if (!NAPI_GRO_CB(skb
)->flush
&&
314 skb_gro_checksum_validate(skb
, IPPROTO_TCP
,
315 inet_gro_compute_pseudo
)) {
316 NAPI_GRO_CB(skb
)->flush
= 1;
320 return tcp_gro_receive(head
, skb
);
323 INDIRECT_CALLABLE_SCOPE
int tcp4_gro_complete(struct sk_buff
*skb
, int thoff
)
325 const struct iphdr
*iph
= ip_hdr(skb
);
326 struct tcphdr
*th
= tcp_hdr(skb
);
328 th
->check
= ~tcp_v4_check(skb
->len
- thoff
, iph
->saddr
,
330 skb_shinfo(skb
)->gso_type
|= SKB_GSO_TCPV4
;
332 if (NAPI_GRO_CB(skb
)->is_atomic
)
333 skb_shinfo(skb
)->gso_type
|= SKB_GSO_TCP_FIXEDID
;
335 return tcp_gro_complete(skb
);
338 static const struct net_offload tcpv4_offload
= {
340 .gso_segment
= tcp4_gso_segment
,
341 .gro_receive
= tcp4_gro_receive
,
342 .gro_complete
= tcp4_gro_complete
,
346 int __init
tcpv4_offload_init(void)
348 return inet_add_offload(&tcpv4_offload
, IPPROTO_TCP
);