]> git.ipfire.org Git - thirdparty/kernel/stable.git/blob - net/ipv6/ip6_output.c
projects / thirdparty / kernel / stable.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'drm-next-2019-03-15' of git://anongit.freedesktop.org/drm/drm
[thirdparty/kernel/stable.git] / net / ipv6 / ip6_output.c
1 /*
2 * IPv6 output functions
3 * Linux INET6 implementation
4 *
5 * Authors:
6 * Pedro Roque <roque@di.fc.ul.pt>
7 *
8 * Based on linux/net/ipv4/ip_output.c
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
14 *
15 * Changes:
16 * A.N.Kuznetsov : airthmetics in fragmentation.
17 * extension headers are implemented.
18 * route changes now work.
19 * ip6_forward does not confuse sniffers.
20 * etc.
21 *
22 * H. von Brand : Added missing #include <linux/string.h>
23 * Imran Patel : frag id should be in NBO
24 * Kazunori MIYAZAWA @USAGI
25 * : add ip6_append_data and related functions
26 * for datagram xmit
27 */
28
29 #include <linux/errno.h>
30 #include <linux/kernel.h>
31 #include <linux/string.h>
32 #include <linux/socket.h>
33 #include <linux/net.h>
34 #include <linux/netdevice.h>
35 #include <linux/if_arp.h>
36 #include <linux/in6.h>
37 #include <linux/tcp.h>
38 #include <linux/route.h>
39 #include <linux/module.h>
40 #include <linux/slab.h>
41
42 #include <linux/bpf-cgroup.h>
43 #include <linux/netfilter.h>
44 #include <linux/netfilter_ipv6.h>
45
46 #include <net/sock.h>
47 #include <net/snmp.h>
48
49 #include <net/ipv6.h>
50 #include <net/ndisc.h>
51 #include <net/protocol.h>
52 #include <net/ip6_route.h>
53 #include <net/addrconf.h>
54 #include <net/rawv6.h>
55 #include <net/icmp.h>
56 #include <net/xfrm.h>
57 #include <net/checksum.h>
58 #include <linux/mroute6.h>
59 #include <net/l3mdev.h>
60 #include <net/lwtunnel.h>
61
62 static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
63 {
64 struct dst_entry *dst = skb_dst(skb);
65 struct net_device *dev = dst->dev;
66 struct neighbour *neigh;
67 struct in6_addr *nexthop;
68 int ret;
69
70 if (ipv6_addr_is_multicast(&ipv6_hdr(skb)->daddr)) {
71 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
72
73 if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) &&
74 ((mroute6_is_socket(net, skb) &&
75 !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) ||
76 ipv6_chk_mcast_addr(dev, &ipv6_hdr(skb)->daddr,
77 &ipv6_hdr(skb)->saddr))) {
78 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
79
80 /* Do not check for IFF_ALLMULTI; multicast routing
81 is not supported in any case.
82 */
83 if (newskb)
84 NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING,
85 net, sk, newskb, NULL, newskb->dev,
86 dev_loopback_xmit);
87
88 if (ipv6_hdr(skb)->hop_limit == 0) {
89 IP6_INC_STATS(net, idev,
90 IPSTATS_MIB_OUTDISCARDS);
91 kfree_skb(skb);
92 return 0;
93 }
94 }
95
96 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUTMCAST, skb->len);
97
98 if (IPV6_ADDR_MC_SCOPE(&ipv6_hdr(skb)->daddr) <=
99 IPV6_ADDR_SCOPE_NODELOCAL &&
100 !(dev->flags & IFF_LOOPBACK)) {
101 kfree_skb(skb);
102 return 0;
103 }
104 }
105
106 if (lwtunnel_xmit_redirect(dst->lwtstate)) {
107 int res = lwtunnel_xmit(skb);
108
109 if (res < 0 || res == LWTUNNEL_XMIT_DONE)
110 return res;
111 }
112
113 rcu_read_lock_bh();
114 nexthop = rt6_nexthop((struct rt6_info *)dst, &ipv6_hdr(skb)->daddr);
115 neigh = __ipv6_neigh_lookup_noref(dst->dev, nexthop);
116 if (unlikely(!neigh))
117 neigh = __neigh_create(&nd_tbl, nexthop, dst->dev, false);
118 if (!IS_ERR(neigh)) {
119 sock_confirm_neigh(skb, neigh);
120 ret = neigh_output(neigh, skb);
121 rcu_read_unlock_bh();
122 return ret;
123 }
124 rcu_read_unlock_bh();
125
126 IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
127 kfree_skb(skb);
128 return -EINVAL;
129 }
130
131 static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
132 {
133 int ret;
134
135 ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb);
136 if (ret) {
137 kfree_skb(skb);
138 return ret;
139 }
140
141 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
142 /* Policy lookup after SNAT yielded a new policy */
143 if (skb_dst(skb)->xfrm) {
144 IPCB(skb)->flags |= IPSKB_REROUTED;
145 return dst_output(net, sk, skb);
146 }
147 #endif
148
149 if ((skb->len > ip6_skb_dst_mtu(skb) && !skb_is_gso(skb)) ||
150 dst_allfrag(skb_dst(skb)) ||
151 (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size))
152 return ip6_fragment(net, sk, skb, ip6_finish_output2);
153 else
154 return ip6_finish_output2(net, sk, skb);
155 }
156
157 int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb)
158 {
159 struct net_device *dev = skb_dst(skb)->dev;
160 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
161
162 skb->protocol = htons(ETH_P_IPV6);
163 skb->dev = dev;
164
165 if (unlikely(idev->cnf.disable_ipv6)) {
166 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS);
167 kfree_skb(skb);
168 return 0;
169 }
170
171 return NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING,
172 net, sk, skb, NULL, dev,
173 ip6_finish_output,
174 !(IP6CB(skb)->flags & IP6SKB_REROUTED));
175 }
176
177 bool ip6_autoflowlabel(struct net *net, const struct ipv6_pinfo *np)
178 {
179 if (!np->autoflowlabel_set)
180 return ip6_default_np_autolabel(net);
181 else
182 return np->autoflowlabel;
183 }
184
185 /*
186 * xmit an sk_buff (used by TCP, SCTP and DCCP)
187 * Note : socket lock is not held for SYNACK packets, but might be modified
188 * by calls to skb_set_owner_w() and ipv6_local_error(),
189 * which are using proper atomic operations or spinlocks.
190 */
191 int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6,
192 __u32 mark, struct ipv6_txoptions *opt, int tclass)
193 {
194 struct net *net = sock_net(sk);
195 const struct ipv6_pinfo *np = inet6_sk(sk);
196 struct in6_addr *first_hop = &fl6->daddr;
197 struct dst_entry *dst = skb_dst(skb);
198 unsigned int head_room;
199 struct ipv6hdr *hdr;
200 u8 proto = fl6->flowi6_proto;
201 int seg_len = skb->len;
202 int hlimit = -1;
203 u32 mtu;
204
205 head_room = sizeof(struct ipv6hdr) + LL_RESERVED_SPACE(dst->dev);
206 if (opt)
207 head_room += opt->opt_nflen + opt->opt_flen;
208
209 if (unlikely(skb_headroom(skb) < head_room)) {
210 struct sk_buff *skb2 = skb_realloc_headroom(skb, head_room);
211 if (!skb2) {
212 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
213 IPSTATS_MIB_OUTDISCARDS);
214 kfree_skb(skb);
215 return -ENOBUFS;
216 }
217 if (skb->sk)
218 skb_set_owner_w(skb2, skb->sk);
219 consume_skb(skb);
220 skb = skb2;
221 }
222
223 if (opt) {
224 seg_len += opt->opt_nflen + opt->opt_flen;
225
226 if (opt->opt_flen)
227 ipv6_push_frag_opts(skb, opt, &proto);
228
229 if (opt->opt_nflen)
230 ipv6_push_nfrag_opts(skb, opt, &proto, &first_hop,
231 &fl6->saddr);
232 }
233
234 skb_push(skb, sizeof(struct ipv6hdr));
235 skb_reset_network_header(skb);
236 hdr = ipv6_hdr(skb);
237
238 /*
239 * Fill in the IPv6 header
240 */
241 if (np)
242 hlimit = np->hop_limit;
243 if (hlimit < 0)
244 hlimit = ip6_dst_hoplimit(dst);
245
246 ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel,
247 ip6_autoflowlabel(net, np), fl6));
248
249 hdr->payload_len = htons(seg_len);
250 hdr->nexthdr = proto;
251 hdr->hop_limit = hlimit;
252
253 hdr->saddr = fl6->saddr;
254 hdr->daddr = *first_hop;
255
256 skb->protocol = htons(ETH_P_IPV6);
257 skb->priority = sk->sk_priority;
258 skb->mark = mark;
259
260 mtu = dst_mtu(dst);
261 if ((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb)) {
262 IP6_UPD_PO_STATS(net, ip6_dst_idev(skb_dst(skb)),
263 IPSTATS_MIB_OUT, skb->len);
264
265 /* if egress device is enslaved to an L3 master device pass the
266 * skb to its handler for processing
267 */
268 skb = l3mdev_ip6_out((struct sock *)sk, skb);
269 if (unlikely(!skb))
270 return 0;
271
272 /* hooks should never assume socket lock is held.
273 * we promote our socket to non const
274 */
275 return NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT,
276 net, (struct sock *)sk, skb, NULL, dst->dev,
277 dst_output);
278 }
279
280 skb->dev = dst->dev;
281 /* ipv6_local_error() does not require socket lock,
282 * we promote our socket to non const
283 */
284 ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu);
285
286 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS);
287 kfree_skb(skb);
288 return -EMSGSIZE;
289 }
290 EXPORT_SYMBOL(ip6_xmit);
291
292 static int ip6_call_ra_chain(struct sk_buff *skb, int sel)
293 {
294 struct ip6_ra_chain *ra;
295 struct sock *last = NULL;
296
297 read_lock(&ip6_ra_lock);
298 for (ra = ip6_ra_chain; ra; ra = ra->next) {
299 struct sock *sk = ra->sk;
300 if (sk && ra->sel == sel &&
301 (!sk->sk_bound_dev_if ||
302 sk->sk_bound_dev_if == skb->dev->ifindex)) {
303 struct ipv6_pinfo *np = inet6_sk(sk);
304
305 if (np && np->rtalert_isolate &&
306 !net_eq(sock_net(sk), dev_net(skb->dev))) {
307 continue;
308 }
309 if (last) {
310 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
311 if (skb2)
312 rawv6_rcv(last, skb2);
313 }
314 last = sk;
315 }
316 }
317
318 if (last) {
319 rawv6_rcv(last, skb);
320 read_unlock(&ip6_ra_lock);
321 return 1;
322 }
323 read_unlock(&ip6_ra_lock);
324 return 0;
325 }
326
327 static int ip6_forward_proxy_check(struct sk_buff *skb)
328 {
329 struct ipv6hdr *hdr = ipv6_hdr(skb);
330 u8 nexthdr = hdr->nexthdr;
331 __be16 frag_off;
332 int offset;
333
334 if (ipv6_ext_hdr(nexthdr)) {
335 offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off);
336 if (offset < 0)
337 return 0;
338 } else
339 offset = sizeof(struct ipv6hdr);
340
341 if (nexthdr == IPPROTO_ICMPV6) {
342 struct icmp6hdr *icmp6;
343
344 if (!pskb_may_pull(skb, (skb_network_header(skb) +
345 offset + 1 - skb->data)))
346 return 0;
347
348 icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset);
349
350 switch (icmp6->icmp6_type) {
351 case NDISC_ROUTER_SOLICITATION:
352 case NDISC_ROUTER_ADVERTISEMENT:
353 case NDISC_NEIGHBOUR_SOLICITATION:
354 case NDISC_NEIGHBOUR_ADVERTISEMENT:
355 case NDISC_REDIRECT:
356 /* For reaction involving unicast neighbor discovery
357 * message destined to the proxied address, pass it to
358 * input function.
359 */
360 return 1;
361 default:
362 break;
363 }
364 }
365
366 /*
367 * The proxying router can't forward traffic sent to a link-local
368 * address, so signal the sender and discard the packet. This
369 * behavior is clarified by the MIPv6 specification.
370 */
371 if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) {
372 dst_link_failure(skb);
373 return -1;
374 }
375
376 return 0;
377 }
378
379 static inline int ip6_forward_finish(struct net *net, struct sock *sk,
380 struct sk_buff *skb)
381 {
382 struct dst_entry *dst = skb_dst(skb);
383
384 __IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS);
385 __IP6_ADD_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTOCTETS, skb->len);
386
387 #ifdef CONFIG_NET_SWITCHDEV
388 if (skb->offload_l3_fwd_mark) {
389 consume_skb(skb);
390 return 0;
391 }
392 #endif
393
394 skb->tstamp = 0;
395 return dst_output(net, sk, skb);
396 }
397
398 static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
399 {
400 if (skb->len <= mtu)
401 return false;
402
403 /* ipv6 conntrack defrag sets max_frag_size + ignore_df */
404 if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu)
405 return true;
406
407 if (skb->ignore_df)
408 return false;
409
410 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
411 return false;
412
413 return true;
414 }
415
416 int ip6_forward(struct sk_buff *skb)
417 {
418 struct inet6_dev *idev = __in6_dev_get_safely(skb->dev);
419 struct dst_entry *dst = skb_dst(skb);
420 struct ipv6hdr *hdr = ipv6_hdr(skb);
421 struct inet6_skb_parm *opt = IP6CB(skb);
422 struct net *net = dev_net(dst->dev);
423 u32 mtu;
424
425 if (net->ipv6.devconf_all->forwarding == 0)
426 goto error;
427
428 if (skb->pkt_type != PACKET_HOST)
429 goto drop;
430
431 if (unlikely(skb->sk))
432 goto drop;
433
434 if (skb_warn_if_lro(skb))
435 goto drop;
436
437 if (!xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) {
438 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
439 goto drop;
440 }
441
442 skb_forward_csum(skb);
443
444 /*
445 * We DO NOT make any processing on
446 * RA packets, pushing them to user level AS IS
447 * without ane WARRANTY that application will be able
448 * to interpret them. The reason is that we
449 * cannot make anything clever here.
450 *
451 * We are not end-node, so that if packet contains
452 * AH/ESP, we cannot make anything.
453 * Defragmentation also would be mistake, RA packets
454 * cannot be fragmented, because there is no warranty
455 * that different fragments will go along one path. --ANK
456 */
457 if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) {
458 if (ip6_call_ra_chain(skb, ntohs(opt->ra)))
459 return 0;
460 }
461
462 /*
463 * check and decrement ttl
464 */
465 if (hdr->hop_limit <= 1) {
466 /* Force OUTPUT device used as source address */
467 skb->dev = dst->dev;
468 icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0);
469 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS);
470
471 kfree_skb(skb);
472 return -ETIMEDOUT;
473 }
474
475 /* XXX: idev->cnf.proxy_ndp? */
476 if (net->ipv6.devconf_all->proxy_ndp &&
477 pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev, 0)) {
478 int proxied = ip6_forward_proxy_check(skb);
479 if (proxied > 0)
480 return ip6_input(skb);
481 else if (proxied < 0) {
482 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
483 goto drop;
484 }
485 }
486
487 if (!xfrm6_route_forward(skb)) {
488 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INDISCARDS);
489 goto drop;
490 }
491 dst = skb_dst(skb);
492
493 /* IPv6 specs say nothing about it, but it is clear that we cannot
494 send redirects to source routed frames.
495 We don't send redirects to frames decapsulated from IPsec.
496 */
497 if (IP6CB(skb)->iif == dst->dev->ifindex &&
498 opt->srcrt == 0 && !skb_sec_path(skb)) {
499 struct in6_addr *target = NULL;
500 struct inet_peer *peer;
501 struct rt6_info *rt;
502
503 /*
504 * incoming and outgoing devices are the same
505 * send a redirect.
506 */
507
508 rt = (struct rt6_info *) dst;
509 if (rt->rt6i_flags & RTF_GATEWAY)
510 target = &rt->rt6i_gateway;
511 else
512 target = &hdr->daddr;
513
514 peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr, 1);
515
516 /* Limit redirects both by destination (here)
517 and by source (inside ndisc_send_redirect)
518 */
519 if (inet_peer_xrlim_allow(peer, 1*HZ))
520 ndisc_send_redirect(skb, target);
521 if (peer)
522 inet_putpeer(peer);
523 } else {
524 int addrtype = ipv6_addr_type(&hdr->saddr);
525
526 /* This check is security critical. */
527 if (addrtype == IPV6_ADDR_ANY ||
528 addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK))
529 goto error;
530 if (addrtype & IPV6_ADDR_LINKLOCAL) {
531 icmpv6_send(skb, ICMPV6_DEST_UNREACH,
532 ICMPV6_NOT_NEIGHBOUR, 0);
533 goto error;
534 }
535 }
536
537 mtu = ip6_dst_mtu_forward(dst);
538 if (mtu < IPV6_MIN_MTU)
539 mtu = IPV6_MIN_MTU;
540
541 if (ip6_pkt_too_big(skb, mtu)) {
542 /* Again, force OUTPUT device used as source address */
543 skb->dev = dst->dev;
544 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
545 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INTOOBIGERRORS);
546 __IP6_INC_STATS(net, ip6_dst_idev(dst),
547 IPSTATS_MIB_FRAGFAILS);
548 kfree_skb(skb);
549 return -EMSGSIZE;
550 }
551
552 if (skb_cow(skb, dst->dev->hard_header_len)) {
553 __IP6_INC_STATS(net, ip6_dst_idev(dst),
554 IPSTATS_MIB_OUTDISCARDS);
555 goto drop;
556 }
557
558 hdr = ipv6_hdr(skb);
559
560 /* Mangling hops number delayed to point after skb COW */
561
562 hdr->hop_limit--;
563
564 return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD,
565 net, NULL, skb, skb->dev, dst->dev,
566 ip6_forward_finish);
567
568 error:
569 __IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS);
570 drop:
571 kfree_skb(skb);
572 return -EINVAL;
573 }
574
575 static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from)
576 {
577 to->pkt_type = from->pkt_type;
578 to->priority = from->priority;
579 to->protocol = from->protocol;
580 skb_dst_drop(to);
581 skb_dst_set(to, dst_clone(skb_dst(from)));
582 to->dev = from->dev;
583 to->mark = from->mark;
584
585 skb_copy_hash(to, from);
586
587 #ifdef CONFIG_NET_SCHED
588 to->tc_index = from->tc_index;
589 #endif
590 nf_copy(to, from);
591 skb_ext_copy(to, from);
592 skb_copy_secmark(to, from);
593 }
594
595 int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
596 int (*output)(struct net *, struct sock *, struct sk_buff *))
597 {
598 struct sk_buff *frag;
599 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb);
600 struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ?
601 inet6_sk(skb->sk) : NULL;
602 struct ipv6hdr *tmp_hdr;
603 struct frag_hdr *fh;
604 unsigned int mtu, hlen, left, len;
605 int hroom, troom;
606 __be32 frag_id;
607 int ptr, offset = 0, err = 0;
608 u8 *prevhdr, nexthdr = 0;
609
610 err = ip6_find_1stfragopt(skb, &prevhdr);
611 if (err < 0)
612 goto fail;
613 hlen = err;
614 nexthdr = *prevhdr;
615
616 mtu = ip6_skb_dst_mtu(skb);
617
618 /* We must not fragment if the socket is set to force MTU discovery
619 * or if the skb it not generated by a local socket.
620 */
621 if (unlikely(!skb->ignore_df && skb->len > mtu))
622 goto fail_toobig;
623
624 if (IP6CB(skb)->frag_max_size) {
625 if (IP6CB(skb)->frag_max_size > mtu)
626 goto fail_toobig;
627
628 /* don't send fragments larger than what we received */
629 mtu = IP6CB(skb)->frag_max_size;
630 if (mtu < IPV6_MIN_MTU)
631 mtu = IPV6_MIN_MTU;
632 }
633
634 if (np && np->frag_size < mtu) {
635 if (np->frag_size)
636 mtu = np->frag_size;
637 }
638 if (mtu < hlen + sizeof(struct frag_hdr) + 8)
639 goto fail_toobig;
640 mtu -= hlen + sizeof(struct frag_hdr);
641
642 frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr,
643 &ipv6_hdr(skb)->saddr);
644
645 if (skb->ip_summed == CHECKSUM_PARTIAL &&
646 (err = skb_checksum_help(skb)))
647 goto fail;
648
649 hroom = LL_RESERVED_SPACE(rt->dst.dev);
650 if (skb_has_frag_list(skb)) {
651 unsigned int first_len = skb_pagelen(skb);
652 struct sk_buff *frag2;
653
654 if (first_len - hlen > mtu ||
655 ((first_len - hlen) & 7) ||
656 skb_cloned(skb) ||
657 skb_headroom(skb) < (hroom + sizeof(struct frag_hdr)))
658 goto slow_path;
659
660 skb_walk_frags(skb, frag) {
661 /* Correct geometry. */
662 if (frag->len > mtu ||
663 ((frag->len & 7) && frag->next) ||
664 skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr)))
665 goto slow_path_clean;
666
667 /* Partially cloned skb? */
668 if (skb_shared(frag))
669 goto slow_path_clean;
670
671 BUG_ON(frag->sk);
672 if (skb->sk) {
673 frag->sk = skb->sk;
674 frag->destructor = sock_wfree;
675 }
676 skb->truesize -= frag->truesize;
677 }
678
679 err = 0;
680 offset = 0;
681 /* BUILD HEADER */
682
683 *prevhdr = NEXTHDR_FRAGMENT;
684 tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC);
685 if (!tmp_hdr) {
686 err = -ENOMEM;
687 goto fail;
688 }
689 frag = skb_shinfo(skb)->frag_list;
690 skb_frag_list_init(skb);
691
692 __skb_pull(skb, hlen);
693 fh = __skb_push(skb, sizeof(struct frag_hdr));
694 __skb_push(skb, hlen);
695 skb_reset_network_header(skb);
696 memcpy(skb_network_header(skb), tmp_hdr, hlen);
697
698 fh->nexthdr = nexthdr;
699 fh->reserved = 0;
700 fh->frag_off = htons(IP6_MF);
701 fh->identification = frag_id;
702
703 first_len = skb_pagelen(skb);
704 skb->data_len = first_len - skb_headlen(skb);
705 skb->len = first_len;
706 ipv6_hdr(skb)->payload_len = htons(first_len -
707 sizeof(struct ipv6hdr));
708
709 for (;;) {
710 /* Prepare header of the next frame,
711 * before previous one went down. */
712 if (frag) {
713 frag->ip_summed = CHECKSUM_NONE;
714 skb_reset_transport_header(frag);
715 fh = __skb_push(frag, sizeof(struct frag_hdr));
716 __skb_push(frag, hlen);
717 skb_reset_network_header(frag);
718 memcpy(skb_network_header(frag), tmp_hdr,
719 hlen);
720 offset += skb->len - hlen - sizeof(struct frag_hdr);
721 fh->nexthdr = nexthdr;
722 fh->reserved = 0;
723 fh->frag_off = htons(offset);
724 if (frag->next)
725 fh->frag_off |= htons(IP6_MF);
726 fh->identification = frag_id;
727 ipv6_hdr(frag)->payload_len =
728 htons(frag->len -
729 sizeof(struct ipv6hdr));
730 ip6_copy_metadata(frag, skb);
731 }
732
733 err = output(net, sk, skb);
734 if (!err)
735 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
736 IPSTATS_MIB_FRAGCREATES);
737
738 if (err || !frag)
739 break;
740
741 skb = frag;
742 frag = skb->next;
743 skb_mark_not_on_list(skb);
744 }
745
746 kfree(tmp_hdr);
747
748 if (err == 0) {
749 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
750 IPSTATS_MIB_FRAGOKS);
751 return 0;
752 }
753
754 kfree_skb_list(frag);
755
756 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
757 IPSTATS_MIB_FRAGFAILS);
758 return err;
759
760 slow_path_clean:
761 skb_walk_frags(skb, frag2) {
762 if (frag2 == frag)
763 break;
764 frag2->sk = NULL;
765 frag2->destructor = NULL;
766 skb->truesize += frag2->truesize;
767 }
768 }
769
770 slow_path:
771 left = skb->len - hlen; /* Space per frame */
772 ptr = hlen; /* Where to start from */
773
774 /*
775 * Fragment the datagram.
776 */
777
778 troom = rt->dst.dev->needed_tailroom;
779
780 /*
781 * Keep copying data until we run out.
782 */
783 while (left > 0) {
784 u8 *fragnexthdr_offset;
785
786 len = left;
787 /* IF: it doesn't fit, use 'mtu' - the data space left */
788 if (len > mtu)
789 len = mtu;
790 /* IF: we are not sending up to and including the packet end
791 then align the next start on an eight byte boundary */
792 if (len < left) {
793 len &= ~7;
794 }
795
796 /* Allocate buffer */
797 frag = alloc_skb(len + hlen + sizeof(struct frag_hdr) +
798 hroom + troom, GFP_ATOMIC);
799 if (!frag) {
800 err = -ENOMEM;
801 goto fail;
802 }
803
804 /*
805 * Set up data on packet
806 */
807
808 ip6_copy_metadata(frag, skb);
809 skb_reserve(frag, hroom);
810 skb_put(frag, len + hlen + sizeof(struct frag_hdr));
811 skb_reset_network_header(frag);
812 fh = (struct frag_hdr *)(skb_network_header(frag) + hlen);
813 frag->transport_header = (frag->network_header + hlen +
814 sizeof(struct frag_hdr));
815
816 /*
817 * Charge the memory for the fragment to any owner
818 * it might possess
819 */
820 if (skb->sk)
821 skb_set_owner_w(frag, skb->sk);
822
823 /*
824 * Copy the packet header into the new buffer.
825 */
826 skb_copy_from_linear_data(skb, skb_network_header(frag), hlen);
827
828 fragnexthdr_offset = skb_network_header(frag);
829 fragnexthdr_offset += prevhdr - skb_network_header(skb);
830 *fragnexthdr_offset = NEXTHDR_FRAGMENT;
831
832 /*
833 * Build fragment header.
834 */
835 fh->nexthdr = nexthdr;
836 fh->reserved = 0;
837 fh->identification = frag_id;
838
839 /*
840 * Copy a block of the IP datagram.
841 */
842 BUG_ON(skb_copy_bits(skb, ptr, skb_transport_header(frag),
843 len));
844 left -= len;
845
846 fh->frag_off = htons(offset);
847 if (left > 0)
848 fh->frag_off |= htons(IP6_MF);
849 ipv6_hdr(frag)->payload_len = htons(frag->len -
850 sizeof(struct ipv6hdr));
851
852 ptr += len;
853 offset += len;
854
855 /*
856 * Put this fragment into the sending queue.
857 */
858 err = output(net, sk, frag);
859 if (err)
860 goto fail;
861
862 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
863 IPSTATS_MIB_FRAGCREATES);
864 }
865 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
866 IPSTATS_MIB_FRAGOKS);
867 consume_skb(skb);
868 return err;
869
870 fail_toobig:
871 if (skb->sk && dst_allfrag(skb_dst(skb)))
872 sk_nocaps_add(skb->sk, NETIF_F_GSO_MASK);
873
874 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
875 err = -EMSGSIZE;
876
877 fail:
878 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
879 IPSTATS_MIB_FRAGFAILS);
880 kfree_skb(skb);
881 return err;
882 }
883
884 static inline int ip6_rt_check(const struct rt6key *rt_key,
885 const struct in6_addr *fl_addr,
886 const struct in6_addr *addr_cache)
887 {
888 return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) &&
889 (!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache));
890 }
891
892 static struct dst_entry *ip6_sk_dst_check(struct sock *sk,
893 struct dst_entry *dst,
894 const struct flowi6 *fl6)
895 {
896 struct ipv6_pinfo *np = inet6_sk(sk);
897 struct rt6_info *rt;
898
899 if (!dst)
900 goto out;
901
902 if (dst->ops->family != AF_INET6) {
903 dst_release(dst);
904 return NULL;
905 }
906
907 rt = (struct rt6_info *)dst;
908 /* Yes, checking route validity in not connected
909 * case is not very simple. Take into account,
910 * that we do not support routing by source, TOS,
911 * and MSG_DONTROUTE --ANK (980726)
912 *
913 * 1. ip6_rt_check(): If route was host route,
914 * check that cached destination is current.
915 * If it is network route, we still may
916 * check its validity using saved pointer
917 * to the last used address: daddr_cache.
918 * We do not want to save whole address now,
919 * (because main consumer of this service
920 * is tcp, which has not this problem),
921 * so that the last trick works only on connected
922 * sockets.
923 * 2. oif also should be the same.
924 */
925 if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr, np->daddr_cache) ||
926 #ifdef CONFIG_IPV6_SUBTREES
927 ip6_rt_check(&rt->rt6i_src, &fl6->saddr, np->saddr_cache) ||
928 #endif
929 (!(fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF) &&
930 (fl6->flowi6_oif && fl6->flowi6_oif != dst->dev->ifindex))) {
931 dst_release(dst);
932 dst = NULL;
933 }
934
935 out:
936 return dst;
937 }
938
939 static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk,
940 struct dst_entry **dst, struct flowi6 *fl6)
941 {
942 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD
943 struct neighbour *n;
944 struct rt6_info *rt;
945 #endif
946 int err;
947 int flags = 0;
948
949 /* The correct way to handle this would be to do
950 * ip6_route_get_saddr, and then ip6_route_output; however,
951 * the route-specific preferred source forces the
952 * ip6_route_output call _before_ ip6_route_get_saddr.
953 *
954 * In source specific routing (no src=any default route),
955 * ip6_route_output will fail given src=any saddr, though, so
956 * that's why we try it again later.
957 */
958 if (ipv6_addr_any(&fl6->saddr) && (!*dst || !(*dst)->error)) {
959 struct fib6_info *from;
960 struct rt6_info *rt;
961 bool had_dst = *dst != NULL;
962
963 if (!had_dst)
964 *dst = ip6_route_output(net, sk, fl6);
965 rt = (*dst)->error ? NULL : (struct rt6_info *)*dst;
966
967 rcu_read_lock();
968 from = rt ? rcu_dereference(rt->from) : NULL;
969 err = ip6_route_get_saddr(net, from, &fl6->daddr,
970 sk ? inet6_sk(sk)->srcprefs : 0,
971 &fl6->saddr);
972 rcu_read_unlock();
973
974 if (err)
975 goto out_err_release;
976
977 /* If we had an erroneous initial result, pretend it
978 * never existed and let the SA-enabled version take
979 * over.
980 */
981 if (!had_dst && (*dst)->error) {
982 dst_release(*dst);
983 *dst = NULL;
984 }
985
986 if (fl6->flowi6_oif)
987 flags |= RT6_LOOKUP_F_IFACE;
988 }
989
990 if (!*dst)
991 *dst = ip6_route_output_flags(net, sk, fl6, flags);
992
993 err = (*dst)->error;
994 if (err)
995 goto out_err_release;
996
997 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD
998 /*
999 * Here if the dst entry we've looked up
1000 * has a neighbour entry that is in the INCOMPLETE
1001 * state and the src address from the flow is
1002 * marked as OPTIMISTIC, we release the found
1003 * dst entry and replace it instead with the
1004 * dst entry of the nexthop router
1005 */
1006 rt = (struct rt6_info *) *dst;
1007 rcu_read_lock_bh();
1008 n = __ipv6_neigh_lookup_noref(rt->dst.dev,
1009 rt6_nexthop(rt, &fl6->daddr));
1010 err = n && !(n->nud_state & NUD_VALID) ? -EINVAL : 0;
1011 rcu_read_unlock_bh();
1012
1013 if (err) {
1014 struct inet6_ifaddr *ifp;
1015 struct flowi6 fl_gw6;
1016 int redirect;
1017
1018 ifp = ipv6_get_ifaddr(net, &fl6->saddr,
1019 (*dst)->dev, 1);
1020
1021 redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC);
1022 if (ifp)
1023 in6_ifa_put(ifp);
1024
1025 if (redirect) {
1026 /*
1027 * We need to get the dst entry for the
1028 * default router instead
1029 */
1030 dst_release(*dst);
1031 memcpy(&fl_gw6, fl6, sizeof(struct flowi6));
1032 memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr));
1033 *dst = ip6_route_output(net, sk, &fl_gw6);
1034 err = (*dst)->error;
1035 if (err)
1036 goto out_err_release;
1037 }
1038 }
1039 #endif
1040 if (ipv6_addr_v4mapped(&fl6->saddr) &&
1041 !(ipv6_addr_v4mapped(&fl6->daddr) || ipv6_addr_any(&fl6->daddr))) {
1042 err = -EAFNOSUPPORT;
1043 goto out_err_release;
1044 }
1045
1046 return 0;
1047
1048 out_err_release:
1049 dst_release(*dst);
1050 *dst = NULL;
1051
1052 if (err == -ENETUNREACH)
1053 IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES);
1054 return err;
1055 }
1056
1057 /**
1058 * ip6_dst_lookup - perform route lookup on flow
1059 * @sk: socket which provides route info
1060 * @dst: pointer to dst_entry * for result
1061 * @fl6: flow to lookup
1062 *
1063 * This function performs a route lookup on the given flow.
1064 *
1065 * It returns zero on success, or a standard errno code on error.
1066 */
1067 int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst,
1068 struct flowi6 *fl6)
1069 {
1070 *dst = NULL;
1071 return ip6_dst_lookup_tail(net, sk, dst, fl6);
1072 }
1073 EXPORT_SYMBOL_GPL(ip6_dst_lookup);
1074
1075 /**
1076 * ip6_dst_lookup_flow - perform route lookup on flow with ipsec
1077 * @sk: socket which provides route info
1078 * @fl6: flow to lookup
1079 * @final_dst: final destination address for ipsec lookup
1080 *
1081 * This function performs a route lookup on the given flow.
1082 *
1083 * It returns a valid dst pointer on success, or a pointer encoded
1084 * error code.
1085 */
1086 struct dst_entry *ip6_dst_lookup_flow(const struct sock *sk, struct flowi6 *fl6,
1087 const struct in6_addr *final_dst)
1088 {
1089 struct dst_entry *dst = NULL;
1090 int err;
1091
1092 err = ip6_dst_lookup_tail(sock_net(sk), sk, &dst, fl6);
1093 if (err)
1094 return ERR_PTR(err);
1095 if (final_dst)
1096 fl6->daddr = *final_dst;
1097
1098 return xfrm_lookup_route(sock_net(sk), dst, flowi6_to_flowi(fl6), sk, 0);
1099 }
1100 EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow);
1101
1102 /**
1103 * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow
1104 * @sk: socket which provides the dst cache and route info
1105 * @fl6: flow to lookup
1106 * @final_dst: final destination address for ipsec lookup
1107 * @connected: whether @sk is connected or not
1108 *
1109 * This function performs a route lookup on the given flow with the
1110 * possibility of using the cached route in the socket if it is valid.
1111 * It will take the socket dst lock when operating on the dst cache.
1112 * As a result, this function can only be used in process context.
1113 *
1114 * In addition, for a connected socket, cache the dst in the socket
1115 * if the current cache is not valid.
1116 *
1117 * It returns a valid dst pointer on success, or a pointer encoded
1118 * error code.
1119 */
1120 struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6,
1121 const struct in6_addr *final_dst,
1122 bool connected)
1123 {
1124 struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie);
1125
1126 dst = ip6_sk_dst_check(sk, dst, fl6);
1127 if (dst)
1128 return dst;
1129
1130 dst = ip6_dst_lookup_flow(sk, fl6, final_dst);
1131 if (connected && !IS_ERR(dst))
1132 ip6_sk_dst_store_flow(sk, dst_clone(dst), fl6);
1133
1134 return dst;
1135 }
1136 EXPORT_SYMBOL_GPL(ip6_sk_dst_lookup_flow);
1137
1138 static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src,
1139 gfp_t gfp)
1140 {
1141 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
1142 }
1143
1144 static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src,
1145 gfp_t gfp)
1146 {
1147 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
1148 }
1149
1150 static void ip6_append_data_mtu(unsigned int *mtu,
1151 int *maxfraglen,
1152 unsigned int fragheaderlen,
1153 struct sk_buff *skb,
1154 struct rt6_info *rt,
1155 unsigned int orig_mtu)
1156 {
1157 if (!(rt->dst.flags & DST_XFRM_TUNNEL)) {
1158 if (!skb) {
1159 /* first fragment, reserve header_len */
1160 *mtu = orig_mtu - rt->dst.header_len;
1161
1162 } else {
1163 /*
1164 * this fragment is not first, the headers
1165 * space is regarded as data space.
1166 */
1167 *mtu = orig_mtu;
1168 }
1169 *maxfraglen = ((*mtu - fragheaderlen) & ~7)
1170 + fragheaderlen - sizeof(struct frag_hdr);
1171 }
1172 }
1173
1174 static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork,
1175 struct inet6_cork *v6_cork, struct ipcm6_cookie *ipc6,
1176 struct rt6_info *rt, struct flowi6 *fl6)
1177 {
1178 struct ipv6_pinfo *np = inet6_sk(sk);
1179 unsigned int mtu;
1180 struct ipv6_txoptions *opt = ipc6->opt;
1181
1182 /*
1183 * setup for corking
1184 */
1185 if (opt) {
1186 if (WARN_ON(v6_cork->opt))
1187 return -EINVAL;
1188
1189 v6_cork->opt = kzalloc(sizeof(*opt), sk->sk_allocation);
1190 if (unlikely(!v6_cork->opt))
1191 return -ENOBUFS;
1192
1193 v6_cork->opt->tot_len = sizeof(*opt);
1194 v6_cork->opt->opt_flen = opt->opt_flen;
1195 v6_cork->opt->opt_nflen = opt->opt_nflen;
1196
1197 v6_cork->opt->dst0opt = ip6_opt_dup(opt->dst0opt,
1198 sk->sk_allocation);
1199 if (opt->dst0opt && !v6_cork->opt->dst0opt)
1200 return -ENOBUFS;
1201
1202 v6_cork->opt->dst1opt = ip6_opt_dup(opt->dst1opt,
1203 sk->sk_allocation);
1204 if (opt->dst1opt && !v6_cork->opt->dst1opt)
1205 return -ENOBUFS;
1206
1207 v6_cork->opt->hopopt = ip6_opt_dup(opt->hopopt,
1208 sk->sk_allocation);
1209 if (opt->hopopt && !v6_cork->opt->hopopt)
1210 return -ENOBUFS;
1211
1212 v6_cork->opt->srcrt = ip6_rthdr_dup(opt->srcrt,
1213 sk->sk_allocation);
1214 if (opt->srcrt && !v6_cork->opt->srcrt)
1215 return -ENOBUFS;
1216
1217 /* need source address above miyazawa*/
1218 }
1219 dst_hold(&rt->dst);
1220 cork->base.dst = &rt->dst;
1221 cork->fl.u.ip6 = *fl6;
1222 v6_cork->hop_limit = ipc6->hlimit;
1223 v6_cork->tclass = ipc6->tclass;
1224 if (rt->dst.flags & DST_XFRM_TUNNEL)
1225 mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ?
1226 READ_ONCE(rt->dst.dev->mtu) : dst_mtu(&rt->dst);
1227 else
1228 mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ?
1229 READ_ONCE(rt->dst.dev->mtu) : dst_mtu(xfrm_dst_path(&rt->dst));
1230 if (np->frag_size < mtu) {
1231 if (np->frag_size)
1232 mtu = np->frag_size;
1233 }
1234 if (mtu < IPV6_MIN_MTU)
1235 return -EINVAL;
1236 cork->base.fragsize = mtu;
1237 cork->base.gso_size = ipc6->gso_size;
1238 cork->base.tx_flags = 0;
1239 sock_tx_timestamp(sk, ipc6->sockc.tsflags, &cork->base.tx_flags);
1240
1241 if (dst_allfrag(xfrm_dst_path(&rt->dst)))
1242 cork->base.flags |= IPCORK_ALLFRAG;
1243 cork->base.length = 0;
1244
1245 cork->base.transmit_time = ipc6->sockc.transmit_time;
1246
1247 return 0;
1248 }
1249
1250 static int __ip6_append_data(struct sock *sk,
1251 struct flowi6 *fl6,
1252 struct sk_buff_head *queue,
1253 struct inet_cork *cork,
1254 struct inet6_cork *v6_cork,
1255 struct page_frag *pfrag,
1256 int getfrag(void *from, char *to, int offset,
1257 int len, int odd, struct sk_buff *skb),
1258 void *from, int length, int transhdrlen,
1259 unsigned int flags, struct ipcm6_cookie *ipc6)
1260 {
1261 struct sk_buff *skb, *skb_prev = NULL;
1262 unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu, pmtu;
1263 struct ubuf_info *uarg = NULL;
1264 int exthdrlen = 0;
1265 int dst_exthdrlen = 0;
1266 int hh_len;
1267 int copy;
1268 int err;
1269 int offset = 0;
1270 u32 tskey = 0;
1271 struct rt6_info *rt = (struct rt6_info *)cork->dst;
1272 struct ipv6_txoptions *opt = v6_cork->opt;
1273 int csummode = CHECKSUM_NONE;
1274 unsigned int maxnonfragsize, headersize;
1275 unsigned int wmem_alloc_delta = 0;
1276 bool paged, extra_uref;
1277
1278 skb = skb_peek_tail(queue);
1279 if (!skb) {
1280 exthdrlen = opt ? opt->opt_flen : 0;
1281 dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
1282 }
1283
1284 paged = !!cork->gso_size;
1285 mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize;
1286 orig_mtu = mtu;
1287
1288 if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP &&
1289 sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)
1290 tskey = sk->sk_tskey++;
1291
1292 hh_len = LL_RESERVED_SPACE(rt->dst.dev);
1293
1294 fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len +
1295 (opt ? opt->opt_nflen : 0);
1296 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen -
1297 sizeof(struct frag_hdr);
1298
1299 headersize = sizeof(struct ipv6hdr) +
1300 (opt ? opt->opt_flen + opt->opt_nflen : 0) +
1301 (dst_allfrag(&rt->dst) ?
1302 sizeof(struct frag_hdr) : 0) +
1303 rt->rt6i_nfheader_len;
1304
1305 /* as per RFC 7112 section 5, the entire IPv6 Header Chain must fit
1306 * the first fragment
1307 */
1308 if (headersize + transhdrlen > mtu)
1309 goto emsgsize;
1310
1311 if (cork->length + length > mtu - headersize && ipc6->dontfrag &&
1312 (sk->sk_protocol == IPPROTO_UDP ||
1313 sk->sk_protocol == IPPROTO_RAW)) {
1314 ipv6_local_rxpmtu(sk, fl6, mtu - headersize +
1315 sizeof(struct ipv6hdr));
1316 goto emsgsize;
1317 }
1318
1319 if (ip6_sk_ignore_df(sk))
1320 maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN;
1321 else
1322 maxnonfragsize = mtu;
1323
1324 if (cork->length + length > maxnonfragsize - headersize) {
1325 emsgsize:
1326 pmtu = max_t(int, mtu - headersize + sizeof(struct ipv6hdr), 0);
1327 ipv6_local_error(sk, EMSGSIZE, fl6, pmtu);
1328 return -EMSGSIZE;
1329 }
1330
1331 /* CHECKSUM_PARTIAL only with no extension headers and when
1332 * we are not going to fragment
1333 */
1334 if (transhdrlen && sk->sk_protocol == IPPROTO_UDP &&
1335 headersize == sizeof(struct ipv6hdr) &&
1336 length <= mtu - headersize &&
1337 (!(flags & MSG_MORE) || cork->gso_size) &&
1338 rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))
1339 csummode = CHECKSUM_PARTIAL;
1340
1341 if (flags & MSG_ZEROCOPY && length && sock_flag(sk, SOCK_ZEROCOPY)) {
1342 uarg = sock_zerocopy_realloc(sk, length, skb_zcopy(skb));
1343 if (!uarg)
1344 return -ENOBUFS;
1345 extra_uref = true;
1346 if (rt->dst.dev->features & NETIF_F_SG &&
1347 csummode == CHECKSUM_PARTIAL) {
1348 paged = true;
1349 } else {
1350 uarg->zerocopy = 0;
1351 skb_zcopy_set(skb, uarg, &extra_uref);
1352 }
1353 }
1354
1355 /*
1356 * Let's try using as much space as possible.
1357 * Use MTU if total length of the message fits into the MTU.
1358 * Otherwise, we need to reserve fragment header and
1359 * fragment alignment (= 8-15 octects, in total).
1360 *
1361 * Note that we may need to "move" the data from the tail of
1362 * of the buffer to the new fragment when we split
1363 * the message.
1364 *
1365 * FIXME: It may be fragmented into multiple chunks
1366 * at once if non-fragmentable extension headers
1367 * are too large.
1368 * --yoshfuji
1369 */
1370
1371 cork->length += length;
1372 if (!skb)
1373 goto alloc_new_skb;
1374
1375 while (length > 0) {
1376 /* Check if the remaining data fits into current packet. */
1377 copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len;
1378 if (copy < length)
1379 copy = maxfraglen - skb->len;
1380
1381 if (copy <= 0) {
1382 char *data;
1383 unsigned int datalen;
1384 unsigned int fraglen;
1385 unsigned int fraggap;
1386 unsigned int alloclen;
1387 unsigned int pagedlen;
1388 alloc_new_skb:
1389 /* There's no room in the current skb */
1390 if (skb)
1391 fraggap = skb->len - maxfraglen;
1392 else
1393 fraggap = 0;
1394 /* update mtu and maxfraglen if necessary */
1395 if (!skb || !skb_prev)
1396 ip6_append_data_mtu(&mtu, &maxfraglen,
1397 fragheaderlen, skb, rt,
1398 orig_mtu);
1399
1400 skb_prev = skb;
1401
1402 /*
1403 * If remaining data exceeds the mtu,
1404 * we know we need more fragment(s).
1405 */
1406 datalen = length + fraggap;
1407
1408 if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen)
1409 datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;
1410 fraglen = datalen + fragheaderlen;
1411 pagedlen = 0;
1412
1413 if ((flags & MSG_MORE) &&
1414 !(rt->dst.dev->features&NETIF_F_SG))
1415 alloclen = mtu;
1416 else if (!paged)
1417 alloclen = fraglen;
1418 else {
1419 alloclen = min_t(int, fraglen, MAX_HEADER);
1420 pagedlen = fraglen - alloclen;
1421 }
1422
1423 alloclen += dst_exthdrlen;
1424
1425 if (datalen != length + fraggap) {
1426 /*
1427 * this is not the last fragment, the trailer
1428 * space is regarded as data space.
1429 */
1430 datalen += rt->dst.trailer_len;
1431 }
1432
1433 alloclen += rt->dst.trailer_len;
1434 fraglen = datalen + fragheaderlen;
1435
1436 /*
1437 * We just reserve space for fragment header.
1438 * Note: this may be overallocation if the message
1439 * (without MSG_MORE) fits into the MTU.
1440 */
1441 alloclen += sizeof(struct frag_hdr);
1442
1443 copy = datalen - transhdrlen - fraggap - pagedlen;
1444 if (copy < 0) {
1445 err = -EINVAL;
1446 goto error;
1447 }
1448 if (transhdrlen) {
1449 skb = sock_alloc_send_skb(sk,
1450 alloclen + hh_len,
1451 (flags & MSG_DONTWAIT), &err);
1452 } else {
1453 skb = NULL;
1454 if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <=
1455 2 * sk->sk_sndbuf)
1456 skb = alloc_skb(alloclen + hh_len,
1457 sk->sk_allocation);
1458 if (unlikely(!skb))
1459 err = -ENOBUFS;
1460 }
1461 if (!skb)
1462 goto error;
1463 /*
1464 * Fill in the control structures
1465 */
1466 skb->protocol = htons(ETH_P_IPV6);
1467 skb->ip_summed = csummode;
1468 skb->csum = 0;
1469 /* reserve for fragmentation and ipsec header */
1470 skb_reserve(skb, hh_len + sizeof(struct frag_hdr) +
1471 dst_exthdrlen);
1472
1473 /*
1474 * Find where to start putting bytes
1475 */
1476 data = skb_put(skb, fraglen - pagedlen);
1477 skb_set_network_header(skb, exthdrlen);
1478 data += fragheaderlen;
1479 skb->transport_header = (skb->network_header +
1480 fragheaderlen);
1481 if (fraggap) {
1482 skb->csum = skb_copy_and_csum_bits(
1483 skb_prev, maxfraglen,
1484 data + transhdrlen, fraggap, 0);
1485 skb_prev->csum = csum_sub(skb_prev->csum,
1486 skb->csum);
1487 data += fraggap;
1488 pskb_trim_unique(skb_prev, maxfraglen);
1489 }
1490 if (copy > 0 &&
1491 getfrag(from, data + transhdrlen, offset,
1492 copy, fraggap, skb) < 0) {
1493 err = -EFAULT;
1494 kfree_skb(skb);
1495 goto error;
1496 }
1497
1498 offset += copy;
1499 length -= copy + transhdrlen;
1500 transhdrlen = 0;
1501 exthdrlen = 0;
1502 dst_exthdrlen = 0;
1503
1504 /* Only the initial fragment is time stamped */
1505 skb_shinfo(skb)->tx_flags = cork->tx_flags;
1506 cork->tx_flags = 0;
1507 skb_shinfo(skb)->tskey = tskey;
1508 tskey = 0;
1509 skb_zcopy_set(skb, uarg, &extra_uref);
1510
1511 if ((flags & MSG_CONFIRM) && !skb_prev)
1512 skb_set_dst_pending_confirm(skb, 1);
1513
1514 /*
1515 * Put the packet on the pending queue
1516 */
1517 if (!skb->destructor) {
1518 skb->destructor = sock_wfree;
1519 skb->sk = sk;
1520 wmem_alloc_delta += skb->truesize;
1521 }
1522 __skb_queue_tail(queue, skb);
1523 continue;
1524 }
1525
1526 if (copy > length)
1527 copy = length;
1528
1529 if (!(rt->dst.dev->features&NETIF_F_SG) &&
1530 skb_tailroom(skb) >= copy) {
1531 unsigned int off;
1532
1533 off = skb->len;
1534 if (getfrag(from, skb_put(skb, copy),
1535 offset, copy, off, skb) < 0) {
1536 __skb_trim(skb, off);
1537 err = -EFAULT;
1538 goto error;
1539 }
1540 } else if (!uarg || !uarg->zerocopy) {
1541 int i = skb_shinfo(skb)->nr_frags;
1542
1543 err = -ENOMEM;
1544 if (!sk_page_frag_refill(sk, pfrag))
1545 goto error;
1546
1547 if (!skb_can_coalesce(skb, i, pfrag->page,
1548 pfrag->offset)) {
1549 err = -EMSGSIZE;
1550 if (i == MAX_SKB_FRAGS)
1551 goto error;
1552
1553 __skb_fill_page_desc(skb, i, pfrag->page,
1554 pfrag->offset, 0);
1555 skb_shinfo(skb)->nr_frags = ++i;
1556 get_page(pfrag->page);
1557 }
1558 copy = min_t(int, copy, pfrag->size - pfrag->offset);
1559 if (getfrag(from,
1560 page_address(pfrag->page) + pfrag->offset,
1561 offset, copy, skb->len, skb) < 0)
1562 goto error_efault;
1563
1564 pfrag->offset += copy;
1565 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1566 skb->len += copy;
1567 skb->data_len += copy;
1568 skb->truesize += copy;
1569 wmem_alloc_delta += copy;
1570 } else {
1571 err = skb_zerocopy_iter_dgram(skb, from, copy);
1572 if (err < 0)
1573 goto error;
1574 }
1575 offset += copy;
1576 length -= copy;
1577 }
1578
1579 if (wmem_alloc_delta)
1580 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
1581 return 0;
1582
1583 error_efault:
1584 err = -EFAULT;
1585 error:
1586 if (uarg)
1587 sock_zerocopy_put_abort(uarg, extra_uref);
1588 cork->length -= length;
1589 IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
1590 refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
1591 return err;
1592 }
1593
1594 int ip6_append_data(struct sock *sk,
1595 int getfrag(void *from, char *to, int offset, int len,
1596 int odd, struct sk_buff *skb),
1597 void *from, int length, int transhdrlen,
1598 struct ipcm6_cookie *ipc6, struct flowi6 *fl6,
1599 struct rt6_info *rt, unsigned int flags)
1600 {
1601 struct inet_sock *inet = inet_sk(sk);
1602 struct ipv6_pinfo *np = inet6_sk(sk);
1603 int exthdrlen;
1604 int err;
1605
1606 if (flags&MSG_PROBE)
1607 return 0;
1608 if (skb_queue_empty(&sk->sk_write_queue)) {
1609 /*
1610 * setup for corking
1611 */
1612 err = ip6_setup_cork(sk, &inet->cork, &np->cork,
1613 ipc6, rt, fl6);
1614 if (err)
1615 return err;
1616
1617 exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0);
1618 length += exthdrlen;
1619 transhdrlen += exthdrlen;
1620 } else {
1621 fl6 = &inet->cork.fl.u.ip6;
1622 transhdrlen = 0;
1623 }
1624
1625 return __ip6_append_data(sk, fl6, &sk->sk_write_queue, &inet->cork.base,
1626 &np->cork, sk_page_frag(sk), getfrag,
1627 from, length, transhdrlen, flags, ipc6);
1628 }
1629 EXPORT_SYMBOL_GPL(ip6_append_data);
1630
1631 static void ip6_cork_release(struct inet_cork_full *cork,
1632 struct inet6_cork *v6_cork)
1633 {
1634 if (v6_cork->opt) {
1635 kfree(v6_cork->opt->dst0opt);
1636 kfree(v6_cork->opt->dst1opt);
1637 kfree(v6_cork->opt->hopopt);
1638 kfree(v6_cork->opt->srcrt);
1639 kfree(v6_cork->opt);
1640 v6_cork->opt = NULL;
1641 }
1642
1643 if (cork->base.dst) {
1644 dst_release(cork->base.dst);
1645 cork->base.dst = NULL;
1646 cork->base.flags &= ~IPCORK_ALLFRAG;
1647 }
1648 memset(&cork->fl, 0, sizeof(cork->fl));
1649 }
1650
1651 struct sk_buff *__ip6_make_skb(struct sock *sk,
1652 struct sk_buff_head *queue,
1653 struct inet_cork_full *cork,
1654 struct inet6_cork *v6_cork)
1655 {
1656 struct sk_buff *skb, *tmp_skb;
1657 struct sk_buff **tail_skb;
1658 struct in6_addr final_dst_buf, *final_dst = &final_dst_buf;
1659 struct ipv6_pinfo *np = inet6_sk(sk);
1660 struct net *net = sock_net(sk);
1661 struct ipv6hdr *hdr;
1662 struct ipv6_txoptions *opt = v6_cork->opt;
1663 struct rt6_info *rt = (struct rt6_info *)cork->base.dst;
1664 struct flowi6 *fl6 = &cork->fl.u.ip6;
1665 unsigned char proto = fl6->flowi6_proto;
1666
1667 skb = __skb_dequeue(queue);
1668 if (!skb)
1669 goto out;
1670 tail_skb = &(skb_shinfo(skb)->frag_list);
1671
1672 /* move skb->data to ip header from ext header */
1673 if (skb->data < skb_network_header(skb))
1674 __skb_pull(skb, skb_network_offset(skb));
1675 while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
1676 __skb_pull(tmp_skb, skb_network_header_len(skb));
1677 *tail_skb = tmp_skb;
1678 tail_skb = &(tmp_skb->next);
1679 skb->len += tmp_skb->len;
1680 skb->data_len += tmp_skb->len;
1681 skb->truesize += tmp_skb->truesize;
1682 tmp_skb->destructor = NULL;
1683 tmp_skb->sk = NULL;
1684 }
1685
1686 /* Allow local fragmentation. */
1687 skb->ignore_df = ip6_sk_ignore_df(sk);
1688
1689 *final_dst = fl6->daddr;
1690 __skb_pull(skb, skb_network_header_len(skb));
1691 if (opt && opt->opt_flen)
1692 ipv6_push_frag_opts(skb, opt, &proto);
1693 if (opt && opt->opt_nflen)
1694 ipv6_push_nfrag_opts(skb, opt, &proto, &final_dst, &fl6->saddr);
1695
1696 skb_push(skb, sizeof(struct ipv6hdr));
1697 skb_reset_network_header(skb);
1698 hdr = ipv6_hdr(skb);
1699
1700 ip6_flow_hdr(hdr, v6_cork->tclass,
1701 ip6_make_flowlabel(net, skb, fl6->flowlabel,
1702 ip6_autoflowlabel(net, np), fl6));
1703 hdr->hop_limit = v6_cork->hop_limit;
1704 hdr->nexthdr = proto;
1705 hdr->saddr = fl6->saddr;
1706 hdr->daddr = *final_dst;
1707
1708 skb->priority = sk->sk_priority;
1709 skb->mark = sk->sk_mark;
1710
1711 skb->tstamp = cork->base.transmit_time;
1712
1713 skb_dst_set(skb, dst_clone(&rt->dst));
1714 IP6_UPD_PO_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUT, skb->len);
1715 if (proto == IPPROTO_ICMPV6) {
1716 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
1717
1718 ICMP6MSGOUT_INC_STATS(net, idev, icmp6_hdr(skb)->icmp6_type);
1719 ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS);
1720 }
1721
1722 ip6_cork_release(cork, v6_cork);
1723 out:
1724 return skb;
1725 }
1726
1727 int ip6_send_skb(struct sk_buff *skb)
1728 {
1729 struct net *net = sock_net(skb->sk);
1730 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb);
1731 int err;
1732
1733 err = ip6_local_out(net, skb->sk, skb);
1734 if (err) {
1735 if (err > 0)
1736 err = net_xmit_errno(err);
1737 if (err)
1738 IP6_INC_STATS(net, rt->rt6i_idev,
1739 IPSTATS_MIB_OUTDISCARDS);
1740 }
1741
1742 return err;
1743 }
1744
1745 int ip6_push_pending_frames(struct sock *sk)
1746 {
1747 struct sk_buff *skb;
1748
1749 skb = ip6_finish_skb(sk);
1750 if (!skb)
1751 return 0;
1752
1753 return ip6_send_skb(skb);
1754 }
1755 EXPORT_SYMBOL_GPL(ip6_push_pending_frames);
1756
1757 static void __ip6_flush_pending_frames(struct sock *sk,
1758 struct sk_buff_head *queue,
1759 struct inet_cork_full *cork,
1760 struct inet6_cork *v6_cork)
1761 {
1762 struct sk_buff *skb;
1763
1764 while ((skb = __skb_dequeue_tail(queue)) != NULL) {
1765 if (skb_dst(skb))
1766 IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)),
1767 IPSTATS_MIB_OUTDISCARDS);
1768 kfree_skb(skb);
1769 }
1770
1771 ip6_cork_release(cork, v6_cork);
1772 }
1773
1774 void ip6_flush_pending_frames(struct sock *sk)
1775 {
1776 __ip6_flush_pending_frames(sk, &sk->sk_write_queue,
1777 &inet_sk(sk)->cork, &inet6_sk(sk)->cork);
1778 }
1779 EXPORT_SYMBOL_GPL(ip6_flush_pending_frames);
1780
1781 struct sk_buff *ip6_make_skb(struct sock *sk,
1782 int getfrag(void *from, char *to, int offset,
1783 int len, int odd, struct sk_buff *skb),
1784 void *from, int length, int transhdrlen,
1785 struct ipcm6_cookie *ipc6, struct flowi6 *fl6,
1786 struct rt6_info *rt, unsigned int flags,
1787 struct inet_cork_full *cork)
1788 {
1789 struct inet6_cork v6_cork;
1790 struct sk_buff_head queue;
1791 int exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0);
1792 int err;
1793
1794 if (flags & MSG_PROBE)
1795 return NULL;
1796
1797 __skb_queue_head_init(&queue);
1798
1799 cork->base.flags = 0;
1800 cork->base.addr = 0;
1801 cork->base.opt = NULL;
1802 cork->base.dst = NULL;
1803 v6_cork.opt = NULL;
1804 err = ip6_setup_cork(sk, cork, &v6_cork, ipc6, rt, fl6);
1805 if (err) {
1806 ip6_cork_release(cork, &v6_cork);
1807 return ERR_PTR(err);
1808 }
1809 if (ipc6->dontfrag < 0)
1810 ipc6->dontfrag = inet6_sk(sk)->dontfrag;
1811
1812 err = __ip6_append_data(sk, fl6, &queue, &cork->base, &v6_cork,
1813 &current->task_frag, getfrag, from,
1814 length + exthdrlen, transhdrlen + exthdrlen,
1815 flags, ipc6);
1816 if (err) {
1817 __ip6_flush_pending_frames(sk, &queue, cork, &v6_cork);
1818 return ERR_PTR(err);
1819 }
1820
1821 return __ip6_make_skb(sk, &queue, cork, &v6_cork);
1822 }
Mirror https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git