1 diff -ruN linux-3.10.27/drivers/net/imq.c linux-3.10.27-imq/drivers/net/imq.c
2 --- linux-3.10.27/drivers/net/imq.c 1970-01-01 01:00:00.000000000 +0100
3 +++ linux-3.10.27-imq/drivers/net/imq.c 2014-01-18 10:19:59.342342913 +0100
6 + * Pseudo-driver for the intermediate queue device.
8 + * This program is free software; you can redistribute it and/or
9 + * modify it under the terms of the GNU General Public License
10 + * as published by the Free Software Foundation; either version
11 + * 2 of the License, or (at your option) any later version.
13 + * Authors: Patrick McHardy, <kaber@trash.net>
15 + * The first version was written by Martin Devera, <devik@cdi.cz>
17 + * Credits: Jan Rafaj <imq2t@cedric.vabo.cz>
18 + * - Update patch to 2.4.21
19 + * Sebastian Strollo <sstrollo@nortelnetworks.com>
20 + * - Fix "Dead-loop on netdevice imq"-issue
21 + * Marcel Sebek <sebek64@post.cz>
22 + * - Update to 2.6.2-rc1
24 + * After some time of inactivity there is a group taking care
25 + * of IMQ again: http://www.linuximq.net
28 + * 2004/06/30 - New version of IMQ patch to kernels <=2.6.7
29 + * including the following changes:
31 + * - Correction of ipv6 support "+"s issue (Hasso Tepper)
32 + * - Correction of imq_init_devs() issue that resulted in
33 + * kernel OOPS unloading IMQ as module (Norbert Buchmuller)
34 + * - Addition of functionality to choose number of IMQ devices
35 + * during kernel config (Andre Correa)
36 + * - Addition of functionality to choose how IMQ hooks on
37 + * PRE and POSTROUTING (after or before NAT) (Andre Correa)
38 + * - Cosmetic corrections (Norbert Buchmuller) (Andre Correa)
41 + * 2005/12/16 - IMQ versions between 2.6.7 and 2.6.13 were
42 + * released with almost no problems. 2.6.14-x was released
43 + * with some important changes: nfcache was removed; After
44 + * some weeks of trouble we figured out that some IMQ fields
45 + * in skb were missing in skbuff.c - skb_clone and copy_skb_header.
46 + * These functions are correctly patched by this new patch version.
48 + * Thanks for all who helped to figure out all the problems with
49 + * 2.6.14.x: Patrick McHardy, Rune Kock, VeNoMouS, Max CtRiX,
50 + * Kevin Shanahan, Richard Lucassen, Valery Dachev (hopefully
51 + * I didn't forget anybody). I apologize again for my lack of time.
54 + * 2008/06/17 - 2.6.25 - Changed imq.c to use qdisc_run() instead
55 + * of qdisc_restart() and moved qdisc_run() to tasklet to avoid
56 + * recursive locking. New initialization routines to fix 'rmmod' not
57 + * working anymore. Used code from ifb.c. (Jussi Kivilinna)
59 + * 2008/08/06 - 2.6.26 - (JK)
60 + * - Replaced tasklet with 'netif_schedule()'.
61 + * - Cleaned up and added comments for imq_nf_queue().
64 + * - Add skb_save_cb/skb_restore_cb helper functions for backuping
65 + * control buffer. This is needed because qdisc-layer on kernels
66 + * 2.6.27 and newer overwrite control buffer. (Jussi Kivilinna)
67 + * - Add better locking for IMQ device. Hopefully this will solve
68 + * SMP issues. (Jussi Kivilinna)
71 + * - Port to 2.6.29 + fix rmmod not working
73 + * 2009/04/20 - (Jussi Kivilinna)
74 + * - Use netdevice feature flags to avoid extra packet handling
75 + * by core networking layer and possibly increase performance.
77 + * 2009/09/26 - (Jussi Kivilinna)
78 + * - Add imq_nf_reinject_lockless to fix deadlock with
79 + * imq_nf_queue/imq_nf_reinject.
81 + * 2009/12/08 - (Jussi Kivilinna)
83 + * - Add check for skb->nf_queue_entry==NULL in imq_dev_xmit()
84 + * - Also add better error checking for skb->nf_queue_entry usage
86 + * 2010/02/25 - (Jussi Kivilinna)
89 + * 2010/08/15 - (Jussi Kivilinna)
91 + * - Simplify hook registration by using nf_register_hooks.
92 + * - nf_reinject doesn't need spinlock around it, therefore remove
93 + * imq_nf_reinject function. Other nf_reinject users protect
94 + * their own data with spinlock. With IMQ however all data is
95 + * needed is stored per skbuff, so no locking is needed.
96 + * - Changed IMQ to use 'separate' NF_IMQ_QUEUE instead of
97 + * NF_QUEUE, this allows working coexistance of IMQ and other
99 + * - Make IMQ multi-queue. Number of IMQ device queues can be
100 + * increased with 'numqueues' module parameters. Default number
101 + * of queues is 1, in other words by default IMQ works as
102 + * single-queue device. Multi-queue selection is based on
103 + * IFB multi-queue patch by Changli Gao <xiaosuo@gmail.com>.
105 + * 2011/03/18 - (Jussi Kivilinna)
108 + * 2011/07/12 - (syoder89@gmail.com)
109 + * - Crash fix that happens when the receiving interface has more
110 + * than one queue (add missing skb_set_queue_mapping in
111 + * imq_select_queue).
113 + * 2011/07/26 - (Jussi Kivilinna)
114 + * - Add queue mapping checks for packets exiting IMQ.
117 + * 2011/08/16 - (Jussi Kivilinna)
118 + * - Clear IFF_TX_SKB_SHARING flag that was added for linux 3.0.2
120 + * 2011/11/03 - Germano Michel <germanomichel@gmail.com>
121 + * - Fix IMQ for net namespaces
123 + * 2011/11/04 - Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
125 + * - Clean-up, move 'get imq device pointer by imqX name' to
126 + * separate function from imq_nf_queue().
128 + * 2012/01/05 - Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
131 + * 2012/03/19 - Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
134 + * 2012/12/12 - Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
136 + * - Fix checkpatch.pl warnings
138 + * 2013/09/10 - Jussi Kivilinna <jussi.kivilinna@iki.fi>
139 + * - Fixed GSO handling for 3.10, see imq_nf_queue() for comments.
140 + * - Don't copy skb->cb_next when copying or cloning skbuffs.
142 + * Also, many thanks to pablo Sebastian Greco for making the initial
143 + * patch and to those who helped the testing.
145 + * More info at: http://www.linuximq.net/ (Andre Correa)
148 +#include <linux/module.h>
149 +#include <linux/kernel.h>
150 +#include <linux/moduleparam.h>
151 +#include <linux/list.h>
152 +#include <linux/skbuff.h>
153 +#include <linux/netdevice.h>
154 +#include <linux/etherdevice.h>
155 +#include <linux/rtnetlink.h>
156 +#include <linux/if_arp.h>
157 +#include <linux/netfilter.h>
158 +#include <linux/netfilter_ipv4.h>
159 +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
160 + #include <linux/netfilter_ipv6.h>
162 +#include <linux/imq.h>
163 +#include <net/pkt_sched.h>
164 +#include <net/netfilter/nf_queue.h>
165 +#include <net/sock.h>
166 +#include <linux/ip.h>
167 +#include <linux/ipv6.h>
168 +#include <linux/if_vlan.h>
169 +#include <linux/if_pppox.h>
171 +#include <net/ipv6.h>
173 +static int imq_nf_queue(struct nf_queue_entry *entry, unsigned queue_num);
175 +static nf_hookfn imq_nf_hook;
177 +static struct nf_hook_ops imq_ops[] = {
179 + /* imq_ingress_ipv4 */
180 + .hook = imq_nf_hook,
181 + .owner = THIS_MODULE,
183 + .hooknum = NF_INET_PRE_ROUTING,
184 +#if defined(CONFIG_IMQ_BEHAVIOR_BA) || defined(CONFIG_IMQ_BEHAVIOR_BB)
185 + .priority = NF_IP_PRI_MANGLE + 1,
187 + .priority = NF_IP_PRI_NAT_DST + 1,
191 + /* imq_egress_ipv4 */
192 + .hook = imq_nf_hook,
193 + .owner = THIS_MODULE,
195 + .hooknum = NF_INET_POST_ROUTING,
196 +#if defined(CONFIG_IMQ_BEHAVIOR_AA) || defined(CONFIG_IMQ_BEHAVIOR_BA)
197 + .priority = NF_IP_PRI_LAST,
199 + .priority = NF_IP_PRI_NAT_SRC - 1,
202 +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
204 + /* imq_ingress_ipv6 */
205 + .hook = imq_nf_hook,
206 + .owner = THIS_MODULE,
208 + .hooknum = NF_INET_PRE_ROUTING,
209 +#if defined(CONFIG_IMQ_BEHAVIOR_BA) || defined(CONFIG_IMQ_BEHAVIOR_BB)
210 + .priority = NF_IP6_PRI_MANGLE + 1,
212 + .priority = NF_IP6_PRI_NAT_DST + 1,
216 + /* imq_egress_ipv6 */
217 + .hook = imq_nf_hook,
218 + .owner = THIS_MODULE,
220 + .hooknum = NF_INET_POST_ROUTING,
221 +#if defined(CONFIG_IMQ_BEHAVIOR_AA) || defined(CONFIG_IMQ_BEHAVIOR_BA)
222 + .priority = NF_IP6_PRI_LAST,
224 + .priority = NF_IP6_PRI_NAT_SRC - 1,
230 +#if defined(CONFIG_IMQ_NUM_DEVS)
231 +static int numdevs = CONFIG_IMQ_NUM_DEVS;
233 +static int numdevs = IMQ_MAX_DEVS;
236 +static struct net_device *imq_devs_cache[IMQ_MAX_DEVS];
238 +#define IMQ_MAX_QUEUES 32
239 +static int numqueues = 1;
240 +static u32 imq_hashrnd;
242 +static inline __be16 pppoe_proto(const struct sk_buff *skb)
244 + return *((__be16 *)(skb_mac_header(skb) + ETH_HLEN +
245 + sizeof(struct pppoe_hdr)));
248 +static u16 imq_hash(struct net_device *dev, struct sk_buff *skb)
250 + unsigned int pull_len;
251 + u16 protocol = skb->protocol;
263 + switch (protocol) {
264 + case htons(ETH_P_8021Q): {
265 + if (unlikely(skb_pull(skb, VLAN_HLEN) == NULL))
268 + pull_len += VLAN_HLEN;
269 + skb->network_header += VLAN_HLEN;
271 + protocol = vlan_eth_hdr(skb)->h_vlan_encapsulated_proto;
275 + case htons(ETH_P_PPP_SES): {
276 + if (unlikely(skb_pull(skb, PPPOE_SES_HLEN) == NULL))
279 + pull_len += PPPOE_SES_HLEN;
280 + skb->network_header += PPPOE_SES_HLEN;
282 + protocol = pppoe_proto(skb);
286 + case htons(ETH_P_IP): {
287 + const struct iphdr *iph = ip_hdr(skb);
289 + if (unlikely(!pskb_may_pull(skb, sizeof(struct iphdr))))
292 + addr1 = iph->daddr;
293 + addr2 = iph->saddr;
295 + ip_proto = !(ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)) ?
297 + ihl = ip_hdrlen(skb);
301 +#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
302 + case htons(ETH_P_IPV6): {
303 + const struct ipv6hdr *iph = ipv6_hdr(skb);
306 + if (unlikely(!pskb_may_pull(skb, sizeof(struct ipv6hdr))))
309 + addr1 = iph->daddr.s6_addr32[3];
310 + addr2 = iph->saddr.s6_addr32[3];
311 + ihl = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &ip_proto,
313 + if (unlikely(ihl < 0))
321 + if (pull_len != 0) {
322 + skb_push(skb, pull_len);
323 + skb->network_header -= pull_len;
326 + return (u16)(ntohs(protocol) % dev->real_num_tx_queues);
330 + swap(addr1, addr2);
332 + switch (ip_proto) {
339 + case IPPROTO_UDPLITE: {
340 + if (likely(skb_copy_bits(skb, ihl, &ports.in32, 4) >= 0)) {
341 + if (ports.in16[0] > ports.in16[1])
342 + swap(ports.in16[0], ports.in16[1]);
352 + if (pull_len != 0) {
353 + skb_push(skb, pull_len);
354 + skb->network_header -= pull_len;
357 + hash = jhash_3words(addr1, addr2, ports.in32, imq_hashrnd ^ ip_proto);
359 + return (u16)(((u64)hash * dev->real_num_tx_queues) >> 32);
362 +static inline bool sk_tx_queue_recorded(struct sock *sk)
364 + return (sk_tx_queue_get(sk) >= 0);
367 +static struct netdev_queue *imq_select_queue(struct net_device *dev,
368 + struct sk_buff *skb)
370 + u16 queue_index = 0;
373 + if (likely(dev->real_num_tx_queues == 1))
376 + /* IMQ can be receiving ingress or engress packets. */
378 + /* Check first for if rx_queue is set */
379 + if (skb_rx_queue_recorded(skb)) {
380 + queue_index = skb_get_rx_queue(skb);
384 + /* Check if socket has tx_queue set */
385 + if (sk_tx_queue_recorded(skb->sk)) {
386 + queue_index = sk_tx_queue_get(skb->sk);
390 + /* Try use socket hash */
391 + if (skb->sk && skb->sk->sk_hash) {
392 + hash = skb->sk->sk_hash;
394 + (u16)(((u64)hash * dev->real_num_tx_queues) >> 32);
398 + /* Generate hash from packet data */
399 + queue_index = imq_hash(dev, skb);
402 + if (unlikely(queue_index >= dev->real_num_tx_queues))
403 + queue_index = (u16)((u32)queue_index % dev->real_num_tx_queues);
405 + skb_set_queue_mapping(skb, queue_index);
406 + return netdev_get_tx_queue(dev, queue_index);
409 +static struct net_device_stats *imq_get_stats(struct net_device *dev)
411 + return &dev->stats;
414 +/* called for packets kfree'd in qdiscs at places other than enqueue */
415 +static void imq_skb_destructor(struct sk_buff *skb)
417 + struct nf_queue_entry *entry = skb->nf_queue_entry;
419 + skb->nf_queue_entry = NULL;
422 + nf_queue_entry_release_refs(entry);
426 + skb_restore_cb(skb); /* kfree backup */
429 +static void imq_done_check_queue_mapping(struct sk_buff *skb,
430 + struct net_device *dev)
432 + unsigned int queue_index;
434 + /* Don't let queue_mapping be left too large after exiting IMQ */
435 + if (likely(skb->dev != dev && skb->dev != NULL)) {
436 + queue_index = skb_get_queue_mapping(skb);
437 + if (unlikely(queue_index >= skb->dev->real_num_tx_queues)) {
438 + queue_index = (u16)((u32)queue_index %
439 + skb->dev->real_num_tx_queues);
440 + skb_set_queue_mapping(skb, queue_index);
443 + /* skb->dev was IMQ device itself or NULL, be on safe side and
444 + * just clear queue mapping.
446 + skb_set_queue_mapping(skb, 0);
450 +static netdev_tx_t imq_dev_xmit(struct sk_buff *skb, struct net_device *dev)
452 + struct nf_queue_entry *entry = skb->nf_queue_entry;
454 + skb->nf_queue_entry = NULL;
455 + dev->trans_start = jiffies;
457 + dev->stats.tx_bytes += skb->len;
458 + dev->stats.tx_packets++;
460 + if (unlikely(entry == NULL)) {
461 + /* We don't know what is going on here.. packet is queued for
462 + * imq device, but (probably) not by us.
464 + * If this packet was not send here by imq_nf_queue(), then
465 + * skb_save_cb() was not used and skb_free() should not show:
466 + * WARNING: IMQ: kfree_skb: skb->cb_next:..
468 + * WARNING: IMQ: kfree_skb: skb->nf_queue_entry...
470 + * However if this message is shown, then IMQ is somehow broken
471 + * and you should report this to linuximq.net.
474 + /* imq_dev_xmit is black hole that eats all packets, report that
475 + * we eat this packet happily and increase dropped counters.
478 + dev->stats.tx_dropped++;
479 + dev_kfree_skb(skb);
481 + return NETDEV_TX_OK;
484 + skb_restore_cb(skb); /* restore skb->cb */
486 + skb->imq_flags = 0;
487 + skb->destructor = NULL;
489 + imq_done_check_queue_mapping(skb, dev);
491 + nf_reinject(entry, NF_ACCEPT);
493 + return NETDEV_TX_OK;
496 +static struct net_device *get_imq_device_by_index(int index)
498 + struct net_device *dev = NULL;
502 + /* get device by name and cache result */
503 + snprintf(buf, sizeof(buf), "imq%d", index);
505 + /* Search device from all namespaces. */
506 + for_each_net(net) {
507 + dev = dev_get_by_name(net, buf);
512 + if (WARN_ON_ONCE(dev == NULL)) {
513 + /* IMQ device not found. Exotic config? */
514 + return ERR_PTR(-ENODEV);
517 + imq_devs_cache[index] = dev;
523 +static struct nf_queue_entry *nf_queue_entry_dup(struct nf_queue_entry *e)
525 + struct nf_queue_entry *entry = kmemdup(e, e->size, GFP_ATOMIC);
527 + if (nf_queue_entry_get_refs(entry))
534 +#ifdef CONFIG_BRIDGE_NETFILTER
535 +/* When called from bridge netfilter, skb->data must point to MAC header
536 + * before calling skb_gso_segment(). Else, original MAC header is lost
537 + * and segmented skbs will be sent to wrong destination.
539 +static void nf_bridge_adjust_skb_data(struct sk_buff *skb)
541 + if (skb->nf_bridge)
542 + __skb_push(skb, skb->network_header - skb->mac_header);
545 +static void nf_bridge_adjust_segmented_data(struct sk_buff *skb)
547 + if (skb->nf_bridge)
548 + __skb_pull(skb, skb->network_header - skb->mac_header);
551 +#define nf_bridge_adjust_skb_data(s) do {} while (0)
552 +#define nf_bridge_adjust_segmented_data(s) do {} while (0)
555 +static void free_entry(struct nf_queue_entry *entry)
557 + nf_queue_entry_release_refs(entry);
561 +static int __imq_nf_queue(struct nf_queue_entry *entry, struct net_device *dev);
563 +static int __imq_nf_queue_gso(struct nf_queue_entry *entry,
564 + struct net_device *dev, struct sk_buff *skb)
567 + struct nf_queue_entry *entry_seg;
569 + nf_bridge_adjust_segmented_data(skb);
571 + if (skb->next == NULL) { /* last packet, no need to copy entry */
572 + struct sk_buff *gso_skb = entry->skb;
574 + ret = __imq_nf_queue(entry, dev);
576 + entry->skb = gso_skb;
582 + entry_seg = nf_queue_entry_dup(entry);
584 + entry_seg->skb = skb;
585 + ret = __imq_nf_queue(entry_seg, dev);
587 + free_entry(entry_seg);
592 +static int imq_nf_queue(struct nf_queue_entry *entry, unsigned queue_num)
594 + struct sk_buff *skb, *segs;
595 + struct net_device *dev;
596 + unsigned int queued;
597 + int index, retval, err;
599 + index = entry->skb->imq_flags & IMQ_F_IFMASK;
600 + if (unlikely(index > numdevs - 1)) {
601 + if (net_ratelimit())
602 + pr_warn("IMQ: invalid device specified, highest is %u\n",
608 + /* check for imq device by index from cache */
609 + dev = imq_devs_cache[index];
610 + if (unlikely(!dev)) {
611 + dev = get_imq_device_by_index(index);
613 + retval = PTR_ERR(dev);
618 + if (unlikely(!(dev->flags & IFF_UP))) {
619 + entry->skb->imq_flags = 0;
620 + retval = -ECANCELED;
624 + if (!skb_is_gso(entry->skb))
625 + return __imq_nf_queue(entry, dev);
627 + /* Since 3.10.x, GSO handling moved here as result of upstream commit
628 + * a5fedd43d5f6c94c71053a66e4c3d2e35f1731a2 (netfilter: move
629 + * skb_gso_segment into nfnetlink_queue module).
631 + * Following code replicates the gso handling from
632 + * 'net/netfilter/nfnetlink_queue_core.c':nfqnl_enqueue_packet().
637 + switch (entry->pf) {
639 + skb->protocol = htons(ETH_P_IP);
642 + skb->protocol = htons(ETH_P_IPV6);
646 + nf_bridge_adjust_skb_data(skb);
647 + segs = skb_gso_segment(skb, 0);
648 + /* Does not use PTR_ERR to limit the number of error codes that can be
649 + * returned by nf_queue. For instance, callers rely on -ECANCELED to
650 + * mean 'ignore this hook'.
658 + struct sk_buff *nskb = segs->next;
659 + if (nskb && nskb->next)
660 + nskb->cb_next = NULL;
662 + err = __imq_nf_queue_gso(entry, dev, segs);
671 + if (err) /* some segments are already queued */
678 + nf_bridge_adjust_segmented_data(skb);
684 +static int __imq_nf_queue(struct nf_queue_entry *entry, struct net_device *dev)
686 + struct sk_buff *skb_orig, *skb, *skb_shared;
688 + struct netdev_queue *txq;
689 + spinlock_t *root_lock;
691 + int retval = -EINVAL;
692 + unsigned int orig_queue_index;
694 + dev->last_rx = jiffies;
699 + /* skb has owner? => make clone */
700 + if (unlikely(skb->destructor)) {
702 + skb = skb_clone(skb, GFP_ATOMIC);
703 + if (unlikely(!skb)) {
707 + skb->cb_next = NULL;
711 + skb->nf_queue_entry = entry;
713 + dev->stats.rx_bytes += skb->len;
714 + dev->stats.rx_packets++;
717 + /* skb->dev == NULL causes problems, try the find cause. */
718 + if (net_ratelimit()) {
719 + dev_warn(&dev->dev,
720 + "received packet with skb->dev == NULL\n");
727 + /* Disables softirqs for lock below */
728 + rcu_read_lock_bh();
730 + /* Multi-queue selection */
731 + orig_queue_index = skb_get_queue_mapping(skb);
732 + txq = imq_select_queue(dev, skb);
734 + q = rcu_dereference(txq->qdisc);
735 + if (unlikely(!q->enqueue))
736 + goto packet_not_eaten_by_imq_dev;
738 + root_lock = qdisc_lock(q);
739 + spin_lock(root_lock);
741 + users = atomic_read(&skb->users);
743 + skb_shared = skb_get(skb); /* increase reference count by one */
745 + /* backup skb->cb, as qdisc layer will overwrite it */
746 + skb_save_cb(skb_shared);
747 + qdisc_enqueue_root(skb_shared, q); /* might kfree_skb */
749 + if (likely(atomic_read(&skb_shared->users) == users + 1)) {
750 + kfree_skb(skb_shared); /* decrease reference count by one */
752 + skb->destructor = &imq_skb_destructor;
755 + if (unlikely(skb_orig))
756 + kfree_skb(skb_orig); /* free original */
758 + spin_unlock(root_lock);
759 + rcu_read_unlock_bh();
761 + /* schedule qdisc dequeue */
762 + __netif_schedule(q);
767 + skb_restore_cb(skb_shared); /* restore skb->cb */
768 + skb->nf_queue_entry = NULL;
770 + * qdisc dropped packet and decreased skb reference count of
771 + * skb, so we don't really want to and try refree as that would
772 + * actually destroy the skb.
774 + spin_unlock(root_lock);
775 + goto packet_not_eaten_by_imq_dev;
778 +packet_not_eaten_by_imq_dev:
779 + skb_set_queue_mapping(skb, orig_queue_index);
780 + rcu_read_unlock_bh();
782 + /* cloned? restore original */
783 + if (unlikely(skb_orig)) {
785 + entry->skb = skb_orig;
792 +static unsigned int imq_nf_hook(unsigned int hook, struct sk_buff *pskb,
793 + const struct net_device *indev,
794 + const struct net_device *outdev,
795 + int (*okfn)(struct sk_buff *))
797 + return (pskb->imq_flags & IMQ_F_ENQUEUE) ? NF_IMQ_QUEUE : NF_ACCEPT;
800 +static int imq_close(struct net_device *dev)
802 + netif_stop_queue(dev);
806 +static int imq_open(struct net_device *dev)
808 + netif_start_queue(dev);
812 +static const struct net_device_ops imq_netdev_ops = {
813 + .ndo_open = imq_open,
814 + .ndo_stop = imq_close,
815 + .ndo_start_xmit = imq_dev_xmit,
816 + .ndo_get_stats = imq_get_stats,
819 +static void imq_setup(struct net_device *dev)
821 + dev->netdev_ops = &imq_netdev_ops;
822 + dev->type = ARPHRD_VOID;
823 + dev->mtu = 16000; /* too small? */
824 + dev->tx_queue_len = 11000; /* too big? */
825 + dev->flags = IFF_NOARP;
826 + dev->features = NETIF_F_SG | NETIF_F_FRAGLIST |
827 + NETIF_F_GSO | NETIF_F_HW_CSUM |
829 + dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE |
830 + IFF_TX_SKB_SHARING);
833 +static int imq_validate(struct nlattr *tb[], struct nlattr *data[])
837 + if (tb[IFLA_ADDRESS]) {
838 + if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) {
842 + if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) {
843 + ret = -EADDRNOTAVAIL;
849 + pr_warn("IMQ: imq_validate failed (%d)\n", ret);
853 +static struct rtnl_link_ops imq_link_ops __read_mostly = {
856 + .setup = imq_setup,
857 + .validate = imq_validate,
860 +static const struct nf_queue_handler imq_nfqh = {
861 + .outfn = imq_nf_queue,
864 +static int __init imq_init_hooks(void)
868 + nf_register_queue_imq_handler(&imq_nfqh);
870 + ret = nf_register_hooks(imq_ops, ARRAY_SIZE(imq_ops));
872 + nf_unregister_queue_imq_handler();
877 +static int __init imq_init_one(int index)
879 + struct net_device *dev;
882 + dev = alloc_netdev_mq(0, "imq%d", imq_setup, numqueues);
886 + ret = dev_alloc_name(dev, dev->name);
890 + dev->rtnl_link_ops = &imq_link_ops;
891 + ret = register_netdevice(dev);
901 +static int __init imq_init_devs(void)
905 + if (numdevs < 1 || numdevs > IMQ_MAX_DEVS) {
906 + pr_err("IMQ: numdevs has to be betweed 1 and %u\n",
911 + if (numqueues < 1 || numqueues > IMQ_MAX_QUEUES) {
912 + pr_err("IMQ: numqueues has to be betweed 1 and %u\n",
917 + get_random_bytes(&imq_hashrnd, sizeof(imq_hashrnd));
920 + err = __rtnl_link_register(&imq_link_ops);
922 + for (i = 0; i < numdevs && !err; i++)
923 + err = imq_init_one(i);
926 + __rtnl_link_unregister(&imq_link_ops);
927 + memset(imq_devs_cache, 0, sizeof(imq_devs_cache));
934 +static int __init imq_init_module(void)
938 +#if defined(CONFIG_IMQ_NUM_DEVS)
939 + BUILD_BUG_ON(CONFIG_IMQ_NUM_DEVS > 16);
940 + BUILD_BUG_ON(CONFIG_IMQ_NUM_DEVS < 2);
941 + BUILD_BUG_ON(CONFIG_IMQ_NUM_DEVS - 1 > IMQ_F_IFMASK);
944 + err = imq_init_devs();
946 + pr_err("IMQ: Error trying imq_init_devs(net)\n");
950 + err = imq_init_hooks();
952 + pr_err(KERN_ERR "IMQ: Error trying imq_init_hooks()\n");
953 + rtnl_link_unregister(&imq_link_ops);
954 + memset(imq_devs_cache, 0, sizeof(imq_devs_cache));
958 + pr_info("IMQ driver loaded successfully. (numdevs = %d, numqueues = %d)\n",
959 + numdevs, numqueues);
961 +#if defined(CONFIG_IMQ_BEHAVIOR_BA) || defined(CONFIG_IMQ_BEHAVIOR_BB)
962 + pr_info("\tHooking IMQ before NAT on PREROUTING.\n");
964 + pr_info("\tHooking IMQ after NAT on PREROUTING.\n");
966 +#if defined(CONFIG_IMQ_BEHAVIOR_AB) || defined(CONFIG_IMQ_BEHAVIOR_BB)
967 + pr_info("\tHooking IMQ before NAT on POSTROUTING.\n");
969 + pr_info("\tHooking IMQ after NAT on POSTROUTING.\n");
975 +static void __exit imq_unhook(void)
977 + nf_unregister_hooks(imq_ops, ARRAY_SIZE(imq_ops));
978 + nf_unregister_queue_imq_handler();
981 +static void __exit imq_cleanup_devs(void)
983 + rtnl_link_unregister(&imq_link_ops);
984 + memset(imq_devs_cache, 0, sizeof(imq_devs_cache));
987 +static void __exit imq_exit_module(void)
990 + imq_cleanup_devs();
991 + pr_info("IMQ driver unloaded successfully.\n");
994 +module_init(imq_init_module);
995 +module_exit(imq_exit_module);
997 +module_param(numdevs, int, 0);
998 +module_param(numqueues, int, 0);
999 +MODULE_PARM_DESC(numdevs, "number of IMQ devices (how many imq* devices will be created)");
1000 +MODULE_PARM_DESC(numqueues, "number of queues per IMQ device");
1001 +MODULE_AUTHOR("http://www.linuximq.net");
1002 +MODULE_DESCRIPTION("Pseudo-driver for the intermediate queue device. See http://www.linuximq.net/ for more information.");
1003 +MODULE_LICENSE("GPL");
1004 +MODULE_ALIAS_RTNL_LINK("imq");
1006 diff -ruN linux-3.10.27/drivers/net/Kconfig linux-3.10.27-imq/drivers/net/Kconfig
1007 --- linux-3.10.27/drivers/net/Kconfig 2014-01-16 00:29:14.000000000 +0100
1008 +++ linux-3.10.27-imq/drivers/net/Kconfig 2014-01-18 10:19:59.341342885 +0100
1009 @@ -207,6 +207,125 @@
1014 + tristate "IMQ (intermediate queueing device) support"
1015 + depends on NETDEVICES && NETFILTER
1017 + The IMQ device(s) is used as placeholder for QoS queueing
1018 + disciplines. Every packet entering/leaving the IP stack can be
1019 + directed through the IMQ device where it's enqueued/dequeued to the
1020 + attached qdisc. This allows you to treat network devices as classes
1021 + and distribute bandwidth among them. Iptables is used to specify
1022 + through which IMQ device, if any, packets travel.
1024 + More information at: http://www.linuximq.net/
1026 + To compile this driver as a module, choose M here: the module
1027 + will be called imq. If unsure, say N.
1030 + prompt "IMQ behavior (PRE/POSTROUTING)"
1032 + default IMQ_BEHAVIOR_AB
1034 + This setting defines how IMQ behaves in respect to its
1035 + hooking in PREROUTING and POSTROUTING.
1037 + IMQ can work in any of the following ways:
1039 + PREROUTING | POSTROUTING
1040 + -----------------|-------------------
1041 + #1 After NAT | After NAT
1042 + #2 After NAT | Before NAT
1043 + #3 Before NAT | After NAT
1044 + #4 Before NAT | Before NAT
1046 + The default behavior is to hook before NAT on PREROUTING
1047 + and after NAT on POSTROUTING (#3).
1049 + This settings are specially usefull when trying to use IMQ
1050 + to shape NATed clients.
1052 + More information can be found at: www.linuximq.net
1054 + If not sure leave the default settings alone.
1056 +config IMQ_BEHAVIOR_AA
1059 + This setting defines how IMQ behaves in respect to its
1060 + hooking in PREROUTING and POSTROUTING.
1062 + Choosing this option will make IMQ hook like this:
1064 + PREROUTING: After NAT
1065 + POSTROUTING: After NAT
1067 + More information can be found at: www.linuximq.net
1069 + If not sure leave the default settings alone.
1071 +config IMQ_BEHAVIOR_AB
1074 + This setting defines how IMQ behaves in respect to its
1075 + hooking in PREROUTING and POSTROUTING.
1077 + Choosing this option will make IMQ hook like this:
1079 + PREROUTING: After NAT
1080 + POSTROUTING: Before NAT
1082 + More information can be found at: www.linuximq.net
1084 + If not sure leave the default settings alone.
1086 +config IMQ_BEHAVIOR_BA
1089 + This setting defines how IMQ behaves in respect to its
1090 + hooking in PREROUTING and POSTROUTING.
1092 + Choosing this option will make IMQ hook like this:
1094 + PREROUTING: Before NAT
1095 + POSTROUTING: After NAT
1097 + More information can be found at: www.linuximq.net
1099 + If not sure leave the default settings alone.
1101 +config IMQ_BEHAVIOR_BB
1104 + This setting defines how IMQ behaves in respect to its
1105 + hooking in PREROUTING and POSTROUTING.
1107 + Choosing this option will make IMQ hook like this:
1109 + PREROUTING: Before NAT
1110 + POSTROUTING: Before NAT
1112 + More information can be found at: www.linuximq.net
1114 + If not sure leave the default settings alone.
1118 +config IMQ_NUM_DEVS
1119 + int "Number of IMQ devices"
1124 + This setting defines how many IMQ devices will be created.
1126 + The default value is 16.
1128 + More information can be found at: www.linuximq.net
1130 + If not sure leave the default settings alone.
1133 tristate "Universal TUN/TAP device driver support"
1135 diff -ruN linux-3.10.27/drivers/net/Makefile linux-3.10.27-imq/drivers/net/Makefile
1136 --- linux-3.10.27/drivers/net/Makefile 2014-01-16 00:29:14.000000000 +0100
1137 +++ linux-3.10.27-imq/drivers/net/Makefile 2014-01-18 10:19:59.341342885 +0100
1139 obj-$(CONFIG_DUMMY) += dummy.o
1140 obj-$(CONFIG_EQUALIZER) += eql.o
1141 obj-$(CONFIG_IFB) += ifb.o
1142 +obj-$(CONFIG_IMQ) += imq.o
1143 obj-$(CONFIG_MACVLAN) += macvlan.o
1144 obj-$(CONFIG_MACVTAP) += macvtap.o
1145 obj-$(CONFIG_MII) += mii.o
1146 diff -ruN linux-3.10.27/include/linux/imq.h linux-3.10.27-imq/include/linux/imq.h
1147 --- linux-3.10.27/include/linux/imq.h 1970-01-01 01:00:00.000000000 +0100
1148 +++ linux-3.10.27-imq/include/linux/imq.h 2014-01-18 10:19:59.342342913 +0100
1153 +/* IFMASK (16 device indexes, 0 to 15) and flag(s) fit in 5 bits */
1154 +#define IMQ_F_BITS 5
1156 +#define IMQ_F_IFMASK 0x0f
1157 +#define IMQ_F_ENQUEUE 0x10
1159 +#define IMQ_MAX_DEVS (IMQ_F_IFMASK + 1)
1161 +#endif /* _IMQ_H */
1163 diff -ruN linux-3.10.27/include/linux/netfilter/xt_IMQ.h linux-3.10.27-imq/include/linux/netfilter/xt_IMQ.h
1164 --- linux-3.10.27/include/linux/netfilter/xt_IMQ.h 1970-01-01 01:00:00.000000000 +0100
1165 +++ linux-3.10.27-imq/include/linux/netfilter/xt_IMQ.h 2014-01-18 10:19:59.342342913 +0100
1170 +struct xt_imq_info {
1171 + unsigned int todev; /* target imq device */
1174 +#endif /* _XT_IMQ_H */
1176 diff -ruN linux-3.10.27/include/linux/netfilter_ipv4/ipt_IMQ.h linux-3.10.27-imq/include/linux/netfilter_ipv4/ipt_IMQ.h
1177 --- linux-3.10.27/include/linux/netfilter_ipv4/ipt_IMQ.h 1970-01-01 01:00:00.000000000 +0100
1178 +++ linux-3.10.27-imq/include/linux/netfilter_ipv4/ipt_IMQ.h 2014-01-18 10:19:59.343342933 +0100
1183 +/* Backwards compatibility for old userspace */
1184 +#include <linux/netfilter/xt_IMQ.h>
1186 +#define ipt_imq_info xt_imq_info
1188 +#endif /* _IPT_IMQ_H */
1190 diff -ruN linux-3.10.27/include/linux/netfilter_ipv6/ip6t_IMQ.h linux-3.10.27-imq/include/linux/netfilter_ipv6/ip6t_IMQ.h
1191 --- linux-3.10.27/include/linux/netfilter_ipv6/ip6t_IMQ.h 1970-01-01 01:00:00.000000000 +0100
1192 +++ linux-3.10.27-imq/include/linux/netfilter_ipv6/ip6t_IMQ.h 2014-01-18 10:19:59.343342933 +0100
1194 +#ifndef _IP6T_IMQ_H
1195 +#define _IP6T_IMQ_H
1197 +/* Backwards compatibility for old userspace */
1198 +#include <linux/netfilter/xt_IMQ.h>
1200 +#define ip6t_imq_info xt_imq_info
1202 +#endif /* _IP6T_IMQ_H */
1204 diff -ruN linux-3.10.27/include/linux/skbuff.h linux-3.10.27-imq/include/linux/skbuff.h
1205 --- linux-3.10.27/include/linux/skbuff.h 2014-01-16 00:29:14.000000000 +0100
1206 +++ linux-3.10.27-imq/include/linux/skbuff.h 2014-01-18 10:18:22.220271201 +0100
1208 #include <linux/dma-mapping.h>
1209 #include <linux/netdev_features.h>
1210 #include <net/flow_keys.h>
1211 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1212 +#include <linux/imq.h>
1215 /* Don't change this without changing skb_csum_unnecessary! */
1216 #define CHECKSUM_NONE 0
1218 * first. This is owned by whoever has the skb queued ATM.
1220 char cb[48] __aligned(8);
1221 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1225 unsigned long _skb_refdst;
1228 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
1229 struct nf_conntrack *nfct;
1231 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1232 + struct nf_queue_entry *nf_queue_entry;
1234 #ifdef CONFIG_BRIDGE_NETFILTER
1235 struct nf_bridge_info *nf_bridge;
1238 __u8 encapsulation:1;
1239 /* 7/9 bit hole (depending on ndisc_nodetype presence) */
1240 kmemcheck_bitfield_end(flags2);
1242 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1243 + __u8 imq_flags:IMQ_F_BITS;
1245 #ifdef CONFIG_NET_DMA
1246 dma_cookie_t dma_cookie;
1248 @@ -616,7 +627,10 @@
1250 return (struct rtable *)skb_dst(skb);
1253 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1254 +extern int skb_save_cb(struct sk_buff *skb);
1255 +extern int skb_restore_cb(struct sk_buff *skb);
1257 extern void kfree_skb(struct sk_buff *skb);
1258 extern void kfree_skb_list(struct sk_buff *segs);
1259 extern void skb_tx_error(struct sk_buff *skb);
1260 @@ -2735,6 +2749,10 @@
1261 nf_conntrack_get(src->nfct);
1262 dst->nfctinfo = src->nfctinfo;
1264 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1265 + dst->imq_flags = src->imq_flags;
1266 + dst->nf_queue_entry = src->nf_queue_entry;
1268 #ifdef CONFIG_BRIDGE_NETFILTER
1269 dst->nf_bridge = src->nf_bridge;
1270 nf_bridge_get(src->nf_bridge);
1271 diff -ruN linux-3.10.27/include/net/netfilter/nf_queue.h linux-3.10.27-imq/include/net/netfilter/nf_queue.h
1272 --- linux-3.10.27/include/net/netfilter/nf_queue.h 2014-01-16 00:29:14.000000000 +0100
1273 +++ linux-3.10.27-imq/include/net/netfilter/nf_queue.h 2014-01-18 10:19:59.345342949 +0100
1275 void nf_register_queue_handler(const struct nf_queue_handler *qh);
1276 void nf_unregister_queue_handler(void);
1277 extern void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict);
1278 +extern void nf_queue_entry_release_refs(struct nf_queue_entry *entry);
1280 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1281 +extern void nf_register_queue_imq_handler(const struct nf_queue_handler *qh);
1282 +extern void nf_unregister_queue_imq_handler(void);
1285 bool nf_queue_entry_get_refs(struct nf_queue_entry *entry);
1286 void nf_queue_entry_release_refs(struct nf_queue_entry *entry);
1287 diff -ruN linux-3.10.27/include/uapi/linux/netfilter.h linux-3.10.27-imq/include/uapi/linux/netfilter.h
1288 --- linux-3.10.27/include/uapi/linux/netfilter.h 2014-01-16 00:29:14.000000000 +0100
1289 +++ linux-3.10.27-imq/include/uapi/linux/netfilter.h 2014-01-18 10:19:59.345342949 +0100
1294 -#define NF_MAX_VERDICT NF_STOP
1295 +#define NF_IMQ_QUEUE 6
1296 +#define NF_MAX_VERDICT NF_IMQ_QUEUE
1298 /* we overload the higher bits for encoding auxiliary data such as the queue
1299 * number or errno values. Not nice, but better than additional function
1300 diff -ruN linux-3.10.27/net/core/dev.c linux-3.10.27-imq/net/core/dev.c
1301 --- linux-3.10.27/net/core/dev.c 2014-01-16 00:29:14.000000000 +0100
1302 +++ linux-3.10.27-imq/net/core/dev.c 2014-01-18 10:19:59.347342963 +0100
1304 #include <linux/inetdevice.h>
1305 #include <linux/cpu_rmap.h>
1306 #include <linux/static_key.h>
1307 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1308 +#include <linux/imq.h>
1311 #include "net-sysfs.h"
1313 @@ -2573,7 +2576,12 @@
1317 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1318 + if (!list_empty(&ptype_all) &&
1319 + !(skb->imq_flags & IMQ_F_ENQUEUE))
1321 if (!list_empty(&ptype_all))
1323 dev_queue_xmit_nit(skb, dev);
1326 diff -ruN linux-3.10.27/net/core/skbuff.c linux-3.10.27-imq/net/core/skbuff.c
1327 --- linux-3.10.27/net/core/skbuff.c 2014-01-16 00:29:14.000000000 +0100
1328 +++ linux-3.10.27-imq/net/core/skbuff.c 2014-01-18 10:19:59.348342972 +0100
1331 struct kmem_cache *skbuff_head_cache __read_mostly;
1332 static struct kmem_cache *skbuff_fclone_cache __read_mostly;
1333 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1334 +static struct kmem_cache *skbuff_cb_store_cache __read_mostly;
1337 static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
1338 struct pipe_buffer *buf)
1343 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1344 +/* Control buffer save/restore for IMQ devices */
1345 +struct skb_cb_table {
1346 + char cb[48] __aligned(8);
1351 +static DEFINE_SPINLOCK(skb_cb_store_lock);
1353 +int skb_save_cb(struct sk_buff *skb)
1355 + struct skb_cb_table *next;
1357 + next = kmem_cache_alloc(skbuff_cb_store_cache, GFP_ATOMIC);
1361 + BUILD_BUG_ON(sizeof(skb->cb) != sizeof(next->cb));
1363 + memcpy(next->cb, skb->cb, sizeof(skb->cb));
1364 + next->cb_next = skb->cb_next;
1366 + atomic_set(&next->refcnt, 1);
1368 + skb->cb_next = next;
1371 +EXPORT_SYMBOL(skb_save_cb);
1373 +int skb_restore_cb(struct sk_buff *skb)
1375 + struct skb_cb_table *next;
1377 + if (!skb->cb_next)
1380 + next = skb->cb_next;
1382 + BUILD_BUG_ON(sizeof(skb->cb) != sizeof(next->cb));
1384 + memcpy(skb->cb, next->cb, sizeof(skb->cb));
1385 + skb->cb_next = next->cb_next;
1387 + spin_lock(&skb_cb_store_lock);
1389 + if (atomic_dec_and_test(&next->refcnt))
1390 + kmem_cache_free(skbuff_cb_store_cache, next);
1392 + spin_unlock(&skb_cb_store_lock);
1396 +EXPORT_SYMBOL(skb_restore_cb);
1398 +static void skb_copy_stored_cb(struct sk_buff *new, const struct sk_buff *__old)
1400 + struct skb_cb_table *next;
1401 + struct sk_buff *old;
1403 + if (!__old->cb_next) {
1404 + new->cb_next = NULL;
1408 + spin_lock(&skb_cb_store_lock);
1410 + old = (struct sk_buff *)__old;
1412 + next = old->cb_next;
1413 + atomic_inc(&next->refcnt);
1414 + new->cb_next = next;
1416 + spin_unlock(&skb_cb_store_lock);
1420 /* Pipe buffer operations for a socket. */
1421 static const struct pipe_buf_operations sock_pipe_buf_ops = {
1422 @@ -582,6 +661,28 @@
1424 skb->destructor(skb);
1426 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1428 + * This should not happen. When it does, avoid memleak by restoring
1429 + * the chain of cb-backups.
1431 + while (skb->cb_next != NULL) {
1432 + if (net_ratelimit())
1433 + pr_warn("IMQ: kfree_skb: skb->cb_next: %08x\n",
1434 + (unsigned int)skb->cb_next);
1436 + skb_restore_cb(skb);
1439 + * This should not happen either, nf_queue_entry is nullified in
1440 + * imq_dev_xmit(). If we have non-NULL nf_queue_entry then we are
1441 + * leaking entry pointers, maybe memory. We don't know if this is
1442 + * pointer to already freed memory, or should this be freed.
1443 + * If this happens we need to add refcounting, etc for nf_queue_entry.
1445 + if (skb->nf_queue_entry && net_ratelimit())
1446 + pr_warn("%s\n", "IMQ: kfree_skb: skb->nf_queue_entry != NULL");
1448 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
1449 nf_conntrack_put(skb->nfct);
1451 @@ -713,6 +814,10 @@
1452 new->sp = secpath_get(old->sp);
1454 memcpy(new->cb, old->cb, sizeof(old->cb));
1455 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1456 + new->cb_next = NULL;
1457 + /*skb_copy_stored_cb(new, old);*/
1459 new->csum = old->csum;
1460 new->local_df = old->local_df;
1461 new->pkt_type = old->pkt_type;
1462 @@ -3093,6 +3198,13 @@
1464 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
1466 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
1467 + skbuff_cb_store_cache = kmem_cache_create("skbuff_cb_store_cache",
1468 + sizeof(struct skb_cb_table),
1470 + SLAB_HWCACHE_ALIGN|SLAB_PANIC,
1476 diff -ruN linux-3.10.27/net/core/skbuff.c.orig linux-3.10.27-imq/net/core/skbuff.c.orig
1477 --- linux-3.10.27/net/core/skbuff.c.orig 1970-01-01 01:00:00.000000000 +0100
1478 +++ linux-3.10.27-imq/net/core/skbuff.c.orig 2014-01-16 00:29:14.000000000 +0100
1481 + * Routines having to do with the 'struct sk_buff' memory handlers.
1483 + * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
1484 + * Florian La Roche <rzsfl@rz.uni-sb.de>
1487 + * Alan Cox : Fixed the worst of the load
1489 + * Dave Platt : Interrupt stacking fix.
1490 + * Richard Kooijman : Timestamp fixes.
1491 + * Alan Cox : Changed buffer format.
1492 + * Alan Cox : destructor hook for AF_UNIX etc.
1493 + * Linus Torvalds : Better skb_clone.
1494 + * Alan Cox : Added skb_copy.
1495 + * Alan Cox : Added all the changed routines Linus
1496 + * only put in the headers
1497 + * Ray VanTassle : Fixed --skb->lock in free
1498 + * Alan Cox : skb_copy copy arp field
1499 + * Andi Kleen : slabified it.
1500 + * Robert Olsson : Removed skb_head_pool
1503 + * The __skb_ routines should be called with interrupts
1504 + * disabled, or you better be *real* sure that the operation is atomic
1505 + * with respect to whatever list is being frobbed (e.g. via lock_sock()
1506 + * or via disabling bottom half handlers, etc).
1508 + * This program is free software; you can redistribute it and/or
1509 + * modify it under the terms of the GNU General Public License
1510 + * as published by the Free Software Foundation; either version
1511 + * 2 of the License, or (at your option) any later version.
1515 + * The functions in this file will not compile correctly with gcc 2.4.x
1518 +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
1520 +#include <linux/module.h>
1521 +#include <linux/types.h>
1522 +#include <linux/kernel.h>
1523 +#include <linux/kmemcheck.h>
1524 +#include <linux/mm.h>
1525 +#include <linux/interrupt.h>
1526 +#include <linux/in.h>
1527 +#include <linux/inet.h>
1528 +#include <linux/slab.h>
1529 +#include <linux/netdevice.h>
1530 +#ifdef CONFIG_NET_CLS_ACT
1531 +#include <net/pkt_sched.h>
1533 +#include <linux/string.h>
1534 +#include <linux/skbuff.h>
1535 +#include <linux/splice.h>
1536 +#include <linux/cache.h>
1537 +#include <linux/rtnetlink.h>
1538 +#include <linux/init.h>
1539 +#include <linux/scatterlist.h>
1540 +#include <linux/errqueue.h>
1541 +#include <linux/prefetch.h>
1543 +#include <net/protocol.h>
1544 +#include <net/dst.h>
1545 +#include <net/sock.h>
1546 +#include <net/checksum.h>
1547 +#include <net/xfrm.h>
1549 +#include <asm/uaccess.h>
1550 +#include <trace/events/skb.h>
1551 +#include <linux/highmem.h>
1553 +struct kmem_cache *skbuff_head_cache __read_mostly;
1554 +static struct kmem_cache *skbuff_fclone_cache __read_mostly;
1556 +static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
1557 + struct pipe_buffer *buf)
1559 + put_page(buf->page);
1562 +static void sock_pipe_buf_get(struct pipe_inode_info *pipe,
1563 + struct pipe_buffer *buf)
1565 + get_page(buf->page);
1568 +static int sock_pipe_buf_steal(struct pipe_inode_info *pipe,
1569 + struct pipe_buffer *buf)
1575 +/* Pipe buffer operations for a socket. */
1576 +static const struct pipe_buf_operations sock_pipe_buf_ops = {
1578 + .map = generic_pipe_buf_map,
1579 + .unmap = generic_pipe_buf_unmap,
1580 + .confirm = generic_pipe_buf_confirm,
1581 + .release = sock_pipe_buf_release,
1582 + .steal = sock_pipe_buf_steal,
1583 + .get = sock_pipe_buf_get,
1587 + * skb_panic - private function for out-of-line support
1591 + * @msg: skb_over_panic or skb_under_panic
1593 + * Out-of-line support for skb_put() and skb_push().
1594 + * Called via the wrapper skb_over_panic() or skb_under_panic().
1595 + * Keep out of line to prevent kernel bloat.
1596 + * __builtin_return_address is not used because it is not always reliable.
1598 +static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
1601 + pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
1602 + msg, addr, skb->len, sz, skb->head, skb->data,
1603 + (unsigned long)skb->tail, (unsigned long)skb->end,
1604 + skb->dev ? skb->dev->name : "<NULL>");
1608 +static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
1610 + skb_panic(skb, sz, addr, __func__);
1613 +static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
1615 + skb_panic(skb, sz, addr, __func__);
1619 + * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
1620 + * the caller if emergency pfmemalloc reserves are being used. If it is and
1621 + * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
1622 + * may be used. Otherwise, the packet data may be discarded until enough
1625 +#define kmalloc_reserve(size, gfp, node, pfmemalloc) \
1626 + __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
1628 +static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
1629 + unsigned long ip, bool *pfmemalloc)
1632 + bool ret_pfmemalloc = false;
1635 + * Try a regular allocation, when that fails and we're not entitled
1636 + * to the reserves, fail.
1638 + obj = kmalloc_node_track_caller(size,
1639 + flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
1641 + if (obj || !(gfp_pfmemalloc_allowed(flags)))
1644 + /* Try again but now we are using pfmemalloc reserves */
1645 + ret_pfmemalloc = true;
1646 + obj = kmalloc_node_track_caller(size, flags, node);
1650 + *pfmemalloc = ret_pfmemalloc;
1655 +/* Allocate a new skbuff. We do this ourselves so we can fill in a few
1656 + * 'private' fields and also do memory statistics to find all the
1661 +struct sk_buff *__alloc_skb_head(gfp_t gfp_mask, int node)
1663 + struct sk_buff *skb;
1665 + /* Get the HEAD */
1666 + skb = kmem_cache_alloc_node(skbuff_head_cache,
1667 + gfp_mask & ~__GFP_DMA, node);
1672 + * Only clear those fields we need to clear, not those that we will
1673 + * actually initialise below. Hence, don't put any more fields after
1674 + * the tail pointer in struct sk_buff!
1676 + memset(skb, 0, offsetof(struct sk_buff, tail));
1678 + skb->truesize = sizeof(struct sk_buff);
1679 + atomic_set(&skb->users, 1);
1681 +#ifdef NET_SKBUFF_DATA_USES_OFFSET
1682 + skb->mac_header = ~0U;
1689 + * __alloc_skb - allocate a network buffer
1690 + * @size: size to allocate
1691 + * @gfp_mask: allocation mask
1692 + * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
1693 + * instead of head cache and allocate a cloned (child) skb.
1694 + * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
1695 + * allocations in case the data is required for writeback
1696 + * @node: numa node to allocate memory on
1698 + * Allocate a new &sk_buff. The returned buffer has no headroom and a
1699 + * tail room of at least size bytes. The object has a reference count
1700 + * of one. The return is the buffer. On a failure the return is %NULL.
1702 + * Buffers may only be allocated from interrupts using a @gfp_mask of
1705 +struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
1706 + int flags, int node)
1708 + struct kmem_cache *cache;
1709 + struct skb_shared_info *shinfo;
1710 + struct sk_buff *skb;
1714 + cache = (flags & SKB_ALLOC_FCLONE)
1715 + ? skbuff_fclone_cache : skbuff_head_cache;
1717 + if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
1718 + gfp_mask |= __GFP_MEMALLOC;
1720 + /* Get the HEAD */
1721 + skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
1726 + /* We do our best to align skb_shared_info on a separate cache
1727 + * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
1728 + * aligned memory blocks, unless SLUB/SLAB debug is enabled.
1729 + * Both skb->head and skb_shared_info are cache line aligned.
1731 + size = SKB_DATA_ALIGN(size);
1732 + size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1733 + data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
1736 + /* kmalloc(size) might give us more room than requested.
1737 + * Put skb_shared_info exactly at the end of allocated zone,
1738 + * to allow max possible filling before reallocation.
1740 + size = SKB_WITH_OVERHEAD(ksize(data));
1741 + prefetchw(data + size);
1744 + * Only clear those fields we need to clear, not those that we will
1745 + * actually initialise below. Hence, don't put any more fields after
1746 + * the tail pointer in struct sk_buff!
1748 + memset(skb, 0, offsetof(struct sk_buff, tail));
1749 + /* Account for allocated memory : skb + skb->head */
1750 + skb->truesize = SKB_TRUESIZE(size);
1751 + skb->pfmemalloc = pfmemalloc;
1752 + atomic_set(&skb->users, 1);
1755 + skb_reset_tail_pointer(skb);
1756 + skb->end = skb->tail + size;
1757 +#ifdef NET_SKBUFF_DATA_USES_OFFSET
1758 + skb->mac_header = ~0U;
1759 + skb->transport_header = ~0U;
1762 + /* make sure we initialize shinfo sequentially */
1763 + shinfo = skb_shinfo(skb);
1764 + memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
1765 + atomic_set(&shinfo->dataref, 1);
1766 + kmemcheck_annotate_variable(shinfo->destructor_arg);
1768 + if (flags & SKB_ALLOC_FCLONE) {
1769 + struct sk_buff *child = skb + 1;
1770 + atomic_t *fclone_ref = (atomic_t *) (child + 1);
1772 + kmemcheck_annotate_bitfield(child, flags1);
1773 + kmemcheck_annotate_bitfield(child, flags2);
1774 + skb->fclone = SKB_FCLONE_ORIG;
1775 + atomic_set(fclone_ref, 1);
1777 + child->fclone = SKB_FCLONE_UNAVAILABLE;
1778 + child->pfmemalloc = pfmemalloc;
1783 + kmem_cache_free(cache, skb);
1787 +EXPORT_SYMBOL(__alloc_skb);
1790 + * build_skb - build a network buffer
1791 + * @data: data buffer provided by caller
1792 + * @frag_size: size of fragment, or 0 if head was kmalloced
1794 + * Allocate a new &sk_buff. Caller provides space holding head and
1795 + * skb_shared_info. @data must have been allocated by kmalloc()
1796 + * The return is the new skb buffer.
1797 + * On a failure the return is %NULL, and @data is not freed.
1799 + * Before IO, driver allocates only data buffer where NIC put incoming frame
1800 + * Driver should add room at head (NET_SKB_PAD) and
1801 + * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
1802 + * After IO, driver calls build_skb(), to allocate sk_buff and populate it
1803 + * before giving packet to stack.
1804 + * RX rings only contains data buffers, not full skbs.
1806 +struct sk_buff *build_skb(void *data, unsigned int frag_size)
1808 + struct skb_shared_info *shinfo;
1809 + struct sk_buff *skb;
1810 + unsigned int size = frag_size ? : ksize(data);
1812 + skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
1816 + size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1818 + memset(skb, 0, offsetof(struct sk_buff, tail));
1819 + skb->truesize = SKB_TRUESIZE(size);
1820 + skb->head_frag = frag_size != 0;
1821 + atomic_set(&skb->users, 1);
1824 + skb_reset_tail_pointer(skb);
1825 + skb->end = skb->tail + size;
1826 +#ifdef NET_SKBUFF_DATA_USES_OFFSET
1827 + skb->mac_header = ~0U;
1828 + skb->transport_header = ~0U;
1831 + /* make sure we initialize shinfo sequentially */
1832 + shinfo = skb_shinfo(skb);
1833 + memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
1834 + atomic_set(&shinfo->dataref, 1);
1835 + kmemcheck_annotate_variable(shinfo->destructor_arg);
1839 +EXPORT_SYMBOL(build_skb);
1841 +struct netdev_alloc_cache {
1842 + struct page_frag frag;
1843 + /* we maintain a pagecount bias, so that we dont dirty cache line
1844 + * containing page->_count every time we allocate a fragment.
1846 + unsigned int pagecnt_bias;
1848 +static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache);
1850 +static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
1852 + struct netdev_alloc_cache *nc;
1853 + void *data = NULL;
1855 + unsigned long flags;
1857 + local_irq_save(flags);
1858 + nc = &__get_cpu_var(netdev_alloc_cache);
1859 + if (unlikely(!nc->frag.page)) {
1861 + for (order = NETDEV_FRAG_PAGE_MAX_ORDER; ;) {
1862 + gfp_t gfp = gfp_mask;
1865 + gfp |= __GFP_COMP | __GFP_NOWARN;
1866 + nc->frag.page = alloc_pages(gfp, order);
1867 + if (likely(nc->frag.page))
1872 + nc->frag.size = PAGE_SIZE << order;
1874 + atomic_set(&nc->frag.page->_count, NETDEV_PAGECNT_MAX_BIAS);
1875 + nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
1876 + nc->frag.offset = 0;
1879 + if (nc->frag.offset + fragsz > nc->frag.size) {
1880 + /* avoid unnecessary locked operations if possible */
1881 + if ((atomic_read(&nc->frag.page->_count) == nc->pagecnt_bias) ||
1882 + atomic_sub_and_test(nc->pagecnt_bias, &nc->frag.page->_count))
1887 + data = page_address(nc->frag.page) + nc->frag.offset;
1888 + nc->frag.offset += fragsz;
1889 + nc->pagecnt_bias--;
1891 + local_irq_restore(flags);
1896 + * netdev_alloc_frag - allocate a page fragment
1897 + * @fragsz: fragment size
1899 + * Allocates a frag from a page for receive buffer.
1900 + * Uses GFP_ATOMIC allocations.
1902 +void *netdev_alloc_frag(unsigned int fragsz)
1904 + return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD);
1906 +EXPORT_SYMBOL(netdev_alloc_frag);
1909 + * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
1910 + * @dev: network device to receive on
1911 + * @length: length to allocate
1912 + * @gfp_mask: get_free_pages mask, passed to alloc_skb
1914 + * Allocate a new &sk_buff and assign it a usage count of one. The
1915 + * buffer has unspecified headroom built in. Users should allocate
1916 + * the headroom they think they need without accounting for the
1917 + * built in space. The built in space is used for optimisations.
1919 + * %NULL is returned if there is no free memory.
1921 +struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
1922 + unsigned int length, gfp_t gfp_mask)
1924 + struct sk_buff *skb = NULL;
1925 + unsigned int fragsz = SKB_DATA_ALIGN(length + NET_SKB_PAD) +
1926 + SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1928 + if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) {
1931 + if (sk_memalloc_socks())
1932 + gfp_mask |= __GFP_MEMALLOC;
1934 + data = __netdev_alloc_frag(fragsz, gfp_mask);
1936 + if (likely(data)) {
1937 + skb = build_skb(data, fragsz);
1938 + if (unlikely(!skb))
1939 + put_page(virt_to_head_page(data));
1942 + skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask,
1943 + SKB_ALLOC_RX, NUMA_NO_NODE);
1945 + if (likely(skb)) {
1946 + skb_reserve(skb, NET_SKB_PAD);
1951 +EXPORT_SYMBOL(__netdev_alloc_skb);
1953 +void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
1954 + int size, unsigned int truesize)
1956 + skb_fill_page_desc(skb, i, page, off, size);
1958 + skb->data_len += size;
1959 + skb->truesize += truesize;
1961 +EXPORT_SYMBOL(skb_add_rx_frag);
1963 +static void skb_drop_list(struct sk_buff **listp)
1965 + kfree_skb_list(*listp);
1969 +static inline void skb_drop_fraglist(struct sk_buff *skb)
1971 + skb_drop_list(&skb_shinfo(skb)->frag_list);
1974 +static void skb_clone_fraglist(struct sk_buff *skb)
1976 + struct sk_buff *list;
1978 + skb_walk_frags(skb, list)
1982 +static void skb_free_head(struct sk_buff *skb)
1984 + if (skb->head_frag)
1985 + put_page(virt_to_head_page(skb->head));
1990 +static void skb_release_data(struct sk_buff *skb)
1992 + if (!skb->cloned ||
1993 + !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
1994 + &skb_shinfo(skb)->dataref)) {
1995 + if (skb_shinfo(skb)->nr_frags) {
1997 + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1998 + skb_frag_unref(skb, i);
2002 + * If skb buf is from userspace, we need to notify the caller
2003 + * the lower device DMA has done;
2005 + if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
2006 + struct ubuf_info *uarg;
2008 + uarg = skb_shinfo(skb)->destructor_arg;
2009 + if (uarg->callback)
2010 + uarg->callback(uarg, true);
2013 + if (skb_has_frag_list(skb))
2014 + skb_drop_fraglist(skb);
2016 + skb_free_head(skb);
2021 + * Free an skbuff by memory without cleaning the state.
2023 +static void kfree_skbmem(struct sk_buff *skb)
2025 + struct sk_buff *other;
2026 + atomic_t *fclone_ref;
2028 + switch (skb->fclone) {
2029 + case SKB_FCLONE_UNAVAILABLE:
2030 + kmem_cache_free(skbuff_head_cache, skb);
2033 + case SKB_FCLONE_ORIG:
2034 + fclone_ref = (atomic_t *) (skb + 2);
2035 + if (atomic_dec_and_test(fclone_ref))
2036 + kmem_cache_free(skbuff_fclone_cache, skb);
2039 + case SKB_FCLONE_CLONE:
2040 + fclone_ref = (atomic_t *) (skb + 1);
2043 + /* The clone portion is available for
2044 + * fast-cloning again.
2046 + skb->fclone = SKB_FCLONE_UNAVAILABLE;
2048 + if (atomic_dec_and_test(fclone_ref))
2049 + kmem_cache_free(skbuff_fclone_cache, other);
2054 +static void skb_release_head_state(struct sk_buff *skb)
2056 + skb_dst_drop(skb);
2058 + secpath_put(skb->sp);
2060 + if (skb->destructor) {
2061 + WARN_ON(in_irq());
2062 + skb->destructor(skb);
2064 +#if IS_ENABLED(CONFIG_NF_CONNTRACK)
2065 + nf_conntrack_put(skb->nfct);
2067 +#ifdef CONFIG_BRIDGE_NETFILTER
2068 + nf_bridge_put(skb->nf_bridge);
2070 +/* XXX: IS this still necessary? - JHS */
2071 +#ifdef CONFIG_NET_SCHED
2072 + skb->tc_index = 0;
2073 +#ifdef CONFIG_NET_CLS_ACT
2079 +/* Free everything but the sk_buff shell. */
2080 +static void skb_release_all(struct sk_buff *skb)
2082 + skb_release_head_state(skb);
2083 + if (likely(skb->head))
2084 + skb_release_data(skb);
2088 + * __kfree_skb - private function
2091 + * Free an sk_buff. Release anything attached to the buffer.
2092 + * Clean the state. This is an internal helper function. Users should
2093 + * always call kfree_skb
2096 +void __kfree_skb(struct sk_buff *skb)
2098 + skb_release_all(skb);
2099 + kfree_skbmem(skb);
2101 +EXPORT_SYMBOL(__kfree_skb);
2104 + * kfree_skb - free an sk_buff
2105 + * @skb: buffer to free
2107 + * Drop a reference to the buffer and free it if the usage count has
2110 +void kfree_skb(struct sk_buff *skb)
2112 + if (unlikely(!skb))
2114 + if (likely(atomic_read(&skb->users) == 1))
2116 + else if (likely(!atomic_dec_and_test(&skb->users)))
2118 + trace_kfree_skb(skb, __builtin_return_address(0));
2121 +EXPORT_SYMBOL(kfree_skb);
2123 +void kfree_skb_list(struct sk_buff *segs)
2126 + struct sk_buff *next = segs->next;
2132 +EXPORT_SYMBOL(kfree_skb_list);
2135 + * skb_tx_error - report an sk_buff xmit error
2136 + * @skb: buffer that triggered an error
2138 + * Report xmit error if a device callback is tracking this skb.
2139 + * skb must be freed afterwards.
2141 +void skb_tx_error(struct sk_buff *skb)
2143 + if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
2144 + struct ubuf_info *uarg;
2146 + uarg = skb_shinfo(skb)->destructor_arg;
2147 + if (uarg->callback)
2148 + uarg->callback(uarg, false);
2149 + skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
2152 +EXPORT_SYMBOL(skb_tx_error);
2155 + * consume_skb - free an skbuff
2156 + * @skb: buffer to free
2158 + * Drop a ref to the buffer and free it if the usage count has hit zero
2159 + * Functions identically to kfree_skb, but kfree_skb assumes that the frame
2160 + * is being dropped after a failure and notes that
2162 +void consume_skb(struct sk_buff *skb)
2164 + if (unlikely(!skb))
2166 + if (likely(atomic_read(&skb->users) == 1))
2168 + else if (likely(!atomic_dec_and_test(&skb->users)))
2170 + trace_consume_skb(skb);
2173 +EXPORT_SYMBOL(consume_skb);
2175 +static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
2177 + new->tstamp = old->tstamp;
2178 + new->dev = old->dev;
2179 + new->transport_header = old->transport_header;
2180 + new->network_header = old->network_header;
2181 + new->mac_header = old->mac_header;
2182 + new->inner_transport_header = old->inner_transport_header;
2183 + new->inner_network_header = old->inner_network_header;
2184 + new->inner_mac_header = old->inner_mac_header;
2185 + skb_dst_copy(new, old);
2186 + new->rxhash = old->rxhash;
2187 + new->ooo_okay = old->ooo_okay;
2188 + new->l4_rxhash = old->l4_rxhash;
2189 + new->no_fcs = old->no_fcs;
2190 + new->encapsulation = old->encapsulation;
2192 + new->sp = secpath_get(old->sp);
2194 + memcpy(new->cb, old->cb, sizeof(old->cb));
2195 + new->csum = old->csum;
2196 + new->local_df = old->local_df;
2197 + new->pkt_type = old->pkt_type;
2198 + new->ip_summed = old->ip_summed;
2199 + skb_copy_queue_mapping(new, old);
2200 + new->priority = old->priority;
2201 +#if IS_ENABLED(CONFIG_IP_VS)
2202 + new->ipvs_property = old->ipvs_property;
2204 + new->pfmemalloc = old->pfmemalloc;
2205 + new->protocol = old->protocol;
2206 + new->mark = old->mark;
2207 + new->skb_iif = old->skb_iif;
2208 + __nf_copy(new, old);
2209 +#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE)
2210 + new->nf_trace = old->nf_trace;
2212 +#ifdef CONFIG_NET_SCHED
2213 + new->tc_index = old->tc_index;
2214 +#ifdef CONFIG_NET_CLS_ACT
2215 + new->tc_verd = old->tc_verd;
2218 + new->vlan_proto = old->vlan_proto;
2219 + new->vlan_tci = old->vlan_tci;
2221 + skb_copy_secmark(new, old);
2225 + * You should not add any new code to this function. Add it to
2226 + * __copy_skb_header above instead.
2228 +static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
2230 +#define C(x) n->x = skb->x
2232 + n->next = n->prev = NULL;
2234 + __copy_skb_header(n, skb);
2239 + n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
2242 + n->destructor = NULL;
2249 + atomic_set(&n->users, 1);
2251 + atomic_inc(&(skb_shinfo(skb)->dataref));
2259 + * skb_morph - morph one skb into another
2260 + * @dst: the skb to receive the contents
2261 + * @src: the skb to supply the contents
2263 + * This is identical to skb_clone except that the target skb is
2264 + * supplied by the user.
2266 + * The target skb is returned upon exit.
2268 +struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
2270 + skb_release_all(dst);
2271 + return __skb_clone(dst, src);
2273 +EXPORT_SYMBOL_GPL(skb_morph);
2276 + * skb_copy_ubufs - copy userspace skb frags buffers to kernel
2277 + * @skb: the skb to modify
2278 + * @gfp_mask: allocation priority
2280 + * This must be called on SKBTX_DEV_ZEROCOPY skb.
2281 + * It will copy all frags into kernel and drop the reference
2282 + * to userspace pages.
2284 + * If this function is called from an interrupt gfp_mask() must be
2287 + * Returns 0 on success or a negative error code on failure
2288 + * to allocate kernel memory to copy to.
2290 +int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
2293 + int num_frags = skb_shinfo(skb)->nr_frags;
2294 + struct page *page, *head = NULL;
2295 + struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg;
2297 + for (i = 0; i < num_frags; i++) {
2299 + skb_frag_t *f = &skb_shinfo(skb)->frags[i];
2301 + page = alloc_page(gfp_mask);
2304 + struct page *next = (struct page *)head->private;
2310 + vaddr = kmap_atomic(skb_frag_page(f));
2311 + memcpy(page_address(page),
2312 + vaddr + f->page_offset, skb_frag_size(f));
2313 + kunmap_atomic(vaddr);
2314 + page->private = (unsigned long)head;
2318 + /* skb frags release userspace buffers */
2319 + for (i = 0; i < num_frags; i++)
2320 + skb_frag_unref(skb, i);
2322 + uarg->callback(uarg, false);
2324 + /* skb frags point to kernel buffers */
2325 + for (i = num_frags - 1; i >= 0; i--) {
2326 + __skb_fill_page_desc(skb, i, head, 0,
2327 + skb_shinfo(skb)->frags[i].size);
2328 + head = (struct page *)head->private;
2331 + skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
2334 +EXPORT_SYMBOL_GPL(skb_copy_ubufs);
2337 + * skb_clone - duplicate an sk_buff
2338 + * @skb: buffer to clone
2339 + * @gfp_mask: allocation priority
2341 + * Duplicate an &sk_buff. The new one is not owned by a socket. Both
2342 + * copies share the same packet data but not structure. The new
2343 + * buffer has a reference count of 1. If the allocation fails the
2344 + * function returns %NULL otherwise the new buffer is returned.
2346 + * If this function is called from an interrupt gfp_mask() must be
2350 +struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
2352 + struct sk_buff *n;
2354 + if (skb_orphan_frags(skb, gfp_mask))
2358 + if (skb->fclone == SKB_FCLONE_ORIG &&
2359 + n->fclone == SKB_FCLONE_UNAVAILABLE) {
2360 + atomic_t *fclone_ref = (atomic_t *) (n + 1);
2361 + n->fclone = SKB_FCLONE_CLONE;
2362 + atomic_inc(fclone_ref);
2364 + if (skb_pfmemalloc(skb))
2365 + gfp_mask |= __GFP_MEMALLOC;
2367 + n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
2371 + kmemcheck_annotate_bitfield(n, flags1);
2372 + kmemcheck_annotate_bitfield(n, flags2);
2373 + n->fclone = SKB_FCLONE_UNAVAILABLE;
2376 + return __skb_clone(n, skb);
2378 +EXPORT_SYMBOL(skb_clone);
2380 +static void skb_headers_offset_update(struct sk_buff *skb, int off)
2382 + /* {transport,network,mac}_header and tail are relative to skb->head */
2383 + skb->transport_header += off;
2384 + skb->network_header += off;
2385 + if (skb_mac_header_was_set(skb))
2386 + skb->mac_header += off;
2387 + skb->inner_transport_header += off;
2388 + skb->inner_network_header += off;
2389 + skb->inner_mac_header += off;
2392 +static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
2394 +#ifndef NET_SKBUFF_DATA_USES_OFFSET
2396 + * Shift between the two data areas in bytes
2398 + unsigned long offset = new->data - old->data;
2401 + __copy_skb_header(new, old);
2403 +#ifndef NET_SKBUFF_DATA_USES_OFFSET
2404 + skb_headers_offset_update(new, offset);
2406 + skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
2407 + skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
2408 + skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
2411 +static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
2413 + if (skb_pfmemalloc(skb))
2414 + return SKB_ALLOC_RX;
2419 + * skb_copy - create private copy of an sk_buff
2420 + * @skb: buffer to copy
2421 + * @gfp_mask: allocation priority
2423 + * Make a copy of both an &sk_buff and its data. This is used when the
2424 + * caller wishes to modify the data and needs a private copy of the
2425 + * data to alter. Returns %NULL on failure or the pointer to the buffer
2426 + * on success. The returned buffer has a reference count of 1.
2428 + * As by-product this function converts non-linear &sk_buff to linear
2429 + * one, so that &sk_buff becomes completely private and caller is allowed
2430 + * to modify all the data of returned buffer. This means that this
2431 + * function is not recommended for use in circumstances when only
2432 + * header is going to be modified. Use pskb_copy() instead.
2435 +struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
2437 + int headerlen = skb_headroom(skb);
2438 + unsigned int size = skb_end_offset(skb) + skb->data_len;
2439 + struct sk_buff *n = __alloc_skb(size, gfp_mask,
2440 + skb_alloc_rx_flag(skb), NUMA_NO_NODE);
2445 + /* Set the data pointer */
2446 + skb_reserve(n, headerlen);
2447 + /* Set the tail pointer and length */
2448 + skb_put(n, skb->len);
2450 + if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
2453 + copy_skb_header(n, skb);
2456 +EXPORT_SYMBOL(skb_copy);
2459 + * __pskb_copy - create copy of an sk_buff with private head.
2460 + * @skb: buffer to copy
2461 + * @headroom: headroom of new skb
2462 + * @gfp_mask: allocation priority
2464 + * Make a copy of both an &sk_buff and part of its data, located
2465 + * in header. Fragmented data remain shared. This is used when
2466 + * the caller wishes to modify only header of &sk_buff and needs
2467 + * private copy of the header to alter. Returns %NULL on failure
2468 + * or the pointer to the buffer on success.
2469 + * The returned buffer has a reference count of 1.
2472 +struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom, gfp_t gfp_mask)
2474 + unsigned int size = skb_headlen(skb) + headroom;
2475 + struct sk_buff *n = __alloc_skb(size, gfp_mask,
2476 + skb_alloc_rx_flag(skb), NUMA_NO_NODE);
2481 + /* Set the data pointer */
2482 + skb_reserve(n, headroom);
2483 + /* Set the tail pointer and length */
2484 + skb_put(n, skb_headlen(skb));
2485 + /* Copy the bytes */
2486 + skb_copy_from_linear_data(skb, n->data, n->len);
2488 + n->truesize += skb->data_len;
2489 + n->data_len = skb->data_len;
2490 + n->len = skb->len;
2492 + if (skb_shinfo(skb)->nr_frags) {
2495 + if (skb_orphan_frags(skb, gfp_mask)) {
2500 + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2501 + skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
2502 + skb_frag_ref(skb, i);
2504 + skb_shinfo(n)->nr_frags = i;
2507 + if (skb_has_frag_list(skb)) {
2508 + skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
2509 + skb_clone_fraglist(n);
2512 + copy_skb_header(n, skb);
2516 +EXPORT_SYMBOL(__pskb_copy);
2519 + * pskb_expand_head - reallocate header of &sk_buff
2520 + * @skb: buffer to reallocate
2521 + * @nhead: room to add at head
2522 + * @ntail: room to add at tail
2523 + * @gfp_mask: allocation priority
2525 + * Expands (or creates identical copy, if &nhead and &ntail are zero)
2526 + * header of skb. &sk_buff itself is not changed. &sk_buff MUST have
2527 + * reference count of 1. Returns zero in the case of success or error,
2528 + * if expansion failed. In the last case, &sk_buff is not changed.
2530 + * All the pointers pointing into skb header may change and must be
2531 + * reloaded after call to this function.
2534 +int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
2539 + int size = nhead + skb_end_offset(skb) + ntail;
2542 + BUG_ON(nhead < 0);
2544 + if (skb_shared(skb))
2547 + size = SKB_DATA_ALIGN(size);
2549 + if (skb_pfmemalloc(skb))
2550 + gfp_mask |= __GFP_MEMALLOC;
2551 + data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
2552 + gfp_mask, NUMA_NO_NODE, NULL);
2555 + size = SKB_WITH_OVERHEAD(ksize(data));
2557 + /* Copy only real data... and, alas, header. This should be
2558 + * optimized for the cases when header is void.
2560 + memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
2562 + memcpy((struct skb_shared_info *)(data + size),
2564 + offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
2567 + * if shinfo is shared we must drop the old head gracefully, but if it
2568 + * is not we can just drop the old head and let the existing refcount
2569 + * be since all we did is relocate the values
2571 + if (skb_cloned(skb)) {
2572 + /* copy this zero copy skb frags */
2573 + if (skb_orphan_frags(skb, gfp_mask))
2575 + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2576 + skb_frag_ref(skb, i);
2578 + if (skb_has_frag_list(skb))
2579 + skb_clone_fraglist(skb);
2581 + skb_release_data(skb);
2583 + skb_free_head(skb);
2585 + off = (data + nhead) - skb->head;
2588 + skb->head_frag = 0;
2590 +#ifdef NET_SKBUFF_DATA_USES_OFFSET
2594 + skb->end = skb->head + size;
2597 + skb_headers_offset_update(skb, off);
2598 + /* Only adjust this if it actually is csum_start rather than csum */
2599 + if (skb->ip_summed == CHECKSUM_PARTIAL)
2600 + skb->csum_start += nhead;
2604 + atomic_set(&skb_shinfo(skb)->dataref, 1);
2612 +EXPORT_SYMBOL(pskb_expand_head);
2614 +/* Make private copy of skb with writable head and some headroom */
2616 +struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
2618 + struct sk_buff *skb2;
2619 + int delta = headroom - skb_headroom(skb);
2622 + skb2 = pskb_copy(skb, GFP_ATOMIC);
2624 + skb2 = skb_clone(skb, GFP_ATOMIC);
2625 + if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
2633 +EXPORT_SYMBOL(skb_realloc_headroom);
2636 + * skb_copy_expand - copy and expand sk_buff
2637 + * @skb: buffer to copy
2638 + * @newheadroom: new free bytes at head
2639 + * @newtailroom: new free bytes at tail
2640 + * @gfp_mask: allocation priority
2642 + * Make a copy of both an &sk_buff and its data and while doing so
2643 + * allocate additional space.
2645 + * This is used when the caller wishes to modify the data and needs a
2646 + * private copy of the data to alter as well as more space for new fields.
2647 + * Returns %NULL on failure or the pointer to the buffer
2648 + * on success. The returned buffer has a reference count of 1.
2650 + * You must pass %GFP_ATOMIC as the allocation priority if this function
2651 + * is called from an interrupt.
2653 +struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
2654 + int newheadroom, int newtailroom,
2658 + * Allocate the copy buffer
2660 + struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
2661 + gfp_mask, skb_alloc_rx_flag(skb),
2663 + int oldheadroom = skb_headroom(skb);
2664 + int head_copy_len, head_copy_off;
2670 + skb_reserve(n, newheadroom);
2672 + /* Set the tail pointer and length */
2673 + skb_put(n, skb->len);
2675 + head_copy_len = oldheadroom;
2676 + head_copy_off = 0;
2677 + if (newheadroom <= head_copy_len)
2678 + head_copy_len = newheadroom;
2680 + head_copy_off = newheadroom - head_copy_len;
2682 + /* Copy the linear header and data. */
2683 + if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
2684 + skb->len + head_copy_len))
2687 + copy_skb_header(n, skb);
2689 + off = newheadroom - oldheadroom;
2690 + if (n->ip_summed == CHECKSUM_PARTIAL)
2691 + n->csum_start += off;
2692 +#ifdef NET_SKBUFF_DATA_USES_OFFSET
2693 + skb_headers_offset_update(n, off);
2698 +EXPORT_SYMBOL(skb_copy_expand);
2701 + * skb_pad - zero pad the tail of an skb
2702 + * @skb: buffer to pad
2703 + * @pad: space to pad
2705 + * Ensure that a buffer is followed by a padding area that is zero
2706 + * filled. Used by network drivers which may DMA or transfer data
2707 + * beyond the buffer end onto the wire.
2709 + * May return error in out of memory cases. The skb is freed on error.
2712 +int skb_pad(struct sk_buff *skb, int pad)
2717 + /* If the skbuff is non linear tailroom is always zero.. */
2718 + if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
2719 + memset(skb->data+skb->len, 0, pad);
2723 + ntail = skb->data_len + pad - (skb->end - skb->tail);
2724 + if (likely(skb_cloned(skb) || ntail > 0)) {
2725 + err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
2726 + if (unlikely(err))
2730 + /* FIXME: The use of this function with non-linear skb's really needs
2733 + err = skb_linearize(skb);
2734 + if (unlikely(err))
2737 + memset(skb->data + skb->len, 0, pad);
2744 +EXPORT_SYMBOL(skb_pad);
2747 + * skb_put - add data to a buffer
2748 + * @skb: buffer to use
2749 + * @len: amount of data to add
2751 + * This function extends the used data area of the buffer. If this would
2752 + * exceed the total buffer size the kernel will panic. A pointer to the
2753 + * first byte of the extra data is returned.
2755 +unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
2757 + unsigned char *tmp = skb_tail_pointer(skb);
2758 + SKB_LINEAR_ASSERT(skb);
2761 + if (unlikely(skb->tail > skb->end))
2762 + skb_over_panic(skb, len, __builtin_return_address(0));
2765 +EXPORT_SYMBOL(skb_put);
2768 + * skb_push - add data to the start of a buffer
2769 + * @skb: buffer to use
2770 + * @len: amount of data to add
2772 + * This function extends the used data area of the buffer at the buffer
2773 + * start. If this would exceed the total buffer headroom the kernel will
2774 + * panic. A pointer to the first byte of the extra data is returned.
2776 +unsigned char *skb_push(struct sk_buff *skb, unsigned int len)
2780 + if (unlikely(skb->data<skb->head))
2781 + skb_under_panic(skb, len, __builtin_return_address(0));
2784 +EXPORT_SYMBOL(skb_push);
2787 + * skb_pull - remove data from the start of a buffer
2788 + * @skb: buffer to use
2789 + * @len: amount of data to remove
2791 + * This function removes data from the start of a buffer, returning
2792 + * the memory to the headroom. A pointer to the next data in the buffer
2793 + * is returned. Once the data has been pulled future pushes will overwrite
2796 +unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
2798 + return skb_pull_inline(skb, len);
2800 +EXPORT_SYMBOL(skb_pull);
2803 + * skb_trim - remove end from a buffer
2804 + * @skb: buffer to alter
2805 + * @len: new length
2807 + * Cut the length of a buffer down by removing data from the tail. If
2808 + * the buffer is already under the length specified it is not modified.
2809 + * The skb must be linear.
2811 +void skb_trim(struct sk_buff *skb, unsigned int len)
2813 + if (skb->len > len)
2814 + __skb_trim(skb, len);
2816 +EXPORT_SYMBOL(skb_trim);
2818 +/* Trims skb to length len. It can change skb pointers.
2821 +int ___pskb_trim(struct sk_buff *skb, unsigned int len)
2823 + struct sk_buff **fragp;
2824 + struct sk_buff *frag;
2825 + int offset = skb_headlen(skb);
2826 + int nfrags = skb_shinfo(skb)->nr_frags;
2830 + if (skb_cloned(skb) &&
2831 + unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
2835 + if (offset >= len)
2838 + for (; i < nfrags; i++) {
2839 + int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2846 + skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
2849 + skb_shinfo(skb)->nr_frags = i;
2851 + for (; i < nfrags; i++)
2852 + skb_frag_unref(skb, i);
2854 + if (skb_has_frag_list(skb))
2855 + skb_drop_fraglist(skb);
2859 + for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
2860 + fragp = &frag->next) {
2861 + int end = offset + frag->len;
2863 + if (skb_shared(frag)) {
2864 + struct sk_buff *nfrag;
2866 + nfrag = skb_clone(frag, GFP_ATOMIC);
2867 + if (unlikely(!nfrag))
2870 + nfrag->next = frag->next;
2871 + consume_skb(frag);
2882 + unlikely((err = pskb_trim(frag, len - offset))))
2886 + skb_drop_list(&frag->next);
2891 + if (len > skb_headlen(skb)) {
2892 + skb->data_len -= skb->len - len;
2896 + skb->data_len = 0;
2897 + skb_set_tail_pointer(skb, len);
2902 +EXPORT_SYMBOL(___pskb_trim);
2905 + * __pskb_pull_tail - advance tail of skb header
2906 + * @skb: buffer to reallocate
2907 + * @delta: number of bytes to advance tail
2909 + * The function makes a sense only on a fragmented &sk_buff,
2910 + * it expands header moving its tail forward and copying necessary
2911 + * data from fragmented part.
2913 + * &sk_buff MUST have reference count of 1.
2915 + * Returns %NULL (and &sk_buff does not change) if pull failed
2916 + * or value of new tail of skb in the case of success.
2918 + * All the pointers pointing into skb header may change and must be
2919 + * reloaded after call to this function.
2922 +/* Moves tail of skb head forward, copying data from fragmented part,
2923 + * when it is necessary.
2924 + * 1. It may fail due to malloc failure.
2925 + * 2. It may change skb pointers.
2927 + * It is pretty complicated. Luckily, it is called only in exceptional cases.
2929 +unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
2931 + /* If skb has not enough free space at tail, get new one
2932 + * plus 128 bytes for future expansions. If we have enough
2933 + * room at tail, reallocate without expansion only if skb is cloned.
2935 + int i, k, eat = (skb->tail + delta) - skb->end;
2937 + if (eat > 0 || skb_cloned(skb)) {
2938 + if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
2943 + if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta))
2946 + /* Optimization: no fragments, no reasons to preestimate
2947 + * size of pulled pages. Superb.
2949 + if (!skb_has_frag_list(skb))
2952 + /* Estimate size of pulled pages. */
2954 + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2955 + int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2962 + /* If we need update frag list, we are in troubles.
2963 + * Certainly, it possible to add an offset to skb data,
2964 + * but taking into account that pulling is expected to
2965 + * be very rare operation, it is worth to fight against
2966 + * further bloating skb head and crucify ourselves here instead.
2967 + * Pure masohism, indeed. 8)8)
2970 + struct sk_buff *list = skb_shinfo(skb)->frag_list;
2971 + struct sk_buff *clone = NULL;
2972 + struct sk_buff *insp = NULL;
2977 + if (list->len <= eat) {
2978 + /* Eaten as whole. */
2980 + list = list->next;
2983 + /* Eaten partially. */
2985 + if (skb_shared(list)) {
2986 + /* Sucks! We need to fork list. :-( */
2987 + clone = skb_clone(list, GFP_ATOMIC);
2990 + insp = list->next;
2993 + /* This may be pulled without
2997 + if (!pskb_pull(list, eat)) {
3005 + /* Free pulled out fragments. */
3006 + while ((list = skb_shinfo(skb)->frag_list) != insp) {
3007 + skb_shinfo(skb)->frag_list = list->next;
3010 + /* And insert new clone at head. */
3012 + clone->next = list;
3013 + skb_shinfo(skb)->frag_list = clone;
3016 + /* Success! Now we may commit changes to skb data. */
3021 + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3022 + int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
3024 + if (size <= eat) {
3025 + skb_frag_unref(skb, i);
3028 + skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
3030 + skb_shinfo(skb)->frags[k].page_offset += eat;
3031 + skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
3037 + skb_shinfo(skb)->nr_frags = k;
3039 + skb->tail += delta;
3040 + skb->data_len -= delta;
3042 + return skb_tail_pointer(skb);
3044 +EXPORT_SYMBOL(__pskb_pull_tail);
3047 + * skb_copy_bits - copy bits from skb to kernel buffer
3048 + * @skb: source skb
3049 + * @offset: offset in source
3050 + * @to: destination buffer
3051 + * @len: number of bytes to copy
3053 + * Copy the specified number of bytes from the source skb to the
3054 + * destination buffer.
3057 + * If its prototype is ever changed,
3058 + * check arch/{*}/net/{*}.S files,
3059 + * since it is called from BPF assembly code.
3061 +int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
3063 + int start = skb_headlen(skb);
3064 + struct sk_buff *frag_iter;
3067 + if (offset > (int)skb->len - len)
3070 + /* Copy header. */
3071 + if ((copy = start - offset) > 0) {
3074 + skb_copy_from_linear_data_offset(skb, offset, to, copy);
3075 + if ((len -= copy) == 0)
3081 + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3083 + skb_frag_t *f = &skb_shinfo(skb)->frags[i];
3085 + WARN_ON(start > offset + len);
3087 + end = start + skb_frag_size(f);
3088 + if ((copy = end - offset) > 0) {
3094 + vaddr = kmap_atomic(skb_frag_page(f));
3096 + vaddr + f->page_offset + offset - start,
3098 + kunmap_atomic(vaddr);
3100 + if ((len -= copy) == 0)
3108 + skb_walk_frags(skb, frag_iter) {
3111 + WARN_ON(start > offset + len);
3113 + end = start + frag_iter->len;
3114 + if ((copy = end - offset) > 0) {
3117 + if (skb_copy_bits(frag_iter, offset - start, to, copy))
3119 + if ((len -= copy) == 0)
3133 +EXPORT_SYMBOL(skb_copy_bits);
3136 + * Callback from splice_to_pipe(), if we need to release some pages
3137 + * at the end of the spd in case we error'ed out in filling the pipe.
3139 +static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3141 + put_page(spd->pages[i]);
3144 +static struct page *linear_to_page(struct page *page, unsigned int *len,
3145 + unsigned int *offset,
3148 + struct page_frag *pfrag = sk_page_frag(sk);
3150 + if (!sk_page_frag_refill(sk, pfrag))
3153 + *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
3155 + memcpy(page_address(pfrag->page) + pfrag->offset,
3156 + page_address(page) + *offset, *len);
3157 + *offset = pfrag->offset;
3158 + pfrag->offset += *len;
3160 + return pfrag->page;
3163 +static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
3164 + struct page *page,
3165 + unsigned int offset)
3167 + return spd->nr_pages &&
3168 + spd->pages[spd->nr_pages - 1] == page &&
3169 + (spd->partial[spd->nr_pages - 1].offset +
3170 + spd->partial[spd->nr_pages - 1].len == offset);
3174 + * Fill page/offset/length into spd, if it can hold more pages.
3176 +static bool spd_fill_page(struct splice_pipe_desc *spd,
3177 + struct pipe_inode_info *pipe, struct page *page,
3178 + unsigned int *len, unsigned int offset,
3182 + if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
3186 + page = linear_to_page(page, len, &offset, sk);
3190 + if (spd_can_coalesce(spd, page, offset)) {
3191 + spd->partial[spd->nr_pages - 1].len += *len;
3195 + spd->pages[spd->nr_pages] = page;
3196 + spd->partial[spd->nr_pages].len = *len;
3197 + spd->partial[spd->nr_pages].offset = offset;
3203 +static bool __splice_segment(struct page *page, unsigned int poff,
3204 + unsigned int plen, unsigned int *off,
3205 + unsigned int *len,
3206 + struct splice_pipe_desc *spd, bool linear,
3208 + struct pipe_inode_info *pipe)
3213 + /* skip this segment if already processed */
3214 + if (*off >= plen) {
3219 + /* ignore any bits we already processed */
3225 + unsigned int flen = min(*len, plen);
3227 + if (spd_fill_page(spd, pipe, page, &flen, poff,
3233 + } while (*len && plen);
3239 + * Map linear and fragment data from the skb to spd. It reports true if the
3240 + * pipe is full or if we already spliced the requested length.
3242 +static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
3243 + unsigned int *offset, unsigned int *len,
3244 + struct splice_pipe_desc *spd, struct sock *sk)
3248 + /* map the linear part :
3249 + * If skb->head_frag is set, this 'linear' part is backed by a
3250 + * fragment, and if the head is not shared with any clones then
3251 + * we can avoid a copy since we own the head portion of this page.
3253 + if (__splice_segment(virt_to_page(skb->data),
3254 + (unsigned long) skb->data & (PAGE_SIZE - 1),
3257 + skb_head_is_locked(skb),
3262 + * then map the fragments
3264 + for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
3265 + const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
3267 + if (__splice_segment(skb_frag_page(f),
3268 + f->page_offset, skb_frag_size(f),
3269 + offset, len, spd, false, sk, pipe))
3277 + * Map data from the skb to a pipe. Should handle both the linear part,
3278 + * the fragments, and the frag list. It does NOT handle frag lists within
3279 + * the frag list, if such a thing exists. We'd probably need to recurse to
3280 + * handle that cleanly.
3282 +int skb_splice_bits(struct sk_buff *skb, unsigned int offset,
3283 + struct pipe_inode_info *pipe, unsigned int tlen,
3284 + unsigned int flags)
3286 + struct partial_page partial[MAX_SKB_FRAGS];
3287 + struct page *pages[MAX_SKB_FRAGS];
3288 + struct splice_pipe_desc spd = {
3290 + .partial = partial,
3291 + .nr_pages_max = MAX_SKB_FRAGS,
3293 + .ops = &sock_pipe_buf_ops,
3294 + .spd_release = sock_spd_release,
3296 + struct sk_buff *frag_iter;
3297 + struct sock *sk = skb->sk;
3301 + * __skb_splice_bits() only fails if the output has no room left,
3302 + * so no point in going over the frag_list for the error case.
3304 + if (__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk))
3310 + * now see if we have a frag_list to map
3312 + skb_walk_frags(skb, frag_iter) {
3315 + if (__skb_splice_bits(frag_iter, pipe, &offset, &tlen, &spd, sk))
3320 + if (spd.nr_pages) {
3322 + * Drop the socket lock, otherwise we have reverse
3323 + * locking dependencies between sk_lock and i_mutex
3324 + * here as compared to sendfile(). We enter here
3325 + * with the socket lock held, and splice_to_pipe() will
3326 + * grab the pipe inode lock. For sendfile() emulation,
3327 + * we call into ->sendpage() with the i_mutex lock held
3328 + * and networking will grab the socket lock.
3331 + ret = splice_to_pipe(pipe, &spd);
3339 + * skb_store_bits - store bits from kernel buffer to skb
3340 + * @skb: destination buffer
3341 + * @offset: offset in destination
3342 + * @from: source buffer
3343 + * @len: number of bytes to copy
3345 + * Copy the specified number of bytes from the source buffer to the
3346 + * destination skb. This function handles all the messy bits of
3347 + * traversing fragment lists and such.
3350 +int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
3352 + int start = skb_headlen(skb);
3353 + struct sk_buff *frag_iter;
3356 + if (offset > (int)skb->len - len)
3359 + if ((copy = start - offset) > 0) {
3362 + skb_copy_to_linear_data_offset(skb, offset, from, copy);
3363 + if ((len -= copy) == 0)
3369 + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3370 + skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3373 + WARN_ON(start > offset + len);
3375 + end = start + skb_frag_size(frag);
3376 + if ((copy = end - offset) > 0) {
3382 + vaddr = kmap_atomic(skb_frag_page(frag));
3383 + memcpy(vaddr + frag->page_offset + offset - start,
3385 + kunmap_atomic(vaddr);
3387 + if ((len -= copy) == 0)
3395 + skb_walk_frags(skb, frag_iter) {
3398 + WARN_ON(start > offset + len);
3400 + end = start + frag_iter->len;
3401 + if ((copy = end - offset) > 0) {
3404 + if (skb_store_bits(frag_iter, offset - start,
3407 + if ((len -= copy) == 0)
3420 +EXPORT_SYMBOL(skb_store_bits);
3422 +/* Checksum skb data. */
3424 +__wsum skb_checksum(const struct sk_buff *skb, int offset,
3425 + int len, __wsum csum)
3427 + int start = skb_headlen(skb);
3428 + int i, copy = start - offset;
3429 + struct sk_buff *frag_iter;
3432 + /* Checksum header. */
3436 + csum = csum_partial(skb->data + offset, copy, csum);
3437 + if ((len -= copy) == 0)
3443 + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3445 + skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3447 + WARN_ON(start > offset + len);
3449 + end = start + skb_frag_size(frag);
3450 + if ((copy = end - offset) > 0) {
3456 + vaddr = kmap_atomic(skb_frag_page(frag));
3457 + csum2 = csum_partial(vaddr + frag->page_offset +
3458 + offset - start, copy, 0);
3459 + kunmap_atomic(vaddr);
3460 + csum = csum_block_add(csum, csum2, pos);
3461 + if (!(len -= copy))
3469 + skb_walk_frags(skb, frag_iter) {
3472 + WARN_ON(start > offset + len);
3474 + end = start + frag_iter->len;
3475 + if ((copy = end - offset) > 0) {
3479 + csum2 = skb_checksum(frag_iter, offset - start,
3481 + csum = csum_block_add(csum, csum2, pos);
3482 + if ((len -= copy) == 0)
3493 +EXPORT_SYMBOL(skb_checksum);
3495 +/* Both of above in one bottle. */
3497 +__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
3498 + u8 *to, int len, __wsum csum)
3500 + int start = skb_headlen(skb);
3501 + int i, copy = start - offset;
3502 + struct sk_buff *frag_iter;
3505 + /* Copy header. */
3509 + csum = csum_partial_copy_nocheck(skb->data + offset, to,
3511 + if ((len -= copy) == 0)
3518 + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3521 + WARN_ON(start > offset + len);
3523 + end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3524 + if ((copy = end - offset) > 0) {
3527 + skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3531 + vaddr = kmap_atomic(skb_frag_page(frag));
3532 + csum2 = csum_partial_copy_nocheck(vaddr +
3533 + frag->page_offset +
3534 + offset - start, to,
3536 + kunmap_atomic(vaddr);
3537 + csum = csum_block_add(csum, csum2, pos);
3538 + if (!(len -= copy))
3547 + skb_walk_frags(skb, frag_iter) {
3551 + WARN_ON(start > offset + len);
3553 + end = start + frag_iter->len;
3554 + if ((copy = end - offset) > 0) {
3557 + csum2 = skb_copy_and_csum_bits(frag_iter,
3560 + csum = csum_block_add(csum, csum2, pos);
3561 + if ((len -= copy) == 0)
3572 +EXPORT_SYMBOL(skb_copy_and_csum_bits);
3574 +void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
3579 + if (skb->ip_summed == CHECKSUM_PARTIAL)
3580 + csstart = skb_checksum_start_offset(skb);
3582 + csstart = skb_headlen(skb);
3584 + BUG_ON(csstart > skb_headlen(skb));
3586 + skb_copy_from_linear_data(skb, to, csstart);
3589 + if (csstart != skb->len)
3590 + csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
3591 + skb->len - csstart, 0);
3593 + if (skb->ip_summed == CHECKSUM_PARTIAL) {
3594 + long csstuff = csstart + skb->csum_offset;
3596 + *((__sum16 *)(to + csstuff)) = csum_fold(csum);
3599 +EXPORT_SYMBOL(skb_copy_and_csum_dev);
3602 + * skb_dequeue - remove from the head of the queue
3603 + * @list: list to dequeue from
3605 + * Remove the head of the list. The list lock is taken so the function
3606 + * may be used safely with other locking list functions. The head item is
3607 + * returned or %NULL if the list is empty.
3610 +struct sk_buff *skb_dequeue(struct sk_buff_head *list)
3612 + unsigned long flags;
3613 + struct sk_buff *result;
3615 + spin_lock_irqsave(&list->lock, flags);
3616 + result = __skb_dequeue(list);
3617 + spin_unlock_irqrestore(&list->lock, flags);
3620 +EXPORT_SYMBOL(skb_dequeue);
3623 + * skb_dequeue_tail - remove from the tail of the queue
3624 + * @list: list to dequeue from
3626 + * Remove the tail of the list. The list lock is taken so the function
3627 + * may be used safely with other locking list functions. The tail item is
3628 + * returned or %NULL if the list is empty.
3630 +struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
3632 + unsigned long flags;
3633 + struct sk_buff *result;
3635 + spin_lock_irqsave(&list->lock, flags);
3636 + result = __skb_dequeue_tail(list);
3637 + spin_unlock_irqrestore(&list->lock, flags);
3640 +EXPORT_SYMBOL(skb_dequeue_tail);
3643 + * skb_queue_purge - empty a list
3644 + * @list: list to empty
3646 + * Delete all buffers on an &sk_buff list. Each buffer is removed from
3647 + * the list and one reference dropped. This function takes the list
3648 + * lock and is atomic with respect to other list locking functions.
3650 +void skb_queue_purge(struct sk_buff_head *list)
3652 + struct sk_buff *skb;
3653 + while ((skb = skb_dequeue(list)) != NULL)
3656 +EXPORT_SYMBOL(skb_queue_purge);
3659 + * skb_queue_head - queue a buffer at the list head
3660 + * @list: list to use
3661 + * @newsk: buffer to queue
3663 + * Queue a buffer at the start of the list. This function takes the
3664 + * list lock and can be used safely with other locking &sk_buff functions
3667 + * A buffer cannot be placed on two lists at the same time.
3669 +void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
3671 + unsigned long flags;
3673 + spin_lock_irqsave(&list->lock, flags);
3674 + __skb_queue_head(list, newsk);
3675 + spin_unlock_irqrestore(&list->lock, flags);
3677 +EXPORT_SYMBOL(skb_queue_head);
3680 + * skb_queue_tail - queue a buffer at the list tail
3681 + * @list: list to use
3682 + * @newsk: buffer to queue
3684 + * Queue a buffer at the tail of the list. This function takes the
3685 + * list lock and can be used safely with other locking &sk_buff functions
3688 + * A buffer cannot be placed on two lists at the same time.
3690 +void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
3692 + unsigned long flags;
3694 + spin_lock_irqsave(&list->lock, flags);
3695 + __skb_queue_tail(list, newsk);
3696 + spin_unlock_irqrestore(&list->lock, flags);
3698 +EXPORT_SYMBOL(skb_queue_tail);
3701 + * skb_unlink - remove a buffer from a list
3702 + * @skb: buffer to remove
3703 + * @list: list to use
3705 + * Remove a packet from a list. The list locks are taken and this
3706 + * function is atomic with respect to other list locked calls
3708 + * You must know what list the SKB is on.
3710 +void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
3712 + unsigned long flags;
3714 + spin_lock_irqsave(&list->lock, flags);
3715 + __skb_unlink(skb, list);
3716 + spin_unlock_irqrestore(&list->lock, flags);
3718 +EXPORT_SYMBOL(skb_unlink);
3721 + * skb_append - append a buffer
3722 + * @old: buffer to insert after
3723 + * @newsk: buffer to insert
3724 + * @list: list to use
3726 + * Place a packet after a given packet in a list. The list locks are taken
3727 + * and this function is atomic with respect to other list locked calls.
3728 + * A buffer cannot be placed on two lists at the same time.
3730 +void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
3732 + unsigned long flags;
3734 + spin_lock_irqsave(&list->lock, flags);
3735 + __skb_queue_after(list, old, newsk);
3736 + spin_unlock_irqrestore(&list->lock, flags);
3738 +EXPORT_SYMBOL(skb_append);
3741 + * skb_insert - insert a buffer
3742 + * @old: buffer to insert before
3743 + * @newsk: buffer to insert
3744 + * @list: list to use
3746 + * Place a packet before a given packet in a list. The list locks are
3747 + * taken and this function is atomic with respect to other list locked
3750 + * A buffer cannot be placed on two lists at the same time.
3752 +void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
3754 + unsigned long flags;
3756 + spin_lock_irqsave(&list->lock, flags);
3757 + __skb_insert(newsk, old->prev, old, list);
3758 + spin_unlock_irqrestore(&list->lock, flags);
3760 +EXPORT_SYMBOL(skb_insert);
3762 +static inline void skb_split_inside_header(struct sk_buff *skb,
3763 + struct sk_buff* skb1,
3764 + const u32 len, const int pos)
3768 + skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
3770 + /* And move data appendix as is. */
3771 + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
3772 + skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
3774 + skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
3775 + skb_shinfo(skb)->nr_frags = 0;
3776 + skb1->data_len = skb->data_len;
3777 + skb1->len += skb1->data_len;
3778 + skb->data_len = 0;
3780 + skb_set_tail_pointer(skb, len);
3783 +static inline void skb_split_no_header(struct sk_buff *skb,
3784 + struct sk_buff* skb1,
3785 + const u32 len, int pos)
3788 + const int nfrags = skb_shinfo(skb)->nr_frags;
3790 + skb_shinfo(skb)->nr_frags = 0;
3791 + skb1->len = skb1->data_len = skb->len - len;
3793 + skb->data_len = len - pos;
3795 + for (i = 0; i < nfrags; i++) {
3796 + int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
3798 + if (pos + size > len) {
3799 + skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
3803 + * We have two variants in this case:
3804 + * 1. Move all the frag to the second
3805 + * part, if it is possible. F.e.
3806 + * this approach is mandatory for TUX,
3807 + * where splitting is expensive.
3808 + * 2. Split is accurately. We make this.
3810 + skb_frag_ref(skb, i);
3811 + skb_shinfo(skb1)->frags[0].page_offset += len - pos;
3812 + skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
3813 + skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
3814 + skb_shinfo(skb)->nr_frags++;
3818 + skb_shinfo(skb)->nr_frags++;
3821 + skb_shinfo(skb1)->nr_frags = k;
3825 + * skb_split - Split fragmented skb to two parts at length len.
3826 + * @skb: the buffer to split
3827 + * @skb1: the buffer to receive the second part
3828 + * @len: new length for skb
3830 +void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
3832 + int pos = skb_headlen(skb);
3834 + skb_shinfo(skb1)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
3835 + if (len < pos) /* Split line is inside header. */
3836 + skb_split_inside_header(skb, skb1, len, pos);
3837 + else /* Second chunk has no header, nothing to copy. */
3838 + skb_split_no_header(skb, skb1, len, pos);
3840 +EXPORT_SYMBOL(skb_split);
3842 +/* Shifting from/to a cloned skb is a no-go.
3844 + * Caller cannot keep skb_shinfo related pointers past calling here!
3846 +static int skb_prepare_for_shift(struct sk_buff *skb)
3848 + return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3852 + * skb_shift - Shifts paged data partially from skb to another
3853 + * @tgt: buffer into which tail data gets added
3854 + * @skb: buffer from which the paged data comes from
3855 + * @shiftlen: shift up to this many bytes
3857 + * Attempts to shift up to shiftlen worth of bytes, which may be less than
3858 + * the length of the skb, from skb to tgt. Returns number bytes shifted.
3859 + * It's up to caller to free skb if everything was shifted.
3861 + * If @tgt runs out of frags, the whole operation is aborted.
3863 + * Skb cannot include anything else but paged data while tgt is allowed
3864 + * to have non-paged data as well.
3866 + * TODO: full sized shift could be optimized but that would need
3867 + * specialized skb free'er to handle frags without up-to-date nr_frags.
3869 +int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
3871 + int from, to, merge, todo;
3872 + struct skb_frag_struct *fragfrom, *fragto;
3874 + BUG_ON(shiftlen > skb->len);
3875 + BUG_ON(skb_headlen(skb)); /* Would corrupt stream */
3879 + to = skb_shinfo(tgt)->nr_frags;
3880 + fragfrom = &skb_shinfo(skb)->frags[from];
3882 + /* Actual merge is delayed until the point when we know we can
3883 + * commit all, so that we don't have to undo partial changes
3886 + !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
3887 + fragfrom->page_offset)) {
3892 + todo -= skb_frag_size(fragfrom);
3894 + if (skb_prepare_for_shift(skb) ||
3895 + skb_prepare_for_shift(tgt))
3898 + /* All previous frag pointers might be stale! */
3899 + fragfrom = &skb_shinfo(skb)->frags[from];
3900 + fragto = &skb_shinfo(tgt)->frags[merge];
3902 + skb_frag_size_add(fragto, shiftlen);
3903 + skb_frag_size_sub(fragfrom, shiftlen);
3904 + fragfrom->page_offset += shiftlen;
3912 + /* Skip full, not-fitting skb to avoid expensive operations */
3913 + if ((shiftlen == skb->len) &&
3914 + (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
3917 + if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
3920 + while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
3921 + if (to == MAX_SKB_FRAGS)
3924 + fragfrom = &skb_shinfo(skb)->frags[from];
3925 + fragto = &skb_shinfo(tgt)->frags[to];
3927 + if (todo >= skb_frag_size(fragfrom)) {
3928 + *fragto = *fragfrom;
3929 + todo -= skb_frag_size(fragfrom);
3934 + __skb_frag_ref(fragfrom);
3935 + fragto->page = fragfrom->page;
3936 + fragto->page_offset = fragfrom->page_offset;
3937 + skb_frag_size_set(fragto, todo);
3939 + fragfrom->page_offset += todo;
3940 + skb_frag_size_sub(fragfrom, todo);
3948 + /* Ready to "commit" this state change to tgt */
3949 + skb_shinfo(tgt)->nr_frags = to;
3952 + fragfrom = &skb_shinfo(skb)->frags[0];
3953 + fragto = &skb_shinfo(tgt)->frags[merge];
3955 + skb_frag_size_add(fragto, skb_frag_size(fragfrom));
3956 + __skb_frag_unref(fragfrom);
3959 + /* Reposition in the original skb */
3961 + while (from < skb_shinfo(skb)->nr_frags)
3962 + skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
3963 + skb_shinfo(skb)->nr_frags = to;
3965 + BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
3968 + /* Most likely the tgt won't ever need its checksum anymore, skb on
3969 + * the other hand might need it if it needs to be resent
3971 + tgt->ip_summed = CHECKSUM_PARTIAL;
3972 + skb->ip_summed = CHECKSUM_PARTIAL;
3974 + /* Yak, is it really working this way? Some helper please? */
3975 + skb->len -= shiftlen;
3976 + skb->data_len -= shiftlen;
3977 + skb->truesize -= shiftlen;
3978 + tgt->len += shiftlen;
3979 + tgt->data_len += shiftlen;
3980 + tgt->truesize += shiftlen;
3986 + * skb_prepare_seq_read - Prepare a sequential read of skb data
3987 + * @skb: the buffer to read
3988 + * @from: lower offset of data to be read
3989 + * @to: upper offset of data to be read
3990 + * @st: state variable
3992 + * Initializes the specified state variable. Must be called before
3993 + * invoking skb_seq_read() for the first time.
3995 +void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
3996 + unsigned int to, struct skb_seq_state *st)
3998 + st->lower_offset = from;
3999 + st->upper_offset = to;
4000 + st->root_skb = st->cur_skb = skb;
4001 + st->frag_idx = st->stepped_offset = 0;
4002 + st->frag_data = NULL;
4004 +EXPORT_SYMBOL(skb_prepare_seq_read);
4007 + * skb_seq_read - Sequentially read skb data
4008 + * @consumed: number of bytes consumed by the caller so far
4009 + * @data: destination pointer for data to be returned
4010 + * @st: state variable
4012 + * Reads a block of skb data at &consumed relative to the
4013 + * lower offset specified to skb_prepare_seq_read(). Assigns
4014 + * the head of the data block to &data and returns the length
4015 + * of the block or 0 if the end of the skb data or the upper
4016 + * offset has been reached.
4018 + * The caller is not required to consume all of the data
4019 + * returned, i.e. &consumed is typically set to the number
4020 + * of bytes already consumed and the next call to
4021 + * skb_seq_read() will return the remaining part of the block.
4023 + * Note 1: The size of each block of data returned can be arbitrary,
4024 + * this limitation is the cost for zerocopy seqeuental
4025 + * reads of potentially non linear data.
4027 + * Note 2: Fragment lists within fragments are not implemented
4028 + * at the moment, state->root_skb could be replaced with
4029 + * a stack for this purpose.
4031 +unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
4032 + struct skb_seq_state *st)
4034 + unsigned int block_limit, abs_offset = consumed + st->lower_offset;
4037 + if (unlikely(abs_offset >= st->upper_offset))
4041 + block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
4043 + if (abs_offset < block_limit && !st->frag_data) {
4044 + *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
4045 + return block_limit - abs_offset;
4048 + if (st->frag_idx == 0 && !st->frag_data)
4049 + st->stepped_offset += skb_headlen(st->cur_skb);
4051 + while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
4052 + frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
4053 + block_limit = skb_frag_size(frag) + st->stepped_offset;
4055 + if (abs_offset < block_limit) {
4056 + if (!st->frag_data)
4057 + st->frag_data = kmap_atomic(skb_frag_page(frag));
4059 + *data = (u8 *) st->frag_data + frag->page_offset +
4060 + (abs_offset - st->stepped_offset);
4062 + return block_limit - abs_offset;
4065 + if (st->frag_data) {
4066 + kunmap_atomic(st->frag_data);
4067 + st->frag_data = NULL;
4071 + st->stepped_offset += skb_frag_size(frag);
4074 + if (st->frag_data) {
4075 + kunmap_atomic(st->frag_data);
4076 + st->frag_data = NULL;
4079 + if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
4080 + st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
4083 + } else if (st->cur_skb->next) {
4084 + st->cur_skb = st->cur_skb->next;
4091 +EXPORT_SYMBOL(skb_seq_read);
4094 + * skb_abort_seq_read - Abort a sequential read of skb data
4095 + * @st: state variable
4097 + * Must be called if skb_seq_read() was not called until it
4100 +void skb_abort_seq_read(struct skb_seq_state *st)
4102 + if (st->frag_data)
4103 + kunmap_atomic(st->frag_data);
4105 +EXPORT_SYMBOL(skb_abort_seq_read);
4107 +#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
4109 +static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
4110 + struct ts_config *conf,
4111 + struct ts_state *state)
4113 + return skb_seq_read(offset, text, TS_SKB_CB(state));
4116 +static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
4118 + skb_abort_seq_read(TS_SKB_CB(state));
4122 + * skb_find_text - Find a text pattern in skb data
4123 + * @skb: the buffer to look in
4124 + * @from: search offset
4125 + * @to: search limit
4126 + * @config: textsearch configuration
4127 + * @state: uninitialized textsearch state variable
4129 + * Finds a pattern in the skb data according to the specified
4130 + * textsearch configuration. Use textsearch_next() to retrieve
4131 + * subsequent occurrences of the pattern. Returns the offset
4132 + * to the first occurrence or UINT_MAX if no match was found.
4134 +unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
4135 + unsigned int to, struct ts_config *config,
4136 + struct ts_state *state)
4140 + config->get_next_block = skb_ts_get_next_block;
4141 + config->finish = skb_ts_finish;
4143 + skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
4145 + ret = textsearch_find(config, state);
4146 + return (ret <= to - from ? ret : UINT_MAX);
4148 +EXPORT_SYMBOL(skb_find_text);
4151 + * skb_append_datato_frags - append the user data to a skb
4152 + * @sk: sock structure
4153 + * @skb: skb structure to be appened with user data.
4154 + * @getfrag: call back function to be used for getting the user data
4155 + * @from: pointer to user message iov
4156 + * @length: length of the iov message
4158 + * Description: This procedure append the user data in the fragment part
4159 + * of the skb if any page alloc fails user this procedure returns -ENOMEM
4161 +int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
4162 + int (*getfrag)(void *from, char *to, int offset,
4163 + int len, int odd, struct sk_buff *skb),
4164 + void *from, int length)
4166 + int frg_cnt = skb_shinfo(skb)->nr_frags;
4170 + struct page_frag *pfrag = ¤t->task_frag;
4173 + /* Return error if we don't have space for new frag */
4174 + if (frg_cnt >= MAX_SKB_FRAGS)
4177 + if (!sk_page_frag_refill(sk, pfrag))
4180 + /* copy the user data to page */
4181 + copy = min_t(int, length, pfrag->size - pfrag->offset);
4183 + ret = getfrag(from, page_address(pfrag->page) + pfrag->offset,
4184 + offset, copy, 0, skb);
4188 + /* copy was successful so update the size parameters */
4189 + skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset,
4192 + pfrag->offset += copy;
4193 + get_page(pfrag->page);
4195 + skb->truesize += copy;
4196 + atomic_add(copy, &sk->sk_wmem_alloc);
4198 + skb->data_len += copy;
4202 + } while (length > 0);
4206 +EXPORT_SYMBOL(skb_append_datato_frags);
4209 + * skb_pull_rcsum - pull skb and update receive checksum
4210 + * @skb: buffer to update
4211 + * @len: length of data pulled
4213 + * This function performs an skb_pull on the packet and updates
4214 + * the CHECKSUM_COMPLETE checksum. It should be used on
4215 + * receive path processing instead of skb_pull unless you know
4216 + * that the checksum difference is zero (e.g., a valid IP header)
4217 + * or you are setting ip_summed to CHECKSUM_NONE.
4219 +unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
4221 + BUG_ON(len > skb->len);
4223 + BUG_ON(skb->len < skb->data_len);
4224 + skb_postpull_rcsum(skb, skb->data, len);
4225 + return skb->data += len;
4227 +EXPORT_SYMBOL_GPL(skb_pull_rcsum);
4230 + * skb_segment - Perform protocol segmentation on skb.
4231 + * @skb: buffer to segment
4232 + * @features: features for the output path (see dev->features)
4234 + * This function performs segmentation on the given skb. It returns
4235 + * a pointer to the first in a list of new skbs for the segments.
4236 + * In case of error it returns ERR_PTR(err).
4238 +struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features)
4240 + struct sk_buff *segs = NULL;
4241 + struct sk_buff *tail = NULL;
4242 + struct sk_buff *fskb = skb_shinfo(skb)->frag_list;
4243 + unsigned int mss = skb_shinfo(skb)->gso_size;
4244 + unsigned int doffset = skb->data - skb_mac_header(skb);
4245 + unsigned int offset = doffset;
4246 + unsigned int tnl_hlen = skb_tnl_header_len(skb);
4247 + unsigned int headroom;
4251 + int sg = !!(features & NETIF_F_SG);
4252 + int nfrags = skb_shinfo(skb)->nr_frags;
4253 + int err = -ENOMEM;
4257 + proto = skb_network_protocol(skb);
4258 + if (unlikely(!proto))
4259 + return ERR_PTR(-EINVAL);
4261 + csum = !!can_checksum_protocol(features, proto);
4262 + __skb_push(skb, doffset);
4263 + headroom = skb_headroom(skb);
4264 + pos = skb_headlen(skb);
4267 + struct sk_buff *nskb;
4272 + len = skb->len - offset;
4276 + hsize = skb_headlen(skb) - offset;
4279 + if (hsize > len || !sg)
4282 + if (!hsize && i >= nfrags) {
4283 + BUG_ON(fskb->len != len);
4286 + nskb = skb_clone(fskb, GFP_ATOMIC);
4287 + fskb = fskb->next;
4289 + if (unlikely(!nskb))
4292 + hsize = skb_end_offset(nskb);
4293 + if (skb_cow_head(nskb, doffset + headroom)) {
4298 + nskb->truesize += skb_end_offset(nskb) - hsize;
4299 + skb_release_head_state(nskb);
4300 + __skb_push(nskb, doffset);
4302 + nskb = __alloc_skb(hsize + doffset + headroom,
4303 + GFP_ATOMIC, skb_alloc_rx_flag(skb),
4306 + if (unlikely(!nskb))
4309 + skb_reserve(nskb, headroom);
4310 + __skb_put(nskb, doffset);
4314 + tail->next = nskb;
4319 + __copy_skb_header(nskb, skb);
4320 + nskb->mac_len = skb->mac_len;
4322 + /* nskb and skb might have different headroom */
4323 + if (nskb->ip_summed == CHECKSUM_PARTIAL)
4324 + nskb->csum_start += skb_headroom(nskb) - headroom;
4326 + skb_reset_mac_header(nskb);
4327 + skb_set_network_header(nskb, skb->mac_len);
4328 + nskb->transport_header = (nskb->network_header +
4329 + skb_network_header_len(skb));
4331 + skb_copy_from_linear_data_offset(skb, -tnl_hlen,
4332 + nskb->data - tnl_hlen,
4333 + doffset + tnl_hlen);
4335 + if (fskb != skb_shinfo(skb)->frag_list)
4336 + goto perform_csum_check;
4339 + nskb->ip_summed = CHECKSUM_NONE;
4340 + nskb->csum = skb_copy_and_csum_bits(skb, offset,
4341 + skb_put(nskb, len),
4346 + frag = skb_shinfo(nskb)->frags;
4348 + skb_copy_from_linear_data_offset(skb, offset,
4349 + skb_put(nskb, hsize), hsize);
4351 + skb_shinfo(nskb)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
4353 + while (pos < offset + len && i < nfrags) {
4354 + *frag = skb_shinfo(skb)->frags[i];
4355 + __skb_frag_ref(frag);
4356 + size = skb_frag_size(frag);
4358 + if (pos < offset) {
4359 + frag->page_offset += offset - pos;
4360 + skb_frag_size_sub(frag, offset - pos);
4363 + skb_shinfo(nskb)->nr_frags++;
4365 + if (pos + size <= offset + len) {
4369 + skb_frag_size_sub(frag, pos + size - (offset + len));
4370 + goto skip_fraglist;
4376 + if (pos < offset + len) {
4377 + struct sk_buff *fskb2 = fskb;
4379 + BUG_ON(pos + fskb->len != offset + len);
4382 + fskb = fskb->next;
4384 + if (fskb2->next) {
4385 + fskb2 = skb_clone(fskb2, GFP_ATOMIC);
4391 + SKB_FRAG_ASSERT(nskb);
4392 + skb_shinfo(nskb)->frag_list = fskb2;
4396 + nskb->data_len = len - hsize;
4397 + nskb->len += nskb->data_len;
4398 + nskb->truesize += nskb->data_len;
4400 +perform_csum_check:
4402 + nskb->csum = skb_checksum(nskb, doffset,
4403 + nskb->len - doffset, 0);
4404 + nskb->ip_summed = CHECKSUM_NONE;
4406 + } while ((offset += len) < skb->len);
4411 + while ((skb = segs)) {
4415 + return ERR_PTR(err);
4417 +EXPORT_SYMBOL_GPL(skb_segment);
4419 +int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
4421 + struct sk_buff *p = *head;
4422 + struct sk_buff *nskb;
4423 + struct skb_shared_info *skbinfo = skb_shinfo(skb);
4424 + struct skb_shared_info *pinfo = skb_shinfo(p);
4425 + unsigned int headroom;
4426 + unsigned int len = skb_gro_len(skb);
4427 + unsigned int offset = skb_gro_offset(skb);
4428 + unsigned int headlen = skb_headlen(skb);
4429 + unsigned int delta_truesize;
4431 + if (p->len + len >= 65536)
4434 + if (pinfo->frag_list)
4436 + else if (headlen <= offset) {
4438 + skb_frag_t *frag2;
4439 + int i = skbinfo->nr_frags;
4440 + int nr_frags = pinfo->nr_frags + i;
4442 + offset -= headlen;
4444 + if (nr_frags > MAX_SKB_FRAGS)
4447 + pinfo->nr_frags = nr_frags;
4448 + skbinfo->nr_frags = 0;
4450 + frag = pinfo->frags + nr_frags;
4451 + frag2 = skbinfo->frags + i;
4453 + *--frag = *--frag2;
4456 + frag->page_offset += offset;
4457 + skb_frag_size_sub(frag, offset);
4459 + /* all fragments truesize : remove (head size + sk_buff) */
4460 + delta_truesize = skb->truesize -
4461 + SKB_TRUESIZE(skb_end_offset(skb));
4463 + skb->truesize -= skb->data_len;
4464 + skb->len -= skb->data_len;
4465 + skb->data_len = 0;
4467 + NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
4469 + } else if (skb->head_frag) {
4470 + int nr_frags = pinfo->nr_frags;
4471 + skb_frag_t *frag = pinfo->frags + nr_frags;
4472 + struct page *page = virt_to_head_page(skb->head);
4473 + unsigned int first_size = headlen - offset;
4474 + unsigned int first_offset;
4476 + if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
4479 + first_offset = skb->data -
4480 + (unsigned char *)page_address(page) +
4483 + pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
4485 + frag->page.p = page;
4486 + frag->page_offset = first_offset;
4487 + skb_frag_size_set(frag, first_size);
4489 + memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
4490 + /* We dont need to clear skbinfo->nr_frags here */
4492 + delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
4493 + NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
4495 + } else if (skb_gro_len(p) != pinfo->gso_size)
4498 + headroom = skb_headroom(p);
4499 + nskb = alloc_skb(headroom + skb_gro_offset(p), GFP_ATOMIC);
4500 + if (unlikely(!nskb))
4503 + __copy_skb_header(nskb, p);
4504 + nskb->mac_len = p->mac_len;
4506 + skb_reserve(nskb, headroom);
4507 + __skb_put(nskb, skb_gro_offset(p));
4509 + skb_set_mac_header(nskb, skb_mac_header(p) - p->data);
4510 + skb_set_network_header(nskb, skb_network_offset(p));
4511 + skb_set_transport_header(nskb, skb_transport_offset(p));
4513 + __skb_pull(p, skb_gro_offset(p));
4514 + memcpy(skb_mac_header(nskb), skb_mac_header(p),
4515 + p->data - skb_mac_header(p));
4517 + skb_shinfo(nskb)->frag_list = p;
4518 + skb_shinfo(nskb)->gso_size = pinfo->gso_size;
4519 + pinfo->gso_size = 0;
4520 + skb_header_release(p);
4521 + NAPI_GRO_CB(nskb)->last = p;
4523 + nskb->data_len += p->len;
4524 + nskb->truesize += p->truesize;
4525 + nskb->len += p->len;
4528 + nskb->next = p->next;
4534 + delta_truesize = skb->truesize;
4535 + if (offset > headlen) {
4536 + unsigned int eat = offset - headlen;
4538 + skbinfo->frags[0].page_offset += eat;
4539 + skb_frag_size_sub(&skbinfo->frags[0], eat);
4540 + skb->data_len -= eat;
4545 + __skb_pull(skb, offset);
4547 + NAPI_GRO_CB(p)->last->next = skb;
4548 + NAPI_GRO_CB(p)->last = skb;
4549 + skb_header_release(skb);
4552 + NAPI_GRO_CB(p)->count++;
4553 + p->data_len += len;
4554 + p->truesize += delta_truesize;
4557 + NAPI_GRO_CB(skb)->same_flow = 1;
4560 +EXPORT_SYMBOL_GPL(skb_gro_receive);
4562 +void __init skb_init(void)
4564 + skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
4565 + sizeof(struct sk_buff),
4567 + SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4569 + skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
4570 + (2*sizeof(struct sk_buff)) +
4573 + SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4578 + * skb_to_sgvec - Fill a scatter-gather list from a socket buffer
4579 + * @skb: Socket buffer containing the buffers to be mapped
4580 + * @sg: The scatter-gather list to map into
4581 + * @offset: The offset into the buffer's contents to start mapping
4582 + * @len: Length of buffer space to be mapped
4584 + * Fill the specified scatter-gather list with mappings/pointers into a
4585 + * region of the buffer space attached to a socket buffer.
4588 +__skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
4590 + int start = skb_headlen(skb);
4591 + int i, copy = start - offset;
4592 + struct sk_buff *frag_iter;
4598 + sg_set_buf(sg, skb->data + offset, copy);
4600 + if ((len -= copy) == 0)
4605 + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4608 + WARN_ON(start > offset + len);
4610 + end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
4611 + if ((copy = end - offset) > 0) {
4612 + skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4616 + sg_set_page(&sg[elt], skb_frag_page(frag), copy,
4617 + frag->page_offset+offset-start);
4619 + if (!(len -= copy))
4626 + skb_walk_frags(skb, frag_iter) {
4629 + WARN_ON(start > offset + len);
4631 + end = start + frag_iter->len;
4632 + if ((copy = end - offset) > 0) {
4635 + elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start,
4637 + if ((len -= copy) == 0)
4647 +int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
4649 + int nsg = __skb_to_sgvec(skb, sg, offset, len);
4651 + sg_mark_end(&sg[nsg - 1]);
4655 +EXPORT_SYMBOL_GPL(skb_to_sgvec);
4658 + * skb_cow_data - Check that a socket buffer's data buffers are writable
4659 + * @skb: The socket buffer to check.
4660 + * @tailbits: Amount of trailing space to be added
4661 + * @trailer: Returned pointer to the skb where the @tailbits space begins
4663 + * Make sure that the data buffers attached to a socket buffer are
4664 + * writable. If they are not, private copies are made of the data buffers
4665 + * and the socket buffer is set to use these instead.
4667 + * If @tailbits is given, make sure that there is space to write @tailbits
4668 + * bytes of data beyond current end of socket buffer. @trailer will be
4669 + * set to point to the skb in which this space begins.
4671 + * The number of scatterlist elements required to completely map the
4672 + * COW'd and extended socket buffer will be returned.
4674 +int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
4678 + struct sk_buff *skb1, **skb_p;
4680 + /* If skb is cloned or its head is paged, reallocate
4681 + * head pulling out all the pages (pages are considered not writable
4682 + * at the moment even if they are anonymous).
4684 + if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
4685 + __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
4688 + /* Easy case. Most of packets will go this way. */
4689 + if (!skb_has_frag_list(skb)) {
4690 + /* A little of trouble, not enough of space for trailer.
4691 + * This should not happen, when stack is tuned to generate
4692 + * good frames. OK, on miss we reallocate and reserve even more
4693 + * space, 128 bytes is fair. */
4695 + if (skb_tailroom(skb) < tailbits &&
4696 + pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
4704 + /* Misery. We are in troubles, going to mincer fragments... */
4707 + skb_p = &skb_shinfo(skb)->frag_list;
4710 + while ((skb1 = *skb_p) != NULL) {
4713 + /* The fragment is partially pulled by someone,
4714 + * this can happen on input. Copy it and everything
4717 + if (skb_shared(skb1))
4720 + /* If the skb is the last, worry about trailer. */
4722 + if (skb1->next == NULL && tailbits) {
4723 + if (skb_shinfo(skb1)->nr_frags ||
4724 + skb_has_frag_list(skb1) ||
4725 + skb_tailroom(skb1) < tailbits)
4726 + ntail = tailbits + 128;
4730 + skb_cloned(skb1) ||
4732 + skb_shinfo(skb1)->nr_frags ||
4733 + skb_has_frag_list(skb1)) {
4734 + struct sk_buff *skb2;
4736 + /* Fuck, we are miserable poor guys... */
4738 + skb2 = skb_copy(skb1, GFP_ATOMIC);
4740 + skb2 = skb_copy_expand(skb1,
4741 + skb_headroom(skb1),
4744 + if (unlikely(skb2 == NULL))
4748 + skb_set_owner_w(skb2, skb1->sk);
4750 + /* Looking around. Are we still alive?
4751 + * OK, link new skb, drop old one */
4753 + skb2->next = skb1->next;
4760 + skb_p = &skb1->next;
4765 +EXPORT_SYMBOL_GPL(skb_cow_data);
4767 +static void sock_rmem_free(struct sk_buff *skb)
4769 + struct sock *sk = skb->sk;
4771 + atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
4775 + * Note: We dont mem charge error packets (no sk_forward_alloc changes)
4777 +int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
4779 + int len = skb->len;
4781 + if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
4782 + (unsigned int)sk->sk_rcvbuf)
4787 + skb->destructor = sock_rmem_free;
4788 + atomic_add(skb->truesize, &sk->sk_rmem_alloc);
4790 + /* before exiting rcu section, make sure dst is refcounted */
4791 + skb_dst_force(skb);
4793 + skb_queue_tail(&sk->sk_error_queue, skb);
4794 + if (!sock_flag(sk, SOCK_DEAD))
4795 + sk->sk_data_ready(sk, len);
4798 +EXPORT_SYMBOL(sock_queue_err_skb);
4800 +void skb_tstamp_tx(struct sk_buff *orig_skb,
4801 + struct skb_shared_hwtstamps *hwtstamps)
4803 + struct sock *sk = orig_skb->sk;
4804 + struct sock_exterr_skb *serr;
4805 + struct sk_buff *skb;
4812 + *skb_hwtstamps(orig_skb) =
4816 + * no hardware time stamps available,
4817 + * so keep the shared tx_flags and only
4818 + * store software time stamp
4820 + orig_skb->tstamp = ktime_get_real();
4823 + skb = skb_clone(orig_skb, GFP_ATOMIC);
4827 + serr = SKB_EXT_ERR(skb);
4828 + memset(serr, 0, sizeof(*serr));
4829 + serr->ee.ee_errno = ENOMSG;
4830 + serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
4832 + err = sock_queue_err_skb(sk, skb);
4837 +EXPORT_SYMBOL_GPL(skb_tstamp_tx);
4839 +void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
4841 + struct sock *sk = skb->sk;
4842 + struct sock_exterr_skb *serr;
4845 + skb->wifi_acked_valid = 1;
4846 + skb->wifi_acked = acked;
4848 + serr = SKB_EXT_ERR(skb);
4849 + memset(serr, 0, sizeof(*serr));
4850 + serr->ee.ee_errno = ENOMSG;
4851 + serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
4853 + err = sock_queue_err_skb(sk, skb);
4857 +EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
4861 + * skb_partial_csum_set - set up and verify partial csum values for packet
4862 + * @skb: the skb to set
4863 + * @start: the number of bytes after skb->data to start checksumming.
4864 + * @off: the offset from start to place the checksum.
4866 + * For untrusted partially-checksummed packets, we need to make sure the values
4867 + * for skb->csum_start and skb->csum_offset are valid so we don't oops.
4869 + * This function checks and sets those values and skb->ip_summed: if this
4870 + * returns false you should drop the packet.
4872 +bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
4874 + if (unlikely(start > skb_headlen(skb)) ||
4875 + unlikely((int)start + off > skb_headlen(skb) - 2)) {
4876 + net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n",
4877 + start, off, skb_headlen(skb));
4880 + skb->ip_summed = CHECKSUM_PARTIAL;
4881 + skb->csum_start = skb_headroom(skb) + start;
4882 + skb->csum_offset = off;
4883 + skb_set_transport_header(skb, start);
4886 +EXPORT_SYMBOL_GPL(skb_partial_csum_set);
4888 +void __skb_warn_lro_forwarding(const struct sk_buff *skb)
4890 + net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
4893 +EXPORT_SYMBOL(__skb_warn_lro_forwarding);
4895 +void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
4897 + if (head_stolen) {
4898 + skb_release_head_state(skb);
4899 + kmem_cache_free(skbuff_head_cache, skb);
4904 +EXPORT_SYMBOL(kfree_skb_partial);
4907 + * skb_try_coalesce - try to merge skb to prior one
4908 + * @to: prior buffer
4909 + * @from: buffer to add
4910 + * @fragstolen: pointer to boolean
4911 + * @delta_truesize: how much more was allocated than was requested
4913 +bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
4914 + bool *fragstolen, int *delta_truesize)
4916 + int i, delta, len = from->len;
4918 + *fragstolen = false;
4920 + if (skb_cloned(to))
4923 + if (len <= skb_tailroom(to)) {
4924 + BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
4925 + *delta_truesize = 0;
4929 + if (skb_has_frag_list(to) || skb_has_frag_list(from))
4932 + if (skb_headlen(from) != 0) {
4933 + struct page *page;
4934 + unsigned int offset;
4936 + if (skb_shinfo(to)->nr_frags +
4937 + skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
4940 + if (skb_head_is_locked(from))
4943 + delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
4945 + page = virt_to_head_page(from->head);
4946 + offset = from->data - (unsigned char *)page_address(page);
4948 + skb_fill_page_desc(to, skb_shinfo(to)->nr_frags,
4949 + page, offset, skb_headlen(from));
4950 + *fragstolen = true;
4952 + if (skb_shinfo(to)->nr_frags +
4953 + skb_shinfo(from)->nr_frags > MAX_SKB_FRAGS)
4956 + delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
4959 + WARN_ON_ONCE(delta < len);
4961 + memcpy(skb_shinfo(to)->frags + skb_shinfo(to)->nr_frags,
4962 + skb_shinfo(from)->frags,
4963 + skb_shinfo(from)->nr_frags * sizeof(skb_frag_t));
4964 + skb_shinfo(to)->nr_frags += skb_shinfo(from)->nr_frags;
4966 + if (!skb_cloned(from))
4967 + skb_shinfo(from)->nr_frags = 0;
4969 + /* if the skb is not cloned this does nothing
4970 + * since we set nr_frags to 0.
4972 + for (i = 0; i < skb_shinfo(from)->nr_frags; i++)
4973 + skb_frag_ref(from, i);
4975 + to->truesize += delta;
4977 + to->data_len += len;
4979 + *delta_truesize = delta;
4982 +EXPORT_SYMBOL(skb_try_coalesce);
4983 diff -ruN linux-3.10.27/net/ipv6/ip6_output.c linux-3.10.27-imq/net/ipv6/ip6_output.c
4984 --- linux-3.10.27/net/ipv6/ip6_output.c 2014-01-16 00:29:14.000000000 +0100
4985 +++ linux-3.10.27-imq/net/ipv6/ip6_output.c 2014-01-18 10:19:59.348342972 +0100
4987 struct in6_addr *nexthop;
4990 - skb->protocol = htons(ETH_P_IPV6);
4993 if (ipv6_addr_is_multicast(&ipv6_hdr(skb)->daddr)) {
4994 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
4996 @@ -168,6 +165,13 @@
5001 + * IMQ-patch: moved setting skb->dev and skb->protocol from
5002 + * ip6_finish_output2 to fix crashing at netif_skb_features().
5004 + skb->protocol = htons(ETH_P_IPV6);
5007 return NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING, skb, NULL, dev,
5009 !(IP6CB(skb)->flags & IP6SKB_REROUTED));
5010 diff -ruN linux-3.10.27/net/ipv6/ip6_output.c.orig linux-3.10.27-imq/net/ipv6/ip6_output.c.orig
5011 --- linux-3.10.27/net/ipv6/ip6_output.c.orig 1970-01-01 01:00:00.000000000 +0100
5012 +++ linux-3.10.27-imq/net/ipv6/ip6_output.c.orig 2014-01-16 00:29:14.000000000 +0100
5015 + * IPv6 output functions
5016 + * Linux INET6 implementation
5019 + * Pedro Roque <roque@di.fc.ul.pt>
5021 + * Based on linux/net/ipv4/ip_output.c
5023 + * This program is free software; you can redistribute it and/or
5024 + * modify it under the terms of the GNU General Public License
5025 + * as published by the Free Software Foundation; either version
5026 + * 2 of the License, or (at your option) any later version.
5029 + * A.N.Kuznetsov : airthmetics in fragmentation.
5030 + * extension headers are implemented.
5031 + * route changes now work.
5032 + * ip6_forward does not confuse sniffers.
5035 + * H. von Brand : Added missing #include <linux/string.h>
5036 + * Imran Patel : frag id should be in NBO
5037 + * Kazunori MIYAZAWA @USAGI
5038 + * : add ip6_append_data and related functions
5039 + * for datagram xmit
5042 +#include <linux/errno.h>
5043 +#include <linux/kernel.h>
5044 +#include <linux/string.h>
5045 +#include <linux/socket.h>
5046 +#include <linux/net.h>
5047 +#include <linux/netdevice.h>
5048 +#include <linux/if_arp.h>
5049 +#include <linux/in6.h>
5050 +#include <linux/tcp.h>
5051 +#include <linux/route.h>
5052 +#include <linux/module.h>
5053 +#include <linux/slab.h>
5055 +#include <linux/netfilter.h>
5056 +#include <linux/netfilter_ipv6.h>
5058 +#include <net/sock.h>
5059 +#include <net/snmp.h>
5061 +#include <net/ipv6.h>
5062 +#include <net/ndisc.h>
5063 +#include <net/protocol.h>
5064 +#include <net/ip6_route.h>
5065 +#include <net/addrconf.h>
5066 +#include <net/rawv6.h>
5067 +#include <net/icmp.h>
5068 +#include <net/xfrm.h>
5069 +#include <net/checksum.h>
5070 +#include <linux/mroute6.h>
5072 +int __ip6_local_out(struct sk_buff *skb)
5076 + len = skb->len - sizeof(struct ipv6hdr);
5077 + if (len > IPV6_MAXPLEN)
5079 + ipv6_hdr(skb)->payload_len = htons(len);
5081 + return nf_hook(NFPROTO_IPV6, NF_INET_LOCAL_OUT, skb, NULL,
5082 + skb_dst(skb)->dev, dst_output);
5085 +int ip6_local_out(struct sk_buff *skb)
5089 + err = __ip6_local_out(skb);
5090 + if (likely(err == 1))
5091 + err = dst_output(skb);
5095 +EXPORT_SYMBOL_GPL(ip6_local_out);
5097 +static int ip6_finish_output2(struct sk_buff *skb)
5099 + struct dst_entry *dst = skb_dst(skb);
5100 + struct net_device *dev = dst->dev;
5101 + struct neighbour *neigh;
5102 + struct in6_addr *nexthop;
5105 + skb->protocol = htons(ETH_P_IPV6);
5108 + if (ipv6_addr_is_multicast(&ipv6_hdr(skb)->daddr)) {
5109 + struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
5111 + if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(skb->sk) &&
5112 + ((mroute6_socket(dev_net(dev), skb) &&
5113 + !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) ||
5114 + ipv6_chk_mcast_addr(dev, &ipv6_hdr(skb)->daddr,
5115 + &ipv6_hdr(skb)->saddr))) {
5116 + struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
5118 + /* Do not check for IFF_ALLMULTI; multicast routing
5119 + is not supported in any case.
5122 + NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING,
5123 + newskb, NULL, newskb->dev,
5124 + dev_loopback_xmit);
5126 + if (ipv6_hdr(skb)->hop_limit == 0) {
5127 + IP6_INC_STATS(dev_net(dev), idev,
5128 + IPSTATS_MIB_OUTDISCARDS);
5134 + IP6_UPD_PO_STATS(dev_net(dev), idev, IPSTATS_MIB_OUTMCAST,
5137 + if (IPV6_ADDR_MC_SCOPE(&ipv6_hdr(skb)->daddr) <=
5138 + IPV6_ADDR_SCOPE_NODELOCAL &&
5139 + !(dev->flags & IFF_LOOPBACK)) {
5145 + rcu_read_lock_bh();
5146 + nexthop = rt6_nexthop((struct rt6_info *)dst);
5147 + neigh = __ipv6_neigh_lookup_noref(dst->dev, nexthop);
5148 + if (unlikely(!neigh))
5149 + neigh = __neigh_create(&nd_tbl, nexthop, dst->dev, false);
5150 + if (!IS_ERR(neigh)) {
5151 + ret = dst_neigh_output(dst, neigh, skb);
5152 + rcu_read_unlock_bh();
5155 + rcu_read_unlock_bh();
5157 + IP6_INC_STATS(dev_net(dst->dev),
5158 + ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
5163 +static int ip6_finish_output(struct sk_buff *skb)
5165 + if ((skb->len > ip6_skb_dst_mtu(skb) && !skb_is_gso(skb)) ||
5166 + dst_allfrag(skb_dst(skb)) ||
5167 + (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size))
5168 + return ip6_fragment(skb, ip6_finish_output2);
5170 + return ip6_finish_output2(skb);
5173 +int ip6_output(struct sk_buff *skb)
5175 + struct net_device *dev = skb_dst(skb)->dev;
5176 + struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
5177 + if (unlikely(idev->cnf.disable_ipv6)) {
5178 + IP6_INC_STATS(dev_net(dev), idev,
5179 + IPSTATS_MIB_OUTDISCARDS);
5184 + return NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING, skb, NULL, dev,
5185 + ip6_finish_output,
5186 + !(IP6CB(skb)->flags & IP6SKB_REROUTED));
5190 + * xmit an sk_buff (used by TCP, SCTP and DCCP)
5193 +int ip6_xmit(struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6,
5194 + struct ipv6_txoptions *opt, int tclass)
5196 + struct net *net = sock_net(sk);
5197 + struct ipv6_pinfo *np = inet6_sk(sk);
5198 + struct in6_addr *first_hop = &fl6->daddr;
5199 + struct dst_entry *dst = skb_dst(skb);
5200 + struct ipv6hdr *hdr;
5201 + u8 proto = fl6->flowi6_proto;
5202 + int seg_len = skb->len;
5207 + unsigned int head_room;
5209 + /* First: exthdrs may take lots of space (~8K for now)
5210 + MAX_HEADER is not enough.
5212 + head_room = opt->opt_nflen + opt->opt_flen;
5213 + seg_len += head_room;
5214 + head_room += sizeof(struct ipv6hdr) + LL_RESERVED_SPACE(dst->dev);
5216 + if (skb_headroom(skb) < head_room) {
5217 + struct sk_buff *skb2 = skb_realloc_headroom(skb, head_room);
5218 + if (skb2 == NULL) {
5219 + IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
5220 + IPSTATS_MIB_OUTDISCARDS);
5226 + skb_set_owner_w(skb, sk);
5228 + if (opt->opt_flen)
5229 + ipv6_push_frag_opts(skb, opt, &proto);
5230 + if (opt->opt_nflen)
5231 + ipv6_push_nfrag_opts(skb, opt, &proto, &first_hop);
5234 + skb_push(skb, sizeof(struct ipv6hdr));
5235 + skb_reset_network_header(skb);
5236 + hdr = ipv6_hdr(skb);
5239 + * Fill in the IPv6 header
5242 + hlimit = np->hop_limit;
5244 + hlimit = ip6_dst_hoplimit(dst);
5246 + ip6_flow_hdr(hdr, tclass, fl6->flowlabel);
5248 + hdr->payload_len = htons(seg_len);
5249 + hdr->nexthdr = proto;
5250 + hdr->hop_limit = hlimit;
5252 + hdr->saddr = fl6->saddr;
5253 + hdr->daddr = *first_hop;
5255 + skb->priority = sk->sk_priority;
5256 + skb->mark = sk->sk_mark;
5258 + mtu = dst_mtu(dst);
5259 + if ((skb->len <= mtu) || skb->local_df || skb_is_gso(skb)) {
5260 + IP6_UPD_PO_STATS(net, ip6_dst_idev(skb_dst(skb)),
5261 + IPSTATS_MIB_OUT, skb->len);
5262 + return NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT, skb, NULL,
5263 + dst->dev, dst_output);
5266 + skb->dev = dst->dev;
5267 + ipv6_local_error(sk, EMSGSIZE, fl6, mtu);
5268 + IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS);
5273 +EXPORT_SYMBOL(ip6_xmit);
5275 +static int ip6_call_ra_chain(struct sk_buff *skb, int sel)
5277 + struct ip6_ra_chain *ra;
5278 + struct sock *last = NULL;
5280 + read_lock(&ip6_ra_lock);
5281 + for (ra = ip6_ra_chain; ra; ra = ra->next) {
5282 + struct sock *sk = ra->sk;
5283 + if (sk && ra->sel == sel &&
5284 + (!sk->sk_bound_dev_if ||
5285 + sk->sk_bound_dev_if == skb->dev->ifindex)) {
5287 + struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
5289 + rawv6_rcv(last, skb2);
5296 + rawv6_rcv(last, skb);
5297 + read_unlock(&ip6_ra_lock);
5300 + read_unlock(&ip6_ra_lock);
5304 +static int ip6_forward_proxy_check(struct sk_buff *skb)
5306 + struct ipv6hdr *hdr = ipv6_hdr(skb);
5307 + u8 nexthdr = hdr->nexthdr;
5311 + if (ipv6_ext_hdr(nexthdr)) {
5312 + offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off);
5316 + offset = sizeof(struct ipv6hdr);
5318 + if (nexthdr == IPPROTO_ICMPV6) {
5319 + struct icmp6hdr *icmp6;
5321 + if (!pskb_may_pull(skb, (skb_network_header(skb) +
5322 + offset + 1 - skb->data)))
5325 + icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset);
5327 + switch (icmp6->icmp6_type) {
5328 + case NDISC_ROUTER_SOLICITATION:
5329 + case NDISC_ROUTER_ADVERTISEMENT:
5330 + case NDISC_NEIGHBOUR_SOLICITATION:
5331 + case NDISC_NEIGHBOUR_ADVERTISEMENT:
5332 + case NDISC_REDIRECT:
5333 + /* For reaction involving unicast neighbor discovery
5334 + * message destined to the proxied address, pass it to
5344 + * The proxying router can't forward traffic sent to a link-local
5345 + * address, so signal the sender and discard the packet. This
5346 + * behavior is clarified by the MIPv6 specification.
5348 + if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) {
5349 + dst_link_failure(skb);
5356 +static inline int ip6_forward_finish(struct sk_buff *skb)
5358 + return dst_output(skb);
5361 +int ip6_forward(struct sk_buff *skb)
5363 + struct dst_entry *dst = skb_dst(skb);
5364 + struct ipv6hdr *hdr = ipv6_hdr(skb);
5365 + struct inet6_skb_parm *opt = IP6CB(skb);
5366 + struct net *net = dev_net(dst->dev);
5369 + if (net->ipv6.devconf_all->forwarding == 0)
5372 + if (skb_warn_if_lro(skb))
5375 + if (!xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) {
5376 + IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_INDISCARDS);
5380 + if (skb->pkt_type != PACKET_HOST)
5383 + skb_forward_csum(skb);
5386 + * We DO NOT make any processing on
5387 + * RA packets, pushing them to user level AS IS
5388 + * without ane WARRANTY that application will be able
5389 + * to interpret them. The reason is that we
5390 + * cannot make anything clever here.
5392 + * We are not end-node, so that if packet contains
5393 + * AH/ESP, we cannot make anything.
5394 + * Defragmentation also would be mistake, RA packets
5395 + * cannot be fragmented, because there is no warranty
5396 + * that different fragments will go along one path. --ANK
5398 + if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) {
5399 + if (ip6_call_ra_chain(skb, ntohs(opt->ra)))
5404 + * check and decrement ttl
5406 + if (hdr->hop_limit <= 1) {
5407 + /* Force OUTPUT device used as source address */
5408 + skb->dev = dst->dev;
5409 + icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0);
5410 + IP6_INC_STATS_BH(net,
5411 + ip6_dst_idev(dst), IPSTATS_MIB_INHDRERRORS);
5414 + return -ETIMEDOUT;
5417 + /* XXX: idev->cnf.proxy_ndp? */
5418 + if (net->ipv6.devconf_all->proxy_ndp &&
5419 + pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev, 0)) {
5420 + int proxied = ip6_forward_proxy_check(skb);
5422 + return ip6_input(skb);
5423 + else if (proxied < 0) {
5424 + IP6_INC_STATS(net, ip6_dst_idev(dst),
5425 + IPSTATS_MIB_INDISCARDS);
5430 + if (!xfrm6_route_forward(skb)) {
5431 + IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_INDISCARDS);
5434 + dst = skb_dst(skb);
5436 + /* IPv6 specs say nothing about it, but it is clear that we cannot
5437 + send redirects to source routed frames.
5438 + We don't send redirects to frames decapsulated from IPsec.
5440 + if (skb->dev == dst->dev && opt->srcrt == 0 && !skb_sec_path(skb)) {
5441 + struct in6_addr *target = NULL;
5442 + struct inet_peer *peer;
5443 + struct rt6_info *rt;
5446 + * incoming and outgoing devices are the same
5447 + * send a redirect.
5450 + rt = (struct rt6_info *) dst;
5451 + if (rt->rt6i_flags & RTF_GATEWAY)
5452 + target = &rt->rt6i_gateway;
5454 + target = &hdr->daddr;
5456 + peer = inet_getpeer_v6(net->ipv6.peers, &rt->rt6i_dst.addr, 1);
5458 + /* Limit redirects both by destination (here)
5459 + and by source (inside ndisc_send_redirect)
5461 + if (inet_peer_xrlim_allow(peer, 1*HZ))
5462 + ndisc_send_redirect(skb, target);
5464 + inet_putpeer(peer);
5466 + int addrtype = ipv6_addr_type(&hdr->saddr);
5468 + /* This check is security critical. */
5469 + if (addrtype == IPV6_ADDR_ANY ||
5470 + addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK))
5472 + if (addrtype & IPV6_ADDR_LINKLOCAL) {
5473 + icmpv6_send(skb, ICMPV6_DEST_UNREACH,
5474 + ICMPV6_NOT_NEIGHBOUR, 0);
5479 + mtu = dst_mtu(dst);
5480 + if (mtu < IPV6_MIN_MTU)
5481 + mtu = IPV6_MIN_MTU;
5483 + if ((!skb->local_df && skb->len > mtu && !skb_is_gso(skb)) ||
5484 + (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu)) {
5485 + /* Again, force OUTPUT device used as source address */
5486 + skb->dev = dst->dev;
5487 + icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
5488 + IP6_INC_STATS_BH(net,
5489 + ip6_dst_idev(dst), IPSTATS_MIB_INTOOBIGERRORS);
5490 + IP6_INC_STATS_BH(net,
5491 + ip6_dst_idev(dst), IPSTATS_MIB_FRAGFAILS);
5496 + if (skb_cow(skb, dst->dev->hard_header_len)) {
5497 + IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTDISCARDS);
5501 + hdr = ipv6_hdr(skb);
5503 + /* Mangling hops number delayed to point after skb COW */
5507 + IP6_INC_STATS_BH(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS);
5508 + IP6_ADD_STATS_BH(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTOCTETS, skb->len);
5509 + return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD, skb, skb->dev, dst->dev,
5510 + ip6_forward_finish);
5513 + IP6_INC_STATS_BH(net, ip6_dst_idev(dst), IPSTATS_MIB_INADDRERRORS);
5519 +static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from)
5521 + to->pkt_type = from->pkt_type;
5522 + to->priority = from->priority;
5523 + to->protocol = from->protocol;
5525 + skb_dst_set(to, dst_clone(skb_dst(from)));
5526 + to->dev = from->dev;
5527 + to->mark = from->mark;
5529 +#ifdef CONFIG_NET_SCHED
5530 + to->tc_index = from->tc_index;
5532 + nf_copy(to, from);
5533 +#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE)
5534 + to->nf_trace = from->nf_trace;
5536 + skb_copy_secmark(to, from);
5539 +int ip6_fragment(struct sk_buff *skb, int (*output)(struct sk_buff *))
5541 + struct sk_buff *frag;
5542 + struct rt6_info *rt = (struct rt6_info*)skb_dst(skb);
5543 + struct ipv6_pinfo *np = skb->sk ? inet6_sk(skb->sk) : NULL;
5544 + struct ipv6hdr *tmp_hdr;
5545 + struct frag_hdr *fh;
5546 + unsigned int mtu, hlen, left, len;
5548 + __be32 frag_id = 0;
5549 + int ptr, offset = 0, err=0;
5550 + u8 *prevhdr, nexthdr = 0;
5551 + struct net *net = dev_net(skb_dst(skb)->dev);
5553 + hlen = ip6_find_1stfragopt(skb, &prevhdr);
5554 + nexthdr = *prevhdr;
5556 + mtu = ip6_skb_dst_mtu(skb);
5558 + /* We must not fragment if the socket is set to force MTU discovery
5559 + * or if the skb it not generated by a local socket.
5561 + if (unlikely(!skb->local_df && skb->len > mtu) ||
5562 + (IP6CB(skb)->frag_max_size &&
5563 + IP6CB(skb)->frag_max_size > mtu)) {
5564 + if (skb->sk && dst_allfrag(skb_dst(skb)))
5565 + sk_nocaps_add(skb->sk, NETIF_F_GSO_MASK);
5567 + skb->dev = skb_dst(skb)->dev;
5568 + icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
5569 + IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
5570 + IPSTATS_MIB_FRAGFAILS);
5575 + if (np && np->frag_size < mtu) {
5576 + if (np->frag_size)
5577 + mtu = np->frag_size;
5579 + mtu -= hlen + sizeof(struct frag_hdr);
5581 + if (skb_has_frag_list(skb)) {
5582 + int first_len = skb_pagelen(skb);
5583 + struct sk_buff *frag2;
5585 + if (first_len - hlen > mtu ||
5586 + ((first_len - hlen) & 7) ||
5590 + skb_walk_frags(skb, frag) {
5591 + /* Correct geometry. */
5592 + if (frag->len > mtu ||
5593 + ((frag->len & 7) && frag->next) ||
5594 + skb_headroom(frag) < hlen)
5595 + goto slow_path_clean;
5597 + /* Partially cloned skb? */
5598 + if (skb_shared(frag))
5599 + goto slow_path_clean;
5603 + frag->sk = skb->sk;
5604 + frag->destructor = sock_wfree;
5606 + skb->truesize -= frag->truesize;
5611 + frag = skb_shinfo(skb)->frag_list;
5612 + skb_frag_list_init(skb);
5613 + /* BUILD HEADER */
5615 + *prevhdr = NEXTHDR_FRAGMENT;
5616 + tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC);
5618 + IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
5619 + IPSTATS_MIB_FRAGFAILS);
5623 + __skb_pull(skb, hlen);
5624 + fh = (struct frag_hdr*)__skb_push(skb, sizeof(struct frag_hdr));
5625 + __skb_push(skb, hlen);
5626 + skb_reset_network_header(skb);
5627 + memcpy(skb_network_header(skb), tmp_hdr, hlen);
5629 + ipv6_select_ident(fh, rt);
5630 + fh->nexthdr = nexthdr;
5632 + fh->frag_off = htons(IP6_MF);
5633 + frag_id = fh->identification;
5635 + first_len = skb_pagelen(skb);
5636 + skb->data_len = first_len - skb_headlen(skb);
5637 + skb->len = first_len;
5638 + ipv6_hdr(skb)->payload_len = htons(first_len -
5639 + sizeof(struct ipv6hdr));
5641 + dst_hold(&rt->dst);
5644 + /* Prepare header of the next frame,
5645 + * before previous one went down. */
5647 + frag->ip_summed = CHECKSUM_NONE;
5648 + skb_reset_transport_header(frag);
5649 + fh = (struct frag_hdr*)__skb_push(frag, sizeof(struct frag_hdr));
5650 + __skb_push(frag, hlen);
5651 + skb_reset_network_header(frag);
5652 + memcpy(skb_network_header(frag), tmp_hdr,
5654 + offset += skb->len - hlen - sizeof(struct frag_hdr);
5655 + fh->nexthdr = nexthdr;
5657 + fh->frag_off = htons(offset);
5658 + if (frag->next != NULL)
5659 + fh->frag_off |= htons(IP6_MF);
5660 + fh->identification = frag_id;
5661 + ipv6_hdr(frag)->payload_len =
5663 + sizeof(struct ipv6hdr));
5664 + ip6_copy_metadata(frag, skb);
5667 + err = output(skb);
5669 + IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
5670 + IPSTATS_MIB_FRAGCREATES);
5683 + IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
5684 + IPSTATS_MIB_FRAGOKS);
5695 + IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
5696 + IPSTATS_MIB_FRAGFAILS);
5701 + skb_walk_frags(skb, frag2) {
5702 + if (frag2 == frag)
5705 + frag2->destructor = NULL;
5706 + skb->truesize += frag2->truesize;
5711 + if ((skb->ip_summed == CHECKSUM_PARTIAL) &&
5712 + skb_checksum_help(skb))
5715 + left = skb->len - hlen; /* Space per frame */
5716 + ptr = hlen; /* Where to start from */
5719 + * Fragment the datagram.
5722 + *prevhdr = NEXTHDR_FRAGMENT;
5723 + hroom = LL_RESERVED_SPACE(rt->dst.dev);
5724 + troom = rt->dst.dev->needed_tailroom;
5727 + * Keep copying data until we run out.
5731 + /* IF: it doesn't fit, use 'mtu' - the data space left */
5734 + /* IF: we are not sending up to and including the packet end
5735 + then align the next start on an eight byte boundary */
5740 + * Allocate buffer.
5743 + if ((frag = alloc_skb(len + hlen + sizeof(struct frag_hdr) +
5744 + hroom + troom, GFP_ATOMIC)) == NULL) {
5745 + NETDEBUG(KERN_INFO "IPv6: frag: no memory for new fragment!\n");
5746 + IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
5747 + IPSTATS_MIB_FRAGFAILS);
5753 + * Set up data on packet
5756 + ip6_copy_metadata(frag, skb);
5757 + skb_reserve(frag, hroom);
5758 + skb_put(frag, len + hlen + sizeof(struct frag_hdr));
5759 + skb_reset_network_header(frag);
5760 + fh = (struct frag_hdr *)(skb_network_header(frag) + hlen);
5761 + frag->transport_header = (frag->network_header + hlen +
5762 + sizeof(struct frag_hdr));
5765 + * Charge the memory for the fragment to any owner
5766 + * it might possess
5769 + skb_set_owner_w(frag, skb->sk);
5772 + * Copy the packet header into the new buffer.
5774 + skb_copy_from_linear_data(skb, skb_network_header(frag), hlen);
5777 + * Build fragment header.
5779 + fh->nexthdr = nexthdr;
5782 + ipv6_select_ident(fh, rt);
5783 + frag_id = fh->identification;
5785 + fh->identification = frag_id;
5788 + * Copy a block of the IP datagram.
5790 + if (skb_copy_bits(skb, ptr, skb_transport_header(frag), len))
5794 + fh->frag_off = htons(offset);
5796 + fh->frag_off |= htons(IP6_MF);
5797 + ipv6_hdr(frag)->payload_len = htons(frag->len -
5798 + sizeof(struct ipv6hdr));
5804 + * Put this fragment into the sending queue.
5806 + err = output(frag);
5810 + IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
5811 + IPSTATS_MIB_FRAGCREATES);
5813 + IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
5814 + IPSTATS_MIB_FRAGOKS);
5819 + IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
5820 + IPSTATS_MIB_FRAGFAILS);
5825 +static inline int ip6_rt_check(const struct rt6key *rt_key,
5826 + const struct in6_addr *fl_addr,
5827 + const struct in6_addr *addr_cache)
5829 + return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) &&
5830 + (addr_cache == NULL || !ipv6_addr_equal(fl_addr, addr_cache));
5833 +static struct dst_entry *ip6_sk_dst_check(struct sock *sk,
5834 + struct dst_entry *dst,
5835 + const struct flowi6 *fl6)
5837 + struct ipv6_pinfo *np = inet6_sk(sk);
5838 + struct rt6_info *rt;
5843 + if (dst->ops->family != AF_INET6) {
5848 + rt = (struct rt6_info *)dst;
5849 + /* Yes, checking route validity in not connected
5850 + * case is not very simple. Take into account,
5851 + * that we do not support routing by source, TOS,
5852 + * and MSG_DONTROUTE --ANK (980726)
5854 + * 1. ip6_rt_check(): If route was host route,
5855 + * check that cached destination is current.
5856 + * If it is network route, we still may
5857 + * check its validity using saved pointer
5858 + * to the last used address: daddr_cache.
5859 + * We do not want to save whole address now,
5860 + * (because main consumer of this service
5861 + * is tcp, which has not this problem),
5862 + * so that the last trick works only on connected
5864 + * 2. oif also should be the same.
5866 + if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr, np->daddr_cache) ||
5867 +#ifdef CONFIG_IPV6_SUBTREES
5868 + ip6_rt_check(&rt->rt6i_src, &fl6->saddr, np->saddr_cache) ||
5870 + (fl6->flowi6_oif && fl6->flowi6_oif != dst->dev->ifindex)) {
5879 +static int ip6_dst_lookup_tail(struct sock *sk,
5880 + struct dst_entry **dst, struct flowi6 *fl6)
5882 + struct net *net = sock_net(sk);
5883 +#ifdef CONFIG_IPV6_OPTIMISTIC_DAD
5884 + struct neighbour *n;
5885 + struct rt6_info *rt;
5890 + *dst = ip6_route_output(net, sk, fl6);
5892 + if ((err = (*dst)->error))
5893 + goto out_err_release;
5895 + if (ipv6_addr_any(&fl6->saddr)) {
5896 + struct rt6_info *rt = (struct rt6_info *) *dst;
5897 + err = ip6_route_get_saddr(net, rt, &fl6->daddr,
5898 + sk ? inet6_sk(sk)->srcprefs : 0,
5901 + goto out_err_release;
5904 +#ifdef CONFIG_IPV6_OPTIMISTIC_DAD
5906 + * Here if the dst entry we've looked up
5907 + * has a neighbour entry that is in the INCOMPLETE
5908 + * state and the src address from the flow is
5909 + * marked as OPTIMISTIC, we release the found
5910 + * dst entry and replace it instead with the
5911 + * dst entry of the nexthop router
5913 + rt = (struct rt6_info *) *dst;
5914 + rcu_read_lock_bh();
5915 + n = __ipv6_neigh_lookup_noref(rt->dst.dev, rt6_nexthop(rt));
5916 + err = n && !(n->nud_state & NUD_VALID) ? -EINVAL : 0;
5917 + rcu_read_unlock_bh();
5920 + struct inet6_ifaddr *ifp;
5921 + struct flowi6 fl_gw6;
5924 + ifp = ipv6_get_ifaddr(net, &fl6->saddr,
5927 + redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC);
5933 + * We need to get the dst entry for the
5934 + * default router instead
5936 + dst_release(*dst);
5937 + memcpy(&fl_gw6, fl6, sizeof(struct flowi6));
5938 + memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr));
5939 + *dst = ip6_route_output(net, sk, &fl_gw6);
5940 + if ((err = (*dst)->error))
5941 + goto out_err_release;
5949 + if (err == -ENETUNREACH)
5950 + IP6_INC_STATS_BH(net, NULL, IPSTATS_MIB_OUTNOROUTES);
5951 + dst_release(*dst);
5957 + * ip6_dst_lookup - perform route lookup on flow
5958 + * @sk: socket which provides route info
5959 + * @dst: pointer to dst_entry * for result
5960 + * @fl6: flow to lookup
5962 + * This function performs a route lookup on the given flow.
5964 + * It returns zero on success, or a standard errno code on error.
5966 +int ip6_dst_lookup(struct sock *sk, struct dst_entry **dst, struct flowi6 *fl6)
5969 + return ip6_dst_lookup_tail(sk, dst, fl6);
5971 +EXPORT_SYMBOL_GPL(ip6_dst_lookup);
5974 + * ip6_dst_lookup_flow - perform route lookup on flow with ipsec
5975 + * @sk: socket which provides route info
5976 + * @fl6: flow to lookup
5977 + * @final_dst: final destination address for ipsec lookup
5978 + * @can_sleep: we are in a sleepable context
5980 + * This function performs a route lookup on the given flow.
5982 + * It returns a valid dst pointer on success, or a pointer encoded
5985 +struct dst_entry *ip6_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6,
5986 + const struct in6_addr *final_dst,
5989 + struct dst_entry *dst = NULL;
5992 + err = ip6_dst_lookup_tail(sk, &dst, fl6);
5994 + return ERR_PTR(err);
5996 + fl6->daddr = *final_dst;
5998 + fl6->flowi6_flags |= FLOWI_FLAG_CAN_SLEEP;
6000 + return xfrm_lookup(sock_net(sk), dst, flowi6_to_flowi(fl6), sk, 0);
6002 +EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow);
6005 + * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow
6006 + * @sk: socket which provides the dst cache and route info
6007 + * @fl6: flow to lookup
6008 + * @final_dst: final destination address for ipsec lookup
6009 + * @can_sleep: we are in a sleepable context
6011 + * This function performs a route lookup on the given flow with the
6012 + * possibility of using the cached route in the socket if it is valid.
6013 + * It will take the socket dst lock when operating on the dst cache.
6014 + * As a result, this function can only be used in process context.
6016 + * It returns a valid dst pointer on success, or a pointer encoded
6019 +struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6,
6020 + const struct in6_addr *final_dst,
6023 + struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie);
6026 + dst = ip6_sk_dst_check(sk, dst, fl6);
6028 + err = ip6_dst_lookup_tail(sk, &dst, fl6);
6030 + return ERR_PTR(err);
6032 + fl6->daddr = *final_dst;
6034 + fl6->flowi6_flags |= FLOWI_FLAG_CAN_SLEEP;
6036 + return xfrm_lookup(sock_net(sk), dst, flowi6_to_flowi(fl6), sk, 0);
6038 +EXPORT_SYMBOL_GPL(ip6_sk_dst_lookup_flow);
6040 +static inline int ip6_ufo_append_data(struct sock *sk,
6041 + int getfrag(void *from, char *to, int offset, int len,
6042 + int odd, struct sk_buff *skb),
6043 + void *from, int length, int hh_len, int fragheaderlen,
6044 + int transhdrlen, int mtu,unsigned int flags,
6045 + struct rt6_info *rt)
6048 + struct sk_buff *skb;
6051 + /* There is support for UDP large send offload by network
6052 + * device, so create one single skb packet containing complete
6055 + if ((skb = skb_peek_tail(&sk->sk_write_queue)) == NULL) {
6056 + struct frag_hdr fhdr;
6058 + skb = sock_alloc_send_skb(sk,
6059 + hh_len + fragheaderlen + transhdrlen + 20,
6060 + (flags & MSG_DONTWAIT), &err);
6064 + /* reserve space for Hardware header */
6065 + skb_reserve(skb, hh_len);
6067 + /* create space for UDP/IP header */
6068 + skb_put(skb,fragheaderlen + transhdrlen);
6070 + /* initialize network header pointer */
6071 + skb_reset_network_header(skb);
6073 + /* initialize protocol header pointer */
6074 + skb->transport_header = skb->network_header + fragheaderlen;
6076 + skb->ip_summed = CHECKSUM_PARTIAL;
6079 + /* Specify the length of each IPv6 datagram fragment.
6080 + * It has to be a multiple of 8.
6082 + skb_shinfo(skb)->gso_size = (mtu - fragheaderlen -
6083 + sizeof(struct frag_hdr)) & ~7;
6084 + skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
6085 + ipv6_select_ident(&fhdr, rt);
6086 + skb_shinfo(skb)->ip6_frag_id = fhdr.identification;
6087 + __skb_queue_tail(&sk->sk_write_queue, skb);
6090 + return skb_append_datato_frags(sk, skb, getfrag, from,
6091 + (length - transhdrlen));
6094 +static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src,
6097 + return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
6100 +static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src,
6103 + return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
6106 +static void ip6_append_data_mtu(unsigned int *mtu,
6108 + unsigned int fragheaderlen,
6109 + struct sk_buff *skb,
6110 + struct rt6_info *rt,
6113 + if (!(rt->dst.flags & DST_XFRM_TUNNEL)) {
6114 + if (skb == NULL) {
6115 + /* first fragment, reserve header_len */
6116 + *mtu = *mtu - rt->dst.header_len;
6120 + * this fragment is not first, the headers
6121 + * space is regarded as data space.
6123 + *mtu = min(*mtu, pmtuprobe ?
6124 + rt->dst.dev->mtu :
6125 + dst_mtu(rt->dst.path));
6127 + *maxfraglen = ((*mtu - fragheaderlen) & ~7)
6128 + + fragheaderlen - sizeof(struct frag_hdr);
6132 +int ip6_append_data(struct sock *sk, int getfrag(void *from, char *to,
6133 + int offset, int len, int odd, struct sk_buff *skb),
6134 + void *from, int length, int transhdrlen,
6135 + int hlimit, int tclass, struct ipv6_txoptions *opt, struct flowi6 *fl6,
6136 + struct rt6_info *rt, unsigned int flags, int dontfrag)
6138 + struct inet_sock *inet = inet_sk(sk);
6139 + struct ipv6_pinfo *np = inet6_sk(sk);
6140 + struct inet_cork *cork;
6141 + struct sk_buff *skb, *skb_prev = NULL;
6142 + unsigned int maxfraglen, fragheaderlen, mtu;
6144 + int dst_exthdrlen;
6149 + __u8 tx_flags = 0;
6151 + if (flags&MSG_PROBE)
6153 + cork = &inet->cork.base;
6154 + if (skb_queue_empty(&sk->sk_write_queue)) {
6156 + * setup for corking
6159 + if (WARN_ON(np->cork.opt))
6162 + np->cork.opt = kzalloc(opt->tot_len, sk->sk_allocation);
6163 + if (unlikely(np->cork.opt == NULL))
6166 + np->cork.opt->tot_len = opt->tot_len;
6167 + np->cork.opt->opt_flen = opt->opt_flen;
6168 + np->cork.opt->opt_nflen = opt->opt_nflen;
6170 + np->cork.opt->dst0opt = ip6_opt_dup(opt->dst0opt,
6171 + sk->sk_allocation);
6172 + if (opt->dst0opt && !np->cork.opt->dst0opt)
6175 + np->cork.opt->dst1opt = ip6_opt_dup(opt->dst1opt,
6176 + sk->sk_allocation);
6177 + if (opt->dst1opt && !np->cork.opt->dst1opt)
6180 + np->cork.opt->hopopt = ip6_opt_dup(opt->hopopt,
6181 + sk->sk_allocation);
6182 + if (opt->hopopt && !np->cork.opt->hopopt)
6185 + np->cork.opt->srcrt = ip6_rthdr_dup(opt->srcrt,
6186 + sk->sk_allocation);
6187 + if (opt->srcrt && !np->cork.opt->srcrt)
6190 + /* need source address above miyazawa*/
6192 + dst_hold(&rt->dst);
6193 + cork->dst = &rt->dst;
6194 + inet->cork.fl.u.ip6 = *fl6;
6195 + np->cork.hop_limit = hlimit;
6196 + np->cork.tclass = tclass;
6197 + if (rt->dst.flags & DST_XFRM_TUNNEL)
6198 + mtu = np->pmtudisc == IPV6_PMTUDISC_PROBE ?
6199 + rt->dst.dev->mtu : dst_mtu(&rt->dst);
6201 + mtu = np->pmtudisc == IPV6_PMTUDISC_PROBE ?
6202 + rt->dst.dev->mtu : dst_mtu(rt->dst.path);
6203 + if (np->frag_size < mtu) {
6204 + if (np->frag_size)
6205 + mtu = np->frag_size;
6207 + cork->fragsize = mtu;
6208 + if (dst_allfrag(rt->dst.path))
6209 + cork->flags |= IPCORK_ALLFRAG;
6211 + exthdrlen = (opt ? opt->opt_flen : 0);
6212 + length += exthdrlen;
6213 + transhdrlen += exthdrlen;
6214 + dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
6216 + rt = (struct rt6_info *)cork->dst;
6217 + fl6 = &inet->cork.fl.u.ip6;
6218 + opt = np->cork.opt;
6221 + dst_exthdrlen = 0;
6222 + mtu = cork->fragsize;
6225 + hh_len = LL_RESERVED_SPACE(rt->dst.dev);
6227 + fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len +
6228 + (opt ? opt->opt_nflen : 0);
6229 + maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen - sizeof(struct frag_hdr);
6231 + if (mtu <= sizeof(struct ipv6hdr) + IPV6_MAXPLEN) {
6232 + if (cork->length + length > sizeof(struct ipv6hdr) + IPV6_MAXPLEN - fragheaderlen) {
6233 + ipv6_local_error(sk, EMSGSIZE, fl6, mtu-exthdrlen);
6238 + /* For UDP, check if TX timestamp is enabled */
6239 + if (sk->sk_type == SOCK_DGRAM)
6240 + sock_tx_timestamp(sk, &tx_flags);
6243 + * Let's try using as much space as possible.
6244 + * Use MTU if total length of the message fits into the MTU.
6245 + * Otherwise, we need to reserve fragment header and
6246 + * fragment alignment (= 8-15 octects, in total).
6248 + * Note that we may need to "move" the data from the tail of
6249 + * of the buffer to the new fragment when we split
6252 + * FIXME: It may be fragmented into multiple chunks
6253 + * at once if non-fragmentable extension headers
6258 + if ((length > mtu) && dontfrag && (sk->sk_protocol == IPPROTO_UDP ||
6259 + sk->sk_protocol == IPPROTO_RAW)) {
6260 + ipv6_local_rxpmtu(sk, fl6, mtu-exthdrlen);
6264 + skb = skb_peek_tail(&sk->sk_write_queue);
6265 + cork->length += length;
6266 + if (((length > mtu) ||
6267 + (skb && skb_has_frags(skb))) &&
6268 + (sk->sk_protocol == IPPROTO_UDP) &&
6269 + (rt->dst.dev->features & NETIF_F_UFO)) {
6270 + err = ip6_ufo_append_data(sk, getfrag, from, length,
6271 + hh_len, fragheaderlen,
6272 + transhdrlen, mtu, flags, rt);
6279 + goto alloc_new_skb;
6281 + while (length > 0) {
6282 + /* Check if the remaining data fits into current packet. */
6283 + copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len;
6284 + if (copy < length)
6285 + copy = maxfraglen - skb->len;
6289 + unsigned int datalen;
6290 + unsigned int fraglen;
6291 + unsigned int fraggap;
6292 + unsigned int alloclen;
6294 + /* There's no room in the current skb */
6296 + fraggap = skb->len - maxfraglen;
6299 + /* update mtu and maxfraglen if necessary */
6300 + if (skb == NULL || skb_prev == NULL)
6301 + ip6_append_data_mtu(&mtu, &maxfraglen,
6302 + fragheaderlen, skb, rt,
6304 + IPV6_PMTUDISC_PROBE);
6309 + * If remaining data exceeds the mtu,
6310 + * we know we need more fragment(s).
6312 + datalen = length + fraggap;
6314 + if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen)
6315 + datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;
6316 + if ((flags & MSG_MORE) &&
6317 + !(rt->dst.dev->features&NETIF_F_SG))
6320 + alloclen = datalen + fragheaderlen;
6322 + alloclen += dst_exthdrlen;
6324 + if (datalen != length + fraggap) {
6326 + * this is not the last fragment, the trailer
6327 + * space is regarded as data space.
6329 + datalen += rt->dst.trailer_len;
6332 + alloclen += rt->dst.trailer_len;
6333 + fraglen = datalen + fragheaderlen;
6336 + * We just reserve space for fragment header.
6337 + * Note: this may be overallocation if the message
6338 + * (without MSG_MORE) fits into the MTU.
6340 + alloclen += sizeof(struct frag_hdr);
6342 + if (transhdrlen) {
6343 + skb = sock_alloc_send_skb(sk,
6344 + alloclen + hh_len,
6345 + (flags & MSG_DONTWAIT), &err);
6348 + if (atomic_read(&sk->sk_wmem_alloc) <=
6349 + 2 * sk->sk_sndbuf)
6350 + skb = sock_wmalloc(sk,
6351 + alloclen + hh_len, 1,
6352 + sk->sk_allocation);
6353 + if (unlikely(skb == NULL))
6356 + /* Only the initial fragment
6357 + * is time stamped.
6365 + * Fill in the control structures
6367 + skb->ip_summed = CHECKSUM_NONE;
6369 + /* reserve for fragmentation and ipsec header */
6370 + skb_reserve(skb, hh_len + sizeof(struct frag_hdr) +
6373 + if (sk->sk_type == SOCK_DGRAM)
6374 + skb_shinfo(skb)->tx_flags = tx_flags;
6377 + * Find where to start putting bytes
6379 + data = skb_put(skb, fraglen);
6380 + skb_set_network_header(skb, exthdrlen);
6381 + data += fragheaderlen;
6382 + skb->transport_header = (skb->network_header +
6385 + skb->csum = skb_copy_and_csum_bits(
6386 + skb_prev, maxfraglen,
6387 + data + transhdrlen, fraggap, 0);
6388 + skb_prev->csum = csum_sub(skb_prev->csum,
6391 + pskb_trim_unique(skb_prev, maxfraglen);
6393 + copy = datalen - transhdrlen - fraggap;
6399 + } else if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
6406 + length -= datalen - fraggap;
6409 + dst_exthdrlen = 0;
6412 + * Put the packet on the pending queue
6414 + __skb_queue_tail(&sk->sk_write_queue, skb);
6418 + if (copy > length)
6421 + if (!(rt->dst.dev->features&NETIF_F_SG)) {
6425 + if (getfrag(from, skb_put(skb, copy),
6426 + offset, copy, off, skb) < 0) {
6427 + __skb_trim(skb, off);
6432 + int i = skb_shinfo(skb)->nr_frags;
6433 + struct page_frag *pfrag = sk_page_frag(sk);
6436 + if (!sk_page_frag_refill(sk, pfrag))
6439 + if (!skb_can_coalesce(skb, i, pfrag->page,
6442 + if (i == MAX_SKB_FRAGS)
6445 + __skb_fill_page_desc(skb, i, pfrag->page,
6446 + pfrag->offset, 0);
6447 + skb_shinfo(skb)->nr_frags = ++i;
6448 + get_page(pfrag->page);
6450 + copy = min_t(int, copy, pfrag->size - pfrag->offset);
6452 + page_address(pfrag->page) + pfrag->offset,
6453 + offset, copy, skb->len, skb) < 0)
6454 + goto error_efault;
6456 + pfrag->offset += copy;
6457 + skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
6459 + skb->data_len += copy;
6460 + skb->truesize += copy;
6461 + atomic_add(copy, &sk->sk_wmem_alloc);
6472 + cork->length -= length;
6473 + IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
6476 +EXPORT_SYMBOL_GPL(ip6_append_data);
6478 +static void ip6_cork_release(struct inet_sock *inet, struct ipv6_pinfo *np)
6480 + if (np->cork.opt) {
6481 + kfree(np->cork.opt->dst0opt);
6482 + kfree(np->cork.opt->dst1opt);
6483 + kfree(np->cork.opt->hopopt);
6484 + kfree(np->cork.opt->srcrt);
6485 + kfree(np->cork.opt);
6486 + np->cork.opt = NULL;
6489 + if (inet->cork.base.dst) {
6490 + dst_release(inet->cork.base.dst);
6491 + inet->cork.base.dst = NULL;
6492 + inet->cork.base.flags &= ~IPCORK_ALLFRAG;
6494 + memset(&inet->cork.fl, 0, sizeof(inet->cork.fl));
6497 +int ip6_push_pending_frames(struct sock *sk)
6499 + struct sk_buff *skb, *tmp_skb;
6500 + struct sk_buff **tail_skb;
6501 + struct in6_addr final_dst_buf, *final_dst = &final_dst_buf;
6502 + struct inet_sock *inet = inet_sk(sk);
6503 + struct ipv6_pinfo *np = inet6_sk(sk);
6504 + struct net *net = sock_net(sk);
6505 + struct ipv6hdr *hdr;
6506 + struct ipv6_txoptions *opt = np->cork.opt;
6507 + struct rt6_info *rt = (struct rt6_info *)inet->cork.base.dst;
6508 + struct flowi6 *fl6 = &inet->cork.fl.u.ip6;
6509 + unsigned char proto = fl6->flowi6_proto;
6512 + if ((skb = __skb_dequeue(&sk->sk_write_queue)) == NULL)
6514 + tail_skb = &(skb_shinfo(skb)->frag_list);
6516 + /* move skb->data to ip header from ext header */
6517 + if (skb->data < skb_network_header(skb))
6518 + __skb_pull(skb, skb_network_offset(skb));
6519 + while ((tmp_skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {
6520 + __skb_pull(tmp_skb, skb_network_header_len(skb));
6521 + *tail_skb = tmp_skb;
6522 + tail_skb = &(tmp_skb->next);
6523 + skb->len += tmp_skb->len;
6524 + skb->data_len += tmp_skb->len;
6525 + skb->truesize += tmp_skb->truesize;
6526 + tmp_skb->destructor = NULL;
6527 + tmp_skb->sk = NULL;
6530 + /* Allow local fragmentation. */
6531 + if (np->pmtudisc < IPV6_PMTUDISC_DO)
6532 + skb->local_df = 1;
6534 + *final_dst = fl6->daddr;
6535 + __skb_pull(skb, skb_network_header_len(skb));
6536 + if (opt && opt->opt_flen)
6537 + ipv6_push_frag_opts(skb, opt, &proto);
6538 + if (opt && opt->opt_nflen)
6539 + ipv6_push_nfrag_opts(skb, opt, &proto, &final_dst);
6541 + skb_push(skb, sizeof(struct ipv6hdr));
6542 + skb_reset_network_header(skb);
6543 + hdr = ipv6_hdr(skb);
6545 + ip6_flow_hdr(hdr, np->cork.tclass, fl6->flowlabel);
6546 + hdr->hop_limit = np->cork.hop_limit;
6547 + hdr->nexthdr = proto;
6548 + hdr->saddr = fl6->saddr;
6549 + hdr->daddr = *final_dst;
6551 + skb->priority = sk->sk_priority;
6552 + skb->mark = sk->sk_mark;
6554 + skb_dst_set(skb, dst_clone(&rt->dst));
6555 + IP6_UPD_PO_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUT, skb->len);
6556 + if (proto == IPPROTO_ICMPV6) {
6557 + struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
6559 + ICMP6MSGOUT_INC_STATS_BH(net, idev, icmp6_hdr(skb)->icmp6_type);
6560 + ICMP6_INC_STATS_BH(net, idev, ICMP6_MIB_OUTMSGS);
6563 + err = ip6_local_out(skb);
6566 + err = net_xmit_errno(err);
6572 + ip6_cork_release(inet, np);
6575 + IP6_INC_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
6578 +EXPORT_SYMBOL_GPL(ip6_push_pending_frames);
6580 +void ip6_flush_pending_frames(struct sock *sk)
6582 + struct sk_buff *skb;
6584 + while ((skb = __skb_dequeue_tail(&sk->sk_write_queue)) != NULL) {
6586 + IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)),
6587 + IPSTATS_MIB_OUTDISCARDS);
6591 + ip6_cork_release(inet_sk(sk), inet6_sk(sk));
6593 +EXPORT_SYMBOL_GPL(ip6_flush_pending_frames);
6594 diff -ruN linux-3.10.27/net/netfilter/core.c linux-3.10.27-imq/net/netfilter/core.c
6595 --- linux-3.10.27/net/netfilter/core.c 2014-01-16 00:29:14.000000000 +0100
6596 +++ linux-3.10.27-imq/net/netfilter/core.c 2014-01-18 10:19:59.349342984 +0100
6597 @@ -191,9 +191,11 @@
6598 ret = NF_DROP_GETERR(verdict);
6601 - } else if ((verdict & NF_VERDICT_MASK) == NF_QUEUE) {
6602 + } else if ((verdict & NF_VERDICT_MASK) == NF_QUEUE ||
6603 + (verdict & NF_VERDICT_MASK) == NF_IMQ_QUEUE) {
6604 int err = nf_queue(skb, elem, pf, hook, indev, outdev, okfn,
6605 - verdict >> NF_VERDICT_QBITS);
6606 + verdict >> NF_VERDICT_QBITS,
6607 + verdict & NF_VERDICT_MASK);
6609 if (err == -ECANCELED)
6611 diff -ruN linux-3.10.27/net/netfilter/Kconfig linux-3.10.27-imq/net/netfilter/Kconfig
6612 --- linux-3.10.27/net/netfilter/Kconfig 2014-01-16 00:29:14.000000000 +0100
6613 +++ linux-3.10.27-imq/net/netfilter/Kconfig 2014-01-18 10:19:59.349342984 +0100
6614 @@ -641,6 +641,18 @@
6616 To compile it as a module, choose M here. If unsure, say N.
6618 +config NETFILTER_XT_TARGET_IMQ
6619 + tristate '"IMQ" target support'
6620 + depends on NETFILTER_XTABLES
6621 + depends on IP_NF_MANGLE || IP6_NF_MANGLE
6623 + default m if NETFILTER_ADVANCED=n
6625 + This option adds a `IMQ' target which is used to specify if and
6626 + to which imq device packets should get enqueued/dequeued.
6628 + To compile it as a module, choose M here. If unsure, say N.
6630 config NETFILTER_XT_TARGET_MARK
6631 tristate '"MARK" target support'
6632 depends on NETFILTER_ADVANCED
6633 diff -ruN linux-3.10.27/net/netfilter/Makefile linux-3.10.27-imq/net/netfilter/Makefile
6634 --- linux-3.10.27/net/netfilter/Makefile 2014-01-16 00:29:14.000000000 +0100
6635 +++ linux-3.10.27-imq/net/netfilter/Makefile 2014-01-18 10:19:59.349342984 +0100
6637 obj-$(CONFIG_NETFILTER_XT_TARGET_DSCP) += xt_DSCP.o
6638 obj-$(CONFIG_NETFILTER_XT_TARGET_HL) += xt_HL.o
6639 obj-$(CONFIG_NETFILTER_XT_TARGET_HMARK) += xt_HMARK.o
6640 +obj-$(CONFIG_NETFILTER_XT_TARGET_IMQ) += xt_IMQ.o
6641 obj-$(CONFIG_NETFILTER_XT_TARGET_LED) += xt_LED.o
6642 obj-$(CONFIG_NETFILTER_XT_TARGET_LOG) += xt_LOG.o
6643 obj-$(CONFIG_NETFILTER_XT_TARGET_NETMAP) += xt_NETMAP.o
6644 diff -ruN linux-3.10.27/net/netfilter/nf_internals.h linux-3.10.27-imq/net/netfilter/nf_internals.h
6645 --- linux-3.10.27/net/netfilter/nf_internals.h 2014-01-16 00:29:14.000000000 +0100
6646 +++ linux-3.10.27-imq/net/netfilter/nf_internals.h 2014-01-18 10:19:59.349342984 +0100
6648 struct net_device *indev,
6649 struct net_device *outdev,
6650 int (*okfn)(struct sk_buff *),
6651 - unsigned int queuenum);
6652 + unsigned int queuenum, unsigned int queuetype);
6653 extern int __init netfilter_queue_init(void);
6656 diff -ruN linux-3.10.27/net/netfilter/nf_queue.c linux-3.10.27-imq/net/netfilter/nf_queue.c
6657 --- linux-3.10.27/net/netfilter/nf_queue.c 2014-01-16 00:29:14.000000000 +0100
6658 +++ linux-3.10.27-imq/net/netfilter/nf_queue.c 2014-01-18 10:19:59.350342998 +0100
6661 static const struct nf_queue_handler __rcu *queue_handler __read_mostly;
6663 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
6664 +static const struct nf_queue_handler __rcu *queue_imq_handler __read_mostly;
6666 +void nf_register_queue_imq_handler(const struct nf_queue_handler *qh)
6668 + rcu_assign_pointer(queue_imq_handler, qh);
6670 +EXPORT_SYMBOL_GPL(nf_register_queue_imq_handler);
6672 +void nf_unregister_queue_imq_handler(void)
6674 + RCU_INIT_POINTER(queue_imq_handler, NULL);
6675 + synchronize_rcu();
6677 +EXPORT_SYMBOL_GPL(nf_unregister_queue_imq_handler);
6680 /* return EBUSY when somebody else is registered, return EEXIST if the
6681 * same handler is registered, return 0 in case of success. */
6682 void nf_register_queue_handler(const struct nf_queue_handler *qh)
6684 struct net_device *indev,
6685 struct net_device *outdev,
6686 int (*okfn)(struct sk_buff *),
6687 - unsigned int queuenum)
6688 + unsigned int queuenum,
6689 + unsigned int queuetype)
6691 int status = -ENOENT;
6692 struct nf_queue_entry *entry = NULL;
6693 @@ -115,7 +133,17 @@
6694 /* QUEUE == DROP if no one is waiting, to be safe. */
6697 - qh = rcu_dereference(queue_handler);
6698 + if (queuetype == NF_IMQ_QUEUE) {
6699 +#if defined(CONFIG_IMQ) || defined(CONFIG_IMQ_MODULE)
6700 + qh = rcu_dereference(queue_imq_handler);
6706 + qh = rcu_dereference(queue_handler);
6712 @@ -205,9 +233,11 @@
6716 + case NF_IMQ_QUEUE:
6717 err = nf_queue(skb, elem, entry->pf, entry->hook,
6718 entry->indev, entry->outdev, entry->okfn,
6719 - verdict >> NF_VERDICT_QBITS);
6720 + verdict >> NF_VERDICT_QBITS,
6721 + verdict & NF_VERDICT_MASK);
6723 if (err == -ECANCELED)
6725 diff -ruN linux-3.10.27/net/netfilter/xt_IMQ.c linux-3.10.27-imq/net/netfilter/xt_IMQ.c
6726 --- linux-3.10.27/net/netfilter/xt_IMQ.c 1970-01-01 01:00:00.000000000 +0100
6727 +++ linux-3.10.27-imq/net/netfilter/xt_IMQ.c 2014-01-18 10:19:59.350342998 +0100
6730 + * This target marks packets to be enqueued to an imq device
6732 +#include <linux/module.h>
6733 +#include <linux/skbuff.h>
6734 +#include <linux/netfilter/x_tables.h>
6735 +#include <linux/netfilter/xt_IMQ.h>
6736 +#include <linux/imq.h>
6738 +static unsigned int imq_target(struct sk_buff *pskb,
6739 + const struct xt_action_param *par)
6741 + const struct xt_imq_info *mr = par->targinfo;
6743 + pskb->imq_flags = (mr->todev & IMQ_F_IFMASK) | IMQ_F_ENQUEUE;
6745 + return XT_CONTINUE;
6748 +static int imq_checkentry(const struct xt_tgchk_param *par)
6750 + struct xt_imq_info *mr = par->targinfo;
6752 + if (mr->todev > IMQ_MAX_DEVS - 1) {
6753 + pr_warn("IMQ: invalid device specified, highest is %u\n",
6754 + IMQ_MAX_DEVS - 1);
6761 +static struct xt_target xt_imq_reg[] __read_mostly = {
6764 + .family = AF_INET,
6765 + .checkentry = imq_checkentry,
6766 + .target = imq_target,
6767 + .targetsize = sizeof(struct xt_imq_info),
6768 + .table = "mangle",
6773 + .family = AF_INET6,
6774 + .checkentry = imq_checkentry,
6775 + .target = imq_target,
6776 + .targetsize = sizeof(struct xt_imq_info),
6777 + .table = "mangle",
6782 +static int __init imq_init(void)
6784 + return xt_register_targets(xt_imq_reg, ARRAY_SIZE(xt_imq_reg));
6787 +static void __exit imq_fini(void)
6789 + xt_unregister_targets(xt_imq_reg, ARRAY_SIZE(xt_imq_reg));
6792 +module_init(imq_init);
6793 +module_exit(imq_fini);
6795 +MODULE_AUTHOR("http://www.linuximq.net");
6796 +MODULE_DESCRIPTION("Pseudo-driver for the intermediate queue device. See http://www.linuximq.net/ for more information.");
6797 +MODULE_LICENSE("GPL");
6798 +MODULE_ALIAS("ipt_IMQ");
6799 +MODULE_ALIAS("ip6t_IMQ");