]> git.ipfire.org Git - thirdparty/linux.git/blob - drivers/net/tun.c
projects / thirdparty / linux.git / blob
? search:


f8c5e2fd04dfa0a9543f322ae570b68cf993da43
[thirdparty/linux.git] / drivers / net / tun.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * TUN - Universal TUN/TAP device driver.
4 * Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com>
5 *
6 * $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $
7 */
8
9 /*
10 * Changes:
11 *
12 * Mike Kershaw <dragorn@kismetwireless.net> 2005/08/14
13 * Add TUNSETLINK ioctl to set the link encapsulation
14 *
15 * Mark Smith <markzzzsmith@yahoo.com.au>
16 * Use eth_random_addr() for tap MAC address.
17 *
18 * Harald Roelle <harald.roelle@ifi.lmu.de> 2004/04/20
19 * Fixes in packet dropping, queue length setting and queue wakeup.
20 * Increased default tx queue length.
21 * Added ethtool API.
22 * Minor cleanups
23 *
24 * Daniel Podlejski <underley@underley.eu.org>
25 * Modifications for 2.3.99-pre5 kernel.
26 */
27
28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
29
30 #define DRV_NAME "tun"
31 #define DRV_VERSION "1.6"
32 #define DRV_DESCRIPTION "Universal TUN/TAP device driver"
33 #define DRV_COPYRIGHT "(C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>"
34
35 #include <linux/module.h>
36 #include <linux/errno.h>
37 #include <linux/kernel.h>
38 #include <linux/sched/signal.h>
39 #include <linux/major.h>
40 #include <linux/slab.h>
41 #include <linux/poll.h>
42 #include <linux/fcntl.h>
43 #include <linux/init.h>
44 #include <linux/skbuff.h>
45 #include <linux/netdevice.h>
46 #include <linux/etherdevice.h>
47 #include <linux/miscdevice.h>
48 #include <linux/ethtool.h>
49 #include <linux/rtnetlink.h>
50 #include <linux/compat.h>
51 #include <linux/if.h>
52 #include <linux/if_arp.h>
53 #include <linux/if_ether.h>
54 #include <linux/if_tun.h>
55 #include <linux/if_vlan.h>
56 #include <linux/crc32.h>
57 #include <linux/math.h>
58 #include <linux/nsproxy.h>
59 #include <linux/virtio_net.h>
60 #include <linux/rcupdate.h>
61 #include <net/net_namespace.h>
62 #include <net/netns/generic.h>
63 #include <net/rtnetlink.h>
64 #include <net/sock.h>
65 #include <net/xdp.h>
66 #include <net/ip_tunnels.h>
67 #include <linux/seq_file.h>
68 #include <linux/uio.h>
69 #include <linux/skb_array.h>
70 #include <linux/bpf.h>
71 #include <linux/bpf_trace.h>
72 #include <linux/mutex.h>
73 #include <linux/ieee802154.h>
74 #include <uapi/linux/if_ltalk.h>
75 #include <uapi/linux/if_fddi.h>
76 #include <uapi/linux/if_hippi.h>
77 #include <uapi/linux/if_fc.h>
78 #include <net/ax25.h>
79 #include <net/rose.h>
80 #include <net/6lowpan.h>
81 #include <net/rps.h>
82
83 #include <linux/uaccess.h>
84 #include <linux/proc_fs.h>
85
86 #include "tun_vnet.h"
87
88 static void tun_default_link_ksettings(struct net_device *dev,
89 struct ethtool_link_ksettings *cmd);
90
91 #define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
92
93 /* TUN device flags */
94
95 /* IFF_ATTACH_QUEUE is never stored in device flags,
96 * overload it to mean fasync when stored there.
97 */
98 #define TUN_FASYNC IFF_ATTACH_QUEUE
99
100 #define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \
101 IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS)
102
103 #define GOODCOPY_LEN 128
104
105 #define FLT_EXACT_COUNT 8
106 struct tap_filter {
107 unsigned int count; /* Number of addrs. Zero means disabled */
108 u32 mask[2]; /* Mask of the hashed addrs */
109 unsigned char addr[FLT_EXACT_COUNT][ETH_ALEN];
110 };
111
112 /* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal
113 * to max number of VCPUs in guest. */
114 #define MAX_TAP_QUEUES 256
115 #define MAX_TAP_FLOWS 4096
116
117 #define TUN_FLOW_EXPIRE (3 * HZ)
118
119 /* A tun_file connects an open character device to a tuntap netdevice. It
120 * also contains all socket related structures (except sock_fprog and tap_filter)
121 * to serve as one transmit queue for tuntap device. The sock_fprog and
122 * tap_filter were kept in tun_struct since they were used for filtering for the
123 * netdevice not for a specific queue (at least I didn't see the requirement for
124 * this).
125 *
126 * RCU usage:
127 * The tun_file and tun_struct are loosely coupled, the pointer from one to the
128 * other can only be read while rcu_read_lock or rtnl_lock is held.
129 */
130 struct tun_file {
131 struct sock sk;
132 struct socket socket;
133 struct tun_struct __rcu *tun;
134 struct fasync_struct *fasync;
135 /* only used for fasnyc */
136 unsigned int flags;
137 union {
138 u16 queue_index;
139 unsigned int ifindex;
140 };
141 struct napi_struct napi;
142 bool napi_enabled;
143 bool napi_frags_enabled;
144 struct mutex napi_mutex; /* Protects access to the above napi */
145 struct list_head next;
146 struct tun_struct *detached;
147 struct ptr_ring tx_ring;
148 struct xdp_rxq_info xdp_rxq;
149 };
150
151 struct tun_page {
152 struct page *page;
153 int count;
154 };
155
156 struct tun_flow_entry {
157 struct hlist_node hash_link;
158 struct rcu_head rcu;
159 struct tun_struct *tun;
160
161 u32 rxhash;
162 u32 rps_rxhash;
163 int queue_index;
164 unsigned long updated ____cacheline_aligned_in_smp;
165 };
166
167 #define TUN_NUM_FLOW_ENTRIES 1024
168 #define TUN_MASK_FLOW_ENTRIES (TUN_NUM_FLOW_ENTRIES - 1)
169
170 struct tun_prog {
171 struct rcu_head rcu;
172 struct bpf_prog *prog;
173 };
174
175 /* Since the socket were moved to tun_file, to preserve the behavior of persist
176 * device, socket filter, sndbuf and vnet header size were restore when the
177 * file were attached to a persist device.
178 */
179 struct tun_struct {
180 struct tun_file __rcu *tfiles[MAX_TAP_QUEUES];
181 unsigned int numqueues;
182 unsigned int flags;
183 kuid_t owner;
184 kgid_t group;
185
186 struct net_device *dev;
187 netdev_features_t set_features;
188 #define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \
189 NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4)
190
191 int align;
192 int vnet_hdr_sz;
193 int sndbuf;
194 struct tap_filter txflt;
195 struct sock_fprog fprog;
196 /* protected by rtnl lock */
197 bool filter_attached;
198 u32 msg_enable;
199 spinlock_t lock;
200 struct hlist_head flows[TUN_NUM_FLOW_ENTRIES];
201 struct timer_list flow_gc_timer;
202 unsigned long ageing_time;
203 unsigned int numdisabled;
204 struct list_head disabled;
205 void *security;
206 u32 flow_count;
207 u32 rx_batched;
208 atomic_long_t rx_frame_errors;
209 struct bpf_prog __rcu *xdp_prog;
210 struct tun_prog __rcu *steering_prog;
211 struct tun_prog __rcu *filter_prog;
212 struct ethtool_link_ksettings link_ksettings;
213 /* init args */
214 struct file *file;
215 struct ifreq *ifr;
216 };
217
218 struct veth {
219 __be16 h_vlan_proto;
220 __be16 h_vlan_TCI;
221 };
222
223 static void tun_flow_init(struct tun_struct *tun);
224 static void tun_flow_uninit(struct tun_struct *tun);
225
226 static int tun_napi_receive(struct napi_struct *napi, int budget)
227 {
228 struct tun_file *tfile = container_of(napi, struct tun_file, napi);
229 struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
230 struct sk_buff_head process_queue;
231 struct sk_buff *skb;
232 int received = 0;
233
234 __skb_queue_head_init(&process_queue);
235
236 spin_lock(&queue->lock);
237 skb_queue_splice_tail_init(queue, &process_queue);
238 spin_unlock(&queue->lock);
239
240 while (received < budget && (skb = __skb_dequeue(&process_queue))) {
241 napi_gro_receive(napi, skb);
242 ++received;
243 }
244
245 if (!skb_queue_empty(&process_queue)) {
246 spin_lock(&queue->lock);
247 skb_queue_splice(&process_queue, queue);
248 spin_unlock(&queue->lock);
249 }
250
251 return received;
252 }
253
254 static int tun_napi_poll(struct napi_struct *napi, int budget)
255 {
256 unsigned int received;
257
258 received = tun_napi_receive(napi, budget);
259
260 if (received < budget)
261 napi_complete_done(napi, received);
262
263 return received;
264 }
265
266 static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,
267 bool napi_en, bool napi_frags)
268 {
269 tfile->napi_enabled = napi_en;
270 tfile->napi_frags_enabled = napi_en && napi_frags;
271 if (napi_en) {
272 netif_napi_add_tx(tun->dev, &tfile->napi, tun_napi_poll);
273 napi_enable(&tfile->napi);
274 }
275 }
276
277 static void tun_napi_enable(struct tun_file *tfile)
278 {
279 if (tfile->napi_enabled)
280 napi_enable(&tfile->napi);
281 }
282
283 static void tun_napi_disable(struct tun_file *tfile)
284 {
285 if (tfile->napi_enabled)
286 napi_disable(&tfile->napi);
287 }
288
289 static void tun_napi_del(struct tun_file *tfile)
290 {
291 if (tfile->napi_enabled)
292 netif_napi_del(&tfile->napi);
293 }
294
295 static bool tun_napi_frags_enabled(const struct tun_file *tfile)
296 {
297 return tfile->napi_frags_enabled;
298 }
299
300 static inline u32 tun_hashfn(u32 rxhash)
301 {
302 return rxhash & TUN_MASK_FLOW_ENTRIES;
303 }
304
305 static struct tun_flow_entry *tun_flow_find(struct hlist_head *head, u32 rxhash)
306 {
307 struct tun_flow_entry *e;
308
309 hlist_for_each_entry_rcu(e, head, hash_link) {
310 if (e->rxhash == rxhash)
311 return e;
312 }
313 return NULL;
314 }
315
316 static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun,
317 struct hlist_head *head,
318 u32 rxhash, u16 queue_index)
319 {
320 struct tun_flow_entry *e = kmalloc(sizeof(*e), GFP_ATOMIC);
321
322 if (e) {
323 netif_info(tun, tx_queued, tun->dev,
324 "create flow: hash %u index %u\n",
325 rxhash, queue_index);
326 e->updated = jiffies;
327 e->rxhash = rxhash;
328 e->rps_rxhash = 0;
329 e->queue_index = queue_index;
330 e->tun = tun;
331 hlist_add_head_rcu(&e->hash_link, head);
332 ++tun->flow_count;
333 }
334 return e;
335 }
336
337 static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e)
338 {
339 netif_info(tun, tx_queued, tun->dev, "delete flow: hash %u index %u\n",
340 e->rxhash, e->queue_index);
341 hlist_del_rcu(&e->hash_link);
342 kfree_rcu(e, rcu);
343 --tun->flow_count;
344 }
345
346 static void tun_flow_flush(struct tun_struct *tun)
347 {
348 int i;
349
350 spin_lock_bh(&tun->lock);
351 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
352 struct tun_flow_entry *e;
353 struct hlist_node *n;
354
355 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link)
356 tun_flow_delete(tun, e);
357 }
358 spin_unlock_bh(&tun->lock);
359 }
360
361 static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index)
362 {
363 int i;
364
365 spin_lock_bh(&tun->lock);
366 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
367 struct tun_flow_entry *e;
368 struct hlist_node *n;
369
370 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
371 if (e->queue_index == queue_index)
372 tun_flow_delete(tun, e);
373 }
374 }
375 spin_unlock_bh(&tun->lock);
376 }
377
378 static void tun_flow_cleanup(struct timer_list *t)
379 {
380 struct tun_struct *tun = timer_container_of(tun, t, flow_gc_timer);
381 unsigned long delay = tun->ageing_time;
382 unsigned long next_timer = jiffies + delay;
383 unsigned long count = 0;
384 int i;
385
386 spin_lock(&tun->lock);
387 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
388 struct tun_flow_entry *e;
389 struct hlist_node *n;
390
391 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
392 unsigned long this_timer;
393
394 this_timer = e->updated + delay;
395 if (time_before_eq(this_timer, jiffies)) {
396 tun_flow_delete(tun, e);
397 continue;
398 }
399 count++;
400 if (time_before(this_timer, next_timer))
401 next_timer = this_timer;
402 }
403 }
404
405 if (count)
406 mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer));
407 spin_unlock(&tun->lock);
408 }
409
410 static void tun_flow_update(struct tun_struct *tun, u32 rxhash,
411 struct tun_file *tfile)
412 {
413 struct hlist_head *head;
414 struct tun_flow_entry *e;
415 unsigned long delay = tun->ageing_time;
416 u16 queue_index = tfile->queue_index;
417
418 head = &tun->flows[tun_hashfn(rxhash)];
419
420 rcu_read_lock();
421
422 e = tun_flow_find(head, rxhash);
423 if (likely(e)) {
424 /* TODO: keep queueing to old queue until it's empty? */
425 if (READ_ONCE(e->queue_index) != queue_index)
426 WRITE_ONCE(e->queue_index, queue_index);
427 if (e->updated != jiffies)
428 e->updated = jiffies;
429 sock_rps_record_flow_hash(e->rps_rxhash);
430 } else {
431 spin_lock_bh(&tun->lock);
432 if (!tun_flow_find(head, rxhash) &&
433 tun->flow_count < MAX_TAP_FLOWS)
434 tun_flow_create(tun, head, rxhash, queue_index);
435
436 if (!timer_pending(&tun->flow_gc_timer))
437 mod_timer(&tun->flow_gc_timer,
438 round_jiffies_up(jiffies + delay));
439 spin_unlock_bh(&tun->lock);
440 }
441
442 rcu_read_unlock();
443 }
444
445 /* Save the hash received in the stack receive path and update the
446 * flow_hash table accordingly.
447 */
448 static inline void tun_flow_save_rps_rxhash(struct tun_flow_entry *e, u32 hash)
449 {
450 if (unlikely(e->rps_rxhash != hash))
451 e->rps_rxhash = hash;
452 }
453
454 /* We try to identify a flow through its rxhash. The reason that
455 * we do not check rxq no. is because some cards(e.g 82599), chooses
456 * the rxq based on the txq where the last packet of the flow comes. As
457 * the userspace application move between processors, we may get a
458 * different rxq no. here.
459 */
460 static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb)
461 {
462 struct tun_flow_entry *e;
463 u32 txq, numqueues;
464
465 numqueues = READ_ONCE(tun->numqueues);
466
467 txq = __skb_get_hash_symmetric(skb);
468 e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq);
469 if (e) {
470 tun_flow_save_rps_rxhash(e, txq);
471 txq = e->queue_index;
472 } else {
473 txq = reciprocal_scale(txq, numqueues);
474 }
475
476 return txq;
477 }
478
479 static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb)
480 {
481 struct tun_prog *prog;
482 u32 numqueues;
483 u16 ret = 0;
484
485 numqueues = READ_ONCE(tun->numqueues);
486 if (!numqueues)
487 return 0;
488
489 prog = rcu_dereference(tun->steering_prog);
490 if (prog)
491 ret = bpf_prog_run_clear_cb(prog->prog, skb);
492
493 return ret % numqueues;
494 }
495
496 static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb,
497 struct net_device *sb_dev)
498 {
499 struct tun_struct *tun = netdev_priv(dev);
500 u16 ret;
501
502 rcu_read_lock();
503 if (rcu_dereference(tun->steering_prog))
504 ret = tun_ebpf_select_queue(tun, skb);
505 else
506 ret = tun_automq_select_queue(tun, skb);
507 rcu_read_unlock();
508
509 return ret;
510 }
511
512 static inline bool tun_not_capable(struct tun_struct *tun)
513 {
514 const struct cred *cred = current_cred();
515 struct net *net = dev_net(tun->dev);
516
517 return ((uid_valid(tun->owner) && !uid_eq(cred->euid, tun->owner)) ||
518 (gid_valid(tun->group) && !in_egroup_p(tun->group))) &&
519 !ns_capable(net->user_ns, CAP_NET_ADMIN);
520 }
521
522 static void tun_set_real_num_queues(struct tun_struct *tun)
523 {
524 netif_set_real_num_tx_queues(tun->dev, tun->numqueues);
525 netif_set_real_num_rx_queues(tun->dev, tun->numqueues);
526 }
527
528 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile)
529 {
530 tfile->detached = tun;
531 list_add_tail(&tfile->next, &tun->disabled);
532 ++tun->numdisabled;
533 }
534
535 static struct tun_struct *tun_enable_queue(struct tun_file *tfile)
536 {
537 struct tun_struct *tun = tfile->detached;
538
539 tfile->detached = NULL;
540 list_del_init(&tfile->next);
541 --tun->numdisabled;
542 return tun;
543 }
544
545 void tun_ptr_free(void *ptr)
546 {
547 if (!ptr)
548 return;
549 if (tun_is_xdp_frame(ptr)) {
550 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
551
552 xdp_return_frame(xdpf);
553 } else {
554 __skb_array_destroy_skb(ptr);
555 }
556 }
557 EXPORT_SYMBOL_GPL(tun_ptr_free);
558
559 static void tun_queue_purge(struct tun_file *tfile)
560 {
561 void *ptr;
562
563 while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL)
564 tun_ptr_free(ptr);
565
566 skb_queue_purge(&tfile->sk.sk_write_queue);
567 skb_queue_purge(&tfile->sk.sk_error_queue);
568 }
569
570 static void __tun_detach(struct tun_file *tfile, bool clean)
571 {
572 struct tun_file *ntfile;
573 struct tun_struct *tun;
574
575 tun = rtnl_dereference(tfile->tun);
576
577 if (tun && clean) {
578 if (!tfile->detached)
579 tun_napi_disable(tfile);
580 tun_napi_del(tfile);
581 }
582
583 if (tun && !tfile->detached) {
584 u16 index = tfile->queue_index;
585 BUG_ON(index >= tun->numqueues);
586
587 rcu_assign_pointer(tun->tfiles[index],
588 tun->tfiles[tun->numqueues - 1]);
589 ntfile = rtnl_dereference(tun->tfiles[index]);
590 ntfile->queue_index = index;
591 ntfile->xdp_rxq.queue_index = index;
592 rcu_assign_pointer(tun->tfiles[tun->numqueues - 1],
593 NULL);
594
595 --tun->numqueues;
596 if (clean) {
597 RCU_INIT_POINTER(tfile->tun, NULL);
598 sock_put(&tfile->sk);
599 } else {
600 tun_disable_queue(tun, tfile);
601 tun_napi_disable(tfile);
602 }
603
604 synchronize_net();
605 tun_flow_delete_by_queue(tun, tun->numqueues + 1);
606 /* Drop read queue */
607 tun_queue_purge(tfile);
608 tun_set_real_num_queues(tun);
609 } else if (tfile->detached && clean) {
610 tun = tun_enable_queue(tfile);
611 sock_put(&tfile->sk);
612 }
613
614 if (clean) {
615 if (tun && tun->numqueues == 0 && tun->numdisabled == 0) {
616 netif_carrier_off(tun->dev);
617
618 if (!(tun->flags & IFF_PERSIST) &&
619 tun->dev->reg_state == NETREG_REGISTERED)
620 unregister_netdevice(tun->dev);
621 }
622 if (tun)
623 xdp_rxq_info_unreg(&tfile->xdp_rxq);
624 ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free);
625 }
626 }
627
628 static void tun_detach(struct tun_file *tfile, bool clean)
629 {
630 struct tun_struct *tun;
631 struct net_device *dev;
632
633 rtnl_lock();
634 tun = rtnl_dereference(tfile->tun);
635 dev = tun ? tun->dev : NULL;
636 __tun_detach(tfile, clean);
637 if (dev)
638 netdev_state_change(dev);
639 rtnl_unlock();
640
641 if (clean)
642 sock_put(&tfile->sk);
643 }
644
645 static void tun_detach_all(struct net_device *dev)
646 {
647 struct tun_struct *tun = netdev_priv(dev);
648 struct tun_file *tfile, *tmp;
649 int i, n = tun->numqueues;
650
651 for (i = 0; i < n; i++) {
652 tfile = rtnl_dereference(tun->tfiles[i]);
653 BUG_ON(!tfile);
654 tun_napi_disable(tfile);
655 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
656 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
657 RCU_INIT_POINTER(tfile->tun, NULL);
658 --tun->numqueues;
659 }
660 list_for_each_entry(tfile, &tun->disabled, next) {
661 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
662 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
663 RCU_INIT_POINTER(tfile->tun, NULL);
664 }
665 BUG_ON(tun->numqueues != 0);
666
667 synchronize_net();
668 for (i = 0; i < n; i++) {
669 tfile = rtnl_dereference(tun->tfiles[i]);
670 tun_napi_del(tfile);
671 /* Drop read queue */
672 tun_queue_purge(tfile);
673 xdp_rxq_info_unreg(&tfile->xdp_rxq);
674 sock_put(&tfile->sk);
675 }
676 list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {
677 tun_napi_del(tfile);
678 tun_enable_queue(tfile);
679 tun_queue_purge(tfile);
680 xdp_rxq_info_unreg(&tfile->xdp_rxq);
681 sock_put(&tfile->sk);
682 }
683 BUG_ON(tun->numdisabled != 0);
684
685 if (tun->flags & IFF_PERSIST)
686 module_put(THIS_MODULE);
687 }
688
689 static int tun_attach(struct tun_struct *tun, struct file *file,
690 bool skip_filter, bool napi, bool napi_frags,
691 bool publish_tun)
692 {
693 struct tun_file *tfile = file->private_data;
694 struct net_device *dev = tun->dev;
695 int err;
696
697 err = security_tun_dev_attach(tfile->socket.sk, tun->security);
698 if (err < 0)
699 goto out;
700
701 err = -EINVAL;
702 if (rtnl_dereference(tfile->tun) && !tfile->detached)
703 goto out;
704
705 err = -EBUSY;
706 if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1)
707 goto out;
708
709 err = -E2BIG;
710 if (!tfile->detached &&
711 tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES)
712 goto out;
713
714 err = 0;
715
716 /* Re-attach the filter to persist device */
717 if (!skip_filter && (tun->filter_attached == true)) {
718 lock_sock(tfile->socket.sk);
719 err = sk_attach_filter(&tun->fprog, tfile->socket.sk);
720 release_sock(tfile->socket.sk);
721 if (!err)
722 goto out;
723 }
724
725 if (!tfile->detached &&
726 ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len,
727 GFP_KERNEL, tun_ptr_free)) {
728 err = -ENOMEM;
729 goto out;
730 }
731
732 tfile->queue_index = tun->numqueues;
733 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;
734
735 if (tfile->detached) {
736 /* Re-attach detached tfile, updating XDP queue_index */
737 WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq));
738
739 if (tfile->xdp_rxq.queue_index != tfile->queue_index)
740 tfile->xdp_rxq.queue_index = tfile->queue_index;
741 } else {
742 /* Setup XDP RX-queue info, for new tfile getting attached */
743 err = xdp_rxq_info_reg(&tfile->xdp_rxq,
744 tun->dev, tfile->queue_index, 0);
745 if (err < 0)
746 goto out;
747 err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq,
748 MEM_TYPE_PAGE_SHARED, NULL);
749 if (err < 0) {
750 xdp_rxq_info_unreg(&tfile->xdp_rxq);
751 goto out;
752 }
753 err = 0;
754 }
755
756 if (tfile->detached) {
757 tun_enable_queue(tfile);
758 tun_napi_enable(tfile);
759 } else {
760 sock_hold(&tfile->sk);
761 tun_napi_init(tun, tfile, napi, napi_frags);
762 }
763
764 if (rtnl_dereference(tun->xdp_prog))
765 sock_set_flag(&tfile->sk, SOCK_XDP);
766
767 /* device is allowed to go away first, so no need to hold extra
768 * refcnt.
769 */
770
771 /* Publish tfile->tun and tun->tfiles only after we've fully
772 * initialized tfile; otherwise we risk using half-initialized
773 * object.
774 */
775 if (publish_tun)
776 rcu_assign_pointer(tfile->tun, tun);
777 rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);
778 tun->numqueues++;
779 tun_set_real_num_queues(tun);
780 out:
781 return err;
782 }
783
784 static struct tun_struct *tun_get(struct tun_file *tfile)
785 {
786 struct tun_struct *tun;
787
788 rcu_read_lock();
789 tun = rcu_dereference(tfile->tun);
790 if (tun)
791 dev_hold(tun->dev);
792 rcu_read_unlock();
793
794 return tun;
795 }
796
797 static void tun_put(struct tun_struct *tun)
798 {
799 dev_put(tun->dev);
800 }
801
802 /* TAP filtering */
803 static void addr_hash_set(u32 *mask, const u8 *addr)
804 {
805 int n = ether_crc(ETH_ALEN, addr) >> 26;
806 mask[n >> 5] |= (1 << (n & 31));
807 }
808
809 static unsigned int addr_hash_test(const u32 *mask, const u8 *addr)
810 {
811 int n = ether_crc(ETH_ALEN, addr) >> 26;
812 return mask[n >> 5] & (1 << (n & 31));
813 }
814
815 static int update_filter(struct tap_filter *filter, void __user *arg)
816 {
817 struct { u8 u[ETH_ALEN]; } *addr;
818 struct tun_filter uf;
819 int err, alen, n, nexact;
820
821 if (copy_from_user(&uf, arg, sizeof(uf)))
822 return -EFAULT;
823
824 if (!uf.count) {
825 /* Disabled */
826 filter->count = 0;
827 return 0;
828 }
829
830 alen = ETH_ALEN * uf.count;
831 addr = memdup_user(arg + sizeof(uf), alen);
832 if (IS_ERR(addr))
833 return PTR_ERR(addr);
834
835 /* The filter is updated without holding any locks. Which is
836 * perfectly safe. We disable it first and in the worst
837 * case we'll accept a few undesired packets. */
838 filter->count = 0;
839 wmb();
840
841 /* Use first set of addresses as an exact filter */
842 for (n = 0; n < uf.count && n < FLT_EXACT_COUNT; n++)
843 memcpy(filter->addr[n], addr[n].u, ETH_ALEN);
844
845 nexact = n;
846
847 /* Remaining multicast addresses are hashed,
848 * unicast will leave the filter disabled. */
849 memset(filter->mask, 0, sizeof(filter->mask));
850 for (; n < uf.count; n++) {
851 if (!is_multicast_ether_addr(addr[n].u)) {
852 err = 0; /* no filter */
853 goto free_addr;
854 }
855 addr_hash_set(filter->mask, addr[n].u);
856 }
857
858 /* For ALLMULTI just set the mask to all ones.
859 * This overrides the mask populated above. */
860 if ((uf.flags & TUN_FLT_ALLMULTI))
861 memset(filter->mask, ~0, sizeof(filter->mask));
862
863 /* Now enable the filter */
864 wmb();
865 filter->count = nexact;
866
867 /* Return the number of exact filters */
868 err = nexact;
869 free_addr:
870 kfree(addr);
871 return err;
872 }
873
874 /* Returns: 0 - drop, !=0 - accept */
875 static int run_filter(struct tap_filter *filter, const struct sk_buff *skb)
876 {
877 /* Cannot use eth_hdr(skb) here because skb_mac_hdr() is incorrect
878 * at this point. */
879 struct ethhdr *eh = (struct ethhdr *) skb->data;
880 int i;
881
882 /* Exact match */
883 for (i = 0; i < filter->count; i++)
884 if (ether_addr_equal(eh->h_dest, filter->addr[i]))
885 return 1;
886
887 /* Inexact match (multicast only) */
888 if (is_multicast_ether_addr(eh->h_dest))
889 return addr_hash_test(filter->mask, eh->h_dest);
890
891 return 0;
892 }
893
894 /*
895 * Checks whether the packet is accepted or not.
896 * Returns: 0 - drop, !=0 - accept
897 */
898 static int check_filter(struct tap_filter *filter, const struct sk_buff *skb)
899 {
900 if (!filter->count)
901 return 1;
902
903 return run_filter(filter, skb);
904 }
905
906 /* Network device part of the driver */
907
908 static const struct ethtool_ops tun_ethtool_ops;
909
910 static int tun_net_init(struct net_device *dev)
911 {
912 struct tun_struct *tun = netdev_priv(dev);
913 struct ifreq *ifr = tun->ifr;
914 int err;
915
916 spin_lock_init(&tun->lock);
917
918 err = security_tun_dev_alloc_security(&tun->security);
919 if (err < 0)
920 return err;
921
922 tun_flow_init(tun);
923
924 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
925 dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST |
926 TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX |
927 NETIF_F_HW_VLAN_STAG_TX;
928 dev->features = dev->hw_features;
929 dev->vlan_features = dev->features &
930 ~(NETIF_F_HW_VLAN_CTAG_TX |
931 NETIF_F_HW_VLAN_STAG_TX);
932 dev->lltx = true;
933
934 tun->flags = (tun->flags & ~TUN_FEATURES) |
935 (ifr->ifr_flags & TUN_FEATURES);
936
937 INIT_LIST_HEAD(&tun->disabled);
938 err = tun_attach(tun, tun->file, false, ifr->ifr_flags & IFF_NAPI,
939 ifr->ifr_flags & IFF_NAPI_FRAGS, false);
940 if (err < 0) {
941 tun_flow_uninit(tun);
942 security_tun_dev_free_security(tun->security);
943 return err;
944 }
945 return 0;
946 }
947
948 /* Net device detach from fd. */
949 static void tun_net_uninit(struct net_device *dev)
950 {
951 tun_detach_all(dev);
952 }
953
954 /* Net device open. */
955 static int tun_net_open(struct net_device *dev)
956 {
957 netif_tx_start_all_queues(dev);
958
959 return 0;
960 }
961
962 /* Net device close. */
963 static int tun_net_close(struct net_device *dev)
964 {
965 netif_tx_stop_all_queues(dev);
966 return 0;
967 }
968
969 /* Net device start xmit */
970 static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb)
971 {
972 #ifdef CONFIG_RPS
973 if (tun->numqueues == 1 && static_branch_unlikely(&rps_needed)) {
974 /* Select queue was not called for the skbuff, so we extract the
975 * RPS hash and save it into the flow_table here.
976 */
977 struct tun_flow_entry *e;
978 __u32 rxhash;
979
980 rxhash = __skb_get_hash_symmetric(skb);
981 e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)], rxhash);
982 if (e)
983 tun_flow_save_rps_rxhash(e, rxhash);
984 }
985 #endif
986 }
987
988 static unsigned int run_ebpf_filter(struct tun_struct *tun,
989 struct sk_buff *skb,
990 int len)
991 {
992 struct tun_prog *prog = rcu_dereference(tun->filter_prog);
993
994 if (prog)
995 len = bpf_prog_run_clear_cb(prog->prog, skb);
996
997 return len;
998 }
999
1000 /* Net device start xmit */
1001 static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
1002 {
1003 struct tun_struct *tun = netdev_priv(dev);
1004 enum skb_drop_reason drop_reason;
1005 int txq = skb->queue_mapping;
1006 struct netdev_queue *queue;
1007 struct tun_file *tfile;
1008 int len = skb->len;
1009
1010 rcu_read_lock();
1011 tfile = rcu_dereference(tun->tfiles[txq]);
1012
1013 /* Drop packet if interface is not attached */
1014 if (!tfile) {
1015 drop_reason = SKB_DROP_REASON_DEV_READY;
1016 goto drop;
1017 }
1018
1019 if (!rcu_dereference(tun->steering_prog))
1020 tun_automq_xmit(tun, skb);
1021
1022 netif_info(tun, tx_queued, tun->dev, "%s %d\n", __func__, skb->len);
1023
1024 /* Drop if the filter does not like it.
1025 * This is a noop if the filter is disabled.
1026 * Filter can be enabled only for the TAP devices. */
1027 if (!check_filter(&tun->txflt, skb)) {
1028 drop_reason = SKB_DROP_REASON_TAP_TXFILTER;
1029 goto drop;
1030 }
1031
1032 if (tfile->socket.sk->sk_filter &&
1033 sk_filter(tfile->socket.sk, skb)) {
1034 drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
1035 goto drop;
1036 }
1037
1038 len = run_ebpf_filter(tun, skb, len);
1039 if (len == 0) {
1040 drop_reason = SKB_DROP_REASON_TAP_FILTER;
1041 goto drop;
1042 }
1043
1044 if (pskb_trim(skb, len)) {
1045 drop_reason = SKB_DROP_REASON_NOMEM;
1046 goto drop;
1047 }
1048
1049 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) {
1050 drop_reason = SKB_DROP_REASON_SKB_UCOPY_FAULT;
1051 goto drop;
1052 }
1053
1054 skb_tx_timestamp(skb);
1055
1056 /* Orphan the skb - required as we might hang on to it
1057 * for indefinite time.
1058 */
1059 skb_orphan(skb);
1060
1061 nf_reset_ct(skb);
1062
1063 if (ptr_ring_produce(&tfile->tx_ring, skb)) {
1064 drop_reason = SKB_DROP_REASON_FULL_RING;
1065 goto drop;
1066 }
1067
1068 /* dev->lltx requires to do our own update of trans_start */
1069 queue = netdev_get_tx_queue(dev, txq);
1070 txq_trans_cond_update(queue);
1071
1072 /* Notify and wake up reader process */
1073 if (tfile->flags & TUN_FASYNC)
1074 kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
1075 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
1076
1077 rcu_read_unlock();
1078 return NETDEV_TX_OK;
1079
1080 drop:
1081 dev_core_stats_tx_dropped_inc(dev);
1082 skb_tx_error(skb);
1083 kfree_skb_reason(skb, drop_reason);
1084 rcu_read_unlock();
1085 return NET_XMIT_DROP;
1086 }
1087
1088 static void tun_net_mclist(struct net_device *dev)
1089 {
1090 /*
1091 * This callback is supposed to deal with mc filter in
1092 * _rx_ path and has nothing to do with the _tx_ path.
1093 * In rx path we always accept everything userspace gives us.
1094 */
1095 }
1096
1097 static netdev_features_t tun_net_fix_features(struct net_device *dev,
1098 netdev_features_t features)
1099 {
1100 struct tun_struct *tun = netdev_priv(dev);
1101
1102 return (features & tun->set_features) | (features & ~TUN_USER_FEATURES);
1103 }
1104
1105 static void tun_set_headroom(struct net_device *dev, int new_hr)
1106 {
1107 struct tun_struct *tun = netdev_priv(dev);
1108
1109 if (new_hr < NET_SKB_PAD)
1110 new_hr = NET_SKB_PAD;
1111
1112 tun->align = new_hr;
1113 }
1114
1115 static void
1116 tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
1117 {
1118 struct tun_struct *tun = netdev_priv(dev);
1119
1120 dev_get_tstats64(dev, stats);
1121
1122 stats->rx_frame_errors +=
1123 (unsigned long)atomic_long_read(&tun->rx_frame_errors);
1124 }
1125
1126 static int tun_xdp_set(struct net_device *dev, struct bpf_prog *prog,
1127 struct netlink_ext_ack *extack)
1128 {
1129 struct tun_struct *tun = netdev_priv(dev);
1130 struct tun_file *tfile;
1131 struct bpf_prog *old_prog;
1132 int i;
1133
1134 old_prog = rtnl_dereference(tun->xdp_prog);
1135 rcu_assign_pointer(tun->xdp_prog, prog);
1136 if (old_prog)
1137 bpf_prog_put(old_prog);
1138
1139 for (i = 0; i < tun->numqueues; i++) {
1140 tfile = rtnl_dereference(tun->tfiles[i]);
1141 if (prog)
1142 sock_set_flag(&tfile->sk, SOCK_XDP);
1143 else
1144 sock_reset_flag(&tfile->sk, SOCK_XDP);
1145 }
1146 list_for_each_entry(tfile, &tun->disabled, next) {
1147 if (prog)
1148 sock_set_flag(&tfile->sk, SOCK_XDP);
1149 else
1150 sock_reset_flag(&tfile->sk, SOCK_XDP);
1151 }
1152
1153 return 0;
1154 }
1155
1156 static int tun_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1157 {
1158 switch (xdp->command) {
1159 case XDP_SETUP_PROG:
1160 return tun_xdp_set(dev, xdp->prog, xdp->extack);
1161 default:
1162 return -EINVAL;
1163 }
1164 }
1165
1166 static int tun_net_change_carrier(struct net_device *dev, bool new_carrier)
1167 {
1168 if (new_carrier) {
1169 struct tun_struct *tun = netdev_priv(dev);
1170
1171 if (!tun->numqueues)
1172 return -EPERM;
1173
1174 netif_carrier_on(dev);
1175 } else {
1176 netif_carrier_off(dev);
1177 }
1178 return 0;
1179 }
1180
1181 static const struct net_device_ops tun_netdev_ops = {
1182 .ndo_init = tun_net_init,
1183 .ndo_uninit = tun_net_uninit,
1184 .ndo_open = tun_net_open,
1185 .ndo_stop = tun_net_close,
1186 .ndo_start_xmit = tun_net_xmit,
1187 .ndo_fix_features = tun_net_fix_features,
1188 .ndo_select_queue = tun_select_queue,
1189 .ndo_set_rx_headroom = tun_set_headroom,
1190 .ndo_get_stats64 = tun_net_get_stats64,
1191 .ndo_change_carrier = tun_net_change_carrier,
1192 };
1193
1194 static void __tun_xdp_flush_tfile(struct tun_file *tfile)
1195 {
1196 /* Notify and wake up reader process */
1197 if (tfile->flags & TUN_FASYNC)
1198 kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
1199 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
1200 }
1201
1202 static int tun_xdp_xmit(struct net_device *dev, int n,
1203 struct xdp_frame **frames, u32 flags)
1204 {
1205 struct tun_struct *tun = netdev_priv(dev);
1206 struct tun_file *tfile;
1207 u32 numqueues;
1208 int nxmit = 0;
1209 int i;
1210
1211 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
1212 return -EINVAL;
1213
1214 rcu_read_lock();
1215
1216 resample:
1217 numqueues = READ_ONCE(tun->numqueues);
1218 if (!numqueues) {
1219 rcu_read_unlock();
1220 return -ENXIO; /* Caller will free/return all frames */
1221 }
1222
1223 tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
1224 numqueues]);
1225 if (unlikely(!tfile))
1226 goto resample;
1227
1228 spin_lock(&tfile->tx_ring.producer_lock);
1229 for (i = 0; i < n; i++) {
1230 struct xdp_frame *xdp = frames[i];
1231 /* Encode the XDP flag into lowest bit for consumer to differ
1232 * XDP buffer from sk_buff.
1233 */
1234 void *frame = tun_xdp_to_ptr(xdp);
1235
1236 if (__ptr_ring_produce(&tfile->tx_ring, frame)) {
1237 dev_core_stats_tx_dropped_inc(dev);
1238 break;
1239 }
1240 nxmit++;
1241 }
1242 spin_unlock(&tfile->tx_ring.producer_lock);
1243
1244 if (flags & XDP_XMIT_FLUSH)
1245 __tun_xdp_flush_tfile(tfile);
1246
1247 rcu_read_unlock();
1248 return nxmit;
1249 }
1250
1251 static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp)
1252 {
1253 struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp);
1254 int nxmit;
1255
1256 if (unlikely(!frame))
1257 return -EOVERFLOW;
1258
1259 nxmit = tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH);
1260 if (!nxmit)
1261 xdp_return_frame_rx_napi(frame);
1262 return nxmit;
1263 }
1264
1265 static const struct net_device_ops tap_netdev_ops = {
1266 .ndo_init = tun_net_init,
1267 .ndo_uninit = tun_net_uninit,
1268 .ndo_open = tun_net_open,
1269 .ndo_stop = tun_net_close,
1270 .ndo_start_xmit = tun_net_xmit,
1271 .ndo_fix_features = tun_net_fix_features,
1272 .ndo_set_rx_mode = tun_net_mclist,
1273 .ndo_set_mac_address = eth_mac_addr,
1274 .ndo_validate_addr = eth_validate_addr,
1275 .ndo_select_queue = tun_select_queue,
1276 .ndo_features_check = passthru_features_check,
1277 .ndo_set_rx_headroom = tun_set_headroom,
1278 .ndo_bpf = tun_xdp,
1279 .ndo_xdp_xmit = tun_xdp_xmit,
1280 .ndo_change_carrier = tun_net_change_carrier,
1281 };
1282
1283 static void tun_flow_init(struct tun_struct *tun)
1284 {
1285 int i;
1286
1287 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++)
1288 INIT_HLIST_HEAD(&tun->flows[i]);
1289
1290 tun->ageing_time = TUN_FLOW_EXPIRE;
1291 timer_setup(&tun->flow_gc_timer, tun_flow_cleanup, 0);
1292 mod_timer(&tun->flow_gc_timer,
1293 round_jiffies_up(jiffies + tun->ageing_time));
1294 }
1295
1296 static void tun_flow_uninit(struct tun_struct *tun)
1297 {
1298 timer_delete_sync(&tun->flow_gc_timer);
1299 tun_flow_flush(tun);
1300 }
1301
1302 #define MIN_MTU 68
1303 #define MAX_MTU 65535
1304
1305 /* Initialize net device. */
1306 static void tun_net_initialize(struct net_device *dev)
1307 {
1308 struct tun_struct *tun = netdev_priv(dev);
1309
1310 switch (tun->flags & TUN_TYPE_MASK) {
1311 case IFF_TUN:
1312 dev->netdev_ops = &tun_netdev_ops;
1313 dev->header_ops = &ip_tunnel_header_ops;
1314
1315 /* Point-to-Point TUN Device */
1316 dev->hard_header_len = 0;
1317 dev->addr_len = 0;
1318 dev->mtu = 1500;
1319
1320 /* Zero header length */
1321 dev->type = ARPHRD_NONE;
1322 dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST;
1323 break;
1324
1325 case IFF_TAP:
1326 dev->netdev_ops = &tap_netdev_ops;
1327 /* Ethernet TAP Device */
1328 ether_setup(dev);
1329 dev->priv_flags &= ~IFF_TX_SKB_SHARING;
1330 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
1331
1332 eth_hw_addr_random(dev);
1333
1334 /* Currently tun does not support XDP, only tap does. */
1335 dev->xdp_features = NETDEV_XDP_ACT_BASIC |
1336 NETDEV_XDP_ACT_REDIRECT |
1337 NETDEV_XDP_ACT_NDO_XMIT;
1338
1339 break;
1340 }
1341
1342 dev->min_mtu = MIN_MTU;
1343 dev->max_mtu = MAX_MTU - dev->hard_header_len;
1344 }
1345
1346 static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile)
1347 {
1348 struct sock *sk = tfile->socket.sk;
1349
1350 return (tun->dev->flags & IFF_UP) && sock_writeable(sk);
1351 }
1352
1353 /* Character device part */
1354
1355 /* Poll */
1356 static __poll_t tun_chr_poll(struct file *file, poll_table *wait)
1357 {
1358 struct tun_file *tfile = file->private_data;
1359 struct tun_struct *tun = tun_get(tfile);
1360 struct sock *sk;
1361 __poll_t mask = 0;
1362
1363 if (!tun)
1364 return EPOLLERR;
1365
1366 sk = tfile->socket.sk;
1367
1368 poll_wait(file, sk_sleep(sk), wait);
1369
1370 if (!ptr_ring_empty(&tfile->tx_ring))
1371 mask |= EPOLLIN | EPOLLRDNORM;
1372
1373 /* Make sure SOCKWQ_ASYNC_NOSPACE is set if not writable to
1374 * guarantee EPOLLOUT to be raised by either here or
1375 * tun_sock_write_space(). Then process could get notification
1376 * after it writes to a down device and meets -EIO.
1377 */
1378 if (tun_sock_writeable(tun, tfile) ||
1379 (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags) &&
1380 tun_sock_writeable(tun, tfile)))
1381 mask |= EPOLLOUT | EPOLLWRNORM;
1382
1383 if (tun->dev->reg_state != NETREG_REGISTERED)
1384 mask = EPOLLERR;
1385
1386 tun_put(tun);
1387 return mask;
1388 }
1389
1390 static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
1391 size_t len,
1392 const struct iov_iter *it)
1393 {
1394 struct sk_buff *skb;
1395 size_t linear;
1396 int err;
1397 int i;
1398
1399 if (it->nr_segs > MAX_SKB_FRAGS + 1 ||
1400 len > (ETH_MAX_MTU - NET_SKB_PAD - NET_IP_ALIGN))
1401 return ERR_PTR(-EMSGSIZE);
1402
1403 local_bh_disable();
1404 skb = napi_get_frags(&tfile->napi);
1405 local_bh_enable();
1406 if (!skb)
1407 return ERR_PTR(-ENOMEM);
1408
1409 linear = iov_iter_single_seg_count(it);
1410 err = __skb_grow(skb, linear);
1411 if (err)
1412 goto free;
1413
1414 skb->len = len;
1415 skb->data_len = len - linear;
1416 skb->truesize += skb->data_len;
1417
1418 for (i = 1; i < it->nr_segs; i++) {
1419 const struct iovec *iov = iter_iov(it) + i;
1420 size_t fragsz = iov->iov_len;
1421 struct page *page;
1422 void *frag;
1423
1424 if (fragsz == 0 || fragsz > PAGE_SIZE) {
1425 err = -EINVAL;
1426 goto free;
1427 }
1428 frag = netdev_alloc_frag(fragsz);
1429 if (!frag) {
1430 err = -ENOMEM;
1431 goto free;
1432 }
1433 page = virt_to_head_page(frag);
1434 skb_fill_page_desc(skb, i - 1, page,
1435 frag - page_address(page), fragsz);
1436 }
1437
1438 return skb;
1439 free:
1440 /* frees skb and all frags allocated with napi_alloc_frag() */
1441 napi_free_frags(&tfile->napi);
1442 return ERR_PTR(err);
1443 }
1444
1445 /* prepad is the amount to reserve at front. len is length after that.
1446 * linear is a hint as to how much to copy (usually headers). */
1447 static struct sk_buff *tun_alloc_skb(struct tun_file *tfile,
1448 size_t prepad, size_t len,
1449 size_t linear, int noblock)
1450 {
1451 struct sock *sk = tfile->socket.sk;
1452 struct sk_buff *skb;
1453 int err;
1454
1455 /* Under a page? Don't bother with paged skb. */
1456 if (prepad + len < PAGE_SIZE)
1457 linear = len;
1458
1459 if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER))
1460 linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER);
1461 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
1462 &err, PAGE_ALLOC_COSTLY_ORDER);
1463 if (!skb)
1464 return ERR_PTR(err);
1465
1466 skb_reserve(skb, prepad);
1467 skb_put(skb, linear);
1468 skb->data_len = len - linear;
1469 skb->len += len - linear;
1470
1471 return skb;
1472 }
1473
1474 static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile,
1475 struct sk_buff *skb, int more)
1476 {
1477 struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
1478 struct sk_buff_head process_queue;
1479 u32 rx_batched = tun->rx_batched;
1480 bool rcv = false;
1481
1482 if (!rx_batched || (!more && skb_queue_empty(queue))) {
1483 local_bh_disable();
1484 skb_record_rx_queue(skb, tfile->queue_index);
1485 netif_receive_skb(skb);
1486 local_bh_enable();
1487 return;
1488 }
1489
1490 spin_lock(&queue->lock);
1491 if (!more || skb_queue_len(queue) == rx_batched) {
1492 __skb_queue_head_init(&process_queue);
1493 skb_queue_splice_tail_init(queue, &process_queue);
1494 rcv = true;
1495 } else {
1496 __skb_queue_tail(queue, skb);
1497 }
1498 spin_unlock(&queue->lock);
1499
1500 if (rcv) {
1501 struct sk_buff *nskb;
1502
1503 local_bh_disable();
1504 while ((nskb = __skb_dequeue(&process_queue))) {
1505 skb_record_rx_queue(nskb, tfile->queue_index);
1506 netif_receive_skb(nskb);
1507 }
1508 skb_record_rx_queue(skb, tfile->queue_index);
1509 netif_receive_skb(skb);
1510 local_bh_enable();
1511 }
1512 }
1513
1514 static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile,
1515 int len, int noblock, bool zerocopy)
1516 {
1517 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
1518 return false;
1519
1520 if (tfile->socket.sk->sk_sndbuf != INT_MAX)
1521 return false;
1522
1523 if (!noblock)
1524 return false;
1525
1526 if (zerocopy)
1527 return false;
1528
1529 if (SKB_DATA_ALIGN(len + TUN_RX_PAD + XDP_PACKET_HEADROOM) +
1530 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE)
1531 return false;
1532
1533 return true;
1534 }
1535
1536 static struct sk_buff *__tun_build_skb(struct tun_file *tfile,
1537 struct page_frag *alloc_frag, char *buf,
1538 int buflen, int len, int pad,
1539 int metasize)
1540 {
1541 struct sk_buff *skb = build_skb(buf, buflen);
1542
1543 if (!skb)
1544 return ERR_PTR(-ENOMEM);
1545
1546 skb_reserve(skb, pad);
1547 skb_put(skb, len);
1548 if (metasize)
1549 skb_metadata_set(skb, metasize);
1550 skb_set_owner_w(skb, tfile->socket.sk);
1551
1552 get_page(alloc_frag->page);
1553 alloc_frag->offset += buflen;
1554
1555 return skb;
1556 }
1557
1558 static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog,
1559 struct xdp_buff *xdp, u32 act)
1560 {
1561 int err;
1562
1563 switch (act) {
1564 case XDP_REDIRECT:
1565 err = xdp_do_redirect(tun->dev, xdp, xdp_prog);
1566 if (err) {
1567 dev_core_stats_rx_dropped_inc(tun->dev);
1568 return err;
1569 }
1570 dev_sw_netstats_rx_add(tun->dev, xdp->data_end - xdp->data);
1571 break;
1572 case XDP_TX:
1573 err = tun_xdp_tx(tun->dev, xdp);
1574 if (err < 0) {
1575 dev_core_stats_rx_dropped_inc(tun->dev);
1576 return err;
1577 }
1578 dev_sw_netstats_rx_add(tun->dev, xdp->data_end - xdp->data);
1579 break;
1580 case XDP_PASS:
1581 break;
1582 default:
1583 bpf_warn_invalid_xdp_action(tun->dev, xdp_prog, act);
1584 fallthrough;
1585 case XDP_ABORTED:
1586 trace_xdp_exception(tun->dev, xdp_prog, act);
1587 fallthrough;
1588 case XDP_DROP:
1589 dev_core_stats_rx_dropped_inc(tun->dev);
1590 break;
1591 }
1592
1593 return act;
1594 }
1595
1596 static struct sk_buff *tun_build_skb(struct tun_struct *tun,
1597 struct tun_file *tfile,
1598 struct iov_iter *from,
1599 struct virtio_net_hdr *hdr,
1600 int len, int *skb_xdp)
1601 {
1602 struct page_frag *alloc_frag = &current->task_frag;
1603 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
1604 struct bpf_prog *xdp_prog;
1605 int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1606 char *buf;
1607 size_t copied;
1608 int pad = TUN_RX_PAD;
1609 int metasize = 0;
1610 int err = 0;
1611
1612 rcu_read_lock();
1613 xdp_prog = rcu_dereference(tun->xdp_prog);
1614 if (xdp_prog)
1615 pad += XDP_PACKET_HEADROOM;
1616 buflen += SKB_DATA_ALIGN(len + pad);
1617 rcu_read_unlock();
1618
1619 alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES);
1620 if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, GFP_KERNEL)))
1621 return ERR_PTR(-ENOMEM);
1622
1623 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1624 copied = copy_page_from_iter(alloc_frag->page,
1625 alloc_frag->offset + pad,
1626 len, from);
1627 if (copied != len)
1628 return ERR_PTR(-EFAULT);
1629
1630 /* There's a small window that XDP may be set after the check
1631 * of xdp_prog above, this should be rare and for simplicity
1632 * we do XDP on skb in case the headroom is not enough.
1633 */
1634 if (hdr->gso_type || !xdp_prog) {
1635 *skb_xdp = 1;
1636 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len,
1637 pad, metasize);
1638 }
1639
1640 *skb_xdp = 0;
1641
1642 local_bh_disable();
1643 rcu_read_lock();
1644 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
1645 xdp_prog = rcu_dereference(tun->xdp_prog);
1646 if (xdp_prog) {
1647 struct xdp_buff xdp;
1648 u32 act;
1649
1650 xdp_init_buff(&xdp, buflen, &tfile->xdp_rxq);
1651 xdp_prepare_buff(&xdp, buf, pad, len, true);
1652
1653 act = bpf_prog_run_xdp(xdp_prog, &xdp);
1654 if (act == XDP_REDIRECT || act == XDP_TX) {
1655 get_page(alloc_frag->page);
1656 alloc_frag->offset += buflen;
1657 }
1658 err = tun_xdp_act(tun, xdp_prog, &xdp, act);
1659 if (err < 0) {
1660 if (act == XDP_REDIRECT || act == XDP_TX)
1661 put_page(alloc_frag->page);
1662 goto out;
1663 }
1664
1665 if (err == XDP_REDIRECT)
1666 xdp_do_flush();
1667 if (err != XDP_PASS)
1668 goto out;
1669
1670 pad = xdp.data - xdp.data_hard_start;
1671 len = xdp.data_end - xdp.data;
1672
1673 /* It is known that the xdp_buff was prepared with metadata
1674 * support, so the metasize will never be negative.
1675 */
1676 metasize = xdp.data - xdp.data_meta;
1677 }
1678 bpf_net_ctx_clear(bpf_net_ctx);
1679 rcu_read_unlock();
1680 local_bh_enable();
1681
1682 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, pad,
1683 metasize);
1684
1685 out:
1686 bpf_net_ctx_clear(bpf_net_ctx);
1687 rcu_read_unlock();
1688 local_bh_enable();
1689 return NULL;
1690 }
1691
1692 /* Get packet from user space buffer */
1693 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1694 void *msg_control, struct iov_iter *from,
1695 int noblock, bool more)
1696 {
1697 struct tun_pi pi = { 0, cpu_to_be16(ETH_P_IP) };
1698 struct sk_buff *skb;
1699 size_t total_len = iov_iter_count(from);
1700 size_t len = total_len, align = tun->align, linear;
1701 struct virtio_net_hdr gso = { 0 };
1702 int good_linear;
1703 int copylen;
1704 int hdr_len = 0;
1705 bool zerocopy = false;
1706 int err;
1707 u32 rxhash = 0;
1708 int skb_xdp = 1;
1709 bool frags = tun_napi_frags_enabled(tfile);
1710 enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1711
1712 if (!(tun->flags & IFF_NO_PI)) {
1713 if (len < sizeof(pi))
1714 return -EINVAL;
1715 len -= sizeof(pi);
1716
1717 if (!copy_from_iter_full(&pi, sizeof(pi), from))
1718 return -EFAULT;
1719 }
1720
1721 if (tun->flags & IFF_VNET_HDR) {
1722 int vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
1723
1724 hdr_len = tun_vnet_hdr_get(vnet_hdr_sz, tun->flags, from, &gso);
1725 if (hdr_len < 0)
1726 return hdr_len;
1727
1728 len -= vnet_hdr_sz;
1729 }
1730
1731 if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) {
1732 align += NET_IP_ALIGN;
1733 if (unlikely(len < ETH_HLEN || (hdr_len && hdr_len < ETH_HLEN)))
1734 return -EINVAL;
1735 }
1736
1737 good_linear = SKB_MAX_HEAD(align);
1738
1739 if (msg_control) {
1740 struct iov_iter i = *from;
1741
1742 /* There are 256 bytes to be copied in skb, so there is
1743 * enough room for skb expand head in case it is used.
1744 * The rest of the buffer is mapped from userspace.
1745 */
1746 copylen = min(hdr_len ? hdr_len : GOODCOPY_LEN, good_linear);
1747 linear = copylen;
1748 iov_iter_advance(&i, copylen);
1749 if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS)
1750 zerocopy = true;
1751 }
1752
1753 if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) {
1754 /* For the packet that is not easy to be processed
1755 * (e.g gso or jumbo packet), we will do it at after
1756 * skb was created with generic XDP routine.
1757 */
1758 skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp);
1759 err = PTR_ERR_OR_ZERO(skb);
1760 if (err)
1761 goto drop;
1762 if (!skb)
1763 return total_len;
1764 } else {
1765 if (!zerocopy) {
1766 copylen = len;
1767 linear = min(hdr_len, good_linear);
1768 }
1769
1770 if (frags) {
1771 mutex_lock(&tfile->napi_mutex);
1772 skb = tun_napi_alloc_frags(tfile, copylen, from);
1773 /* tun_napi_alloc_frags() enforces a layout for the skb.
1774 * If zerocopy is enabled, then this layout will be
1775 * overwritten by zerocopy_sg_from_iter().
1776 */
1777 zerocopy = false;
1778 } else {
1779 if (!linear)
1780 linear = min_t(size_t, good_linear, copylen);
1781
1782 skb = tun_alloc_skb(tfile, align, copylen, linear,
1783 noblock);
1784 }
1785
1786 err = PTR_ERR_OR_ZERO(skb);
1787 if (err)
1788 goto drop;
1789
1790 if (zerocopy)
1791 err = zerocopy_sg_from_iter(skb, from);
1792 else
1793 err = skb_copy_datagram_from_iter(skb, 0, from, len);
1794
1795 if (err) {
1796 err = -EFAULT;
1797 drop_reason = SKB_DROP_REASON_SKB_UCOPY_FAULT;
1798 goto drop;
1799 }
1800 }
1801
1802 if (tun_vnet_hdr_to_skb(tun->flags, skb, &gso)) {
1803 atomic_long_inc(&tun->rx_frame_errors);
1804 err = -EINVAL;
1805 goto free_skb;
1806 }
1807
1808 switch (tun->flags & TUN_TYPE_MASK) {
1809 case IFF_TUN:
1810 if (tun->flags & IFF_NO_PI) {
1811 u8 ip_version = skb->len ? (skb->data[0] >> 4) : 0;
1812
1813 switch (ip_version) {
1814 case 4:
1815 pi.proto = htons(ETH_P_IP);
1816 break;
1817 case 6:
1818 pi.proto = htons(ETH_P_IPV6);
1819 break;
1820 default:
1821 err = -EINVAL;
1822 goto drop;
1823 }
1824 }
1825
1826 skb_reset_mac_header(skb);
1827 skb->protocol = pi.proto;
1828 skb->dev = tun->dev;
1829 break;
1830 case IFF_TAP:
1831 if (frags && !pskb_may_pull(skb, ETH_HLEN)) {
1832 err = -ENOMEM;
1833 drop_reason = SKB_DROP_REASON_HDR_TRUNC;
1834 goto drop;
1835 }
1836 skb->protocol = eth_type_trans(skb, tun->dev);
1837 break;
1838 }
1839
1840 /* copy skb_ubuf_info for callback when skb has no error */
1841 if (zerocopy) {
1842 skb_zcopy_init(skb, msg_control);
1843 } else if (msg_control) {
1844 struct ubuf_info *uarg = msg_control;
1845 uarg->ops->complete(NULL, uarg, false);
1846 }
1847
1848 skb_reset_network_header(skb);
1849 skb_probe_transport_header(skb);
1850 skb_record_rx_queue(skb, tfile->queue_index);
1851
1852 if (skb_xdp) {
1853 struct bpf_prog *xdp_prog;
1854 int ret;
1855
1856 local_bh_disable();
1857 rcu_read_lock();
1858 xdp_prog = rcu_dereference(tun->xdp_prog);
1859 if (xdp_prog) {
1860 ret = do_xdp_generic(xdp_prog, &skb);
1861 if (ret != XDP_PASS) {
1862 rcu_read_unlock();
1863 local_bh_enable();
1864 goto unlock_frags;
1865 }
1866 }
1867 rcu_read_unlock();
1868 local_bh_enable();
1869 }
1870
1871 /* Compute the costly rx hash only if needed for flow updates.
1872 * We may get a very small possibility of OOO during switching, not
1873 * worth to optimize.
1874 */
1875 if (!rcu_access_pointer(tun->steering_prog) && tun->numqueues > 1 &&
1876 !tfile->detached)
1877 rxhash = __skb_get_hash_symmetric(skb);
1878
1879 rcu_read_lock();
1880 if (unlikely(!(tun->dev->flags & IFF_UP))) {
1881 err = -EIO;
1882 rcu_read_unlock();
1883 drop_reason = SKB_DROP_REASON_DEV_READY;
1884 goto drop;
1885 }
1886
1887 if (frags) {
1888 u32 headlen;
1889
1890 /* Exercise flow dissector code path. */
1891 skb_push(skb, ETH_HLEN);
1892 headlen = eth_get_headlen(tun->dev, skb->data,
1893 skb_headlen(skb));
1894
1895 if (unlikely(headlen > skb_headlen(skb))) {
1896 WARN_ON_ONCE(1);
1897 err = -ENOMEM;
1898 dev_core_stats_rx_dropped_inc(tun->dev);
1899 napi_busy:
1900 napi_free_frags(&tfile->napi);
1901 rcu_read_unlock();
1902 mutex_unlock(&tfile->napi_mutex);
1903 return err;
1904 }
1905
1906 if (likely(napi_schedule_prep(&tfile->napi))) {
1907 local_bh_disable();
1908 napi_gro_frags(&tfile->napi);
1909 napi_complete(&tfile->napi);
1910 local_bh_enable();
1911 } else {
1912 err = -EBUSY;
1913 goto napi_busy;
1914 }
1915 mutex_unlock(&tfile->napi_mutex);
1916 } else if (tfile->napi_enabled) {
1917 struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
1918 int queue_len;
1919
1920 spin_lock_bh(&queue->lock);
1921
1922 if (unlikely(tfile->detached)) {
1923 spin_unlock_bh(&queue->lock);
1924 rcu_read_unlock();
1925 err = -EBUSY;
1926 goto free_skb;
1927 }
1928
1929 __skb_queue_tail(queue, skb);
1930 queue_len = skb_queue_len(queue);
1931 spin_unlock(&queue->lock);
1932
1933 if (!more || queue_len > NAPI_POLL_WEIGHT)
1934 napi_schedule(&tfile->napi);
1935
1936 local_bh_enable();
1937 } else if (!IS_ENABLED(CONFIG_4KSTACKS)) {
1938 tun_rx_batched(tun, tfile, skb, more);
1939 } else {
1940 netif_rx(skb);
1941 }
1942 rcu_read_unlock();
1943
1944 preempt_disable();
1945 dev_sw_netstats_rx_add(tun->dev, len);
1946 preempt_enable();
1947
1948 if (rxhash)
1949 tun_flow_update(tun, rxhash, tfile);
1950
1951 return total_len;
1952
1953 drop:
1954 if (err != -EAGAIN)
1955 dev_core_stats_rx_dropped_inc(tun->dev);
1956
1957 free_skb:
1958 if (!IS_ERR_OR_NULL(skb))
1959 kfree_skb_reason(skb, drop_reason);
1960
1961 unlock_frags:
1962 if (frags) {
1963 tfile->napi.skb = NULL;
1964 mutex_unlock(&tfile->napi_mutex);
1965 }
1966
1967 return err ?: total_len;
1968 }
1969
1970 static ssize_t tun_chr_write_iter(struct kiocb *iocb, struct iov_iter *from)
1971 {
1972 struct file *file = iocb->ki_filp;
1973 struct tun_file *tfile = file->private_data;
1974 struct tun_struct *tun = tun_get(tfile);
1975 ssize_t result;
1976 int noblock = 0;
1977
1978 if (!tun)
1979 return -EBADFD;
1980
1981 if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT))
1982 noblock = 1;
1983
1984 result = tun_get_user(tun, tfile, NULL, from, noblock, false);
1985
1986 tun_put(tun);
1987 return result;
1988 }
1989
1990 static ssize_t tun_put_user_xdp(struct tun_struct *tun,
1991 struct tun_file *tfile,
1992 struct xdp_frame *xdp_frame,
1993 struct iov_iter *iter)
1994 {
1995 int vnet_hdr_sz = 0;
1996 size_t size = xdp_frame->len;
1997 ssize_t ret;
1998
1999 if (tun->flags & IFF_VNET_HDR) {
2000 struct virtio_net_hdr gso = { 0 };
2001
2002 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
2003 ret = tun_vnet_hdr_put(vnet_hdr_sz, iter, &gso);
2004 if (ret)
2005 return ret;
2006 }
2007
2008 ret = copy_to_iter(xdp_frame->data, size, iter) + vnet_hdr_sz;
2009
2010 preempt_disable();
2011 dev_sw_netstats_tx_add(tun->dev, 1, ret);
2012 preempt_enable();
2013
2014 return ret;
2015 }
2016
2017 /* Put packet to the user space buffer */
2018 static ssize_t tun_put_user(struct tun_struct *tun,
2019 struct tun_file *tfile,
2020 struct sk_buff *skb,
2021 struct iov_iter *iter)
2022 {
2023 struct tun_pi pi = { 0, skb->protocol };
2024 ssize_t total;
2025 int vlan_offset = 0;
2026 int vlan_hlen = 0;
2027 int vnet_hdr_sz = 0;
2028 int ret;
2029
2030 if (skb_vlan_tag_present(skb))
2031 vlan_hlen = VLAN_HLEN;
2032
2033 if (tun->flags & IFF_VNET_HDR)
2034 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
2035
2036 total = skb->len + vlan_hlen + vnet_hdr_sz;
2037
2038 if (!(tun->flags & IFF_NO_PI)) {
2039 if (iov_iter_count(iter) < sizeof(pi))
2040 return -EINVAL;
2041
2042 total += sizeof(pi);
2043 if (iov_iter_count(iter) < total) {
2044 /* Packet will be striped */
2045 pi.flags |= TUN_PKT_STRIP;
2046 }
2047
2048 if (copy_to_iter(&pi, sizeof(pi), iter) != sizeof(pi))
2049 return -EFAULT;
2050 }
2051
2052 if (vnet_hdr_sz) {
2053 struct virtio_net_hdr gso;
2054
2055 ret = tun_vnet_hdr_from_skb(tun->flags, tun->dev, skb, &gso);
2056 if (ret)
2057 return ret;
2058
2059 ret = tun_vnet_hdr_put(vnet_hdr_sz, iter, &gso);
2060 if (ret)
2061 return ret;
2062 }
2063
2064 if (vlan_hlen) {
2065 int ret;
2066 struct veth veth;
2067
2068 veth.h_vlan_proto = skb->vlan_proto;
2069 veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb));
2070
2071 vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto);
2072
2073 ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset);
2074 if (ret || !iov_iter_count(iter))
2075 goto done;
2076
2077 ret = copy_to_iter(&veth, sizeof(veth), iter);
2078 if (ret != sizeof(veth) || !iov_iter_count(iter))
2079 goto done;
2080 }
2081
2082 skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset);
2083
2084 done:
2085 /* caller is in process context, */
2086 preempt_disable();
2087 dev_sw_netstats_tx_add(tun->dev, 1, skb->len + vlan_hlen);
2088 preempt_enable();
2089
2090 return total;
2091 }
2092
2093 static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
2094 {
2095 DECLARE_WAITQUEUE(wait, current);
2096 void *ptr = NULL;
2097 int error = 0;
2098
2099 ptr = ptr_ring_consume(&tfile->tx_ring);
2100 if (ptr)
2101 goto out;
2102 if (noblock) {
2103 error = -EAGAIN;
2104 goto out;
2105 }
2106
2107 add_wait_queue(&tfile->socket.wq.wait, &wait);
2108
2109 while (1) {
2110 set_current_state(TASK_INTERRUPTIBLE);
2111 ptr = ptr_ring_consume(&tfile->tx_ring);
2112 if (ptr)
2113 break;
2114 if (signal_pending(current)) {
2115 error = -ERESTARTSYS;
2116 break;
2117 }
2118 if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) {
2119 error = -EFAULT;
2120 break;
2121 }
2122
2123 schedule();
2124 }
2125
2126 __set_current_state(TASK_RUNNING);
2127 remove_wait_queue(&tfile->socket.wq.wait, &wait);
2128
2129 out:
2130 *err = error;
2131 return ptr;
2132 }
2133
2134 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
2135 struct iov_iter *to,
2136 int noblock, void *ptr)
2137 {
2138 ssize_t ret;
2139 int err;
2140
2141 if (!iov_iter_count(to)) {
2142 tun_ptr_free(ptr);
2143 return 0;
2144 }
2145
2146 if (!ptr) {
2147 /* Read frames from ring */
2148 ptr = tun_ring_recv(tfile, noblock, &err);
2149 if (!ptr)
2150 return err;
2151 }
2152
2153 if (tun_is_xdp_frame(ptr)) {
2154 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
2155
2156 ret = tun_put_user_xdp(tun, tfile, xdpf, to);
2157 xdp_return_frame(xdpf);
2158 } else {
2159 struct sk_buff *skb = ptr;
2160
2161 ret = tun_put_user(tun, tfile, skb, to);
2162 if (unlikely(ret < 0))
2163 kfree_skb(skb);
2164 else
2165 consume_skb(skb);
2166 }
2167
2168 return ret;
2169 }
2170
2171 static ssize_t tun_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
2172 {
2173 struct file *file = iocb->ki_filp;
2174 struct tun_file *tfile = file->private_data;
2175 struct tun_struct *tun = tun_get(tfile);
2176 ssize_t len = iov_iter_count(to), ret;
2177 int noblock = 0;
2178
2179 if (!tun)
2180 return -EBADFD;
2181
2182 if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT))
2183 noblock = 1;
2184
2185 ret = tun_do_read(tun, tfile, to, noblock, NULL);
2186 ret = min_t(ssize_t, ret, len);
2187 if (ret > 0)
2188 iocb->ki_pos = ret;
2189 tun_put(tun);
2190 return ret;
2191 }
2192
2193 static void tun_prog_free(struct rcu_head *rcu)
2194 {
2195 struct tun_prog *prog = container_of(rcu, struct tun_prog, rcu);
2196
2197 bpf_prog_destroy(prog->prog);
2198 kfree(prog);
2199 }
2200
2201 static int __tun_set_ebpf(struct tun_struct *tun,
2202 struct tun_prog __rcu **prog_p,
2203 struct bpf_prog *prog)
2204 {
2205 struct tun_prog *old, *new = NULL;
2206
2207 if (prog) {
2208 new = kmalloc(sizeof(*new), GFP_KERNEL);
2209 if (!new)
2210 return -ENOMEM;
2211 new->prog = prog;
2212 }
2213
2214 spin_lock_bh(&tun->lock);
2215 old = rcu_dereference_protected(*prog_p,
2216 lockdep_is_held(&tun->lock));
2217 rcu_assign_pointer(*prog_p, new);
2218 spin_unlock_bh(&tun->lock);
2219
2220 if (old)
2221 call_rcu(&old->rcu, tun_prog_free);
2222
2223 return 0;
2224 }
2225
2226 static void tun_free_netdev(struct net_device *dev)
2227 {
2228 struct tun_struct *tun = netdev_priv(dev);
2229
2230 BUG_ON(!(list_empty(&tun->disabled)));
2231
2232 tun_flow_uninit(tun);
2233 security_tun_dev_free_security(tun->security);
2234 __tun_set_ebpf(tun, &tun->steering_prog, NULL);
2235 __tun_set_ebpf(tun, &tun->filter_prog, NULL);
2236 }
2237
2238 static void tun_setup(struct net_device *dev)
2239 {
2240 struct tun_struct *tun = netdev_priv(dev);
2241
2242 tun->owner = INVALID_UID;
2243 tun->group = INVALID_GID;
2244 tun_default_link_ksettings(dev, &tun->link_ksettings);
2245
2246 dev->ethtool_ops = &tun_ethtool_ops;
2247 dev->needs_free_netdev = true;
2248 dev->priv_destructor = tun_free_netdev;
2249 /* We prefer our own queue length */
2250 dev->tx_queue_len = TUN_READQ_SIZE;
2251 }
2252
2253 /* Trivial set of netlink ops to allow deleting tun or tap
2254 * device with netlink.
2255 */
2256 static int tun_validate(struct nlattr *tb[], struct nlattr *data[],
2257 struct netlink_ext_ack *extack)
2258 {
2259 NL_SET_ERR_MSG(extack,
2260 "tun/tap creation via rtnetlink is not supported.");
2261 return -EOPNOTSUPP;
2262 }
2263
2264 static size_t tun_get_size(const struct net_device *dev)
2265 {
2266 BUILD_BUG_ON(sizeof(u32) != sizeof(uid_t));
2267 BUILD_BUG_ON(sizeof(u32) != sizeof(gid_t));
2268
2269 return nla_total_size(sizeof(uid_t)) + /* OWNER */
2270 nla_total_size(sizeof(gid_t)) + /* GROUP */
2271 nla_total_size(sizeof(u8)) + /* TYPE */
2272 nla_total_size(sizeof(u8)) + /* PI */
2273 nla_total_size(sizeof(u8)) + /* VNET_HDR */
2274 nla_total_size(sizeof(u8)) + /* PERSIST */
2275 nla_total_size(sizeof(u8)) + /* MULTI_QUEUE */
2276 nla_total_size(sizeof(u32)) + /* NUM_QUEUES */
2277 nla_total_size(sizeof(u32)) + /* NUM_DISABLED_QUEUES */
2278 0;
2279 }
2280
2281 static int tun_fill_info(struct sk_buff *skb, const struct net_device *dev)
2282 {
2283 struct tun_struct *tun = netdev_priv(dev);
2284
2285 if (nla_put_u8(skb, IFLA_TUN_TYPE, tun->flags & TUN_TYPE_MASK))
2286 goto nla_put_failure;
2287 if (uid_valid(tun->owner) &&
2288 nla_put_u32(skb, IFLA_TUN_OWNER,
2289 from_kuid_munged(current_user_ns(), tun->owner)))
2290 goto nla_put_failure;
2291 if (gid_valid(tun->group) &&
2292 nla_put_u32(skb, IFLA_TUN_GROUP,
2293 from_kgid_munged(current_user_ns(), tun->group)))
2294 goto nla_put_failure;
2295 if (nla_put_u8(skb, IFLA_TUN_PI, !(tun->flags & IFF_NO_PI)))
2296 goto nla_put_failure;
2297 if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(tun->flags & IFF_VNET_HDR)))
2298 goto nla_put_failure;
2299 if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(tun->flags & IFF_PERSIST)))
2300 goto nla_put_failure;
2301 if (nla_put_u8(skb, IFLA_TUN_MULTI_QUEUE,
2302 !!(tun->flags & IFF_MULTI_QUEUE)))
2303 goto nla_put_failure;
2304 if (tun->flags & IFF_MULTI_QUEUE) {
2305 if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES, tun->numqueues))
2306 goto nla_put_failure;
2307 if (nla_put_u32(skb, IFLA_TUN_NUM_DISABLED_QUEUES,
2308 tun->numdisabled))
2309 goto nla_put_failure;
2310 }
2311
2312 return 0;
2313
2314 nla_put_failure:
2315 return -EMSGSIZE;
2316 }
2317
2318 static struct rtnl_link_ops tun_link_ops __read_mostly = {
2319 .kind = DRV_NAME,
2320 .priv_size = sizeof(struct tun_struct),
2321 .setup = tun_setup,
2322 .validate = tun_validate,
2323 .get_size = tun_get_size,
2324 .fill_info = tun_fill_info,
2325 };
2326
2327 static void tun_sock_write_space(struct sock *sk)
2328 {
2329 struct tun_file *tfile;
2330 wait_queue_head_t *wqueue;
2331
2332 if (!sock_writeable(sk))
2333 return;
2334
2335 if (!test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags))
2336 return;
2337
2338 wqueue = sk_sleep(sk);
2339 if (wqueue && waitqueue_active(wqueue))
2340 wake_up_interruptible_sync_poll(wqueue, EPOLLOUT |
2341 EPOLLWRNORM | EPOLLWRBAND);
2342
2343 tfile = container_of(sk, struct tun_file, sk);
2344 kill_fasync(&tfile->fasync, SIGIO, POLL_OUT);
2345 }
2346
2347 static void tun_put_page(struct tun_page *tpage)
2348 {
2349 if (tpage->page)
2350 __page_frag_cache_drain(tpage->page, tpage->count);
2351 }
2352
2353 static int tun_xdp_one(struct tun_struct *tun,
2354 struct tun_file *tfile,
2355 struct xdp_buff *xdp, int *flush,
2356 struct tun_page *tpage)
2357 {
2358 unsigned int datasize = xdp->data_end - xdp->data;
2359 struct tun_xdp_hdr *hdr = xdp->data_hard_start;
2360 struct virtio_net_hdr *gso = &hdr->gso;
2361 struct bpf_prog *xdp_prog;
2362 struct sk_buff *skb = NULL;
2363 struct sk_buff_head *queue;
2364 u32 rxhash = 0, act;
2365 int buflen = hdr->buflen;
2366 int metasize = 0;
2367 int ret = 0;
2368 bool skb_xdp = false;
2369 struct page *page;
2370
2371 if (unlikely(datasize < ETH_HLEN))
2372 return -EINVAL;
2373
2374 xdp_prog = rcu_dereference(tun->xdp_prog);
2375 if (xdp_prog) {
2376 if (gso->gso_type) {
2377 skb_xdp = true;
2378 goto build;
2379 }
2380
2381 xdp_init_buff(xdp, buflen, &tfile->xdp_rxq);
2382
2383 act = bpf_prog_run_xdp(xdp_prog, xdp);
2384 ret = tun_xdp_act(tun, xdp_prog, xdp, act);
2385 if (ret < 0) {
2386 put_page(virt_to_head_page(xdp->data));
2387 return ret;
2388 }
2389
2390 switch (ret) {
2391 case XDP_REDIRECT:
2392 *flush = true;
2393 fallthrough;
2394 case XDP_TX:
2395 return 0;
2396 case XDP_PASS:
2397 break;
2398 default:
2399 page = virt_to_head_page(xdp->data);
2400 if (tpage->page == page) {
2401 ++tpage->count;
2402 } else {
2403 tun_put_page(tpage);
2404 tpage->page = page;
2405 tpage->count = 1;
2406 }
2407 return 0;
2408 }
2409 }
2410
2411 build:
2412 skb = build_skb(xdp->data_hard_start, buflen);
2413 if (!skb) {
2414 ret = -ENOMEM;
2415 goto out;
2416 }
2417
2418 skb_reserve(skb, xdp->data - xdp->data_hard_start);
2419 skb_put(skb, xdp->data_end - xdp->data);
2420
2421 /* The externally provided xdp_buff may have no metadata support, which
2422 * is marked by xdp->data_meta being xdp->data + 1. This will lead to a
2423 * metasize of -1 and is the reason why the condition checks for > 0.
2424 */
2425 metasize = xdp->data - xdp->data_meta;
2426 if (metasize > 0)
2427 skb_metadata_set(skb, metasize);
2428
2429 if (tun_vnet_hdr_to_skb(tun->flags, skb, gso)) {
2430 atomic_long_inc(&tun->rx_frame_errors);
2431 kfree_skb(skb);
2432 ret = -EINVAL;
2433 goto out;
2434 }
2435
2436 skb->protocol = eth_type_trans(skb, tun->dev);
2437 skb_reset_network_header(skb);
2438 skb_probe_transport_header(skb);
2439 skb_record_rx_queue(skb, tfile->queue_index);
2440
2441 if (skb_xdp) {
2442 ret = do_xdp_generic(xdp_prog, &skb);
2443 if (ret != XDP_PASS) {
2444 ret = 0;
2445 goto out;
2446 }
2447 }
2448
2449 if (!rcu_dereference(tun->steering_prog) && tun->numqueues > 1 &&
2450 !tfile->detached)
2451 rxhash = __skb_get_hash_symmetric(skb);
2452
2453 if (tfile->napi_enabled) {
2454 queue = &tfile->sk.sk_write_queue;
2455 spin_lock(&queue->lock);
2456
2457 if (unlikely(tfile->detached)) {
2458 spin_unlock(&queue->lock);
2459 kfree_skb(skb);
2460 return -EBUSY;
2461 }
2462
2463 __skb_queue_tail(queue, skb);
2464 spin_unlock(&queue->lock);
2465 ret = 1;
2466 } else {
2467 netif_receive_skb(skb);
2468 ret = 0;
2469 }
2470
2471 /* No need to disable preemption here since this function is
2472 * always called with bh disabled
2473 */
2474 dev_sw_netstats_rx_add(tun->dev, datasize);
2475
2476 if (rxhash)
2477 tun_flow_update(tun, rxhash, tfile);
2478
2479 out:
2480 return ret;
2481 }
2482
2483 static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
2484 {
2485 int ret, i;
2486 struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2487 struct tun_struct *tun = tun_get(tfile);
2488 struct tun_msg_ctl *ctl = m->msg_control;
2489 struct xdp_buff *xdp;
2490
2491 if (!tun)
2492 return -EBADFD;
2493
2494 if (m->msg_controllen == sizeof(struct tun_msg_ctl) &&
2495 ctl && ctl->type == TUN_MSG_PTR) {
2496 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
2497 struct tun_page tpage;
2498 int n = ctl->num;
2499 int flush = 0, queued = 0;
2500
2501 memset(&tpage, 0, sizeof(tpage));
2502
2503 local_bh_disable();
2504 rcu_read_lock();
2505 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
2506
2507 for (i = 0; i < n; i++) {
2508 xdp = &((struct xdp_buff *)ctl->ptr)[i];
2509 ret = tun_xdp_one(tun, tfile, xdp, &flush, &tpage);
2510 if (ret > 0)
2511 queued += ret;
2512 }
2513
2514 if (flush)
2515 xdp_do_flush();
2516
2517 if (tfile->napi_enabled && queued > 0)
2518 napi_schedule(&tfile->napi);
2519
2520 bpf_net_ctx_clear(bpf_net_ctx);
2521 rcu_read_unlock();
2522 local_bh_enable();
2523
2524 tun_put_page(&tpage);
2525
2526 ret = total_len;
2527 goto out;
2528 }
2529
2530 ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter,
2531 m->msg_flags & MSG_DONTWAIT,
2532 m->msg_flags & MSG_MORE);
2533 out:
2534 tun_put(tun);
2535 return ret;
2536 }
2537
2538 static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
2539 int flags)
2540 {
2541 struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2542 struct tun_struct *tun = tun_get(tfile);
2543 void *ptr = m->msg_control;
2544 int ret;
2545
2546 if (!tun) {
2547 ret = -EBADFD;
2548 goto out_free;
2549 }
2550
2551 if (flags & ~(MSG_DONTWAIT|MSG_TRUNC|MSG_ERRQUEUE)) {
2552 ret = -EINVAL;
2553 goto out_put_tun;
2554 }
2555 if (flags & MSG_ERRQUEUE) {
2556 ret = sock_recv_errqueue(sock->sk, m, total_len,
2557 SOL_PACKET, TUN_TX_TIMESTAMP);
2558 goto out;
2559 }
2560 ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr);
2561 if (ret > (ssize_t)total_len) {
2562 m->msg_flags |= MSG_TRUNC;
2563 ret = flags & MSG_TRUNC ? ret : total_len;
2564 }
2565 out:
2566 tun_put(tun);
2567 return ret;
2568
2569 out_put_tun:
2570 tun_put(tun);
2571 out_free:
2572 tun_ptr_free(ptr);
2573 return ret;
2574 }
2575
2576 static int tun_ptr_peek_len(void *ptr)
2577 {
2578 if (likely(ptr)) {
2579 if (tun_is_xdp_frame(ptr)) {
2580 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
2581
2582 return xdpf->len;
2583 }
2584 return __skb_array_len_with_tag(ptr);
2585 } else {
2586 return 0;
2587 }
2588 }
2589
2590 static int tun_peek_len(struct socket *sock)
2591 {
2592 struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2593 struct tun_struct *tun;
2594 int ret = 0;
2595
2596 tun = tun_get(tfile);
2597 if (!tun)
2598 return 0;
2599
2600 ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len);
2601 tun_put(tun);
2602
2603 return ret;
2604 }
2605
2606 /* Ops structure to mimic raw sockets with tun */
2607 static const struct proto_ops tun_socket_ops = {
2608 .peek_len = tun_peek_len,
2609 .sendmsg = tun_sendmsg,
2610 .recvmsg = tun_recvmsg,
2611 };
2612
2613 static struct proto tun_proto = {
2614 .name = "tun",
2615 .owner = THIS_MODULE,
2616 .obj_size = sizeof(struct tun_file),
2617 };
2618
2619 static int tun_flags(struct tun_struct *tun)
2620 {
2621 return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP);
2622 }
2623
2624 static ssize_t tun_flags_show(struct device *dev, struct device_attribute *attr,
2625 char *buf)
2626 {
2627 struct tun_struct *tun = netdev_priv(to_net_dev(dev));
2628 return sysfs_emit(buf, "0x%x\n", tun_flags(tun));
2629 }
2630
2631 static ssize_t owner_show(struct device *dev, struct device_attribute *attr,
2632 char *buf)
2633 {
2634 struct tun_struct *tun = netdev_priv(to_net_dev(dev));
2635 return uid_valid(tun->owner)?
2636 sysfs_emit(buf, "%u\n",
2637 from_kuid_munged(current_user_ns(), tun->owner)) :
2638 sysfs_emit(buf, "-1\n");
2639 }
2640
2641 static ssize_t group_show(struct device *dev, struct device_attribute *attr,
2642 char *buf)
2643 {
2644 struct tun_struct *tun = netdev_priv(to_net_dev(dev));
2645 return gid_valid(tun->group) ?
2646 sysfs_emit(buf, "%u\n",
2647 from_kgid_munged(current_user_ns(), tun->group)) :
2648 sysfs_emit(buf, "-1\n");
2649 }
2650
2651 static DEVICE_ATTR_RO(tun_flags);
2652 static DEVICE_ATTR_RO(owner);
2653 static DEVICE_ATTR_RO(group);
2654
2655 static struct attribute *tun_dev_attrs[] = {
2656 &dev_attr_tun_flags.attr,
2657 &dev_attr_owner.attr,
2658 &dev_attr_group.attr,
2659 NULL
2660 };
2661
2662 static const struct attribute_group tun_attr_group = {
2663 .attrs = tun_dev_attrs
2664 };
2665
2666 static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
2667 {
2668 struct tun_struct *tun;
2669 struct tun_file *tfile = file->private_data;
2670 struct net_device *dev;
2671 int err;
2672
2673 if (tfile->detached)
2674 return -EINVAL;
2675
2676 if ((ifr->ifr_flags & IFF_NAPI_FRAGS)) {
2677 if (!capable(CAP_NET_ADMIN))
2678 return -EPERM;
2679
2680 if (!(ifr->ifr_flags & IFF_NAPI) ||
2681 (ifr->ifr_flags & TUN_TYPE_MASK) != IFF_TAP)
2682 return -EINVAL;
2683 }
2684
2685 dev = __dev_get_by_name(net, ifr->ifr_name);
2686 if (dev) {
2687 if (ifr->ifr_flags & IFF_TUN_EXCL)
2688 return -EBUSY;
2689 if ((ifr->ifr_flags & IFF_TUN) && dev->netdev_ops == &tun_netdev_ops)
2690 tun = netdev_priv(dev);
2691 else if ((ifr->ifr_flags & IFF_TAP) && dev->netdev_ops == &tap_netdev_ops)
2692 tun = netdev_priv(dev);
2693 else
2694 return -EINVAL;
2695
2696 if (!!(ifr->ifr_flags & IFF_MULTI_QUEUE) !=
2697 !!(tun->flags & IFF_MULTI_QUEUE))
2698 return -EINVAL;
2699
2700 if (tun_not_capable(tun))
2701 return -EPERM;
2702 err = security_tun_dev_open(tun->security);
2703 if (err < 0)
2704 return err;
2705
2706 err = tun_attach(tun, file, ifr->ifr_flags & IFF_NOFILTER,
2707 ifr->ifr_flags & IFF_NAPI,
2708 ifr->ifr_flags & IFF_NAPI_FRAGS, true);
2709 if (err < 0)
2710 return err;
2711
2712 if (tun->flags & IFF_MULTI_QUEUE &&
2713 (tun->numqueues + tun->numdisabled > 1)) {
2714 /* One or more queue has already been attached, no need
2715 * to initialize the device again.
2716 */
2717 netdev_state_change(dev);
2718 return 0;
2719 }
2720
2721 tun->flags = (tun->flags & ~TUN_FEATURES) |
2722 (ifr->ifr_flags & TUN_FEATURES);
2723
2724 netdev_state_change(dev);
2725 } else {
2726 char *name;
2727 unsigned long flags = 0;
2728 int queues = ifr->ifr_flags & IFF_MULTI_QUEUE ?
2729 MAX_TAP_QUEUES : 1;
2730
2731 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
2732 return -EPERM;
2733 err = security_tun_dev_create();
2734 if (err < 0)
2735 return err;
2736
2737 /* Set dev type */
2738 if (ifr->ifr_flags & IFF_TUN) {
2739 /* TUN device */
2740 flags |= IFF_TUN;
2741 name = "tun%d";
2742 } else if (ifr->ifr_flags & IFF_TAP) {
2743 /* TAP device */
2744 flags |= IFF_TAP;
2745 name = "tap%d";
2746 } else
2747 return -EINVAL;
2748
2749 if (*ifr->ifr_name)
2750 name = ifr->ifr_name;
2751
2752 dev = alloc_netdev_mqs(sizeof(struct tun_struct), name,
2753 NET_NAME_UNKNOWN, tun_setup, queues,
2754 queues);
2755
2756 if (!dev)
2757 return -ENOMEM;
2758
2759 dev_net_set(dev, net);
2760 dev->rtnl_link_ops = &tun_link_ops;
2761 dev->ifindex = tfile->ifindex;
2762 dev->sysfs_groups[0] = &tun_attr_group;
2763
2764 tun = netdev_priv(dev);
2765 tun->dev = dev;
2766 tun->flags = flags;
2767 tun->txflt.count = 0;
2768 tun->vnet_hdr_sz = sizeof(struct virtio_net_hdr);
2769
2770 tun->align = NET_SKB_PAD;
2771 tun->filter_attached = false;
2772 tun->sndbuf = tfile->socket.sk->sk_sndbuf;
2773 tun->rx_batched = 0;
2774 RCU_INIT_POINTER(tun->steering_prog, NULL);
2775
2776 tun->ifr = ifr;
2777 tun->file = file;
2778
2779 tun_net_initialize(dev);
2780
2781 err = register_netdevice(tun->dev);
2782 if (err < 0) {
2783 free_netdev(dev);
2784 return err;
2785 }
2786 /* free_netdev() won't check refcnt, to avoid race
2787 * with dev_put() we need publish tun after registration.
2788 */
2789 rcu_assign_pointer(tfile->tun, tun);
2790 }
2791
2792 if (ifr->ifr_flags & IFF_NO_CARRIER)
2793 netif_carrier_off(tun->dev);
2794 else
2795 netif_carrier_on(tun->dev);
2796
2797 /* Make sure persistent devices do not get stuck in
2798 * xoff state.
2799 */
2800 if (netif_running(tun->dev))
2801 netif_tx_wake_all_queues(tun->dev);
2802
2803 strcpy(ifr->ifr_name, tun->dev->name);
2804 return 0;
2805 }
2806
2807 static void tun_get_iff(struct tun_struct *tun, struct ifreq *ifr)
2808 {
2809 strcpy(ifr->ifr_name, tun->dev->name);
2810
2811 ifr->ifr_flags = tun_flags(tun);
2812
2813 }
2814
2815 /* This is like a cut-down ethtool ops, except done via tun fd so no
2816 * privs required. */
2817 static int set_offload(struct tun_struct *tun, unsigned long arg)
2818 {
2819 netdev_features_t features = 0;
2820
2821 if (arg & TUN_F_CSUM) {
2822 features |= NETIF_F_HW_CSUM;
2823 arg &= ~TUN_F_CSUM;
2824
2825 if (arg & (TUN_F_TSO4|TUN_F_TSO6)) {
2826 if (arg & TUN_F_TSO_ECN) {
2827 features |= NETIF_F_TSO_ECN;
2828 arg &= ~TUN_F_TSO_ECN;
2829 }
2830 if (arg & TUN_F_TSO4)
2831 features |= NETIF_F_TSO;
2832 if (arg & TUN_F_TSO6)
2833 features |= NETIF_F_TSO6;
2834 arg &= ~(TUN_F_TSO4|TUN_F_TSO6);
2835 }
2836
2837 arg &= ~TUN_F_UFO;
2838
2839 /* TODO: for now USO4 and USO6 should work simultaneously */
2840 if (arg & TUN_F_USO4 && arg & TUN_F_USO6) {
2841 features |= NETIF_F_GSO_UDP_L4;
2842 arg &= ~(TUN_F_USO4 | TUN_F_USO6);
2843 }
2844 }
2845
2846 /* This gives the user a way to test for new features in future by
2847 * trying to set them. */
2848 if (arg)
2849 return -EINVAL;
2850
2851 tun->set_features = features;
2852 tun->dev->wanted_features &= ~TUN_USER_FEATURES;
2853 tun->dev->wanted_features |= features;
2854 netdev_update_features(tun->dev);
2855
2856 return 0;
2857 }
2858
2859 static void tun_detach_filter(struct tun_struct *tun, int n)
2860 {
2861 int i;
2862 struct tun_file *tfile;
2863
2864 for (i = 0; i < n; i++) {
2865 tfile = rtnl_dereference(tun->tfiles[i]);
2866 lock_sock(tfile->socket.sk);
2867 sk_detach_filter(tfile->socket.sk);
2868 release_sock(tfile->socket.sk);
2869 }
2870
2871 tun->filter_attached = false;
2872 }
2873
2874 static int tun_attach_filter(struct tun_struct *tun)
2875 {
2876 int i, ret = 0;
2877 struct tun_file *tfile;
2878
2879 for (i = 0; i < tun->numqueues; i++) {
2880 tfile = rtnl_dereference(tun->tfiles[i]);
2881 lock_sock(tfile->socket.sk);
2882 ret = sk_attach_filter(&tun->fprog, tfile->socket.sk);
2883 release_sock(tfile->socket.sk);
2884 if (ret) {
2885 tun_detach_filter(tun, i);
2886 return ret;
2887 }
2888 }
2889
2890 tun->filter_attached = true;
2891 return ret;
2892 }
2893
2894 static void tun_set_sndbuf(struct tun_struct *tun)
2895 {
2896 struct tun_file *tfile;
2897 int i;
2898
2899 for (i = 0; i < tun->numqueues; i++) {
2900 tfile = rtnl_dereference(tun->tfiles[i]);
2901 tfile->socket.sk->sk_sndbuf = tun->sndbuf;
2902 }
2903 }
2904
2905 static int tun_set_queue(struct file *file, struct ifreq *ifr)
2906 {
2907 struct tun_file *tfile = file->private_data;
2908 struct tun_struct *tun;
2909 int ret = 0;
2910
2911 rtnl_lock();
2912
2913 if (ifr->ifr_flags & IFF_ATTACH_QUEUE) {
2914 tun = tfile->detached;
2915 if (!tun) {
2916 ret = -EINVAL;
2917 goto unlock;
2918 }
2919 ret = security_tun_dev_attach_queue(tun->security);
2920 if (ret < 0)
2921 goto unlock;
2922 ret = tun_attach(tun, file, false, tun->flags & IFF_NAPI,
2923 tun->flags & IFF_NAPI_FRAGS, true);
2924 } else if (ifr->ifr_flags & IFF_DETACH_QUEUE) {
2925 tun = rtnl_dereference(tfile->tun);
2926 if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached)
2927 ret = -EINVAL;
2928 else
2929 __tun_detach(tfile, false);
2930 } else
2931 ret = -EINVAL;
2932
2933 if (ret >= 0)
2934 netdev_state_change(tun->dev);
2935
2936 unlock:
2937 rtnl_unlock();
2938 return ret;
2939 }
2940
2941 static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog __rcu **prog_p,
2942 void __user *data)
2943 {
2944 struct bpf_prog *prog;
2945 int fd;
2946
2947 if (copy_from_user(&fd, data, sizeof(fd)))
2948 return -EFAULT;
2949
2950 if (fd == -1) {
2951 prog = NULL;
2952 } else {
2953 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
2954 if (IS_ERR(prog))
2955 return PTR_ERR(prog);
2956 }
2957
2958 return __tun_set_ebpf(tun, prog_p, prog);
2959 }
2960
2961 /* Return correct value for tun->dev->addr_len based on tun->dev->type. */
2962 static unsigned char tun_get_addr_len(unsigned short type)
2963 {
2964 switch (type) {
2965 case ARPHRD_IP6GRE:
2966 case ARPHRD_TUNNEL6:
2967 return sizeof(struct in6_addr);
2968 case ARPHRD_IPGRE:
2969 case ARPHRD_TUNNEL:
2970 case ARPHRD_SIT:
2971 return 4;
2972 case ARPHRD_ETHER:
2973 return ETH_ALEN;
2974 case ARPHRD_IEEE802154:
2975 case ARPHRD_IEEE802154_MONITOR:
2976 return IEEE802154_EXTENDED_ADDR_LEN;
2977 case ARPHRD_PHONET_PIPE:
2978 case ARPHRD_PPP:
2979 case ARPHRD_NONE:
2980 return 0;
2981 case ARPHRD_6LOWPAN:
2982 return EUI64_ADDR_LEN;
2983 case ARPHRD_FDDI:
2984 return FDDI_K_ALEN;
2985 case ARPHRD_HIPPI:
2986 return HIPPI_ALEN;
2987 case ARPHRD_IEEE802:
2988 return FC_ALEN;
2989 case ARPHRD_ROSE:
2990 return ROSE_ADDR_LEN;
2991 case ARPHRD_NETROM:
2992 return AX25_ADDR_LEN;
2993 case ARPHRD_LOCALTLK:
2994 return LTALK_ALEN;
2995 default:
2996 return 0;
2997 }
2998 }
2999
3000 static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
3001 unsigned long arg, int ifreq_len)
3002 {
3003 struct tun_file *tfile = file->private_data;
3004 struct net *net = sock_net(&tfile->sk);
3005 struct tun_struct *tun;
3006 void __user* argp = (void __user*)arg;
3007 unsigned int carrier;
3008 struct ifreq ifr;
3009 kuid_t owner;
3010 kgid_t group;
3011 int ifindex;
3012 int sndbuf;
3013 int ret;
3014 bool do_notify = false;
3015
3016 if (cmd == TUNSETIFF || cmd == TUNSETQUEUE ||
3017 (_IOC_TYPE(cmd) == SOCK_IOC_TYPE && cmd != SIOCGSKNS)) {
3018 if (copy_from_user(&ifr, argp, ifreq_len))
3019 return -EFAULT;
3020 } else {
3021 memset(&ifr, 0, sizeof(ifr));
3022 }
3023 if (cmd == TUNGETFEATURES) {
3024 /* Currently this just means: "what IFF flags are valid?".
3025 * This is needed because we never checked for invalid flags on
3026 * TUNSETIFF.
3027 */
3028 return put_user(IFF_TUN | IFF_TAP | IFF_NO_CARRIER |
3029 TUN_FEATURES, (unsigned int __user*)argp);
3030 } else if (cmd == TUNSETQUEUE) {
3031 return tun_set_queue(file, &ifr);
3032 } else if (cmd == SIOCGSKNS) {
3033 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
3034 return -EPERM;
3035 return open_related_ns(&net->ns, get_net_ns);
3036 }
3037
3038 rtnl_lock();
3039
3040 tun = tun_get(tfile);
3041 if (cmd == TUNSETIFF) {
3042 ret = -EEXIST;
3043 if (tun)
3044 goto unlock;
3045
3046 ifr.ifr_name[IFNAMSIZ-1] = '\0';
3047
3048 ret = tun_set_iff(net, file, &ifr);
3049
3050 if (ret)
3051 goto unlock;
3052
3053 if (copy_to_user(argp, &ifr, ifreq_len))
3054 ret = -EFAULT;
3055 goto unlock;
3056 }
3057 if (cmd == TUNSETIFINDEX) {
3058 ret = -EPERM;
3059 if (tun)
3060 goto unlock;
3061
3062 ret = -EFAULT;
3063 if (copy_from_user(&ifindex, argp, sizeof(ifindex)))
3064 goto unlock;
3065 ret = -EINVAL;
3066 if (ifindex < 0)
3067 goto unlock;
3068 ret = 0;
3069 tfile->ifindex = ifindex;
3070 goto unlock;
3071 }
3072
3073 ret = -EBADFD;
3074 if (!tun)
3075 goto unlock;
3076
3077 netif_info(tun, drv, tun->dev, "tun_chr_ioctl cmd %u\n", cmd);
3078
3079 net = dev_net(tun->dev);
3080 ret = 0;
3081 switch (cmd) {
3082 case TUNGETIFF:
3083 tun_get_iff(tun, &ifr);
3084
3085 if (tfile->detached)
3086 ifr.ifr_flags |= IFF_DETACH_QUEUE;
3087 if (!tfile->socket.sk->sk_filter)
3088 ifr.ifr_flags |= IFF_NOFILTER;
3089
3090 if (copy_to_user(argp, &ifr, ifreq_len))
3091 ret = -EFAULT;
3092 break;
3093
3094 case TUNSETNOCSUM:
3095 /* Disable/Enable checksum */
3096
3097 /* [unimplemented] */
3098 netif_info(tun, drv, tun->dev, "ignored: set checksum %s\n",
3099 arg ? "disabled" : "enabled");
3100 break;
3101
3102 case TUNSETPERSIST:
3103 /* Disable/Enable persist mode. Keep an extra reference to the
3104 * module to prevent the module being unprobed.
3105 */
3106 if (arg && !(tun->flags & IFF_PERSIST)) {
3107 tun->flags |= IFF_PERSIST;
3108 __module_get(THIS_MODULE);
3109 do_notify = true;
3110 }
3111 if (!arg && (tun->flags & IFF_PERSIST)) {
3112 tun->flags &= ~IFF_PERSIST;
3113 module_put(THIS_MODULE);
3114 do_notify = true;
3115 }
3116
3117 netif_info(tun, drv, tun->dev, "persist %s\n",
3118 arg ? "enabled" : "disabled");
3119 break;
3120
3121 case TUNSETOWNER:
3122 /* Set owner of the device */
3123 owner = make_kuid(current_user_ns(), arg);
3124 if (!uid_valid(owner)) {
3125 ret = -EINVAL;
3126 break;
3127 }
3128 tun->owner = owner;
3129 do_notify = true;
3130 netif_info(tun, drv, tun->dev, "owner set to %u\n",
3131 from_kuid(&init_user_ns, tun->owner));
3132 break;
3133
3134 case TUNSETGROUP:
3135 /* Set group of the device */
3136 group = make_kgid(current_user_ns(), arg);
3137 if (!gid_valid(group)) {
3138 ret = -EINVAL;
3139 break;
3140 }
3141 tun->group = group;
3142 do_notify = true;
3143 netif_info(tun, drv, tun->dev, "group set to %u\n",
3144 from_kgid(&init_user_ns, tun->group));
3145 break;
3146
3147 case TUNSETLINK:
3148 /* Only allow setting the type when the interface is down */
3149 if (tun->dev->flags & IFF_UP) {
3150 netif_info(tun, drv, tun->dev,
3151 "Linktype set failed because interface is up\n");
3152 ret = -EBUSY;
3153 } else {
3154 ret = call_netdevice_notifiers(NETDEV_PRE_TYPE_CHANGE,
3155 tun->dev);
3156 ret = notifier_to_errno(ret);
3157 if (ret) {
3158 netif_info(tun, drv, tun->dev,
3159 "Refused to change device type\n");
3160 break;
3161 }
3162 tun->dev->type = (int) arg;
3163 tun->dev->addr_len = tun_get_addr_len(tun->dev->type);
3164 netif_info(tun, drv, tun->dev, "linktype set to %d\n",
3165 tun->dev->type);
3166 call_netdevice_notifiers(NETDEV_POST_TYPE_CHANGE,
3167 tun->dev);
3168 }
3169 break;
3170
3171 case TUNSETDEBUG:
3172 tun->msg_enable = (u32)arg;
3173 break;
3174
3175 case TUNSETOFFLOAD:
3176 ret = set_offload(tun, arg);
3177 break;
3178
3179 case TUNSETTXFILTER:
3180 /* Can be set only for TAPs */
3181 ret = -EINVAL;
3182 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3183 break;
3184 ret = update_filter(&tun->txflt, (void __user *)arg);
3185 break;
3186
3187 case SIOCGIFHWADDR:
3188 /* Get hw address */
3189 dev_get_mac_address(&ifr.ifr_hwaddr, net, tun->dev->name);
3190 if (copy_to_user(argp, &ifr, ifreq_len))
3191 ret = -EFAULT;
3192 break;
3193
3194 case SIOCSIFHWADDR:
3195 /* Set hw address */
3196 if (tun->dev->addr_len > sizeof(ifr.ifr_hwaddr)) {
3197 ret = -EINVAL;
3198 break;
3199 }
3200 ret = dev_set_mac_address_user(tun->dev,
3201 (struct sockaddr_storage *)&ifr.ifr_hwaddr,
3202 NULL);
3203 break;
3204
3205 case TUNGETSNDBUF:
3206 sndbuf = tfile->socket.sk->sk_sndbuf;
3207 if (copy_to_user(argp, &sndbuf, sizeof(sndbuf)))
3208 ret = -EFAULT;
3209 break;
3210
3211 case TUNSETSNDBUF:
3212 if (copy_from_user(&sndbuf, argp, sizeof(sndbuf))) {
3213 ret = -EFAULT;
3214 break;
3215 }
3216 if (sndbuf <= 0) {
3217 ret = -EINVAL;
3218 break;
3219 }
3220
3221 tun->sndbuf = sndbuf;
3222 tun_set_sndbuf(tun);
3223 break;
3224
3225 case TUNATTACHFILTER:
3226 /* Can be set only for TAPs */
3227 ret = -EINVAL;
3228 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3229 break;
3230 ret = -EFAULT;
3231 if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog)))
3232 break;
3233
3234 ret = tun_attach_filter(tun);
3235 break;
3236
3237 case TUNDETACHFILTER:
3238 /* Can be set only for TAPs */
3239 ret = -EINVAL;
3240 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3241 break;
3242 ret = 0;
3243 tun_detach_filter(tun, tun->numqueues);
3244 break;
3245
3246 case TUNGETFILTER:
3247 ret = -EINVAL;
3248 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3249 break;
3250 ret = -EFAULT;
3251 if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog)))
3252 break;
3253 ret = 0;
3254 break;
3255
3256 case TUNSETSTEERINGEBPF:
3257 ret = tun_set_ebpf(tun, &tun->steering_prog, argp);
3258 break;
3259
3260 case TUNSETFILTEREBPF:
3261 ret = tun_set_ebpf(tun, &tun->filter_prog, argp);
3262 break;
3263
3264 case TUNSETCARRIER:
3265 ret = -EFAULT;
3266 if (copy_from_user(&carrier, argp, sizeof(carrier)))
3267 goto unlock;
3268
3269 ret = tun_net_change_carrier(tun->dev, (bool)carrier);
3270 break;
3271
3272 case TUNGETDEVNETNS:
3273 ret = -EPERM;
3274 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
3275 goto unlock;
3276 ret = open_related_ns(&net->ns, get_net_ns);
3277 break;
3278
3279 default:
3280 ret = tun_vnet_ioctl(&tun->vnet_hdr_sz, &tun->flags, cmd, argp);
3281 break;
3282 }
3283
3284 if (do_notify)
3285 netdev_state_change(tun->dev);
3286
3287 unlock:
3288 rtnl_unlock();
3289 if (tun)
3290 tun_put(tun);
3291 return ret;
3292 }
3293
3294 static long tun_chr_ioctl(struct file *file,
3295 unsigned int cmd, unsigned long arg)
3296 {
3297 return __tun_chr_ioctl(file, cmd, arg, sizeof (struct ifreq));
3298 }
3299
3300 #ifdef CONFIG_COMPAT
3301 static long tun_chr_compat_ioctl(struct file *file,
3302 unsigned int cmd, unsigned long arg)
3303 {
3304 switch (cmd) {
3305 case TUNSETIFF:
3306 case TUNGETIFF:
3307 case TUNSETTXFILTER:
3308 case TUNGETSNDBUF:
3309 case TUNSETSNDBUF:
3310 case SIOCGIFHWADDR:
3311 case SIOCSIFHWADDR:
3312 arg = (unsigned long)compat_ptr(arg);
3313 break;
3314 default:
3315 arg = (compat_ulong_t)arg;
3316 break;
3317 }
3318
3319 /*
3320 * compat_ifreq is shorter than ifreq, so we must not access beyond
3321 * the end of that structure. All fields that are used in this
3322 * driver are compatible though, we don't need to convert the
3323 * contents.
3324 */
3325 return __tun_chr_ioctl(file, cmd, arg, sizeof(struct compat_ifreq));
3326 }
3327 #endif /* CONFIG_COMPAT */
3328
3329 static int tun_chr_fasync(int fd, struct file *file, int on)
3330 {
3331 struct tun_file *tfile = file->private_data;
3332 int ret;
3333
3334 if (on) {
3335 ret = file_f_owner_allocate(file);
3336 if (ret)
3337 goto out;
3338 }
3339
3340 if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0)
3341 goto out;
3342
3343 if (on) {
3344 __f_setown(file, task_pid(current), PIDTYPE_TGID, 0);
3345 tfile->flags |= TUN_FASYNC;
3346 } else
3347 tfile->flags &= ~TUN_FASYNC;
3348 ret = 0;
3349 out:
3350 return ret;
3351 }
3352
3353 static int tun_chr_open(struct inode *inode, struct file * file)
3354 {
3355 struct net *net = current->nsproxy->net_ns;
3356 struct tun_file *tfile;
3357
3358 tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,
3359 &tun_proto, 0);
3360 if (!tfile)
3361 return -ENOMEM;
3362 if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) {
3363 sk_free(&tfile->sk);
3364 return -ENOMEM;
3365 }
3366
3367 mutex_init(&tfile->napi_mutex);
3368 RCU_INIT_POINTER(tfile->tun, NULL);
3369 tfile->flags = 0;
3370 tfile->ifindex = 0;
3371
3372 init_waitqueue_head(&tfile->socket.wq.wait);
3373
3374 tfile->socket.file = file;
3375 tfile->socket.ops = &tun_socket_ops;
3376
3377 sock_init_data_uid(&tfile->socket, &tfile->sk, current_fsuid());
3378
3379 tfile->sk.sk_write_space = tun_sock_write_space;
3380 tfile->sk.sk_sndbuf = INT_MAX;
3381
3382 file->private_data = tfile;
3383 INIT_LIST_HEAD(&tfile->next);
3384
3385 sock_set_flag(&tfile->sk, SOCK_ZEROCOPY);
3386
3387 /* tun groks IOCB_NOWAIT just fine, mark it as such */
3388 file->f_mode |= FMODE_NOWAIT;
3389 return 0;
3390 }
3391
3392 static int tun_chr_close(struct inode *inode, struct file *file)
3393 {
3394 struct tun_file *tfile = file->private_data;
3395
3396 tun_detach(tfile, true);
3397
3398 return 0;
3399 }
3400
3401 #ifdef CONFIG_PROC_FS
3402 static void tun_chr_show_fdinfo(struct seq_file *m, struct file *file)
3403 {
3404 struct tun_file *tfile = file->private_data;
3405 struct tun_struct *tun;
3406 struct ifreq ifr;
3407
3408 memset(&ifr, 0, sizeof(ifr));
3409
3410 rtnl_lock();
3411 tun = tun_get(tfile);
3412 if (tun)
3413 tun_get_iff(tun, &ifr);
3414 rtnl_unlock();
3415
3416 if (tun)
3417 tun_put(tun);
3418
3419 seq_printf(m, "iff:\t%s\n", ifr.ifr_name);
3420 }
3421 #endif
3422
3423 static const struct file_operations tun_fops = {
3424 .owner = THIS_MODULE,
3425 .read_iter = tun_chr_read_iter,
3426 .write_iter = tun_chr_write_iter,
3427 .poll = tun_chr_poll,
3428 .unlocked_ioctl = tun_chr_ioctl,
3429 #ifdef CONFIG_COMPAT
3430 .compat_ioctl = tun_chr_compat_ioctl,
3431 #endif
3432 .open = tun_chr_open,
3433 .release = tun_chr_close,
3434 .fasync = tun_chr_fasync,
3435 #ifdef CONFIG_PROC_FS
3436 .show_fdinfo = tun_chr_show_fdinfo,
3437 #endif
3438 };
3439
3440 static struct miscdevice tun_miscdev = {
3441 .minor = TUN_MINOR,
3442 .name = "tun",
3443 .nodename = "net/tun",
3444 .fops = &tun_fops,
3445 };
3446
3447 /* ethtool interface */
3448
3449 static void tun_default_link_ksettings(struct net_device *dev,
3450 struct ethtool_link_ksettings *cmd)
3451 {
3452 ethtool_link_ksettings_zero_link_mode(cmd, supported);
3453 ethtool_link_ksettings_zero_link_mode(cmd, advertising);
3454 cmd->base.speed = SPEED_10000;
3455 cmd->base.duplex = DUPLEX_FULL;
3456 cmd->base.port = PORT_TP;
3457 cmd->base.phy_address = 0;
3458 cmd->base.autoneg = AUTONEG_DISABLE;
3459 }
3460
3461 static int tun_get_link_ksettings(struct net_device *dev,
3462 struct ethtool_link_ksettings *cmd)
3463 {
3464 struct tun_struct *tun = netdev_priv(dev);
3465
3466 memcpy(cmd, &tun->link_ksettings, sizeof(*cmd));
3467 return 0;
3468 }
3469
3470 static int tun_set_link_ksettings(struct net_device *dev,
3471 const struct ethtool_link_ksettings *cmd)
3472 {
3473 struct tun_struct *tun = netdev_priv(dev);
3474
3475 memcpy(&tun->link_ksettings, cmd, sizeof(*cmd));
3476 return 0;
3477 }
3478
3479 static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
3480 {
3481 struct tun_struct *tun = netdev_priv(dev);
3482
3483 strscpy(info->driver, DRV_NAME, sizeof(info->driver));
3484 strscpy(info->version, DRV_VERSION, sizeof(info->version));
3485
3486 switch (tun->flags & TUN_TYPE_MASK) {
3487 case IFF_TUN:
3488 strscpy(info->bus_info, "tun", sizeof(info->bus_info));
3489 break;
3490 case IFF_TAP:
3491 strscpy(info->bus_info, "tap", sizeof(info->bus_info));
3492 break;
3493 }
3494 }
3495
3496 static u32 tun_get_msglevel(struct net_device *dev)
3497 {
3498 struct tun_struct *tun = netdev_priv(dev);
3499
3500 return tun->msg_enable;
3501 }
3502
3503 static void tun_set_msglevel(struct net_device *dev, u32 value)
3504 {
3505 struct tun_struct *tun = netdev_priv(dev);
3506
3507 tun->msg_enable = value;
3508 }
3509
3510 static int tun_get_coalesce(struct net_device *dev,
3511 struct ethtool_coalesce *ec,
3512 struct kernel_ethtool_coalesce *kernel_coal,
3513 struct netlink_ext_ack *extack)
3514 {
3515 struct tun_struct *tun = netdev_priv(dev);
3516
3517 ec->rx_max_coalesced_frames = tun->rx_batched;
3518
3519 return 0;
3520 }
3521
3522 static int tun_set_coalesce(struct net_device *dev,
3523 struct ethtool_coalesce *ec,
3524 struct kernel_ethtool_coalesce *kernel_coal,
3525 struct netlink_ext_ack *extack)
3526 {
3527 struct tun_struct *tun = netdev_priv(dev);
3528
3529 if (ec->rx_max_coalesced_frames > NAPI_POLL_WEIGHT)
3530 tun->rx_batched = NAPI_POLL_WEIGHT;
3531 else
3532 tun->rx_batched = ec->rx_max_coalesced_frames;
3533
3534 return 0;
3535 }
3536
3537 static void tun_get_channels(struct net_device *dev,
3538 struct ethtool_channels *channels)
3539 {
3540 struct tun_struct *tun = netdev_priv(dev);
3541
3542 channels->combined_count = tun->numqueues;
3543 channels->max_combined = tun->flags & IFF_MULTI_QUEUE ? MAX_TAP_QUEUES : 1;
3544 }
3545
3546 static const struct ethtool_ops tun_ethtool_ops = {
3547 .supported_coalesce_params = ETHTOOL_COALESCE_RX_MAX_FRAMES,
3548 .get_drvinfo = tun_get_drvinfo,
3549 .get_msglevel = tun_get_msglevel,
3550 .set_msglevel = tun_set_msglevel,
3551 .get_link = ethtool_op_get_link,
3552 .get_channels = tun_get_channels,
3553 .get_ts_info = ethtool_op_get_ts_info,
3554 .get_coalesce = tun_get_coalesce,
3555 .set_coalesce = tun_set_coalesce,
3556 .get_link_ksettings = tun_get_link_ksettings,
3557 .set_link_ksettings = tun_set_link_ksettings,
3558 };
3559
3560 static int tun_queue_resize(struct tun_struct *tun)
3561 {
3562 struct net_device *dev = tun->dev;
3563 struct tun_file *tfile;
3564 struct ptr_ring **rings;
3565 int n = tun->numqueues + tun->numdisabled;
3566 int ret, i;
3567
3568 rings = kmalloc_array(n, sizeof(*rings), GFP_KERNEL);
3569 if (!rings)
3570 return -ENOMEM;
3571
3572 for (i = 0; i < tun->numqueues; i++) {
3573 tfile = rtnl_dereference(tun->tfiles[i]);
3574 rings[i] = &tfile->tx_ring;
3575 }
3576 list_for_each_entry(tfile, &tun->disabled, next)
3577 rings[i++] = &tfile->tx_ring;
3578
3579 ret = ptr_ring_resize_multiple_bh(rings, n,
3580 dev->tx_queue_len, GFP_KERNEL,
3581 tun_ptr_free);
3582
3583 kfree(rings);
3584 return ret;
3585 }
3586
3587 static int tun_device_event(struct notifier_block *unused,
3588 unsigned long event, void *ptr)
3589 {
3590 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3591 struct tun_struct *tun = netdev_priv(dev);
3592 int i;
3593
3594 if (dev->rtnl_link_ops != &tun_link_ops)
3595 return NOTIFY_DONE;
3596
3597 switch (event) {
3598 case NETDEV_CHANGE_TX_QUEUE_LEN:
3599 if (tun_queue_resize(tun))
3600 return NOTIFY_BAD;
3601 break;
3602 case NETDEV_UP:
3603 for (i = 0; i < tun->numqueues; i++) {
3604 struct tun_file *tfile;
3605
3606 tfile = rtnl_dereference(tun->tfiles[i]);
3607 tfile->socket.sk->sk_write_space(tfile->socket.sk);
3608 }
3609 break;
3610 default:
3611 break;
3612 }
3613
3614 return NOTIFY_DONE;
3615 }
3616
3617 static struct notifier_block tun_notifier_block __read_mostly = {
3618 .notifier_call = tun_device_event,
3619 };
3620
3621 static int __init tun_init(void)
3622 {
3623 int ret = 0;
3624
3625 pr_info("%s, %s\n", DRV_DESCRIPTION, DRV_VERSION);
3626
3627 ret = rtnl_link_register(&tun_link_ops);
3628 if (ret) {
3629 pr_err("Can't register link_ops\n");
3630 goto err_linkops;
3631 }
3632
3633 ret = misc_register(&tun_miscdev);
3634 if (ret) {
3635 pr_err("Can't register misc device %d\n", TUN_MINOR);
3636 goto err_misc;
3637 }
3638
3639 ret = register_netdevice_notifier(&tun_notifier_block);
3640 if (ret) {
3641 pr_err("Can't register netdevice notifier\n");
3642 goto err_notifier;
3643 }
3644
3645 return 0;
3646
3647 err_notifier:
3648 misc_deregister(&tun_miscdev);
3649 err_misc:
3650 rtnl_link_unregister(&tun_link_ops);
3651 err_linkops:
3652 return ret;
3653 }
3654
3655 static void __exit tun_cleanup(void)
3656 {
3657 misc_deregister(&tun_miscdev);
3658 rtnl_link_unregister(&tun_link_ops);
3659 unregister_netdevice_notifier(&tun_notifier_block);
3660 }
3661
3662 /* Get an underlying socket object from tun file. Returns error unless file is
3663 * attached to a device. The returned object works like a packet socket, it
3664 * can be used for sock_sendmsg/sock_recvmsg. The caller is responsible for
3665 * holding a reference to the file for as long as the socket is in use. */
3666 struct socket *tun_get_socket(struct file *file)
3667 {
3668 struct tun_file *tfile;
3669 if (file->f_op != &tun_fops)
3670 return ERR_PTR(-EINVAL);
3671 tfile = file->private_data;
3672 if (!tfile)
3673 return ERR_PTR(-EBADFD);
3674 return &tfile->socket;
3675 }
3676 EXPORT_SYMBOL_GPL(tun_get_socket);
3677
3678 struct ptr_ring *tun_get_tx_ring(struct file *file)
3679 {
3680 struct tun_file *tfile;
3681
3682 if (file->f_op != &tun_fops)
3683 return ERR_PTR(-EINVAL);
3684 tfile = file->private_data;
3685 if (!tfile)
3686 return ERR_PTR(-EBADFD);
3687 return &tfile->tx_ring;
3688 }
3689 EXPORT_SYMBOL_GPL(tun_get_tx_ring);
3690
3691 module_init(tun_init);
3692 module_exit(tun_cleanup);
3693 MODULE_DESCRIPTION(DRV_DESCRIPTION);
3694 MODULE_AUTHOR(DRV_COPYRIGHT);
3695 MODULE_LICENSE("GPL");
3696 MODULE_ALIAS_MISCDEV(TUN_MINOR);
3697 MODULE_ALIAS("devname:net/tun");
A mirror of Linus' kernel repository