1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2009, Microsoft Corporation.
6 * Haiyang Zhang <haiyangz@microsoft.com>
7 * Hank Janssen <hjanssen@microsoft.com>
9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11 #include <linux/init.h>
12 #include <linux/atomic.h>
13 #include <linux/module.h>
14 #include <linux/highmem.h>
15 #include <linux/device.h>
17 #include <linux/delay.h>
18 #include <linux/netdevice.h>
19 #include <linux/inetdevice.h>
20 #include <linux/etherdevice.h>
21 #include <linux/pci.h>
22 #include <linux/skbuff.h>
23 #include <linux/if_vlan.h>
25 #include <linux/slab.h>
26 #include <linux/rtnetlink.h>
27 #include <linux/netpoll.h>
30 #include <net/route.h>
32 #include <net/pkt_sched.h>
33 #include <net/checksum.h>
34 #include <net/ip6_checksum.h>
36 #include "hyperv_net.h"
38 #define RING_SIZE_MIN 64
39 #define RETRY_US_LO 5000
40 #define RETRY_US_HI 10000
41 #define RETRY_MAX 2000 /* >10 sec */
43 #define LINKCHANGE_INT (2 * HZ)
44 #define VF_TAKEOVER_INT (HZ / 10)
46 static unsigned int ring_size __ro_after_init
= 128;
47 module_param(ring_size
, uint
, 0444);
48 MODULE_PARM_DESC(ring_size
, "Ring buffer size (# of pages)");
49 unsigned int netvsc_ring_bytes __ro_after_init
;
51 static const u32 default_msg
= NETIF_MSG_DRV
| NETIF_MSG_PROBE
|
52 NETIF_MSG_LINK
| NETIF_MSG_IFUP
|
53 NETIF_MSG_IFDOWN
| NETIF_MSG_RX_ERR
|
56 static int debug
= -1;
57 module_param(debug
, int, 0444);
58 MODULE_PARM_DESC(debug
, "Debug level (0=none,...,16=all)");
60 static LIST_HEAD(netvsc_dev_list
);
62 static void netvsc_change_rx_flags(struct net_device
*net
, int change
)
64 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
65 struct net_device
*vf_netdev
= rtnl_dereference(ndev_ctx
->vf_netdev
);
71 if (change
& IFF_PROMISC
) {
72 inc
= (net
->flags
& IFF_PROMISC
) ? 1 : -1;
73 dev_set_promiscuity(vf_netdev
, inc
);
76 if (change
& IFF_ALLMULTI
) {
77 inc
= (net
->flags
& IFF_ALLMULTI
) ? 1 : -1;
78 dev_set_allmulti(vf_netdev
, inc
);
82 static void netvsc_set_rx_mode(struct net_device
*net
)
84 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
85 struct net_device
*vf_netdev
;
86 struct netvsc_device
*nvdev
;
89 vf_netdev
= rcu_dereference(ndev_ctx
->vf_netdev
);
91 dev_uc_sync(vf_netdev
, net
);
92 dev_mc_sync(vf_netdev
, net
);
95 nvdev
= rcu_dereference(ndev_ctx
->nvdev
);
97 rndis_filter_update(nvdev
);
101 static void netvsc_tx_enable(struct netvsc_device
*nvscdev
,
102 struct net_device
*ndev
)
104 nvscdev
->tx_disable
= false;
105 virt_wmb(); /* ensure queue wake up mechanism is on */
107 netif_tx_wake_all_queues(ndev
);
110 static int netvsc_open(struct net_device
*net
)
112 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
113 struct net_device
*vf_netdev
= rtnl_dereference(ndev_ctx
->vf_netdev
);
114 struct netvsc_device
*nvdev
= rtnl_dereference(ndev_ctx
->nvdev
);
115 struct rndis_device
*rdev
;
118 netif_carrier_off(net
);
120 /* Open up the device */
121 ret
= rndis_filter_open(nvdev
);
123 netdev_err(net
, "unable to open device (ret %d).\n", ret
);
127 rdev
= nvdev
->extension
;
128 if (!rdev
->link_state
) {
129 netif_carrier_on(net
);
130 netvsc_tx_enable(nvdev
, net
);
134 /* Setting synthetic device up transparently sets
135 * slave as up. If open fails, then slave will be
136 * still be offline (and not used).
138 ret
= dev_open(vf_netdev
, NULL
);
141 "unable to open slave: %s: %d\n",
142 vf_netdev
->name
, ret
);
147 static int netvsc_wait_until_empty(struct netvsc_device
*nvdev
)
149 unsigned int retry
= 0;
152 /* Ensure pending bytes in ring are read */
156 for (i
= 0; i
< nvdev
->num_chn
; i
++) {
157 struct vmbus_channel
*chn
158 = nvdev
->chan_table
[i
].channel
;
163 /* make sure receive not running now */
164 napi_synchronize(&nvdev
->chan_table
[i
].napi
);
166 aread
= hv_get_bytes_to_read(&chn
->inbound
);
170 aread
= hv_get_bytes_to_read(&chn
->outbound
);
178 if (++retry
> RETRY_MAX
)
181 usleep_range(RETRY_US_LO
, RETRY_US_HI
);
185 static void netvsc_tx_disable(struct netvsc_device
*nvscdev
,
186 struct net_device
*ndev
)
189 nvscdev
->tx_disable
= true;
190 virt_wmb(); /* ensure txq will not wake up after stop */
193 netif_tx_disable(ndev
);
196 static int netvsc_close(struct net_device
*net
)
198 struct net_device_context
*net_device_ctx
= netdev_priv(net
);
199 struct net_device
*vf_netdev
200 = rtnl_dereference(net_device_ctx
->vf_netdev
);
201 struct netvsc_device
*nvdev
= rtnl_dereference(net_device_ctx
->nvdev
);
204 netvsc_tx_disable(nvdev
, net
);
206 /* No need to close rndis filter if it is removed already */
210 ret
= rndis_filter_close(nvdev
);
212 netdev_err(net
, "unable to close device (ret %d).\n", ret
);
216 ret
= netvsc_wait_until_empty(nvdev
);
218 netdev_err(net
, "Ring buffer not empty after closing rndis\n");
221 dev_close(vf_netdev
);
226 static inline void *init_ppi_data(struct rndis_message
*msg
,
227 u32 ppi_size
, u32 pkt_type
)
229 struct rndis_packet
*rndis_pkt
= &msg
->msg
.pkt
;
230 struct rndis_per_packet_info
*ppi
;
232 rndis_pkt
->data_offset
+= ppi_size
;
233 ppi
= (void *)rndis_pkt
+ rndis_pkt
->per_pkt_info_offset
234 + rndis_pkt
->per_pkt_info_len
;
236 ppi
->size
= ppi_size
;
237 ppi
->type
= pkt_type
;
239 ppi
->ppi_offset
= sizeof(struct rndis_per_packet_info
);
241 rndis_pkt
->per_pkt_info_len
+= ppi_size
;
246 /* Azure hosts don't support non-TCP port numbers in hashing for fragmented
247 * packets. We can use ethtool to change UDP hash level when necessary.
249 static inline u32
netvsc_get_hash(
251 const struct net_device_context
*ndc
)
253 struct flow_keys flow
;
254 u32 hash
, pkt_proto
= 0;
255 static u32 hashrnd __read_mostly
;
257 net_get_random_once(&hashrnd
, sizeof(hashrnd
));
259 if (!skb_flow_dissect_flow_keys(skb
, &flow
, 0))
262 switch (flow
.basic
.ip_proto
) {
264 if (flow
.basic
.n_proto
== htons(ETH_P_IP
))
265 pkt_proto
= HV_TCP4_L4HASH
;
266 else if (flow
.basic
.n_proto
== htons(ETH_P_IPV6
))
267 pkt_proto
= HV_TCP6_L4HASH
;
272 if (flow
.basic
.n_proto
== htons(ETH_P_IP
))
273 pkt_proto
= HV_UDP4_L4HASH
;
274 else if (flow
.basic
.n_proto
== htons(ETH_P_IPV6
))
275 pkt_proto
= HV_UDP6_L4HASH
;
280 if (pkt_proto
& ndc
->l4_hash
) {
281 return skb_get_hash(skb
);
283 if (flow
.basic
.n_proto
== htons(ETH_P_IP
))
284 hash
= jhash2((u32
*)&flow
.addrs
.v4addrs
, 2, hashrnd
);
285 else if (flow
.basic
.n_proto
== htons(ETH_P_IPV6
))
286 hash
= jhash2((u32
*)&flow
.addrs
.v6addrs
, 8, hashrnd
);
290 skb_set_hash(skb
, hash
, PKT_HASH_TYPE_L3
);
296 static inline int netvsc_get_tx_queue(struct net_device
*ndev
,
297 struct sk_buff
*skb
, int old_idx
)
299 const struct net_device_context
*ndc
= netdev_priv(ndev
);
300 struct sock
*sk
= skb
->sk
;
303 q_idx
= ndc
->tx_table
[netvsc_get_hash(skb
, ndc
) &
304 (VRSS_SEND_TAB_SIZE
- 1)];
306 /* If queue index changed record the new value */
307 if (q_idx
!= old_idx
&&
308 sk
&& sk_fullsock(sk
) && rcu_access_pointer(sk
->sk_dst_cache
))
309 sk_tx_queue_set(sk
, q_idx
);
315 * Select queue for transmit.
317 * If a valid queue has already been assigned, then use that.
318 * Otherwise compute tx queue based on hash and the send table.
320 * This is basically similar to default (netdev_pick_tx) with the added step
321 * of using the host send_table when no other queue has been assigned.
323 * TODO support XPS - but get_xps_queue not exported
325 static u16
netvsc_pick_tx(struct net_device
*ndev
, struct sk_buff
*skb
)
327 int q_idx
= sk_tx_queue_get(skb
->sk
);
329 if (q_idx
< 0 || skb
->ooo_okay
|| q_idx
>= ndev
->real_num_tx_queues
) {
330 /* If forwarding a packet, we use the recorded queue when
331 * available for better cache locality.
333 if (skb_rx_queue_recorded(skb
))
334 q_idx
= skb_get_rx_queue(skb
);
336 q_idx
= netvsc_get_tx_queue(ndev
, skb
, q_idx
);
342 static u16
netvsc_select_queue(struct net_device
*ndev
, struct sk_buff
*skb
,
343 struct net_device
*sb_dev
)
345 struct net_device_context
*ndc
= netdev_priv(ndev
);
346 struct net_device
*vf_netdev
;
350 vf_netdev
= rcu_dereference(ndc
->vf_netdev
);
352 const struct net_device_ops
*vf_ops
= vf_netdev
->netdev_ops
;
354 if (vf_ops
->ndo_select_queue
)
355 txq
= vf_ops
->ndo_select_queue(vf_netdev
, skb
, sb_dev
);
357 txq
= netdev_pick_tx(vf_netdev
, skb
, NULL
);
359 /* Record the queue selected by VF so that it can be
360 * used for common case where VF has more queues than
361 * the synthetic device.
363 qdisc_skb_cb(skb
)->slave_dev_queue_mapping
= txq
;
365 txq
= netvsc_pick_tx(ndev
, skb
);
369 while (unlikely(txq
>= ndev
->real_num_tx_queues
))
370 txq
-= ndev
->real_num_tx_queues
;
375 static u32
fill_pg_buf(struct page
*page
, u32 offset
, u32 len
,
376 struct hv_page_buffer
*pb
)
380 /* Deal with compound pages by ignoring unused part
383 page
+= (offset
>> PAGE_SHIFT
);
384 offset
&= ~PAGE_MASK
;
389 bytes
= PAGE_SIZE
- offset
;
392 pb
[j
].pfn
= page_to_pfn(page
);
393 pb
[j
].offset
= offset
;
399 if (offset
== PAGE_SIZE
&& len
) {
409 static u32
init_page_array(void *hdr
, u32 len
, struct sk_buff
*skb
,
410 struct hv_netvsc_packet
*packet
,
411 struct hv_page_buffer
*pb
)
414 char *data
= skb
->data
;
415 int frags
= skb_shinfo(skb
)->nr_frags
;
418 /* The packet is laid out thus:
419 * 1. hdr: RNDIS header and PPI
421 * 3. skb fragment data
423 slots_used
+= fill_pg_buf(virt_to_page(hdr
),
425 len
, &pb
[slots_used
]);
427 packet
->rmsg_size
= len
;
428 packet
->rmsg_pgcnt
= slots_used
;
430 slots_used
+= fill_pg_buf(virt_to_page(data
),
431 offset_in_page(data
),
432 skb_headlen(skb
), &pb
[slots_used
]);
434 for (i
= 0; i
< frags
; i
++) {
435 skb_frag_t
*frag
= skb_shinfo(skb
)->frags
+ i
;
437 slots_used
+= fill_pg_buf(skb_frag_page(frag
),
439 skb_frag_size(frag
), &pb
[slots_used
]);
444 static int count_skb_frag_slots(struct sk_buff
*skb
)
446 int i
, frags
= skb_shinfo(skb
)->nr_frags
;
449 for (i
= 0; i
< frags
; i
++) {
450 skb_frag_t
*frag
= skb_shinfo(skb
)->frags
+ i
;
451 unsigned long size
= skb_frag_size(frag
);
452 unsigned long offset
= frag
->page_offset
;
454 /* Skip unused frames from start of page */
455 offset
&= ~PAGE_MASK
;
456 pages
+= PFN_UP(offset
+ size
);
461 static int netvsc_get_slots(struct sk_buff
*skb
)
463 char *data
= skb
->data
;
464 unsigned int offset
= offset_in_page(data
);
465 unsigned int len
= skb_headlen(skb
);
469 slots
= DIV_ROUND_UP(offset
+ len
, PAGE_SIZE
);
470 frag_slots
= count_skb_frag_slots(skb
);
471 return slots
+ frag_slots
;
474 static u32
net_checksum_info(struct sk_buff
*skb
)
476 if (skb
->protocol
== htons(ETH_P_IP
)) {
477 struct iphdr
*ip
= ip_hdr(skb
);
479 if (ip
->protocol
== IPPROTO_TCP
)
480 return TRANSPORT_INFO_IPV4_TCP
;
481 else if (ip
->protocol
== IPPROTO_UDP
)
482 return TRANSPORT_INFO_IPV4_UDP
;
484 struct ipv6hdr
*ip6
= ipv6_hdr(skb
);
486 if (ip6
->nexthdr
== IPPROTO_TCP
)
487 return TRANSPORT_INFO_IPV6_TCP
;
488 else if (ip6
->nexthdr
== IPPROTO_UDP
)
489 return TRANSPORT_INFO_IPV6_UDP
;
492 return TRANSPORT_INFO_NOT_IP
;
495 /* Send skb on the slave VF device. */
496 static int netvsc_vf_xmit(struct net_device
*net
, struct net_device
*vf_netdev
,
499 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
500 unsigned int len
= skb
->len
;
503 skb
->dev
= vf_netdev
;
504 skb
->queue_mapping
= qdisc_skb_cb(skb
)->slave_dev_queue_mapping
;
506 rc
= dev_queue_xmit(skb
);
507 if (likely(rc
== NET_XMIT_SUCCESS
|| rc
== NET_XMIT_CN
)) {
508 struct netvsc_vf_pcpu_stats
*pcpu_stats
509 = this_cpu_ptr(ndev_ctx
->vf_stats
);
511 u64_stats_update_begin(&pcpu_stats
->syncp
);
512 pcpu_stats
->tx_packets
++;
513 pcpu_stats
->tx_bytes
+= len
;
514 u64_stats_update_end(&pcpu_stats
->syncp
);
516 this_cpu_inc(ndev_ctx
->vf_stats
->tx_dropped
);
522 static int netvsc_start_xmit(struct sk_buff
*skb
, struct net_device
*net
)
524 struct net_device_context
*net_device_ctx
= netdev_priv(net
);
525 struct hv_netvsc_packet
*packet
= NULL
;
527 unsigned int num_data_pgs
;
528 struct rndis_message
*rndis_msg
;
529 struct net_device
*vf_netdev
;
532 struct hv_page_buffer pb
[MAX_PAGE_BUFFER_COUNT
];
534 /* if VF is present and up then redirect packets
535 * already called with rcu_read_lock_bh
537 vf_netdev
= rcu_dereference_bh(net_device_ctx
->vf_netdev
);
538 if (vf_netdev
&& netif_running(vf_netdev
) &&
539 !netpoll_tx_running(net
))
540 return netvsc_vf_xmit(net
, vf_netdev
, skb
);
542 /* We will atmost need two pages to describe the rndis
543 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number
544 * of pages in a single packet. If skb is scattered around
545 * more pages we try linearizing it.
548 num_data_pgs
= netvsc_get_slots(skb
) + 2;
550 if (unlikely(num_data_pgs
> MAX_PAGE_BUFFER_COUNT
)) {
551 ++net_device_ctx
->eth_stats
.tx_scattered
;
553 if (skb_linearize(skb
))
556 num_data_pgs
= netvsc_get_slots(skb
) + 2;
557 if (num_data_pgs
> MAX_PAGE_BUFFER_COUNT
) {
558 ++net_device_ctx
->eth_stats
.tx_too_big
;
564 * Place the rndis header in the skb head room and
565 * the skb->cb will be used for hv_netvsc_packet
568 ret
= skb_cow_head(skb
, RNDIS_AND_PPI_SIZE
);
572 /* Use the skb control buffer for building up the packet */
573 BUILD_BUG_ON(sizeof(struct hv_netvsc_packet
) >
574 FIELD_SIZEOF(struct sk_buff
, cb
));
575 packet
= (struct hv_netvsc_packet
*)skb
->cb
;
577 packet
->q_idx
= skb_get_queue_mapping(skb
);
579 packet
->total_data_buflen
= skb
->len
;
580 packet
->total_bytes
= skb
->len
;
581 packet
->total_packets
= 1;
583 rndis_msg
= (struct rndis_message
*)skb
->head
;
585 /* Add the rndis header */
586 rndis_msg
->ndis_msg_type
= RNDIS_MSG_PACKET
;
587 rndis_msg
->msg_len
= packet
->total_data_buflen
;
589 rndis_msg
->msg
.pkt
= (struct rndis_packet
) {
590 .data_offset
= sizeof(struct rndis_packet
),
591 .data_len
= packet
->total_data_buflen
,
592 .per_pkt_info_offset
= sizeof(struct rndis_packet
),
595 rndis_msg_size
= RNDIS_MESSAGE_SIZE(struct rndis_packet
);
597 hash
= skb_get_hash_raw(skb
);
598 if (hash
!= 0 && net
->real_num_tx_queues
> 1) {
601 rndis_msg_size
+= NDIS_HASH_PPI_SIZE
;
602 hash_info
= init_ppi_data(rndis_msg
, NDIS_HASH_PPI_SIZE
,
607 if (skb_vlan_tag_present(skb
)) {
608 struct ndis_pkt_8021q_info
*vlan
;
610 rndis_msg_size
+= NDIS_VLAN_PPI_SIZE
;
611 vlan
= init_ppi_data(rndis_msg
, NDIS_VLAN_PPI_SIZE
,
615 vlan
->vlanid
= skb_vlan_tag_get_id(skb
);
616 vlan
->cfi
= skb_vlan_tag_get_cfi(skb
);
617 vlan
->pri
= skb_vlan_tag_get_prio(skb
);
620 if (skb_is_gso(skb
)) {
621 struct ndis_tcp_lso_info
*lso_info
;
623 rndis_msg_size
+= NDIS_LSO_PPI_SIZE
;
624 lso_info
= init_ppi_data(rndis_msg
, NDIS_LSO_PPI_SIZE
,
625 TCP_LARGESEND_PKTINFO
);
628 lso_info
->lso_v2_transmit
.type
= NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE
;
629 if (skb
->protocol
== htons(ETH_P_IP
)) {
630 lso_info
->lso_v2_transmit
.ip_version
=
631 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4
;
632 ip_hdr(skb
)->tot_len
= 0;
633 ip_hdr(skb
)->check
= 0;
634 tcp_hdr(skb
)->check
=
635 ~csum_tcpudp_magic(ip_hdr(skb
)->saddr
,
636 ip_hdr(skb
)->daddr
, 0, IPPROTO_TCP
, 0);
638 lso_info
->lso_v2_transmit
.ip_version
=
639 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6
;
640 ipv6_hdr(skb
)->payload_len
= 0;
641 tcp_hdr(skb
)->check
=
642 ~csum_ipv6_magic(&ipv6_hdr(skb
)->saddr
,
643 &ipv6_hdr(skb
)->daddr
, 0, IPPROTO_TCP
, 0);
645 lso_info
->lso_v2_transmit
.tcp_header_offset
= skb_transport_offset(skb
);
646 lso_info
->lso_v2_transmit
.mss
= skb_shinfo(skb
)->gso_size
;
647 } else if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
648 if (net_checksum_info(skb
) & net_device_ctx
->tx_checksum_mask
) {
649 struct ndis_tcp_ip_checksum_info
*csum_info
;
651 rndis_msg_size
+= NDIS_CSUM_PPI_SIZE
;
652 csum_info
= init_ppi_data(rndis_msg
, NDIS_CSUM_PPI_SIZE
,
653 TCPIP_CHKSUM_PKTINFO
);
655 csum_info
->value
= 0;
656 csum_info
->transmit
.tcp_header_offset
= skb_transport_offset(skb
);
658 if (skb
->protocol
== htons(ETH_P_IP
)) {
659 csum_info
->transmit
.is_ipv4
= 1;
661 if (ip_hdr(skb
)->protocol
== IPPROTO_TCP
)
662 csum_info
->transmit
.tcp_checksum
= 1;
664 csum_info
->transmit
.udp_checksum
= 1;
666 csum_info
->transmit
.is_ipv6
= 1;
668 if (ipv6_hdr(skb
)->nexthdr
== IPPROTO_TCP
)
669 csum_info
->transmit
.tcp_checksum
= 1;
671 csum_info
->transmit
.udp_checksum
= 1;
674 /* Can't do offload of this type of checksum */
675 if (skb_checksum_help(skb
))
680 /* Start filling in the page buffers with the rndis hdr */
681 rndis_msg
->msg_len
+= rndis_msg_size
;
682 packet
->total_data_buflen
= rndis_msg
->msg_len
;
683 packet
->page_buf_cnt
= init_page_array(rndis_msg
, rndis_msg_size
,
686 /* timestamp packet in software */
687 skb_tx_timestamp(skb
);
689 ret
= netvsc_send(net
, packet
, rndis_msg
, pb
, skb
);
690 if (likely(ret
== 0))
693 if (ret
== -EAGAIN
) {
694 ++net_device_ctx
->eth_stats
.tx_busy
;
695 return NETDEV_TX_BUSY
;
699 ++net_device_ctx
->eth_stats
.tx_no_space
;
702 dev_kfree_skb_any(skb
);
703 net
->stats
.tx_dropped
++;
708 ++net_device_ctx
->eth_stats
.tx_no_memory
;
713 * netvsc_linkstatus_callback - Link up/down notification
715 void netvsc_linkstatus_callback(struct net_device
*net
,
716 struct rndis_message
*resp
)
718 struct rndis_indicate_status
*indicate
= &resp
->msg
.indicate_status
;
719 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
720 struct netvsc_reconfig
*event
;
723 /* Update the physical link speed when changing to another vSwitch */
724 if (indicate
->status
== RNDIS_STATUS_LINK_SPEED_CHANGE
) {
727 speed
= *(u32
*)((void *)indicate
728 + indicate
->status_buf_offset
) / 10000;
729 ndev_ctx
->speed
= speed
;
733 /* Handle these link change statuses below */
734 if (indicate
->status
!= RNDIS_STATUS_NETWORK_CHANGE
&&
735 indicate
->status
!= RNDIS_STATUS_MEDIA_CONNECT
&&
736 indicate
->status
!= RNDIS_STATUS_MEDIA_DISCONNECT
)
739 if (net
->reg_state
!= NETREG_REGISTERED
)
742 event
= kzalloc(sizeof(*event
), GFP_ATOMIC
);
745 event
->event
= indicate
->status
;
747 spin_lock_irqsave(&ndev_ctx
->lock
, flags
);
748 list_add_tail(&event
->list
, &ndev_ctx
->reconfig_events
);
749 spin_unlock_irqrestore(&ndev_ctx
->lock
, flags
);
751 schedule_delayed_work(&ndev_ctx
->dwork
, 0);
754 static void netvsc_comp_ipcsum(struct sk_buff
*skb
)
756 struct iphdr
*iph
= (struct iphdr
*)skb
->data
;
759 iph
->check
= ip_fast_csum(iph
, iph
->ihl
);
762 static struct sk_buff
*netvsc_alloc_recv_skb(struct net_device
*net
,
763 struct netvsc_channel
*nvchan
)
765 struct napi_struct
*napi
= &nvchan
->napi
;
766 const struct ndis_pkt_8021q_info
*vlan
= nvchan
->rsc
.vlan
;
767 const struct ndis_tcp_ip_checksum_info
*csum_info
=
768 nvchan
->rsc
.csum_info
;
772 skb
= napi_alloc_skb(napi
, nvchan
->rsc
.pktlen
);
777 * Copy to skb. This copy is needed here since the memory pointed by
778 * hv_netvsc_packet cannot be deallocated
780 for (i
= 0; i
< nvchan
->rsc
.cnt
; i
++)
781 skb_put_data(skb
, nvchan
->rsc
.data
[i
], nvchan
->rsc
.len
[i
]);
783 skb
->protocol
= eth_type_trans(skb
, net
);
785 /* skb is already created with CHECKSUM_NONE */
786 skb_checksum_none_assert(skb
);
788 /* Incoming packets may have IP header checksum verified by the host.
789 * They may not have IP header checksum computed after coalescing.
790 * We compute it here if the flags are set, because on Linux, the IP
791 * checksum is always checked.
793 if (csum_info
&& csum_info
->receive
.ip_checksum_value_invalid
&&
794 csum_info
->receive
.ip_checksum_succeeded
&&
795 skb
->protocol
== htons(ETH_P_IP
))
796 netvsc_comp_ipcsum(skb
);
798 /* Do L4 checksum offload if enabled and present.
800 if (csum_info
&& (net
->features
& NETIF_F_RXCSUM
)) {
801 if (csum_info
->receive
.tcp_checksum_succeeded
||
802 csum_info
->receive
.udp_checksum_succeeded
)
803 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
807 u16 vlan_tci
= vlan
->vlanid
| (vlan
->pri
<< VLAN_PRIO_SHIFT
) |
808 (vlan
->cfi
? VLAN_CFI_MASK
: 0);
810 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
),
818 * netvsc_recv_callback - Callback when we receive a packet from the
819 * "wire" on the specified device.
821 int netvsc_recv_callback(struct net_device
*net
,
822 struct netvsc_device
*net_device
,
823 struct netvsc_channel
*nvchan
)
825 struct net_device_context
*net_device_ctx
= netdev_priv(net
);
826 struct vmbus_channel
*channel
= nvchan
->channel
;
827 u16 q_idx
= channel
->offermsg
.offer
.sub_channel_index
;
829 struct netvsc_stats
*rx_stats
;
831 if (net
->reg_state
!= NETREG_REGISTERED
)
832 return NVSP_STAT_FAIL
;
834 /* Allocate a skb - TODO direct I/O to pages? */
835 skb
= netvsc_alloc_recv_skb(net
, nvchan
);
837 if (unlikely(!skb
)) {
838 ++net_device_ctx
->eth_stats
.rx_no_memory
;
839 return NVSP_STAT_FAIL
;
842 skb_record_rx_queue(skb
, q_idx
);
845 * Even if injecting the packet, record the statistics
846 * on the synthetic device because modifying the VF device
847 * statistics will not work correctly.
849 rx_stats
= &nvchan
->rx_stats
;
850 u64_stats_update_begin(&rx_stats
->syncp
);
852 rx_stats
->bytes
+= nvchan
->rsc
.pktlen
;
854 if (skb
->pkt_type
== PACKET_BROADCAST
)
855 ++rx_stats
->broadcast
;
856 else if (skb
->pkt_type
== PACKET_MULTICAST
)
857 ++rx_stats
->multicast
;
858 u64_stats_update_end(&rx_stats
->syncp
);
860 napi_gro_receive(&nvchan
->napi
, skb
);
861 return NVSP_STAT_SUCCESS
;
864 static void netvsc_get_drvinfo(struct net_device
*net
,
865 struct ethtool_drvinfo
*info
)
867 strlcpy(info
->driver
, KBUILD_MODNAME
, sizeof(info
->driver
));
868 strlcpy(info
->fw_version
, "N/A", sizeof(info
->fw_version
));
871 static void netvsc_get_channels(struct net_device
*net
,
872 struct ethtool_channels
*channel
)
874 struct net_device_context
*net_device_ctx
= netdev_priv(net
);
875 struct netvsc_device
*nvdev
= rtnl_dereference(net_device_ctx
->nvdev
);
878 channel
->max_combined
= nvdev
->max_chn
;
879 channel
->combined_count
= nvdev
->num_chn
;
883 /* Alloc struct netvsc_device_info, and initialize it from either existing
884 * struct netvsc_device, or from default values.
886 static struct netvsc_device_info
*netvsc_devinfo_get
887 (struct netvsc_device
*nvdev
)
889 struct netvsc_device_info
*dev_info
;
891 dev_info
= kzalloc(sizeof(*dev_info
), GFP_ATOMIC
);
897 dev_info
->num_chn
= nvdev
->num_chn
;
898 dev_info
->send_sections
= nvdev
->send_section_cnt
;
899 dev_info
->send_section_size
= nvdev
->send_section_size
;
900 dev_info
->recv_sections
= nvdev
->recv_section_cnt
;
901 dev_info
->recv_section_size
= nvdev
->recv_section_size
;
903 memcpy(dev_info
->rss_key
, nvdev
->extension
->rss_key
,
906 dev_info
->num_chn
= VRSS_CHANNEL_DEFAULT
;
907 dev_info
->send_sections
= NETVSC_DEFAULT_TX
;
908 dev_info
->send_section_size
= NETVSC_SEND_SECTION_SIZE
;
909 dev_info
->recv_sections
= NETVSC_DEFAULT_RX
;
910 dev_info
->recv_section_size
= NETVSC_RECV_SECTION_SIZE
;
916 static int netvsc_detach(struct net_device
*ndev
,
917 struct netvsc_device
*nvdev
)
919 struct net_device_context
*ndev_ctx
= netdev_priv(ndev
);
920 struct hv_device
*hdev
= ndev_ctx
->device_ctx
;
923 /* Don't try continuing to try and setup sub channels */
924 if (cancel_work_sync(&nvdev
->subchan_work
))
927 /* If device was up (receiving) then shutdown */
928 if (netif_running(ndev
)) {
929 netvsc_tx_disable(nvdev
, ndev
);
931 ret
= rndis_filter_close(nvdev
);
934 "unable to close device (ret %d).\n", ret
);
938 ret
= netvsc_wait_until_empty(nvdev
);
941 "Ring buffer not empty after closing rndis\n");
946 netif_device_detach(ndev
);
948 rndis_filter_device_remove(hdev
, nvdev
);
953 static int netvsc_attach(struct net_device
*ndev
,
954 struct netvsc_device_info
*dev_info
)
956 struct net_device_context
*ndev_ctx
= netdev_priv(ndev
);
957 struct hv_device
*hdev
= ndev_ctx
->device_ctx
;
958 struct netvsc_device
*nvdev
;
959 struct rndis_device
*rdev
;
962 nvdev
= rndis_filter_device_add(hdev
, dev_info
);
964 return PTR_ERR(nvdev
);
966 if (nvdev
->num_chn
> 1) {
967 ret
= rndis_set_subchannel(ndev
, nvdev
, dev_info
);
969 /* if unavailable, just proceed with one queue */
976 /* In any case device is now ready */
977 netif_device_attach(ndev
);
979 /* Note: enable and attach happen when sub-channels setup */
980 netif_carrier_off(ndev
);
982 if (netif_running(ndev
)) {
983 ret
= rndis_filter_open(nvdev
);
987 rdev
= nvdev
->extension
;
988 if (!rdev
->link_state
)
989 netif_carrier_on(ndev
);
995 static int netvsc_set_channels(struct net_device
*net
,
996 struct ethtool_channels
*channels
)
998 struct net_device_context
*net_device_ctx
= netdev_priv(net
);
999 struct netvsc_device
*nvdev
= rtnl_dereference(net_device_ctx
->nvdev
);
1000 unsigned int orig
, count
= channels
->combined_count
;
1001 struct netvsc_device_info
*device_info
;
1004 /* We do not support separate count for rx, tx, or other */
1006 channels
->rx_count
|| channels
->tx_count
|| channels
->other_count
)
1009 if (!nvdev
|| nvdev
->destroy
)
1012 if (nvdev
->nvsp_version
< NVSP_PROTOCOL_VERSION_5
)
1015 if (count
> nvdev
->max_chn
)
1018 orig
= nvdev
->num_chn
;
1020 device_info
= netvsc_devinfo_get(nvdev
);
1025 device_info
->num_chn
= count
;
1027 ret
= netvsc_detach(net
, nvdev
);
1031 ret
= netvsc_attach(net
, device_info
);
1033 device_info
->num_chn
= orig
;
1034 if (netvsc_attach(net
, device_info
))
1035 netdev_err(net
, "restoring channel setting failed\n");
1044 netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings
*cmd
)
1046 struct ethtool_link_ksettings diff1
= *cmd
;
1047 struct ethtool_link_ksettings diff2
= {};
1049 diff1
.base
.speed
= 0;
1050 diff1
.base
.duplex
= 0;
1051 /* advertising and cmd are usually set */
1052 ethtool_link_ksettings_zero_link_mode(&diff1
, advertising
);
1054 /* We set port to PORT_OTHER */
1055 diff2
.base
.port
= PORT_OTHER
;
1057 return !memcmp(&diff1
, &diff2
, sizeof(diff1
));
1060 static void netvsc_init_settings(struct net_device
*dev
)
1062 struct net_device_context
*ndc
= netdev_priv(dev
);
1064 ndc
->l4_hash
= HV_DEFAULT_L4HASH
;
1066 ndc
->speed
= SPEED_UNKNOWN
;
1067 ndc
->duplex
= DUPLEX_FULL
;
1069 dev
->features
= NETIF_F_LRO
;
1072 static int netvsc_get_link_ksettings(struct net_device
*dev
,
1073 struct ethtool_link_ksettings
*cmd
)
1075 struct net_device_context
*ndc
= netdev_priv(dev
);
1077 cmd
->base
.speed
= ndc
->speed
;
1078 cmd
->base
.duplex
= ndc
->duplex
;
1079 cmd
->base
.port
= PORT_OTHER
;
1084 static int netvsc_set_link_ksettings(struct net_device
*dev
,
1085 const struct ethtool_link_ksettings
*cmd
)
1087 struct net_device_context
*ndc
= netdev_priv(dev
);
1090 speed
= cmd
->base
.speed
;
1091 if (!ethtool_validate_speed(speed
) ||
1092 !ethtool_validate_duplex(cmd
->base
.duplex
) ||
1093 !netvsc_validate_ethtool_ss_cmd(cmd
))
1097 ndc
->duplex
= cmd
->base
.duplex
;
1102 static int netvsc_change_mtu(struct net_device
*ndev
, int mtu
)
1104 struct net_device_context
*ndevctx
= netdev_priv(ndev
);
1105 struct net_device
*vf_netdev
= rtnl_dereference(ndevctx
->vf_netdev
);
1106 struct netvsc_device
*nvdev
= rtnl_dereference(ndevctx
->nvdev
);
1107 int orig_mtu
= ndev
->mtu
;
1108 struct netvsc_device_info
*device_info
;
1111 if (!nvdev
|| nvdev
->destroy
)
1114 device_info
= netvsc_devinfo_get(nvdev
);
1119 /* Change MTU of underlying VF netdev first. */
1121 ret
= dev_set_mtu(vf_netdev
, mtu
);
1126 ret
= netvsc_detach(ndev
, nvdev
);
1132 ret
= netvsc_attach(ndev
, device_info
);
1136 /* Attempt rollback to original MTU */
1137 ndev
->mtu
= orig_mtu
;
1139 if (netvsc_attach(ndev
, device_info
))
1140 netdev_err(ndev
, "restoring mtu failed\n");
1143 dev_set_mtu(vf_netdev
, orig_mtu
);
1150 static void netvsc_get_vf_stats(struct net_device
*net
,
1151 struct netvsc_vf_pcpu_stats
*tot
)
1153 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
1156 memset(tot
, 0, sizeof(*tot
));
1158 for_each_possible_cpu(i
) {
1159 const struct netvsc_vf_pcpu_stats
*stats
1160 = per_cpu_ptr(ndev_ctx
->vf_stats
, i
);
1161 u64 rx_packets
, rx_bytes
, tx_packets
, tx_bytes
;
1165 start
= u64_stats_fetch_begin_irq(&stats
->syncp
);
1166 rx_packets
= stats
->rx_packets
;
1167 tx_packets
= stats
->tx_packets
;
1168 rx_bytes
= stats
->rx_bytes
;
1169 tx_bytes
= stats
->tx_bytes
;
1170 } while (u64_stats_fetch_retry_irq(&stats
->syncp
, start
));
1172 tot
->rx_packets
+= rx_packets
;
1173 tot
->tx_packets
+= tx_packets
;
1174 tot
->rx_bytes
+= rx_bytes
;
1175 tot
->tx_bytes
+= tx_bytes
;
1176 tot
->tx_dropped
+= stats
->tx_dropped
;
1180 static void netvsc_get_pcpu_stats(struct net_device
*net
,
1181 struct netvsc_ethtool_pcpu_stats
*pcpu_tot
)
1183 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
1184 struct netvsc_device
*nvdev
= rcu_dereference_rtnl(ndev_ctx
->nvdev
);
1187 /* fetch percpu stats of vf */
1188 for_each_possible_cpu(i
) {
1189 const struct netvsc_vf_pcpu_stats
*stats
=
1190 per_cpu_ptr(ndev_ctx
->vf_stats
, i
);
1191 struct netvsc_ethtool_pcpu_stats
*this_tot
= &pcpu_tot
[i
];
1195 start
= u64_stats_fetch_begin_irq(&stats
->syncp
);
1196 this_tot
->vf_rx_packets
= stats
->rx_packets
;
1197 this_tot
->vf_tx_packets
= stats
->tx_packets
;
1198 this_tot
->vf_rx_bytes
= stats
->rx_bytes
;
1199 this_tot
->vf_tx_bytes
= stats
->tx_bytes
;
1200 } while (u64_stats_fetch_retry_irq(&stats
->syncp
, start
));
1201 this_tot
->rx_packets
= this_tot
->vf_rx_packets
;
1202 this_tot
->tx_packets
= this_tot
->vf_tx_packets
;
1203 this_tot
->rx_bytes
= this_tot
->vf_rx_bytes
;
1204 this_tot
->tx_bytes
= this_tot
->vf_tx_bytes
;
1207 /* fetch percpu stats of netvsc */
1208 for (i
= 0; i
< nvdev
->num_chn
; i
++) {
1209 const struct netvsc_channel
*nvchan
= &nvdev
->chan_table
[i
];
1210 const struct netvsc_stats
*stats
;
1211 struct netvsc_ethtool_pcpu_stats
*this_tot
=
1212 &pcpu_tot
[nvchan
->channel
->target_cpu
];
1216 stats
= &nvchan
->tx_stats
;
1218 start
= u64_stats_fetch_begin_irq(&stats
->syncp
);
1219 packets
= stats
->packets
;
1220 bytes
= stats
->bytes
;
1221 } while (u64_stats_fetch_retry_irq(&stats
->syncp
, start
));
1223 this_tot
->tx_bytes
+= bytes
;
1224 this_tot
->tx_packets
+= packets
;
1226 stats
= &nvchan
->rx_stats
;
1228 start
= u64_stats_fetch_begin_irq(&stats
->syncp
);
1229 packets
= stats
->packets
;
1230 bytes
= stats
->bytes
;
1231 } while (u64_stats_fetch_retry_irq(&stats
->syncp
, start
));
1233 this_tot
->rx_bytes
+= bytes
;
1234 this_tot
->rx_packets
+= packets
;
1238 static void netvsc_get_stats64(struct net_device
*net
,
1239 struct rtnl_link_stats64
*t
)
1241 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
1242 struct netvsc_device
*nvdev
;
1243 struct netvsc_vf_pcpu_stats vf_tot
;
1248 nvdev
= rcu_dereference(ndev_ctx
->nvdev
);
1252 netdev_stats_to_stats64(t
, &net
->stats
);
1254 netvsc_get_vf_stats(net
, &vf_tot
);
1255 t
->rx_packets
+= vf_tot
.rx_packets
;
1256 t
->tx_packets
+= vf_tot
.tx_packets
;
1257 t
->rx_bytes
+= vf_tot
.rx_bytes
;
1258 t
->tx_bytes
+= vf_tot
.tx_bytes
;
1259 t
->tx_dropped
+= vf_tot
.tx_dropped
;
1261 for (i
= 0; i
< nvdev
->num_chn
; i
++) {
1262 const struct netvsc_channel
*nvchan
= &nvdev
->chan_table
[i
];
1263 const struct netvsc_stats
*stats
;
1264 u64 packets
, bytes
, multicast
;
1267 stats
= &nvchan
->tx_stats
;
1269 start
= u64_stats_fetch_begin_irq(&stats
->syncp
);
1270 packets
= stats
->packets
;
1271 bytes
= stats
->bytes
;
1272 } while (u64_stats_fetch_retry_irq(&stats
->syncp
, start
));
1274 t
->tx_bytes
+= bytes
;
1275 t
->tx_packets
+= packets
;
1277 stats
= &nvchan
->rx_stats
;
1279 start
= u64_stats_fetch_begin_irq(&stats
->syncp
);
1280 packets
= stats
->packets
;
1281 bytes
= stats
->bytes
;
1282 multicast
= stats
->multicast
+ stats
->broadcast
;
1283 } while (u64_stats_fetch_retry_irq(&stats
->syncp
, start
));
1285 t
->rx_bytes
+= bytes
;
1286 t
->rx_packets
+= packets
;
1287 t
->multicast
+= multicast
;
1293 static int netvsc_set_mac_addr(struct net_device
*ndev
, void *p
)
1295 struct net_device_context
*ndc
= netdev_priv(ndev
);
1296 struct net_device
*vf_netdev
= rtnl_dereference(ndc
->vf_netdev
);
1297 struct netvsc_device
*nvdev
= rtnl_dereference(ndc
->nvdev
);
1298 struct sockaddr
*addr
= p
;
1301 err
= eth_prepare_mac_addr_change(ndev
, p
);
1309 err
= dev_set_mac_address(vf_netdev
, addr
, NULL
);
1314 err
= rndis_filter_set_device_mac(nvdev
, addr
->sa_data
);
1316 eth_commit_mac_addr_change(ndev
, p
);
1317 } else if (vf_netdev
) {
1318 /* rollback change on VF */
1319 memcpy(addr
->sa_data
, ndev
->dev_addr
, ETH_ALEN
);
1320 dev_set_mac_address(vf_netdev
, addr
, NULL
);
1326 static const struct {
1327 char name
[ETH_GSTRING_LEN
];
1329 } netvsc_stats
[] = {
1330 { "tx_scattered", offsetof(struct netvsc_ethtool_stats
, tx_scattered
) },
1331 { "tx_no_memory", offsetof(struct netvsc_ethtool_stats
, tx_no_memory
) },
1332 { "tx_no_space", offsetof(struct netvsc_ethtool_stats
, tx_no_space
) },
1333 { "tx_too_big", offsetof(struct netvsc_ethtool_stats
, tx_too_big
) },
1334 { "tx_busy", offsetof(struct netvsc_ethtool_stats
, tx_busy
) },
1335 { "tx_send_full", offsetof(struct netvsc_ethtool_stats
, tx_send_full
) },
1336 { "rx_comp_busy", offsetof(struct netvsc_ethtool_stats
, rx_comp_busy
) },
1337 { "rx_no_memory", offsetof(struct netvsc_ethtool_stats
, rx_no_memory
) },
1338 { "stop_queue", offsetof(struct netvsc_ethtool_stats
, stop_queue
) },
1339 { "wake_queue", offsetof(struct netvsc_ethtool_stats
, wake_queue
) },
1341 { "cpu%u_rx_packets",
1342 offsetof(struct netvsc_ethtool_pcpu_stats
, rx_packets
) },
1344 offsetof(struct netvsc_ethtool_pcpu_stats
, rx_bytes
) },
1345 { "cpu%u_tx_packets",
1346 offsetof(struct netvsc_ethtool_pcpu_stats
, tx_packets
) },
1348 offsetof(struct netvsc_ethtool_pcpu_stats
, tx_bytes
) },
1349 { "cpu%u_vf_rx_packets",
1350 offsetof(struct netvsc_ethtool_pcpu_stats
, vf_rx_packets
) },
1351 { "cpu%u_vf_rx_bytes",
1352 offsetof(struct netvsc_ethtool_pcpu_stats
, vf_rx_bytes
) },
1353 { "cpu%u_vf_tx_packets",
1354 offsetof(struct netvsc_ethtool_pcpu_stats
, vf_tx_packets
) },
1355 { "cpu%u_vf_tx_bytes",
1356 offsetof(struct netvsc_ethtool_pcpu_stats
, vf_tx_bytes
) },
1358 { "vf_rx_packets", offsetof(struct netvsc_vf_pcpu_stats
, rx_packets
) },
1359 { "vf_rx_bytes", offsetof(struct netvsc_vf_pcpu_stats
, rx_bytes
) },
1360 { "vf_tx_packets", offsetof(struct netvsc_vf_pcpu_stats
, tx_packets
) },
1361 { "vf_tx_bytes", offsetof(struct netvsc_vf_pcpu_stats
, tx_bytes
) },
1362 { "vf_tx_dropped", offsetof(struct netvsc_vf_pcpu_stats
, tx_dropped
) },
1365 #define NETVSC_GLOBAL_STATS_LEN ARRAY_SIZE(netvsc_stats)
1366 #define NETVSC_VF_STATS_LEN ARRAY_SIZE(vf_stats)
1368 /* statistics per queue (rx/tx packets/bytes) */
1369 #define NETVSC_PCPU_STATS_LEN (num_present_cpus() * ARRAY_SIZE(pcpu_stats))
1371 /* 4 statistics per queue (rx/tx packets/bytes) */
1372 #define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 4)
1374 static int netvsc_get_sset_count(struct net_device
*dev
, int string_set
)
1376 struct net_device_context
*ndc
= netdev_priv(dev
);
1377 struct netvsc_device
*nvdev
= rtnl_dereference(ndc
->nvdev
);
1382 switch (string_set
) {
1384 return NETVSC_GLOBAL_STATS_LEN
1385 + NETVSC_VF_STATS_LEN
1386 + NETVSC_QUEUE_STATS_LEN(nvdev
)
1387 + NETVSC_PCPU_STATS_LEN
;
1393 static void netvsc_get_ethtool_stats(struct net_device
*dev
,
1394 struct ethtool_stats
*stats
, u64
*data
)
1396 struct net_device_context
*ndc
= netdev_priv(dev
);
1397 struct netvsc_device
*nvdev
= rtnl_dereference(ndc
->nvdev
);
1398 const void *nds
= &ndc
->eth_stats
;
1399 const struct netvsc_stats
*qstats
;
1400 struct netvsc_vf_pcpu_stats sum
;
1401 struct netvsc_ethtool_pcpu_stats
*pcpu_sum
;
1409 for (i
= 0; i
< NETVSC_GLOBAL_STATS_LEN
; i
++)
1410 data
[i
] = *(unsigned long *)(nds
+ netvsc_stats
[i
].offset
);
1412 netvsc_get_vf_stats(dev
, &sum
);
1413 for (j
= 0; j
< NETVSC_VF_STATS_LEN
; j
++)
1414 data
[i
++] = *(u64
*)((void *)&sum
+ vf_stats
[j
].offset
);
1416 for (j
= 0; j
< nvdev
->num_chn
; j
++) {
1417 qstats
= &nvdev
->chan_table
[j
].tx_stats
;
1420 start
= u64_stats_fetch_begin_irq(&qstats
->syncp
);
1421 packets
= qstats
->packets
;
1422 bytes
= qstats
->bytes
;
1423 } while (u64_stats_fetch_retry_irq(&qstats
->syncp
, start
));
1424 data
[i
++] = packets
;
1427 qstats
= &nvdev
->chan_table
[j
].rx_stats
;
1429 start
= u64_stats_fetch_begin_irq(&qstats
->syncp
);
1430 packets
= qstats
->packets
;
1431 bytes
= qstats
->bytes
;
1432 } while (u64_stats_fetch_retry_irq(&qstats
->syncp
, start
));
1433 data
[i
++] = packets
;
1437 pcpu_sum
= kvmalloc_array(num_possible_cpus(),
1438 sizeof(struct netvsc_ethtool_pcpu_stats
),
1440 netvsc_get_pcpu_stats(dev
, pcpu_sum
);
1441 for_each_present_cpu(cpu
) {
1442 struct netvsc_ethtool_pcpu_stats
*this_sum
= &pcpu_sum
[cpu
];
1444 for (j
= 0; j
< ARRAY_SIZE(pcpu_stats
); j
++)
1445 data
[i
++] = *(u64
*)((void *)this_sum
1446 + pcpu_stats
[j
].offset
);
1451 static void netvsc_get_strings(struct net_device
*dev
, u32 stringset
, u8
*data
)
1453 struct net_device_context
*ndc
= netdev_priv(dev
);
1454 struct netvsc_device
*nvdev
= rtnl_dereference(ndc
->nvdev
);
1461 switch (stringset
) {
1463 for (i
= 0; i
< ARRAY_SIZE(netvsc_stats
); i
++) {
1464 memcpy(p
, netvsc_stats
[i
].name
, ETH_GSTRING_LEN
);
1465 p
+= ETH_GSTRING_LEN
;
1468 for (i
= 0; i
< ARRAY_SIZE(vf_stats
); i
++) {
1469 memcpy(p
, vf_stats
[i
].name
, ETH_GSTRING_LEN
);
1470 p
+= ETH_GSTRING_LEN
;
1473 for (i
= 0; i
< nvdev
->num_chn
; i
++) {
1474 sprintf(p
, "tx_queue_%u_packets", i
);
1475 p
+= ETH_GSTRING_LEN
;
1476 sprintf(p
, "tx_queue_%u_bytes", i
);
1477 p
+= ETH_GSTRING_LEN
;
1478 sprintf(p
, "rx_queue_%u_packets", i
);
1479 p
+= ETH_GSTRING_LEN
;
1480 sprintf(p
, "rx_queue_%u_bytes", i
);
1481 p
+= ETH_GSTRING_LEN
;
1484 for_each_present_cpu(cpu
) {
1485 for (i
= 0; i
< ARRAY_SIZE(pcpu_stats
); i
++) {
1486 sprintf(p
, pcpu_stats
[i
].name
, cpu
);
1487 p
+= ETH_GSTRING_LEN
;
1496 netvsc_get_rss_hash_opts(struct net_device_context
*ndc
,
1497 struct ethtool_rxnfc
*info
)
1499 const u32 l4_flag
= RXH_L4_B_0_1
| RXH_L4_B_2_3
;
1501 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
1503 switch (info
->flow_type
) {
1505 if (ndc
->l4_hash
& HV_TCP4_L4HASH
)
1506 info
->data
|= l4_flag
;
1511 if (ndc
->l4_hash
& HV_TCP6_L4HASH
)
1512 info
->data
|= l4_flag
;
1517 if (ndc
->l4_hash
& HV_UDP4_L4HASH
)
1518 info
->data
|= l4_flag
;
1523 if (ndc
->l4_hash
& HV_UDP6_L4HASH
)
1524 info
->data
|= l4_flag
;
1540 netvsc_get_rxnfc(struct net_device
*dev
, struct ethtool_rxnfc
*info
,
1543 struct net_device_context
*ndc
= netdev_priv(dev
);
1544 struct netvsc_device
*nvdev
= rtnl_dereference(ndc
->nvdev
);
1549 switch (info
->cmd
) {
1550 case ETHTOOL_GRXRINGS
:
1551 info
->data
= nvdev
->num_chn
;
1555 return netvsc_get_rss_hash_opts(ndc
, info
);
1560 static int netvsc_set_rss_hash_opts(struct net_device_context
*ndc
,
1561 struct ethtool_rxnfc
*info
)
1563 if (info
->data
== (RXH_IP_SRC
| RXH_IP_DST
|
1564 RXH_L4_B_0_1
| RXH_L4_B_2_3
)) {
1565 switch (info
->flow_type
) {
1567 ndc
->l4_hash
|= HV_TCP4_L4HASH
;
1571 ndc
->l4_hash
|= HV_TCP6_L4HASH
;
1575 ndc
->l4_hash
|= HV_UDP4_L4HASH
;
1579 ndc
->l4_hash
|= HV_UDP6_L4HASH
;
1589 if (info
->data
== (RXH_IP_SRC
| RXH_IP_DST
)) {
1590 switch (info
->flow_type
) {
1592 ndc
->l4_hash
&= ~HV_TCP4_L4HASH
;
1596 ndc
->l4_hash
&= ~HV_TCP6_L4HASH
;
1600 ndc
->l4_hash
&= ~HV_UDP4_L4HASH
;
1604 ndc
->l4_hash
&= ~HV_UDP6_L4HASH
;
1618 netvsc_set_rxnfc(struct net_device
*ndev
, struct ethtool_rxnfc
*info
)
1620 struct net_device_context
*ndc
= netdev_priv(ndev
);
1622 if (info
->cmd
== ETHTOOL_SRXFH
)
1623 return netvsc_set_rss_hash_opts(ndc
, info
);
1628 static u32
netvsc_get_rxfh_key_size(struct net_device
*dev
)
1630 return NETVSC_HASH_KEYLEN
;
1633 static u32
netvsc_rss_indir_size(struct net_device
*dev
)
1638 static int netvsc_get_rxfh(struct net_device
*dev
, u32
*indir
, u8
*key
,
1641 struct net_device_context
*ndc
= netdev_priv(dev
);
1642 struct netvsc_device
*ndev
= rtnl_dereference(ndc
->nvdev
);
1643 struct rndis_device
*rndis_dev
;
1650 *hfunc
= ETH_RSS_HASH_TOP
; /* Toeplitz */
1652 rndis_dev
= ndev
->extension
;
1654 for (i
= 0; i
< ITAB_NUM
; i
++)
1655 indir
[i
] = rndis_dev
->rx_table
[i
];
1659 memcpy(key
, rndis_dev
->rss_key
, NETVSC_HASH_KEYLEN
);
1664 static int netvsc_set_rxfh(struct net_device
*dev
, const u32
*indir
,
1665 const u8
*key
, const u8 hfunc
)
1667 struct net_device_context
*ndc
= netdev_priv(dev
);
1668 struct netvsc_device
*ndev
= rtnl_dereference(ndc
->nvdev
);
1669 struct rndis_device
*rndis_dev
;
1675 if (hfunc
!= ETH_RSS_HASH_NO_CHANGE
&& hfunc
!= ETH_RSS_HASH_TOP
)
1678 rndis_dev
= ndev
->extension
;
1680 for (i
= 0; i
< ITAB_NUM
; i
++)
1681 if (indir
[i
] >= ndev
->num_chn
)
1684 for (i
= 0; i
< ITAB_NUM
; i
++)
1685 rndis_dev
->rx_table
[i
] = indir
[i
];
1692 key
= rndis_dev
->rss_key
;
1695 return rndis_filter_set_rss_param(rndis_dev
, key
);
1698 /* Hyper-V RNDIS protocol does not have ring in the HW sense.
1699 * It does have pre-allocated receive area which is divided into sections.
1701 static void __netvsc_get_ringparam(struct netvsc_device
*nvdev
,
1702 struct ethtool_ringparam
*ring
)
1706 ring
->rx_pending
= nvdev
->recv_section_cnt
;
1707 ring
->tx_pending
= nvdev
->send_section_cnt
;
1709 if (nvdev
->nvsp_version
<= NVSP_PROTOCOL_VERSION_2
)
1710 max_buf_size
= NETVSC_RECEIVE_BUFFER_SIZE_LEGACY
;
1712 max_buf_size
= NETVSC_RECEIVE_BUFFER_SIZE
;
1714 ring
->rx_max_pending
= max_buf_size
/ nvdev
->recv_section_size
;
1715 ring
->tx_max_pending
= NETVSC_SEND_BUFFER_SIZE
1716 / nvdev
->send_section_size
;
1719 static void netvsc_get_ringparam(struct net_device
*ndev
,
1720 struct ethtool_ringparam
*ring
)
1722 struct net_device_context
*ndevctx
= netdev_priv(ndev
);
1723 struct netvsc_device
*nvdev
= rtnl_dereference(ndevctx
->nvdev
);
1728 __netvsc_get_ringparam(nvdev
, ring
);
1731 static int netvsc_set_ringparam(struct net_device
*ndev
,
1732 struct ethtool_ringparam
*ring
)
1734 struct net_device_context
*ndevctx
= netdev_priv(ndev
);
1735 struct netvsc_device
*nvdev
= rtnl_dereference(ndevctx
->nvdev
);
1736 struct netvsc_device_info
*device_info
;
1737 struct ethtool_ringparam orig
;
1741 if (!nvdev
|| nvdev
->destroy
)
1744 memset(&orig
, 0, sizeof(orig
));
1745 __netvsc_get_ringparam(nvdev
, &orig
);
1747 new_tx
= clamp_t(u32
, ring
->tx_pending
,
1748 NETVSC_MIN_TX_SECTIONS
, orig
.tx_max_pending
);
1749 new_rx
= clamp_t(u32
, ring
->rx_pending
,
1750 NETVSC_MIN_RX_SECTIONS
, orig
.rx_max_pending
);
1752 if (new_tx
== orig
.tx_pending
&&
1753 new_rx
== orig
.rx_pending
)
1754 return 0; /* no change */
1756 device_info
= netvsc_devinfo_get(nvdev
);
1761 device_info
->send_sections
= new_tx
;
1762 device_info
->recv_sections
= new_rx
;
1764 ret
= netvsc_detach(ndev
, nvdev
);
1768 ret
= netvsc_attach(ndev
, device_info
);
1770 device_info
->send_sections
= orig
.tx_pending
;
1771 device_info
->recv_sections
= orig
.rx_pending
;
1773 if (netvsc_attach(ndev
, device_info
))
1774 netdev_err(ndev
, "restoring ringparam failed");
1782 static int netvsc_set_features(struct net_device
*ndev
,
1783 netdev_features_t features
)
1785 netdev_features_t change
= features
^ ndev
->features
;
1786 struct net_device_context
*ndevctx
= netdev_priv(ndev
);
1787 struct netvsc_device
*nvdev
= rtnl_dereference(ndevctx
->nvdev
);
1788 struct ndis_offload_params offloads
;
1790 if (!nvdev
|| nvdev
->destroy
)
1793 if (!(change
& NETIF_F_LRO
))
1796 memset(&offloads
, 0, sizeof(struct ndis_offload_params
));
1798 if (features
& NETIF_F_LRO
) {
1799 offloads
.rsc_ip_v4
= NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED
;
1800 offloads
.rsc_ip_v6
= NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED
;
1802 offloads
.rsc_ip_v4
= NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED
;
1803 offloads
.rsc_ip_v6
= NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED
;
1806 return rndis_filter_set_offload_params(ndev
, nvdev
, &offloads
);
1809 static u32
netvsc_get_msglevel(struct net_device
*ndev
)
1811 struct net_device_context
*ndev_ctx
= netdev_priv(ndev
);
1813 return ndev_ctx
->msg_enable
;
1816 static void netvsc_set_msglevel(struct net_device
*ndev
, u32 val
)
1818 struct net_device_context
*ndev_ctx
= netdev_priv(ndev
);
1820 ndev_ctx
->msg_enable
= val
;
1823 static const struct ethtool_ops ethtool_ops
= {
1824 .get_drvinfo
= netvsc_get_drvinfo
,
1825 .get_msglevel
= netvsc_get_msglevel
,
1826 .set_msglevel
= netvsc_set_msglevel
,
1827 .get_link
= ethtool_op_get_link
,
1828 .get_ethtool_stats
= netvsc_get_ethtool_stats
,
1829 .get_sset_count
= netvsc_get_sset_count
,
1830 .get_strings
= netvsc_get_strings
,
1831 .get_channels
= netvsc_get_channels
,
1832 .set_channels
= netvsc_set_channels
,
1833 .get_ts_info
= ethtool_op_get_ts_info
,
1834 .get_rxnfc
= netvsc_get_rxnfc
,
1835 .set_rxnfc
= netvsc_set_rxnfc
,
1836 .get_rxfh_key_size
= netvsc_get_rxfh_key_size
,
1837 .get_rxfh_indir_size
= netvsc_rss_indir_size
,
1838 .get_rxfh
= netvsc_get_rxfh
,
1839 .set_rxfh
= netvsc_set_rxfh
,
1840 .get_link_ksettings
= netvsc_get_link_ksettings
,
1841 .set_link_ksettings
= netvsc_set_link_ksettings
,
1842 .get_ringparam
= netvsc_get_ringparam
,
1843 .set_ringparam
= netvsc_set_ringparam
,
1846 static const struct net_device_ops device_ops
= {
1847 .ndo_open
= netvsc_open
,
1848 .ndo_stop
= netvsc_close
,
1849 .ndo_start_xmit
= netvsc_start_xmit
,
1850 .ndo_change_rx_flags
= netvsc_change_rx_flags
,
1851 .ndo_set_rx_mode
= netvsc_set_rx_mode
,
1852 .ndo_set_features
= netvsc_set_features
,
1853 .ndo_change_mtu
= netvsc_change_mtu
,
1854 .ndo_validate_addr
= eth_validate_addr
,
1855 .ndo_set_mac_address
= netvsc_set_mac_addr
,
1856 .ndo_select_queue
= netvsc_select_queue
,
1857 .ndo_get_stats64
= netvsc_get_stats64
,
1861 * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link
1862 * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is
1863 * present send GARP packet to network peers with netif_notify_peers().
1865 static void netvsc_link_change(struct work_struct
*w
)
1867 struct net_device_context
*ndev_ctx
=
1868 container_of(w
, struct net_device_context
, dwork
.work
);
1869 struct hv_device
*device_obj
= ndev_ctx
->device_ctx
;
1870 struct net_device
*net
= hv_get_drvdata(device_obj
);
1871 struct netvsc_device
*net_device
;
1872 struct rndis_device
*rdev
;
1873 struct netvsc_reconfig
*event
= NULL
;
1874 bool notify
= false, reschedule
= false;
1875 unsigned long flags
, next_reconfig
, delay
;
1877 /* if changes are happening, comeback later */
1878 if (!rtnl_trylock()) {
1879 schedule_delayed_work(&ndev_ctx
->dwork
, LINKCHANGE_INT
);
1883 net_device
= rtnl_dereference(ndev_ctx
->nvdev
);
1887 rdev
= net_device
->extension
;
1889 next_reconfig
= ndev_ctx
->last_reconfig
+ LINKCHANGE_INT
;
1890 if (time_is_after_jiffies(next_reconfig
)) {
1891 /* link_watch only sends one notification with current state
1892 * per second, avoid doing reconfig more frequently. Handle
1895 delay
= next_reconfig
- jiffies
;
1896 delay
= delay
< LINKCHANGE_INT
? delay
: LINKCHANGE_INT
;
1897 schedule_delayed_work(&ndev_ctx
->dwork
, delay
);
1900 ndev_ctx
->last_reconfig
= jiffies
;
1902 spin_lock_irqsave(&ndev_ctx
->lock
, flags
);
1903 if (!list_empty(&ndev_ctx
->reconfig_events
)) {
1904 event
= list_first_entry(&ndev_ctx
->reconfig_events
,
1905 struct netvsc_reconfig
, list
);
1906 list_del(&event
->list
);
1907 reschedule
= !list_empty(&ndev_ctx
->reconfig_events
);
1909 spin_unlock_irqrestore(&ndev_ctx
->lock
, flags
);
1914 switch (event
->event
) {
1915 /* Only the following events are possible due to the check in
1916 * netvsc_linkstatus_callback()
1918 case RNDIS_STATUS_MEDIA_CONNECT
:
1919 if (rdev
->link_state
) {
1920 rdev
->link_state
= false;
1921 netif_carrier_on(net
);
1922 netvsc_tx_enable(net_device
, net
);
1928 case RNDIS_STATUS_MEDIA_DISCONNECT
:
1929 if (!rdev
->link_state
) {
1930 rdev
->link_state
= true;
1931 netif_carrier_off(net
);
1932 netvsc_tx_disable(net_device
, net
);
1936 case RNDIS_STATUS_NETWORK_CHANGE
:
1937 /* Only makes sense if carrier is present */
1938 if (!rdev
->link_state
) {
1939 rdev
->link_state
= true;
1940 netif_carrier_off(net
);
1941 netvsc_tx_disable(net_device
, net
);
1942 event
->event
= RNDIS_STATUS_MEDIA_CONNECT
;
1943 spin_lock_irqsave(&ndev_ctx
->lock
, flags
);
1944 list_add(&event
->list
, &ndev_ctx
->reconfig_events
);
1945 spin_unlock_irqrestore(&ndev_ctx
->lock
, flags
);
1954 netdev_notify_peers(net
);
1956 /* link_watch only sends one notification with current state per
1957 * second, handle next reconfig event in 2 seconds.
1960 schedule_delayed_work(&ndev_ctx
->dwork
, LINKCHANGE_INT
);
1968 static struct net_device
*get_netvsc_byref(struct net_device
*vf_netdev
)
1970 struct net_device_context
*net_device_ctx
;
1971 struct net_device
*dev
;
1973 dev
= netdev_master_upper_dev_get(vf_netdev
);
1974 if (!dev
|| dev
->netdev_ops
!= &device_ops
)
1975 return NULL
; /* not a netvsc device */
1977 net_device_ctx
= netdev_priv(dev
);
1978 if (!rtnl_dereference(net_device_ctx
->nvdev
))
1979 return NULL
; /* device is removed */
1984 /* Called when VF is injecting data into network stack.
1985 * Change the associated network device from VF to netvsc.
1986 * note: already called with rcu_read_lock
1988 static rx_handler_result_t
netvsc_vf_handle_frame(struct sk_buff
**pskb
)
1990 struct sk_buff
*skb
= *pskb
;
1991 struct net_device
*ndev
= rcu_dereference(skb
->dev
->rx_handler_data
);
1992 struct net_device_context
*ndev_ctx
= netdev_priv(ndev
);
1993 struct netvsc_vf_pcpu_stats
*pcpu_stats
1994 = this_cpu_ptr(ndev_ctx
->vf_stats
);
1996 skb
= skb_share_check(skb
, GFP_ATOMIC
);
1998 return RX_HANDLER_CONSUMED
;
2004 u64_stats_update_begin(&pcpu_stats
->syncp
);
2005 pcpu_stats
->rx_packets
++;
2006 pcpu_stats
->rx_bytes
+= skb
->len
;
2007 u64_stats_update_end(&pcpu_stats
->syncp
);
2009 return RX_HANDLER_ANOTHER
;
2012 static int netvsc_vf_join(struct net_device
*vf_netdev
,
2013 struct net_device
*ndev
)
2015 struct net_device_context
*ndev_ctx
= netdev_priv(ndev
);
2018 ret
= netdev_rx_handler_register(vf_netdev
,
2019 netvsc_vf_handle_frame
, ndev
);
2021 netdev_err(vf_netdev
,
2022 "can not register netvsc VF receive handler (err = %d)\n",
2024 goto rx_handler_failed
;
2027 ret
= netdev_master_upper_dev_link(vf_netdev
, ndev
,
2030 netdev_err(vf_netdev
,
2031 "can not set master device %s (err = %d)\n",
2033 goto upper_link_failed
;
2036 /* set slave flag before open to prevent IPv6 addrconf */
2037 vf_netdev
->flags
|= IFF_SLAVE
;
2039 schedule_delayed_work(&ndev_ctx
->vf_takeover
, VF_TAKEOVER_INT
);
2041 call_netdevice_notifiers(NETDEV_JOIN
, vf_netdev
);
2043 netdev_info(vf_netdev
, "joined to %s\n", ndev
->name
);
2047 netdev_rx_handler_unregister(vf_netdev
);
2052 static void __netvsc_vf_setup(struct net_device
*ndev
,
2053 struct net_device
*vf_netdev
)
2057 /* Align MTU of VF with master */
2058 ret
= dev_set_mtu(vf_netdev
, ndev
->mtu
);
2060 netdev_warn(vf_netdev
,
2061 "unable to change mtu to %u\n", ndev
->mtu
);
2063 /* set multicast etc flags on VF */
2064 dev_change_flags(vf_netdev
, ndev
->flags
| IFF_SLAVE
, NULL
);
2066 /* sync address list from ndev to VF */
2067 netif_addr_lock_bh(ndev
);
2068 dev_uc_sync(vf_netdev
, ndev
);
2069 dev_mc_sync(vf_netdev
, ndev
);
2070 netif_addr_unlock_bh(ndev
);
2072 if (netif_running(ndev
)) {
2073 ret
= dev_open(vf_netdev
, NULL
);
2075 netdev_warn(vf_netdev
,
2076 "unable to open: %d\n", ret
);
2080 /* Setup VF as slave of the synthetic device.
2081 * Runs in workqueue to avoid recursion in netlink callbacks.
2083 static void netvsc_vf_setup(struct work_struct
*w
)
2085 struct net_device_context
*ndev_ctx
2086 = container_of(w
, struct net_device_context
, vf_takeover
.work
);
2087 struct net_device
*ndev
= hv_get_drvdata(ndev_ctx
->device_ctx
);
2088 struct net_device
*vf_netdev
;
2090 if (!rtnl_trylock()) {
2091 schedule_delayed_work(&ndev_ctx
->vf_takeover
, 0);
2095 vf_netdev
= rtnl_dereference(ndev_ctx
->vf_netdev
);
2097 __netvsc_vf_setup(ndev
, vf_netdev
);
2102 /* Find netvsc by VF serial number.
2103 * The PCI hyperv controller records the serial number as the slot kobj name.
2105 static struct net_device
*get_netvsc_byslot(const struct net_device
*vf_netdev
)
2107 struct device
*parent
= vf_netdev
->dev
.parent
;
2108 struct net_device_context
*ndev_ctx
;
2109 struct pci_dev
*pdev
;
2112 if (!parent
|| !dev_is_pci(parent
))
2113 return NULL
; /* not a PCI device */
2115 pdev
= to_pci_dev(parent
);
2117 netdev_notice(vf_netdev
, "no PCI slot information\n");
2121 if (kstrtou32(pci_slot_name(pdev
->slot
), 10, &serial
)) {
2122 netdev_notice(vf_netdev
, "Invalid vf serial:%s\n",
2123 pci_slot_name(pdev
->slot
));
2127 list_for_each_entry(ndev_ctx
, &netvsc_dev_list
, list
) {
2128 if (!ndev_ctx
->vf_alloc
)
2131 if (ndev_ctx
->vf_serial
== serial
)
2132 return hv_get_drvdata(ndev_ctx
->device_ctx
);
2135 netdev_notice(vf_netdev
,
2136 "no netdev found for vf serial:%u\n", serial
);
2140 static int netvsc_register_vf(struct net_device
*vf_netdev
)
2142 struct net_device_context
*net_device_ctx
;
2143 struct netvsc_device
*netvsc_dev
;
2144 struct net_device
*ndev
;
2147 if (vf_netdev
->addr_len
!= ETH_ALEN
)
2150 ndev
= get_netvsc_byslot(vf_netdev
);
2154 net_device_ctx
= netdev_priv(ndev
);
2155 netvsc_dev
= rtnl_dereference(net_device_ctx
->nvdev
);
2156 if (!netvsc_dev
|| rtnl_dereference(net_device_ctx
->vf_netdev
))
2159 /* if synthetic interface is a different namespace,
2160 * then move the VF to that namespace; join will be
2161 * done again in that context.
2163 if (!net_eq(dev_net(ndev
), dev_net(vf_netdev
))) {
2164 ret
= dev_change_net_namespace(vf_netdev
,
2165 dev_net(ndev
), "eth%d");
2167 netdev_err(vf_netdev
,
2168 "could not move to same namespace as %s: %d\n",
2171 netdev_info(vf_netdev
,
2172 "VF moved to namespace with: %s\n",
2177 netdev_info(ndev
, "VF registering: %s\n", vf_netdev
->name
);
2179 if (netvsc_vf_join(vf_netdev
, ndev
) != 0)
2182 dev_hold(vf_netdev
);
2183 rcu_assign_pointer(net_device_ctx
->vf_netdev
, vf_netdev
);
2187 /* VF up/down change detected, schedule to change data path */
2188 static int netvsc_vf_changed(struct net_device
*vf_netdev
)
2190 struct net_device_context
*net_device_ctx
;
2191 struct netvsc_device
*netvsc_dev
;
2192 struct net_device
*ndev
;
2193 bool vf_is_up
= netif_running(vf_netdev
);
2195 ndev
= get_netvsc_byref(vf_netdev
);
2199 net_device_ctx
= netdev_priv(ndev
);
2200 netvsc_dev
= rtnl_dereference(net_device_ctx
->nvdev
);
2204 netvsc_switch_datapath(ndev
, vf_is_up
);
2205 netdev_info(ndev
, "Data path switched %s VF: %s\n",
2206 vf_is_up
? "to" : "from", vf_netdev
->name
);
2211 static int netvsc_unregister_vf(struct net_device
*vf_netdev
)
2213 struct net_device
*ndev
;
2214 struct net_device_context
*net_device_ctx
;
2216 ndev
= get_netvsc_byref(vf_netdev
);
2220 net_device_ctx
= netdev_priv(ndev
);
2221 cancel_delayed_work_sync(&net_device_ctx
->vf_takeover
);
2223 netdev_info(ndev
, "VF unregistering: %s\n", vf_netdev
->name
);
2225 netdev_rx_handler_unregister(vf_netdev
);
2226 netdev_upper_dev_unlink(vf_netdev
, ndev
);
2227 RCU_INIT_POINTER(net_device_ctx
->vf_netdev
, NULL
);
2233 static int netvsc_probe(struct hv_device
*dev
,
2234 const struct hv_vmbus_device_id
*dev_id
)
2236 struct net_device
*net
= NULL
;
2237 struct net_device_context
*net_device_ctx
;
2238 struct netvsc_device_info
*device_info
= NULL
;
2239 struct netvsc_device
*nvdev
;
2242 net
= alloc_etherdev_mq(sizeof(struct net_device_context
),
2247 netif_carrier_off(net
);
2249 netvsc_init_settings(net
);
2251 net_device_ctx
= netdev_priv(net
);
2252 net_device_ctx
->device_ctx
= dev
;
2253 net_device_ctx
->msg_enable
= netif_msg_init(debug
, default_msg
);
2254 if (netif_msg_probe(net_device_ctx
))
2255 netdev_dbg(net
, "netvsc msg_enable: %d\n",
2256 net_device_ctx
->msg_enable
);
2258 hv_set_drvdata(dev
, net
);
2260 INIT_DELAYED_WORK(&net_device_ctx
->dwork
, netvsc_link_change
);
2262 spin_lock_init(&net_device_ctx
->lock
);
2263 INIT_LIST_HEAD(&net_device_ctx
->reconfig_events
);
2264 INIT_DELAYED_WORK(&net_device_ctx
->vf_takeover
, netvsc_vf_setup
);
2266 net_device_ctx
->vf_stats
2267 = netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats
);
2268 if (!net_device_ctx
->vf_stats
)
2271 net
->netdev_ops
= &device_ops
;
2272 net
->ethtool_ops
= ðtool_ops
;
2273 SET_NETDEV_DEV(net
, &dev
->device
);
2275 /* We always need headroom for rndis header */
2276 net
->needed_headroom
= RNDIS_AND_PPI_SIZE
;
2278 /* Initialize the number of queues to be 1, we may change it if more
2279 * channels are offered later.
2281 netif_set_real_num_tx_queues(net
, 1);
2282 netif_set_real_num_rx_queues(net
, 1);
2284 /* Notify the netvsc driver of the new device */
2285 device_info
= netvsc_devinfo_get(NULL
);
2289 goto devinfo_failed
;
2292 nvdev
= rndis_filter_device_add(dev
, device_info
);
2293 if (IS_ERR(nvdev
)) {
2294 ret
= PTR_ERR(nvdev
);
2295 netdev_err(net
, "unable to add netvsc device (ret %d)\n", ret
);
2299 memcpy(net
->dev_addr
, device_info
->mac_adr
, ETH_ALEN
);
2301 /* We must get rtnl lock before scheduling nvdev->subchan_work,
2302 * otherwise netvsc_subchan_work() can get rtnl lock first and wait
2303 * all subchannels to show up, but that may not happen because
2304 * netvsc_probe() can't get rtnl lock and as a result vmbus_onoffer()
2305 * -> ... -> device_add() -> ... -> __device_attach() can't get
2306 * the device lock, so all the subchannels can't be processed --
2307 * finally netvsc_subchan_work() hangs forever.
2311 if (nvdev
->num_chn
> 1)
2312 schedule_work(&nvdev
->subchan_work
);
2314 /* hw_features computed in rndis_netdev_set_hwcaps() */
2315 net
->features
= net
->hw_features
|
2316 NETIF_F_HIGHDMA
| NETIF_F_SG
|
2317 NETIF_F_HW_VLAN_CTAG_TX
| NETIF_F_HW_VLAN_CTAG_RX
;
2318 net
->vlan_features
= net
->features
;
2320 netdev_lockdep_set_classes(net
);
2322 /* MTU range: 68 - 1500 or 65521 */
2323 net
->min_mtu
= NETVSC_MTU_MIN
;
2324 if (nvdev
->nvsp_version
>= NVSP_PROTOCOL_VERSION_2
)
2325 net
->max_mtu
= NETVSC_MTU
- ETH_HLEN
;
2327 net
->max_mtu
= ETH_DATA_LEN
;
2329 ret
= register_netdevice(net
);
2331 pr_err("Unable to register netdev.\n");
2332 goto register_failed
;
2335 list_add(&net_device_ctx
->list
, &netvsc_dev_list
);
2343 rndis_filter_device_remove(dev
, nvdev
);
2347 free_percpu(net_device_ctx
->vf_stats
);
2349 hv_set_drvdata(dev
, NULL
);
2355 static int netvsc_remove(struct hv_device
*dev
)
2357 struct net_device_context
*ndev_ctx
;
2358 struct net_device
*vf_netdev
, *net
;
2359 struct netvsc_device
*nvdev
;
2361 net
= hv_get_drvdata(dev
);
2363 dev_err(&dev
->device
, "No net device to remove\n");
2367 ndev_ctx
= netdev_priv(net
);
2369 cancel_delayed_work_sync(&ndev_ctx
->dwork
);
2372 nvdev
= rtnl_dereference(ndev_ctx
->nvdev
);
2374 cancel_work_sync(&nvdev
->subchan_work
);
2377 * Call to the vsc driver to let it know that the device is being
2378 * removed. Also blocks mtu and channel changes.
2380 vf_netdev
= rtnl_dereference(ndev_ctx
->vf_netdev
);
2382 netvsc_unregister_vf(vf_netdev
);
2385 rndis_filter_device_remove(dev
, nvdev
);
2387 unregister_netdevice(net
);
2388 list_del(&ndev_ctx
->list
);
2392 hv_set_drvdata(dev
, NULL
);
2394 free_percpu(ndev_ctx
->vf_stats
);
2399 static const struct hv_vmbus_device_id id_table
[] = {
2405 MODULE_DEVICE_TABLE(vmbus
, id_table
);
2407 /* The one and only one */
2408 static struct hv_driver netvsc_drv
= {
2409 .name
= KBUILD_MODNAME
,
2410 .id_table
= id_table
,
2411 .probe
= netvsc_probe
,
2412 .remove
= netvsc_remove
,
2414 .probe_type
= PROBE_FORCE_SYNCHRONOUS
,
2419 * On Hyper-V, every VF interface is matched with a corresponding
2420 * synthetic interface. The synthetic interface is presented first
2421 * to the guest. When the corresponding VF instance is registered,
2422 * we will take care of switching the data path.
2424 static int netvsc_netdev_event(struct notifier_block
*this,
2425 unsigned long event
, void *ptr
)
2427 struct net_device
*event_dev
= netdev_notifier_info_to_dev(ptr
);
2429 /* Skip our own events */
2430 if (event_dev
->netdev_ops
== &device_ops
)
2433 /* Avoid non-Ethernet type devices */
2434 if (event_dev
->type
!= ARPHRD_ETHER
)
2437 /* Avoid Vlan dev with same MAC registering as VF */
2438 if (is_vlan_dev(event_dev
))
2441 /* Avoid Bonding master dev with same MAC registering as VF */
2442 if ((event_dev
->priv_flags
& IFF_BONDING
) &&
2443 (event_dev
->flags
& IFF_MASTER
))
2447 case NETDEV_REGISTER
:
2448 return netvsc_register_vf(event_dev
);
2449 case NETDEV_UNREGISTER
:
2450 return netvsc_unregister_vf(event_dev
);
2453 return netvsc_vf_changed(event_dev
);
2459 static struct notifier_block netvsc_netdev_notifier
= {
2460 .notifier_call
= netvsc_netdev_event
,
2463 static void __exit
netvsc_drv_exit(void)
2465 unregister_netdevice_notifier(&netvsc_netdev_notifier
);
2466 vmbus_driver_unregister(&netvsc_drv
);
2469 static int __init
netvsc_drv_init(void)
2473 if (ring_size
< RING_SIZE_MIN
) {
2474 ring_size
= RING_SIZE_MIN
;
2475 pr_info("Increased ring_size to %u (min allowed)\n",
2478 netvsc_ring_bytes
= ring_size
* PAGE_SIZE
;
2480 ret
= vmbus_driver_register(&netvsc_drv
);
2484 register_netdevice_notifier(&netvsc_netdev_notifier
);
2488 MODULE_LICENSE("GPL");
2489 MODULE_DESCRIPTION("Microsoft Hyper-V network driver");
2491 module_init(netvsc_drv_init
);
2492 module_exit(netvsc_drv_exit
);