]> git.ipfire.org Git - thirdparty/linux.git/blame - drivers/net/cxgb4/cxgb4_main.c
projects / thirdparty / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
cxgb4: advertise NETIF_F_TSO_ECN
[thirdparty/linux.git] / drivers / net / cxgb4 / cxgb4_main.c
CommitLineData
b8ff05a9
DM		 1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
44#include <linux/if_vlan.h>
45#include <linux/init.h>
46#include <linux/log2.h>
47#include <linux/mdio.h>
48#include <linux/module.h>
49#include <linux/moduleparam.h>
50#include <linux/mutex.h>
51#include <linux/netdevice.h>
52#include <linux/pci.h>
53#include <linux/aer.h>
54#include <linux/rtnetlink.h>
55#include <linux/sched.h>
56#include <linux/seq_file.h>
57#include <linux/sockios.h>
58#include <linux/vmalloc.h>
59#include <linux/workqueue.h>
60#include <net/neighbour.h>
61#include <net/netevent.h>
62#include <asm/uaccess.h>
63
64#include "cxgb4.h"
65#include "t4_regs.h"
66#include "t4_msg.h"
67#include "t4fw_api.h"
68#include "l2t.h"
69
70#define DRV_VERSION "1.0.0-ko"
71#define DRV_DESC "Chelsio T4 Network Driver"
72
73/*
74 * Max interrupt hold-off timer value in us. Queues fall back to this value
75 * under extreme memory pressure so it's largish to give the system time to
76 * recover.
77 */
78#define MAX_SGE_TIMERVAL 200U
79
7ee9ff94
CL		 80#ifdef CONFIG_PCI_IOV
81/*
82 * Virtual Function provisioning constants. We need two extra Ingress Queues
83 * with Interrupt capability to serve as the VF's Firmware Event Queue and
84 * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
85 * Lists associated with them). For each Ethernet/Control Egress Queue and
86 * for each Free List, we need an Egress Context.
87 */
88enum {
89 VFRES_NPORTS = 1, /* # of "ports" per VF */
90 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
91
92 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
93 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
94 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
95 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
96 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
97 VFRES_TC = 0, /* PCI-E traffic class */
98 VFRES_NEXACTF = 16, /* # of exact MPS filters */
99
100 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
101 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
102};
103
104/*
105 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
106 * static and likely not to be useful in the long run. We really need to
107 * implement some form of persistent configuration which the firmware
108 * controls.
109 */
110static unsigned int pfvfres_pmask(struct adapter *adapter,
111 unsigned int pf, unsigned int vf)
112{
113 unsigned int portn, portvec;
114
115 /*
116 * Give PF's access to all of the ports.
117 */
118 if (vf == 0)
119 return FW_PFVF_CMD_PMASK_MASK;
120
121 /*
122 * For VFs, we'll assign them access to the ports based purely on the
123 * PF. We assign active ports in order, wrapping around if there are
124 * fewer active ports than PFs: e.g. active port[pf % nports].
125 * Unfortunately the adapter's port_info structs haven't been
126 * initialized yet so we have to compute this.
127 */
128 if (adapter->params.nports == 0)
129 return 0;
130
131 portn = pf % adapter->params.nports;
132 portvec = adapter->params.portvec;
133 for (;;) {
134 /*
135 * Isolate the lowest set bit in the port vector. If we're at
136 * the port number that we want, return that as the pmask.
137 * otherwise mask that bit out of the port vector and
138 * decrement our port number ...
139 */
140 unsigned int pmask = portvec ^ (portvec & (portvec-1));
141 if (portn == 0)
142 return pmask;
143 portn--;
144 portvec &= ~pmask;
145 }
146 /*NOTREACHED*/
147}
148#endif
149
b8ff05a9
DM		 150enum {
151 MEMWIN0_APERTURE = 65536,
152 MEMWIN0_BASE = 0x30000,
153 MEMWIN1_APERTURE = 32768,
154 MEMWIN1_BASE = 0x28000,
155 MEMWIN2_APERTURE = 2048,
156 MEMWIN2_BASE = 0x1b800,
157};
158
159enum {
160 MAX_TXQ_ENTRIES = 16384,
161 MAX_CTRL_TXQ_ENTRIES = 1024,
162 MAX_RSPQ_ENTRIES = 16384,
163 MAX_RX_BUFFERS = 16384,
164 MIN_TXQ_ENTRIES = 32,
165 MIN_CTRL_TXQ_ENTRIES = 32,
166 MIN_RSPQ_ENTRIES = 128,
167 MIN_FL_ENTRIES = 16
168};
169
170#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
171 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
172 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
173
174#define CH_DEVICE(devid) { PCI_VDEVICE(CHELSIO, devid), 0 }
175
176static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
177 CH_DEVICE(0xa000), /* PE10K */
178 { 0, }
179};
180
181#define FW_FNAME "cxgb4/t4fw.bin"
182
183MODULE_DESCRIPTION(DRV_DESC);
184MODULE_AUTHOR("Chelsio Communications");
185MODULE_LICENSE("Dual BSD/GPL");
186MODULE_VERSION(DRV_VERSION);
187MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
188MODULE_FIRMWARE(FW_FNAME);
189
190static int dflt_msg_enable = DFLT_MSG_ENABLE;
191
192module_param(dflt_msg_enable, int, 0644);
193MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
194
195/*
196 * The driver uses the best interrupt scheme available on a platform in the
197 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
198 * of these schemes the driver may consider as follows:
199 *
200 * msi = 2: choose from among all three options
201 * msi = 1: only consider MSI and INTx interrupts
202 * msi = 0: force INTx interrupts
203 */
204static int msi = 2;
205
206module_param(msi, int, 0644);
207MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
208
209/*
210 * Queue interrupt hold-off timer values. Queues default to the first of these
211 * upon creation.
212 */
213static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
214
215module_param_array(intr_holdoff, uint, NULL, 0644);
216MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
217 "0..4 in microseconds");
218
219static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
220
221module_param_array(intr_cnt, uint, NULL, 0644);
222MODULE_PARM_DESC(intr_cnt,
223 "thresholds 1..3 for queue interrupt packet counters");
224
225static int vf_acls;
226
227#ifdef CONFIG_PCI_IOV
228module_param(vf_acls, bool, 0644);
229MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
230
231static unsigned int num_vf[4];
232
233module_param_array(num_vf, uint, NULL, 0644);
234MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
235#endif
236
237static struct dentry *cxgb4_debugfs_root;
238
239static LIST_HEAD(adapter_list);
240static DEFINE_MUTEX(uld_mutex);
241static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
242static const char *uld_str[] = { "RDMA", "iSCSI" };
243
244static void link_report(struct net_device *dev)
245{
246 if (!netif_carrier_ok(dev))
247 netdev_info(dev, "link down\n");
248 else {
249 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
250
251 const char *s = "10Mbps";
252 const struct port_info *p = netdev_priv(dev);
253
254 switch (p->link_cfg.speed) {
255 case SPEED_10000:
256 s = "10Gbps";
257 break;
258 case SPEED_1000:
259 s = "1000Mbps";
260 break;
261 case SPEED_100:
262 s = "100Mbps";
263 break;
264 }
265
266 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
267 fc[p->link_cfg.fc]);
268 }
269}
270
271void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
272{
273 struct net_device *dev = adapter->port[port_id];
274
275 /* Skip changes from disabled ports. */
276 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
277 if (link_stat)
278 netif_carrier_on(dev);
279 else
280 netif_carrier_off(dev);
281
282 link_report(dev);
283 }
284}
285
286void t4_os_portmod_changed(const struct adapter *adap, int port_id)
287{
288 static const char *mod_str[] = {
a0881cab 289 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
b8ff05a9
DM		 290 };
291
292 const struct net_device *dev = adap->port[port_id];
293 const struct port_info *pi = netdev_priv(dev);
294
295 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
296 netdev_info(dev, "port module unplugged\n");
a0881cab 297 else if (pi->mod_type < ARRAY_SIZE(mod_str))
b8ff05a9
DM		 298 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
299}
300
301/*
302 * Configure the exact and hash address filters to handle a port's multicast
303 * and secondary unicast MAC addresses.
304 */
305static int set_addr_filters(const struct net_device *dev, bool sleep)
306{
307 u64 mhash = 0;
308 u64 uhash = 0;
309 bool free = true;
310 u16 filt_idx[7];
311 const u8 *addr[7];
312 int ret, naddr = 0;
b8ff05a9
DM		 313 const struct netdev_hw_addr *ha;
314 int uc_cnt = netdev_uc_count(dev);
4a35ecf8 315 int mc_cnt = netdev_mc_count(dev);
b8ff05a9
DM		 316 const struct port_info *pi = netdev_priv(dev);
317
318 /* first do the secondary unicast addresses */
319 netdev_for_each_uc_addr(ha, dev) {
320 addr[naddr++] = ha->addr;
321 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
322 ret = t4_alloc_mac_filt(pi->adapter, 0, pi->viid, free,
323 naddr, addr, filt_idx, &uhash, sleep);
324 if (ret < 0)
325 return ret;
326
327 free = false;
328 naddr = 0;
329 }
330 }
331
332 /* next set up the multicast addresses */
4a35ecf8
DM		 333 netdev_for_each_mc_addr(ha, dev) {
334 addr[naddr++] = ha->addr;
335 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
b8ff05a9
DM		 336 ret = t4_alloc_mac_filt(pi->adapter, 0, pi->viid, free,
337 naddr, addr, filt_idx, &mhash, sleep);
338 if (ret < 0)
339 return ret;
340
341 free = false;
342 naddr = 0;
343 }
344 }
345
346 return t4_set_addr_hash(pi->adapter, 0, pi->viid, uhash != 0,
347 uhash | mhash, sleep);
348}
349
350/*
351 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
352 * If @mtu is -1 it is left unchanged.
353 */
354static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
355{
356 int ret;
357 struct port_info *pi = netdev_priv(dev);
358
359 ret = set_addr_filters(dev, sleep_ok);
360 if (ret == 0)
361 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, mtu,
362 (dev->flags & IFF_PROMISC) ? 1 : 0,
f8f5aafa 363 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
b8ff05a9
DM		 364 sleep_ok);
365 return ret;
366}
367
368/**
369 * link_start - enable a port
370 * @dev: the port to enable
371 *
372 * Performs the MAC and PHY actions needed to enable a port.
373 */
374static int link_start(struct net_device *dev)
375{
376 int ret;
377 struct port_info *pi = netdev_priv(dev);
378
379 /*
380 * We do not set address filters and promiscuity here, the stack does
381 * that step explicitly.
382 */
383 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, dev->mtu, -1, -1, -1,
f8f5aafa 384 pi->vlan_grp != NULL, true);
b8ff05a9
DM		 385 if (ret == 0) {
386 ret = t4_change_mac(pi->adapter, 0, pi->viid,
387 pi->xact_addr_filt, dev->dev_addr, true,
b6bd29e7 388 true);
b8ff05a9
DM		 389 if (ret >= 0) {
390 pi->xact_addr_filt = ret;
391 ret = 0;
392 }
393 }
394 if (ret == 0)
395 ret = t4_link_start(pi->adapter, 0, pi->tx_chan, &pi->link_cfg);
396 if (ret == 0)
397 ret = t4_enable_vi(pi->adapter, 0, pi->viid, true, true);
398 return ret;
399}
400
401/*
402 * Response queue handler for the FW event queue.
403 */
404static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
405 const struct pkt_gl *gl)
406{
407 u8 opcode = ((const struct rss_header *)rsp)->opcode;
408
409 rsp++; /* skip RSS header */
410 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
411 const struct cpl_sge_egr_update *p = (void *)rsp;
412 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
413 struct sge_txq *txq = q->adap->sge.egr_map[qid];
414
415 txq->restarts++;
416 if ((u8 *)txq < (u8 *)q->adap->sge.ethrxq) {
417 struct sge_eth_txq *eq;
418
419 eq = container_of(txq, struct sge_eth_txq, q);
420 netif_tx_wake_queue(eq->txq);
421 } else {
422 struct sge_ofld_txq *oq;
423
424 oq = container_of(txq, struct sge_ofld_txq, q);
425 tasklet_schedule(&oq->qresume_tsk);
426 }
427 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
428 const struct cpl_fw6_msg *p = (void *)rsp;
429
430 if (p->type == 0)
431 t4_handle_fw_rpl(q->adap, p->data);
432 } else if (opcode == CPL_L2T_WRITE_RPL) {
433 const struct cpl_l2t_write_rpl *p = (void *)rsp;
434
435 do_l2t_write_rpl(q->adap, p);
436 } else
437 dev_err(q->adap->pdev_dev,
438 "unexpected CPL %#x on FW event queue\n", opcode);
439 return 0;
440}
441
442/**
443 * uldrx_handler - response queue handler for ULD queues
444 * @q: the response queue that received the packet
445 * @rsp: the response queue descriptor holding the offload message
446 * @gl: the gather list of packet fragments
447 *
448 * Deliver an ingress offload packet to a ULD. All processing is done by
449 * the ULD, we just maintain statistics.
450 */
451static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
452 const struct pkt_gl *gl)
453{
454 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
455
456 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
457 rxq->stats.nomem++;
458 return -1;
459 }
460 if (gl == NULL)
461 rxq->stats.imm++;
462 else if (gl == CXGB4_MSG_AN)
463 rxq->stats.an++;
464 else
465 rxq->stats.pkts++;
466 return 0;
467}
468
469static void disable_msi(struct adapter *adapter)
470{
471 if (adapter->flags & USING_MSIX) {
472 pci_disable_msix(adapter->pdev);
473 adapter->flags &= ~USING_MSIX;
474 } else if (adapter->flags & USING_MSI) {
475 pci_disable_msi(adapter->pdev);
476 adapter->flags &= ~USING_MSI;
477 }
478}
479
480/*
481 * Interrupt handler for non-data events used with MSI-X.
482 */
483static irqreturn_t t4_nondata_intr(int irq, void *cookie)
484{
485 struct adapter *adap = cookie;
486
487 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
488 if (v & PFSW) {
489 adap->swintr = 1;
490 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
491 }
492 t4_slow_intr_handler(adap);
493 return IRQ_HANDLED;
494}
495
496/*
497 * Name the MSI-X interrupts.
498 */
499static void name_msix_vecs(struct adapter *adap)
500{
501 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc) - 1;
502
503 /* non-data interrupts */
504 snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
505 adap->msix_info[0].desc[n] = 0;
506
507 /* FW events */
508 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", adap->name);
509 adap->msix_info[1].desc[n] = 0;
510
511 /* Ethernet queues */
512 for_each_port(adap, j) {
513 struct net_device *d = adap->port[j];
514 const struct port_info *pi = netdev_priv(d);
515
516 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
517 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
518 d->name, i);
519 adap->msix_info[msi_idx].desc[n] = 0;
520 }
521 }
522
523 /* offload queues */
524 for_each_ofldrxq(&adap->sge, i) {
525 snprintf(adap->msix_info[msi_idx].desc, n, "%s-ofld%d",
526 adap->name, i);
527 adap->msix_info[msi_idx++].desc[n] = 0;
528 }
529 for_each_rdmarxq(&adap->sge, i) {
530 snprintf(adap->msix_info[msi_idx].desc, n, "%s-rdma%d",
531 adap->name, i);
532 adap->msix_info[msi_idx++].desc[n] = 0;
533 }
534}
535
536static int request_msix_queue_irqs(struct adapter *adap)
537{
538 struct sge *s = &adap->sge;
539 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
540
541 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
542 adap->msix_info[1].desc, &s->fw_evtq);
543 if (err)
544 return err;
545
546 for_each_ethrxq(s, ethqidx) {
547 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
548 adap->msix_info[msi].desc,
549 &s->ethrxq[ethqidx].rspq);
550 if (err)
551 goto unwind;
552 msi++;
553 }
554 for_each_ofldrxq(s, ofldqidx) {
555 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
556 adap->msix_info[msi].desc,
557 &s->ofldrxq[ofldqidx].rspq);
558 if (err)
559 goto unwind;
560 msi++;
561 }
562 for_each_rdmarxq(s, rdmaqidx) {
563 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
564 adap->msix_info[msi].desc,
565 &s->rdmarxq[rdmaqidx].rspq);
566 if (err)
567 goto unwind;
568 msi++;
569 }
570 return 0;
571
572unwind:
573 while (--rdmaqidx >= 0)
574 free_irq(adap->msix_info[--msi].vec,
575 &s->rdmarxq[rdmaqidx].rspq);
576 while (--ofldqidx >= 0)
577 free_irq(adap->msix_info[--msi].vec,
578 &s->ofldrxq[ofldqidx].rspq);
579 while (--ethqidx >= 0)
580 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
581 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
582 return err;
583}
584
585static void free_msix_queue_irqs(struct adapter *adap)
586{
587 int i, msi = 2;
588 struct sge *s = &adap->sge;
589
590 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
591 for_each_ethrxq(s, i)
592 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
593 for_each_ofldrxq(s, i)
594 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
595 for_each_rdmarxq(s, i)
596 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
597}
598
671b0060
DM		 599/**
600 * write_rss - write the RSS table for a given port
601 * @pi: the port
602 * @queues: array of queue indices for RSS
603 *
604 * Sets up the portion of the HW RSS table for the port's VI to distribute
605 * packets to the Rx queues in @queues.
606 */
607static int write_rss(const struct port_info *pi, const u16 *queues)
608{
609 u16 *rss;
610 int i, err;
611 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
612
613 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
614 if (!rss)
615 return -ENOMEM;
616
617 /* map the queue indices to queue ids */
618 for (i = 0; i < pi->rss_size; i++, queues++)
619 rss[i] = q[*queues].rspq.abs_id;
620
621 err = t4_config_rss_range(pi->adapter, 0, pi->viid, 0, pi->rss_size,
622 rss, pi->rss_size);
623 kfree(rss);
624 return err;
625}
626
b8ff05a9
DM		 627/**
628 * setup_rss - configure RSS
629 * @adap: the adapter
630 *
671b0060 631 * Sets up RSS for each port.
b8ff05a9
DM		 632 */
633static int setup_rss(struct adapter *adap)
634{
671b0060 635 int i, err;
b8ff05a9
DM		 636
637 for_each_port(adap, i) {
638 const struct port_info *pi = adap2pinfo(adap, i);
b8ff05a9 639
671b0060 640 err = write_rss(pi, pi->rss);
b8ff05a9
DM		 641 if (err)
642 return err;
643 }
644 return 0;
645}
646
647/*
648 * Wait until all NAPI handlers are descheduled.
649 */
650static void quiesce_rx(struct adapter *adap)
651{
652 int i;
653
654 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
655 struct sge_rspq *q = adap->sge.ingr_map[i];
656
657 if (q && q->handler)
658 napi_disable(&q->napi);
659 }
660}
661
662/*
663 * Enable NAPI scheduling and interrupt generation for all Rx queues.
664 */
665static void enable_rx(struct adapter *adap)
666{
667 int i;
668
669 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
670 struct sge_rspq *q = adap->sge.ingr_map[i];
671
672 if (!q)
673 continue;
674 if (q->handler)
675 napi_enable(&q->napi);
676 /* 0-increment GTS to start the timer and enable interrupts */
677 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
678 SEINTARM(q->intr_params) |
679 INGRESSQID(q->cntxt_id));
680 }
681}
682
683/**
684 * setup_sge_queues - configure SGE Tx/Rx/response queues
685 * @adap: the adapter
686 *
687 * Determines how many sets of SGE queues to use and initializes them.
688 * We support multiple queue sets per port if we have MSI-X, otherwise
689 * just one queue set per port.
690 */
691static int setup_sge_queues(struct adapter *adap)
692{
693 int err, msi_idx, i, j;
694 struct sge *s = &adap->sge;
695
696 bitmap_zero(s->starving_fl, MAX_EGRQ);
697 bitmap_zero(s->txq_maperr, MAX_EGRQ);
698
699 if (adap->flags & USING_MSIX)
700 msi_idx = 1; /* vector 0 is for non-queue interrupts */
701 else {
702 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
703 NULL, NULL);
704 if (err)
705 return err;
706 msi_idx = -((int)s->intrq.abs_id + 1);
707 }
708
709 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
710 msi_idx, NULL, fwevtq_handler);
711 if (err) {
712freeout: t4_free_sge_resources(adap);
713 return err;
714 }
715
716 for_each_port(adap, i) {
717 struct net_device *dev = adap->port[i];
718 struct port_info *pi = netdev_priv(dev);
719 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
720 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
721
722 for (j = 0; j < pi->nqsets; j++, q++) {
723 if (msi_idx > 0)
724 msi_idx++;
725 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
726 msi_idx, &q->fl,
727 t4_ethrx_handler);
728 if (err)
729 goto freeout;
730 q->rspq.idx = j;
731 memset(&q->stats, 0, sizeof(q->stats));
732 }
733 for (j = 0; j < pi->nqsets; j++, t++) {
734 err = t4_sge_alloc_eth_txq(adap, t, dev,
735 netdev_get_tx_queue(dev, j),
736 s->fw_evtq.cntxt_id);
737 if (err)
738 goto freeout;
739 }
740 }
741
742 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
743 for_each_ofldrxq(s, i) {
744 struct sge_ofld_rxq *q = &s->ofldrxq[i];
745 struct net_device *dev = adap->port[i / j];
746
747 if (msi_idx > 0)
748 msi_idx++;
749 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
750 &q->fl, uldrx_handler);
751 if (err)
752 goto freeout;
753 memset(&q->stats, 0, sizeof(q->stats));
754 s->ofld_rxq[i] = q->rspq.abs_id;
755 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
756 s->fw_evtq.cntxt_id);
757 if (err)
758 goto freeout;
759 }
760
761 for_each_rdmarxq(s, i) {
762 struct sge_ofld_rxq *q = &s->rdmarxq[i];
763
764 if (msi_idx > 0)
765 msi_idx++;
766 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
767 msi_idx, &q->fl, uldrx_handler);
768 if (err)
769 goto freeout;
770 memset(&q->stats, 0, sizeof(q->stats));
771 s->rdma_rxq[i] = q->rspq.abs_id;
772 }
773
774 for_each_port(adap, i) {
775 /*
776 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
777 * have RDMA queues, and that's the right value.
778 */
779 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
780 s->fw_evtq.cntxt_id,
781 s->rdmarxq[i].rspq.cntxt_id);
782 if (err)
783 goto freeout;
784 }
785
786 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
787 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
788 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
789 return 0;
790}
791
792/*
793 * Returns 0 if new FW was successfully loaded, a positive errno if a load was
794 * started but failed, and a negative errno if flash load couldn't start.
795 */
796static int upgrade_fw(struct adapter *adap)
797{
798 int ret;
799 u32 vers;
800 const struct fw_hdr *hdr;
801 const struct firmware *fw;
802 struct device *dev = adap->pdev_dev;
803
804 ret = request_firmware(&fw, FW_FNAME, dev);
805 if (ret < 0) {
806 dev_err(dev, "unable to load firmware image " FW_FNAME
807 ", error %d\n", ret);
808 return ret;
809 }
810
811 hdr = (const struct fw_hdr *)fw->data;
812 vers = ntohl(hdr->fw_ver);
813 if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
814 ret = -EINVAL; /* wrong major version, won't do */
815 goto out;
816 }
817
818 /*
819 * If the flash FW is unusable or we found something newer, load it.
820 */
821 if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
822 vers > adap->params.fw_vers) {
823 ret = -t4_load_fw(adap, fw->data, fw->size);
824 if (!ret)
825 dev_info(dev, "firmware upgraded to version %pI4 from "
826 FW_FNAME "\n", &hdr->fw_ver);
827 }
828out: release_firmware(fw);
829 return ret;
830}
831
832/*
833 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
834 * The allocated memory is cleared.
835 */
836void *t4_alloc_mem(size_t size)
837{
838 void *p = kmalloc(size, GFP_KERNEL);
839
840 if (!p)
841 p = vmalloc(size);
842 if (p)
843 memset(p, 0, size);
844 return p;
845}
846
847/*
848 * Free memory allocated through alloc_mem().
849 */
850void t4_free_mem(void *addr)
851{
852 if (is_vmalloc_addr(addr))
853 vfree(addr);
854 else
855 kfree(addr);
856}
857
858static inline int is_offload(const struct adapter *adap)
859{
860 return adap->params.offload;
861}
862
863/*
864 * Implementation of ethtool operations.
865 */
866
867static u32 get_msglevel(struct net_device *dev)
868{
869 return netdev2adap(dev)->msg_enable;
870}
871
872static void set_msglevel(struct net_device *dev, u32 val)
873{
874 netdev2adap(dev)->msg_enable = val;
875}
876
877static char stats_strings[][ETH_GSTRING_LEN] = {
878 "TxOctetsOK ",
879 "TxFramesOK ",
880 "TxBroadcastFrames ",
881 "TxMulticastFrames ",
882 "TxUnicastFrames ",
883 "TxErrorFrames ",
884
885 "TxFrames64 ",
886 "TxFrames65To127 ",
887 "TxFrames128To255 ",
888 "TxFrames256To511 ",
889 "TxFrames512To1023 ",
890 "TxFrames1024To1518 ",
891 "TxFrames1519ToMax ",
892
893 "TxFramesDropped ",
894 "TxPauseFrames ",
895 "TxPPP0Frames ",
896 "TxPPP1Frames ",
897 "TxPPP2Frames ",
898 "TxPPP3Frames ",
899 "TxPPP4Frames ",
900 "TxPPP5Frames ",
901 "TxPPP6Frames ",
902 "TxPPP7Frames ",
903
904 "RxOctetsOK ",
905 "RxFramesOK ",
906 "RxBroadcastFrames ",
907 "RxMulticastFrames ",
908 "RxUnicastFrames ",
909
910 "RxFramesTooLong ",
911 "RxJabberErrors ",
912 "RxFCSErrors ",
913 "RxLengthErrors ",
914 "RxSymbolErrors ",
915 "RxRuntFrames ",
916
917 "RxFrames64 ",
918 "RxFrames65To127 ",
919 "RxFrames128To255 ",
920 "RxFrames256To511 ",
921 "RxFrames512To1023 ",
922 "RxFrames1024To1518 ",
923 "RxFrames1519ToMax ",
924
925 "RxPauseFrames ",
926 "RxPPP0Frames ",
927 "RxPPP1Frames ",
928 "RxPPP2Frames ",
929 "RxPPP3Frames ",
930 "RxPPP4Frames ",
931 "RxPPP5Frames ",
932 "RxPPP6Frames ",
933 "RxPPP7Frames ",
934
935 "RxBG0FramesDropped ",
936 "RxBG1FramesDropped ",
937 "RxBG2FramesDropped ",
938 "RxBG3FramesDropped ",
939 "RxBG0FramesTrunc ",
940 "RxBG1FramesTrunc ",
941 "RxBG2FramesTrunc ",
942 "RxBG3FramesTrunc ",
943
944 "TSO ",
945 "TxCsumOffload ",
946 "RxCsumGood ",
947 "VLANextractions ",
948 "VLANinsertions ",
4a6346d4
DM		 949 "GROpackets ",
950 "GROmerged ",
b8ff05a9
DM		 951};
952
953static int get_sset_count(struct net_device *dev, int sset)
954{
955 switch (sset) {
956 case ETH_SS_STATS:
957 return ARRAY_SIZE(stats_strings);
958 default:
959 return -EOPNOTSUPP;
960 }
961}
962
963#define T4_REGMAP_SIZE (160 * 1024)
964
965static int get_regs_len(struct net_device *dev)
966{
967 return T4_REGMAP_SIZE;
968}
969
970static int get_eeprom_len(struct net_device *dev)
971{
972 return EEPROMSIZE;
973}
974
975static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
976{
977 struct adapter *adapter = netdev2adap(dev);
978
979 strcpy(info->driver, KBUILD_MODNAME);
980 strcpy(info->version, DRV_VERSION);
981 strcpy(info->bus_info, pci_name(adapter->pdev));
982
983 if (!adapter->params.fw_vers)
984 strcpy(info->fw_version, "N/A");
985 else
986 snprintf(info->fw_version, sizeof(info->fw_version),
987 "%u.%u.%u.%u, TP %u.%u.%u.%u",
988 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
989 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
990 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
991 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
992 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
993 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
994 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
995 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
996}
997
998static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
999{
1000 if (stringset == ETH_SS_STATS)
1001 memcpy(data, stats_strings, sizeof(stats_strings));
1002}
1003
1004/*
1005 * port stats maintained per queue of the port. They should be in the same
1006 * order as in stats_strings above.
1007 */
1008struct queue_port_stats {
1009 u64 tso;
1010 u64 tx_csum;
1011 u64 rx_csum;
1012 u64 vlan_ex;
1013 u64 vlan_ins;
4a6346d4
DM		 1014 u64 gro_pkts;
1015 u64 gro_merged;
b8ff05a9
DM		 1016};
1017
1018static void collect_sge_port_stats(const struct adapter *adap,
1019 const struct port_info *p, struct queue_port_stats *s)
1020{
1021 int i;
1022 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1023 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1024
1025 memset(s, 0, sizeof(*s));
1026 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1027 s->tso += tx->tso;
1028 s->tx_csum += tx->tx_cso;
1029 s->rx_csum += rx->stats.rx_cso;
1030 s->vlan_ex += rx->stats.vlan_ex;
1031 s->vlan_ins += tx->vlan_ins;
4a6346d4
DM		 1032 s->gro_pkts += rx->stats.lro_pkts;
1033 s->gro_merged += rx->stats.lro_merged;
b8ff05a9
DM		 1034 }
1035}
1036
1037static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1038 u64 *data)
1039{
1040 struct port_info *pi = netdev_priv(dev);
1041 struct adapter *adapter = pi->adapter;
1042
1043 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1044
1045 data += sizeof(struct port_stats) / sizeof(u64);
1046 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1047}
1048
1049/*
1050 * Return a version number to identify the type of adapter. The scheme is:
1051 * - bits 0..9: chip version
1052 * - bits 10..15: chip revision
835bb606 1053 * - bits 16..23: register dump version
b8ff05a9
DM		 1054 */
1055static inline unsigned int mk_adap_vers(const struct adapter *ap)
1056{
835bb606 1057 return 4 | (ap->params.rev << 10) | (1 << 16);
b8ff05a9
DM		 1058}
1059
1060static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1061 unsigned int end)
1062{
1063 u32 *p = buf + start;
1064
1065 for ( ; start <= end; start += sizeof(u32))
1066 *p++ = t4_read_reg(ap, start);
1067}
1068
1069static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1070 void *buf)
1071{
1072 static const unsigned int reg_ranges[] = {
1073 0x1008, 0x1108,
1074 0x1180, 0x11b4,
1075 0x11fc, 0x123c,
1076 0x1300, 0x173c,
1077 0x1800, 0x18fc,
1078 0x3000, 0x30d8,
1079 0x30e0, 0x5924,
1080 0x5960, 0x59d4,
1081 0x5a00, 0x5af8,
1082 0x6000, 0x6098,
1083 0x6100, 0x6150,
1084 0x6200, 0x6208,
1085 0x6240, 0x6248,
1086 0x6280, 0x6338,
1087 0x6370, 0x638c,
1088 0x6400, 0x643c,
1089 0x6500, 0x6524,
1090 0x6a00, 0x6a38,
1091 0x6a60, 0x6a78,
1092 0x6b00, 0x6b84,
1093 0x6bf0, 0x6c84,
1094 0x6cf0, 0x6d84,
1095 0x6df0, 0x6e84,
1096 0x6ef0, 0x6f84,
1097 0x6ff0, 0x7084,
1098 0x70f0, 0x7184,
1099 0x71f0, 0x7284,
1100 0x72f0, 0x7384,
1101 0x73f0, 0x7450,
1102 0x7500, 0x7530,
1103 0x7600, 0x761c,
1104 0x7680, 0x76cc,
1105 0x7700, 0x7798,
1106 0x77c0, 0x77fc,
1107 0x7900, 0x79fc,
1108 0x7b00, 0x7c38,
1109 0x7d00, 0x7efc,
1110 0x8dc0, 0x8e1c,
1111 0x8e30, 0x8e78,
1112 0x8ea0, 0x8f6c,
1113 0x8fc0, 0x9074,
1114 0x90fc, 0x90fc,
1115 0x9400, 0x9458,
1116 0x9600, 0x96bc,
1117 0x9800, 0x9808,
1118 0x9820, 0x983c,
1119 0x9850, 0x9864,
1120 0x9c00, 0x9c6c,
1121 0x9c80, 0x9cec,
1122 0x9d00, 0x9d6c,
1123 0x9d80, 0x9dec,
1124 0x9e00, 0x9e6c,
1125 0x9e80, 0x9eec,
1126 0x9f00, 0x9f6c,
1127 0x9f80, 0x9fec,
1128 0xd004, 0xd03c,
1129 0xdfc0, 0xdfe0,
1130 0xe000, 0xea7c,
1131 0xf000, 0x11190,
835bb606
DM		 1132 0x19040, 0x1906c,
1133 0x19078, 0x19080,
1134 0x1908c, 0x19124,
b8ff05a9
DM		 1135 0x19150, 0x191b0,
1136 0x191d0, 0x191e8,
1137 0x19238, 0x1924c,
1138 0x193f8, 0x19474,
1139 0x19490, 0x194f8,
1140 0x19800, 0x19f30,
1141 0x1a000, 0x1a06c,
1142 0x1a0b0, 0x1a120,
1143 0x1a128, 0x1a138,
1144 0x1a190, 0x1a1c4,
1145 0x1a1fc, 0x1a1fc,
1146 0x1e040, 0x1e04c,
835bb606 1147 0x1e284, 0x1e28c,
b8ff05a9
DM		 1148 0x1e2c0, 0x1e2c0,
1149 0x1e2e0, 0x1e2e0,
1150 0x1e300, 0x1e384,
1151 0x1e3c0, 0x1e3c8,
1152 0x1e440, 0x1e44c,
835bb606 1153 0x1e684, 0x1e68c,
b8ff05a9
DM		 1154 0x1e6c0, 0x1e6c0,
1155 0x1e6e0, 0x1e6e0,
1156 0x1e700, 0x1e784,
1157 0x1e7c0, 0x1e7c8,
1158 0x1e840, 0x1e84c,
835bb606 1159 0x1ea84, 0x1ea8c,
b8ff05a9
DM		 1160 0x1eac0, 0x1eac0,
1161 0x1eae0, 0x1eae0,
1162 0x1eb00, 0x1eb84,
1163 0x1ebc0, 0x1ebc8,
1164 0x1ec40, 0x1ec4c,
835bb606 1165 0x1ee84, 0x1ee8c,
b8ff05a9
DM		 1166 0x1eec0, 0x1eec0,
1167 0x1eee0, 0x1eee0,
1168 0x1ef00, 0x1ef84,
1169 0x1efc0, 0x1efc8,
1170 0x1f040, 0x1f04c,
835bb606 1171 0x1f284, 0x1f28c,
b8ff05a9
DM		 1172 0x1f2c0, 0x1f2c0,
1173 0x1f2e0, 0x1f2e0,
1174 0x1f300, 0x1f384,
1175 0x1f3c0, 0x1f3c8,
1176 0x1f440, 0x1f44c,
835bb606 1177 0x1f684, 0x1f68c,
b8ff05a9
DM		 1178 0x1f6c0, 0x1f6c0,
1179 0x1f6e0, 0x1f6e0,
1180 0x1f700, 0x1f784,
1181 0x1f7c0, 0x1f7c8,
1182 0x1f840, 0x1f84c,
835bb606 1183 0x1fa84, 0x1fa8c,
b8ff05a9
DM		 1184 0x1fac0, 0x1fac0,
1185 0x1fae0, 0x1fae0,
1186 0x1fb00, 0x1fb84,
1187 0x1fbc0, 0x1fbc8,
1188 0x1fc40, 0x1fc4c,
835bb606 1189 0x1fe84, 0x1fe8c,
b8ff05a9
DM		 1190 0x1fec0, 0x1fec0,
1191 0x1fee0, 0x1fee0,
1192 0x1ff00, 0x1ff84,
1193 0x1ffc0, 0x1ffc8,
1194 0x20000, 0x2002c,
1195 0x20100, 0x2013c,
1196 0x20190, 0x201c8,
1197 0x20200, 0x20318,
1198 0x20400, 0x20528,
1199 0x20540, 0x20614,
1200 0x21000, 0x21040,
1201 0x2104c, 0x21060,
1202 0x210c0, 0x210ec,
1203 0x21200, 0x21268,
1204 0x21270, 0x21284,
1205 0x212fc, 0x21388,
1206 0x21400, 0x21404,
1207 0x21500, 0x21518,
1208 0x2152c, 0x2153c,
1209 0x21550, 0x21554,
1210 0x21600, 0x21600,
1211 0x21608, 0x21628,
1212 0x21630, 0x2163c,
1213 0x21700, 0x2171c,
1214 0x21780, 0x2178c,
1215 0x21800, 0x21c38,
1216 0x21c80, 0x21d7c,
1217 0x21e00, 0x21e04,
1218 0x22000, 0x2202c,
1219 0x22100, 0x2213c,
1220 0x22190, 0x221c8,
1221 0x22200, 0x22318,
1222 0x22400, 0x22528,
1223 0x22540, 0x22614,
1224 0x23000, 0x23040,
1225 0x2304c, 0x23060,
1226 0x230c0, 0x230ec,
1227 0x23200, 0x23268,
1228 0x23270, 0x23284,
1229 0x232fc, 0x23388,
1230 0x23400, 0x23404,
1231 0x23500, 0x23518,
1232 0x2352c, 0x2353c,
1233 0x23550, 0x23554,
1234 0x23600, 0x23600,
1235 0x23608, 0x23628,
1236 0x23630, 0x2363c,
1237 0x23700, 0x2371c,
1238 0x23780, 0x2378c,
1239 0x23800, 0x23c38,
1240 0x23c80, 0x23d7c,
1241 0x23e00, 0x23e04,
1242 0x24000, 0x2402c,
1243 0x24100, 0x2413c,
1244 0x24190, 0x241c8,
1245 0x24200, 0x24318,
1246 0x24400, 0x24528,
1247 0x24540, 0x24614,
1248 0x25000, 0x25040,
1249 0x2504c, 0x25060,
1250 0x250c0, 0x250ec,
1251 0x25200, 0x25268,
1252 0x25270, 0x25284,
1253 0x252fc, 0x25388,
1254 0x25400, 0x25404,
1255 0x25500, 0x25518,
1256 0x2552c, 0x2553c,
1257 0x25550, 0x25554,
1258 0x25600, 0x25600,
1259 0x25608, 0x25628,
1260 0x25630, 0x2563c,
1261 0x25700, 0x2571c,
1262 0x25780, 0x2578c,
1263 0x25800, 0x25c38,
1264 0x25c80, 0x25d7c,
1265 0x25e00, 0x25e04,
1266 0x26000, 0x2602c,
1267 0x26100, 0x2613c,
1268 0x26190, 0x261c8,
1269 0x26200, 0x26318,
1270 0x26400, 0x26528,
1271 0x26540, 0x26614,
1272 0x27000, 0x27040,
1273 0x2704c, 0x27060,
1274 0x270c0, 0x270ec,
1275 0x27200, 0x27268,
1276 0x27270, 0x27284,
1277 0x272fc, 0x27388,
1278 0x27400, 0x27404,
1279 0x27500, 0x27518,
1280 0x2752c, 0x2753c,
1281 0x27550, 0x27554,
1282 0x27600, 0x27600,
1283 0x27608, 0x27628,
1284 0x27630, 0x2763c,
1285 0x27700, 0x2771c,
1286 0x27780, 0x2778c,
1287 0x27800, 0x27c38,
1288 0x27c80, 0x27d7c,
1289 0x27e00, 0x27e04
1290 };
1291
1292 int i;
1293 struct adapter *ap = netdev2adap(dev);
1294
1295 regs->version = mk_adap_vers(ap);
1296
1297 memset(buf, 0, T4_REGMAP_SIZE);
1298 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1299 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1300}
1301
1302static int restart_autoneg(struct net_device *dev)
1303{
1304 struct port_info *p = netdev_priv(dev);
1305
1306 if (!netif_running(dev))
1307 return -EAGAIN;
1308 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1309 return -EINVAL;
1310 t4_restart_aneg(p->adapter, 0, p->tx_chan);
1311 return 0;
1312}
1313
1314static int identify_port(struct net_device *dev, u32 data)
1315{
1316 if (data == 0)
1317 data = 2; /* default to 2 seconds */
1318
1319 return t4_identify_port(netdev2adap(dev), 0, netdev2pinfo(dev)->viid,
1320 data * 5);
1321}
1322
1323static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1324{
1325 unsigned int v = 0;
1326
a0881cab
DM		 1327 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1328 type == FW_PORT_TYPE_BT_XAUI) {
b8ff05a9
DM		 1329 v |= SUPPORTED_TP;
1330 if (caps & FW_PORT_CAP_SPEED_100M)
1331 v |= SUPPORTED_100baseT_Full;
1332 if (caps & FW_PORT_CAP_SPEED_1G)
1333 v |= SUPPORTED_1000baseT_Full;
1334 if (caps & FW_PORT_CAP_SPEED_10G)
1335 v |= SUPPORTED_10000baseT_Full;
1336 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1337 v |= SUPPORTED_Backplane;
1338 if (caps & FW_PORT_CAP_SPEED_1G)
1339 v |= SUPPORTED_1000baseKX_Full;
1340 if (caps & FW_PORT_CAP_SPEED_10G)
1341 v |= SUPPORTED_10000baseKX4_Full;
1342 } else if (type == FW_PORT_TYPE_KR)
1343 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
a0881cab
DM		 1344 else if (type == FW_PORT_TYPE_BP_AP)
1345 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC;
1346 else if (type == FW_PORT_TYPE_FIBER_XFI ||
1347 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
b8ff05a9
DM		 1348 v |= SUPPORTED_FIBRE;
1349
1350 if (caps & FW_PORT_CAP_ANEG)
1351 v |= SUPPORTED_Autoneg;
1352 return v;
1353}
1354
1355static unsigned int to_fw_linkcaps(unsigned int caps)
1356{
1357 unsigned int v = 0;
1358
1359 if (caps & ADVERTISED_100baseT_Full)
1360 v |= FW_PORT_CAP_SPEED_100M;
1361 if (caps & ADVERTISED_1000baseT_Full)
1362 v |= FW_PORT_CAP_SPEED_1G;
1363 if (caps & ADVERTISED_10000baseT_Full)
1364 v |= FW_PORT_CAP_SPEED_10G;
1365 return v;
1366}
1367
1368static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1369{
1370 const struct port_info *p = netdev_priv(dev);
1371
1372 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
a0881cab 1373 p->port_type == FW_PORT_TYPE_BT_XFI ||
b8ff05a9
DM		 1374 p->port_type == FW_PORT_TYPE_BT_XAUI)
1375 cmd->port = PORT_TP;
a0881cab
DM		 1376 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1377 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
b8ff05a9 1378 cmd->port = PORT_FIBRE;
a0881cab
DM		 1379 else if (p->port_type == FW_PORT_TYPE_SFP) {
1380 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1381 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1382 cmd->port = PORT_DA;
1383 else
1384 cmd->port = PORT_FIBRE;
1385 } else
b8ff05a9
DM		 1386 cmd->port = PORT_OTHER;
1387
1388 if (p->mdio_addr >= 0) {
1389 cmd->phy_address = p->mdio_addr;
1390 cmd->transceiver = XCVR_EXTERNAL;
1391 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1392 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1393 } else {
1394 cmd->phy_address = 0; /* not really, but no better option */
1395 cmd->transceiver = XCVR_INTERNAL;
1396 cmd->mdio_support = 0;
1397 }
1398
1399 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1400 cmd->advertising = from_fw_linkcaps(p->port_type,
1401 p->link_cfg.advertising);
1402 cmd->speed = netif_carrier_ok(dev) ? p->link_cfg.speed : 0;
1403 cmd->duplex = DUPLEX_FULL;
1404 cmd->autoneg = p->link_cfg.autoneg;
1405 cmd->maxtxpkt = 0;
1406 cmd->maxrxpkt = 0;
1407 return 0;
1408}
1409
1410static unsigned int speed_to_caps(int speed)
1411{
1412 if (speed == SPEED_100)
1413 return FW_PORT_CAP_SPEED_100M;
1414 if (speed == SPEED_1000)
1415 return FW_PORT_CAP_SPEED_1G;
1416 if (speed == SPEED_10000)
1417 return FW_PORT_CAP_SPEED_10G;
1418 return 0;
1419}
1420
1421static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1422{
1423 unsigned int cap;
1424 struct port_info *p = netdev_priv(dev);
1425 struct link_config *lc = &p->link_cfg;
1426
1427 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
1428 return -EINVAL;
1429
1430 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1431 /*
1432 * PHY offers a single speed. See if that's what's
1433 * being requested.
1434 */
1435 if (cmd->autoneg == AUTONEG_DISABLE &&
1436 (lc->supported & speed_to_caps(cmd->speed)))
1437 return 0;
1438 return -EINVAL;
1439 }
1440
1441 if (cmd->autoneg == AUTONEG_DISABLE) {
1442 cap = speed_to_caps(cmd->speed);
1443
1444 if (!(lc->supported & cap) || cmd->speed == SPEED_1000 ||
1445 cmd->speed == SPEED_10000)
1446 return -EINVAL;
1447 lc->requested_speed = cap;
1448 lc->advertising = 0;
1449 } else {
1450 cap = to_fw_linkcaps(cmd->advertising);
1451 if (!(lc->supported & cap))
1452 return -EINVAL;
1453 lc->requested_speed = 0;
1454 lc->advertising = cap | FW_PORT_CAP_ANEG;
1455 }
1456 lc->autoneg = cmd->autoneg;
1457
1458 if (netif_running(dev))
1459 return t4_link_start(p->adapter, 0, p->tx_chan, lc);
1460 return 0;
1461}
1462
1463static void get_pauseparam(struct net_device *dev,
1464 struct ethtool_pauseparam *epause)
1465{
1466 struct port_info *p = netdev_priv(dev);
1467
1468 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1469 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1470 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1471}
1472
1473static int set_pauseparam(struct net_device *dev,
1474 struct ethtool_pauseparam *epause)
1475{
1476 struct port_info *p = netdev_priv(dev);
1477 struct link_config *lc = &p->link_cfg;
1478
1479 if (epause->autoneg == AUTONEG_DISABLE)
1480 lc->requested_fc = 0;
1481 else if (lc->supported & FW_PORT_CAP_ANEG)
1482 lc->requested_fc = PAUSE_AUTONEG;
1483 else
1484 return -EINVAL;
1485
1486 if (epause->rx_pause)
1487 lc->requested_fc |= PAUSE_RX;
1488 if (epause->tx_pause)
1489 lc->requested_fc |= PAUSE_TX;
1490 if (netif_running(dev))
1491 return t4_link_start(p->adapter, 0, p->tx_chan, lc);
1492 return 0;
1493}
1494
1495static u32 get_rx_csum(struct net_device *dev)
1496{
1497 struct port_info *p = netdev_priv(dev);
1498
1499 return p->rx_offload & RX_CSO;
1500}
1501
1502static int set_rx_csum(struct net_device *dev, u32 data)
1503{
1504 struct port_info *p = netdev_priv(dev);
1505
1506 if (data)
1507 p->rx_offload |= RX_CSO;
1508 else
1509 p->rx_offload &= ~RX_CSO;
1510 return 0;
1511}
1512
1513static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1514{
1515 const struct port_info *pi = netdev_priv(dev);
1516 const struct sge *s = &pi->adapter->sge;
1517
1518 e->rx_max_pending = MAX_RX_BUFFERS;
1519 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1520 e->rx_jumbo_max_pending = 0;
1521 e->tx_max_pending = MAX_TXQ_ENTRIES;
1522
1523 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1524 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1525 e->rx_jumbo_pending = 0;
1526 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1527}
1528
1529static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1530{
1531 int i;
1532 const struct port_info *pi = netdev_priv(dev);
1533 struct adapter *adapter = pi->adapter;
1534 struct sge *s = &adapter->sge;
1535
1536 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1537 e->tx_pending > MAX_TXQ_ENTRIES ||
1538 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1539 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1540 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1541 return -EINVAL;
1542
1543 if (adapter->flags & FULL_INIT_DONE)
1544 return -EBUSY;
1545
1546 for (i = 0; i < pi->nqsets; ++i) {
1547 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1548 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1549 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1550 }
1551 return 0;
1552}
1553
1554static int closest_timer(const struct sge *s, int time)
1555{
1556 int i, delta, match = 0, min_delta = INT_MAX;
1557
1558 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1559 delta = time - s->timer_val[i];
1560 if (delta < 0)
1561 delta = -delta;
1562 if (delta < min_delta) {
1563 min_delta = delta;
1564 match = i;
1565 }
1566 }
1567 return match;
1568}
1569
1570static int closest_thres(const struct sge *s, int thres)
1571{
1572 int i, delta, match = 0, min_delta = INT_MAX;
1573
1574 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1575 delta = thres - s->counter_val[i];
1576 if (delta < 0)
1577 delta = -delta;
1578 if (delta < min_delta) {
1579 min_delta = delta;
1580 match = i;
1581 }
1582 }
1583 return match;
1584}
1585
1586/*
1587 * Return a queue's interrupt hold-off time in us. 0 means no timer.
1588 */
1589static unsigned int qtimer_val(const struct adapter *adap,
1590 const struct sge_rspq *q)
1591{
1592 unsigned int idx = q->intr_params >> 1;
1593
1594 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1595}
1596
1597/**
1598 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
1599 * @adap: the adapter
1600 * @q: the Rx queue
1601 * @us: the hold-off time in us, or 0 to disable timer
1602 * @cnt: the hold-off packet count, or 0 to disable counter
1603 *
1604 * Sets an Rx queue's interrupt hold-off time and packet count. At least
1605 * one of the two needs to be enabled for the queue to generate interrupts.
1606 */
1607static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1608 unsigned int us, unsigned int cnt)
1609{
1610 if ((us | cnt) == 0)
1611 cnt = 1;
1612
1613 if (cnt) {
1614 int err;
1615 u32 v, new_idx;
1616
1617 new_idx = closest_thres(&adap->sge, cnt);
1618 if (q->desc && q->pktcnt_idx != new_idx) {
1619 /* the queue has already been created, update it */
1620 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1621 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1622 FW_PARAMS_PARAM_YZ(q->cntxt_id);
1623 err = t4_set_params(adap, 0, 0, 0, 1, &v, &new_idx);
1624 if (err)
1625 return err;
1626 }
1627 q->pktcnt_idx = new_idx;
1628 }
1629
1630 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1631 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1632 return 0;
1633}
1634
1635static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1636{
1637 const struct port_info *pi = netdev_priv(dev);
1638 struct adapter *adap = pi->adapter;
1639
1640 return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1641 c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1642}
1643
1644static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1645{
1646 const struct port_info *pi = netdev_priv(dev);
1647 const struct adapter *adap = pi->adapter;
1648 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1649
1650 c->rx_coalesce_usecs = qtimer_val(adap, rq);
1651 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1652 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1653 return 0;
1654}
1655
1656/*
1657 * Translate a physical EEPROM address to virtual. The first 1K is accessed
1658 * through virtual addresses starting at 31K, the rest is accessed through
1659 * virtual addresses starting at 0. This mapping is correct only for PF0.
1660 */
1661static int eeprom_ptov(unsigned int phys_addr)
1662{
1663 if (phys_addr < 1024)
1664 return phys_addr + (31 << 10);
1665 if (phys_addr < EEPROMSIZE)
1666 return phys_addr - 1024;
1667 return -EINVAL;
1668}
1669
1670/*
1671 * The next two routines implement eeprom read/write from physical addresses.
1672 * The physical->virtual translation is correct only for PF0.
1673 */
1674static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1675{
1676 int vaddr = eeprom_ptov(phys_addr);
1677
1678 if (vaddr >= 0)
1679 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1680 return vaddr < 0 ? vaddr : 0;
1681}
1682
1683static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1684{
1685 int vaddr = eeprom_ptov(phys_addr);
1686
1687 if (vaddr >= 0)
1688 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1689 return vaddr < 0 ? vaddr : 0;
1690}
1691
1692#define EEPROM_MAGIC 0x38E2F10C
1693
1694static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1695 u8 *data)
1696{
1697 int i, err = 0;
1698 struct adapter *adapter = netdev2adap(dev);
1699
1700 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1701 if (!buf)
1702 return -ENOMEM;
1703
1704 e->magic = EEPROM_MAGIC;
1705 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1706 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1707
1708 if (!err)
1709 memcpy(data, buf + e->offset, e->len);
1710 kfree(buf);
1711 return err;
1712}
1713
1714static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1715 u8 *data)
1716{
1717 u8 *buf;
1718 int err = 0;
1719 u32 aligned_offset, aligned_len, *p;
1720 struct adapter *adapter = netdev2adap(dev);
1721
1722 if (eeprom->magic != EEPROM_MAGIC)
1723 return -EINVAL;
1724
1725 aligned_offset = eeprom->offset & ~3;
1726 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1727
1728 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1729 /*
1730 * RMW possibly needed for first or last words.
1731 */
1732 buf = kmalloc(aligned_len, GFP_KERNEL);
1733 if (!buf)
1734 return -ENOMEM;
1735 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1736 if (!err && aligned_len > 4)
1737 err = eeprom_rd_phys(adapter,
1738 aligned_offset + aligned_len - 4,
1739 (u32 *)&buf[aligned_len - 4]);
1740 if (err)
1741 goto out;
1742 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1743 } else
1744 buf = data;
1745
1746 err = t4_seeprom_wp(adapter, false);
1747 if (err)
1748 goto out;
1749
1750 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1751 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1752 aligned_offset += 4;
1753 }
1754
1755 if (!err)
1756 err = t4_seeprom_wp(adapter, true);
1757out:
1758 if (buf != data)
1759 kfree(buf);
1760 return err;
1761}
1762
1763static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1764{
1765 int ret;
1766 const struct firmware *fw;
1767 struct adapter *adap = netdev2adap(netdev);
1768
1769 ef->data[sizeof(ef->data) - 1] = '\0';
1770 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1771 if (ret < 0)
1772 return ret;
1773
1774 ret = t4_load_fw(adap, fw->data, fw->size);
1775 release_firmware(fw);
1776 if (!ret)
1777 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1778 return ret;
1779}
1780
1781#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1782#define BCAST_CRC 0xa0ccc1a6
1783
1784static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1785{
1786 wol->supported = WAKE_BCAST | WAKE_MAGIC;
1787 wol->wolopts = netdev2adap(dev)->wol;
1788 memset(&wol->sopass, 0, sizeof(wol->sopass));
1789}
1790
1791static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1792{
1793 int err = 0;
1794 struct port_info *pi = netdev_priv(dev);
1795
1796 if (wol->wolopts & ~WOL_SUPPORTED)
1797 return -EINVAL;
1798 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1799 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1800 if (wol->wolopts & WAKE_BCAST) {
1801 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1802 ~0ULL, 0, false);
1803 if (!err)
1804 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1805 ~6ULL, ~0ULL, BCAST_CRC, true);
1806 } else
1807 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1808 return err;
1809}
1810
35d35682
DM		 1811#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
1812
b8ff05a9
DM		 1813static int set_tso(struct net_device *dev, u32 value)
1814{
1815 if (value)
35d35682 1816 dev->features |= TSO_FLAGS;
b8ff05a9 1817 else
35d35682 1818 dev->features &= ~TSO_FLAGS;
b8ff05a9
DM		 1819 return 0;
1820}
1821
87b6cf51
DM		 1822static int set_flags(struct net_device *dev, u32 flags)
1823{
1437ce39 1824 return ethtool_op_set_flags(dev, flags, ETH_FLAG_RXHASH);
87b6cf51
DM		 1825}
1826
671b0060
DM		 1827static int get_rss_table(struct net_device *dev, struct ethtool_rxfh_indir *p)
1828{
1829 const struct port_info *pi = netdev_priv(dev);
1830 unsigned int n = min_t(unsigned int, p->size, pi->rss_size);
1831
1832 p->size = pi->rss_size;
1833 while (n--)
1834 p->ring_index[n] = pi->rss[n];
1835 return 0;
1836}
1837
1838static int set_rss_table(struct net_device *dev,
1839 const struct ethtool_rxfh_indir *p)
1840{
1841 unsigned int i;
1842 struct port_info *pi = netdev_priv(dev);
1843
1844 if (p->size != pi->rss_size)
1845 return -EINVAL;
1846 for (i = 0; i < p->size; i++)
1847 if (p->ring_index[i] >= pi->nqsets)
1848 return -EINVAL;
1849 for (i = 0; i < p->size; i++)
1850 pi->rss[i] = p->ring_index[i];
1851 if (pi->adapter->flags & FULL_INIT_DONE)
1852 return write_rss(pi, pi->rss);
1853 return 0;
1854}
1855
1856static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1857 void *rules)
1858{
f796564a
DM		 1859 const struct port_info *pi = netdev_priv(dev);
1860
671b0060 1861 switch (info->cmd) {
f796564a
DM		 1862 case ETHTOOL_GRXFH: {
1863 unsigned int v = pi->rss_mode;
1864
1865 info->data = 0;
1866 switch (info->flow_type) {
1867 case TCP_V4_FLOW:
1868 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
1869 info->data = RXH_IP_SRC | RXH_IP_DST |
1870 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1871 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1872 info->data = RXH_IP_SRC | RXH_IP_DST;
1873 break;
1874 case UDP_V4_FLOW:
1875 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
1876 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1877 info->data = RXH_IP_SRC | RXH_IP_DST |
1878 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1879 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1880 info->data = RXH_IP_SRC | RXH_IP_DST;
1881 break;
1882 case SCTP_V4_FLOW:
1883 case AH_ESP_V4_FLOW:
1884 case IPV4_FLOW:
1885 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1886 info->data = RXH_IP_SRC | RXH_IP_DST;
1887 break;
1888 case TCP_V6_FLOW:
1889 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
1890 info->data = RXH_IP_SRC | RXH_IP_DST |
1891 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1892 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1893 info->data = RXH_IP_SRC | RXH_IP_DST;
1894 break;
1895 case UDP_V6_FLOW:
1896 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
1897 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1898 info->data = RXH_IP_SRC | RXH_IP_DST |
1899 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1900 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1901 info->data = RXH_IP_SRC | RXH_IP_DST;
1902 break;
1903 case SCTP_V6_FLOW:
1904 case AH_ESP_V6_FLOW:
1905 case IPV6_FLOW:
1906 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1907 info->data = RXH_IP_SRC | RXH_IP_DST;
1908 break;
1909 }
1910 return 0;
1911 }
671b0060 1912 case ETHTOOL_GRXRINGS:
f796564a 1913 info->data = pi->nqsets;
671b0060
DM		 1914 return 0;
1915 }
1916 return -EOPNOTSUPP;
1917}
1918
b8ff05a9
DM		 1919static struct ethtool_ops cxgb_ethtool_ops = {
1920 .get_settings = get_settings,
1921 .set_settings = set_settings,
1922 .get_drvinfo = get_drvinfo,
1923 .get_msglevel = get_msglevel,
1924 .set_msglevel = set_msglevel,
1925 .get_ringparam = get_sge_param,
1926 .set_ringparam = set_sge_param,
1927 .get_coalesce = get_coalesce,
1928 .set_coalesce = set_coalesce,
1929 .get_eeprom_len = get_eeprom_len,
1930 .get_eeprom = get_eeprom,
1931 .set_eeprom = set_eeprom,
1932 .get_pauseparam = get_pauseparam,
1933 .set_pauseparam = set_pauseparam,
1934 .get_rx_csum = get_rx_csum,
1935 .set_rx_csum = set_rx_csum,
1936 .set_tx_csum = ethtool_op_set_tx_ipv6_csum,
1937 .set_sg = ethtool_op_set_sg,
1938 .get_link = ethtool_op_get_link,
1939 .get_strings = get_strings,
1940 .phys_id = identify_port,
1941 .nway_reset = restart_autoneg,
1942 .get_sset_count = get_sset_count,
1943 .get_ethtool_stats = get_stats,
1944 .get_regs_len = get_regs_len,
1945 .get_regs = get_regs,
1946 .get_wol = get_wol,
1947 .set_wol = set_wol,
1948 .set_tso = set_tso,
87b6cf51 1949 .set_flags = set_flags,
671b0060
DM		 1950 .get_rxnfc = get_rxnfc,
1951 .get_rxfh_indir = get_rss_table,
1952 .set_rxfh_indir = set_rss_table,
b8ff05a9
DM		 1953 .flash_device = set_flash,
1954};
1955
1956/*
1957 * debugfs support
1958 */
1959
1960static int mem_open(struct inode *inode, struct file *file)
1961{
1962 file->private_data = inode->i_private;
1963 return 0;
1964}
1965
1966static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
1967 loff_t *ppos)
1968{
1969 loff_t pos = *ppos;
1970 loff_t avail = file->f_path.dentry->d_inode->i_size;
1971 unsigned int mem = (uintptr_t)file->private_data & 3;
1972 struct adapter *adap = file->private_data - mem;
1973
1974 if (pos < 0)
1975 return -EINVAL;
1976 if (pos >= avail)
1977 return 0;
1978 if (count > avail - pos)
1979 count = avail - pos;
1980
1981 while (count) {
1982 size_t len;
1983 int ret, ofst;
1984 __be32 data[16];
1985
1986 if (mem == MEM_MC)
1987 ret = t4_mc_read(adap, pos, data, NULL);
1988 else
1989 ret = t4_edc_read(adap, mem, pos, data, NULL);
1990 if (ret)
1991 return ret;
1992
1993 ofst = pos % sizeof(data);
1994 len = min(count, sizeof(data) - ofst);
1995 if (copy_to_user(buf, (u8 *)data + ofst, len))
1996 return -EFAULT;
1997
1998 buf += len;
1999 pos += len;
2000 count -= len;
2001 }
2002 count = pos - *ppos;
2003 *ppos = pos;
2004 return count;
2005}
2006
2007static const struct file_operations mem_debugfs_fops = {
2008 .owner = THIS_MODULE,
2009 .open = mem_open,
2010 .read = mem_read,
2011};
2012
2013static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2014 unsigned int idx, unsigned int size_mb)
2015{
2016 struct dentry *de;
2017
2018 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2019 (void *)adap + idx, &mem_debugfs_fops);
2020 if (de && de->d_inode)
2021 de->d_inode->i_size = size_mb << 20;
2022}
2023
2024static int __devinit setup_debugfs(struct adapter *adap)
2025{
2026 int i;
2027
2028 if (IS_ERR_OR_NULL(adap->debugfs_root))
2029 return -1;
2030
2031 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2032 if (i & EDRAM0_ENABLE)
2033 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2034 if (i & EDRAM1_ENABLE)
2035 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2036 if (i & EXT_MEM_ENABLE)
2037 add_debugfs_mem(adap, "mc", MEM_MC,
2038 EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2039 if (adap->l2t)
2040 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2041 &t4_l2t_fops);
2042 return 0;
2043}
2044
2045/*
2046 * upper-layer driver support
2047 */
2048
2049/*
2050 * Allocate an active-open TID and set it to the supplied value.
2051 */
2052int cxgb4_alloc_atid(struct tid_info *t, void *data)
2053{
2054 int atid = -1;
2055
2056 spin_lock_bh(&t->atid_lock);
2057 if (t->afree) {
2058 union aopen_entry *p = t->afree;
2059
2060 atid = p - t->atid_tab;
2061 t->afree = p->next;
2062 p->data = data;
2063 t->atids_in_use++;
2064 }
2065 spin_unlock_bh(&t->atid_lock);
2066 return atid;
2067}
2068EXPORT_SYMBOL(cxgb4_alloc_atid);
2069
2070/*
2071 * Release an active-open TID.
2072 */
2073void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2074{
2075 union aopen_entry *p = &t->atid_tab[atid];
2076
2077 spin_lock_bh(&t->atid_lock);
2078 p->next = t->afree;
2079 t->afree = p;
2080 t->atids_in_use--;
2081 spin_unlock_bh(&t->atid_lock);
2082}
2083EXPORT_SYMBOL(cxgb4_free_atid);
2084
2085/*
2086 * Allocate a server TID and set it to the supplied value.
2087 */
2088int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2089{
2090 int stid;
2091
2092 spin_lock_bh(&t->stid_lock);
2093 if (family == PF_INET) {
2094 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2095 if (stid < t->nstids)
2096 __set_bit(stid, t->stid_bmap);
2097 else
2098 stid = -1;
2099 } else {
2100 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2101 if (stid < 0)
2102 stid = -1;
2103 }
2104 if (stid >= 0) {
2105 t->stid_tab[stid].data = data;
2106 stid += t->stid_base;
2107 t->stids_in_use++;
2108 }
2109 spin_unlock_bh(&t->stid_lock);
2110 return stid;
2111}
2112EXPORT_SYMBOL(cxgb4_alloc_stid);
2113
2114/*
2115 * Release a server TID.
2116 */
2117void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2118{
2119 stid -= t->stid_base;
2120 spin_lock_bh(&t->stid_lock);
2121 if (family == PF_INET)
2122 __clear_bit(stid, t->stid_bmap);
2123 else
2124 bitmap_release_region(t->stid_bmap, stid, 2);
2125 t->stid_tab[stid].data = NULL;
2126 t->stids_in_use--;
2127 spin_unlock_bh(&t->stid_lock);
2128}
2129EXPORT_SYMBOL(cxgb4_free_stid);
2130
2131/*
2132 * Populate a TID_RELEASE WR. Caller must properly size the skb.
2133 */
2134static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2135 unsigned int tid)
2136{
2137 struct cpl_tid_release *req;
2138
2139 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2140 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2141 INIT_TP_WR(req, tid);
2142 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2143}
2144
2145/*
2146 * Queue a TID release request and if necessary schedule a work queue to
2147 * process it.
2148 */
2149void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2150 unsigned int tid)
2151{
2152 void **p = &t->tid_tab[tid];
2153 struct adapter *adap = container_of(t, struct adapter, tids);
2154
2155 spin_lock_bh(&adap->tid_release_lock);
2156 *p = adap->tid_release_head;
2157 /* Low 2 bits encode the Tx channel number */
2158 adap->tid_release_head = (void **)((uintptr_t)p | chan);
2159 if (!adap->tid_release_task_busy) {
2160 adap->tid_release_task_busy = true;
2161 schedule_work(&adap->tid_release_task);
2162 }
2163 spin_unlock_bh(&adap->tid_release_lock);
2164}
2165EXPORT_SYMBOL(cxgb4_queue_tid_release);
2166
2167/*
2168 * Process the list of pending TID release requests.
2169 */
2170static void process_tid_release_list(struct work_struct *work)
2171{
2172 struct sk_buff *skb;
2173 struct adapter *adap;
2174
2175 adap = container_of(work, struct adapter, tid_release_task);
2176
2177 spin_lock_bh(&adap->tid_release_lock);
2178 while (adap->tid_release_head) {
2179 void **p = adap->tid_release_head;
2180 unsigned int chan = (uintptr_t)p & 3;
2181 p = (void *)p - chan;
2182
2183 adap->tid_release_head = *p;
2184 *p = NULL;
2185 spin_unlock_bh(&adap->tid_release_lock);
2186
2187 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2188 GFP_KERNEL)))
2189 schedule_timeout_uninterruptible(1);
2190
2191 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2192 t4_ofld_send(adap, skb);
2193 spin_lock_bh(&adap->tid_release_lock);
2194 }
2195 adap->tid_release_task_busy = false;
2196 spin_unlock_bh(&adap->tid_release_lock);
2197}
2198
2199/*
2200 * Release a TID and inform HW. If we are unable to allocate the release
2201 * message we defer to a work queue.
2202 */
2203void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2204{
2205 void *old;
2206 struct sk_buff *skb;
2207 struct adapter *adap = container_of(t, struct adapter, tids);
2208
2209 old = t->tid_tab[tid];
2210 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2211 if (likely(skb)) {
2212 t->tid_tab[tid] = NULL;
2213 mk_tid_release(skb, chan, tid);
2214 t4_ofld_send(adap, skb);
2215 } else
2216 cxgb4_queue_tid_release(t, chan, tid);
2217 if (old)
2218 atomic_dec(&t->tids_in_use);
2219}
2220EXPORT_SYMBOL(cxgb4_remove_tid);
2221
2222/*
2223 * Allocate and initialize the TID tables. Returns 0 on success.
2224 */
2225static int tid_init(struct tid_info *t)
2226{
2227 size_t size;
2228 unsigned int natids = t->natids;
2229
2230 size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2231 t->nstids * sizeof(*t->stid_tab) +
2232 BITS_TO_LONGS(t->nstids) * sizeof(long);
2233 t->tid_tab = t4_alloc_mem(size);
2234 if (!t->tid_tab)
2235 return -ENOMEM;
2236
2237 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2238 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2239 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2240 spin_lock_init(&t->stid_lock);
2241 spin_lock_init(&t->atid_lock);
2242
2243 t->stids_in_use = 0;
2244 t->afree = NULL;
2245 t->atids_in_use = 0;
2246 atomic_set(&t->tids_in_use, 0);
2247
2248 /* Setup the free list for atid_tab and clear the stid bitmap. */
2249 if (natids) {
2250 while (--natids)
2251 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2252 t->afree = t->atid_tab;
2253 }
2254 bitmap_zero(t->stid_bmap, t->nstids);
2255 return 0;
2256}
2257
2258/**
2259 * cxgb4_create_server - create an IP server
2260 * @dev: the device
2261 * @stid: the server TID
2262 * @sip: local IP address to bind server to
2263 * @sport: the server's TCP port
2264 * @queue: queue to direct messages from this server to
2265 *
2266 * Create an IP server for the given port and address.
2267 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2268 */
2269int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2270 __be32 sip, __be16 sport, unsigned int queue)
2271{
2272 unsigned int chan;
2273 struct sk_buff *skb;
2274 struct adapter *adap;
2275 struct cpl_pass_open_req *req;
2276
2277 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2278 if (!skb)
2279 return -ENOMEM;
2280
2281 adap = netdev2adap(dev);
2282 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2283 INIT_TP_WR(req, 0);
2284 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2285 req->local_port = sport;
2286 req->peer_port = htons(0);
2287 req->local_ip = sip;
2288 req->peer_ip = htonl(0);
2289 chan = netdev2pinfo(adap->sge.ingr_map[queue]->netdev)->tx_chan;
2290 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2291 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2292 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2293 return t4_mgmt_tx(adap, skb);
2294}
2295EXPORT_SYMBOL(cxgb4_create_server);
2296
2297/**
2298 * cxgb4_create_server6 - create an IPv6 server
2299 * @dev: the device
2300 * @stid: the server TID
2301 * @sip: local IPv6 address to bind server to
2302 * @sport: the server's TCP port
2303 * @queue: queue to direct messages from this server to
2304 *
2305 * Create an IPv6 server for the given port and address.
2306 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2307 */
2308int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
2309 const struct in6_addr *sip, __be16 sport,
2310 unsigned int queue)
2311{
2312 unsigned int chan;
2313 struct sk_buff *skb;
2314 struct adapter *adap;
2315 struct cpl_pass_open_req6 *req;
2316
2317 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2318 if (!skb)
2319 return -ENOMEM;
2320
2321 adap = netdev2adap(dev);
2322 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
2323 INIT_TP_WR(req, 0);
2324 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
2325 req->local_port = sport;
2326 req->peer_port = htons(0);
2327 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
2328 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
2329 req->peer_ip_hi = cpu_to_be64(0);
2330 req->peer_ip_lo = cpu_to_be64(0);
2331 chan = netdev2pinfo(adap->sge.ingr_map[queue]->netdev)->tx_chan;
2332 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2333 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2334 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2335 return t4_mgmt_tx(adap, skb);
2336}
2337EXPORT_SYMBOL(cxgb4_create_server6);
2338
2339/**
2340 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2341 * @mtus: the HW MTU table
2342 * @mtu: the target MTU
2343 * @idx: index of selected entry in the MTU table
2344 *
2345 * Returns the index and the value in the HW MTU table that is closest to
2346 * but does not exceed @mtu, unless @mtu is smaller than any value in the
2347 * table, in which case that smallest available value is selected.
2348 */
2349unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2350 unsigned int *idx)
2351{
2352 unsigned int i = 0;
2353
2354 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2355 ++i;
2356 if (idx)
2357 *idx = i;
2358 return mtus[i];
2359}
2360EXPORT_SYMBOL(cxgb4_best_mtu);
2361
2362/**
2363 * cxgb4_port_chan - get the HW channel of a port
2364 * @dev: the net device for the port
2365 *
2366 * Return the HW Tx channel of the given port.
2367 */
2368unsigned int cxgb4_port_chan(const struct net_device *dev)
2369{
2370 return netdev2pinfo(dev)->tx_chan;
2371}
2372EXPORT_SYMBOL(cxgb4_port_chan);
2373
2374/**
2375 * cxgb4_port_viid - get the VI id of a port
2376 * @dev: the net device for the port
2377 *
2378 * Return the VI id of the given port.
2379 */
2380unsigned int cxgb4_port_viid(const struct net_device *dev)
2381{
2382 return netdev2pinfo(dev)->viid;
2383}
2384EXPORT_SYMBOL(cxgb4_port_viid);
2385
2386/**
2387 * cxgb4_port_idx - get the index of a port
2388 * @dev: the net device for the port
2389 *
2390 * Return the index of the given port.
2391 */
2392unsigned int cxgb4_port_idx(const struct net_device *dev)
2393{
2394 return netdev2pinfo(dev)->port_id;
2395}
2396EXPORT_SYMBOL(cxgb4_port_idx);
2397
2398/**
2399 * cxgb4_netdev_by_hwid - return the net device of a HW port
2400 * @pdev: identifies the adapter
2401 * @id: the HW port id
2402 *
2403 * Return the net device associated with the interface with the given HW
2404 * id.
2405 */
2406struct net_device *cxgb4_netdev_by_hwid(struct pci_dev *pdev, unsigned int id)
2407{
2408 const struct adapter *adap = pci_get_drvdata(pdev);
2409
2410 if (!adap || id >= NCHAN)
2411 return NULL;
2412 id = adap->chan_map[id];
2413 return id < MAX_NPORTS ? adap->port[id] : NULL;
2414}
2415EXPORT_SYMBOL(cxgb4_netdev_by_hwid);
2416
2417void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2418 struct tp_tcp_stats *v6)
2419{
2420 struct adapter *adap = pci_get_drvdata(pdev);
2421
2422 spin_lock(&adap->stats_lock);
2423 t4_tp_get_tcp_stats(adap, v4, v6);
2424 spin_unlock(&adap->stats_lock);
2425}
2426EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2427
2428void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2429 const unsigned int *pgsz_order)
2430{
2431 struct adapter *adap = netdev2adap(dev);
2432
2433 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2434 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2435 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2436 HPZ3(pgsz_order[3]));
2437}
2438EXPORT_SYMBOL(cxgb4_iscsi_init);
2439
2440static struct pci_driver cxgb4_driver;
2441
2442static void check_neigh_update(struct neighbour *neigh)
2443{
2444 const struct device *parent;
2445 const struct net_device *netdev = neigh->dev;
2446
2447 if (netdev->priv_flags & IFF_802_1Q_VLAN)
2448 netdev = vlan_dev_real_dev(netdev);
2449 parent = netdev->dev.parent;
2450 if (parent && parent->driver == &cxgb4_driver.driver)
2451 t4_l2t_update(dev_get_drvdata(parent), neigh);
2452}
2453
2454static int netevent_cb(struct notifier_block *nb, unsigned long event,
2455 void *data)
2456{
2457 switch (event) {
2458 case NETEVENT_NEIGH_UPDATE:
2459 check_neigh_update(data);
2460 break;
2461 case NETEVENT_PMTU_UPDATE:
2462 case NETEVENT_REDIRECT:
2463 default:
2464 break;
2465 }
2466 return 0;
2467}
2468
2469static bool netevent_registered;
2470static struct notifier_block cxgb4_netevent_nb = {
2471 .notifier_call = netevent_cb
2472};
2473
2474static void uld_attach(struct adapter *adap, unsigned int uld)
2475{
2476 void *handle;
2477 struct cxgb4_lld_info lli;
2478
2479 lli.pdev = adap->pdev;
2480 lli.l2t = adap->l2t;
2481 lli.tids = &adap->tids;
2482 lli.ports = adap->port;
2483 lli.vr = &adap->vres;
2484 lli.mtus = adap->params.mtus;
2485 if (uld == CXGB4_ULD_RDMA) {
2486 lli.rxq_ids = adap->sge.rdma_rxq;
2487 lli.nrxq = adap->sge.rdmaqs;
2488 } else if (uld == CXGB4_ULD_ISCSI) {
2489 lli.rxq_ids = adap->sge.ofld_rxq;
2490 lli.nrxq = adap->sge.ofldqsets;
2491 }
2492 lli.ntxq = adap->sge.ofldqsets;
2493 lli.nchan = adap->params.nports;
2494 lli.nports = adap->params.nports;
2495 lli.wr_cred = adap->params.ofldq_wr_cred;
2496 lli.adapter_type = adap->params.rev;
2497 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2498 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
2499 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF));
2500 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
2501 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF));
2502 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2503 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2504 lli.fw_vers = adap->params.fw_vers;
2505
2506 handle = ulds[uld].add(&lli);
2507 if (IS_ERR(handle)) {
2508 dev_warn(adap->pdev_dev,
2509 "could not attach to the %s driver, error %ld\n",
2510 uld_str[uld], PTR_ERR(handle));
2511 return;
2512 }
2513
2514 adap->uld_handle[uld] = handle;
2515
2516 if (!netevent_registered) {
2517 register_netevent_notifier(&cxgb4_netevent_nb);
2518 netevent_registered = true;
2519 }
e29f5dbc
DM		 2520
2521 if (adap->flags & FULL_INIT_DONE)
2522 ulds[uld].state_change(handle, CXGB4_STATE_UP);
b8ff05a9
DM		 2523}
2524
2525static void attach_ulds(struct adapter *adap)
2526{
2527 unsigned int i;
2528
2529 mutex_lock(&uld_mutex);
2530 list_add_tail(&adap->list_node, &adapter_list);
2531 for (i = 0; i < CXGB4_ULD_MAX; i++)
2532 if (ulds[i].add)
2533 uld_attach(adap, i);
2534 mutex_unlock(&uld_mutex);
2535}
2536
2537static void detach_ulds(struct adapter *adap)
2538{
2539 unsigned int i;
2540
2541 mutex_lock(&uld_mutex);
2542 list_del(&adap->list_node);
2543 for (i = 0; i < CXGB4_ULD_MAX; i++)
2544 if (adap->uld_handle[i]) {
2545 ulds[i].state_change(adap->uld_handle[i],
2546 CXGB4_STATE_DETACH);
2547 adap->uld_handle[i] = NULL;
2548 }
2549 if (netevent_registered && list_empty(&adapter_list)) {
2550 unregister_netevent_notifier(&cxgb4_netevent_nb);
2551 netevent_registered = false;
2552 }
2553 mutex_unlock(&uld_mutex);
2554}
2555
2556static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2557{
2558 unsigned int i;
2559
2560 mutex_lock(&uld_mutex);
2561 for (i = 0; i < CXGB4_ULD_MAX; i++)
2562 if (adap->uld_handle[i])
2563 ulds[i].state_change(adap->uld_handle[i], new_state);
2564 mutex_unlock(&uld_mutex);
2565}
2566
2567/**
2568 * cxgb4_register_uld - register an upper-layer driver
2569 * @type: the ULD type
2570 * @p: the ULD methods
2571 *
2572 * Registers an upper-layer driver with this driver and notifies the ULD
2573 * about any presently available devices that support its type. Returns
2574 * %-EBUSY if a ULD of the same type is already registered.
2575 */
2576int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2577{
2578 int ret = 0;
2579 struct adapter *adap;
2580
2581 if (type >= CXGB4_ULD_MAX)
2582 return -EINVAL;
2583 mutex_lock(&uld_mutex);
2584 if (ulds[type].add) {
2585 ret = -EBUSY;
2586 goto out;
2587 }
2588 ulds[type] = *p;
2589 list_for_each_entry(adap, &adapter_list, list_node)
2590 uld_attach(adap, type);
2591out: mutex_unlock(&uld_mutex);
2592 return ret;
2593}
2594EXPORT_SYMBOL(cxgb4_register_uld);
2595
2596/**
2597 * cxgb4_unregister_uld - unregister an upper-layer driver
2598 * @type: the ULD type
2599 *
2600 * Unregisters an existing upper-layer driver.
2601 */
2602int cxgb4_unregister_uld(enum cxgb4_uld type)
2603{
2604 struct adapter *adap;
2605
2606 if (type >= CXGB4_ULD_MAX)
2607 return -EINVAL;
2608 mutex_lock(&uld_mutex);
2609 list_for_each_entry(adap, &adapter_list, list_node)
2610 adap->uld_handle[type] = NULL;
2611 ulds[type].add = NULL;
2612 mutex_unlock(&uld_mutex);
2613 return 0;
2614}
2615EXPORT_SYMBOL(cxgb4_unregister_uld);
2616
2617/**
2618 * cxgb_up - enable the adapter
2619 * @adap: adapter being enabled
2620 *
2621 * Called when the first port is enabled, this function performs the
2622 * actions necessary to make an adapter operational, such as completing
2623 * the initialization of HW modules, and enabling interrupts.
2624 *
2625 * Must be called with the rtnl lock held.
2626 */
2627static int cxgb_up(struct adapter *adap)
2628{
aaefae9b 2629 int err;
b8ff05a9 2630
aaefae9b
DM		 2631 err = setup_sge_queues(adap);
2632 if (err)
2633 goto out;
2634 err = setup_rss(adap);
2635 if (err)
2636 goto freeq;
b8ff05a9
DM		 2637
2638 if (adap->flags & USING_MSIX) {
aaefae9b 2639 name_msix_vecs(adap);
b8ff05a9
DM		 2640 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2641 adap->msix_info[0].desc, adap);
2642 if (err)
2643 goto irq_err;
2644
2645 err = request_msix_queue_irqs(adap);
2646 if (err) {
2647 free_irq(adap->msix_info[0].vec, adap);
2648 goto irq_err;
2649 }
2650 } else {
2651 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2652 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2653 adap->name, adap);
2654 if (err)
2655 goto irq_err;
2656 }
2657 enable_rx(adap);
2658 t4_sge_start(adap);
2659 t4_intr_enable(adap);
aaefae9b 2660 adap->flags |= FULL_INIT_DONE;
b8ff05a9
DM		 2661 notify_ulds(adap, CXGB4_STATE_UP);
2662 out:
2663 return err;
2664 irq_err:
2665 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
aaefae9b
DM		 2666 freeq:
2667 t4_free_sge_resources(adap);
b8ff05a9
DM		 2668 goto out;
2669}
2670
2671static void cxgb_down(struct adapter *adapter)
2672{
2673 t4_intr_disable(adapter);
2674 cancel_work_sync(&adapter->tid_release_task);
2675 adapter->tid_release_task_busy = false;
204dc3c0 2676 adapter->tid_release_head = NULL;
b8ff05a9
DM		 2677
2678 if (adapter->flags & USING_MSIX) {
2679 free_msix_queue_irqs(adapter);
2680 free_irq(adapter->msix_info[0].vec, adapter);
2681 } else
2682 free_irq(adapter->pdev->irq, adapter);
2683 quiesce_rx(adapter);
aaefae9b
DM		 2684 t4_sge_stop(adapter);
2685 t4_free_sge_resources(adapter);
2686 adapter->flags &= ~FULL_INIT_DONE;
b8ff05a9
DM		 2687}
2688
2689/*
2690 * net_device operations
2691 */
2692static int cxgb_open(struct net_device *dev)
2693{
2694 int err;
2695 struct port_info *pi = netdev_priv(dev);
2696 struct adapter *adapter = pi->adapter;
2697
aaefae9b
DM		 2698 if (!(adapter->flags & FULL_INIT_DONE)) {
2699 err = cxgb_up(adapter);
2700 if (err < 0)
2701 return err;
2702 }
b8ff05a9
DM		 2703
2704 dev->real_num_tx_queues = pi->nqsets;
f68707b8
DM		 2705 err = link_start(dev);
2706 if (!err)
2707 netif_tx_start_all_queues(dev);
2708 return err;
b8ff05a9
DM		 2709}
2710
2711static int cxgb_close(struct net_device *dev)
2712{
b8ff05a9
DM		 2713 struct port_info *pi = netdev_priv(dev);
2714 struct adapter *adapter = pi->adapter;
2715
2716 netif_tx_stop_all_queues(dev);
2717 netif_carrier_off(dev);
aaefae9b 2718 return t4_enable_vi(adapter, 0, pi->viid, false, false);
b8ff05a9
DM		 2719}
2720
f5152c90
DM		 2721static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2722 struct rtnl_link_stats64 *ns)
b8ff05a9
DM		 2723{
2724 struct port_stats stats;
2725 struct port_info *p = netdev_priv(dev);
2726 struct adapter *adapter = p->adapter;
b8ff05a9
DM		 2727
2728 spin_lock(&adapter->stats_lock);
2729 t4_get_port_stats(adapter, p->tx_chan, &stats);
2730 spin_unlock(&adapter->stats_lock);
2731
2732 ns->tx_bytes = stats.tx_octets;
2733 ns->tx_packets = stats.tx_frames;
2734 ns->rx_bytes = stats.rx_octets;
2735 ns->rx_packets = stats.rx_frames;
2736 ns->multicast = stats.rx_mcast_frames;
2737
2738 /* detailed rx_errors */
2739 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2740 stats.rx_runt;
2741 ns->rx_over_errors = 0;
2742 ns->rx_crc_errors = stats.rx_fcs_err;
2743 ns->rx_frame_errors = stats.rx_symbol_err;
2744 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
2745 stats.rx_ovflow2 + stats.rx_ovflow3 +
2746 stats.rx_trunc0 + stats.rx_trunc1 +
2747 stats.rx_trunc2 + stats.rx_trunc3;
2748 ns->rx_missed_errors = 0;
2749
2750 /* detailed tx_errors */
2751 ns->tx_aborted_errors = 0;
2752 ns->tx_carrier_errors = 0;
2753 ns->tx_fifo_errors = 0;
2754 ns->tx_heartbeat_errors = 0;
2755 ns->tx_window_errors = 0;
2756
2757 ns->tx_errors = stats.tx_error_frames;
2758 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2759 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2760 return ns;
2761}
2762
2763static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2764{
2765 int ret = 0, prtad, devad;
2766 struct port_info *pi = netdev_priv(dev);
2767 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2768
2769 switch (cmd) {
2770 case SIOCGMIIPHY:
2771 if (pi->mdio_addr < 0)
2772 return -EOPNOTSUPP;
2773 data->phy_id = pi->mdio_addr;
2774 break;
2775 case SIOCGMIIREG:
2776 case SIOCSMIIREG:
2777 if (mdio_phy_id_is_c45(data->phy_id)) {
2778 prtad = mdio_phy_id_prtad(data->phy_id);
2779 devad = mdio_phy_id_devad(data->phy_id);
2780 } else if (data->phy_id < 32) {
2781 prtad = data->phy_id;
2782 devad = 0;
2783 data->reg_num &= 0x1f;
2784 } else
2785 return -EINVAL;
2786
2787 if (cmd == SIOCGMIIREG)
2788 ret = t4_mdio_rd(pi->adapter, 0, prtad, devad,
2789 data->reg_num, &data->val_out);
2790 else
2791 ret = t4_mdio_wr(pi->adapter, 0, prtad, devad,
2792 data->reg_num, data->val_in);
2793 break;
2794 default:
2795 return -EOPNOTSUPP;
2796 }
2797 return ret;
2798}
2799
2800static void cxgb_set_rxmode(struct net_device *dev)
2801{
2802 /* unfortunately we can't return errors to the stack */
2803 set_rxmode(dev, -1, false);
2804}
2805
2806static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2807{
2808 int ret;
2809 struct port_info *pi = netdev_priv(dev);
2810
2811 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
2812 return -EINVAL;
f8f5aafa 2813 ret = t4_set_rxmode(pi->adapter, 0, pi->viid, new_mtu, -1, -1, -1, -1,
b8ff05a9
DM		 2814 true);
2815 if (!ret)
2816 dev->mtu = new_mtu;
2817 return ret;
2818}
2819
2820static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2821{
2822 int ret;
2823 struct sockaddr *addr = p;
2824 struct port_info *pi = netdev_priv(dev);
2825
2826 if (!is_valid_ether_addr(addr->sa_data))
2827 return -EINVAL;
2828
2829 ret = t4_change_mac(pi->adapter, 0, pi->viid, pi->xact_addr_filt,
2830 addr->sa_data, true, true);
2831 if (ret < 0)
2832 return ret;
2833
2834 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2835 pi->xact_addr_filt = ret;
2836 return 0;
2837}
2838
2839static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2840{
2841 struct port_info *pi = netdev_priv(dev);
2842
2843 pi->vlan_grp = grp;
f8f5aafa
DM		 2844 t4_set_rxmode(pi->adapter, 0, pi->viid, -1, -1, -1, -1, grp != NULL,
2845 true);
b8ff05a9
DM		 2846}
2847
2848#ifdef CONFIG_NET_POLL_CONTROLLER
2849static void cxgb_netpoll(struct net_device *dev)
2850{
2851 struct port_info *pi = netdev_priv(dev);
2852 struct adapter *adap = pi->adapter;
2853
2854 if (adap->flags & USING_MSIX) {
2855 int i;
2856 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2857
2858 for (i = pi->nqsets; i; i--, rx++)
2859 t4_sge_intr_msix(0, &rx->rspq);
2860 } else
2861 t4_intr_handler(adap)(0, adap);
2862}
2863#endif
2864
2865static const struct net_device_ops cxgb4_netdev_ops = {
2866 .ndo_open = cxgb_open,
2867 .ndo_stop = cxgb_close,
2868 .ndo_start_xmit = t4_eth_xmit,
9be793bf 2869 .ndo_get_stats64 = cxgb_get_stats,
b8ff05a9
DM		 2870 .ndo_set_rx_mode = cxgb_set_rxmode,
2871 .ndo_set_mac_address = cxgb_set_mac_addr,
2872 .ndo_validate_addr = eth_validate_addr,
2873 .ndo_do_ioctl = cxgb_ioctl,
2874 .ndo_change_mtu = cxgb_change_mtu,
2875 .ndo_vlan_rx_register = vlan_rx_register,
2876#ifdef CONFIG_NET_POLL_CONTROLLER
2877 .ndo_poll_controller = cxgb_netpoll,
2878#endif
2879};
2880
2881void t4_fatal_err(struct adapter *adap)
2882{
2883 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
2884 t4_intr_disable(adap);
2885 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2886}
2887
2888static void setup_memwin(struct adapter *adap)
2889{
2890 u32 bar0;
2891
2892 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
2893 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
2894 (bar0 + MEMWIN0_BASE) | BIR(0) |
2895 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
2896 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
2897 (bar0 + MEMWIN1_BASE) | BIR(0) |
2898 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
2899 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
2900 (bar0 + MEMWIN2_BASE) | BIR(0) |
2901 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
1ae970e0
DM		 2902 if (adap->vres.ocq.size) {
2903 unsigned int start, sz_kb;
2904
2905 start = pci_resource_start(adap->pdev, 2) +
2906 OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
2907 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
2908 t4_write_reg(adap,
2909 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
2910 start | BIR(1) | WINDOW(ilog2(sz_kb)));
2911 t4_write_reg(adap,
2912 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
2913 adap->vres.ocq.start);
2914 t4_read_reg(adap,
2915 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
2916 }
b8ff05a9
DM		 2917}
2918
02b5fb8e
DM		 2919static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
2920{
2921 u32 v;
2922 int ret;
2923
2924 /* get device capabilities */
2925 memset(c, 0, sizeof(*c));
2926 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2927 FW_CMD_REQUEST | FW_CMD_READ);
2928 c->retval_len16 = htonl(FW_LEN16(*c));
2929 ret = t4_wr_mbox(adap, 0, c, sizeof(*c), c);
2930 if (ret < 0)
2931 return ret;
2932
2933 /* select capabilities we'll be using */
2934 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
2935 if (!vf_acls)
2936 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
2937 else
2938 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
2939 } else if (vf_acls) {
2940 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
2941 return ret;
2942 }
2943 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2944 FW_CMD_REQUEST | FW_CMD_WRITE);
2945 ret = t4_wr_mbox(adap, 0, c, sizeof(*c), NULL);
2946 if (ret < 0)
2947 return ret;
2948
2949 ret = t4_config_glbl_rss(adap, 0,
2950 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
2951 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
2952 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
2953 if (ret < 0)
2954 return ret;
2955
20c0da65
DM		 2956 ret = t4_cfg_pfvf(adap, 0, 0, 0, MAX_EGRQ, 64, MAX_INGQ, 0, 0, 4,
2957 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
02b5fb8e
DM		 2958 if (ret < 0)
2959 return ret;
2960
2961 t4_sge_init(adap);
2962
2963 /* get basic stuff going */
2964 ret = t4_early_init(adap, 0);
2965 if (ret < 0)
2966 return ret;
2967
2968 /* tweak some settings */
2969 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
2970 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
2971 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
2972 v = t4_read_reg(adap, TP_PIO_DATA);
2973 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
02b5fb8e
DM		 2974 return 0;
2975}
2976
b8ff05a9
DM		 2977/*
2978 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
2979 */
2980#define MAX_ATIDS 8192U
2981
2982/*
2983 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
2984 */
2985static int adap_init0(struct adapter *adap)
2986{
2987 int ret;
2988 u32 v, port_vec;
2989 enum dev_state state;
2990 u32 params[7], val[7];
2991 struct fw_caps_config_cmd c;
2992
2993 ret = t4_check_fw_version(adap);
2994 if (ret == -EINVAL || ret > 0) {
2995 if (upgrade_fw(adap) >= 0) /* recache FW version */
2996 ret = t4_check_fw_version(adap);
2997 }
2998 if (ret < 0)
2999 return ret;
3000
3001 /* contact FW, request master */
3002 ret = t4_fw_hello(adap, 0, 0, MASTER_MUST, &state);
3003 if (ret < 0) {
3004 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
3005 ret);
3006 return ret;
3007 }
3008
3009 /* reset device */
3010 ret = t4_fw_reset(adap, 0, PIORSTMODE | PIORST);
3011 if (ret < 0)
3012 goto bye;
3013
b8ff05a9
DM		 3014 for (v = 0; v < SGE_NTIMERS - 1; v++)
3015 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
3016 adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
3017 adap->sge.counter_val[0] = 1;
3018 for (v = 1; v < SGE_NCOUNTERS; v++)
3019 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
3020 THRESHOLD_3_MASK);
b8ff05a9
DM		 3021#define FW_PARAM_DEV(param) \
3022 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3023 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3024
a0881cab
DM		 3025 params[0] = FW_PARAM_DEV(CCLK);
3026 ret = t4_query_params(adap, 0, 0, 0, 1, params, val);
3027 if (ret < 0)
3028 goto bye;
3029 adap->params.vpd.cclk = val[0];
3030
3031 ret = adap_init1(adap, &c);
3032 if (ret < 0)
3033 goto bye;
3034
b8ff05a9
DM		 3035#define FW_PARAM_PFVF(param) \
3036 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3037 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
3038
3039 params[0] = FW_PARAM_DEV(PORTVEC);
3040 params[1] = FW_PARAM_PFVF(L2T_START);
3041 params[2] = FW_PARAM_PFVF(L2T_END);
3042 params[3] = FW_PARAM_PFVF(FILTER_START);
3043 params[4] = FW_PARAM_PFVF(FILTER_END);
3044 ret = t4_query_params(adap, 0, 0, 0, 5, params, val);
3045 if (ret < 0)
3046 goto bye;
3047 port_vec = val[0];
3048 adap->tids.ftid_base = val[3];
3049 adap->tids.nftids = val[4] - val[3] + 1;
3050
3051 if (c.ofldcaps) {
3052 /* query offload-related parameters */
3053 params[0] = FW_PARAM_DEV(NTID);
3054 params[1] = FW_PARAM_PFVF(SERVER_START);
3055 params[2] = FW_PARAM_PFVF(SERVER_END);
3056 params[3] = FW_PARAM_PFVF(TDDP_START);
3057 params[4] = FW_PARAM_PFVF(TDDP_END);
3058 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3059 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
3060 if (ret < 0)
3061 goto bye;
3062 adap->tids.ntids = val[0];
3063 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3064 adap->tids.stid_base = val[1];
3065 adap->tids.nstids = val[2] - val[1] + 1;
3066 adap->vres.ddp.start = val[3];
3067 adap->vres.ddp.size = val[4] - val[3] + 1;
3068 adap->params.ofldq_wr_cred = val[5];
3069 adap->params.offload = 1;
3070 }
3071 if (c.rdmacaps) {
3072 params[0] = FW_PARAM_PFVF(STAG_START);
3073 params[1] = FW_PARAM_PFVF(STAG_END);
3074 params[2] = FW_PARAM_PFVF(RQ_START);
3075 params[3] = FW_PARAM_PFVF(RQ_END);
3076 params[4] = FW_PARAM_PFVF(PBL_START);
3077 params[5] = FW_PARAM_PFVF(PBL_END);
3078 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
3079 if (ret < 0)
3080 goto bye;
3081 adap->vres.stag.start = val[0];
3082 adap->vres.stag.size = val[1] - val[0] + 1;
3083 adap->vres.rq.start = val[2];
3084 adap->vres.rq.size = val[3] - val[2] + 1;
3085 adap->vres.pbl.start = val[4];
3086 adap->vres.pbl.size = val[5] - val[4] + 1;
a0881cab
DM		 3087
3088 params[0] = FW_PARAM_PFVF(SQRQ_START);
3089 params[1] = FW_PARAM_PFVF(SQRQ_END);
3090 params[2] = FW_PARAM_PFVF(CQ_START);
3091 params[3] = FW_PARAM_PFVF(CQ_END);
1ae970e0
DM		 3092 params[4] = FW_PARAM_PFVF(OCQ_START);
3093 params[5] = FW_PARAM_PFVF(OCQ_END);
3094 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
a0881cab
DM		 3095 if (ret < 0)
3096 goto bye;
3097 adap->vres.qp.start = val[0];
3098 adap->vres.qp.size = val[1] - val[0] + 1;
3099 adap->vres.cq.start = val[2];
3100 adap->vres.cq.size = val[3] - val[2] + 1;
1ae970e0
DM		 3101 adap->vres.ocq.start = val[4];
3102 adap->vres.ocq.size = val[5] - val[4] + 1;
b8ff05a9
DM		 3103 }
3104 if (c.iscsicaps) {
3105 params[0] = FW_PARAM_PFVF(ISCSI_START);
3106 params[1] = FW_PARAM_PFVF(ISCSI_END);
3107 ret = t4_query_params(adap, 0, 0, 0, 2, params, val);
3108 if (ret < 0)
3109 goto bye;
3110 adap->vres.iscsi.start = val[0];
3111 adap->vres.iscsi.size = val[1] - val[0] + 1;
3112 }
3113#undef FW_PARAM_PFVF
3114#undef FW_PARAM_DEV
3115
3116 adap->params.nports = hweight32(port_vec);
3117 adap->params.portvec = port_vec;
3118 adap->flags |= FW_OK;
3119
3120 /* These are finalized by FW initialization, load their values now */
3121 v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
3122 adap->params.tp.tre = TIMERRESOLUTION_GET(v);
3123 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3124 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3125 adap->params.b_wnd);
7ee9ff94
CL		 3126
3127#ifdef CONFIG_PCI_IOV
3128 /*
3129 * Provision resource limits for Virtual Functions. We currently
3130 * grant them all the same static resource limits except for the Port
3131 * Access Rights Mask which we're assigning based on the PF. All of
3132 * the static provisioning stuff for both the PF and VF really needs
3133 * to be managed in a persistent manner for each device which the
3134 * firmware controls.
3135 */
3136 {
3137 int pf, vf;
3138
3139 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3140 if (num_vf[pf] <= 0)
3141 continue;
3142
3143 /* VF numbering starts at 1! */
3144 for (vf = 1; vf <= num_vf[pf]; vf++) {
3145 ret = t4_cfg_pfvf(adap, 0, pf, vf,
3146 VFRES_NEQ, VFRES_NETHCTRL,
3147 VFRES_NIQFLINT, VFRES_NIQ,
3148 VFRES_TC, VFRES_NVI,
3149 FW_PFVF_CMD_CMASK_MASK,
3150 pfvfres_pmask(adap, pf, vf),
3151 VFRES_NEXACTF,
3152 VFRES_R_CAPS, VFRES_WX_CAPS);
3153 if (ret < 0)
3154 dev_warn(adap->pdev_dev, "failed to "
3155 "provision pf/vf=%d/%d; "
3156 "err=%d\n", pf, vf, ret);
3157 }
3158 }
3159 }
3160#endif
3161
1ae970e0 3162 setup_memwin(adap);
b8ff05a9
DM		 3163 return 0;
3164
3165 /*
3166 * If a command timed out or failed with EIO FW does not operate within
3167 * its spec or something catastrophic happened to HW/FW, stop issuing
3168 * commands.
3169 */
3170bye: if (ret != -ETIMEDOUT && ret != -EIO)
3171 t4_fw_bye(adap, 0);
3172 return ret;
3173}
3174
204dc3c0
DM		 3175/* EEH callbacks */
3176
3177static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3178 pci_channel_state_t state)
3179{
3180 int i;
3181 struct adapter *adap = pci_get_drvdata(pdev);
3182
3183 if (!adap)
3184 goto out;
3185
3186 rtnl_lock();
3187 adap->flags &= ~FW_OK;
3188 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3189 for_each_port(adap, i) {
3190 struct net_device *dev = adap->port[i];
3191
3192 netif_device_detach(dev);
3193 netif_carrier_off(dev);
3194 }
3195 if (adap->flags & FULL_INIT_DONE)
3196 cxgb_down(adap);
3197 rtnl_unlock();
3198 pci_disable_device(pdev);
3199out: return state == pci_channel_io_perm_failure ?
3200 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3201}
3202
3203static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3204{
3205 int i, ret;
3206 struct fw_caps_config_cmd c;
3207 struct adapter *adap = pci_get_drvdata(pdev);
3208
3209 if (!adap) {
3210 pci_restore_state(pdev);
3211 pci_save_state(pdev);
3212 return PCI_ERS_RESULT_RECOVERED;
3213 }
3214
3215 if (pci_enable_device(pdev)) {
3216 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3217 return PCI_ERS_RESULT_DISCONNECT;
3218 }
3219
3220 pci_set_master(pdev);
3221 pci_restore_state(pdev);
3222 pci_save_state(pdev);
3223 pci_cleanup_aer_uncorrect_error_status(pdev);
3224
3225 if (t4_wait_dev_ready(adap) < 0)
3226 return PCI_ERS_RESULT_DISCONNECT;
3227 if (t4_fw_hello(adap, 0, 0, MASTER_MUST, NULL))
3228 return PCI_ERS_RESULT_DISCONNECT;
3229 adap->flags |= FW_OK;
3230 if (adap_init1(adap, &c))
3231 return PCI_ERS_RESULT_DISCONNECT;
3232
3233 for_each_port(adap, i) {
3234 struct port_info *p = adap2pinfo(adap, i);
3235
3236 ret = t4_alloc_vi(adap, 0, p->tx_chan, 0, 0, 1, NULL, NULL);
3237 if (ret < 0)
3238 return PCI_ERS_RESULT_DISCONNECT;
3239 p->viid = ret;
3240 p->xact_addr_filt = -1;
3241 }
3242
3243 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3244 adap->params.b_wnd);
1ae970e0 3245 setup_memwin(adap);
204dc3c0
DM		 3246 if (cxgb_up(adap))
3247 return PCI_ERS_RESULT_DISCONNECT;
3248 return PCI_ERS_RESULT_RECOVERED;
3249}
3250
3251static void eeh_resume(struct pci_dev *pdev)
3252{
3253 int i;
3254 struct adapter *adap = pci_get_drvdata(pdev);
3255
3256 if (!adap)
3257 return;
3258
3259 rtnl_lock();
3260 for_each_port(adap, i) {
3261 struct net_device *dev = adap->port[i];
3262
3263 if (netif_running(dev)) {
3264 link_start(dev);
3265 cxgb_set_rxmode(dev);
3266 }
3267 netif_device_attach(dev);
3268 }
3269 rtnl_unlock();
3270}
3271
3272static struct pci_error_handlers cxgb4_eeh = {
3273 .error_detected = eeh_err_detected,
3274 .slot_reset = eeh_slot_reset,
3275 .resume = eeh_resume,
3276};
3277
b8ff05a9
DM		 3278static inline bool is_10g_port(const struct link_config *lc)
3279{
3280 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3281}
3282
3283static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3284 unsigned int size, unsigned int iqe_size)
3285{
3286 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3287 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3288 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3289 q->iqe_len = iqe_size;
3290 q->size = size;
3291}
3292
3293/*
3294 * Perform default configuration of DMA queues depending on the number and type
3295 * of ports we found and the number of available CPUs. Most settings can be
3296 * modified by the admin prior to actual use.
3297 */
3298static void __devinit cfg_queues(struct adapter *adap)
3299{
3300 struct sge *s = &adap->sge;
3301 int i, q10g = 0, n10g = 0, qidx = 0;
3302
3303 for_each_port(adap, i)
3304 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3305
3306 /*
3307 * We default to 1 queue per non-10G port and up to # of cores queues
3308 * per 10G port.
3309 */
3310 if (n10g)
3311 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
3312 if (q10g > num_online_cpus())
3313 q10g = num_online_cpus();
3314
3315 for_each_port(adap, i) {
3316 struct port_info *pi = adap2pinfo(adap, i);
3317
3318 pi->first_qset = qidx;
3319 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3320 qidx += pi->nqsets;
3321 }
3322
3323 s->ethqsets = qidx;
3324 s->max_ethqsets = qidx; /* MSI-X may lower it later */
3325
3326 if (is_offload(adap)) {
3327 /*
3328 * For offload we use 1 queue/channel if all ports are up to 1G,
3329 * otherwise we divide all available queues amongst the channels
3330 * capped by the number of available cores.
3331 */
3332 if (n10g) {
3333 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3334 num_online_cpus());
3335 s->ofldqsets = roundup(i, adap->params.nports);
3336 } else
3337 s->ofldqsets = adap->params.nports;
3338 /* For RDMA one Rx queue per channel suffices */
3339 s->rdmaqs = adap->params.nports;
3340 }
3341
3342 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3343 struct sge_eth_rxq *r = &s->ethrxq[i];
3344
3345 init_rspq(&r->rspq, 0, 0, 1024, 64);
3346 r->fl.size = 72;
3347 }
3348
3349 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3350 s->ethtxq[i].q.size = 1024;
3351
3352 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3353 s->ctrlq[i].q.size = 512;
3354
3355 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3356 s->ofldtxq[i].q.size = 1024;
3357
3358 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3359 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3360
3361 init_rspq(&r->rspq, 0, 0, 1024, 64);
3362 r->rspq.uld = CXGB4_ULD_ISCSI;
3363 r->fl.size = 72;
3364 }
3365
3366 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3367 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3368
3369 init_rspq(&r->rspq, 0, 0, 511, 64);
3370 r->rspq.uld = CXGB4_ULD_RDMA;
3371 r->fl.size = 72;
3372 }
3373
3374 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3375 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3376}
3377
3378/*
3379 * Reduce the number of Ethernet queues across all ports to at most n.
3380 * n provides at least one queue per port.
3381 */
3382static void __devinit reduce_ethqs(struct adapter *adap, int n)
3383{
3384 int i;
3385 struct port_info *pi;
3386
3387 while (n < adap->sge.ethqsets)
3388 for_each_port(adap, i) {
3389 pi = adap2pinfo(adap, i);
3390 if (pi->nqsets > 1) {
3391 pi->nqsets--;
3392 adap->sge.ethqsets--;
3393 if (adap->sge.ethqsets <= n)
3394 break;
3395 }
3396 }
3397
3398 n = 0;
3399 for_each_port(adap, i) {
3400 pi = adap2pinfo(adap, i);
3401 pi->first_qset = n;
3402 n += pi->nqsets;
3403 }
3404}
3405
3406/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3407#define EXTRA_VECS 2
3408
3409static int __devinit enable_msix(struct adapter *adap)
3410{
3411 int ofld_need = 0;
3412 int i, err, want, need;
3413 struct sge *s = &adap->sge;
3414 unsigned int nchan = adap->params.nports;
3415 struct msix_entry entries[MAX_INGQ + 1];
3416
3417 for (i = 0; i < ARRAY_SIZE(entries); ++i)
3418 entries[i].entry = i;
3419
3420 want = s->max_ethqsets + EXTRA_VECS;
3421 if (is_offload(adap)) {
3422 want += s->rdmaqs + s->ofldqsets;
3423 /* need nchan for each possible ULD */
3424 ofld_need = 2 * nchan;
3425 }
3426 need = adap->params.nports + EXTRA_VECS + ofld_need;
3427
3428 while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3429 want = err;
3430
3431 if (!err) {
3432 /*
3433 * Distribute available vectors to the various queue groups.
3434 * Every group gets its minimum requirement and NIC gets top
3435 * priority for leftovers.
3436 */
3437 i = want - EXTRA_VECS - ofld_need;
3438 if (i < s->max_ethqsets) {
3439 s->max_ethqsets = i;
3440 if (i < s->ethqsets)
3441 reduce_ethqs(adap, i);
3442 }
3443 if (is_offload(adap)) {
3444 i = want - EXTRA_VECS - s->max_ethqsets;
3445 i -= ofld_need - nchan;
3446 s->ofldqsets = (i / nchan) * nchan; /* round down */
3447 }
3448 for (i = 0; i < want; ++i)
3449 adap->msix_info[i].vec = entries[i].vector;
3450 } else if (err > 0)
3451 dev_info(adap->pdev_dev,
3452 "only %d MSI-X vectors left, not using MSI-X\n", err);
3453 return err;
3454}
3455
3456#undef EXTRA_VECS
3457
671b0060
DM		 3458static int __devinit init_rss(struct adapter *adap)
3459{
3460 unsigned int i, j;
3461
3462 for_each_port(adap, i) {
3463 struct port_info *pi = adap2pinfo(adap, i);
3464
3465 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
3466 if (!pi->rss)
3467 return -ENOMEM;
3468 for (j = 0; j < pi->rss_size; j++)
3469 pi->rss[j] = j % pi->nqsets;
3470 }
3471 return 0;
3472}
3473
b8ff05a9
DM		 3474static void __devinit print_port_info(struct adapter *adap)
3475{
3476 static const char *base[] = {
a0881cab
DM		 3477 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
3478 "KX", "KR", "KR SFP+", "KR FEC"
b8ff05a9
DM		 3479 };
3480
3481 int i;
3482 char buf[80];
f1a051b9
DM		 3483 const char *spd = "";
3484
3485 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
3486 spd = " 2.5 GT/s";
3487 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
3488 spd = " 5 GT/s";
b8ff05a9
DM		 3489
3490 for_each_port(adap, i) {
3491 struct net_device *dev = adap->port[i];
3492 const struct port_info *pi = netdev_priv(dev);
3493 char *bufp = buf;
3494
3495 if (!test_bit(i, &adap->registered_device_map))
3496 continue;
3497
3498 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3499 bufp += sprintf(bufp, "100/");
3500 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3501 bufp += sprintf(bufp, "1000/");
3502 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3503 bufp += sprintf(bufp, "10G/");
3504 if (bufp != buf)
3505 --bufp;
3506 sprintf(bufp, "BASE-%s", base[pi->port_type]);
3507
f1a051b9 3508 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
b8ff05a9
DM		 3509 adap->params.vpd.id, adap->params.rev,
3510 buf, is_offload(adap) ? "R" : "",
f1a051b9 3511 adap->params.pci.width, spd,
b8ff05a9
DM		 3512 (adap->flags & USING_MSIX) ? " MSI-X" :
3513 (adap->flags & USING_MSI) ? " MSI" : "");
3514 if (adap->name == dev->name)
3515 netdev_info(dev, "S/N: %s, E/C: %s\n",
3516 adap->params.vpd.sn, adap->params.vpd.ec);
3517 }
3518}
3519
06546391
DM		 3520/*
3521 * Free the following resources:
3522 * - memory used for tables
3523 * - MSI/MSI-X
3524 * - net devices
3525 * - resources FW is holding for us
3526 */
3527static void free_some_resources(struct adapter *adapter)
3528{
3529 unsigned int i;
3530
3531 t4_free_mem(adapter->l2t);
3532 t4_free_mem(adapter->tids.tid_tab);
3533 disable_msi(adapter);
3534
3535 for_each_port(adapter, i)
671b0060
DM		 3536 if (adapter->port[i]) {
3537 kfree(adap2pinfo(adapter, i)->rss);
06546391 3538 free_netdev(adapter->port[i]);
671b0060 3539 }
06546391
DM		 3540 if (adapter->flags & FW_OK)
3541 t4_fw_bye(adapter, 0);
3542}
3543
35d35682 3544#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
b8ff05a9
DM		 3545 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3546
3547static int __devinit init_one(struct pci_dev *pdev,
3548 const struct pci_device_id *ent)
3549{
3550 int func, i, err;
3551 struct port_info *pi;
3552 unsigned int highdma = 0;
3553 struct adapter *adapter = NULL;
3554
3555 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3556
3557 err = pci_request_regions(pdev, KBUILD_MODNAME);
3558 if (err) {
3559 /* Just info, some other driver may have claimed the device. */
3560 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3561 return err;
3562 }
3563
3564 /* We control everything through PF 0 */
3565 func = PCI_FUNC(pdev->devfn);
204dc3c0
DM		 3566 if (func > 0) {
3567 pci_save_state(pdev); /* to restore SR-IOV later */
b8ff05a9 3568 goto sriov;
204dc3c0 3569 }
b8ff05a9
DM		 3570
3571 err = pci_enable_device(pdev);
3572 if (err) {
3573 dev_err(&pdev->dev, "cannot enable PCI device\n");
3574 goto out_release_regions;
3575 }
3576
3577 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3578 highdma = NETIF_F_HIGHDMA;
3579 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3580 if (err) {
3581 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3582 "coherent allocations\n");
3583 goto out_disable_device;
3584 }
3585 } else {
3586 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3587 if (err) {
3588 dev_err(&pdev->dev, "no usable DMA configuration\n");
3589 goto out_disable_device;
3590 }
3591 }
3592
3593 pci_enable_pcie_error_reporting(pdev);
3594 pci_set_master(pdev);
3595 pci_save_state(pdev);
3596
3597 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3598 if (!adapter) {
3599 err = -ENOMEM;
3600 goto out_disable_device;
3601 }
3602
3603 adapter->regs = pci_ioremap_bar(pdev, 0);
3604 if (!adapter->regs) {
3605 dev_err(&pdev->dev, "cannot map device registers\n");
3606 err = -ENOMEM;
3607 goto out_free_adapter;
3608 }
3609
3610 adapter->pdev = pdev;
3611 adapter->pdev_dev = &pdev->dev;
3612 adapter->name = pci_name(pdev);
3613 adapter->msg_enable = dflt_msg_enable;
3614 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3615
3616 spin_lock_init(&adapter->stats_lock);
3617 spin_lock_init(&adapter->tid_release_lock);
3618
3619 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3620
3621 err = t4_prep_adapter(adapter);
3622 if (err)
3623 goto out_unmap_bar;
3624 err = adap_init0(adapter);
3625 if (err)
3626 goto out_unmap_bar;
3627
3628 for_each_port(adapter, i) {
3629 struct net_device *netdev;
3630
3631 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3632 MAX_ETH_QSETS);
3633 if (!netdev) {
3634 err = -ENOMEM;
3635 goto out_free_dev;
3636 }
3637
3638 SET_NETDEV_DEV(netdev, &pdev->dev);
3639
3640 adapter->port[i] = netdev;
3641 pi = netdev_priv(netdev);
3642 pi->adapter = adapter;
3643 pi->xact_addr_filt = -1;
3644 pi->rx_offload = RX_CSO;
3645 pi->port_id = i;
3646 netif_carrier_off(netdev);
3647 netif_tx_stop_all_queues(netdev);
3648 netdev->irq = pdev->irq;
3649
35d35682 3650 netdev->features |= NETIF_F_SG | TSO_FLAGS;
b8ff05a9 3651 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
87b6cf51 3652 netdev->features |= NETIF_F_GRO | NETIF_F_RXHASH | highdma;
b8ff05a9
DM		 3653 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3654 netdev->vlan_features = netdev->features & VLAN_FEAT;
3655
3656 netdev->netdev_ops = &cxgb4_netdev_ops;
3657 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3658 }
3659
3660 pci_set_drvdata(pdev, adapter);
3661
3662 if (adapter->flags & FW_OK) {
3663 err = t4_port_init(adapter, 0, 0, 0);
3664 if (err)
3665 goto out_free_dev;
3666 }
3667
3668 /*
3669 * Configure queues and allocate tables now, they can be needed as
3670 * soon as the first register_netdev completes.
3671 */
3672 cfg_queues(adapter);
3673
3674 adapter->l2t = t4_init_l2t();
3675 if (!adapter->l2t) {
3676 /* We tolerate a lack of L2T, giving up some functionality */
3677 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3678 adapter->params.offload = 0;
3679 }
3680
3681 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3682 dev_warn(&pdev->dev, "could not allocate TID table, "
3683 "continuing\n");
3684 adapter->params.offload = 0;
3685 }
3686
f7cabcdd
DM		 3687 /* See what interrupts we'll be using */
3688 if (msi > 1 && enable_msix(adapter) == 0)
3689 adapter->flags |= USING_MSIX;
3690 else if (msi > 0 && pci_enable_msi(pdev) == 0)
3691 adapter->flags |= USING_MSI;
3692
671b0060
DM		 3693 err = init_rss(adapter);
3694 if (err)
3695 goto out_free_dev;
3696
b8ff05a9
DM		 3697 /*
3698 * The card is now ready to go. If any errors occur during device
3699 * registration we do not fail the whole card but rather proceed only
3700 * with the ports we manage to register successfully. However we must
3701 * register at least one net device.
3702 */
3703 for_each_port(adapter, i) {
3704 err = register_netdev(adapter->port[i]);
3705 if (err)
3706 dev_warn(&pdev->dev,
3707 "cannot register net device %s, skipping\n",
3708 adapter->port[i]->name);
3709 else {
3710 /*
3711 * Change the name we use for messages to the name of
3712 * the first successfully registered interface.
3713 */
3714 if (!adapter->registered_device_map)
3715 adapter->name = adapter->port[i]->name;
3716
3717 __set_bit(i, &adapter->registered_device_map);
3718 adapter->chan_map[adap2pinfo(adapter, i)->tx_chan] = i;
3719 }
3720 }
3721 if (!adapter->registered_device_map) {
3722 dev_err(&pdev->dev, "could not register any net devices\n");
3723 goto out_free_dev;
3724 }
3725
3726 if (cxgb4_debugfs_root) {
3727 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3728 cxgb4_debugfs_root);
3729 setup_debugfs(adapter);
3730 }
3731
b8ff05a9
DM		 3732 if (is_offload(adapter))
3733 attach_ulds(adapter);
3734
3735 print_port_info(adapter);
3736
3737sriov:
3738#ifdef CONFIG_PCI_IOV
3739 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3740 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3741 dev_info(&pdev->dev,
3742 "instantiated %u virtual functions\n",
3743 num_vf[func]);
3744#endif
3745 return 0;
3746
3747 out_free_dev:
06546391 3748 free_some_resources(adapter);
b8ff05a9
DM		 3749 out_unmap_bar:
3750 iounmap(adapter->regs);
3751 out_free_adapter:
3752 kfree(adapter);
3753 out_disable_device:
3754 pci_disable_pcie_error_reporting(pdev);
3755 pci_disable_device(pdev);
3756 out_release_regions:
3757 pci_release_regions(pdev);
3758 pci_set_drvdata(pdev, NULL);
3759 return err;
3760}
3761
3762static void __devexit remove_one(struct pci_dev *pdev)
3763{
3764 struct adapter *adapter = pci_get_drvdata(pdev);
3765
3766 pci_disable_sriov(pdev);
3767
3768 if (adapter) {
3769 int i;
3770
3771 if (is_offload(adapter))
3772 detach_ulds(adapter);
3773
3774 for_each_port(adapter, i)
3775 if (test_bit(i, &adapter->registered_device_map))
3776 unregister_netdev(adapter->port[i]);
3777
3778 if (adapter->debugfs_root)
3779 debugfs_remove_recursive(adapter->debugfs_root);
3780
aaefae9b
DM		 3781 if (adapter->flags & FULL_INIT_DONE)
3782 cxgb_down(adapter);
b8ff05a9 3783
06546391 3784 free_some_resources(adapter);
b8ff05a9
DM		 3785 iounmap(adapter->regs);
3786 kfree(adapter);
3787 pci_disable_pcie_error_reporting(pdev);
3788 pci_disable_device(pdev);
3789 pci_release_regions(pdev);
3790 pci_set_drvdata(pdev, NULL);
3791 } else if (PCI_FUNC(pdev->devfn) > 0)
3792 pci_release_regions(pdev);
3793}
3794
3795static struct pci_driver cxgb4_driver = {
3796 .name = KBUILD_MODNAME,
3797 .id_table = cxgb4_pci_tbl,
3798 .probe = init_one,
3799 .remove = __devexit_p(remove_one),
204dc3c0 3800 .err_handler = &cxgb4_eeh,
b8ff05a9
DM		 3801};
3802
3803static int __init cxgb4_init_module(void)
3804{
3805 int ret;
3806
3807 /* Debugfs support is optional, just warn if this fails */
3808 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3809 if (!cxgb4_debugfs_root)
3810 pr_warning("could not create debugfs entry, continuing\n");
3811
3812 ret = pci_register_driver(&cxgb4_driver);
3813 if (ret < 0)
3814 debugfs_remove(cxgb4_debugfs_root);
3815 return ret;
3816}
3817
3818static void __exit cxgb4_cleanup_module(void)
3819{
3820 pci_unregister_driver(&cxgb4_driver);
3821 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
3822}
3823
3824module_init(cxgb4_init_module);
3825module_exit(cxgb4_cleanup_module);
A mirror of Linus' kernel repository