]> git.ipfire.org Git - thirdparty/linux.git/blame - drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
projects / thirdparty / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
cxgb4: add cxgb4_fcoe.h and macro definitions for FCoE
[thirdparty/linux.git] / drivers / net / ethernet / chelsio / cxgb4 / cxgb4_main.c
CommitLineData
b8ff05a9
DM		 1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
ce100b8b 4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
b8ff05a9
DM		 5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
01789349 44#include <linux/if.h>
b8ff05a9
DM		 45#include <linux/if_vlan.h>
46#include <linux/init.h>
47#include <linux/log2.h>
48#include <linux/mdio.h>
49#include <linux/module.h>
50#include <linux/moduleparam.h>
51#include <linux/mutex.h>
52#include <linux/netdevice.h>
53#include <linux/pci.h>
54#include <linux/aer.h>
55#include <linux/rtnetlink.h>
56#include <linux/sched.h>
57#include <linux/seq_file.h>
58#include <linux/sockios.h>
59#include <linux/vmalloc.h>
60#include <linux/workqueue.h>
61#include <net/neighbour.h>
62#include <net/netevent.h>
01bcca68 63#include <net/addrconf.h>
1ef8019b 64#include <net/bonding.h>
b5a02f50 65#include <net/addrconf.h>
b8ff05a9
DM		 66#include <asm/uaccess.h>
67
68#include "cxgb4.h"
69#include "t4_regs.h"
f612b815 70#include "t4_values.h"
b8ff05a9
DM		 71#include "t4_msg.h"
72#include "t4fw_api.h"
cd6c2f12 73#include "t4fw_version.h"
688848b1 74#include "cxgb4_dcb.h"
fd88b31a 75#include "cxgb4_debugfs.h"
b5a02f50 76#include "clip_tbl.h"
b8ff05a9
DM		 77#include "l2t.h"
78
01bcca68
VP		 79#ifdef DRV_VERSION
80#undef DRV_VERSION
81#endif
3a7f8554
SR		 82#define DRV_VERSION "2.0.0-ko"
83#define DRV_DESC "Chelsio T4/T5 Network Driver"
b8ff05a9 84
b8ff05a9
DM		 85enum {
86 MAX_TXQ_ENTRIES = 16384,
87 MAX_CTRL_TXQ_ENTRIES = 1024,
88 MAX_RSPQ_ENTRIES = 16384,
89 MAX_RX_BUFFERS = 16384,
90 MIN_TXQ_ENTRIES = 32,
91 MIN_CTRL_TXQ_ENTRIES = 32,
92 MIN_RSPQ_ENTRIES = 128,
93 MIN_FL_ENTRIES = 16
94};
95
f2b7e78d
VP		 96/* Host shadow copy of ingress filter entry. This is in host native format
97 * and doesn't match the ordering or bit order, etc. of the hardware of the
98 * firmware command. The use of bit-field structure elements is purely to
99 * remind ourselves of the field size limitations and save memory in the case
100 * where the filter table is large.
101 */
102struct filter_entry {
103 /* Administrative fields for filter.
104 */
105 u32 valid:1; /* filter allocated and valid */
106 u32 locked:1; /* filter is administratively locked */
107
108 u32 pending:1; /* filter action is pending firmware reply */
109 u32 smtidx:8; /* Source MAC Table index for smac */
110 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */
111
112 /* The filter itself. Most of this is a straight copy of information
113 * provided by the extended ioctl(). Some fields are translated to
114 * internal forms -- for instance the Ingress Queue ID passed in from
115 * the ioctl() is translated into the Absolute Ingress Queue ID.
116 */
117 struct ch_filter_specification fs;
118};
119
b8ff05a9
DM		 120#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
121 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
122 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
123
3fedeab1
HS		 124/* Macros needed to support the PCI Device ID Table ...
125 */
126#define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
768ffc66 127 static const struct pci_device_id cxgb4_pci_tbl[] = {
3fedeab1 128#define CH_PCI_DEVICE_ID_FUNCTION 0x4
b8ff05a9 129
3fedeab1
HS		 130/* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is
131 * called for both.
132 */
133#define CH_PCI_DEVICE_ID_FUNCTION2 0x0
134
135#define CH_PCI_ID_TABLE_ENTRY(devid) \
136 {PCI_VDEVICE(CHELSIO, (devid)), 4}
137
138#define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
139 { 0, } \
140 }
141
142#include "t4_pci_id_tbl.h"
b8ff05a9 143
16e47624 144#define FW4_FNAME "cxgb4/t4fw.bin"
0a57a536 145#define FW5_FNAME "cxgb4/t5fw.bin"
16e47624 146#define FW4_CFNAME "cxgb4/t4-config.txt"
0a57a536 147#define FW5_CFNAME "cxgb4/t5-config.txt"
b8ff05a9
DM		 148
149MODULE_DESCRIPTION(DRV_DESC);
150MODULE_AUTHOR("Chelsio Communications");
151MODULE_LICENSE("Dual BSD/GPL");
152MODULE_VERSION(DRV_VERSION);
153MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
16e47624 154MODULE_FIRMWARE(FW4_FNAME);
0a57a536 155MODULE_FIRMWARE(FW5_FNAME);
b8ff05a9 156
636f9d37
VP		 157/*
158 * Normally we're willing to become the firmware's Master PF but will be happy
159 * if another PF has already become the Master and initialized the adapter.
160 * Setting "force_init" will cause this driver to forcibly establish itself as
161 * the Master PF and initialize the adapter.
162 */
163static uint force_init;
164
165module_param(force_init, uint, 0644);
166MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
167
13ee15d3
VP		 168/*
169 * Normally if the firmware we connect to has Configuration File support, we
170 * use that and only fall back to the old Driver-based initialization if the
171 * Configuration File fails for some reason. If force_old_init is set, then
172 * we'll always use the old Driver-based initialization sequence.
173 */
174static uint force_old_init;
175
176module_param(force_old_init, uint, 0644);
06640310
HS		 177MODULE_PARM_DESC(force_old_init, "Force old initialization sequence, deprecated"
178 " parameter");
13ee15d3 179
b8ff05a9
DM		 180static int dflt_msg_enable = DFLT_MSG_ENABLE;
181
182module_param(dflt_msg_enable, int, 0644);
183MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
184
185/*
186 * The driver uses the best interrupt scheme available on a platform in the
187 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
188 * of these schemes the driver may consider as follows:
189 *
190 * msi = 2: choose from among all three options
191 * msi = 1: only consider MSI and INTx interrupts
192 * msi = 0: force INTx interrupts
193 */
194static int msi = 2;
195
196module_param(msi, int, 0644);
197MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
198
199/*
200 * Queue interrupt hold-off timer values. Queues default to the first of these
201 * upon creation.
202 */
203static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
204
205module_param_array(intr_holdoff, uint, NULL, 0644);
206MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
06640310 207 "0..4 in microseconds, deprecated parameter");
b8ff05a9
DM		 208
209static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
210
211module_param_array(intr_cnt, uint, NULL, 0644);
212MODULE_PARM_DESC(intr_cnt,
06640310
HS		 213 "thresholds 1..3 for queue interrupt packet counters, "
214 "deprecated parameter");
b8ff05a9 215
636f9d37
VP		 216/*
217 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
218 * offset by 2 bytes in order to have the IP headers line up on 4-byte
219 * boundaries. This is a requirement for many architectures which will throw
220 * a machine check fault if an attempt is made to access one of the 4-byte IP
221 * header fields on a non-4-byte boundary. And it's a major performance issue
222 * even on some architectures which allow it like some implementations of the
223 * x86 ISA. However, some architectures don't mind this and for some very
224 * edge-case performance sensitive applications (like forwarding large volumes
225 * of small packets), setting this DMA offset to 0 will decrease the number of
226 * PCI-E Bus transfers enough to measurably affect performance.
227 */
228static int rx_dma_offset = 2;
229
eb939922 230static bool vf_acls;
b8ff05a9
DM		 231
232#ifdef CONFIG_PCI_IOV
233module_param(vf_acls, bool, 0644);
06640310
HS		 234MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement, "
235 "deprecated parameter");
b8ff05a9 236
7d6727cf
SR		 237/* Configure the number of PCI-E Virtual Function which are to be instantiated
238 * on SR-IOV Capable Physical Functions.
0a57a536 239 */
7d6727cf 240static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
b8ff05a9
DM		 241
242module_param_array(num_vf, uint, NULL, 0644);
7d6727cf 243MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
b8ff05a9
DM		 244#endif
245
688848b1
AB		 246/* TX Queue select used to determine what algorithm to use for selecting TX
247 * queue. Select between the kernel provided function (select_queue=0) or user
248 * cxgb_select_queue function (select_queue=1)
249 *
250 * Default: select_queue=0
251 */
252static int select_queue;
253module_param(select_queue, int, 0644);
254MODULE_PARM_DESC(select_queue,
255 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method.");
256
06640310 257static unsigned int tp_vlan_pri_map = HW_TPL_FR_MT_PR_IV_P_FC;
13ee15d3 258
f2b7e78d 259module_param(tp_vlan_pri_map, uint, 0644);
06640310
HS		 260MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration, "
261 "deprecated parameter");
f2b7e78d 262
b8ff05a9
DM		 263static struct dentry *cxgb4_debugfs_root;
264
265static LIST_HEAD(adapter_list);
266static DEFINE_MUTEX(uld_mutex);
01bcca68
VP		 267/* Adapter list to be accessed from atomic context */
268static LIST_HEAD(adap_rcu_list);
269static DEFINE_SPINLOCK(adap_rcu_lock);
b8ff05a9
DM		 270static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
271static const char *uld_str[] = { "RDMA", "iSCSI" };
272
273static void link_report(struct net_device *dev)
274{
275 if (!netif_carrier_ok(dev))
276 netdev_info(dev, "link down\n");
277 else {
278 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
279
280 const char *s = "10Mbps";
281 const struct port_info *p = netdev_priv(dev);
282
283 switch (p->link_cfg.speed) {
e8b39015 284 case 10000:
b8ff05a9
DM		 285 s = "10Gbps";
286 break;
e8b39015 287 case 1000:
b8ff05a9
DM		 288 s = "1000Mbps";
289 break;
e8b39015 290 case 100:
b8ff05a9
DM		 291 s = "100Mbps";
292 break;
e8b39015 293 case 40000:
72aca4bf
KS		 294 s = "40Gbps";
295 break;
b8ff05a9
DM		 296 }
297
298 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
299 fc[p->link_cfg.fc]);
300 }
301}
302
688848b1
AB		 303#ifdef CONFIG_CHELSIO_T4_DCB
304/* Set up/tear down Data Center Bridging Priority mapping for a net device. */
305static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable)
306{
307 struct port_info *pi = netdev_priv(dev);
308 struct adapter *adap = pi->adapter;
309 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset];
310 int i;
311
312 /* We use a simple mapping of Port TX Queue Index to DCB
313 * Priority when we're enabling DCB.
314 */
315 for (i = 0; i < pi->nqsets; i++, txq++) {
316 u32 name, value;
317 int err;
318
5167865a
HS		 319 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
320 FW_PARAMS_PARAM_X_V(
321 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) |
322 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id));
688848b1
AB		 323 value = enable ? i : 0xffffffff;
324
325 /* Since we can be called while atomic (from "interrupt
326 * level") we need to issue the Set Parameters Commannd
327 * without sleeping (timeout < 0).
328 */
329 err = t4_set_params_nosleep(adap, adap->mbox, adap->fn, 0, 1,
330 &name, &value);
331
332 if (err)
333 dev_err(adap->pdev_dev,
334 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n",
335 enable ? "set" : "unset", pi->port_id, i, -err);
10b00466
AB		 336 else
337 txq->dcb_prio = value;
688848b1
AB		 338 }
339}
340#endif /* CONFIG_CHELSIO_T4_DCB */
341
b8ff05a9
DM		 342void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
343{
344 struct net_device *dev = adapter->port[port_id];
345
346 /* Skip changes from disabled ports. */
347 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
348 if (link_stat)
349 netif_carrier_on(dev);
688848b1
AB		 350 else {
351#ifdef CONFIG_CHELSIO_T4_DCB
352 cxgb4_dcb_state_init(dev);
353 dcb_tx_queue_prio_enable(dev, false);
354#endif /* CONFIG_CHELSIO_T4_DCB */
b8ff05a9 355 netif_carrier_off(dev);
688848b1 356 }
b8ff05a9
DM		 357
358 link_report(dev);
359 }
360}
361
362void t4_os_portmod_changed(const struct adapter *adap, int port_id)
363{
364 static const char *mod_str[] = {
a0881cab 365 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
b8ff05a9
DM		 366 };
367
368 const struct net_device *dev = adap->port[port_id];
369 const struct port_info *pi = netdev_priv(dev);
370
371 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
372 netdev_info(dev, "port module unplugged\n");
a0881cab 373 else if (pi->mod_type < ARRAY_SIZE(mod_str))
b8ff05a9
DM		 374 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
375}
376
377/*
378 * Configure the exact and hash address filters to handle a port's multicast
379 * and secondary unicast MAC addresses.
380 */
381static int set_addr_filters(const struct net_device *dev, bool sleep)
382{
383 u64 mhash = 0;
384 u64 uhash = 0;
385 bool free = true;
386 u16 filt_idx[7];
387 const u8 *addr[7];
388 int ret, naddr = 0;
b8ff05a9
DM		 389 const struct netdev_hw_addr *ha;
390 int uc_cnt = netdev_uc_count(dev);
4a35ecf8 391 int mc_cnt = netdev_mc_count(dev);
b8ff05a9 392 const struct port_info *pi = netdev_priv(dev);
060e0c75 393 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 394
395 /* first do the secondary unicast addresses */
396 netdev_for_each_uc_addr(ha, dev) {
397 addr[naddr++] = ha->addr;
398 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 399 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 400 naddr, addr, filt_idx, &uhash, sleep);
401 if (ret < 0)
402 return ret;
403
404 free = false;
405 naddr = 0;
406 }
407 }
408
409 /* next set up the multicast addresses */
4a35ecf8
DM		 410 netdev_for_each_mc_addr(ha, dev) {
411 addr[naddr++] = ha->addr;
412 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 413 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 414 naddr, addr, filt_idx, &mhash, sleep);
415 if (ret < 0)
416 return ret;
417
418 free = false;
419 naddr = 0;
420 }
421 }
422
060e0c75 423 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
b8ff05a9
DM		 424 uhash | mhash, sleep);
425}
426
3069ee9b
VP		 427int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
428module_param(dbfifo_int_thresh, int, 0644);
429MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
430
404d9e3f
VP		 431/*
432 * usecs to sleep while draining the dbfifo
433 */
434static int dbfifo_drain_delay = 1000;
3069ee9b
VP		 435module_param(dbfifo_drain_delay, int, 0644);
436MODULE_PARM_DESC(dbfifo_drain_delay,
437 "usecs to sleep while draining the dbfifo");
438
b8ff05a9
DM		 439/*
440 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
441 * If @mtu is -1 it is left unchanged.
442 */
443static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
444{
445 int ret;
446 struct port_info *pi = netdev_priv(dev);
447
448 ret = set_addr_filters(dev, sleep_ok);
449 if (ret == 0)
060e0c75 450 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
b8ff05a9 451 (dev->flags & IFF_PROMISC) ? 1 : 0,
f8f5aafa 452 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
b8ff05a9
DM		 453 sleep_ok);
454 return ret;
455}
456
457/**
458 * link_start - enable a port
459 * @dev: the port to enable
460 *
461 * Performs the MAC and PHY actions needed to enable a port.
462 */
463static int link_start(struct net_device *dev)
464{
465 int ret;
466 struct port_info *pi = netdev_priv(dev);
060e0c75 467 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 468
469 /*
470 * We do not set address filters and promiscuity here, the stack does
471 * that step explicitly.
472 */
060e0c75 473 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
f646968f 474 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
b8ff05a9 475 if (ret == 0) {
060e0c75 476 ret = t4_change_mac(pi->adapter, mb, pi->viid,
b8ff05a9 477 pi->xact_addr_filt, dev->dev_addr, true,
b6bd29e7 478 true);
b8ff05a9
DM		 479 if (ret >= 0) {
480 pi->xact_addr_filt = ret;
481 ret = 0;
482 }
483 }
484 if (ret == 0)
060e0c75
DM		 485 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
486 &pi->link_cfg);
30f00847
AB		 487 if (ret == 0) {
488 local_bh_disable();
688848b1
AB		 489 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true,
490 true, CXGB4_DCB_ENABLED);
30f00847
AB		 491 local_bh_enable();
492 }
688848b1 493
b8ff05a9
DM		 494 return ret;
495}
496
688848b1
AB		 497int cxgb4_dcb_enabled(const struct net_device *dev)
498{
499#ifdef CONFIG_CHELSIO_T4_DCB
500 struct port_info *pi = netdev_priv(dev);
501
3bb06261
AB		 502 if (!pi->dcb.enabled)
503 return 0;
504
505 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) ||
506 (pi->dcb.state == CXGB4_DCB_STATE_HOST));
688848b1
AB		 507#else
508 return 0;
509#endif
510}
511EXPORT_SYMBOL(cxgb4_dcb_enabled);
512
513#ifdef CONFIG_CHELSIO_T4_DCB
514/* Handle a Data Center Bridging update message from the firmware. */
515static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd)
516{
2b5fb1f2 517 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid));
688848b1
AB		 518 struct net_device *dev = adap->port[port];
519 int old_dcb_enabled = cxgb4_dcb_enabled(dev);
520 int new_dcb_enabled;
521
522 cxgb4_dcb_handle_fw_update(adap, pcmd);
523 new_dcb_enabled = cxgb4_dcb_enabled(dev);
524
525 /* If the DCB has become enabled or disabled on the port then we're
526 * going to need to set up/tear down DCB Priority parameters for the
527 * TX Queues associated with the port.
528 */
529 if (new_dcb_enabled != old_dcb_enabled)
530 dcb_tx_queue_prio_enable(dev, new_dcb_enabled);
531}
532#endif /* CONFIG_CHELSIO_T4_DCB */
533
f2b7e78d
VP		 534/* Clear a filter and release any of its resources that we own. This also
535 * clears the filter's "pending" status.
536 */
537static void clear_filter(struct adapter *adap, struct filter_entry *f)
538{
539 /* If the new or old filter have loopback rewriteing rules then we'll
540 * need to free any existing Layer Two Table (L2T) entries of the old
541 * filter rule. The firmware will handle freeing up any Source MAC
542 * Table (SMT) entries used for rewriting Source MAC Addresses in
543 * loopback rules.
544 */
545 if (f->l2t)
546 cxgb4_l2t_release(f->l2t);
547
548 /* The zeroing of the filter rule below clears the filter valid,
549 * pending, locked flags, l2t pointer, etc. so it's all we need for
550 * this operation.
551 */
552 memset(f, 0, sizeof(*f));
553}
554
555/* Handle a filter write/deletion reply.
556 */
557static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
558{
559 unsigned int idx = GET_TID(rpl);
560 unsigned int nidx = idx - adap->tids.ftid_base;
561 unsigned int ret;
562 struct filter_entry *f;
563
564 if (idx >= adap->tids.ftid_base && nidx <
565 (adap->tids.nftids + adap->tids.nsftids)) {
566 idx = nidx;
bdc590b9 567 ret = TCB_COOKIE_G(rpl->cookie);
f2b7e78d
VP		 568 f = &adap->tids.ftid_tab[idx];
569
570 if (ret == FW_FILTER_WR_FLT_DELETED) {
571 /* Clear the filter when we get confirmation from the
572 * hardware that the filter has been deleted.
573 */
574 clear_filter(adap, f);
575 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
576 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
577 idx);
578 clear_filter(adap, f);
579 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
580 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
581 f->pending = 0; /* asynchronous setup completed */
582 f->valid = 1;
583 } else {
584 /* Something went wrong. Issue a warning about the
585 * problem and clear everything out.
586 */
587 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
588 idx, ret);
589 clear_filter(adap, f);
590 }
591 }
592}
593
594/* Response queue handler for the FW event queue.
b8ff05a9
DM		 595 */
596static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
597 const struct pkt_gl *gl)
598{
599 u8 opcode = ((const struct rss_header *)rsp)->opcode;
600
601 rsp++; /* skip RSS header */
b407a4a9
VP		 602
603 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
604 */
605 if (unlikely(opcode == CPL_FW4_MSG &&
606 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
607 rsp++;
608 opcode = ((const struct rss_header *)rsp)->opcode;
609 rsp++;
610 if (opcode != CPL_SGE_EGR_UPDATE) {
611 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
612 , opcode);
613 goto out;
614 }
615 }
616
b8ff05a9
DM		 617 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
618 const struct cpl_sge_egr_update *p = (void *)rsp;
bdc590b9 619 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid));
e46dab4d 620 struct sge_txq *txq;
b8ff05a9 621
e46dab4d 622 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
b8ff05a9 623 txq->restarts++;
e46dab4d 624 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
b8ff05a9
DM		 625 struct sge_eth_txq *eq;
626
627 eq = container_of(txq, struct sge_eth_txq, q);
628 netif_tx_wake_queue(eq->txq);
629 } else {
630 struct sge_ofld_txq *oq;
631
632 oq = container_of(txq, struct sge_ofld_txq, q);
633 tasklet_schedule(&oq->qresume_tsk);
634 }
635 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
636 const struct cpl_fw6_msg *p = (void *)rsp;
637
688848b1
AB		 638#ifdef CONFIG_CHELSIO_T4_DCB
639 const struct fw_port_cmd *pcmd = (const void *)p->data;
e2ac9628 640 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid));
688848b1 641 unsigned int action =
2b5fb1f2 642 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16));
688848b1
AB		 643
644 if (cmd == FW_PORT_CMD &&
645 action == FW_PORT_ACTION_GET_PORT_INFO) {
2b5fb1f2 646 int port = FW_PORT_CMD_PORTID_G(
688848b1
AB		 647 be32_to_cpu(pcmd->op_to_portid));
648 struct net_device *dev = q->adap->port[port];
649 int state_input = ((pcmd->u.info.dcbxdis_pkd &
2b5fb1f2 650 FW_PORT_CMD_DCBXDIS_F)
688848b1
AB		 651 ? CXGB4_DCB_INPUT_FW_DISABLED
652 : CXGB4_DCB_INPUT_FW_ENABLED);
653
654 cxgb4_dcb_state_fsm(dev, state_input);
655 }
656
657 if (cmd == FW_PORT_CMD &&
658 action == FW_PORT_ACTION_L2_DCB_CFG)
659 dcb_rpl(q->adap, pcmd);
660 else
661#endif
662 if (p->type == 0)
663 t4_handle_fw_rpl(q->adap, p->data);
b8ff05a9
DM		 664 } else if (opcode == CPL_L2T_WRITE_RPL) {
665 const struct cpl_l2t_write_rpl *p = (void *)rsp;
666
667 do_l2t_write_rpl(q->adap, p);
f2b7e78d
VP		 668 } else if (opcode == CPL_SET_TCB_RPL) {
669 const struct cpl_set_tcb_rpl *p = (void *)rsp;
670
671 filter_rpl(q->adap, p);
b8ff05a9
DM		 672 } else
673 dev_err(q->adap->pdev_dev,
674 "unexpected CPL %#x on FW event queue\n", opcode);
b407a4a9 675out:
b8ff05a9
DM		 676 return 0;
677}
678
679/**
680 * uldrx_handler - response queue handler for ULD queues
681 * @q: the response queue that received the packet
682 * @rsp: the response queue descriptor holding the offload message
683 * @gl: the gather list of packet fragments
684 *
685 * Deliver an ingress offload packet to a ULD. All processing is done by
686 * the ULD, we just maintain statistics.
687 */
688static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
689 const struct pkt_gl *gl)
690{
691 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
692
b407a4a9
VP		 693 /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
694 */
695 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
696 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
697 rsp += 2;
698
b8ff05a9
DM		 699 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
700 rxq->stats.nomem++;
701 return -1;
702 }
703 if (gl == NULL)
704 rxq->stats.imm++;
705 else if (gl == CXGB4_MSG_AN)
706 rxq->stats.an++;
707 else
708 rxq->stats.pkts++;
709 return 0;
710}
711
712static void disable_msi(struct adapter *adapter)
713{
714 if (adapter->flags & USING_MSIX) {
715 pci_disable_msix(adapter->pdev);
716 adapter->flags &= ~USING_MSIX;
717 } else if (adapter->flags & USING_MSI) {
718 pci_disable_msi(adapter->pdev);
719 adapter->flags &= ~USING_MSI;
720 }
721}
722
723/*
724 * Interrupt handler for non-data events used with MSI-X.
725 */
726static irqreturn_t t4_nondata_intr(int irq, void *cookie)
727{
728 struct adapter *adap = cookie;
0d804338 729 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A));
b8ff05a9 730
0d804338 731 if (v & PFSW_F) {
b8ff05a9 732 adap->swintr = 1;
0d804338 733 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v);
b8ff05a9
DM		 734 }
735 t4_slow_intr_handler(adap);
736 return IRQ_HANDLED;
737}
738
739/*
740 * Name the MSI-X interrupts.
741 */
742static void name_msix_vecs(struct adapter *adap)
743{
ba27816c 744 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
b8ff05a9
DM		 745
746 /* non-data interrupts */
b1a3c2b6 747 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
b8ff05a9
DM		 748
749 /* FW events */
b1a3c2b6
DM		 750 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
751 adap->port[0]->name);
b8ff05a9
DM		 752
753 /* Ethernet queues */
754 for_each_port(adap, j) {
755 struct net_device *d = adap->port[j];
756 const struct port_info *pi = netdev_priv(d);
757
ba27816c 758 for (i = 0; i < pi->nqsets; i++, msi_idx++)
b8ff05a9
DM		 759 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
760 d->name, i);
b8ff05a9
DM		 761 }
762
763 /* offload queues */
ba27816c
DM		 764 for_each_ofldrxq(&adap->sge, i)
765 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
b1a3c2b6 766 adap->port[0]->name, i);
ba27816c
DM		 767
768 for_each_rdmarxq(&adap->sge, i)
769 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
b1a3c2b6 770 adap->port[0]->name, i);
cf38be6d
HS		 771
772 for_each_rdmaciq(&adap->sge, i)
773 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d",
774 adap->port[0]->name, i);
b8ff05a9
DM		 775}
776
777static int request_msix_queue_irqs(struct adapter *adap)
778{
779 struct sge *s = &adap->sge;
cf38be6d
HS		 780 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0;
781 int msi_index = 2;
b8ff05a9
DM		 782
783 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
784 adap->msix_info[1].desc, &s->fw_evtq);
785 if (err)
786 return err;
787
788 for_each_ethrxq(s, ethqidx) {
404d9e3f
VP		 789 err = request_irq(adap->msix_info[msi_index].vec,
790 t4_sge_intr_msix, 0,
791 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 792 &s->ethrxq[ethqidx].rspq);
793 if (err)
794 goto unwind;
404d9e3f 795 msi_index++;
b8ff05a9
DM		 796 }
797 for_each_ofldrxq(s, ofldqidx) {
404d9e3f
VP		 798 err = request_irq(adap->msix_info[msi_index].vec,
799 t4_sge_intr_msix, 0,
800 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 801 &s->ofldrxq[ofldqidx].rspq);
802 if (err)
803 goto unwind;
404d9e3f 804 msi_index++;
b8ff05a9
DM		 805 }
806 for_each_rdmarxq(s, rdmaqidx) {
404d9e3f
VP		 807 err = request_irq(adap->msix_info[msi_index].vec,
808 t4_sge_intr_msix, 0,
809 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 810 &s->rdmarxq[rdmaqidx].rspq);
811 if (err)
812 goto unwind;
404d9e3f 813 msi_index++;
b8ff05a9 814 }
cf38be6d
HS		 815 for_each_rdmaciq(s, rdmaciqqidx) {
816 err = request_irq(adap->msix_info[msi_index].vec,
817 t4_sge_intr_msix, 0,
818 adap->msix_info[msi_index].desc,
819 &s->rdmaciq[rdmaciqqidx].rspq);
820 if (err)
821 goto unwind;
822 msi_index++;
823 }
b8ff05a9
DM		 824 return 0;
825
826unwind:
cf38be6d
HS		 827 while (--rdmaciqqidx >= 0)
828 free_irq(adap->msix_info[--msi_index].vec,
829 &s->rdmaciq[rdmaciqqidx].rspq);
b8ff05a9 830 while (--rdmaqidx >= 0)
404d9e3f 831 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 832 &s->rdmarxq[rdmaqidx].rspq);
833 while (--ofldqidx >= 0)
404d9e3f 834 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 835 &s->ofldrxq[ofldqidx].rspq);
836 while (--ethqidx >= 0)
404d9e3f
VP		 837 free_irq(adap->msix_info[--msi_index].vec,
838 &s->ethrxq[ethqidx].rspq);
b8ff05a9
DM		 839 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
840 return err;
841}
842
843static void free_msix_queue_irqs(struct adapter *adap)
844{
404d9e3f 845 int i, msi_index = 2;
b8ff05a9
DM		 846 struct sge *s = &adap->sge;
847
848 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
849 for_each_ethrxq(s, i)
404d9e3f 850 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
b8ff05a9 851 for_each_ofldrxq(s, i)
404d9e3f 852 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
b8ff05a9 853 for_each_rdmarxq(s, i)
404d9e3f 854 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
cf38be6d
HS		 855 for_each_rdmaciq(s, i)
856 free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq);
b8ff05a9
DM		 857}
858
671b0060
DM		 859/**
860 * write_rss - write the RSS table for a given port
861 * @pi: the port
862 * @queues: array of queue indices for RSS
863 *
864 * Sets up the portion of the HW RSS table for the port's VI to distribute
865 * packets to the Rx queues in @queues.
866 */
867static int write_rss(const struct port_info *pi, const u16 *queues)
868{
869 u16 *rss;
870 int i, err;
871 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
872
873 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
874 if (!rss)
875 return -ENOMEM;
876
877 /* map the queue indices to queue ids */
878 for (i = 0; i < pi->rss_size; i++, queues++)
879 rss[i] = q[*queues].rspq.abs_id;
880
060e0c75
DM		 881 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
882 pi->rss_size, rss, pi->rss_size);
671b0060
DM		 883 kfree(rss);
884 return err;
885}
886
b8ff05a9
DM		 887/**
888 * setup_rss - configure RSS
889 * @adap: the adapter
890 *
671b0060 891 * Sets up RSS for each port.
b8ff05a9
DM		 892 */
893static int setup_rss(struct adapter *adap)
894{
671b0060 895 int i, err;
b8ff05a9
DM		 896
897 for_each_port(adap, i) {
898 const struct port_info *pi = adap2pinfo(adap, i);
b8ff05a9 899
671b0060 900 err = write_rss(pi, pi->rss);
b8ff05a9
DM		 901 if (err)
902 return err;
903 }
904 return 0;
905}
906
e46dab4d
DM		 907/*
908 * Return the channel of the ingress queue with the given qid.
909 */
910static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
911{
912 qid -= p->ingr_start;
913 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
914}
915
b8ff05a9
DM		 916/*
917 * Wait until all NAPI handlers are descheduled.
918 */
919static void quiesce_rx(struct adapter *adap)
920{
921 int i;
922
923 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
924 struct sge_rspq *q = adap->sge.ingr_map[i];
925
3a336cb1 926 if (q && q->handler) {
b8ff05a9 927 napi_disable(&q->napi);
3a336cb1
HS		 928 local_bh_disable();
929 while (!cxgb_poll_lock_napi(q))
930 mdelay(1);
931 local_bh_enable();
932 }
933
b8ff05a9
DM		 934 }
935}
936
937/*
938 * Enable NAPI scheduling and interrupt generation for all Rx queues.
939 */
940static void enable_rx(struct adapter *adap)
941{
942 int i;
943
944 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
945 struct sge_rspq *q = adap->sge.ingr_map[i];
946
947 if (!q)
948 continue;
3a336cb1
HS		 949 if (q->handler) {
950 cxgb_busy_poll_init_lock(q);
b8ff05a9 951 napi_enable(&q->napi);
3a336cb1 952 }
b8ff05a9 953 /* 0-increment GTS to start the timer and enable interrupts */
f612b815
HS		 954 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
955 SEINTARM_V(q->intr_params) |
956 INGRESSQID_V(q->cntxt_id));
b8ff05a9
DM		 957 }
958}
959
1c6a5b0e
HS		 960static int alloc_ofld_rxqs(struct adapter *adap, struct sge_ofld_rxq *q,
961 unsigned int nq, unsigned int per_chan, int msi_idx,
962 u16 *ids)
963{
964 int i, err;
965
966 for (i = 0; i < nq; i++, q++) {
967 if (msi_idx > 0)
968 msi_idx++;
969 err = t4_sge_alloc_rxq(adap, &q->rspq, false,
970 adap->port[i / per_chan],
971 msi_idx, q->fl.size ? &q->fl : NULL,
972 uldrx_handler);
973 if (err)
974 return err;
975 memset(&q->stats, 0, sizeof(q->stats));
976 if (ids)
977 ids[i] = q->rspq.abs_id;
978 }
979 return 0;
980}
981
b8ff05a9
DM		 982/**
983 * setup_sge_queues - configure SGE Tx/Rx/response queues
984 * @adap: the adapter
985 *
986 * Determines how many sets of SGE queues to use and initializes them.
987 * We support multiple queue sets per port if we have MSI-X, otherwise
988 * just one queue set per port.
989 */
990static int setup_sge_queues(struct adapter *adap)
991{
992 int err, msi_idx, i, j;
993 struct sge *s = &adap->sge;
994
995 bitmap_zero(s->starving_fl, MAX_EGRQ);
996 bitmap_zero(s->txq_maperr, MAX_EGRQ);
997
998 if (adap->flags & USING_MSIX)
999 msi_idx = 1; /* vector 0 is for non-queue interrupts */
1000 else {
1001 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
1002 NULL, NULL);
1003 if (err)
1004 return err;
1005 msi_idx = -((int)s->intrq.abs_id + 1);
1006 }
1007
1008 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
1009 msi_idx, NULL, fwevtq_handler);
1010 if (err) {
1011freeout: t4_free_sge_resources(adap);
1012 return err;
1013 }
1014
1015 for_each_port(adap, i) {
1016 struct net_device *dev = adap->port[i];
1017 struct port_info *pi = netdev_priv(dev);
1018 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
1019 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
1020
1021 for (j = 0; j < pi->nqsets; j++, q++) {
1022 if (msi_idx > 0)
1023 msi_idx++;
1024 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1025 msi_idx, &q->fl,
1026 t4_ethrx_handler);
1027 if (err)
1028 goto freeout;
1029 q->rspq.idx = j;
1030 memset(&q->stats, 0, sizeof(q->stats));
1031 }
1032 for (j = 0; j < pi->nqsets; j++, t++) {
1033 err = t4_sge_alloc_eth_txq(adap, t, dev,
1034 netdev_get_tx_queue(dev, j),
1035 s->fw_evtq.cntxt_id);
1036 if (err)
1037 goto freeout;
1038 }
1039 }
1040
1041 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1042 for_each_ofldrxq(s, i) {
1c6a5b0e
HS		 1043 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i],
1044 adap->port[i / j],
b8ff05a9
DM		 1045 s->fw_evtq.cntxt_id);
1046 if (err)
1047 goto freeout;
1048 }
1049
1c6a5b0e
HS		 1050#define ALLOC_OFLD_RXQS(firstq, nq, per_chan, ids) do { \
1051 err = alloc_ofld_rxqs(adap, firstq, nq, per_chan, msi_idx, ids); \
1052 if (err) \
1053 goto freeout; \
1054 if (msi_idx > 0) \
1055 msi_idx += nq; \
1056} while (0)
b8ff05a9 1057
1c6a5b0e
HS		 1058 ALLOC_OFLD_RXQS(s->ofldrxq, s->ofldqsets, j, s->ofld_rxq);
1059 ALLOC_OFLD_RXQS(s->rdmarxq, s->rdmaqs, 1, s->rdma_rxq);
f36e58e5
HS		 1060 j = s->rdmaciqs / adap->params.nports; /* rdmaq queues per channel */
1061 ALLOC_OFLD_RXQS(s->rdmaciq, s->rdmaciqs, j, s->rdma_ciq);
b8ff05a9 1062
1c6a5b0e 1063#undef ALLOC_OFLD_RXQS
cf38be6d 1064
b8ff05a9
DM		 1065 for_each_port(adap, i) {
1066 /*
1067 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1068 * have RDMA queues, and that's the right value.
1069 */
1070 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1071 s->fw_evtq.cntxt_id,
1072 s->rdmarxq[i].rspq.cntxt_id);
1073 if (err)
1074 goto freeout;
1075 }
1076
9bb59b96 1077 t4_write_reg(adap, is_t4(adap->params.chip) ?
837e4a42
HS		 1078 MPS_TRC_RSS_CONTROL_A :
1079 MPS_T5_TRC_RSS_CONTROL_A,
1080 RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) |
1081 QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id));
b8ff05a9
DM		 1082 return 0;
1083}
1084
b8ff05a9
DM		 1085/*
1086 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1087 * The allocated memory is cleared.
1088 */
1089void *t4_alloc_mem(size_t size)
1090{
8be04b93 1091 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
b8ff05a9
DM		 1092
1093 if (!p)
89bf67f1 1094 p = vzalloc(size);
b8ff05a9
DM		 1095 return p;
1096}
1097
1098/*
1099 * Free memory allocated through alloc_mem().
1100 */
fd88b31a 1101void t4_free_mem(void *addr)
b8ff05a9
DM		 1102{
1103 if (is_vmalloc_addr(addr))
1104 vfree(addr);
1105 else
1106 kfree(addr);
1107}
1108
f2b7e78d
VP		 1109/* Send a Work Request to write the filter at a specified index. We construct
1110 * a Firmware Filter Work Request to have the work done and put the indicated
1111 * filter into "pending" mode which will prevent any further actions against
1112 * it till we get a reply from the firmware on the completion status of the
1113 * request.
1114 */
1115static int set_filter_wr(struct adapter *adapter, int fidx)
1116{
1117 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1118 struct sk_buff *skb;
1119 struct fw_filter_wr *fwr;
1120 unsigned int ftid;
1121
1122 /* If the new filter requires loopback Destination MAC and/or VLAN
1123 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1124 * the filter.
1125 */
1126 if (f->fs.newdmac || f->fs.newvlan) {
1127 /* allocate L2T entry for new filter */
1128 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1129 if (f->l2t == NULL)
1130 return -EAGAIN;
1131 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1132 f->fs.eport, f->fs.dmac)) {
1133 cxgb4_l2t_release(f->l2t);
1134 f->l2t = NULL;
1135 return -ENOMEM;
1136 }
1137 }
1138
1139 ftid = adapter->tids.ftid_base + fidx;
1140
1141 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1142 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1143 memset(fwr, 0, sizeof(*fwr));
1144
1145 /* It would be nice to put most of the following in t4_hw.c but most
1146 * of the work is translating the cxgbtool ch_filter_specification
1147 * into the Work Request and the definition of that structure is
1148 * currently in cxgbtool.h which isn't appropriate to pull into the
1149 * common code. We may eventually try to come up with a more neutral
1150 * filter specification structure but for now it's easiest to simply
1151 * put this fairly direct code in line ...
1152 */
e2ac9628
HS		 1153 fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR));
1154 fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr)/16));
f2b7e78d 1155 fwr->tid_to_iq =
77a80e23
HS		 1156 htonl(FW_FILTER_WR_TID_V(ftid) |
1157 FW_FILTER_WR_RQTYPE_V(f->fs.type) |
1158 FW_FILTER_WR_NOREPLY_V(0) |
1159 FW_FILTER_WR_IQ_V(f->fs.iq));
f2b7e78d 1160 fwr->del_filter_to_l2tix =
77a80e23
HS		 1161 htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) |
1162 FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) |
1163 FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) |
1164 FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) |
1165 FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) |
1166 FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) |
1167 FW_FILTER_WR_DMAC_V(f->fs.newdmac) |
1168 FW_FILTER_WR_SMAC_V(f->fs.newsmac) |
1169 FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT ||
f2b7e78d 1170 f->fs.newvlan == VLAN_REWRITE) |
77a80e23 1171 FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE ||
f2b7e78d 1172 f->fs.newvlan == VLAN_REWRITE) |
77a80e23
HS		 1173 FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) |
1174 FW_FILTER_WR_TXCHAN_V(f->fs.eport) |
1175 FW_FILTER_WR_PRIO_V(f->fs.prio) |
1176 FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0));
f2b7e78d
VP		 1177 fwr->ethtype = htons(f->fs.val.ethtype);
1178 fwr->ethtypem = htons(f->fs.mask.ethtype);
1179 fwr->frag_to_ovlan_vldm =
77a80e23
HS		 1180 (FW_FILTER_WR_FRAG_V(f->fs.val.frag) |
1181 FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) |
1182 FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) |
1183 FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) |
1184 FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) |
1185 FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld));
f2b7e78d
VP		 1186 fwr->smac_sel = 0;
1187 fwr->rx_chan_rx_rpl_iq =
77a80e23
HS		 1188 htons(FW_FILTER_WR_RX_CHAN_V(0) |
1189 FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id));
f2b7e78d 1190 fwr->maci_to_matchtypem =
77a80e23
HS		 1191 htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) |
1192 FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) |
1193 FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) |
1194 FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) |
1195 FW_FILTER_WR_PORT_V(f->fs.val.iport) |
1196 FW_FILTER_WR_PORTM_V(f->fs.mask.iport) |
1197 FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) |
1198 FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype));
f2b7e78d
VP		 1199 fwr->ptcl = f->fs.val.proto;
1200 fwr->ptclm = f->fs.mask.proto;
1201 fwr->ttyp = f->fs.val.tos;
1202 fwr->ttypm = f->fs.mask.tos;
1203 fwr->ivlan = htons(f->fs.val.ivlan);
1204 fwr->ivlanm = htons(f->fs.mask.ivlan);
1205 fwr->ovlan = htons(f->fs.val.ovlan);
1206 fwr->ovlanm = htons(f->fs.mask.ovlan);
1207 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1208 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1209 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1210 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1211 fwr->lp = htons(f->fs.val.lport);
1212 fwr->lpm = htons(f->fs.mask.lport);
1213 fwr->fp = htons(f->fs.val.fport);
1214 fwr->fpm = htons(f->fs.mask.fport);
1215 if (f->fs.newsmac)
1216 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1217
1218 /* Mark the filter as "pending" and ship off the Filter Work Request.
1219 * When we get the Work Request Reply we'll clear the pending status.
1220 */
1221 f->pending = 1;
1222 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1223 t4_ofld_send(adapter, skb);
1224 return 0;
1225}
1226
1227/* Delete the filter at a specified index.
1228 */
1229static int del_filter_wr(struct adapter *adapter, int fidx)
1230{
1231 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1232 struct sk_buff *skb;
1233 struct fw_filter_wr *fwr;
1234 unsigned int len, ftid;
1235
1236 len = sizeof(*fwr);
1237 ftid = adapter->tids.ftid_base + fidx;
1238
1239 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1240 fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1241 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1242
1243 /* Mark the filter as "pending" and ship off the Filter Work Request.
1244 * When we get the Work Request Reply we'll clear the pending status.
1245 */
1246 f->pending = 1;
1247 t4_mgmt_tx(adapter, skb);
1248 return 0;
1249}
1250
688848b1
AB		 1251static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
1252 void *accel_priv, select_queue_fallback_t fallback)
1253{
1254 int txq;
1255
1256#ifdef CONFIG_CHELSIO_T4_DCB
1257 /* If a Data Center Bridging has been successfully negotiated on this
1258 * link then we'll use the skb's priority to map it to a TX Queue.
1259 * The skb's priority is determined via the VLAN Tag Priority Code
1260 * Point field.
1261 */
1262 if (cxgb4_dcb_enabled(dev)) {
1263 u16 vlan_tci;
1264 int err;
1265
1266 err = vlan_get_tag(skb, &vlan_tci);
1267 if (unlikely(err)) {
1268 if (net_ratelimit())
1269 netdev_warn(dev,
1270 "TX Packet without VLAN Tag on DCB Link\n");
1271 txq = 0;
1272 } else {
1273 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1274 }
1275 return txq;
1276 }
1277#endif /* CONFIG_CHELSIO_T4_DCB */
1278
1279 if (select_queue) {
1280 txq = (skb_rx_queue_recorded(skb)
1281 ? skb_get_rx_queue(skb)
1282 : smp_processor_id());
1283
1284 while (unlikely(txq >= dev->real_num_tx_queues))
1285 txq -= dev->real_num_tx_queues;
1286
1287 return txq;
1288 }
1289
1290 return fallback(dev, skb) % dev->real_num_tx_queues;
1291}
1292
b8ff05a9
DM		 1293static inline int is_offload(const struct adapter *adap)
1294{
1295 return adap->params.offload;
1296}
1297
1298/*
1299 * Implementation of ethtool operations.
1300 */
1301
1302static u32 get_msglevel(struct net_device *dev)
1303{
1304 return netdev2adap(dev)->msg_enable;
1305}
1306
1307static void set_msglevel(struct net_device *dev, u32 val)
1308{
1309 netdev2adap(dev)->msg_enable = val;
1310}
1311
1312static char stats_strings[][ETH_GSTRING_LEN] = {
1313 "TxOctetsOK ",
1314 "TxFramesOK ",
1315 "TxBroadcastFrames ",
1316 "TxMulticastFrames ",
1317 "TxUnicastFrames ",
1318 "TxErrorFrames ",
1319
1320 "TxFrames64 ",
1321 "TxFrames65To127 ",
1322 "TxFrames128To255 ",
1323 "TxFrames256To511 ",
1324 "TxFrames512To1023 ",
1325 "TxFrames1024To1518 ",
1326 "TxFrames1519ToMax ",
1327
1328 "TxFramesDropped ",
1329 "TxPauseFrames ",
1330 "TxPPP0Frames ",
1331 "TxPPP1Frames ",
1332 "TxPPP2Frames ",
1333 "TxPPP3Frames ",
1334 "TxPPP4Frames ",
1335 "TxPPP5Frames ",
1336 "TxPPP6Frames ",
1337 "TxPPP7Frames ",
1338
1339 "RxOctetsOK ",
1340 "RxFramesOK ",
1341 "RxBroadcastFrames ",
1342 "RxMulticastFrames ",
1343 "RxUnicastFrames ",
1344
1345 "RxFramesTooLong ",
1346 "RxJabberErrors ",
1347 "RxFCSErrors ",
1348 "RxLengthErrors ",
1349 "RxSymbolErrors ",
1350 "RxRuntFrames ",
1351
1352 "RxFrames64 ",
1353 "RxFrames65To127 ",
1354 "RxFrames128To255 ",
1355 "RxFrames256To511 ",
1356 "RxFrames512To1023 ",
1357 "RxFrames1024To1518 ",
1358 "RxFrames1519ToMax ",
1359
1360 "RxPauseFrames ",
1361 "RxPPP0Frames ",
1362 "RxPPP1Frames ",
1363 "RxPPP2Frames ",
1364 "RxPPP3Frames ",
1365 "RxPPP4Frames ",
1366 "RxPPP5Frames ",
1367 "RxPPP6Frames ",
1368 "RxPPP7Frames ",
1369
1370 "RxBG0FramesDropped ",
1371 "RxBG1FramesDropped ",
1372 "RxBG2FramesDropped ",
1373 "RxBG3FramesDropped ",
1374 "RxBG0FramesTrunc ",
1375 "RxBG1FramesTrunc ",
1376 "RxBG2FramesTrunc ",
1377 "RxBG3FramesTrunc ",
1378
1379 "TSO ",
1380 "TxCsumOffload ",
1381 "RxCsumGood ",
1382 "VLANextractions ",
1383 "VLANinsertions ",
4a6346d4
DM		 1384 "GROpackets ",
1385 "GROmerged ",
22adfe0a
SR		 1386 "WriteCoalSuccess ",
1387 "WriteCoalFail ",
b8ff05a9
DM		 1388};
1389
1390static int get_sset_count(struct net_device *dev, int sset)
1391{
1392 switch (sset) {
1393 case ETH_SS_STATS:
1394 return ARRAY_SIZE(stats_strings);
1395 default:
1396 return -EOPNOTSUPP;
1397 }
1398}
1399
1400#define T4_REGMAP_SIZE (160 * 1024)
251f9e88 1401#define T5_REGMAP_SIZE (332 * 1024)
b8ff05a9
DM		 1402
1403static int get_regs_len(struct net_device *dev)
1404{
251f9e88 1405 struct adapter *adap = netdev2adap(dev);
d14807dd 1406 if (is_t4(adap->params.chip))
251f9e88
SR		 1407 return T4_REGMAP_SIZE;
1408 else
1409 return T5_REGMAP_SIZE;
b8ff05a9
DM		 1410}
1411
1412static int get_eeprom_len(struct net_device *dev)
1413{
1414 return EEPROMSIZE;
1415}
1416
1417static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1418{
1419 struct adapter *adapter = netdev2adap(dev);
ba3f8cd5 1420 u32 exprom_vers;
b8ff05a9 1421
23020ab3
RJ		 1422 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1423 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1424 strlcpy(info->bus_info, pci_name(adapter->pdev),
1425 sizeof(info->bus_info));
b8ff05a9 1426
84b40501 1427 if (adapter->params.fw_vers)
b8ff05a9
DM		 1428 snprintf(info->fw_version, sizeof(info->fw_version),
1429 "%u.%u.%u.%u, TP %u.%u.%u.%u",
b2e1a3f0
HS		 1430 FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
1431 FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
1432 FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
1433 FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers),
1434 FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
1435 FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
1436 FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
1437 FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
ba3f8cd5
HS		 1438
1439 if (!t4_get_exprom_version(adapter, &exprom_vers))
1440 snprintf(info->erom_version, sizeof(info->erom_version),
1441 "%u.%u.%u.%u",
1442 FW_HDR_FW_VER_MAJOR_G(exprom_vers),
1443 FW_HDR_FW_VER_MINOR_G(exprom_vers),
1444 FW_HDR_FW_VER_MICRO_G(exprom_vers),
1445 FW_HDR_FW_VER_BUILD_G(exprom_vers));
b8ff05a9
DM		 1446}
1447
1448static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1449{
1450 if (stringset == ETH_SS_STATS)
1451 memcpy(data, stats_strings, sizeof(stats_strings));
1452}
1453
1454/*
1455 * port stats maintained per queue of the port. They should be in the same
1456 * order as in stats_strings above.
1457 */
1458struct queue_port_stats {
1459 u64 tso;
1460 u64 tx_csum;
1461 u64 rx_csum;
1462 u64 vlan_ex;
1463 u64 vlan_ins;
4a6346d4
DM		 1464 u64 gro_pkts;
1465 u64 gro_merged;
b8ff05a9
DM		 1466};
1467
1468static void collect_sge_port_stats(const struct adapter *adap,
1469 const struct port_info *p, struct queue_port_stats *s)
1470{
1471 int i;
1472 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1473 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1474
1475 memset(s, 0, sizeof(*s));
1476 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1477 s->tso += tx->tso;
1478 s->tx_csum += tx->tx_cso;
1479 s->rx_csum += rx->stats.rx_cso;
1480 s->vlan_ex += rx->stats.vlan_ex;
1481 s->vlan_ins += tx->vlan_ins;
4a6346d4
DM		 1482 s->gro_pkts += rx->stats.lro_pkts;
1483 s->gro_merged += rx->stats.lro_merged;
b8ff05a9
DM		 1484 }
1485}
1486
1487static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1488 u64 *data)
1489{
1490 struct port_info *pi = netdev_priv(dev);
1491 struct adapter *adapter = pi->adapter;
22adfe0a 1492 u32 val1, val2;
b8ff05a9
DM		 1493
1494 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1495
1496 data += sizeof(struct port_stats) / sizeof(u64);
1497 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
22adfe0a 1498 data += sizeof(struct queue_port_stats) / sizeof(u64);
d14807dd 1499 if (!is_t4(adapter->params.chip)) {
f061de42
HS		 1500 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7));
1501 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL_A);
1502 val2 = t4_read_reg(adapter, SGE_STAT_MATCH_A);
22adfe0a
SR		 1503 *data = val1 - val2;
1504 data++;
1505 *data = val2;
1506 data++;
1507 } else {
1508 memset(data, 0, 2 * sizeof(u64));
1509 *data += 2;
1510 }
b8ff05a9
DM		 1511}
1512
1513/*
1514 * Return a version number to identify the type of adapter. The scheme is:
1515 * - bits 0..9: chip version
1516 * - bits 10..15: chip revision
835bb606 1517 * - bits 16..23: register dump version
b8ff05a9
DM		 1518 */
1519static inline unsigned int mk_adap_vers(const struct adapter *ap)
1520{
d14807dd
HS		 1521 return CHELSIO_CHIP_VERSION(ap->params.chip) |
1522 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
b8ff05a9
DM		 1523}
1524
1525static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1526 unsigned int end)
1527{
1528 u32 *p = buf + start;
1529
1530 for ( ; start <= end; start += sizeof(u32))
1531 *p++ = t4_read_reg(ap, start);
1532}
1533
1534static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1535 void *buf)
1536{
251f9e88 1537 static const unsigned int t4_reg_ranges[] = {
b8ff05a9
DM		 1538 0x1008, 0x1108,
1539 0x1180, 0x11b4,
1540 0x11fc, 0x123c,
1541 0x1300, 0x173c,
1542 0x1800, 0x18fc,
1543 0x3000, 0x30d8,
1544 0x30e0, 0x5924,
1545 0x5960, 0x59d4,
1546 0x5a00, 0x5af8,
1547 0x6000, 0x6098,
1548 0x6100, 0x6150,
1549 0x6200, 0x6208,
1550 0x6240, 0x6248,
1551 0x6280, 0x6338,
1552 0x6370, 0x638c,
1553 0x6400, 0x643c,
1554 0x6500, 0x6524,
1555 0x6a00, 0x6a38,
1556 0x6a60, 0x6a78,
1557 0x6b00, 0x6b84,
1558 0x6bf0, 0x6c84,
1559 0x6cf0, 0x6d84,
1560 0x6df0, 0x6e84,
1561 0x6ef0, 0x6f84,
1562 0x6ff0, 0x7084,
1563 0x70f0, 0x7184,
1564 0x71f0, 0x7284,
1565 0x72f0, 0x7384,
1566 0x73f0, 0x7450,
1567 0x7500, 0x7530,
1568 0x7600, 0x761c,
1569 0x7680, 0x76cc,
1570 0x7700, 0x7798,
1571 0x77c0, 0x77fc,
1572 0x7900, 0x79fc,
1573 0x7b00, 0x7c38,
1574 0x7d00, 0x7efc,
1575 0x8dc0, 0x8e1c,
1576 0x8e30, 0x8e78,
1577 0x8ea0, 0x8f6c,
1578 0x8fc0, 0x9074,
1579 0x90fc, 0x90fc,
1580 0x9400, 0x9458,
1581 0x9600, 0x96bc,
1582 0x9800, 0x9808,
1583 0x9820, 0x983c,
1584 0x9850, 0x9864,
1585 0x9c00, 0x9c6c,
1586 0x9c80, 0x9cec,
1587 0x9d00, 0x9d6c,
1588 0x9d80, 0x9dec,
1589 0x9e00, 0x9e6c,
1590 0x9e80, 0x9eec,
1591 0x9f00, 0x9f6c,
1592 0x9f80, 0x9fec,
1593 0xd004, 0xd03c,
1594 0xdfc0, 0xdfe0,
1595 0xe000, 0xea7c,
3d9103f8
HS		 1596 0xf000, 0x11110,
1597 0x11118, 0x11190,
835bb606
DM		 1598 0x19040, 0x1906c,
1599 0x19078, 0x19080,
1600 0x1908c, 0x19124,
b8ff05a9
DM		 1601 0x19150, 0x191b0,
1602 0x191d0, 0x191e8,
1603 0x19238, 0x1924c,
1604 0x193f8, 0x19474,
1605 0x19490, 0x194f8,
1606 0x19800, 0x19f30,
1607 0x1a000, 0x1a06c,
1608 0x1a0b0, 0x1a120,
1609 0x1a128, 0x1a138,
1610 0x1a190, 0x1a1c4,
1611 0x1a1fc, 0x1a1fc,
1612 0x1e040, 0x1e04c,
835bb606 1613 0x1e284, 0x1e28c,
b8ff05a9
DM		 1614 0x1e2c0, 0x1e2c0,
1615 0x1e2e0, 0x1e2e0,
1616 0x1e300, 0x1e384,
1617 0x1e3c0, 0x1e3c8,
1618 0x1e440, 0x1e44c,
835bb606 1619 0x1e684, 0x1e68c,
b8ff05a9
DM		 1620 0x1e6c0, 0x1e6c0,
1621 0x1e6e0, 0x1e6e0,
1622 0x1e700, 0x1e784,
1623 0x1e7c0, 0x1e7c8,
1624 0x1e840, 0x1e84c,
835bb606 1625 0x1ea84, 0x1ea8c,
b8ff05a9
DM		 1626 0x1eac0, 0x1eac0,
1627 0x1eae0, 0x1eae0,
1628 0x1eb00, 0x1eb84,
1629 0x1ebc0, 0x1ebc8,
1630 0x1ec40, 0x1ec4c,
835bb606 1631 0x1ee84, 0x1ee8c,
b8ff05a9
DM		 1632 0x1eec0, 0x1eec0,
1633 0x1eee0, 0x1eee0,
1634 0x1ef00, 0x1ef84,
1635 0x1efc0, 0x1efc8,
1636 0x1f040, 0x1f04c,
835bb606 1637 0x1f284, 0x1f28c,
b8ff05a9
DM		 1638 0x1f2c0, 0x1f2c0,
1639 0x1f2e0, 0x1f2e0,
1640 0x1f300, 0x1f384,
1641 0x1f3c0, 0x1f3c8,
1642 0x1f440, 0x1f44c,
835bb606 1643 0x1f684, 0x1f68c,
b8ff05a9
DM		 1644 0x1f6c0, 0x1f6c0,
1645 0x1f6e0, 0x1f6e0,
1646 0x1f700, 0x1f784,
1647 0x1f7c0, 0x1f7c8,
1648 0x1f840, 0x1f84c,
835bb606 1649 0x1fa84, 0x1fa8c,
b8ff05a9
DM		 1650 0x1fac0, 0x1fac0,
1651 0x1fae0, 0x1fae0,
1652 0x1fb00, 0x1fb84,
1653 0x1fbc0, 0x1fbc8,
1654 0x1fc40, 0x1fc4c,
835bb606 1655 0x1fe84, 0x1fe8c,
b8ff05a9
DM		 1656 0x1fec0, 0x1fec0,
1657 0x1fee0, 0x1fee0,
1658 0x1ff00, 0x1ff84,
1659 0x1ffc0, 0x1ffc8,
1660 0x20000, 0x2002c,
1661 0x20100, 0x2013c,
1662 0x20190, 0x201c8,
1663 0x20200, 0x20318,
1664 0x20400, 0x20528,
1665 0x20540, 0x20614,
1666 0x21000, 0x21040,
1667 0x2104c, 0x21060,
1668 0x210c0, 0x210ec,
1669 0x21200, 0x21268,
1670 0x21270, 0x21284,
1671 0x212fc, 0x21388,
1672 0x21400, 0x21404,
1673 0x21500, 0x21518,
1674 0x2152c, 0x2153c,
1675 0x21550, 0x21554,
1676 0x21600, 0x21600,
1677 0x21608, 0x21628,
1678 0x21630, 0x2163c,
1679 0x21700, 0x2171c,
1680 0x21780, 0x2178c,
1681 0x21800, 0x21c38,
1682 0x21c80, 0x21d7c,
1683 0x21e00, 0x21e04,
1684 0x22000, 0x2202c,
1685 0x22100, 0x2213c,
1686 0x22190, 0x221c8,
1687 0x22200, 0x22318,
1688 0x22400, 0x22528,
1689 0x22540, 0x22614,
1690 0x23000, 0x23040,
1691 0x2304c, 0x23060,
1692 0x230c0, 0x230ec,
1693 0x23200, 0x23268,
1694 0x23270, 0x23284,
1695 0x232fc, 0x23388,
1696 0x23400, 0x23404,
1697 0x23500, 0x23518,
1698 0x2352c, 0x2353c,
1699 0x23550, 0x23554,
1700 0x23600, 0x23600,
1701 0x23608, 0x23628,
1702 0x23630, 0x2363c,
1703 0x23700, 0x2371c,
1704 0x23780, 0x2378c,
1705 0x23800, 0x23c38,
1706 0x23c80, 0x23d7c,
1707 0x23e00, 0x23e04,
1708 0x24000, 0x2402c,
1709 0x24100, 0x2413c,
1710 0x24190, 0x241c8,
1711 0x24200, 0x24318,
1712 0x24400, 0x24528,
1713 0x24540, 0x24614,
1714 0x25000, 0x25040,
1715 0x2504c, 0x25060,
1716 0x250c0, 0x250ec,
1717 0x25200, 0x25268,
1718 0x25270, 0x25284,
1719 0x252fc, 0x25388,
1720 0x25400, 0x25404,
1721 0x25500, 0x25518,
1722 0x2552c, 0x2553c,
1723 0x25550, 0x25554,
1724 0x25600, 0x25600,
1725 0x25608, 0x25628,
1726 0x25630, 0x2563c,
1727 0x25700, 0x2571c,
1728 0x25780, 0x2578c,
1729 0x25800, 0x25c38,
1730 0x25c80, 0x25d7c,
1731 0x25e00, 0x25e04,
1732 0x26000, 0x2602c,
1733 0x26100, 0x2613c,
1734 0x26190, 0x261c8,
1735 0x26200, 0x26318,
1736 0x26400, 0x26528,
1737 0x26540, 0x26614,
1738 0x27000, 0x27040,
1739 0x2704c, 0x27060,
1740 0x270c0, 0x270ec,
1741 0x27200, 0x27268,
1742 0x27270, 0x27284,
1743 0x272fc, 0x27388,
1744 0x27400, 0x27404,
1745 0x27500, 0x27518,
1746 0x2752c, 0x2753c,
1747 0x27550, 0x27554,
1748 0x27600, 0x27600,
1749 0x27608, 0x27628,
1750 0x27630, 0x2763c,
1751 0x27700, 0x2771c,
1752 0x27780, 0x2778c,
1753 0x27800, 0x27c38,
1754 0x27c80, 0x27d7c,
1755 0x27e00, 0x27e04
1756 };
1757
251f9e88
SR		 1758 static const unsigned int t5_reg_ranges[] = {
1759 0x1008, 0x1148,
1760 0x1180, 0x11b4,
1761 0x11fc, 0x123c,
1762 0x1280, 0x173c,
1763 0x1800, 0x18fc,
1764 0x3000, 0x3028,
1765 0x3060, 0x30d8,
1766 0x30e0, 0x30fc,
1767 0x3140, 0x357c,
1768 0x35a8, 0x35cc,
1769 0x35ec, 0x35ec,
1770 0x3600, 0x5624,
1771 0x56cc, 0x575c,
1772 0x580c, 0x5814,
1773 0x5890, 0x58bc,
1774 0x5940, 0x59dc,
1775 0x59fc, 0x5a18,
1776 0x5a60, 0x5a9c,
1777 0x5b9c, 0x5bfc,
1778 0x6000, 0x6040,
1779 0x6058, 0x614c,
1780 0x7700, 0x7798,
1781 0x77c0, 0x78fc,
1782 0x7b00, 0x7c54,
1783 0x7d00, 0x7efc,
1784 0x8dc0, 0x8de0,
1785 0x8df8, 0x8e84,
1786 0x8ea0, 0x8f84,
1787 0x8fc0, 0x90f8,
1788 0x9400, 0x9470,
1789 0x9600, 0x96f4,
1790 0x9800, 0x9808,
1791 0x9820, 0x983c,
1792 0x9850, 0x9864,
1793 0x9c00, 0x9c6c,
1794 0x9c80, 0x9cec,
1795 0x9d00, 0x9d6c,
1796 0x9d80, 0x9dec,
1797 0x9e00, 0x9e6c,
1798 0x9e80, 0x9eec,
1799 0x9f00, 0x9f6c,
1800 0x9f80, 0xa020,
1801 0xd004, 0xd03c,
1802 0xdfc0, 0xdfe0,
1803 0xe000, 0x11088,
3d9103f8
HS		 1804 0x1109c, 0x11110,
1805 0x11118, 0x1117c,
251f9e88
SR		 1806 0x11190, 0x11204,
1807 0x19040, 0x1906c,
1808 0x19078, 0x19080,
1809 0x1908c, 0x19124,
1810 0x19150, 0x191b0,
1811 0x191d0, 0x191e8,
1812 0x19238, 0x19290,
1813 0x193f8, 0x19474,
1814 0x19490, 0x194cc,
1815 0x194f0, 0x194f8,
1816 0x19c00, 0x19c60,
1817 0x19c94, 0x19e10,
1818 0x19e50, 0x19f34,
1819 0x19f40, 0x19f50,
1820 0x19f90, 0x19fe4,
1821 0x1a000, 0x1a06c,
1822 0x1a0b0, 0x1a120,
1823 0x1a128, 0x1a138,
1824 0x1a190, 0x1a1c4,
1825 0x1a1fc, 0x1a1fc,
1826 0x1e008, 0x1e00c,
1827 0x1e040, 0x1e04c,
1828 0x1e284, 0x1e290,
1829 0x1e2c0, 0x1e2c0,
1830 0x1e2e0, 0x1e2e0,
1831 0x1e300, 0x1e384,
1832 0x1e3c0, 0x1e3c8,
1833 0x1e408, 0x1e40c,
1834 0x1e440, 0x1e44c,
1835 0x1e684, 0x1e690,
1836 0x1e6c0, 0x1e6c0,
1837 0x1e6e0, 0x1e6e0,
1838 0x1e700, 0x1e784,
1839 0x1e7c0, 0x1e7c8,
1840 0x1e808, 0x1e80c,
1841 0x1e840, 0x1e84c,
1842 0x1ea84, 0x1ea90,
1843 0x1eac0, 0x1eac0,
1844 0x1eae0, 0x1eae0,
1845 0x1eb00, 0x1eb84,
1846 0x1ebc0, 0x1ebc8,
1847 0x1ec08, 0x1ec0c,
1848 0x1ec40, 0x1ec4c,
1849 0x1ee84, 0x1ee90,
1850 0x1eec0, 0x1eec0,
1851 0x1eee0, 0x1eee0,
1852 0x1ef00, 0x1ef84,
1853 0x1efc0, 0x1efc8,
1854 0x1f008, 0x1f00c,
1855 0x1f040, 0x1f04c,
1856 0x1f284, 0x1f290,
1857 0x1f2c0, 0x1f2c0,
1858 0x1f2e0, 0x1f2e0,
1859 0x1f300, 0x1f384,
1860 0x1f3c0, 0x1f3c8,
1861 0x1f408, 0x1f40c,
1862 0x1f440, 0x1f44c,
1863 0x1f684, 0x1f690,
1864 0x1f6c0, 0x1f6c0,
1865 0x1f6e0, 0x1f6e0,
1866 0x1f700, 0x1f784,
1867 0x1f7c0, 0x1f7c8,
1868 0x1f808, 0x1f80c,
1869 0x1f840, 0x1f84c,
1870 0x1fa84, 0x1fa90,
1871 0x1fac0, 0x1fac0,
1872 0x1fae0, 0x1fae0,
1873 0x1fb00, 0x1fb84,
1874 0x1fbc0, 0x1fbc8,
1875 0x1fc08, 0x1fc0c,
1876 0x1fc40, 0x1fc4c,
1877 0x1fe84, 0x1fe90,
1878 0x1fec0, 0x1fec0,
1879 0x1fee0, 0x1fee0,
1880 0x1ff00, 0x1ff84,
1881 0x1ffc0, 0x1ffc8,
1882 0x30000, 0x30030,
1883 0x30100, 0x30144,
1884 0x30190, 0x301d0,
1885 0x30200, 0x30318,
1886 0x30400, 0x3052c,
1887 0x30540, 0x3061c,
1888 0x30800, 0x30834,
1889 0x308c0, 0x30908,
1890 0x30910, 0x309ac,
1891 0x30a00, 0x30a04,
1892 0x30a0c, 0x30a2c,
1893 0x30a44, 0x30a50,
1894 0x30a74, 0x30c24,
1895 0x30d08, 0x30d14,
1896 0x30d1c, 0x30d20,
1897 0x30d3c, 0x30d50,
1898 0x31200, 0x3120c,
1899 0x31220, 0x31220,
1900 0x31240, 0x31240,
1901 0x31600, 0x31600,
1902 0x31608, 0x3160c,
1903 0x31a00, 0x31a1c,
1904 0x31e04, 0x31e20,
1905 0x31e38, 0x31e3c,
1906 0x31e80, 0x31e80,
1907 0x31e88, 0x31ea8,
1908 0x31eb0, 0x31eb4,
1909 0x31ec8, 0x31ed4,
1910 0x31fb8, 0x32004,
1911 0x32208, 0x3223c,
1912 0x32600, 0x32630,
1913 0x32a00, 0x32abc,
1914 0x32b00, 0x32b70,
1915 0x33000, 0x33048,
1916 0x33060, 0x3309c,
1917 0x330f0, 0x33148,
1918 0x33160, 0x3319c,
1919 0x331f0, 0x332e4,
1920 0x332f8, 0x333e4,
1921 0x333f8, 0x33448,
1922 0x33460, 0x3349c,
1923 0x334f0, 0x33548,
1924 0x33560, 0x3359c,
1925 0x335f0, 0x336e4,
1926 0x336f8, 0x337e4,
1927 0x337f8, 0x337fc,
1928 0x33814, 0x33814,
1929 0x3382c, 0x3382c,
1930 0x33880, 0x3388c,
1931 0x338e8, 0x338ec,
1932 0x33900, 0x33948,
1933 0x33960, 0x3399c,
1934 0x339f0, 0x33ae4,
1935 0x33af8, 0x33b10,
1936 0x33b28, 0x33b28,
1937 0x33b3c, 0x33b50,
1938 0x33bf0, 0x33c10,
1939 0x33c28, 0x33c28,
1940 0x33c3c, 0x33c50,
1941 0x33cf0, 0x33cfc,
1942 0x34000, 0x34030,
1943 0x34100, 0x34144,
1944 0x34190, 0x341d0,
1945 0x34200, 0x34318,
1946 0x34400, 0x3452c,
1947 0x34540, 0x3461c,
1948 0x34800, 0x34834,
1949 0x348c0, 0x34908,
1950 0x34910, 0x349ac,
1951 0x34a00, 0x34a04,
1952 0x34a0c, 0x34a2c,
1953 0x34a44, 0x34a50,
1954 0x34a74, 0x34c24,
1955 0x34d08, 0x34d14,
1956 0x34d1c, 0x34d20,
1957 0x34d3c, 0x34d50,
1958 0x35200, 0x3520c,
1959 0x35220, 0x35220,
1960 0x35240, 0x35240,
1961 0x35600, 0x35600,
1962 0x35608, 0x3560c,
1963 0x35a00, 0x35a1c,
1964 0x35e04, 0x35e20,
1965 0x35e38, 0x35e3c,
1966 0x35e80, 0x35e80,
1967 0x35e88, 0x35ea8,
1968 0x35eb0, 0x35eb4,
1969 0x35ec8, 0x35ed4,
1970 0x35fb8, 0x36004,
1971 0x36208, 0x3623c,
1972 0x36600, 0x36630,
1973 0x36a00, 0x36abc,
1974 0x36b00, 0x36b70,
1975 0x37000, 0x37048,
1976 0x37060, 0x3709c,
1977 0x370f0, 0x37148,
1978 0x37160, 0x3719c,
1979 0x371f0, 0x372e4,
1980 0x372f8, 0x373e4,
1981 0x373f8, 0x37448,
1982 0x37460, 0x3749c,
1983 0x374f0, 0x37548,
1984 0x37560, 0x3759c,
1985 0x375f0, 0x376e4,
1986 0x376f8, 0x377e4,
1987 0x377f8, 0x377fc,
1988 0x37814, 0x37814,
1989 0x3782c, 0x3782c,
1990 0x37880, 0x3788c,
1991 0x378e8, 0x378ec,
1992 0x37900, 0x37948,
1993 0x37960, 0x3799c,
1994 0x379f0, 0x37ae4,
1995 0x37af8, 0x37b10,
1996 0x37b28, 0x37b28,
1997 0x37b3c, 0x37b50,
1998 0x37bf0, 0x37c10,
1999 0x37c28, 0x37c28,
2000 0x37c3c, 0x37c50,
2001 0x37cf0, 0x37cfc,
2002 0x38000, 0x38030,
2003 0x38100, 0x38144,
2004 0x38190, 0x381d0,
2005 0x38200, 0x38318,
2006 0x38400, 0x3852c,
2007 0x38540, 0x3861c,
2008 0x38800, 0x38834,
2009 0x388c0, 0x38908,
2010 0x38910, 0x389ac,
2011 0x38a00, 0x38a04,
2012 0x38a0c, 0x38a2c,
2013 0x38a44, 0x38a50,
2014 0x38a74, 0x38c24,
2015 0x38d08, 0x38d14,
2016 0x38d1c, 0x38d20,
2017 0x38d3c, 0x38d50,
2018 0x39200, 0x3920c,
2019 0x39220, 0x39220,
2020 0x39240, 0x39240,
2021 0x39600, 0x39600,
2022 0x39608, 0x3960c,
2023 0x39a00, 0x39a1c,
2024 0x39e04, 0x39e20,
2025 0x39e38, 0x39e3c,
2026 0x39e80, 0x39e80,
2027 0x39e88, 0x39ea8,
2028 0x39eb0, 0x39eb4,
2029 0x39ec8, 0x39ed4,
2030 0x39fb8, 0x3a004,
2031 0x3a208, 0x3a23c,
2032 0x3a600, 0x3a630,
2033 0x3aa00, 0x3aabc,
2034 0x3ab00, 0x3ab70,
2035 0x3b000, 0x3b048,
2036 0x3b060, 0x3b09c,
2037 0x3b0f0, 0x3b148,
2038 0x3b160, 0x3b19c,
2039 0x3b1f0, 0x3b2e4,
2040 0x3b2f8, 0x3b3e4,
2041 0x3b3f8, 0x3b448,
2042 0x3b460, 0x3b49c,
2043 0x3b4f0, 0x3b548,
2044 0x3b560, 0x3b59c,
2045 0x3b5f0, 0x3b6e4,
2046 0x3b6f8, 0x3b7e4,
2047 0x3b7f8, 0x3b7fc,
2048 0x3b814, 0x3b814,
2049 0x3b82c, 0x3b82c,
2050 0x3b880, 0x3b88c,
2051 0x3b8e8, 0x3b8ec,
2052 0x3b900, 0x3b948,
2053 0x3b960, 0x3b99c,
2054 0x3b9f0, 0x3bae4,
2055 0x3baf8, 0x3bb10,
2056 0x3bb28, 0x3bb28,
2057 0x3bb3c, 0x3bb50,
2058 0x3bbf0, 0x3bc10,
2059 0x3bc28, 0x3bc28,
2060 0x3bc3c, 0x3bc50,
2061 0x3bcf0, 0x3bcfc,
2062 0x3c000, 0x3c030,
2063 0x3c100, 0x3c144,
2064 0x3c190, 0x3c1d0,
2065 0x3c200, 0x3c318,
2066 0x3c400, 0x3c52c,
2067 0x3c540, 0x3c61c,
2068 0x3c800, 0x3c834,
2069 0x3c8c0, 0x3c908,
2070 0x3c910, 0x3c9ac,
2071 0x3ca00, 0x3ca04,
2072 0x3ca0c, 0x3ca2c,
2073 0x3ca44, 0x3ca50,
2074 0x3ca74, 0x3cc24,
2075 0x3cd08, 0x3cd14,
2076 0x3cd1c, 0x3cd20,
2077 0x3cd3c, 0x3cd50,
2078 0x3d200, 0x3d20c,
2079 0x3d220, 0x3d220,
2080 0x3d240, 0x3d240,
2081 0x3d600, 0x3d600,
2082 0x3d608, 0x3d60c,
2083 0x3da00, 0x3da1c,
2084 0x3de04, 0x3de20,
2085 0x3de38, 0x3de3c,
2086 0x3de80, 0x3de80,
2087 0x3de88, 0x3dea8,
2088 0x3deb0, 0x3deb4,
2089 0x3dec8, 0x3ded4,
2090 0x3dfb8, 0x3e004,
2091 0x3e208, 0x3e23c,
2092 0x3e600, 0x3e630,
2093 0x3ea00, 0x3eabc,
2094 0x3eb00, 0x3eb70,
2095 0x3f000, 0x3f048,
2096 0x3f060, 0x3f09c,
2097 0x3f0f0, 0x3f148,
2098 0x3f160, 0x3f19c,
2099 0x3f1f0, 0x3f2e4,
2100 0x3f2f8, 0x3f3e4,
2101 0x3f3f8, 0x3f448,
2102 0x3f460, 0x3f49c,
2103 0x3f4f0, 0x3f548,
2104 0x3f560, 0x3f59c,
2105 0x3f5f0, 0x3f6e4,
2106 0x3f6f8, 0x3f7e4,
2107 0x3f7f8, 0x3f7fc,
2108 0x3f814, 0x3f814,
2109 0x3f82c, 0x3f82c,
2110 0x3f880, 0x3f88c,
2111 0x3f8e8, 0x3f8ec,
2112 0x3f900, 0x3f948,
2113 0x3f960, 0x3f99c,
2114 0x3f9f0, 0x3fae4,
2115 0x3faf8, 0x3fb10,
2116 0x3fb28, 0x3fb28,
2117 0x3fb3c, 0x3fb50,
2118 0x3fbf0, 0x3fc10,
2119 0x3fc28, 0x3fc28,
2120 0x3fc3c, 0x3fc50,
2121 0x3fcf0, 0x3fcfc,
2122 0x40000, 0x4000c,
2123 0x40040, 0x40068,
2124 0x40080, 0x40144,
2125 0x40180, 0x4018c,
2126 0x40200, 0x40298,
2127 0x402ac, 0x4033c,
2128 0x403f8, 0x403fc,
c1f49e3e 2129 0x41304, 0x413c4,
251f9e88
SR		 2130 0x41400, 0x4141c,
2131 0x41480, 0x414d0,
2132 0x44000, 0x44078,
2133 0x440c0, 0x44278,
2134 0x442c0, 0x44478,
2135 0x444c0, 0x44678,
2136 0x446c0, 0x44878,
2137 0x448c0, 0x449fc,
2138 0x45000, 0x45068,
2139 0x45080, 0x45084,
2140 0x450a0, 0x450b0,
2141 0x45200, 0x45268,
2142 0x45280, 0x45284,
2143 0x452a0, 0x452b0,
2144 0x460c0, 0x460e4,
2145 0x47000, 0x4708c,
2146 0x47200, 0x47250,
2147 0x47400, 0x47420,
2148 0x47600, 0x47618,
2149 0x47800, 0x47814,
2150 0x48000, 0x4800c,
2151 0x48040, 0x48068,
2152 0x48080, 0x48144,
2153 0x48180, 0x4818c,
2154 0x48200, 0x48298,
2155 0x482ac, 0x4833c,
2156 0x483f8, 0x483fc,
c1f49e3e 2157 0x49304, 0x493c4,
251f9e88
SR		 2158 0x49400, 0x4941c,
2159 0x49480, 0x494d0,
2160 0x4c000, 0x4c078,
2161 0x4c0c0, 0x4c278,
2162 0x4c2c0, 0x4c478,
2163 0x4c4c0, 0x4c678,
2164 0x4c6c0, 0x4c878,
2165 0x4c8c0, 0x4c9fc,
2166 0x4d000, 0x4d068,
2167 0x4d080, 0x4d084,
2168 0x4d0a0, 0x4d0b0,
2169 0x4d200, 0x4d268,
2170 0x4d280, 0x4d284,
2171 0x4d2a0, 0x4d2b0,
2172 0x4e0c0, 0x4e0e4,
2173 0x4f000, 0x4f08c,
2174 0x4f200, 0x4f250,
2175 0x4f400, 0x4f420,
2176 0x4f600, 0x4f618,
2177 0x4f800, 0x4f814,
2178 0x50000, 0x500cc,
2179 0x50400, 0x50400,
2180 0x50800, 0x508cc,
2181 0x50c00, 0x50c00,
2182 0x51000, 0x5101c,
2183 0x51300, 0x51308,
2184 };
2185
b8ff05a9
DM		 2186 int i;
2187 struct adapter *ap = netdev2adap(dev);
251f9e88
SR		 2188 static const unsigned int *reg_ranges;
2189 int arr_size = 0, buf_size = 0;
2190
d14807dd 2191 if (is_t4(ap->params.chip)) {
251f9e88
SR		 2192 reg_ranges = &t4_reg_ranges[0];
2193 arr_size = ARRAY_SIZE(t4_reg_ranges);
2194 buf_size = T4_REGMAP_SIZE;
2195 } else {
2196 reg_ranges = &t5_reg_ranges[0];
2197 arr_size = ARRAY_SIZE(t5_reg_ranges);
2198 buf_size = T5_REGMAP_SIZE;
2199 }
b8ff05a9
DM		 2200
2201 regs->version = mk_adap_vers(ap);
2202
251f9e88
SR		 2203 memset(buf, 0, buf_size);
2204 for (i = 0; i < arr_size; i += 2)
b8ff05a9
DM		 2205 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2206}
2207
2208static int restart_autoneg(struct net_device *dev)
2209{
2210 struct port_info *p = netdev_priv(dev);
2211
2212 if (!netif_running(dev))
2213 return -EAGAIN;
2214 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2215 return -EINVAL;
060e0c75 2216 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
b8ff05a9
DM		 2217 return 0;
2218}
2219
c5e06360
DM		 2220static int identify_port(struct net_device *dev,
2221 enum ethtool_phys_id_state state)
b8ff05a9 2222{
c5e06360 2223 unsigned int val;
060e0c75
DM		 2224 struct adapter *adap = netdev2adap(dev);
2225
c5e06360
DM		 2226 if (state == ETHTOOL_ID_ACTIVE)
2227 val = 0xffff;
2228 else if (state == ETHTOOL_ID_INACTIVE)
2229 val = 0;
2230 else
2231 return -EINVAL;
b8ff05a9 2232
c5e06360 2233 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
b8ff05a9
DM		 2234}
2235
40e9de4b 2236static unsigned int from_fw_linkcaps(enum fw_port_type type, unsigned int caps)
b8ff05a9
DM		 2237{
2238 unsigned int v = 0;
2239
a0881cab
DM		 2240 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2241 type == FW_PORT_TYPE_BT_XAUI) {
b8ff05a9
DM		 2242 v |= SUPPORTED_TP;
2243 if (caps & FW_PORT_CAP_SPEED_100M)
2244 v |= SUPPORTED_100baseT_Full;
2245 if (caps & FW_PORT_CAP_SPEED_1G)
2246 v |= SUPPORTED_1000baseT_Full;
2247 if (caps & FW_PORT_CAP_SPEED_10G)
2248 v |= SUPPORTED_10000baseT_Full;
2249 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2250 v |= SUPPORTED_Backplane;
2251 if (caps & FW_PORT_CAP_SPEED_1G)
2252 v |= SUPPORTED_1000baseKX_Full;
2253 if (caps & FW_PORT_CAP_SPEED_10G)
2254 v |= SUPPORTED_10000baseKX4_Full;
2255 } else if (type == FW_PORT_TYPE_KR)
2256 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
a0881cab 2257 else if (type == FW_PORT_TYPE_BP_AP)
7d5e77aa
DM		 2258 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2259 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2260 else if (type == FW_PORT_TYPE_BP4_AP)
2261 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2262 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2263 SUPPORTED_10000baseKX4_Full;
a0881cab 2264 else if (type == FW_PORT_TYPE_FIBER_XFI ||
40e9de4b
HS		 2265 type == FW_PORT_TYPE_FIBER_XAUI ||
2266 type == FW_PORT_TYPE_SFP ||
2267 type == FW_PORT_TYPE_QSFP_10G ||
2268 type == FW_PORT_TYPE_QSA) {
b8ff05a9 2269 v |= SUPPORTED_FIBRE;
4c2d5186
HS		 2270 if (caps & FW_PORT_CAP_SPEED_1G)
2271 v |= SUPPORTED_1000baseT_Full;
2272 if (caps & FW_PORT_CAP_SPEED_10G)
2273 v |= SUPPORTED_10000baseT_Full;
40e9de4b
HS		 2274 } else if (type == FW_PORT_TYPE_BP40_BA ||
2275 type == FW_PORT_TYPE_QSFP) {
72aca4bf 2276 v |= SUPPORTED_40000baseSR4_Full;
40e9de4b
HS		 2277 v |= SUPPORTED_FIBRE;
2278 }
b8ff05a9
DM		 2279
2280 if (caps & FW_PORT_CAP_ANEG)
2281 v |= SUPPORTED_Autoneg;
2282 return v;
2283}
2284
2285static unsigned int to_fw_linkcaps(unsigned int caps)
2286{
2287 unsigned int v = 0;
2288
2289 if (caps & ADVERTISED_100baseT_Full)
2290 v |= FW_PORT_CAP_SPEED_100M;
2291 if (caps & ADVERTISED_1000baseT_Full)
2292 v |= FW_PORT_CAP_SPEED_1G;
2293 if (caps & ADVERTISED_10000baseT_Full)
2294 v |= FW_PORT_CAP_SPEED_10G;
72aca4bf
KS		 2295 if (caps & ADVERTISED_40000baseSR4_Full)
2296 v |= FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2297 return v;
2298}
2299
2300static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2301{
2302 const struct port_info *p = netdev_priv(dev);
2303
2304 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
a0881cab 2305 p->port_type == FW_PORT_TYPE_BT_XFI ||
b8ff05a9
DM		 2306 p->port_type == FW_PORT_TYPE_BT_XAUI)
2307 cmd->port = PORT_TP;
a0881cab
DM		 2308 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2309 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
b8ff05a9 2310 cmd->port = PORT_FIBRE;
3e00a509
HS		 2311 else if (p->port_type == FW_PORT_TYPE_SFP ||
2312 p->port_type == FW_PORT_TYPE_QSFP_10G ||
40e9de4b 2313 p->port_type == FW_PORT_TYPE_QSA ||
3e00a509
HS		 2314 p->port_type == FW_PORT_TYPE_QSFP) {
2315 if (p->mod_type == FW_PORT_MOD_TYPE_LR ||
2316 p->mod_type == FW_PORT_MOD_TYPE_SR ||
2317 p->mod_type == FW_PORT_MOD_TYPE_ER ||
2318 p->mod_type == FW_PORT_MOD_TYPE_LRM)
2319 cmd->port = PORT_FIBRE;
2320 else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2321 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
a0881cab
DM		 2322 cmd->port = PORT_DA;
2323 else
3e00a509 2324 cmd->port = PORT_OTHER;
a0881cab 2325 } else
b8ff05a9
DM		 2326 cmd->port = PORT_OTHER;
2327
2328 if (p->mdio_addr >= 0) {
2329 cmd->phy_address = p->mdio_addr;
2330 cmd->transceiver = XCVR_EXTERNAL;
2331 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2332 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2333 } else {
2334 cmd->phy_address = 0; /* not really, but no better option */
2335 cmd->transceiver = XCVR_INTERNAL;
2336 cmd->mdio_support = 0;
2337 }
2338
2339 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2340 cmd->advertising = from_fw_linkcaps(p->port_type,
2341 p->link_cfg.advertising);
70739497
DD		 2342 ethtool_cmd_speed_set(cmd,
2343 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
b8ff05a9
DM		 2344 cmd->duplex = DUPLEX_FULL;
2345 cmd->autoneg = p->link_cfg.autoneg;
2346 cmd->maxtxpkt = 0;
2347 cmd->maxrxpkt = 0;
2348 return 0;
2349}
2350
2351static unsigned int speed_to_caps(int speed)
2352{
e8b39015 2353 if (speed == 100)
b8ff05a9 2354 return FW_PORT_CAP_SPEED_100M;
e8b39015 2355 if (speed == 1000)
b8ff05a9 2356 return FW_PORT_CAP_SPEED_1G;
e8b39015 2357 if (speed == 10000)
b8ff05a9 2358 return FW_PORT_CAP_SPEED_10G;
e8b39015 2359 if (speed == 40000)
72aca4bf 2360 return FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2361 return 0;
2362}
2363
2364static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2365{
2366 unsigned int cap;
2367 struct port_info *p = netdev_priv(dev);
2368 struct link_config *lc = &p->link_cfg;
25db0338 2369 u32 speed = ethtool_cmd_speed(cmd);
b8ff05a9
DM		 2370
2371 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
2372 return -EINVAL;
2373
2374 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2375 /*
2376 * PHY offers a single speed. See if that's what's
2377 * being requested.
2378 */
2379 if (cmd->autoneg == AUTONEG_DISABLE &&
25db0338
DD		 2380 (lc->supported & speed_to_caps(speed)))
2381 return 0;
b8ff05a9
DM		 2382 return -EINVAL;
2383 }
2384
2385 if (cmd->autoneg == AUTONEG_DISABLE) {
25db0338 2386 cap = speed_to_caps(speed);
b8ff05a9 2387
72aca4bf 2388 if (!(lc->supported & cap) ||
e8b39015
BH		 2389 (speed == 1000) ||
2390 (speed == 10000) ||
72aca4bf 2391 (speed == 40000))
b8ff05a9
DM		 2392 return -EINVAL;
2393 lc->requested_speed = cap;
2394 lc->advertising = 0;
2395 } else {
2396 cap = to_fw_linkcaps(cmd->advertising);
2397 if (!(lc->supported & cap))
2398 return -EINVAL;
2399 lc->requested_speed = 0;
2400 lc->advertising = cap | FW_PORT_CAP_ANEG;
2401 }
2402 lc->autoneg = cmd->autoneg;
2403
2404 if (netif_running(dev))
060e0c75
DM		 2405 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2406 lc);
b8ff05a9
DM		 2407 return 0;
2408}
2409
2410static void get_pauseparam(struct net_device *dev,
2411 struct ethtool_pauseparam *epause)
2412{
2413 struct port_info *p = netdev_priv(dev);
2414
2415 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2416 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2417 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2418}
2419
2420static int set_pauseparam(struct net_device *dev,
2421 struct ethtool_pauseparam *epause)
2422{
2423 struct port_info *p = netdev_priv(dev);
2424 struct link_config *lc = &p->link_cfg;
2425
2426 if (epause->autoneg == AUTONEG_DISABLE)
2427 lc->requested_fc = 0;
2428 else if (lc->supported & FW_PORT_CAP_ANEG)
2429 lc->requested_fc = PAUSE_AUTONEG;
2430 else
2431 return -EINVAL;
2432
2433 if (epause->rx_pause)
2434 lc->requested_fc |= PAUSE_RX;
2435 if (epause->tx_pause)
2436 lc->requested_fc |= PAUSE_TX;
2437 if (netif_running(dev))
060e0c75
DM		 2438 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2439 lc);
b8ff05a9
DM		 2440 return 0;
2441}
2442
b8ff05a9
DM		 2443static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2444{
2445 const struct port_info *pi = netdev_priv(dev);
2446 const struct sge *s = &pi->adapter->sge;
2447
2448 e->rx_max_pending = MAX_RX_BUFFERS;
2449 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2450 e->rx_jumbo_max_pending = 0;
2451 e->tx_max_pending = MAX_TXQ_ENTRIES;
2452
2453 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2454 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2455 e->rx_jumbo_pending = 0;
2456 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2457}
2458
2459static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2460{
2461 int i;
2462 const struct port_info *pi = netdev_priv(dev);
2463 struct adapter *adapter = pi->adapter;
2464 struct sge *s = &adapter->sge;
2465
2466 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2467 e->tx_pending > MAX_TXQ_ENTRIES ||
2468 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2469 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2470 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2471 return -EINVAL;
2472
2473 if (adapter->flags & FULL_INIT_DONE)
2474 return -EBUSY;
2475
2476 for (i = 0; i < pi->nqsets; ++i) {
2477 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2478 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2479 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2480 }
2481 return 0;
2482}
2483
2484static int closest_timer(const struct sge *s, int time)
2485{
2486 int i, delta, match = 0, min_delta = INT_MAX;
2487
2488 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2489 delta = time - s->timer_val[i];
2490 if (delta < 0)
2491 delta = -delta;
2492 if (delta < min_delta) {
2493 min_delta = delta;
2494 match = i;
2495 }
2496 }
2497 return match;
2498}
2499
2500static int closest_thres(const struct sge *s, int thres)
2501{
2502 int i, delta, match = 0, min_delta = INT_MAX;
2503
2504 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2505 delta = thres - s->counter_val[i];
2506 if (delta < 0)
2507 delta = -delta;
2508 if (delta < min_delta) {
2509 min_delta = delta;
2510 match = i;
2511 }
2512 }
2513 return match;
2514}
2515
2516/*
2517 * Return a queue's interrupt hold-off time in us. 0 means no timer.
2518 */
dc9daab2
HS		 2519unsigned int qtimer_val(const struct adapter *adap,
2520 const struct sge_rspq *q)
b8ff05a9
DM		 2521{
2522 unsigned int idx = q->intr_params >> 1;
2523
2524 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2525}
2526
2527/**
c887ad0e 2528 * set_rspq_intr_params - set a queue's interrupt holdoff parameters
b8ff05a9
DM		 2529 * @q: the Rx queue
2530 * @us: the hold-off time in us, or 0 to disable timer
2531 * @cnt: the hold-off packet count, or 0 to disable counter
2532 *
2533 * Sets an Rx queue's interrupt hold-off time and packet count. At least
2534 * one of the two needs to be enabled for the queue to generate interrupts.
2535 */
c887ad0e
HS		 2536static int set_rspq_intr_params(struct sge_rspq *q,
2537 unsigned int us, unsigned int cnt)
b8ff05a9 2538{
c887ad0e
HS		 2539 struct adapter *adap = q->adap;
2540
b8ff05a9
DM		 2541 if ((us | cnt) == 0)
2542 cnt = 1;
2543
2544 if (cnt) {
2545 int err;
2546 u32 v, new_idx;
2547
2548 new_idx = closest_thres(&adap->sge, cnt);
2549 if (q->desc && q->pktcnt_idx != new_idx) {
2550 /* the queue has already been created, update it */
5167865a
HS		 2551 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
2552 FW_PARAMS_PARAM_X_V(
2553 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2554 FW_PARAMS_PARAM_YZ_V(q->cntxt_id);
060e0c75
DM		 2555 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2556 &new_idx);
b8ff05a9
DM		 2557 if (err)
2558 return err;
2559 }
2560 q->pktcnt_idx = new_idx;
2561 }
2562
2563 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2564 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2565 return 0;
2566}
2567
c887ad0e
HS		 2568/**
2569 * set_rx_intr_params - set a net devices's RX interrupt holdoff paramete!
2570 * @dev: the network device
2571 * @us: the hold-off time in us, or 0 to disable timer
2572 * @cnt: the hold-off packet count, or 0 to disable counter
2573 *
2574 * Set the RX interrupt hold-off parameters for a network device.
2575 */
2576static int set_rx_intr_params(struct net_device *dev,
2577 unsigned int us, unsigned int cnt)
b8ff05a9 2578{
c887ad0e
HS		 2579 int i, err;
2580 struct port_info *pi = netdev_priv(dev);
b8ff05a9 2581 struct adapter *adap = pi->adapter;
c887ad0e
HS		 2582 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2583
2584 for (i = 0; i < pi->nqsets; i++, q++) {
2585 err = set_rspq_intr_params(&q->rspq, us, cnt);
2586 if (err)
2587 return err;
d4fc9dc2 2588 }
c887ad0e
HS		 2589 return 0;
2590}
2591
e553ec3f
HS		 2592static int set_adaptive_rx_setting(struct net_device *dev, int adaptive_rx)
2593{
2594 int i;
2595 struct port_info *pi = netdev_priv(dev);
2596 struct adapter *adap = pi->adapter;
2597 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2598
2599 for (i = 0; i < pi->nqsets; i++, q++)
2600 q->rspq.adaptive_rx = adaptive_rx;
2601
2602 return 0;
2603}
2604
2605static int get_adaptive_rx_setting(struct net_device *dev)
2606{
2607 struct port_info *pi = netdev_priv(dev);
2608 struct adapter *adap = pi->adapter;
2609 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2610
2611 return q->rspq.adaptive_rx;
2612}
2613
c887ad0e
HS		 2614static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2615{
e553ec3f 2616 set_adaptive_rx_setting(dev, c->use_adaptive_rx_coalesce);
c887ad0e
HS		 2617 return set_rx_intr_params(dev, c->rx_coalesce_usecs,
2618 c->rx_max_coalesced_frames);
b8ff05a9
DM		 2619}
2620
2621static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2622{
2623 const struct port_info *pi = netdev_priv(dev);
2624 const struct adapter *adap = pi->adapter;
2625 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2626
2627 c->rx_coalesce_usecs = qtimer_val(adap, rq);
2628 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2629 adap->sge.counter_val[rq->pktcnt_idx] : 0;
e553ec3f 2630 c->use_adaptive_rx_coalesce = get_adaptive_rx_setting(dev);
b8ff05a9
DM		 2631 return 0;
2632}
2633
1478b3ee
DM		 2634/**
2635 * eeprom_ptov - translate a physical EEPROM address to virtual
2636 * @phys_addr: the physical EEPROM address
2637 * @fn: the PCI function number
2638 * @sz: size of function-specific area
2639 *
2640 * Translate a physical EEPROM address to virtual. The first 1K is
2641 * accessed through virtual addresses starting at 31K, the rest is
2642 * accessed through virtual addresses starting at 0.
2643 *
2644 * The mapping is as follows:
2645 * [0..1K) -> [31K..32K)
2646 * [1K..1K+A) -> [31K-A..31K)
2647 * [1K+A..ES) -> [0..ES-A-1K)
2648 *
2649 * where A = @fn * @sz, and ES = EEPROM size.
b8ff05a9 2650 */
1478b3ee 2651static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
b8ff05a9 2652{
1478b3ee 2653 fn *= sz;
b8ff05a9
DM		 2654 if (phys_addr < 1024)
2655 return phys_addr + (31 << 10);
1478b3ee
DM		 2656 if (phys_addr < 1024 + fn)
2657 return 31744 - fn + phys_addr - 1024;
b8ff05a9 2658 if (phys_addr < EEPROMSIZE)
1478b3ee 2659 return phys_addr - 1024 - fn;
b8ff05a9
DM		 2660 return -EINVAL;
2661}
2662
2663/*
2664 * The next two routines implement eeprom read/write from physical addresses.
b8ff05a9
DM		 2665 */
2666static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2667{
1478b3ee 2668 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2669
2670 if (vaddr >= 0)
2671 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2672 return vaddr < 0 ? vaddr : 0;
2673}
2674
2675static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2676{
1478b3ee 2677 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2678
2679 if (vaddr >= 0)
2680 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2681 return vaddr < 0 ? vaddr : 0;
2682}
2683
2684#define EEPROM_MAGIC 0x38E2F10C
2685
2686static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2687 u8 *data)
2688{
2689 int i, err = 0;
2690 struct adapter *adapter = netdev2adap(dev);
2691
2692 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2693 if (!buf)
2694 return -ENOMEM;
2695
2696 e->magic = EEPROM_MAGIC;
2697 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2698 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2699
2700 if (!err)
2701 memcpy(data, buf + e->offset, e->len);
2702 kfree(buf);
2703 return err;
2704}
2705
2706static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2707 u8 *data)
2708{
2709 u8 *buf;
2710 int err = 0;
2711 u32 aligned_offset, aligned_len, *p;
2712 struct adapter *adapter = netdev2adap(dev);
2713
2714 if (eeprom->magic != EEPROM_MAGIC)
2715 return -EINVAL;
2716
2717 aligned_offset = eeprom->offset & ~3;
2718 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2719
1478b3ee
DM		 2720 if (adapter->fn > 0) {
2721 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2722
2723 if (aligned_offset < start ||
2724 aligned_offset + aligned_len > start + EEPROMPFSIZE)
2725 return -EPERM;
2726 }
2727
b8ff05a9
DM		 2728 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2729 /*
2730 * RMW possibly needed for first or last words.
2731 */
2732 buf = kmalloc(aligned_len, GFP_KERNEL);
2733 if (!buf)
2734 return -ENOMEM;
2735 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2736 if (!err && aligned_len > 4)
2737 err = eeprom_rd_phys(adapter,
2738 aligned_offset + aligned_len - 4,
2739 (u32 *)&buf[aligned_len - 4]);
2740 if (err)
2741 goto out;
2742 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2743 } else
2744 buf = data;
2745
2746 err = t4_seeprom_wp(adapter, false);
2747 if (err)
2748 goto out;
2749
2750 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2751 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2752 aligned_offset += 4;
2753 }
2754
2755 if (!err)
2756 err = t4_seeprom_wp(adapter, true);
2757out:
2758 if (buf != data)
2759 kfree(buf);
2760 return err;
2761}
2762
2763static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2764{
2765 int ret;
2766 const struct firmware *fw;
2767 struct adapter *adap = netdev2adap(netdev);
b2e1a3f0 2768 unsigned int mbox = PCIE_FW_MASTER_M + 1;
b8ff05a9
DM		 2769
2770 ef->data[sizeof(ef->data) - 1] = '\0';
2771 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2772 if (ret < 0)
2773 return ret;
2774
22c0b963
HS		 2775 /* If the adapter has been fully initialized then we'll go ahead and
2776 * try to get the firmware's cooperation in upgrading to the new
2777 * firmware image otherwise we'll try to do the entire job from the
2778 * host ... and we always "force" the operation in this path.
2779 */
2780 if (adap->flags & FULL_INIT_DONE)
2781 mbox = adap->mbox;
2782
2783 ret = t4_fw_upgrade(adap, mbox, fw->data, fw->size, 1);
b8ff05a9
DM		 2784 release_firmware(fw);
2785 if (!ret)
22c0b963
HS		 2786 dev_info(adap->pdev_dev, "loaded firmware %s,"
2787 " reload cxgb4 driver\n", ef->data);
b8ff05a9
DM		 2788 return ret;
2789}
2790
2791#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2792#define BCAST_CRC 0xa0ccc1a6
2793
2794static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2795{
2796 wol->supported = WAKE_BCAST | WAKE_MAGIC;
2797 wol->wolopts = netdev2adap(dev)->wol;
2798 memset(&wol->sopass, 0, sizeof(wol->sopass));
2799}
2800
2801static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2802{
2803 int err = 0;
2804 struct port_info *pi = netdev_priv(dev);
2805
2806 if (wol->wolopts & ~WOL_SUPPORTED)
2807 return -EINVAL;
2808 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2809 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2810 if (wol->wolopts & WAKE_BCAST) {
2811 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2812 ~0ULL, 0, false);
2813 if (!err)
2814 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2815 ~6ULL, ~0ULL, BCAST_CRC, true);
2816 } else
2817 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2818 return err;
2819}
2820
c8f44aff 2821static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
87b6cf51 2822{
2ed28baa 2823 const struct port_info *pi = netdev_priv(dev);
c8f44aff 2824 netdev_features_t changed = dev->features ^ features;
19ecae2c 2825 int err;
19ecae2c 2826
f646968f 2827 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2ed28baa 2828 return 0;
19ecae2c 2829
2ed28baa
MM		 2830 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2831 -1, -1, -1,
f646968f 2832 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2ed28baa 2833 if (unlikely(err))
f646968f 2834 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
19ecae2c 2835 return err;
87b6cf51
DM		 2836}
2837
7850f63f 2838static u32 get_rss_table_size(struct net_device *dev)
671b0060
DM		 2839{
2840 const struct port_info *pi = netdev_priv(dev);
671b0060 2841
7850f63f
BH		 2842 return pi->rss_size;
2843}
2844
892311f6 2845static int get_rss_table(struct net_device *dev, u32 *p, u8 *key, u8 *hfunc)
7850f63f
BH		 2846{
2847 const struct port_info *pi = netdev_priv(dev);
2848 unsigned int n = pi->rss_size;
2849
892311f6
EP		 2850 if (hfunc)
2851 *hfunc = ETH_RSS_HASH_TOP;
2852 if (!p)
2853 return 0;
671b0060 2854 while (n--)
7850f63f 2855 p[n] = pi->rss[n];
671b0060
DM		 2856 return 0;
2857}
2858
892311f6
EP		 2859static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key,
2860 const u8 hfunc)
671b0060
DM		 2861{
2862 unsigned int i;
2863 struct port_info *pi = netdev_priv(dev);
2864
892311f6
EP		 2865 /* We require at least one supported parameter to be changed and no
2866 * change in any of the unsupported parameters
2867 */
2868 if (key ||
2869 (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
2870 return -EOPNOTSUPP;
2871 if (!p)
2872 return 0;
2873
7850f63f
BH		 2874 for (i = 0; i < pi->rss_size; i++)
2875 pi->rss[i] = p[i];
671b0060
DM		 2876 if (pi->adapter->flags & FULL_INIT_DONE)
2877 return write_rss(pi, pi->rss);
2878 return 0;
2879}
2880
2881static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
815c7db5 2882 u32 *rules)
671b0060 2883{
f796564a
DM		 2884 const struct port_info *pi = netdev_priv(dev);
2885
671b0060 2886 switch (info->cmd) {
f796564a
DM		 2887 case ETHTOOL_GRXFH: {
2888 unsigned int v = pi->rss_mode;
2889
2890 info->data = 0;
2891 switch (info->flow_type) {
2892 case TCP_V4_FLOW:
b2e1a3f0 2893 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F)
f796564a
DM		 2894 info->data = RXH_IP_SRC | RXH_IP_DST |
2895 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2896 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
f796564a
DM		 2897 info->data = RXH_IP_SRC | RXH_IP_DST;
2898 break;
2899 case UDP_V4_FLOW:
b2e1a3f0
HS		 2900 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) &&
2901 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
f796564a
DM		 2902 info->data = RXH_IP_SRC | RXH_IP_DST |
2903 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2904 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
f796564a
DM		 2905 info->data = RXH_IP_SRC | RXH_IP_DST;
2906 break;
2907 case SCTP_V4_FLOW:
2908 case AH_ESP_V4_FLOW:
2909 case IPV4_FLOW:
b2e1a3f0 2910 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
f796564a
DM		 2911 info->data = RXH_IP_SRC | RXH_IP_DST;
2912 break;
2913 case TCP_V6_FLOW:
b2e1a3f0 2914 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F)
f796564a
DM		 2915 info->data = RXH_IP_SRC | RXH_IP_DST |
2916 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2917 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
f796564a
DM		 2918 info->data = RXH_IP_SRC | RXH_IP_DST;
2919 break;
2920 case UDP_V6_FLOW:
b2e1a3f0
HS		 2921 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) &&
2922 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
f796564a
DM		 2923 info->data = RXH_IP_SRC | RXH_IP_DST |
2924 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2925 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
f796564a
DM		 2926 info->data = RXH_IP_SRC | RXH_IP_DST;
2927 break;
2928 case SCTP_V6_FLOW:
2929 case AH_ESP_V6_FLOW:
2930 case IPV6_FLOW:
b2e1a3f0 2931 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
f796564a
DM		 2932 info->data = RXH_IP_SRC | RXH_IP_DST;
2933 break;
2934 }
2935 return 0;
2936 }
671b0060 2937 case ETHTOOL_GRXRINGS:
f796564a 2938 info->data = pi->nqsets;
671b0060
DM		 2939 return 0;
2940 }
2941 return -EOPNOTSUPP;
2942}
2943
9b07be4b 2944static const struct ethtool_ops cxgb_ethtool_ops = {
b8ff05a9
DM		 2945 .get_settings = get_settings,
2946 .set_settings = set_settings,
2947 .get_drvinfo = get_drvinfo,
2948 .get_msglevel = get_msglevel,
2949 .set_msglevel = set_msglevel,
2950 .get_ringparam = get_sge_param,
2951 .set_ringparam = set_sge_param,
2952 .get_coalesce = get_coalesce,
2953 .set_coalesce = set_coalesce,
2954 .get_eeprom_len = get_eeprom_len,
2955 .get_eeprom = get_eeprom,
2956 .set_eeprom = set_eeprom,
2957 .get_pauseparam = get_pauseparam,
2958 .set_pauseparam = set_pauseparam,
b8ff05a9
DM		 2959 .get_link = ethtool_op_get_link,
2960 .get_strings = get_strings,
c5e06360 2961 .set_phys_id = identify_port,
b8ff05a9
DM		 2962 .nway_reset = restart_autoneg,
2963 .get_sset_count = get_sset_count,
2964 .get_ethtool_stats = get_stats,
2965 .get_regs_len = get_regs_len,
2966 .get_regs = get_regs,
2967 .get_wol = get_wol,
2968 .set_wol = set_wol,
671b0060 2969 .get_rxnfc = get_rxnfc,
7850f63f 2970 .get_rxfh_indir_size = get_rss_table_size,
fe62d001
BH		 2971 .get_rxfh = get_rss_table,
2972 .set_rxfh = set_rss_table,
b8ff05a9
DM		 2973 .flash_device = set_flash,
2974};
2975
91744948 2976static int setup_debugfs(struct adapter *adap)
b8ff05a9 2977{
b8ff05a9
DM		 2978 if (IS_ERR_OR_NULL(adap->debugfs_root))
2979 return -1;
2980
fd88b31a
HS		 2981#ifdef CONFIG_DEBUG_FS
2982 t4_setup_debugfs(adap);
2983#endif
b8ff05a9
DM		 2984 return 0;
2985}
2986
2987/*
2988 * upper-layer driver support
2989 */
2990
2991/*
2992 * Allocate an active-open TID and set it to the supplied value.
2993 */
2994int cxgb4_alloc_atid(struct tid_info *t, void *data)
2995{
2996 int atid = -1;
2997
2998 spin_lock_bh(&t->atid_lock);
2999 if (t->afree) {
3000 union aopen_entry *p = t->afree;
3001
f2b7e78d 3002 atid = (p - t->atid_tab) + t->atid_base;
b8ff05a9
DM		 3003 t->afree = p->next;
3004 p->data = data;
3005 t->atids_in_use++;
3006 }
3007 spin_unlock_bh(&t->atid_lock);
3008 return atid;
3009}
3010EXPORT_SYMBOL(cxgb4_alloc_atid);
3011
3012/*
3013 * Release an active-open TID.
3014 */
3015void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
3016{
f2b7e78d 3017 union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
b8ff05a9
DM		 3018
3019 spin_lock_bh(&t->atid_lock);
3020 p->next = t->afree;
3021 t->afree = p;
3022 t->atids_in_use--;
3023 spin_unlock_bh(&t->atid_lock);
3024}
3025EXPORT_SYMBOL(cxgb4_free_atid);
3026
3027/*
3028 * Allocate a server TID and set it to the supplied value.
3029 */
3030int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
3031{
3032 int stid;
3033
3034 spin_lock_bh(&t->stid_lock);
3035 if (family == PF_INET) {
3036 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
3037 if (stid < t->nstids)
3038 __set_bit(stid, t->stid_bmap);
3039 else
3040 stid = -1;
3041 } else {
3042 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
3043 if (stid < 0)
3044 stid = -1;
3045 }
3046 if (stid >= 0) {
3047 t->stid_tab[stid].data = data;
3048 stid += t->stid_base;
15f63b74
KS		 3049 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3050 * This is equivalent to 4 TIDs. With CLIP enabled it
3051 * needs 2 TIDs.
3052 */
3053 if (family == PF_INET)
3054 t->stids_in_use++;
3055 else
3056 t->stids_in_use += 4;
b8ff05a9
DM		 3057 }
3058 spin_unlock_bh(&t->stid_lock);
3059 return stid;
3060}
3061EXPORT_SYMBOL(cxgb4_alloc_stid);
3062
dca4faeb
VP		 3063/* Allocate a server filter TID and set it to the supplied value.
3064 */
3065int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3066{
3067 int stid;
3068
3069 spin_lock_bh(&t->stid_lock);
3070 if (family == PF_INET) {
3071 stid = find_next_zero_bit(t->stid_bmap,
3072 t->nstids + t->nsftids, t->nstids);
3073 if (stid < (t->nstids + t->nsftids))
3074 __set_bit(stid, t->stid_bmap);
3075 else
3076 stid = -1;
3077 } else {
3078 stid = -1;
3079 }
3080 if (stid >= 0) {
3081 t->stid_tab[stid].data = data;
470c60c4
KS		 3082 stid -= t->nstids;
3083 stid += t->sftid_base;
dca4faeb
VP		 3084 t->stids_in_use++;
3085 }
3086 spin_unlock_bh(&t->stid_lock);
3087 return stid;
3088}
3089EXPORT_SYMBOL(cxgb4_alloc_sftid);
3090
3091/* Release a server TID.
b8ff05a9
DM		 3092 */
3093void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3094{
470c60c4
KS		 3095 /* Is it a server filter TID? */
3096 if (t->nsftids && (stid >= t->sftid_base)) {
3097 stid -= t->sftid_base;
3098 stid += t->nstids;
3099 } else {
3100 stid -= t->stid_base;
3101 }
3102
b8ff05a9
DM		 3103 spin_lock_bh(&t->stid_lock);
3104 if (family == PF_INET)
3105 __clear_bit(stid, t->stid_bmap);
3106 else
3107 bitmap_release_region(t->stid_bmap, stid, 2);
3108 t->stid_tab[stid].data = NULL;
15f63b74
KS		 3109 if (family == PF_INET)
3110 t->stids_in_use--;
3111 else
3112 t->stids_in_use -= 4;
b8ff05a9
DM		 3113 spin_unlock_bh(&t->stid_lock);
3114}
3115EXPORT_SYMBOL(cxgb4_free_stid);
3116
3117/*
3118 * Populate a TID_RELEASE WR. Caller must properly size the skb.
3119 */
3120static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3121 unsigned int tid)
3122{
3123 struct cpl_tid_release *req;
3124
3125 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3126 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3127 INIT_TP_WR(req, tid);
3128 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3129}
3130
3131/*
3132 * Queue a TID release request and if necessary schedule a work queue to
3133 * process it.
3134 */
31b9c19b 3135static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3136 unsigned int tid)
b8ff05a9
DM		 3137{
3138 void **p = &t->tid_tab[tid];
3139 struct adapter *adap = container_of(t, struct adapter, tids);
3140
3141 spin_lock_bh(&adap->tid_release_lock);
3142 *p = adap->tid_release_head;
3143 /* Low 2 bits encode the Tx channel number */
3144 adap->tid_release_head = (void **)((uintptr_t)p | chan);
3145 if (!adap->tid_release_task_busy) {
3146 adap->tid_release_task_busy = true;
29aaee65 3147 queue_work(adap->workq, &adap->tid_release_task);
b8ff05a9
DM		 3148 }
3149 spin_unlock_bh(&adap->tid_release_lock);
3150}
b8ff05a9
DM		 3151
3152/*
3153 * Process the list of pending TID release requests.
3154 */
3155static void process_tid_release_list(struct work_struct *work)
3156{
3157 struct sk_buff *skb;
3158 struct adapter *adap;
3159
3160 adap = container_of(work, struct adapter, tid_release_task);
3161
3162 spin_lock_bh(&adap->tid_release_lock);
3163 while (adap->tid_release_head) {
3164 void **p = adap->tid_release_head;
3165 unsigned int chan = (uintptr_t)p & 3;
3166 p = (void *)p - chan;
3167
3168 adap->tid_release_head = *p;
3169 *p = NULL;
3170 spin_unlock_bh(&adap->tid_release_lock);
3171
3172 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3173 GFP_KERNEL)))
3174 schedule_timeout_uninterruptible(1);
3175
3176 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3177 t4_ofld_send(adap, skb);
3178 spin_lock_bh(&adap->tid_release_lock);
3179 }
3180 adap->tid_release_task_busy = false;
3181 spin_unlock_bh(&adap->tid_release_lock);
3182}
3183
3184/*
3185 * Release a TID and inform HW. If we are unable to allocate the release
3186 * message we defer to a work queue.
3187 */
3188void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3189{
3190 void *old;
3191 struct sk_buff *skb;
3192 struct adapter *adap = container_of(t, struct adapter, tids);
3193
3194 old = t->tid_tab[tid];
3195 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3196 if (likely(skb)) {
3197 t->tid_tab[tid] = NULL;
3198 mk_tid_release(skb, chan, tid);
3199 t4_ofld_send(adap, skb);
3200 } else
3201 cxgb4_queue_tid_release(t, chan, tid);
3202 if (old)
3203 atomic_dec(&t->tids_in_use);
3204}
3205EXPORT_SYMBOL(cxgb4_remove_tid);
3206
3207/*
3208 * Allocate and initialize the TID tables. Returns 0 on success.
3209 */
3210static int tid_init(struct tid_info *t)
3211{
3212 size_t size;
f2b7e78d 3213 unsigned int stid_bmap_size;
b8ff05a9 3214 unsigned int natids = t->natids;
b6f8eaec 3215 struct adapter *adap = container_of(t, struct adapter, tids);
b8ff05a9 3216
dca4faeb 3217 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
f2b7e78d
VP		 3218 size = t->ntids * sizeof(*t->tid_tab) +
3219 natids * sizeof(*t->atid_tab) +
b8ff05a9 3220 t->nstids * sizeof(*t->stid_tab) +
dca4faeb 3221 t->nsftids * sizeof(*t->stid_tab) +
f2b7e78d 3222 stid_bmap_size * sizeof(long) +
dca4faeb
VP		 3223 t->nftids * sizeof(*t->ftid_tab) +
3224 t->nsftids * sizeof(*t->ftid_tab);
f2b7e78d 3225
b8ff05a9
DM		 3226 t->tid_tab = t4_alloc_mem(size);
3227 if (!t->tid_tab)
3228 return -ENOMEM;
3229
3230 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3231 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
dca4faeb 3232 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
f2b7e78d 3233 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
b8ff05a9
DM		 3234 spin_lock_init(&t->stid_lock);
3235 spin_lock_init(&t->atid_lock);
3236
3237 t->stids_in_use = 0;
3238 t->afree = NULL;
3239 t->atids_in_use = 0;
3240 atomic_set(&t->tids_in_use, 0);
3241
3242 /* Setup the free list for atid_tab and clear the stid bitmap. */
3243 if (natids) {
3244 while (--natids)
3245 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3246 t->afree = t->atid_tab;
3247 }
dca4faeb 3248 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
b6f8eaec
KS		 3249 /* Reserve stid 0 for T4/T5 adapters */
3250 if (!t->stid_base &&
3251 (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3252 __set_bit(0, t->stid_bmap);
3253
b8ff05a9
DM		 3254 return 0;
3255}
3256
3257/**
3258 * cxgb4_create_server - create an IP server
3259 * @dev: the device
3260 * @stid: the server TID
3261 * @sip: local IP address to bind server to
3262 * @sport: the server's TCP port
3263 * @queue: queue to direct messages from this server to
3264 *
3265 * Create an IP server for the given port and address.
3266 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3267 */
3268int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
793dad94
VP		 3269 __be32 sip, __be16 sport, __be16 vlan,
3270 unsigned int queue)
b8ff05a9
DM		 3271{
3272 unsigned int chan;
3273 struct sk_buff *skb;
3274 struct adapter *adap;
3275 struct cpl_pass_open_req *req;
80f40c1f 3276 int ret;
b8ff05a9
DM		 3277
3278 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3279 if (!skb)
3280 return -ENOMEM;
3281
3282 adap = netdev2adap(dev);
3283 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3284 INIT_TP_WR(req, 0);
3285 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3286 req->local_port = sport;
3287 req->peer_port = htons(0);
3288 req->local_ip = sip;
3289 req->peer_ip = htonl(0);
e46dab4d 3290 chan = rxq_to_chan(&adap->sge, queue);
d7990b0c 3291 req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
6c53e938
HS		 3292 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
3293 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
80f40c1f
VP		 3294 ret = t4_mgmt_tx(adap, skb);
3295 return net_xmit_eval(ret);
b8ff05a9
DM		 3296}
3297EXPORT_SYMBOL(cxgb4_create_server);
3298
80f40c1f
VP		 3299/* cxgb4_create_server6 - create an IPv6 server
3300 * @dev: the device
3301 * @stid: the server TID
3302 * @sip: local IPv6 address to bind server to
3303 * @sport: the server's TCP port
3304 * @queue: queue to direct messages from this server to
3305 *
3306 * Create an IPv6 server for the given port and address.
3307 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3308 */
3309int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3310 const struct in6_addr *sip, __be16 sport,
3311 unsigned int queue)
3312{
3313 unsigned int chan;
3314 struct sk_buff *skb;
3315 struct adapter *adap;
3316 struct cpl_pass_open_req6 *req;
3317 int ret;
3318
3319 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3320 if (!skb)
3321 return -ENOMEM;
3322
3323 adap = netdev2adap(dev);
3324 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3325 INIT_TP_WR(req, 0);
3326 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3327 req->local_port = sport;
3328 req->peer_port = htons(0);
3329 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3330 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3331 req->peer_ip_hi = cpu_to_be64(0);
3332 req->peer_ip_lo = cpu_to_be64(0);
3333 chan = rxq_to_chan(&adap->sge, queue);
d7990b0c 3334 req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
6c53e938
HS		 3335 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
3336 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
80f40c1f
VP		 3337 ret = t4_mgmt_tx(adap, skb);
3338 return net_xmit_eval(ret);
3339}
3340EXPORT_SYMBOL(cxgb4_create_server6);
3341
3342int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3343 unsigned int queue, bool ipv6)
3344{
3345 struct sk_buff *skb;
3346 struct adapter *adap;
3347 struct cpl_close_listsvr_req *req;
3348 int ret;
3349
3350 adap = netdev2adap(dev);
3351
3352 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3353 if (!skb)
3354 return -ENOMEM;
3355
3356 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3357 INIT_TP_WR(req, 0);
3358 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
bdc590b9
HS		 3359 req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) :
3360 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue));
80f40c1f
VP		 3361 ret = t4_mgmt_tx(adap, skb);
3362 return net_xmit_eval(ret);
3363}
3364EXPORT_SYMBOL(cxgb4_remove_server);
3365
b8ff05a9
DM		 3366/**
3367 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3368 * @mtus: the HW MTU table
3369 * @mtu: the target MTU
3370 * @idx: index of selected entry in the MTU table
3371 *
3372 * Returns the index and the value in the HW MTU table that is closest to
3373 * but does not exceed @mtu, unless @mtu is smaller than any value in the
3374 * table, in which case that smallest available value is selected.
3375 */
3376unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3377 unsigned int *idx)
3378{
3379 unsigned int i = 0;
3380
3381 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3382 ++i;
3383 if (idx)
3384 *idx = i;
3385 return mtus[i];
3386}
3387EXPORT_SYMBOL(cxgb4_best_mtu);
3388
92e7ae71
HS		 3389/**
3390 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned
3391 * @mtus: the HW MTU table
3392 * @header_size: Header Size
3393 * @data_size_max: maximum Data Segment Size
3394 * @data_size_align: desired Data Segment Size Alignment (2^N)
3395 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL)
3396 *
3397 * Similar to cxgb4_best_mtu() but instead of searching the Hardware
3398 * MTU Table based solely on a Maximum MTU parameter, we break that
3399 * parameter up into a Header Size and Maximum Data Segment Size, and
3400 * provide a desired Data Segment Size Alignment. If we find an MTU in
3401 * the Hardware MTU Table which will result in a Data Segment Size with
3402 * the requested alignment _and_ that MTU isn't "too far" from the
3403 * closest MTU, then we'll return that rather than the closest MTU.
3404 */
3405unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus,
3406 unsigned short header_size,
3407 unsigned short data_size_max,
3408 unsigned short data_size_align,
3409 unsigned int *mtu_idxp)
3410{
3411 unsigned short max_mtu = header_size + data_size_max;
3412 unsigned short data_size_align_mask = data_size_align - 1;
3413 int mtu_idx, aligned_mtu_idx;
3414
3415 /* Scan the MTU Table till we find an MTU which is larger than our
3416 * Maximum MTU or we reach the end of the table. Along the way,
3417 * record the last MTU found, if any, which will result in a Data
3418 * Segment Length matching the requested alignment.
3419 */
3420 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) {
3421 unsigned short data_size = mtus[mtu_idx] - header_size;
3422
3423 /* If this MTU minus the Header Size would result in a
3424 * Data Segment Size of the desired alignment, remember it.
3425 */
3426 if ((data_size & data_size_align_mask) == 0)
3427 aligned_mtu_idx = mtu_idx;
3428
3429 /* If we're not at the end of the Hardware MTU Table and the
3430 * next element is larger than our Maximum MTU, drop out of
3431 * the loop.
3432 */
3433 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu)
3434 break;
3435 }
3436
3437 /* If we fell out of the loop because we ran to the end of the table,
3438 * then we just have to use the last [largest] entry.
3439 */
3440 if (mtu_idx == NMTUS)
3441 mtu_idx--;
3442
3443 /* If we found an MTU which resulted in the requested Data Segment
3444 * Length alignment and that's "not far" from the largest MTU which is
3445 * less than or equal to the maximum MTU, then use that.
3446 */
3447 if (aligned_mtu_idx >= 0 &&
3448 mtu_idx - aligned_mtu_idx <= 1)
3449 mtu_idx = aligned_mtu_idx;
3450
3451 /* If the caller has passed in an MTU Index pointer, pass the
3452 * MTU Index back. Return the MTU value.
3453 */
3454 if (mtu_idxp)
3455 *mtu_idxp = mtu_idx;
3456 return mtus[mtu_idx];
3457}
3458EXPORT_SYMBOL(cxgb4_best_aligned_mtu);
3459
b8ff05a9
DM		 3460/**
3461 * cxgb4_port_chan - get the HW channel of a port
3462 * @dev: the net device for the port
3463 *
3464 * Return the HW Tx channel of the given port.
3465 */
3466unsigned int cxgb4_port_chan(const struct net_device *dev)
3467{
3468 return netdev2pinfo(dev)->tx_chan;
3469}
3470EXPORT_SYMBOL(cxgb4_port_chan);
3471
881806bc
VP		 3472unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3473{
3474 struct adapter *adap = netdev2adap(dev);
2cc301d2 3475 u32 v1, v2, lp_count, hp_count;
881806bc 3476
f061de42
HS		 3477 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3478 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
d14807dd 3479 if (is_t4(adap->params.chip)) {
f061de42
HS		 3480 lp_count = LP_COUNT_G(v1);
3481 hp_count = HP_COUNT_G(v1);
2cc301d2 3482 } else {
f061de42
HS		 3483 lp_count = LP_COUNT_T5_G(v1);
3484 hp_count = HP_COUNT_T5_G(v2);
2cc301d2
SR		 3485 }
3486 return lpfifo ? lp_count : hp_count;
881806bc
VP		 3487}
3488EXPORT_SYMBOL(cxgb4_dbfifo_count);
3489
b8ff05a9
DM		 3490/**
3491 * cxgb4_port_viid - get the VI id of a port
3492 * @dev: the net device for the port
3493 *
3494 * Return the VI id of the given port.
3495 */
3496unsigned int cxgb4_port_viid(const struct net_device *dev)
3497{
3498 return netdev2pinfo(dev)->viid;
3499}
3500EXPORT_SYMBOL(cxgb4_port_viid);
3501
3502/**
3503 * cxgb4_port_idx - get the index of a port
3504 * @dev: the net device for the port
3505 *
3506 * Return the index of the given port.
3507 */
3508unsigned int cxgb4_port_idx(const struct net_device *dev)
3509{
3510 return netdev2pinfo(dev)->port_id;
3511}
3512EXPORT_SYMBOL(cxgb4_port_idx);
3513
b8ff05a9
DM		 3514void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3515 struct tp_tcp_stats *v6)
3516{
3517 struct adapter *adap = pci_get_drvdata(pdev);
3518
3519 spin_lock(&adap->stats_lock);
3520 t4_tp_get_tcp_stats(adap, v4, v6);
3521 spin_unlock(&adap->stats_lock);
3522}
3523EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3524
3525void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3526 const unsigned int *pgsz_order)
3527{
3528 struct adapter *adap = netdev2adap(dev);
3529
0d804338
HS		 3530 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask);
3531 t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) |
3532 HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) |
3533 HPZ3_V(pgsz_order[3]));
b8ff05a9
DM		 3534}
3535EXPORT_SYMBOL(cxgb4_iscsi_init);
3536
3069ee9b
VP		 3537int cxgb4_flush_eq_cache(struct net_device *dev)
3538{
3539 struct adapter *adap = netdev2adap(dev);
3540 int ret;
3541
3542 ret = t4_fwaddrspace_write(adap, adap->mbox,
f061de42 3543 0xe1000000 + SGE_CTXT_CMD_A, 0x20000000);
3069ee9b
VP		 3544 return ret;
3545}
3546EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3547
3548static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3549{
f061de42 3550 u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8;
3069ee9b
VP		 3551 __be64 indices;
3552 int ret;
3553
fc5ab020
HS		 3554 spin_lock(&adap->win0_lock);
3555 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr,
3556 sizeof(indices), (__be32 *)&indices,
3557 T4_MEMORY_READ);
3558 spin_unlock(&adap->win0_lock);
3069ee9b 3559 if (!ret) {
404d9e3f
VP		 3560 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3561 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3069ee9b
VP		 3562 }
3563 return ret;
3564}
3565
3566int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3567 u16 size)
3568{
3569 struct adapter *adap = netdev2adap(dev);
3570 u16 hw_pidx, hw_cidx;
3571 int ret;
3572
3573 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3574 if (ret)
3575 goto out;
3576
3577 if (pidx != hw_pidx) {
3578 u16 delta;
f612b815 3579 u32 val;
3069ee9b
VP		 3580
3581 if (pidx >= hw_pidx)
3582 delta = pidx - hw_pidx;
3583 else
3584 delta = size - hw_pidx + pidx;
f612b815
HS		 3585
3586 if (is_t4(adap->params.chip))
3587 val = PIDX_V(delta);
3588 else
3589 val = PIDX_T5_V(delta);
3069ee9b 3590 wmb();
f612b815
HS		 3591 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3592 QID_V(qid) | val);
3069ee9b
VP		 3593 }
3594out:
3595 return ret;
3596}
3597EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3598
3cbdb928
VP		 3599void cxgb4_disable_db_coalescing(struct net_device *dev)
3600{
3601 struct adapter *adap;
3602
3603 adap = netdev2adap(dev);
f061de42 3604 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F,
f612b815 3605 NOCOALESCE_F);
3cbdb928
VP		 3606}
3607EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3608
3609void cxgb4_enable_db_coalescing(struct net_device *dev)
3610{
3611 struct adapter *adap;
3612
3613 adap = netdev2adap(dev);
f061de42 3614 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F, 0);
3cbdb928
VP		 3615}
3616EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3617
031cf476
HS		 3618int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte)
3619{
3620 struct adapter *adap;
3621 u32 offset, memtype, memaddr;
6559a7e8 3622 u32 edc0_size, edc1_size, mc0_size, mc1_size, size;
031cf476
HS		 3623 u32 edc0_end, edc1_end, mc0_end, mc1_end;
3624 int ret;
3625
3626 adap = netdev2adap(dev);
3627
3628 offset = ((stag >> 8) * 32) + adap->vres.stag.start;
3629
3630 /* Figure out where the offset lands in the Memory Type/Address scheme.
3631 * This code assumes that the memory is laid out starting at offset 0
3632 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0
3633 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have
3634 * MC0, and some have both MC0 and MC1.
3635 */
6559a7e8
HS		 3636 size = t4_read_reg(adap, MA_EDRAM0_BAR_A);
3637 edc0_size = EDRAM0_SIZE_G(size) << 20;
3638 size = t4_read_reg(adap, MA_EDRAM1_BAR_A);
3639 edc1_size = EDRAM1_SIZE_G(size) << 20;
3640 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A);
3641 mc0_size = EXT_MEM0_SIZE_G(size) << 20;
031cf476
HS		 3642
3643 edc0_end = edc0_size;
3644 edc1_end = edc0_end + edc1_size;
3645 mc0_end = edc1_end + mc0_size;
3646
3647 if (offset < edc0_end) {
3648 memtype = MEM_EDC0;
3649 memaddr = offset;
3650 } else if (offset < edc1_end) {
3651 memtype = MEM_EDC1;
3652 memaddr = offset - edc0_end;
3653 } else {
3654 if (offset < mc0_end) {
3655 memtype = MEM_MC0;
3656 memaddr = offset - edc1_end;
3657 } else if (is_t4(adap->params.chip)) {
3658 /* T4 only has a single memory channel */
3659 goto err;
3660 } else {
6559a7e8
HS		 3661 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
3662 mc1_size = EXT_MEM1_SIZE_G(size) << 20;
031cf476
HS		 3663 mc1_end = mc0_end + mc1_size;
3664 if (offset < mc1_end) {
3665 memtype = MEM_MC1;
3666 memaddr = offset - mc0_end;
3667 } else {
3668 /* offset beyond the end of any memory */
3669 goto err;
3670 }
3671 }
3672 }
3673
3674 spin_lock(&adap->win0_lock);
3675 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ);
3676 spin_unlock(&adap->win0_lock);
3677 return ret;
3678
3679err:
3680 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n",
3681 stag, offset);
3682 return -EINVAL;
3683}
3684EXPORT_SYMBOL(cxgb4_read_tpte);
3685
7730b4c7
HS		 3686u64 cxgb4_read_sge_timestamp(struct net_device *dev)
3687{
3688 u32 hi, lo;
3689 struct adapter *adap;
3690
3691 adap = netdev2adap(dev);
f612b815
HS		 3692 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A);
3693 hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A));
7730b4c7
HS		 3694
3695 return ((u64)hi << 32) | (u64)lo;
3696}
3697EXPORT_SYMBOL(cxgb4_read_sge_timestamp);
3698
df64e4d3
HS		 3699int cxgb4_bar2_sge_qregs(struct net_device *dev,
3700 unsigned int qid,
3701 enum cxgb4_bar2_qtype qtype,
3702 u64 *pbar2_qoffset,
3703 unsigned int *pbar2_qid)
3704{
dd0bcc0b 3705 return cxgb4_t4_bar2_sge_qregs(netdev2adap(dev),
df64e4d3
HS		 3706 qid,
3707 (qtype == CXGB4_BAR2_QTYPE_EGRESS
3708 ? T4_BAR2_QTYPE_EGRESS
3709 : T4_BAR2_QTYPE_INGRESS),
3710 pbar2_qoffset,
3711 pbar2_qid);
3712}
3713EXPORT_SYMBOL(cxgb4_bar2_sge_qregs);
3714
b8ff05a9
DM		 3715static struct pci_driver cxgb4_driver;
3716
3717static void check_neigh_update(struct neighbour *neigh)
3718{
3719 const struct device *parent;
3720 const struct net_device *netdev = neigh->dev;
3721
3722 if (netdev->priv_flags & IFF_802_1Q_VLAN)
3723 netdev = vlan_dev_real_dev(netdev);
3724 parent = netdev->dev.parent;
3725 if (parent && parent->driver == &cxgb4_driver.driver)
3726 t4_l2t_update(dev_get_drvdata(parent), neigh);
3727}
3728
3729static int netevent_cb(struct notifier_block *nb, unsigned long event,
3730 void *data)
3731{
3732 switch (event) {
3733 case NETEVENT_NEIGH_UPDATE:
3734 check_neigh_update(data);
3735 break;
b8ff05a9
DM		 3736 case NETEVENT_REDIRECT:
3737 default:
3738 break;
3739 }
3740 return 0;
3741}
3742
3743static bool netevent_registered;
3744static struct notifier_block cxgb4_netevent_nb = {
3745 .notifier_call = netevent_cb
3746};
3747
3069ee9b
VP		 3748static void drain_db_fifo(struct adapter *adap, int usecs)
3749{
2cc301d2 3750 u32 v1, v2, lp_count, hp_count;
3069ee9b
VP		 3751
3752 do {
f061de42
HS		 3753 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3754 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
d14807dd 3755 if (is_t4(adap->params.chip)) {
f061de42
HS		 3756 lp_count = LP_COUNT_G(v1);
3757 hp_count = HP_COUNT_G(v1);
2cc301d2 3758 } else {
f061de42
HS		 3759 lp_count = LP_COUNT_T5_G(v1);
3760 hp_count = HP_COUNT_T5_G(v2);
2cc301d2
SR		 3761 }
3762
3763 if (lp_count == 0 && hp_count == 0)
3764 break;
3069ee9b
VP		 3765 set_current_state(TASK_UNINTERRUPTIBLE);
3766 schedule_timeout(usecs_to_jiffies(usecs));
3069ee9b
VP		 3767 } while (1);
3768}
3769
3770static void disable_txq_db(struct sge_txq *q)
3771{
05eb2389
SW		 3772 unsigned long flags;
3773
3774 spin_lock_irqsave(&q->db_lock, flags);
3069ee9b 3775 q->db_disabled = 1;
05eb2389 3776 spin_unlock_irqrestore(&q->db_lock, flags);
3069ee9b
VP		 3777}
3778
05eb2389 3779static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
3069ee9b
VP		 3780{
3781 spin_lock_irq(&q->db_lock);
05eb2389
SW		 3782 if (q->db_pidx_inc) {
3783 /* Make sure that all writes to the TX descriptors
3784 * are committed before we tell HW about them.
3785 */
3786 wmb();
f612b815
HS		 3787 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3788 QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc));
05eb2389
SW		 3789 q->db_pidx_inc = 0;
3790 }
3069ee9b
VP		 3791 q->db_disabled = 0;
3792 spin_unlock_irq(&q->db_lock);
3793}
3794
3795static void disable_dbs(struct adapter *adap)
3796{
3797 int i;
3798
3799 for_each_ethrxq(&adap->sge, i)
3800 disable_txq_db(&adap->sge.ethtxq[i].q);
3801 for_each_ofldrxq(&adap->sge, i)
3802 disable_txq_db(&adap->sge.ofldtxq[i].q);
3803 for_each_port(adap, i)
3804 disable_txq_db(&adap->sge.ctrlq[i].q);
3805}
3806
3807static void enable_dbs(struct adapter *adap)
3808{
3809 int i;
3810
3811 for_each_ethrxq(&adap->sge, i)
05eb2389 3812 enable_txq_db(adap, &adap->sge.ethtxq[i].q);
3069ee9b 3813 for_each_ofldrxq(&adap->sge, i)
05eb2389 3814 enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
3069ee9b 3815 for_each_port(adap, i)
05eb2389
SW		 3816 enable_txq_db(adap, &adap->sge.ctrlq[i].q);
3817}
3818
3819static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3820{
3821 if (adap->uld_handle[CXGB4_ULD_RDMA])
3822 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3823 cmd);
3824}
3825
3826static void process_db_full(struct work_struct *work)
3827{
3828 struct adapter *adap;
3829
3830 adap = container_of(work, struct adapter, db_full_task);
3831
3832 drain_db_fifo(adap, dbfifo_drain_delay);
3833 enable_dbs(adap);
3834 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
f612b815
HS		 3835 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3836 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F,
3837 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F);
3069ee9b
VP		 3838}
3839
3840static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3841{
3842 u16 hw_pidx, hw_cidx;
3843 int ret;
3844
05eb2389 3845 spin_lock_irq(&q->db_lock);
3069ee9b
VP		 3846 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3847 if (ret)
3848 goto out;
3849 if (q->db_pidx != hw_pidx) {
3850 u16 delta;
f612b815 3851 u32 val;
3069ee9b
VP		 3852
3853 if (q->db_pidx >= hw_pidx)
3854 delta = q->db_pidx - hw_pidx;
3855 else
3856 delta = q->size - hw_pidx + q->db_pidx;
f612b815
HS		 3857
3858 if (is_t4(adap->params.chip))
3859 val = PIDX_V(delta);
3860 else
3861 val = PIDX_T5_V(delta);
3069ee9b 3862 wmb();
f612b815
HS		 3863 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3864 QID_V(q->cntxt_id) | val);
3069ee9b
VP		 3865 }
3866out:
3867 q->db_disabled = 0;
05eb2389
SW		 3868 q->db_pidx_inc = 0;
3869 spin_unlock_irq(&q->db_lock);
3069ee9b
VP		 3870 if (ret)
3871 CH_WARN(adap, "DB drop recovery failed.\n");
3872}
3873static void recover_all_queues(struct adapter *adap)
3874{
3875 int i;
3876
3877 for_each_ethrxq(&adap->sge, i)
3878 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3879 for_each_ofldrxq(&adap->sge, i)
3880 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3881 for_each_port(adap, i)
3882 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3883}
3884
881806bc
VP		 3885static void process_db_drop(struct work_struct *work)
3886{
3887 struct adapter *adap;
881806bc 3888
3069ee9b 3889 adap = container_of(work, struct adapter, db_drop_task);
881806bc 3890
d14807dd 3891 if (is_t4(adap->params.chip)) {
05eb2389 3892 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3893 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
05eb2389 3894 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3895 recover_all_queues(adap);
05eb2389 3896 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 3897 enable_dbs(adap);
05eb2389 3898 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
2cc301d2
SR		 3899 } else {
3900 u32 dropped_db = t4_read_reg(adap, 0x010ac);
3901 u16 qid = (dropped_db >> 15) & 0x1ffff;
3902 u16 pidx_inc = dropped_db & 0x1fff;
df64e4d3
HS		 3903 u64 bar2_qoffset;
3904 unsigned int bar2_qid;
3905 int ret;
2cc301d2 3906
dd0bcc0b 3907 ret = cxgb4_t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS,
df64e4d3
HS		 3908 &bar2_qoffset, &bar2_qid);
3909 if (ret)
3910 dev_err(adap->pdev_dev, "doorbell drop recovery: "
3911 "qid=%d, pidx_inc=%d\n", qid, pidx_inc);
3912 else
f612b815 3913 writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid),
df64e4d3 3914 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL);
2cc301d2
SR		 3915
3916 /* Re-enable BAR2 WC */
3917 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
3918 }
3919
f061de42 3920 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0);
881806bc
VP		 3921}
3922
3923void t4_db_full(struct adapter *adap)
3924{
d14807dd 3925 if (is_t4(adap->params.chip)) {
05eb2389
SW		 3926 disable_dbs(adap);
3927 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
f612b815
HS		 3928 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3929 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0);
29aaee65 3930 queue_work(adap->workq, &adap->db_full_task);
2cc301d2 3931 }
881806bc
VP		 3932}
3933
3934void t4_db_dropped(struct adapter *adap)
3935{
05eb2389
SW		 3936 if (is_t4(adap->params.chip)) {
3937 disable_dbs(adap);
3938 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3939 }
29aaee65 3940 queue_work(adap->workq, &adap->db_drop_task);
881806bc
VP		 3941}
3942
b8ff05a9
DM		 3943static void uld_attach(struct adapter *adap, unsigned int uld)
3944{
3945 void *handle;
3946 struct cxgb4_lld_info lli;
dca4faeb 3947 unsigned short i;
b8ff05a9
DM		 3948
3949 lli.pdev = adap->pdev;
35b1de55 3950 lli.pf = adap->fn;
b8ff05a9
DM		 3951 lli.l2t = adap->l2t;
3952 lli.tids = &adap->tids;
3953 lli.ports = adap->port;
3954 lli.vr = &adap->vres;
3955 lli.mtus = adap->params.mtus;
3956 if (uld == CXGB4_ULD_RDMA) {
3957 lli.rxq_ids = adap->sge.rdma_rxq;
cf38be6d 3958 lli.ciq_ids = adap->sge.rdma_ciq;
b8ff05a9 3959 lli.nrxq = adap->sge.rdmaqs;
cf38be6d 3960 lli.nciq = adap->sge.rdmaciqs;
b8ff05a9
DM		 3961 } else if (uld == CXGB4_ULD_ISCSI) {
3962 lli.rxq_ids = adap->sge.ofld_rxq;
3963 lli.nrxq = adap->sge.ofldqsets;
3964 }
3965 lli.ntxq = adap->sge.ofldqsets;
3966 lli.nchan = adap->params.nports;
3967 lli.nports = adap->params.nports;
3968 lli.wr_cred = adap->params.ofldq_wr_cred;
d14807dd 3969 lli.adapter_type = adap->params.chip;
837e4a42 3970 lli.iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
7730b4c7 3971 lli.cclk_ps = 1000000000 / adap->params.vpd.cclk;
df64e4d3
HS		 3972 lli.udb_density = 1 << adap->params.sge.eq_qpp;
3973 lli.ucq_density = 1 << adap->params.sge.iq_qpp;
dcf7b6f5 3974 lli.filt_mode = adap->params.tp.vlan_pri_map;
dca4faeb
VP		 3975 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
3976 for (i = 0; i < NCHAN; i++)
3977 lli.tx_modq[i] = i;
f612b815
HS		 3978 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
3979 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
b8ff05a9 3980 lli.fw_vers = adap->params.fw_vers;
3069ee9b 3981 lli.dbfifo_int_thresh = dbfifo_int_thresh;
04e10e21
HS		 3982 lli.sge_ingpadboundary = adap->sge.fl_align;
3983 lli.sge_egrstatuspagesize = adap->sge.stat_len;
dca4faeb
VP		 3984 lli.sge_pktshift = adap->sge.pktshift;
3985 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
4c2c5763
HS		 3986 lli.max_ordird_qp = adap->params.max_ordird_qp;
3987 lli.max_ird_adapter = adap->params.max_ird_adapter;
1ac0f095 3988 lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
b8ff05a9
DM		 3989
3990 handle = ulds[uld].add(&lli);
3991 if (IS_ERR(handle)) {
3992 dev_warn(adap->pdev_dev,
3993 "could not attach to the %s driver, error %ld\n",
3994 uld_str[uld], PTR_ERR(handle));
3995 return;
3996 }
3997
3998 adap->uld_handle[uld] = handle;
3999
4000 if (!netevent_registered) {
4001 register_netevent_notifier(&cxgb4_netevent_nb);
4002 netevent_registered = true;
4003 }
e29f5dbc
DM		 4004
4005 if (adap->flags & FULL_INIT_DONE)
4006 ulds[uld].state_change(handle, CXGB4_STATE_UP);
b8ff05a9
DM		 4007}
4008
4009static void attach_ulds(struct adapter *adap)
4010{
4011 unsigned int i;
4012
01bcca68
VP		 4013 spin_lock(&adap_rcu_lock);
4014 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
4015 spin_unlock(&adap_rcu_lock);
4016
b8ff05a9
DM		 4017 mutex_lock(&uld_mutex);
4018 list_add_tail(&adap->list_node, &adapter_list);
4019 for (i = 0; i < CXGB4_ULD_MAX; i++)
4020 if (ulds[i].add)
4021 uld_attach(adap, i);
4022 mutex_unlock(&uld_mutex);
4023}
4024
4025static void detach_ulds(struct adapter *adap)
4026{
4027 unsigned int i;
4028
4029 mutex_lock(&uld_mutex);
4030 list_del(&adap->list_node);
4031 for (i = 0; i < CXGB4_ULD_MAX; i++)
4032 if (adap->uld_handle[i]) {
4033 ulds[i].state_change(adap->uld_handle[i],
4034 CXGB4_STATE_DETACH);
4035 adap->uld_handle[i] = NULL;
4036 }
4037 if (netevent_registered && list_empty(&adapter_list)) {
4038 unregister_netevent_notifier(&cxgb4_netevent_nb);
4039 netevent_registered = false;
4040 }
4041 mutex_unlock(&uld_mutex);
01bcca68
VP		 4042
4043 spin_lock(&adap_rcu_lock);
4044 list_del_rcu(&adap->rcu_node);
4045 spin_unlock(&adap_rcu_lock);
b8ff05a9
DM		 4046}
4047
4048static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
4049{
4050 unsigned int i;
4051
4052 mutex_lock(&uld_mutex);
4053 for (i = 0; i < CXGB4_ULD_MAX; i++)
4054 if (adap->uld_handle[i])
4055 ulds[i].state_change(adap->uld_handle[i], new_state);
4056 mutex_unlock(&uld_mutex);
4057}
4058
4059/**
4060 * cxgb4_register_uld - register an upper-layer driver
4061 * @type: the ULD type
4062 * @p: the ULD methods
4063 *
4064 * Registers an upper-layer driver with this driver and notifies the ULD
4065 * about any presently available devices that support its type. Returns
4066 * %-EBUSY if a ULD of the same type is already registered.
4067 */
4068int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
4069{
4070 int ret = 0;
4071 struct adapter *adap;
4072
4073 if (type >= CXGB4_ULD_MAX)
4074 return -EINVAL;
4075 mutex_lock(&uld_mutex);
4076 if (ulds[type].add) {
4077 ret = -EBUSY;
4078 goto out;
4079 }
4080 ulds[type] = *p;
4081 list_for_each_entry(adap, &adapter_list, list_node)
4082 uld_attach(adap, type);
4083out: mutex_unlock(&uld_mutex);
4084 return ret;
4085}
4086EXPORT_SYMBOL(cxgb4_register_uld);
4087
4088/**
4089 * cxgb4_unregister_uld - unregister an upper-layer driver
4090 * @type: the ULD type
4091 *
4092 * Unregisters an existing upper-layer driver.
4093 */
4094int cxgb4_unregister_uld(enum cxgb4_uld type)
4095{
4096 struct adapter *adap;
4097
4098 if (type >= CXGB4_ULD_MAX)
4099 return -EINVAL;
4100 mutex_lock(&uld_mutex);
4101 list_for_each_entry(adap, &adapter_list, list_node)
4102 adap->uld_handle[type] = NULL;
4103 ulds[type].add = NULL;
4104 mutex_unlock(&uld_mutex);
4105 return 0;
4106}
4107EXPORT_SYMBOL(cxgb4_unregister_uld);
4108
1bb60376 4109#if IS_ENABLED(CONFIG_IPV6)
b5a02f50
AB		 4110static int cxgb4_inet6addr_handler(struct notifier_block *this,
4111 unsigned long event, void *data)
01bcca68 4112{
b5a02f50
AB		 4113 struct inet6_ifaddr *ifa = data;
4114 struct net_device *event_dev = ifa->idev->dev;
4115 const struct device *parent = NULL;
4116#if IS_ENABLED(CONFIG_BONDING)
01bcca68 4117 struct adapter *adap;
b5a02f50
AB		 4118#endif
4119 if (event_dev->priv_flags & IFF_802_1Q_VLAN)
4120 event_dev = vlan_dev_real_dev(event_dev);
4121#if IS_ENABLED(CONFIG_BONDING)
4122 if (event_dev->flags & IFF_MASTER) {
4123 list_for_each_entry(adap, &adapter_list, list_node) {
4124 switch (event) {
4125 case NETDEV_UP:
4126 cxgb4_clip_get(adap->port[0],
4127 (const u32 *)ifa, 1);
4128 break;
4129 case NETDEV_DOWN:
4130 cxgb4_clip_release(adap->port[0],
4131 (const u32 *)ifa, 1);
4132 break;
4133 default:
4134 break;
4135 }
4136 }
4137 return NOTIFY_OK;
4138 }
4139#endif
01bcca68 4140
b5a02f50
AB		 4141 if (event_dev)
4142 parent = event_dev->dev.parent;
01bcca68 4143
b5a02f50 4144 if (parent && parent->driver == &cxgb4_driver.driver) {
01bcca68
VP		 4145 switch (event) {
4146 case NETDEV_UP:
b5a02f50 4147 cxgb4_clip_get(event_dev, (const u32 *)ifa, 1);
01bcca68
VP		 4148 break;
4149 case NETDEV_DOWN:
b5a02f50 4150 cxgb4_clip_release(event_dev, (const u32 *)ifa, 1);
01bcca68
VP		 4151 break;
4152 default:
4153 break;
4154 }
4155 }
b5a02f50 4156 return NOTIFY_OK;
01bcca68
VP		 4157}
4158
b5a02f50 4159static bool inet6addr_registered;
01bcca68
VP		 4160static struct notifier_block cxgb4_inet6addr_notifier = {
4161 .notifier_call = cxgb4_inet6addr_handler
4162};
4163
01bcca68
VP		 4164static void update_clip(const struct adapter *adap)
4165{
4166 int i;
4167 struct net_device *dev;
4168 int ret;
4169
4170 rcu_read_lock();
4171
4172 for (i = 0; i < MAX_NPORTS; i++) {
4173 dev = adap->port[i];
4174 ret = 0;
4175
4176 if (dev)
b5a02f50 4177 ret = cxgb4_update_root_dev_clip(dev);
01bcca68
VP		 4178
4179 if (ret < 0)
4180 break;
4181 }
4182 rcu_read_unlock();
4183}
1bb60376 4184#endif /* IS_ENABLED(CONFIG_IPV6) */
01bcca68 4185
b8ff05a9
DM		 4186/**
4187 * cxgb_up - enable the adapter
4188 * @adap: adapter being enabled
4189 *
4190 * Called when the first port is enabled, this function performs the
4191 * actions necessary to make an adapter operational, such as completing
4192 * the initialization of HW modules, and enabling interrupts.
4193 *
4194 * Must be called with the rtnl lock held.
4195 */
4196static int cxgb_up(struct adapter *adap)
4197{
aaefae9b 4198 int err;
b8ff05a9 4199
aaefae9b
DM		 4200 err = setup_sge_queues(adap);
4201 if (err)
4202 goto out;
4203 err = setup_rss(adap);
4204 if (err)
4205 goto freeq;
b8ff05a9
DM		 4206
4207 if (adap->flags & USING_MSIX) {
aaefae9b 4208 name_msix_vecs(adap);
b8ff05a9
DM		 4209 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4210 adap->msix_info[0].desc, adap);
4211 if (err)
4212 goto irq_err;
4213
4214 err = request_msix_queue_irqs(adap);
4215 if (err) {
4216 free_irq(adap->msix_info[0].vec, adap);
4217 goto irq_err;
4218 }
4219 } else {
4220 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4221 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
b1a3c2b6 4222 adap->port[0]->name, adap);
b8ff05a9
DM		 4223 if (err)
4224 goto irq_err;
4225 }
4226 enable_rx(adap);
4227 t4_sge_start(adap);
4228 t4_intr_enable(adap);
aaefae9b 4229 adap->flags |= FULL_INIT_DONE;
b8ff05a9 4230 notify_ulds(adap, CXGB4_STATE_UP);
1bb60376 4231#if IS_ENABLED(CONFIG_IPV6)
01bcca68 4232 update_clip(adap);
1bb60376 4233#endif
b8ff05a9
DM		 4234 out:
4235 return err;
4236 irq_err:
4237 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
aaefae9b
DM		 4238 freeq:
4239 t4_free_sge_resources(adap);
b8ff05a9
DM		 4240 goto out;
4241}
4242
4243static void cxgb_down(struct adapter *adapter)
4244{
4245 t4_intr_disable(adapter);
4246 cancel_work_sync(&adapter->tid_release_task);
881806bc
VP		 4247 cancel_work_sync(&adapter->db_full_task);
4248 cancel_work_sync(&adapter->db_drop_task);
b8ff05a9 4249 adapter->tid_release_task_busy = false;
204dc3c0 4250 adapter->tid_release_head = NULL;
b8ff05a9
DM		 4251
4252 if (adapter->flags & USING_MSIX) {
4253 free_msix_queue_irqs(adapter);
4254 free_irq(adapter->msix_info[0].vec, adapter);
4255 } else
4256 free_irq(adapter->pdev->irq, adapter);
4257 quiesce_rx(adapter);
aaefae9b
DM		 4258 t4_sge_stop(adapter);
4259 t4_free_sge_resources(adapter);
4260 adapter->flags &= ~FULL_INIT_DONE;
b8ff05a9
DM		 4261}
4262
4263/*
4264 * net_device operations
4265 */
4266static int cxgb_open(struct net_device *dev)
4267{
4268 int err;
4269 struct port_info *pi = netdev_priv(dev);
4270 struct adapter *adapter = pi->adapter;
4271
6a3c869a
DM		 4272 netif_carrier_off(dev);
4273
aaefae9b
DM		 4274 if (!(adapter->flags & FULL_INIT_DONE)) {
4275 err = cxgb_up(adapter);
4276 if (err < 0)
4277 return err;
4278 }
b8ff05a9 4279
f68707b8
DM		 4280 err = link_start(dev);
4281 if (!err)
4282 netif_tx_start_all_queues(dev);
4283 return err;
b8ff05a9
DM		 4284}
4285
4286static int cxgb_close(struct net_device *dev)
4287{
b8ff05a9
DM		 4288 struct port_info *pi = netdev_priv(dev);
4289 struct adapter *adapter = pi->adapter;
4290
4291 netif_tx_stop_all_queues(dev);
4292 netif_carrier_off(dev);
060e0c75 4293 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
b8ff05a9
DM		 4294}
4295
f2b7e78d
VP		 4296/* Return an error number if the indicated filter isn't writable ...
4297 */
4298static int writable_filter(struct filter_entry *f)
4299{
4300 if (f->locked)
4301 return -EPERM;
4302 if (f->pending)
4303 return -EBUSY;
4304
4305 return 0;
4306}
4307
4308/* Delete the filter at the specified index (if valid). The checks for all
4309 * the common problems with doing this like the filter being locked, currently
4310 * pending in another operation, etc.
4311 */
4312static int delete_filter(struct adapter *adapter, unsigned int fidx)
4313{
4314 struct filter_entry *f;
4315 int ret;
4316
dca4faeb 4317 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
f2b7e78d
VP		 4318 return -EINVAL;
4319
4320 f = &adapter->tids.ftid_tab[fidx];
4321 ret = writable_filter(f);
4322 if (ret)
4323 return ret;
4324 if (f->valid)
4325 return del_filter_wr(adapter, fidx);
4326
4327 return 0;
4328}
4329
dca4faeb 4330int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
793dad94
VP		 4331 __be32 sip, __be16 sport, __be16 vlan,
4332 unsigned int queue, unsigned char port, unsigned char mask)
dca4faeb
VP		 4333{
4334 int ret;
4335 struct filter_entry *f;
4336 struct adapter *adap;
4337 int i;
4338 u8 *val;
4339
4340 adap = netdev2adap(dev);
4341
1cab775c 4342 /* Adjust stid to correct filter index */
470c60c4 4343 stid -= adap->tids.sftid_base;
1cab775c
VP		 4344 stid += adap->tids.nftids;
4345
dca4faeb
VP		 4346 /* Check to make sure the filter requested is writable ...
4347 */
4348 f = &adap->tids.ftid_tab[stid];
4349 ret = writable_filter(f);
4350 if (ret)
4351 return ret;
4352
4353 /* Clear out any old resources being used by the filter before
4354 * we start constructing the new filter.
4355 */
4356 if (f->valid)
4357 clear_filter(adap, f);
4358
4359 /* Clear out filter specifications */
4360 memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4361 f->fs.val.lport = cpu_to_be16(sport);
4362 f->fs.mask.lport = ~0;
4363 val = (u8 *)&sip;
793dad94 4364 if ((val[0] | val[1] | val[2] | val[3]) != 0) {
dca4faeb
VP		 4365 for (i = 0; i < 4; i++) {
4366 f->fs.val.lip[i] = val[i];
4367 f->fs.mask.lip[i] = ~0;
4368 }
0d804338 4369 if (adap->params.tp.vlan_pri_map & PORT_F) {
793dad94
VP		 4370 f->fs.val.iport = port;
4371 f->fs.mask.iport = mask;
4372 }
4373 }
dca4faeb 4374
0d804338 4375 if (adap->params.tp.vlan_pri_map & PROTOCOL_F) {
7c89e555
KS		 4376 f->fs.val.proto = IPPROTO_TCP;
4377 f->fs.mask.proto = ~0;
4378 }
4379
dca4faeb
VP		 4380 f->fs.dirsteer = 1;
4381 f->fs.iq = queue;
4382 /* Mark filter as locked */
4383 f->locked = 1;
4384 f->fs.rpttid = 1;
4385
4386 ret = set_filter_wr(adap, stid);
4387 if (ret) {
4388 clear_filter(adap, f);
4389 return ret;
4390 }
4391
4392 return 0;
4393}
4394EXPORT_SYMBOL(cxgb4_create_server_filter);
4395
4396int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4397 unsigned int queue, bool ipv6)
4398{
4399 int ret;
4400 struct filter_entry *f;
4401 struct adapter *adap;
4402
4403 adap = netdev2adap(dev);
1cab775c
VP		 4404
4405 /* Adjust stid to correct filter index */
470c60c4 4406 stid -= adap->tids.sftid_base;
1cab775c
VP		 4407 stid += adap->tids.nftids;
4408
dca4faeb
VP		 4409 f = &adap->tids.ftid_tab[stid];
4410 /* Unlock the filter */
4411 f->locked = 0;
4412
4413 ret = delete_filter(adap, stid);
4414 if (ret)
4415 return ret;
4416
4417 return 0;
4418}
4419EXPORT_SYMBOL(cxgb4_remove_server_filter);
4420
f5152c90
DM		 4421static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4422 struct rtnl_link_stats64 *ns)
b8ff05a9
DM		 4423{
4424 struct port_stats stats;
4425 struct port_info *p = netdev_priv(dev);
4426 struct adapter *adapter = p->adapter;
b8ff05a9 4427
9fe6cb58
GS		 4428 /* Block retrieving statistics during EEH error
4429 * recovery. Otherwise, the recovery might fail
4430 * and the PCI device will be removed permanently
4431 */
b8ff05a9 4432 spin_lock(&adapter->stats_lock);
9fe6cb58
GS		 4433 if (!netif_device_present(dev)) {
4434 spin_unlock(&adapter->stats_lock);
4435 return ns;
4436 }
b8ff05a9
DM		 4437 t4_get_port_stats(adapter, p->tx_chan, &stats);
4438 spin_unlock(&adapter->stats_lock);
4439
4440 ns->tx_bytes = stats.tx_octets;
4441 ns->tx_packets = stats.tx_frames;
4442 ns->rx_bytes = stats.rx_octets;
4443 ns->rx_packets = stats.rx_frames;
4444 ns->multicast = stats.rx_mcast_frames;
4445
4446 /* detailed rx_errors */
4447 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4448 stats.rx_runt;
4449 ns->rx_over_errors = 0;
4450 ns->rx_crc_errors = stats.rx_fcs_err;
4451 ns->rx_frame_errors = stats.rx_symbol_err;
4452 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
4453 stats.rx_ovflow2 + stats.rx_ovflow3 +
4454 stats.rx_trunc0 + stats.rx_trunc1 +
4455 stats.rx_trunc2 + stats.rx_trunc3;
4456 ns->rx_missed_errors = 0;
4457
4458 /* detailed tx_errors */
4459 ns->tx_aborted_errors = 0;
4460 ns->tx_carrier_errors = 0;
4461 ns->tx_fifo_errors = 0;
4462 ns->tx_heartbeat_errors = 0;
4463 ns->tx_window_errors = 0;
4464
4465 ns->tx_errors = stats.tx_error_frames;
4466 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4467 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4468 return ns;
4469}
4470
4471static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4472{
060e0c75 4473 unsigned int mbox;
b8ff05a9
DM		 4474 int ret = 0, prtad, devad;
4475 struct port_info *pi = netdev_priv(dev);
4476 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4477
4478 switch (cmd) {
4479 case SIOCGMIIPHY:
4480 if (pi->mdio_addr < 0)
4481 return -EOPNOTSUPP;
4482 data->phy_id = pi->mdio_addr;
4483 break;
4484 case SIOCGMIIREG:
4485 case SIOCSMIIREG:
4486 if (mdio_phy_id_is_c45(data->phy_id)) {
4487 prtad = mdio_phy_id_prtad(data->phy_id);
4488 devad = mdio_phy_id_devad(data->phy_id);
4489 } else if (data->phy_id < 32) {
4490 prtad = data->phy_id;
4491 devad = 0;
4492 data->reg_num &= 0x1f;
4493 } else
4494 return -EINVAL;
4495
060e0c75 4496 mbox = pi->adapter->fn;
b8ff05a9 4497 if (cmd == SIOCGMIIREG)
060e0c75 4498 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4499 data->reg_num, &data->val_out);
4500 else
060e0c75 4501 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4502 data->reg_num, data->val_in);
4503 break;
4504 default:
4505 return -EOPNOTSUPP;
4506 }
4507 return ret;
4508}
4509
4510static void cxgb_set_rxmode(struct net_device *dev)
4511{
4512 /* unfortunately we can't return errors to the stack */
4513 set_rxmode(dev, -1, false);
4514}
4515
4516static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4517{
4518 int ret;
4519 struct port_info *pi = netdev_priv(dev);
4520
4521 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
4522 return -EINVAL;
060e0c75
DM		 4523 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4524 -1, -1, -1, true);
b8ff05a9
DM		 4525 if (!ret)
4526 dev->mtu = new_mtu;
4527 return ret;
4528}
4529
4530static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4531{
4532 int ret;
4533 struct sockaddr *addr = p;
4534 struct port_info *pi = netdev_priv(dev);
4535
4536 if (!is_valid_ether_addr(addr->sa_data))
504f9b5a 4537 return -EADDRNOTAVAIL;
b8ff05a9 4538
060e0c75
DM		 4539 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4540 pi->xact_addr_filt, addr->sa_data, true, true);
b8ff05a9
DM		 4541 if (ret < 0)
4542 return ret;
4543
4544 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4545 pi->xact_addr_filt = ret;
4546 return 0;
4547}
4548
b8ff05a9
DM		 4549#ifdef CONFIG_NET_POLL_CONTROLLER
4550static void cxgb_netpoll(struct net_device *dev)
4551{
4552 struct port_info *pi = netdev_priv(dev);
4553 struct adapter *adap = pi->adapter;
4554
4555 if (adap->flags & USING_MSIX) {
4556 int i;
4557 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4558
4559 for (i = pi->nqsets; i; i--, rx++)
4560 t4_sge_intr_msix(0, &rx->rspq);
4561 } else
4562 t4_intr_handler(adap)(0, adap);
4563}
4564#endif
4565
4566static const struct net_device_ops cxgb4_netdev_ops = {
4567 .ndo_open = cxgb_open,
4568 .ndo_stop = cxgb_close,
4569 .ndo_start_xmit = t4_eth_xmit,
688848b1 4570 .ndo_select_queue = cxgb_select_queue,
9be793bf 4571 .ndo_get_stats64 = cxgb_get_stats,
b8ff05a9
DM		 4572 .ndo_set_rx_mode = cxgb_set_rxmode,
4573 .ndo_set_mac_address = cxgb_set_mac_addr,
2ed28baa 4574 .ndo_set_features = cxgb_set_features,
b8ff05a9
DM		 4575 .ndo_validate_addr = eth_validate_addr,
4576 .ndo_do_ioctl = cxgb_ioctl,
4577 .ndo_change_mtu = cxgb_change_mtu,
b8ff05a9
DM		 4578#ifdef CONFIG_NET_POLL_CONTROLLER
4579 .ndo_poll_controller = cxgb_netpoll,
4580#endif
3a336cb1
HS		 4581#ifdef CONFIG_NET_RX_BUSY_POLL
4582 .ndo_busy_poll = cxgb_busy_poll,
4583#endif
4584
b8ff05a9
DM		 4585};
4586
4587void t4_fatal_err(struct adapter *adap)
4588{
f612b815 4589 t4_set_reg_field(adap, SGE_CONTROL_A, GLOBALENABLE_F, 0);
b8ff05a9
DM		 4590 t4_intr_disable(adap);
4591 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4592}
4593
0abfd152
HS		 4594/* Return the specified PCI-E Configuration Space register from our Physical
4595 * Function. We try first via a Firmware LDST Command since we prefer to let
4596 * the firmware own all of these registers, but if that fails we go for it
4597 * directly ourselves.
4598 */
4599static u32 t4_read_pcie_cfg4(struct adapter *adap, int reg)
4600{
4601 struct fw_ldst_cmd ldst_cmd;
4602 u32 val;
4603 int ret;
4604
4605 /* Construct and send the Firmware LDST Command to retrieve the
4606 * specified PCI-E Configuration Space register.
4607 */
4608 memset(&ldst_cmd, 0, sizeof(ldst_cmd));
4609 ldst_cmd.op_to_addrspace =
e2ac9628
HS		 4610 htonl(FW_CMD_OP_V(FW_LDST_CMD) |
4611 FW_CMD_REQUEST_F |
4612 FW_CMD_READ_F |
5167865a 4613 FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE));
0abfd152 4614 ldst_cmd.cycles_to_len16 = htonl(FW_LEN16(ldst_cmd));
5167865a 4615 ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1);
0abfd152 4616 ldst_cmd.u.pcie.ctrl_to_fn =
5167865a 4617 (FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->fn));
0abfd152
HS		 4618 ldst_cmd.u.pcie.r = reg;
4619 ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
4620 &ldst_cmd);
4621
4622 /* If the LDST Command suucceeded, exctract the returned register
4623 * value. Otherwise read it directly ourself.
4624 */
4625 if (ret == 0)
4626 val = ntohl(ldst_cmd.u.pcie.data[0]);
4627 else
4628 t4_hw_pci_read_cfg4(adap, reg, &val);
4629
4630 return val;
4631}
4632
b8ff05a9
DM		 4633static void setup_memwin(struct adapter *adap)
4634{
0abfd152 4635 u32 mem_win0_base, mem_win1_base, mem_win2_base, mem_win2_aperture;
b8ff05a9 4636
d14807dd 4637 if (is_t4(adap->params.chip)) {
0abfd152
HS		 4638 u32 bar0;
4639
4640 /* Truncation intentional: we only read the bottom 32-bits of
4641 * the 64-bit BAR0/BAR1 ... We use the hardware backdoor
4642 * mechanism to read BAR0 instead of using
4643 * pci_resource_start() because we could be operating from
4644 * within a Virtual Machine which is trapping our accesses to
4645 * our Configuration Space and we need to set up the PCI-E
4646 * Memory Window decoders with the actual addresses which will
4647 * be coming across the PCI-E link.
4648 */
4649 bar0 = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_0);
4650 bar0 &= PCI_BASE_ADDRESS_MEM_MASK;
4651 adap->t4_bar0 = bar0;
4652
19dd37ba
SR		 4653 mem_win0_base = bar0 + MEMWIN0_BASE;
4654 mem_win1_base = bar0 + MEMWIN1_BASE;
4655 mem_win2_base = bar0 + MEMWIN2_BASE;
0abfd152 4656 mem_win2_aperture = MEMWIN2_APERTURE;
19dd37ba
SR		 4657 } else {
4658 /* For T5, only relative offset inside the PCIe BAR is passed */
4659 mem_win0_base = MEMWIN0_BASE;
0abfd152 4660 mem_win1_base = MEMWIN1_BASE;
19dd37ba 4661 mem_win2_base = MEMWIN2_BASE_T5;
0abfd152 4662 mem_win2_aperture = MEMWIN2_APERTURE_T5;
19dd37ba 4663 }
f061de42
HS		 4664 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 0),
4665 mem_win0_base | BIR_V(0) |
4666 WINDOW_V(ilog2(MEMWIN0_APERTURE) - 10));
4667 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 1),
4668 mem_win1_base | BIR_V(0) |
4669 WINDOW_V(ilog2(MEMWIN1_APERTURE) - 10));
4670 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2),
4671 mem_win2_base | BIR_V(0) |
4672 WINDOW_V(ilog2(mem_win2_aperture) - 10));
4673 t4_read_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2));
636f9d37
VP		 4674}
4675
4676static void setup_memwin_rdma(struct adapter *adap)
4677{
1ae970e0 4678 if (adap->vres.ocq.size) {
0abfd152
HS		 4679 u32 start;
4680 unsigned int sz_kb;
1ae970e0 4681
0abfd152
HS		 4682 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2);
4683 start &= PCI_BASE_ADDRESS_MEM_MASK;
4684 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
1ae970e0
DM		 4685 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4686 t4_write_reg(adap,
f061de42
HS		 4687 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3),
4688 start | BIR_V(1) | WINDOW_V(ilog2(sz_kb)));
1ae970e0 4689 t4_write_reg(adap,
f061de42 4690 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3),
1ae970e0
DM		 4691 adap->vres.ocq.start);
4692 t4_read_reg(adap,
f061de42 4693 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3));
1ae970e0 4694 }
b8ff05a9
DM		 4695}
4696
02b5fb8e
DM		 4697static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4698{
4699 u32 v;
4700 int ret;
4701
4702 /* get device capabilities */
4703 memset(c, 0, sizeof(*c));
e2ac9628
HS		 4704 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4705 FW_CMD_REQUEST_F | FW_CMD_READ_F);
ce91a923 4706 c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
060e0c75 4707 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
02b5fb8e
DM		 4708 if (ret < 0)
4709 return ret;
4710
4711 /* select capabilities we'll be using */
4712 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4713 if (!vf_acls)
4714 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4715 else
4716 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4717 } else if (vf_acls) {
4718 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4719 return ret;
4720 }
e2ac9628
HS		 4721 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4722 FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
060e0c75 4723 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
02b5fb8e
DM		 4724 if (ret < 0)
4725 return ret;
4726
060e0c75 4727 ret = t4_config_glbl_rss(adap, adap->fn,
02b5fb8e 4728 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
b2e1a3f0
HS		 4729 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F |
4730 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F);
02b5fb8e
DM		 4731 if (ret < 0)
4732 return ret;
4733
060e0c75
DM		 4734 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
4735 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
02b5fb8e
DM		 4736 if (ret < 0)
4737 return ret;
4738
4739 t4_sge_init(adap);
4740
02b5fb8e 4741 /* tweak some settings */
837e4a42 4742 t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849);
0d804338 4743 t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12));
837e4a42
HS		 4744 t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A);
4745 v = t4_read_reg(adap, TP_PIO_DATA_A);
4746 t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F);
060e0c75 4747
dca4faeb
VP		 4748 /* first 4 Tx modulation queues point to consecutive Tx channels */
4749 adap->params.tp.tx_modq_map = 0xE4;
0d804338
HS		 4750 t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A,
4751 TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map));
dca4faeb
VP		 4752
4753 /* associate each Tx modulation queue with consecutive Tx channels */
4754 v = 0x84218421;
837e4a42 4755 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
0d804338 4756 &v, 1, TP_TX_SCHED_HDR_A);
837e4a42 4757 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
0d804338 4758 &v, 1, TP_TX_SCHED_FIFO_A);
837e4a42 4759 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
0d804338 4760 &v, 1, TP_TX_SCHED_PCMD_A);
dca4faeb
VP		 4761
4762#define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4763 if (is_offload(adap)) {
0d804338
HS		 4764 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A,
4765 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4766 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4767 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4768 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4769 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A,
4770 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4771 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4772 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4773 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
dca4faeb
VP		 4774 }
4775
060e0c75
DM		 4776 /* get basic stuff going */
4777 return t4_early_init(adap, adap->fn);
02b5fb8e
DM		 4778}
4779
b8ff05a9
DM		 4780/*
4781 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
4782 */
4783#define MAX_ATIDS 8192U
4784
636f9d37
VP		 4785/*
4786 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4787 *
4788 * If the firmware we're dealing with has Configuration File support, then
4789 * we use that to perform all configuration
4790 */
4791
4792/*
4793 * Tweak configuration based on module parameters, etc. Most of these have
4794 * defaults assigned to them by Firmware Configuration Files (if we're using
4795 * them) but need to be explicitly set if we're using hard-coded
4796 * initialization. But even in the case of using Firmware Configuration
4797 * Files, we'd like to expose the ability to change these via module
4798 * parameters so these are essentially common tweaks/settings for
4799 * Configuration Files and hard-coded initialization ...
4800 */
4801static int adap_init0_tweaks(struct adapter *adapter)
4802{
4803 /*
4804 * Fix up various Host-Dependent Parameters like Page Size, Cache
4805 * Line Size, etc. The firmware default is for a 4KB Page Size and
4806 * 64B Cache Line Size ...
4807 */
4808 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
4809
4810 /*
4811 * Process module parameters which affect early initialization.
4812 */
4813 if (rx_dma_offset != 2 && rx_dma_offset != 0) {
4814 dev_err(&adapter->pdev->dev,
4815 "Ignoring illegal rx_dma_offset=%d, using 2\n",
4816 rx_dma_offset);
4817 rx_dma_offset = 2;
4818 }
f612b815
HS		 4819 t4_set_reg_field(adapter, SGE_CONTROL_A,
4820 PKTSHIFT_V(PKTSHIFT_M),
4821 PKTSHIFT_V(rx_dma_offset));
636f9d37
VP		 4822
4823 /*
4824 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4825 * adds the pseudo header itself.
4826 */
837e4a42
HS		 4827 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A,
4828 CSUM_HAS_PSEUDO_HDR_F, 0);
636f9d37
VP		 4829
4830 return 0;
4831}
4832
4833/*
4834 * Attempt to initialize the adapter via a Firmware Configuration File.
4835 */
4836static int adap_init0_config(struct adapter *adapter, int reset)
4837{
4838 struct fw_caps_config_cmd caps_cmd;
4839 const struct firmware *cf;
4840 unsigned long mtype = 0, maddr = 0;
4841 u32 finiver, finicsum, cfcsum;
16e47624
HS		 4842 int ret;
4843 int config_issued = 0;
0a57a536 4844 char *fw_config_file, fw_config_file_path[256];
16e47624 4845 char *config_name = NULL;
636f9d37
VP		 4846
4847 /*
4848 * Reset device if necessary.
4849 */
4850 if (reset) {
4851 ret = t4_fw_reset(adapter, adapter->mbox,
0d804338 4852 PIORSTMODE_F | PIORST_F);
636f9d37
VP		 4853 if (ret < 0)
4854 goto bye;
4855 }
4856
4857 /*
4858 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4859 * then use that. Otherwise, use the configuration file stored
4860 * in the adapter flash ...
4861 */
d14807dd 4862 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
0a57a536 4863 case CHELSIO_T4:
16e47624 4864 fw_config_file = FW4_CFNAME;
0a57a536
SR		 4865 break;
4866 case CHELSIO_T5:
4867 fw_config_file = FW5_CFNAME;
4868 break;
4869 default:
4870 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
4871 adapter->pdev->device);
4872 ret = -EINVAL;
4873 goto bye;
4874 }
4875
4876 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
636f9d37 4877 if (ret < 0) {
16e47624 4878 config_name = "On FLASH";
636f9d37
VP		 4879 mtype = FW_MEMTYPE_CF_FLASH;
4880 maddr = t4_flash_cfg_addr(adapter);
4881 } else {
4882 u32 params[7], val[7];
4883
16e47624
HS		 4884 sprintf(fw_config_file_path,
4885 "/lib/firmware/%s", fw_config_file);
4886 config_name = fw_config_file_path;
4887
636f9d37
VP		 4888 if (cf->size >= FLASH_CFG_MAX_SIZE)
4889 ret = -ENOMEM;
4890 else {
5167865a
HS		 4891 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
4892 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
636f9d37
VP		 4893 ret = t4_query_params(adapter, adapter->mbox,
4894 adapter->fn, 0, 1, params, val);
4895 if (ret == 0) {
4896 /*
fc5ab020 4897 * For t4_memory_rw() below addresses and
636f9d37
VP		 4898 * sizes have to be in terms of multiples of 4
4899 * bytes. So, if the Configuration File isn't
4900 * a multiple of 4 bytes in length we'll have
4901 * to write that out separately since we can't
4902 * guarantee that the bytes following the
4903 * residual byte in the buffer returned by
4904 * request_firmware() are zeroed out ...
4905 */
4906 size_t resid = cf->size & 0x3;
4907 size_t size = cf->size & ~0x3;
4908 __be32 *data = (__be32 *)cf->data;
4909
5167865a
HS		 4910 mtype = FW_PARAMS_PARAM_Y_G(val[0]);
4911 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16;
636f9d37 4912
fc5ab020
HS		 4913 spin_lock(&adapter->win0_lock);
4914 ret = t4_memory_rw(adapter, 0, mtype, maddr,
4915 size, data, T4_MEMORY_WRITE);
636f9d37
VP		 4916 if (ret == 0 && resid != 0) {
4917 union {
4918 __be32 word;
4919 char buf[4];
4920 } last;
4921 int i;
4922
4923 last.word = data[size >> 2];
4924 for (i = resid; i < 4; i++)
4925 last.buf[i] = 0;
fc5ab020
HS		 4926 ret = t4_memory_rw(adapter, 0, mtype,
4927 maddr + size,
4928 4, &last.word,
4929 T4_MEMORY_WRITE);
636f9d37 4930 }
fc5ab020 4931 spin_unlock(&adapter->win0_lock);
636f9d37
VP		 4932 }
4933 }
4934
4935 release_firmware(cf);
4936 if (ret)
4937 goto bye;
4938 }
4939
4940 /*
4941 * Issue a Capability Configuration command to the firmware to get it
4942 * to parse the Configuration File. We don't use t4_fw_config_file()
4943 * because we want the ability to modify various features after we've
4944 * processed the configuration file ...
4945 */
4946 memset(&caps_cmd, 0, sizeof(caps_cmd));
4947 caps_cmd.op_to_write =
e2ac9628
HS		 4948 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4949 FW_CMD_REQUEST_F |
4950 FW_CMD_READ_F);
ce91a923 4951 caps_cmd.cfvalid_to_len16 =
5167865a
HS		 4952 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F |
4953 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) |
4954 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) |
636f9d37
VP		 4955 FW_LEN16(caps_cmd));
4956 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4957 &caps_cmd);
16e47624
HS		 4958
4959 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
4960 * Configuration File in FLASH), our last gasp effort is to use the
4961 * Firmware Configuration File which is embedded in the firmware. A
4962 * very few early versions of the firmware didn't have one embedded
4963 * but we can ignore those.
4964 */
4965 if (ret == -ENOENT) {
4966 memset(&caps_cmd, 0, sizeof(caps_cmd));
4967 caps_cmd.op_to_write =
e2ac9628
HS		 4968 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4969 FW_CMD_REQUEST_F |
4970 FW_CMD_READ_F);
16e47624
HS		 4971 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4972 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
4973 sizeof(caps_cmd), &caps_cmd);
4974 config_name = "Firmware Default";
4975 }
4976
4977 config_issued = 1;
636f9d37
VP		 4978 if (ret < 0)
4979 goto bye;
4980
4981 finiver = ntohl(caps_cmd.finiver);
4982 finicsum = ntohl(caps_cmd.finicsum);
4983 cfcsum = ntohl(caps_cmd.cfcsum);
4984 if (finicsum != cfcsum)
4985 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
4986 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
4987 finicsum, cfcsum);
4988
636f9d37
VP		 4989 /*
4990 * And now tell the firmware to use the configuration we just loaded.
4991 */
4992 caps_cmd.op_to_write =
e2ac9628
HS		 4993 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4994 FW_CMD_REQUEST_F |
4995 FW_CMD_WRITE_F);
ce91a923 4996 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 4997 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4998 NULL);
4999 if (ret < 0)
5000 goto bye;
5001
5002 /*
5003 * Tweak configuration based on system architecture, module
5004 * parameters, etc.
5005 */
5006 ret = adap_init0_tweaks(adapter);
5007 if (ret < 0)
5008 goto bye;
5009
5010 /*
5011 * And finally tell the firmware to initialize itself using the
5012 * parameters from the Configuration File.
5013 */
5014 ret = t4_fw_initialize(adapter, adapter->mbox);
5015 if (ret < 0)
5016 goto bye;
5017
06640310
HS		 5018 /* Emit Firmware Configuration File information and return
5019 * successfully.
636f9d37 5020 */
636f9d37 5021 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
16e47624
HS		 5022 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
5023 config_name, finiver, cfcsum);
636f9d37
VP		 5024 return 0;
5025
5026 /*
5027 * Something bad happened. Return the error ... (If the "error"
5028 * is that there's no Configuration File on the adapter we don't
5029 * want to issue a warning since this is fairly common.)
5030 */
5031bye:
16e47624
HS		 5032 if (config_issued && ret != -ENOENT)
5033 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
5034 config_name, -ret);
636f9d37
VP		 5035 return ret;
5036}
5037
16e47624
HS		 5038static struct fw_info fw_info_array[] = {
5039 {
5040 .chip = CHELSIO_T4,
5041 .fs_name = FW4_CFNAME,
5042 .fw_mod_name = FW4_FNAME,
5043 .fw_hdr = {
5044 .chip = FW_HDR_CHIP_T4,
5045 .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5046 .intfver_nic = FW_INTFVER(T4, NIC),
5047 .intfver_vnic = FW_INTFVER(T4, VNIC),
5048 .intfver_ri = FW_INTFVER(T4, RI),
5049 .intfver_iscsi = FW_INTFVER(T4, ISCSI),
5050 .intfver_fcoe = FW_INTFVER(T4, FCOE),
5051 },
5052 }, {
5053 .chip = CHELSIO_T5,
5054 .fs_name = FW5_CFNAME,
5055 .fw_mod_name = FW5_FNAME,
5056 .fw_hdr = {
5057 .chip = FW_HDR_CHIP_T5,
5058 .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5059 .intfver_nic = FW_INTFVER(T5, NIC),
5060 .intfver_vnic = FW_INTFVER(T5, VNIC),
5061 .intfver_ri = FW_INTFVER(T5, RI),
5062 .intfver_iscsi = FW_INTFVER(T5, ISCSI),
5063 .intfver_fcoe = FW_INTFVER(T5, FCOE),
5064 },
5065 }
5066};
5067
5068static struct fw_info *find_fw_info(int chip)
5069{
5070 int i;
5071
5072 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5073 if (fw_info_array[i].chip == chip)
5074 return &fw_info_array[i];
5075 }
5076 return NULL;
5077}
5078
b8ff05a9
DM		 5079/*
5080 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5081 */
5082static int adap_init0(struct adapter *adap)
5083{
5084 int ret;
5085 u32 v, port_vec;
5086 enum dev_state state;
5087 u32 params[7], val[7];
9a4da2cd 5088 struct fw_caps_config_cmd caps_cmd;
49aa284f
HS		 5089 struct fw_devlog_cmd devlog_cmd;
5090 u32 devlog_meminfo;
dcf7b6f5 5091 int reset = 1;
b8ff05a9 5092
666224d4
HS		 5093 /* Contact FW, advertising Master capability */
5094 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state);
b8ff05a9
DM		 5095 if (ret < 0) {
5096 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5097 ret);
5098 return ret;
5099 }
636f9d37
VP		 5100 if (ret == adap->mbox)
5101 adap->flags |= MASTER_PF;
b8ff05a9 5102
636f9d37
VP		 5103 /*
5104 * If we're the Master PF Driver and the device is uninitialized,
5105 * then let's consider upgrading the firmware ... (We always want
5106 * to check the firmware version number in order to A. get it for
5107 * later reporting and B. to warn if the currently loaded firmware
5108 * is excessively mismatched relative to the driver.)
5109 */
16e47624
HS		 5110 t4_get_fw_version(adap, &adap->params.fw_vers);
5111 t4_get_tp_version(adap, &adap->params.tp_vers);
636f9d37 5112 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
16e47624
HS		 5113 struct fw_info *fw_info;
5114 struct fw_hdr *card_fw;
5115 const struct firmware *fw;
5116 const u8 *fw_data = NULL;
5117 unsigned int fw_size = 0;
5118
5119 /* This is the firmware whose headers the driver was compiled
5120 * against
5121 */
5122 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5123 if (fw_info == NULL) {
5124 dev_err(adap->pdev_dev,
5125 "unable to get firmware info for chip %d.\n",
5126 CHELSIO_CHIP_VERSION(adap->params.chip));
5127 return -EINVAL;
636f9d37 5128 }
16e47624
HS		 5129
5130 /* allocate memory to read the header of the firmware on the
5131 * card
5132 */
5133 card_fw = t4_alloc_mem(sizeof(*card_fw));
5134
5135 /* Get FW from from /lib/firmware/ */
5136 ret = request_firmware(&fw, fw_info->fw_mod_name,
5137 adap->pdev_dev);
5138 if (ret < 0) {
5139 dev_err(adap->pdev_dev,
5140 "unable to load firmware image %s, error %d\n",
5141 fw_info->fw_mod_name, ret);
5142 } else {
5143 fw_data = fw->data;
5144 fw_size = fw->size;
5145 }
5146
5147 /* upgrade FW logic */
5148 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5149 state, &reset);
5150
5151 /* Cleaning up */
0b5b6bee 5152 release_firmware(fw);
16e47624
HS		 5153 t4_free_mem(card_fw);
5154
636f9d37 5155 if (ret < 0)
16e47624 5156 goto bye;
636f9d37 5157 }
b8ff05a9 5158
636f9d37
VP		 5159 /*
5160 * Grab VPD parameters. This should be done after we establish a
5161 * connection to the firmware since some of the VPD parameters
5162 * (notably the Core Clock frequency) are retrieved via requests to
5163 * the firmware. On the other hand, we need these fairly early on
5164 * so we do this right after getting ahold of the firmware.
5165 */
5166 ret = get_vpd_params(adap, &adap->params.vpd);
a0881cab
DM		 5167 if (ret < 0)
5168 goto bye;
a0881cab 5169
49aa284f
HS		 5170 /* Read firmware device log parameters. We really need to find a way
5171 * to get these parameters initialized with some default values (which
5172 * are likely to be correct) for the case where we either don't
5173 * attache to the firmware or it's crashed when we probe the adapter.
5174 * That way we'll still be able to perform early firmware startup
5175 * debugging ... If the request to get the Firmware's Device Log
5176 * parameters fails, we'll live so we don't make that a fatal error.
5177 */
5178 memset(&devlog_cmd, 0, sizeof(devlog_cmd));
5179 devlog_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_DEVLOG_CMD) |
5180 FW_CMD_REQUEST_F | FW_CMD_READ_F);
5181 devlog_cmd.retval_len16 = htonl(FW_LEN16(devlog_cmd));
5182 ret = t4_wr_mbox(adap, adap->mbox, &devlog_cmd, sizeof(devlog_cmd),
5183 &devlog_cmd);
5184 if (ret == 0) {
5185 devlog_meminfo =
5186 ntohl(devlog_cmd.memtype_devlog_memaddr16_devlog);
5187 adap->params.devlog.memtype =
5188 FW_DEVLOG_CMD_MEMTYPE_DEVLOG_G(devlog_meminfo);
5189 adap->params.devlog.start =
5190 FW_DEVLOG_CMD_MEMADDR16_DEVLOG_G(devlog_meminfo) << 4;
5191 adap->params.devlog.size = ntohl(devlog_cmd.memsize_devlog);
5192 }
5193
636f9d37 5194 /*
13ee15d3
VP		 5195 * Find out what ports are available to us. Note that we need to do
5196 * this before calling adap_init0_no_config() since it needs nports
5197 * and portvec ...
636f9d37
VP		 5198 */
5199 v =
5167865a
HS		 5200 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5201 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC);
636f9d37 5202 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
a0881cab
DM		 5203 if (ret < 0)
5204 goto bye;
5205
636f9d37
VP		 5206 adap->params.nports = hweight32(port_vec);
5207 adap->params.portvec = port_vec;
5208
06640310
HS		 5209 /* If the firmware is initialized already, emit a simply note to that
5210 * effect. Otherwise, it's time to try initializing the adapter.
636f9d37
VP		 5211 */
5212 if (state == DEV_STATE_INIT) {
5213 dev_info(adap->pdev_dev, "Coming up as %s: "\
5214 "Adapter already initialized\n",
5215 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
636f9d37
VP		 5216 } else {
5217 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5218 "Initializing adapter\n");
06640310
HS		 5219
5220 /* Find out whether we're dealing with a version of the
5221 * firmware which has configuration file support.
636f9d37 5222 */
06640310
HS		 5223 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5224 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
5225 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5226 params, val);
13ee15d3 5227
06640310
HS		 5228 /* If the firmware doesn't support Configuration Files,
5229 * return an error.
5230 */
5231 if (ret < 0) {
5232 dev_err(adap->pdev_dev, "firmware doesn't support "
5233 "Firmware Configuration Files\n");
5234 goto bye;
5235 }
5236
5237 /* The firmware provides us with a memory buffer where we can
5238 * load a Configuration File from the host if we want to
5239 * override the Configuration File in flash.
5240 */
5241 ret = adap_init0_config(adap, reset);
5242 if (ret == -ENOENT) {
5243 dev_err(adap->pdev_dev, "no Configuration File "
5244 "present on adapter.\n");
5245 goto bye;
636f9d37
VP		 5246 }
5247 if (ret < 0) {
06640310
HS		 5248 dev_err(adap->pdev_dev, "could not initialize "
5249 "adapter, error %d\n", -ret);
636f9d37
VP		 5250 goto bye;
5251 }
5252 }
5253
06640310
HS		 5254 /* Give the SGE code a chance to pull in anything that it needs ...
5255 * Note that this must be called after we retrieve our VPD parameters
5256 * in order to know how to convert core ticks to seconds, etc.
636f9d37 5257 */
06640310
HS		 5258 ret = t4_sge_init(adap);
5259 if (ret < 0)
5260 goto bye;
636f9d37 5261
9a4da2cd
VP		 5262 if (is_bypass_device(adap->pdev->device))
5263 adap->params.bypass = 1;
5264
636f9d37
VP		 5265 /*
5266 * Grab some of our basic fundamental operating parameters.
5267 */
5268#define FW_PARAM_DEV(param) \
5167865a
HS		 5269 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \
5270 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param))
636f9d37 5271
b8ff05a9 5272#define FW_PARAM_PFVF(param) \
5167865a
HS		 5273 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \
5274 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \
5275 FW_PARAMS_PARAM_Y_V(0) | \
5276 FW_PARAMS_PARAM_Z_V(0)
b8ff05a9 5277
636f9d37 5278 params[0] = FW_PARAM_PFVF(EQ_START);
b8ff05a9
DM		 5279 params[1] = FW_PARAM_PFVF(L2T_START);
5280 params[2] = FW_PARAM_PFVF(L2T_END);
5281 params[3] = FW_PARAM_PFVF(FILTER_START);
5282 params[4] = FW_PARAM_PFVF(FILTER_END);
e46dab4d 5283 params[5] = FW_PARAM_PFVF(IQFLINT_START);
636f9d37 5284 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
b8ff05a9
DM		 5285 if (ret < 0)
5286 goto bye;
636f9d37
VP		 5287 adap->sge.egr_start = val[0];
5288 adap->l2t_start = val[1];
5289 adap->l2t_end = val[2];
b8ff05a9
DM		 5290 adap->tids.ftid_base = val[3];
5291 adap->tids.nftids = val[4] - val[3] + 1;
e46dab4d 5292 adap->sge.ingr_start = val[5];
b8ff05a9 5293
b5a02f50
AB		 5294 params[0] = FW_PARAM_PFVF(CLIP_START);
5295 params[1] = FW_PARAM_PFVF(CLIP_END);
5296 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5297 if (ret < 0)
5298 goto bye;
5299 adap->clipt_start = val[0];
5300 adap->clipt_end = val[1];
5301
636f9d37
VP		 5302 /* query params related to active filter region */
5303 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5304 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5305 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5306 /* If Active filter size is set we enable establishing
5307 * offload connection through firmware work request
5308 */
5309 if ((val[0] != val[1]) && (ret >= 0)) {
5310 adap->flags |= FW_OFLD_CONN;
5311 adap->tids.aftid_base = val[0];
5312 adap->tids.aftid_end = val[1];
5313 }
5314
b407a4a9
VP		 5315 /* If we're running on newer firmware, let it know that we're
5316 * prepared to deal with encapsulated CPL messages. Older
5317 * firmware won't understand this and we'll just get
5318 * unencapsulated messages ...
5319 */
5320 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5321 val[0] = 1;
5322 (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5323
1ac0f095
KS		 5324 /*
5325 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5326 * capability. Earlier versions of the firmware didn't have the
5327 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5328 * permission to use ULPTX MEMWRITE DSGL.
5329 */
5330 if (is_t4(adap->params.chip)) {
5331 adap->params.ulptx_memwrite_dsgl = false;
5332 } else {
5333 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5334 ret = t4_query_params(adap, adap->mbox, adap->fn, 0,
5335 1, params, val);
5336 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
5337 }
5338
636f9d37
VP		 5339 /*
5340 * Get device capabilities so we can determine what resources we need
5341 * to manage.
5342 */
5343 memset(&caps_cmd, 0, sizeof(caps_cmd));
e2ac9628
HS		 5344 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5345 FW_CMD_REQUEST_F | FW_CMD_READ_F);
ce91a923 5346 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 5347 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5348 &caps_cmd);
5349 if (ret < 0)
5350 goto bye;
5351
13ee15d3 5352 if (caps_cmd.ofldcaps) {
b8ff05a9
DM		 5353 /* query offload-related parameters */
5354 params[0] = FW_PARAM_DEV(NTID);
5355 params[1] = FW_PARAM_PFVF(SERVER_START);
5356 params[2] = FW_PARAM_PFVF(SERVER_END);
5357 params[3] = FW_PARAM_PFVF(TDDP_START);
5358 params[4] = FW_PARAM_PFVF(TDDP_END);
5359 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
636f9d37
VP		 5360 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5361 params, val);
b8ff05a9
DM		 5362 if (ret < 0)
5363 goto bye;
5364 adap->tids.ntids = val[0];
5365 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5366 adap->tids.stid_base = val[1];
5367 adap->tids.nstids = val[2] - val[1] + 1;
636f9d37 5368 /*
dbedd44e 5369 * Setup server filter region. Divide the available filter
636f9d37
VP		 5370 * region into two parts. Regular filters get 1/3rd and server
5371 * filters get 2/3rd part. This is only enabled if workarond
5372 * path is enabled.
5373 * 1. For regular filters.
5374 * 2. Server filter: This are special filters which are used
5375 * to redirect SYN packets to offload queue.
5376 */
5377 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5378 adap->tids.sftid_base = adap->tids.ftid_base +
5379 DIV_ROUND_UP(adap->tids.nftids, 3);
5380 adap->tids.nsftids = adap->tids.nftids -
5381 DIV_ROUND_UP(adap->tids.nftids, 3);
5382 adap->tids.nftids = adap->tids.sftid_base -
5383 adap->tids.ftid_base;
5384 }
b8ff05a9
DM		 5385 adap->vres.ddp.start = val[3];
5386 adap->vres.ddp.size = val[4] - val[3] + 1;
5387 adap->params.ofldq_wr_cred = val[5];
636f9d37 5388
b8ff05a9
DM		 5389 adap->params.offload = 1;
5390 }
636f9d37 5391 if (caps_cmd.rdmacaps) {
b8ff05a9
DM		 5392 params[0] = FW_PARAM_PFVF(STAG_START);
5393 params[1] = FW_PARAM_PFVF(STAG_END);
5394 params[2] = FW_PARAM_PFVF(RQ_START);
5395 params[3] = FW_PARAM_PFVF(RQ_END);
5396 params[4] = FW_PARAM_PFVF(PBL_START);
5397 params[5] = FW_PARAM_PFVF(PBL_END);
636f9d37
VP		 5398 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5399 params, val);
b8ff05a9
DM		 5400 if (ret < 0)
5401 goto bye;
5402 adap->vres.stag.start = val[0];
5403 adap->vres.stag.size = val[1] - val[0] + 1;
5404 adap->vres.rq.start = val[2];
5405 adap->vres.rq.size = val[3] - val[2] + 1;
5406 adap->vres.pbl.start = val[4];
5407 adap->vres.pbl.size = val[5] - val[4] + 1;
a0881cab
DM		 5408
5409 params[0] = FW_PARAM_PFVF(SQRQ_START);
5410 params[1] = FW_PARAM_PFVF(SQRQ_END);
5411 params[2] = FW_PARAM_PFVF(CQ_START);
5412 params[3] = FW_PARAM_PFVF(CQ_END);
1ae970e0
DM		 5413 params[4] = FW_PARAM_PFVF(OCQ_START);
5414 params[5] = FW_PARAM_PFVF(OCQ_END);
5c937dd3
HS		 5415 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params,
5416 val);
a0881cab
DM		 5417 if (ret < 0)
5418 goto bye;
5419 adap->vres.qp.start = val[0];
5420 adap->vres.qp.size = val[1] - val[0] + 1;
5421 adap->vres.cq.start = val[2];
5422 adap->vres.cq.size = val[3] - val[2] + 1;
1ae970e0
DM		 5423 adap->vres.ocq.start = val[4];
5424 adap->vres.ocq.size = val[5] - val[4] + 1;
4c2c5763
HS		 5425
5426 params[0] = FW_PARAM_DEV(MAXORDIRD_QP);
5427 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER);
5c937dd3
HS		 5428 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params,
5429 val);
4c2c5763
HS		 5430 if (ret < 0) {
5431 adap->params.max_ordird_qp = 8;
5432 adap->params.max_ird_adapter = 32 * adap->tids.ntids;
5433 ret = 0;
5434 } else {
5435 adap->params.max_ordird_qp = val[0];
5436 adap->params.max_ird_adapter = val[1];
5437 }
5438 dev_info(adap->pdev_dev,
5439 "max_ordird_qp %d max_ird_adapter %d\n",
5440 adap->params.max_ordird_qp,
5441 adap->params.max_ird_adapter);
b8ff05a9 5442 }
636f9d37 5443 if (caps_cmd.iscsicaps) {
b8ff05a9
DM		 5444 params[0] = FW_PARAM_PFVF(ISCSI_START);
5445 params[1] = FW_PARAM_PFVF(ISCSI_END);
636f9d37
VP		 5446 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5447 params, val);
b8ff05a9
DM		 5448 if (ret < 0)
5449 goto bye;
5450 adap->vres.iscsi.start = val[0];
5451 adap->vres.iscsi.size = val[1] - val[0] + 1;
5452 }
5453#undef FW_PARAM_PFVF
5454#undef FW_PARAM_DEV
5455
92e7ae71
HS		 5456 /* The MTU/MSS Table is initialized by now, so load their values. If
5457 * we're initializing the adapter, then we'll make any modifications
5458 * we want to the MTU/MSS Table and also initialize the congestion
5459 * parameters.
636f9d37 5460 */
b8ff05a9 5461 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
92e7ae71
HS		 5462 if (state != DEV_STATE_INIT) {
5463 int i;
5464
5465 /* The default MTU Table contains values 1492 and 1500.
5466 * However, for TCP, it's better to have two values which are
5467 * a multiple of 8 +/- 4 bytes apart near this popular MTU.
5468 * This allows us to have a TCP Data Payload which is a
5469 * multiple of 8 regardless of what combination of TCP Options
5470 * are in use (always a multiple of 4 bytes) which is
5471 * important for performance reasons. For instance, if no
5472 * options are in use, then we have a 20-byte IP header and a
5473 * 20-byte TCP header. In this case, a 1500-byte MSS would
5474 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
5475 * which is not a multiple of 8. So using an MSS of 1488 in
5476 * this case results in a TCP Data Payload of 1448 bytes which
5477 * is a multiple of 8. On the other hand, if 12-byte TCP Time
5478 * Stamps have been negotiated, then an MTU of 1500 bytes
5479 * results in a TCP Data Payload of 1448 bytes which, as
5480 * above, is a multiple of 8 bytes ...
5481 */
5482 for (i = 0; i < NMTUS; i++)
5483 if (adap->params.mtus[i] == 1492) {
5484 adap->params.mtus[i] = 1488;
5485 break;
5486 }
7ee9ff94 5487
92e7ae71
HS		 5488 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5489 adap->params.b_wnd);
5490 }
df64e4d3 5491 t4_init_sge_params(adap);
dcf7b6f5 5492 t4_init_tp_params(adap);
636f9d37 5493 adap->flags |= FW_OK;
b8ff05a9
DM		 5494 return 0;
5495
5496 /*
636f9d37
VP		 5497 * Something bad happened. If a command timed out or failed with EIO
5498 * FW does not operate within its spec or something catastrophic
5499 * happened to HW/FW, stop issuing commands.
b8ff05a9 5500 */
636f9d37
VP		 5501bye:
5502 if (ret != -ETIMEDOUT && ret != -EIO)
5503 t4_fw_bye(adap, adap->mbox);
b8ff05a9
DM		 5504 return ret;
5505}
5506
204dc3c0
DM		 5507/* EEH callbacks */
5508
5509static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5510 pci_channel_state_t state)
5511{
5512 int i;
5513 struct adapter *adap = pci_get_drvdata(pdev);
5514
5515 if (!adap)
5516 goto out;
5517
5518 rtnl_lock();
5519 adap->flags &= ~FW_OK;
5520 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
9fe6cb58 5521 spin_lock(&adap->stats_lock);
204dc3c0
DM		 5522 for_each_port(adap, i) {
5523 struct net_device *dev = adap->port[i];
5524
5525 netif_device_detach(dev);
5526 netif_carrier_off(dev);
5527 }
9fe6cb58 5528 spin_unlock(&adap->stats_lock);
204dc3c0
DM		 5529 if (adap->flags & FULL_INIT_DONE)
5530 cxgb_down(adap);
5531 rtnl_unlock();
144be3d9
GS		 5532 if ((adap->flags & DEV_ENABLED)) {
5533 pci_disable_device(pdev);
5534 adap->flags &= ~DEV_ENABLED;
5535 }
204dc3c0
DM		 5536out: return state == pci_channel_io_perm_failure ?
5537 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5538}
5539
5540static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5541{
5542 int i, ret;
5543 struct fw_caps_config_cmd c;
5544 struct adapter *adap = pci_get_drvdata(pdev);
5545
5546 if (!adap) {
5547 pci_restore_state(pdev);
5548 pci_save_state(pdev);
5549 return PCI_ERS_RESULT_RECOVERED;
5550 }
5551
144be3d9
GS		 5552 if (!(adap->flags & DEV_ENABLED)) {
5553 if (pci_enable_device(pdev)) {
5554 dev_err(&pdev->dev, "Cannot reenable PCI "
5555 "device after reset\n");
5556 return PCI_ERS_RESULT_DISCONNECT;
5557 }
5558 adap->flags |= DEV_ENABLED;
204dc3c0
DM		 5559 }
5560
5561 pci_set_master(pdev);
5562 pci_restore_state(pdev);
5563 pci_save_state(pdev);
5564 pci_cleanup_aer_uncorrect_error_status(pdev);
5565
8203b509 5566 if (t4_wait_dev_ready(adap->regs) < 0)
204dc3c0 5567 return PCI_ERS_RESULT_DISCONNECT;
777c2300 5568 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
204dc3c0
DM		 5569 return PCI_ERS_RESULT_DISCONNECT;
5570 adap->flags |= FW_OK;
5571 if (adap_init1(adap, &c))
5572 return PCI_ERS_RESULT_DISCONNECT;
5573
5574 for_each_port(adap, i) {
5575 struct port_info *p = adap2pinfo(adap, i);
5576
060e0c75
DM		 5577 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
5578 NULL, NULL);
204dc3c0
DM		 5579 if (ret < 0)
5580 return PCI_ERS_RESULT_DISCONNECT;
5581 p->viid = ret;
5582 p->xact_addr_filt = -1;
5583 }
5584
5585 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5586 adap->params.b_wnd);
1ae970e0 5587 setup_memwin(adap);
204dc3c0
DM		 5588 if (cxgb_up(adap))
5589 return PCI_ERS_RESULT_DISCONNECT;
5590 return PCI_ERS_RESULT_RECOVERED;
5591}
5592
5593static void eeh_resume(struct pci_dev *pdev)
5594{
5595 int i;
5596 struct adapter *adap = pci_get_drvdata(pdev);
5597
5598 if (!adap)
5599 return;
5600
5601 rtnl_lock();
5602 for_each_port(adap, i) {
5603 struct net_device *dev = adap->port[i];
5604
5605 if (netif_running(dev)) {
5606 link_start(dev);
5607 cxgb_set_rxmode(dev);
5608 }
5609 netif_device_attach(dev);
5610 }
5611 rtnl_unlock();
5612}
5613
3646f0e5 5614static const struct pci_error_handlers cxgb4_eeh = {
204dc3c0
DM		 5615 .error_detected = eeh_err_detected,
5616 .slot_reset = eeh_slot_reset,
5617 .resume = eeh_resume,
5618};
5619
57d8b764 5620static inline bool is_x_10g_port(const struct link_config *lc)
b8ff05a9 5621{
57d8b764
KS		 5622 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
5623 (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
b8ff05a9
DM		 5624}
5625
c887ad0e
HS		 5626static inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
5627 unsigned int us, unsigned int cnt,
b8ff05a9
DM		 5628 unsigned int size, unsigned int iqe_size)
5629{
c887ad0e
HS		 5630 q->adap = adap;
5631 set_rspq_intr_params(q, us, cnt);
b8ff05a9
DM		 5632 q->iqe_len = iqe_size;
5633 q->size = size;
5634}
5635
5636/*
5637 * Perform default configuration of DMA queues depending on the number and type
5638 * of ports we found and the number of available CPUs. Most settings can be
5639 * modified by the admin prior to actual use.
5640 */
91744948 5641static void cfg_queues(struct adapter *adap)
b8ff05a9
DM		 5642{
5643 struct sge *s = &adap->sge;
688848b1
AB		 5644 int i, n10g = 0, qidx = 0;
5645#ifndef CONFIG_CHELSIO_T4_DCB
5646 int q10g = 0;
5647#endif
cf38be6d 5648 int ciq_size;
b8ff05a9
DM		 5649
5650 for_each_port(adap, i)
57d8b764 5651 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
688848b1
AB		 5652#ifdef CONFIG_CHELSIO_T4_DCB
5653 /* For Data Center Bridging support we need to be able to support up
5654 * to 8 Traffic Priorities; each of which will be assigned to its
5655 * own TX Queue in order to prevent Head-Of-Line Blocking.
5656 */
5657 if (adap->params.nports * 8 > MAX_ETH_QSETS) {
5658 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n",
5659 MAX_ETH_QSETS, adap->params.nports * 8);
5660 BUG_ON(1);
5661 }
b8ff05a9 5662
688848b1
AB		 5663 for_each_port(adap, i) {
5664 struct port_info *pi = adap2pinfo(adap, i);
5665
5666 pi->first_qset = qidx;
5667 pi->nqsets = 8;
5668 qidx += pi->nqsets;
5669 }
5670#else /* !CONFIG_CHELSIO_T4_DCB */
b8ff05a9
DM		 5671 /*
5672 * We default to 1 queue per non-10G port and up to # of cores queues
5673 * per 10G port.
5674 */
5675 if (n10g)
5676 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5952dde7
YM		 5677 if (q10g > netif_get_num_default_rss_queues())
5678 q10g = netif_get_num_default_rss_queues();
b8ff05a9
DM		 5679
5680 for_each_port(adap, i) {
5681 struct port_info *pi = adap2pinfo(adap, i);
5682
5683 pi->first_qset = qidx;
57d8b764 5684 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
b8ff05a9
DM		 5685 qidx += pi->nqsets;
5686 }
688848b1 5687#endif /* !CONFIG_CHELSIO_T4_DCB */
b8ff05a9
DM		 5688
5689 s->ethqsets = qidx;
5690 s->max_ethqsets = qidx; /* MSI-X may lower it later */
5691
5692 if (is_offload(adap)) {
5693 /*
5694 * For offload we use 1 queue/channel if all ports are up to 1G,
5695 * otherwise we divide all available queues amongst the channels
5696 * capped by the number of available cores.
5697 */
5698 if (n10g) {
5699 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
5700 num_online_cpus());
5701 s->ofldqsets = roundup(i, adap->params.nports);
5702 } else
5703 s->ofldqsets = adap->params.nports;
5704 /* For RDMA one Rx queue per channel suffices */
5705 s->rdmaqs = adap->params.nports;
f36e58e5
HS		 5706 /* Try and allow at least 1 CIQ per cpu rounding down
5707 * to the number of ports, with a minimum of 1 per port.
5708 * A 2 port card in a 6 cpu system: 6 CIQs, 3 / port.
5709 * A 4 port card in a 6 cpu system: 4 CIQs, 1 / port.
5710 * A 4 port card in a 2 cpu system: 4 CIQs, 1 / port.
5711 */
5712 s->rdmaciqs = min_t(int, MAX_RDMA_CIQS, num_online_cpus());
5713 s->rdmaciqs = (s->rdmaciqs / adap->params.nports) *
5714 adap->params.nports;
5715 s->rdmaciqs = max_t(int, s->rdmaciqs, adap->params.nports);
b8ff05a9
DM		 5716 }
5717
5718 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
5719 struct sge_eth_rxq *r = &s->ethrxq[i];
5720
c887ad0e 5721 init_rspq(adap, &r->rspq, 5, 10, 1024, 64);
b8ff05a9
DM		 5722 r->fl.size = 72;
5723 }
5724
5725 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
5726 s->ethtxq[i].q.size = 1024;
5727
5728 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
5729 s->ctrlq[i].q.size = 512;
5730
5731 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
5732 s->ofldtxq[i].q.size = 1024;
5733
5734 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
5735 struct sge_ofld_rxq *r = &s->ofldrxq[i];
5736
c887ad0e 5737 init_rspq(adap, &r->rspq, 5, 1, 1024, 64);
b8ff05a9
DM		 5738 r->rspq.uld = CXGB4_ULD_ISCSI;
5739 r->fl.size = 72;
5740 }
5741
5742 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
5743 struct sge_ofld_rxq *r = &s->rdmarxq[i];
5744
c887ad0e 5745 init_rspq(adap, &r->rspq, 5, 1, 511, 64);
b8ff05a9
DM		 5746 r->rspq.uld = CXGB4_ULD_RDMA;
5747 r->fl.size = 72;
5748 }
5749
cf38be6d
HS		 5750 ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
5751 if (ciq_size > SGE_MAX_IQ_SIZE) {
5752 CH_WARN(adap, "CIQ size too small for available IQs\n");
5753 ciq_size = SGE_MAX_IQ_SIZE;
5754 }
5755
5756 for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) {
5757 struct sge_ofld_rxq *r = &s->rdmaciq[i];
5758
c887ad0e 5759 init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64);
cf38be6d
HS		 5760 r->rspq.uld = CXGB4_ULD_RDMA;
5761 }
5762
c887ad0e
HS		 5763 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64);
5764 init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64);
b8ff05a9
DM		 5765}
5766
5767/*
5768 * Reduce the number of Ethernet queues across all ports to at most n.
5769 * n provides at least one queue per port.
5770 */
91744948 5771static void reduce_ethqs(struct adapter *adap, int n)
b8ff05a9
DM		 5772{
5773 int i;
5774 struct port_info *pi;
5775
5776 while (n < adap->sge.ethqsets)
5777 for_each_port(adap, i) {
5778 pi = adap2pinfo(adap, i);
5779 if (pi->nqsets > 1) {
5780 pi->nqsets--;
5781 adap->sge.ethqsets--;
5782 if (adap->sge.ethqsets <= n)
5783 break;
5784 }
5785 }
5786
5787 n = 0;
5788 for_each_port(adap, i) {
5789 pi = adap2pinfo(adap, i);
5790 pi->first_qset = n;
5791 n += pi->nqsets;
5792 }
5793}
5794
5795/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5796#define EXTRA_VECS 2
5797
91744948 5798static int enable_msix(struct adapter *adap)
b8ff05a9
DM		 5799{
5800 int ofld_need = 0;
f36e58e5 5801 int i, want, need, allocated;
b8ff05a9
DM		 5802 struct sge *s = &adap->sge;
5803 unsigned int nchan = adap->params.nports;
f36e58e5
HS		 5804 struct msix_entry *entries;
5805
5806 entries = kmalloc(sizeof(*entries) * (MAX_INGQ + 1),
5807 GFP_KERNEL);
5808 if (!entries)
5809 return -ENOMEM;
b8ff05a9 5810
f36e58e5 5811 for (i = 0; i < MAX_INGQ + 1; ++i)
b8ff05a9
DM		 5812 entries[i].entry = i;
5813
5814 want = s->max_ethqsets + EXTRA_VECS;
5815 if (is_offload(adap)) {
cf38be6d 5816 want += s->rdmaqs + s->rdmaciqs + s->ofldqsets;
b8ff05a9 5817 /* need nchan for each possible ULD */
cf38be6d 5818 ofld_need = 3 * nchan;
b8ff05a9 5819 }
688848b1
AB		 5820#ifdef CONFIG_CHELSIO_T4_DCB
5821 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
5822 * each port.
5823 */
5824 need = 8 * adap->params.nports + EXTRA_VECS + ofld_need;
5825#else
b8ff05a9 5826 need = adap->params.nports + EXTRA_VECS + ofld_need;
688848b1 5827#endif
f36e58e5
HS		 5828 allocated = pci_enable_msix_range(adap->pdev, entries, need, want);
5829 if (allocated < 0) {
5830 dev_info(adap->pdev_dev, "not enough MSI-X vectors left,"
5831 " not using MSI-X\n");
5832 kfree(entries);
5833 return allocated;
5834 }
b8ff05a9 5835
f36e58e5 5836 /* Distribute available vectors to the various queue groups.
c32ad224
AG		 5837 * Every group gets its minimum requirement and NIC gets top
5838 * priority for leftovers.
5839 */
f36e58e5 5840 i = allocated - EXTRA_VECS - ofld_need;
c32ad224
AG		 5841 if (i < s->max_ethqsets) {
5842 s->max_ethqsets = i;
5843 if (i < s->ethqsets)
5844 reduce_ethqs(adap, i);
5845 }
5846 if (is_offload(adap)) {
f36e58e5
HS		 5847 if (allocated < want) {
5848 s->rdmaqs = nchan;
5849 s->rdmaciqs = nchan;
5850 }
5851
5852 /* leftovers go to OFLD */
5853 i = allocated - EXTRA_VECS - s->max_ethqsets -
5854 s->rdmaqs - s->rdmaciqs;
c32ad224
AG		 5855 s->ofldqsets = (i / nchan) * nchan; /* round down */
5856 }
f36e58e5 5857 for (i = 0; i < allocated; ++i)
c32ad224
AG		 5858 adap->msix_info[i].vec = entries[i].vector;
5859
f36e58e5 5860 kfree(entries);
c32ad224 5861 return 0;
b8ff05a9
DM		 5862}
5863
5864#undef EXTRA_VECS
5865
91744948 5866static int init_rss(struct adapter *adap)
671b0060
DM		 5867{
5868 unsigned int i, j;
5869
5870 for_each_port(adap, i) {
5871 struct port_info *pi = adap2pinfo(adap, i);
5872
5873 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
5874 if (!pi->rss)
5875 return -ENOMEM;
5876 for (j = 0; j < pi->rss_size; j++)
278bc429 5877 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
671b0060
DM		 5878 }
5879 return 0;
5880}
5881
91744948 5882static void print_port_info(const struct net_device *dev)
b8ff05a9 5883{
b8ff05a9 5884 char buf[80];
118969ed 5885 char *bufp = buf;
f1a051b9 5886 const char *spd = "";
118969ed
DM		 5887 const struct port_info *pi = netdev_priv(dev);
5888 const struct adapter *adap = pi->adapter;
f1a051b9
DM		 5889
5890 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
5891 spd = " 2.5 GT/s";
5892 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
5893 spd = " 5 GT/s";
d2e752db
RD		 5894 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB)
5895 spd = " 8 GT/s";
b8ff05a9 5896
118969ed
DM		 5897 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
5898 bufp += sprintf(bufp, "100/");
5899 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
5900 bufp += sprintf(bufp, "1000/");
5901 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
5902 bufp += sprintf(bufp, "10G/");
72aca4bf
KS		 5903 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
5904 bufp += sprintf(bufp, "40G/");
118969ed
DM		 5905 if (bufp != buf)
5906 --bufp;
72aca4bf 5907 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
118969ed
DM		 5908
5909 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
0a57a536 5910 adap->params.vpd.id,
d14807dd 5911 CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
118969ed
DM		 5912 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
5913 (adap->flags & USING_MSIX) ? " MSI-X" :
5914 (adap->flags & USING_MSI) ? " MSI" : "");
a94cd705
KS		 5915 netdev_info(dev, "S/N: %s, P/N: %s\n",
5916 adap->params.vpd.sn, adap->params.vpd.pn);
b8ff05a9
DM		 5917}
5918
91744948 5919static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
ef306b50 5920{
e5c8ae5f 5921 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
ef306b50
DM		 5922}
5923
06546391
DM		 5924/*
5925 * Free the following resources:
5926 * - memory used for tables
5927 * - MSI/MSI-X
5928 * - net devices
5929 * - resources FW is holding for us
5930 */
5931static void free_some_resources(struct adapter *adapter)
5932{
5933 unsigned int i;
5934
5935 t4_free_mem(adapter->l2t);
5936 t4_free_mem(adapter->tids.tid_tab);
5937 disable_msi(adapter);
5938
5939 for_each_port(adapter, i)
671b0060
DM		 5940 if (adapter->port[i]) {
5941 kfree(adap2pinfo(adapter, i)->rss);
06546391 5942 free_netdev(adapter->port[i]);
671b0060 5943 }
06546391 5944 if (adapter->flags & FW_OK)
060e0c75 5945 t4_fw_bye(adapter, adapter->fn);
06546391
DM		 5946}
5947
2ed28baa 5948#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
35d35682 5949#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
b8ff05a9 5950 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
22adfe0a 5951#define SEGMENT_SIZE 128
b8ff05a9 5952
1dd06ae8 5953static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
b8ff05a9 5954{
22adfe0a 5955 int func, i, err, s_qpp, qpp, num_seg;
b8ff05a9 5956 struct port_info *pi;
c8f44aff 5957 bool highdma = false;
b8ff05a9 5958 struct adapter *adapter = NULL;
d6ce2628 5959 void __iomem *regs;
b8ff05a9
DM		 5960
5961 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
5962
5963 err = pci_request_regions(pdev, KBUILD_MODNAME);
5964 if (err) {
5965 /* Just info, some other driver may have claimed the device. */
5966 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
5967 return err;
5968 }
5969
b8ff05a9
DM		 5970 err = pci_enable_device(pdev);
5971 if (err) {
5972 dev_err(&pdev->dev, "cannot enable PCI device\n");
5973 goto out_release_regions;
5974 }
5975
d6ce2628
HS		 5976 regs = pci_ioremap_bar(pdev, 0);
5977 if (!regs) {
5978 dev_err(&pdev->dev, "cannot map device registers\n");
5979 err = -ENOMEM;
5980 goto out_disable_device;
5981 }
5982
8203b509
HS		 5983 err = t4_wait_dev_ready(regs);
5984 if (err < 0)
5985 goto out_unmap_bar0;
5986
d6ce2628 5987 /* We control everything through one PF */
0d804338 5988 func = SOURCEPF_G(readl(regs + PL_WHOAMI_A));
d6ce2628
HS		 5989 if (func != ent->driver_data) {
5990 iounmap(regs);
5991 pci_disable_device(pdev);
5992 pci_save_state(pdev); /* to restore SR-IOV later */
5993 goto sriov;
5994 }
5995
b8ff05a9 5996 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
c8f44aff 5997 highdma = true;
b8ff05a9
DM		 5998 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
5999 if (err) {
6000 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
6001 "coherent allocations\n");
d6ce2628 6002 goto out_unmap_bar0;
b8ff05a9
DM		 6003 }
6004 } else {
6005 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
6006 if (err) {
6007 dev_err(&pdev->dev, "no usable DMA configuration\n");
d6ce2628 6008 goto out_unmap_bar0;
b8ff05a9
DM		 6009 }
6010 }
6011
6012 pci_enable_pcie_error_reporting(pdev);
ef306b50 6013 enable_pcie_relaxed_ordering(pdev);
b8ff05a9
DM		 6014 pci_set_master(pdev);
6015 pci_save_state(pdev);
6016
6017 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
6018 if (!adapter) {
6019 err = -ENOMEM;
d6ce2628 6020 goto out_unmap_bar0;
b8ff05a9
DM		 6021 }
6022
29aaee65
AB		 6023 adapter->workq = create_singlethread_workqueue("cxgb4");
6024 if (!adapter->workq) {
6025 err = -ENOMEM;
6026 goto out_free_adapter;
6027 }
6028
144be3d9
GS		 6029 /* PCI device has been enabled */
6030 adapter->flags |= DEV_ENABLED;
6031
d6ce2628 6032 adapter->regs = regs;
b8ff05a9
DM		 6033 adapter->pdev = pdev;
6034 adapter->pdev_dev = &pdev->dev;
3069ee9b 6035 adapter->mbox = func;
060e0c75 6036 adapter->fn = func;
b8ff05a9
DM		 6037 adapter->msg_enable = dflt_msg_enable;
6038 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
6039
6040 spin_lock_init(&adapter->stats_lock);
6041 spin_lock_init(&adapter->tid_release_lock);
e327c225 6042 spin_lock_init(&adapter->win0_lock);
b8ff05a9
DM		 6043
6044 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
881806bc
VP		 6045 INIT_WORK(&adapter->db_full_task, process_db_full);
6046 INIT_WORK(&adapter->db_drop_task, process_db_drop);
b8ff05a9
DM		 6047
6048 err = t4_prep_adapter(adapter);
6049 if (err)
d6ce2628
HS		 6050 goto out_free_adapter;
6051
22adfe0a 6052
d14807dd 6053 if (!is_t4(adapter->params.chip)) {
f612b815
HS		 6054 s_qpp = (QUEUESPERPAGEPF0_S +
6055 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) *
6056 adapter->fn);
6057 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter,
6058 SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp);
22adfe0a
SR		 6059 num_seg = PAGE_SIZE / SEGMENT_SIZE;
6060
6061 /* Each segment size is 128B. Write coalescing is enabled only
6062 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
6063 * queue is less no of segments that can be accommodated in
6064 * a page size.
6065 */
6066 if (qpp > num_seg) {
6067 dev_err(&pdev->dev,
6068 "Incorrect number of egress queues per page\n");
6069 err = -EINVAL;
d6ce2628 6070 goto out_free_adapter;
22adfe0a
SR		 6071 }
6072 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
6073 pci_resource_len(pdev, 2));
6074 if (!adapter->bar2) {
6075 dev_err(&pdev->dev, "cannot map device bar2 region\n");
6076 err = -ENOMEM;
d6ce2628 6077 goto out_free_adapter;
22adfe0a
SR		 6078 }
6079 }
6080
636f9d37 6081 setup_memwin(adapter);
b8ff05a9 6082 err = adap_init0(adapter);
636f9d37 6083 setup_memwin_rdma(adapter);
b8ff05a9
DM		 6084 if (err)
6085 goto out_unmap_bar;
6086
6087 for_each_port(adapter, i) {
6088 struct net_device *netdev;
6089
6090 netdev = alloc_etherdev_mq(sizeof(struct port_info),
6091 MAX_ETH_QSETS);
6092 if (!netdev) {
6093 err = -ENOMEM;
6094 goto out_free_dev;
6095 }
6096
6097 SET_NETDEV_DEV(netdev, &pdev->dev);
6098
6099 adapter->port[i] = netdev;
6100 pi = netdev_priv(netdev);
6101 pi->adapter = adapter;
6102 pi->xact_addr_filt = -1;
b8ff05a9 6103 pi->port_id = i;
b8ff05a9
DM		 6104 netdev->irq = pdev->irq;
6105
2ed28baa
MM		 6106 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6107 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6108 NETIF_F_RXCSUM | NETIF_F_RXHASH |
f646968f 6109 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
c8f44aff
MM		 6110 if (highdma)
6111 netdev->hw_features |= NETIF_F_HIGHDMA;
6112 netdev->features |= netdev->hw_features;
b8ff05a9
DM		 6113 netdev->vlan_features = netdev->features & VLAN_FEAT;
6114
01789349
JP		 6115 netdev->priv_flags |= IFF_UNICAST_FLT;
6116
b8ff05a9 6117 netdev->netdev_ops = &cxgb4_netdev_ops;
688848b1
AB		 6118#ifdef CONFIG_CHELSIO_T4_DCB
6119 netdev->dcbnl_ops = &cxgb4_dcb_ops;
6120 cxgb4_dcb_state_init(netdev);
6121#endif
7ad24ea4 6122 netdev->ethtool_ops = &cxgb_ethtool_ops;
b8ff05a9
DM		 6123 }
6124
6125 pci_set_drvdata(pdev, adapter);
6126
6127 if (adapter->flags & FW_OK) {
060e0c75 6128 err = t4_port_init(adapter, func, func, 0);
b8ff05a9
DM		 6129 if (err)
6130 goto out_free_dev;
6131 }
6132
6133 /*
6134 * Configure queues and allocate tables now, they can be needed as
6135 * soon as the first register_netdev completes.
6136 */
6137 cfg_queues(adapter);
6138
6139 adapter->l2t = t4_init_l2t();
6140 if (!adapter->l2t) {
6141 /* We tolerate a lack of L2T, giving up some functionality */
6142 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6143 adapter->params.offload = 0;
6144 }
6145
b5a02f50
AB		 6146#if IS_ENABLED(CONFIG_IPV6)
6147 adapter->clipt = t4_init_clip_tbl(adapter->clipt_start,
6148 adapter->clipt_end);
6149 if (!adapter->clipt) {
6150 /* We tolerate a lack of clip_table, giving up
6151 * some functionality
6152 */
6153 dev_warn(&pdev->dev,
6154 "could not allocate Clip table, continuing\n");
6155 adapter->params.offload = 0;
6156 }
6157#endif
b8ff05a9
DM		 6158 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6159 dev_warn(&pdev->dev, "could not allocate TID table, "
6160 "continuing\n");
6161 adapter->params.offload = 0;
6162 }
6163
f7cabcdd
DM		 6164 /* See what interrupts we'll be using */
6165 if (msi > 1 && enable_msix(adapter) == 0)
6166 adapter->flags |= USING_MSIX;
6167 else if (msi > 0 && pci_enable_msi(pdev) == 0)
6168 adapter->flags |= USING_MSI;
6169
671b0060
DM		 6170 err = init_rss(adapter);
6171 if (err)
6172 goto out_free_dev;
6173
b8ff05a9
DM		 6174 /*
6175 * The card is now ready to go. If any errors occur during device
6176 * registration we do not fail the whole card but rather proceed only
6177 * with the ports we manage to register successfully. However we must
6178 * register at least one net device.
6179 */
6180 for_each_port(adapter, i) {
a57cabe0
DM		 6181 pi = adap2pinfo(adapter, i);
6182 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6183 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6184
b8ff05a9
DM		 6185 err = register_netdev(adapter->port[i]);
6186 if (err)
b1a3c2b6 6187 break;
b1a3c2b6
DM		 6188 adapter->chan_map[pi->tx_chan] = i;
6189 print_port_info(adapter->port[i]);
b8ff05a9 6190 }
b1a3c2b6 6191 if (i == 0) {
b8ff05a9
DM		 6192 dev_err(&pdev->dev, "could not register any net devices\n");
6193 goto out_free_dev;
6194 }
b1a3c2b6
DM		 6195 if (err) {
6196 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6197 err = 0;
6403eab1 6198 }
b8ff05a9
DM		 6199
6200 if (cxgb4_debugfs_root) {
6201 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6202 cxgb4_debugfs_root);
6203 setup_debugfs(adapter);
6204 }
6205
6482aa7c
DLR		 6206 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6207 pdev->needs_freset = 1;
6208
b8ff05a9
DM		 6209 if (is_offload(adapter))
6210 attach_ulds(adapter);
6211
8e1e6059 6212sriov:
b8ff05a9 6213#ifdef CONFIG_PCI_IOV
7d6727cf 6214 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
b8ff05a9
DM		 6215 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6216 dev_info(&pdev->dev,
6217 "instantiated %u virtual functions\n",
6218 num_vf[func]);
6219#endif
6220 return 0;
6221
6222 out_free_dev:
06546391 6223 free_some_resources(adapter);
b8ff05a9 6224 out_unmap_bar:
d14807dd 6225 if (!is_t4(adapter->params.chip))
22adfe0a 6226 iounmap(adapter->bar2);
b8ff05a9 6227 out_free_adapter:
29aaee65
AB		 6228 if (adapter->workq)
6229 destroy_workqueue(adapter->workq);
6230
b8ff05a9 6231 kfree(adapter);
d6ce2628
HS		 6232 out_unmap_bar0:
6233 iounmap(regs);
b8ff05a9
DM		 6234 out_disable_device:
6235 pci_disable_pcie_error_reporting(pdev);
6236 pci_disable_device(pdev);
6237 out_release_regions:
6238 pci_release_regions(pdev);
b8ff05a9
DM		 6239 return err;
6240}
6241
91744948 6242static void remove_one(struct pci_dev *pdev)
b8ff05a9
DM		 6243{
6244 struct adapter *adapter = pci_get_drvdata(pdev);
6245
636f9d37 6246#ifdef CONFIG_PCI_IOV
b8ff05a9
DM		 6247 pci_disable_sriov(pdev);
6248
636f9d37
VP		 6249#endif
6250
b8ff05a9
DM		 6251 if (adapter) {
6252 int i;
6253
29aaee65
AB		 6254 /* Tear down per-adapter Work Queue first since it can contain
6255 * references to our adapter data structure.
6256 */
6257 destroy_workqueue(adapter->workq);
6258
b8ff05a9
DM		 6259 if (is_offload(adapter))
6260 detach_ulds(adapter);
6261
6262 for_each_port(adapter, i)
8f3a7676 6263 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
b8ff05a9
DM		 6264 unregister_netdev(adapter->port[i]);
6265
9f16dc2e 6266 debugfs_remove_recursive(adapter->debugfs_root);
b8ff05a9 6267
f2b7e78d
VP		 6268 /* If we allocated filters, free up state associated with any
6269 * valid filters ...
6270 */
6271 if (adapter->tids.ftid_tab) {
6272 struct filter_entry *f = &adapter->tids.ftid_tab[0];
dca4faeb
VP		 6273 for (i = 0; i < (adapter->tids.nftids +
6274 adapter->tids.nsftids); i++, f++)
f2b7e78d
VP		 6275 if (f->valid)
6276 clear_filter(adapter, f);
6277 }
6278
aaefae9b
DM		 6279 if (adapter->flags & FULL_INIT_DONE)
6280 cxgb_down(adapter);
b8ff05a9 6281
06546391 6282 free_some_resources(adapter);
b5a02f50
AB		 6283#if IS_ENABLED(CONFIG_IPV6)
6284 t4_cleanup_clip_tbl(adapter);
6285#endif
b8ff05a9 6286 iounmap(adapter->regs);
d14807dd 6287 if (!is_t4(adapter->params.chip))
22adfe0a 6288 iounmap(adapter->bar2);
b8ff05a9 6289 pci_disable_pcie_error_reporting(pdev);
144be3d9
GS		 6290 if ((adapter->flags & DEV_ENABLED)) {
6291 pci_disable_device(pdev);
6292 adapter->flags &= ~DEV_ENABLED;
6293 }
b8ff05a9 6294 pci_release_regions(pdev);
ee9a33b2 6295 synchronize_rcu();
8b662fe7 6296 kfree(adapter);
a069ec91 6297 } else
b8ff05a9
DM		 6298 pci_release_regions(pdev);
6299}
6300
6301static struct pci_driver cxgb4_driver = {
6302 .name = KBUILD_MODNAME,
6303 .id_table = cxgb4_pci_tbl,
6304 .probe = init_one,
91744948 6305 .remove = remove_one,
687d705c 6306 .shutdown = remove_one,
204dc3c0 6307 .err_handler = &cxgb4_eeh,
b8ff05a9
DM		 6308};
6309
6310static int __init cxgb4_init_module(void)
6311{
6312 int ret;
6313
6314 /* Debugfs support is optional, just warn if this fails */
6315 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6316 if (!cxgb4_debugfs_root)
428ac43f 6317 pr_warn("could not create debugfs entry, continuing\n");
b8ff05a9
DM		 6318
6319 ret = pci_register_driver(&cxgb4_driver);
29aaee65 6320 if (ret < 0)
b8ff05a9 6321 debugfs_remove(cxgb4_debugfs_root);
01bcca68 6322
1bb60376 6323#if IS_ENABLED(CONFIG_IPV6)
b5a02f50
AB		 6324 if (!inet6addr_registered) {
6325 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6326 inet6addr_registered = true;
6327 }
1bb60376 6328#endif
01bcca68 6329
b8ff05a9
DM		 6330 return ret;
6331}
6332
6333static void __exit cxgb4_cleanup_module(void)
6334{
1bb60376 6335#if IS_ENABLED(CONFIG_IPV6)
1793c798 6336 if (inet6addr_registered) {
b5a02f50
AB		 6337 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6338 inet6addr_registered = false;
6339 }
1bb60376 6340#endif
b8ff05a9
DM		 6341 pci_unregister_driver(&cxgb4_driver);
6342 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
6343}
6344
6345module_init(cxgb4_init_module);
6346module_exit(cxgb4_cleanup_module);
A mirror of Linus' kernel repository