]> git.ipfire.org Git - thirdparty/linux.git/blame - drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
projects / thirdparty / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bridge: Add ability to enable TSO
[thirdparty/linux.git] / drivers / net / ethernet / chelsio / cxgb4 / cxgb4_main.c
CommitLineData
b8ff05a9
DM		 1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
ce100b8b 4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
b8ff05a9
DM		 5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
01789349 44#include <linux/if.h>
b8ff05a9
DM		 45#include <linux/if_vlan.h>
46#include <linux/init.h>
47#include <linux/log2.h>
48#include <linux/mdio.h>
49#include <linux/module.h>
50#include <linux/moduleparam.h>
51#include <linux/mutex.h>
52#include <linux/netdevice.h>
53#include <linux/pci.h>
54#include <linux/aer.h>
55#include <linux/rtnetlink.h>
56#include <linux/sched.h>
57#include <linux/seq_file.h>
58#include <linux/sockios.h>
59#include <linux/vmalloc.h>
60#include <linux/workqueue.h>
61#include <net/neighbour.h>
62#include <net/netevent.h>
01bcca68 63#include <net/addrconf.h>
1ef8019b 64#include <net/bonding.h>
b8ff05a9
DM		 65#include <asm/uaccess.h>
66
67#include "cxgb4.h"
68#include "t4_regs.h"
f612b815 69#include "t4_values.h"
b8ff05a9
DM		 70#include "t4_msg.h"
71#include "t4fw_api.h"
688848b1 72#include "cxgb4_dcb.h"
fd88b31a 73#include "cxgb4_debugfs.h"
b8ff05a9
DM		 74#include "l2t.h"
75
01bcca68
VP		 76#ifdef DRV_VERSION
77#undef DRV_VERSION
78#endif
3a7f8554
SR		 79#define DRV_VERSION "2.0.0-ko"
80#define DRV_DESC "Chelsio T4/T5 Network Driver"
b8ff05a9
DM		 81
82/*
83 * Max interrupt hold-off timer value in us. Queues fall back to this value
84 * under extreme memory pressure so it's largish to give the system time to
85 * recover.
86 */
87#define MAX_SGE_TIMERVAL 200U
88
7ee9ff94 89enum {
13ee15d3
VP		 90 /*
91 * Physical Function provisioning constants.
92 */
93 PFRES_NVI = 4, /* # of Virtual Interfaces */
94 PFRES_NETHCTRL = 128, /* # of EQs used for ETH or CTRL Qs */
95 PFRES_NIQFLINT = 128, /* # of ingress Qs/w Free List(s)/intr
96 */
97 PFRES_NEQ = 256, /* # of egress queues */
98 PFRES_NIQ = 0, /* # of ingress queues */
99 PFRES_TC = 0, /* PCI-E traffic class */
100 PFRES_NEXACTF = 128, /* # of exact MPS filters */
101
102 PFRES_R_CAPS = FW_CMD_CAP_PF,
103 PFRES_WX_CAPS = FW_CMD_CAP_PF,
104
105#ifdef CONFIG_PCI_IOV
106 /*
107 * Virtual Function provisioning constants. We need two extra Ingress
108 * Queues with Interrupt capability to serve as the VF's Firmware
109 * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
110 * neither will have Free Lists associated with them). For each
111 * Ethernet/Control Egress Queue and for each Free List, we need an
112 * Egress Context.
113 */
7ee9ff94
CL		 114 VFRES_NPORTS = 1, /* # of "ports" per VF */
115 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
116
117 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
118 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
119 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
7ee9ff94 120 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
13ee15d3 121 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
7ee9ff94
CL		 122 VFRES_TC = 0, /* PCI-E traffic class */
123 VFRES_NEXACTF = 16, /* # of exact MPS filters */
124
125 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
126 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
13ee15d3 127#endif
7ee9ff94
CL		 128};
129
130/*
131 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
132 * static and likely not to be useful in the long run. We really need to
133 * implement some form of persistent configuration which the firmware
134 * controls.
135 */
136static unsigned int pfvfres_pmask(struct adapter *adapter,
137 unsigned int pf, unsigned int vf)
138{
139 unsigned int portn, portvec;
140
141 /*
142 * Give PF's access to all of the ports.
143 */
144 if (vf == 0)
5167865a 145 return FW_PFVF_CMD_PMASK_M;
7ee9ff94
CL		 146
147 /*
148 * For VFs, we'll assign them access to the ports based purely on the
149 * PF. We assign active ports in order, wrapping around if there are
150 * fewer active ports than PFs: e.g. active port[pf % nports].
151 * Unfortunately the adapter's port_info structs haven't been
152 * initialized yet so we have to compute this.
153 */
154 if (adapter->params.nports == 0)
155 return 0;
156
157 portn = pf % adapter->params.nports;
158 portvec = adapter->params.portvec;
159 for (;;) {
160 /*
161 * Isolate the lowest set bit in the port vector. If we're at
162 * the port number that we want, return that as the pmask.
163 * otherwise mask that bit out of the port vector and
164 * decrement our port number ...
165 */
166 unsigned int pmask = portvec ^ (portvec & (portvec-1));
167 if (portn == 0)
168 return pmask;
169 portn--;
170 portvec &= ~pmask;
171 }
172 /*NOTREACHED*/
173}
7ee9ff94 174
b8ff05a9
DM		 175enum {
176 MAX_TXQ_ENTRIES = 16384,
177 MAX_CTRL_TXQ_ENTRIES = 1024,
178 MAX_RSPQ_ENTRIES = 16384,
179 MAX_RX_BUFFERS = 16384,
180 MIN_TXQ_ENTRIES = 32,
181 MIN_CTRL_TXQ_ENTRIES = 32,
182 MIN_RSPQ_ENTRIES = 128,
183 MIN_FL_ENTRIES = 16
184};
185
f2b7e78d
VP		 186/* Host shadow copy of ingress filter entry. This is in host native format
187 * and doesn't match the ordering or bit order, etc. of the hardware of the
188 * firmware command. The use of bit-field structure elements is purely to
189 * remind ourselves of the field size limitations and save memory in the case
190 * where the filter table is large.
191 */
192struct filter_entry {
193 /* Administrative fields for filter.
194 */
195 u32 valid:1; /* filter allocated and valid */
196 u32 locked:1; /* filter is administratively locked */
197
198 u32 pending:1; /* filter action is pending firmware reply */
199 u32 smtidx:8; /* Source MAC Table index for smac */
200 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */
201
202 /* The filter itself. Most of this is a straight copy of information
203 * provided by the extended ioctl(). Some fields are translated to
204 * internal forms -- for instance the Ingress Queue ID passed in from
205 * the ioctl() is translated into the Absolute Ingress Queue ID.
206 */
207 struct ch_filter_specification fs;
208};
209
b8ff05a9
DM		 210#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
211 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
212 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
213
3fedeab1
HS		 214/* Macros needed to support the PCI Device ID Table ...
215 */
216#define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
217 static struct pci_device_id cxgb4_pci_tbl[] = {
218#define CH_PCI_DEVICE_ID_FUNCTION 0x4
b8ff05a9 219
3fedeab1
HS		 220/* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is
221 * called for both.
222 */
223#define CH_PCI_DEVICE_ID_FUNCTION2 0x0
224
225#define CH_PCI_ID_TABLE_ENTRY(devid) \
226 {PCI_VDEVICE(CHELSIO, (devid)), 4}
227
228#define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
229 { 0, } \
230 }
231
232#include "t4_pci_id_tbl.h"
b8ff05a9 233
16e47624 234#define FW4_FNAME "cxgb4/t4fw.bin"
0a57a536 235#define FW5_FNAME "cxgb4/t5fw.bin"
16e47624 236#define FW4_CFNAME "cxgb4/t4-config.txt"
0a57a536 237#define FW5_CFNAME "cxgb4/t5-config.txt"
b8ff05a9
DM		 238
239MODULE_DESCRIPTION(DRV_DESC);
240MODULE_AUTHOR("Chelsio Communications");
241MODULE_LICENSE("Dual BSD/GPL");
242MODULE_VERSION(DRV_VERSION);
243MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
16e47624 244MODULE_FIRMWARE(FW4_FNAME);
0a57a536 245MODULE_FIRMWARE(FW5_FNAME);
b8ff05a9 246
636f9d37
VP		 247/*
248 * Normally we're willing to become the firmware's Master PF but will be happy
249 * if another PF has already become the Master and initialized the adapter.
250 * Setting "force_init" will cause this driver to forcibly establish itself as
251 * the Master PF and initialize the adapter.
252 */
253static uint force_init;
254
255module_param(force_init, uint, 0644);
256MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
257
13ee15d3
VP		 258/*
259 * Normally if the firmware we connect to has Configuration File support, we
260 * use that and only fall back to the old Driver-based initialization if the
261 * Configuration File fails for some reason. If force_old_init is set, then
262 * we'll always use the old Driver-based initialization sequence.
263 */
264static uint force_old_init;
265
266module_param(force_old_init, uint, 0644);
267MODULE_PARM_DESC(force_old_init, "Force old initialization sequence");
268
b8ff05a9
DM		 269static int dflt_msg_enable = DFLT_MSG_ENABLE;
270
271module_param(dflt_msg_enable, int, 0644);
272MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
273
274/*
275 * The driver uses the best interrupt scheme available on a platform in the
276 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
277 * of these schemes the driver may consider as follows:
278 *
279 * msi = 2: choose from among all three options
280 * msi = 1: only consider MSI and INTx interrupts
281 * msi = 0: force INTx interrupts
282 */
283static int msi = 2;
284
285module_param(msi, int, 0644);
286MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
287
288/*
289 * Queue interrupt hold-off timer values. Queues default to the first of these
290 * upon creation.
291 */
292static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
293
294module_param_array(intr_holdoff, uint, NULL, 0644);
295MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
296 "0..4 in microseconds");
297
298static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
299
300module_param_array(intr_cnt, uint, NULL, 0644);
301MODULE_PARM_DESC(intr_cnt,
302 "thresholds 1..3 for queue interrupt packet counters");
303
636f9d37
VP		 304/*
305 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
306 * offset by 2 bytes in order to have the IP headers line up on 4-byte
307 * boundaries. This is a requirement for many architectures which will throw
308 * a machine check fault if an attempt is made to access one of the 4-byte IP
309 * header fields on a non-4-byte boundary. And it's a major performance issue
310 * even on some architectures which allow it like some implementations of the
311 * x86 ISA. However, some architectures don't mind this and for some very
312 * edge-case performance sensitive applications (like forwarding large volumes
313 * of small packets), setting this DMA offset to 0 will decrease the number of
314 * PCI-E Bus transfers enough to measurably affect performance.
315 */
316static int rx_dma_offset = 2;
317
eb939922 318static bool vf_acls;
b8ff05a9
DM		 319
320#ifdef CONFIG_PCI_IOV
321module_param(vf_acls, bool, 0644);
322MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
323
7d6727cf
SR		 324/* Configure the number of PCI-E Virtual Function which are to be instantiated
325 * on SR-IOV Capable Physical Functions.
0a57a536 326 */
7d6727cf 327static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
b8ff05a9
DM		 328
329module_param_array(num_vf, uint, NULL, 0644);
7d6727cf 330MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
b8ff05a9
DM		 331#endif
332
688848b1
AB		 333/* TX Queue select used to determine what algorithm to use for selecting TX
334 * queue. Select between the kernel provided function (select_queue=0) or user
335 * cxgb_select_queue function (select_queue=1)
336 *
337 * Default: select_queue=0
338 */
339static int select_queue;
340module_param(select_queue, int, 0644);
341MODULE_PARM_DESC(select_queue,
342 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method.");
343
13ee15d3
VP		 344/*
345 * The filter TCAM has a fixed portion and a variable portion. The fixed
346 * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
347 * ports. The variable portion is 36 bits which can include things like Exact
348 * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
349 * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
350 * far exceed the 36-bit budget for this "compressed" header portion of the
351 * filter. Thus, we have a scarce resource which must be carefully managed.
352 *
353 * By default we set this up to mostly match the set of filter matching
354 * capabilities of T3 but with accommodations for some of T4's more
355 * interesting features:
356 *
357 * { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
358 * [Inner] VLAN (17), Port (3), FCoE (1) }
359 */
360enum {
361 TP_VLAN_PRI_MAP_DEFAULT = HW_TPL_FR_MT_PR_IV_P_FC,
837e4a42
HS		 362 TP_VLAN_PRI_MAP_FIRST = FCOE_S,
363 TP_VLAN_PRI_MAP_LAST = FRAGMENTATION_S,
13ee15d3
VP		 364};
365
366static unsigned int tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
367
f2b7e78d
VP		 368module_param(tp_vlan_pri_map, uint, 0644);
369MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration");
370
b8ff05a9
DM		 371static struct dentry *cxgb4_debugfs_root;
372
373static LIST_HEAD(adapter_list);
374static DEFINE_MUTEX(uld_mutex);
01bcca68
VP		 375/* Adapter list to be accessed from atomic context */
376static LIST_HEAD(adap_rcu_list);
377static DEFINE_SPINLOCK(adap_rcu_lock);
b8ff05a9
DM		 378static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
379static const char *uld_str[] = { "RDMA", "iSCSI" };
380
381static void link_report(struct net_device *dev)
382{
383 if (!netif_carrier_ok(dev))
384 netdev_info(dev, "link down\n");
385 else {
386 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
387
388 const char *s = "10Mbps";
389 const struct port_info *p = netdev_priv(dev);
390
391 switch (p->link_cfg.speed) {
e8b39015 392 case 10000:
b8ff05a9
DM		 393 s = "10Gbps";
394 break;
e8b39015 395 case 1000:
b8ff05a9
DM		 396 s = "1000Mbps";
397 break;
e8b39015 398 case 100:
b8ff05a9
DM		 399 s = "100Mbps";
400 break;
e8b39015 401 case 40000:
72aca4bf
KS		 402 s = "40Gbps";
403 break;
b8ff05a9
DM		 404 }
405
406 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
407 fc[p->link_cfg.fc]);
408 }
409}
410
688848b1
AB		 411#ifdef CONFIG_CHELSIO_T4_DCB
412/* Set up/tear down Data Center Bridging Priority mapping for a net device. */
413static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable)
414{
415 struct port_info *pi = netdev_priv(dev);
416 struct adapter *adap = pi->adapter;
417 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset];
418 int i;
419
420 /* We use a simple mapping of Port TX Queue Index to DCB
421 * Priority when we're enabling DCB.
422 */
423 for (i = 0; i < pi->nqsets; i++, txq++) {
424 u32 name, value;
425 int err;
426
5167865a
HS		 427 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
428 FW_PARAMS_PARAM_X_V(
429 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) |
430 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id));
688848b1
AB		 431 value = enable ? i : 0xffffffff;
432
433 /* Since we can be called while atomic (from "interrupt
434 * level") we need to issue the Set Parameters Commannd
435 * without sleeping (timeout < 0).
436 */
437 err = t4_set_params_nosleep(adap, adap->mbox, adap->fn, 0, 1,
438 &name, &value);
439
440 if (err)
441 dev_err(adap->pdev_dev,
442 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n",
443 enable ? "set" : "unset", pi->port_id, i, -err);
10b00466
AB		 444 else
445 txq->dcb_prio = value;
688848b1
AB		 446 }
447}
448#endif /* CONFIG_CHELSIO_T4_DCB */
449
b8ff05a9
DM		 450void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
451{
452 struct net_device *dev = adapter->port[port_id];
453
454 /* Skip changes from disabled ports. */
455 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
456 if (link_stat)
457 netif_carrier_on(dev);
688848b1
AB		 458 else {
459#ifdef CONFIG_CHELSIO_T4_DCB
460 cxgb4_dcb_state_init(dev);
461 dcb_tx_queue_prio_enable(dev, false);
462#endif /* CONFIG_CHELSIO_T4_DCB */
b8ff05a9 463 netif_carrier_off(dev);
688848b1 464 }
b8ff05a9
DM		 465
466 link_report(dev);
467 }
468}
469
470void t4_os_portmod_changed(const struct adapter *adap, int port_id)
471{
472 static const char *mod_str[] = {
a0881cab 473 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
b8ff05a9
DM		 474 };
475
476 const struct net_device *dev = adap->port[port_id];
477 const struct port_info *pi = netdev_priv(dev);
478
479 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
480 netdev_info(dev, "port module unplugged\n");
a0881cab 481 else if (pi->mod_type < ARRAY_SIZE(mod_str))
b8ff05a9
DM		 482 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
483}
484
485/*
486 * Configure the exact and hash address filters to handle a port's multicast
487 * and secondary unicast MAC addresses.
488 */
489static int set_addr_filters(const struct net_device *dev, bool sleep)
490{
491 u64 mhash = 0;
492 u64 uhash = 0;
493 bool free = true;
494 u16 filt_idx[7];
495 const u8 *addr[7];
496 int ret, naddr = 0;
b8ff05a9
DM		 497 const struct netdev_hw_addr *ha;
498 int uc_cnt = netdev_uc_count(dev);
4a35ecf8 499 int mc_cnt = netdev_mc_count(dev);
b8ff05a9 500 const struct port_info *pi = netdev_priv(dev);
060e0c75 501 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 502
503 /* first do the secondary unicast addresses */
504 netdev_for_each_uc_addr(ha, dev) {
505 addr[naddr++] = ha->addr;
506 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 507 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 508 naddr, addr, filt_idx, &uhash, sleep);
509 if (ret < 0)
510 return ret;
511
512 free = false;
513 naddr = 0;
514 }
515 }
516
517 /* next set up the multicast addresses */
4a35ecf8
DM		 518 netdev_for_each_mc_addr(ha, dev) {
519 addr[naddr++] = ha->addr;
520 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 521 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 522 naddr, addr, filt_idx, &mhash, sleep);
523 if (ret < 0)
524 return ret;
525
526 free = false;
527 naddr = 0;
528 }
529 }
530
060e0c75 531 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
b8ff05a9
DM		 532 uhash | mhash, sleep);
533}
534
3069ee9b
VP		 535int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
536module_param(dbfifo_int_thresh, int, 0644);
537MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
538
404d9e3f
VP		 539/*
540 * usecs to sleep while draining the dbfifo
541 */
542static int dbfifo_drain_delay = 1000;
3069ee9b
VP		 543module_param(dbfifo_drain_delay, int, 0644);
544MODULE_PARM_DESC(dbfifo_drain_delay,
545 "usecs to sleep while draining the dbfifo");
546
b8ff05a9
DM		 547/*
548 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
549 * If @mtu is -1 it is left unchanged.
550 */
551static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
552{
553 int ret;
554 struct port_info *pi = netdev_priv(dev);
555
556 ret = set_addr_filters(dev, sleep_ok);
557 if (ret == 0)
060e0c75 558 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
b8ff05a9 559 (dev->flags & IFF_PROMISC) ? 1 : 0,
f8f5aafa 560 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
b8ff05a9
DM		 561 sleep_ok);
562 return ret;
563}
564
565/**
566 * link_start - enable a port
567 * @dev: the port to enable
568 *
569 * Performs the MAC and PHY actions needed to enable a port.
570 */
571static int link_start(struct net_device *dev)
572{
573 int ret;
574 struct port_info *pi = netdev_priv(dev);
060e0c75 575 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 576
577 /*
578 * We do not set address filters and promiscuity here, the stack does
579 * that step explicitly.
580 */
060e0c75 581 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
f646968f 582 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
b8ff05a9 583 if (ret == 0) {
060e0c75 584 ret = t4_change_mac(pi->adapter, mb, pi->viid,
b8ff05a9 585 pi->xact_addr_filt, dev->dev_addr, true,
b6bd29e7 586 true);
b8ff05a9
DM		 587 if (ret >= 0) {
588 pi->xact_addr_filt = ret;
589 ret = 0;
590 }
591 }
592 if (ret == 0)
060e0c75
DM		 593 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
594 &pi->link_cfg);
30f00847
AB		 595 if (ret == 0) {
596 local_bh_disable();
688848b1
AB		 597 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true,
598 true, CXGB4_DCB_ENABLED);
30f00847
AB		 599 local_bh_enable();
600 }
688848b1 601
b8ff05a9
DM		 602 return ret;
603}
604
688848b1
AB		 605int cxgb4_dcb_enabled(const struct net_device *dev)
606{
607#ifdef CONFIG_CHELSIO_T4_DCB
608 struct port_info *pi = netdev_priv(dev);
609
3bb06261
AB		 610 if (!pi->dcb.enabled)
611 return 0;
612
613 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) ||
614 (pi->dcb.state == CXGB4_DCB_STATE_HOST));
688848b1
AB		 615#else
616 return 0;
617#endif
618}
619EXPORT_SYMBOL(cxgb4_dcb_enabled);
620
621#ifdef CONFIG_CHELSIO_T4_DCB
622/* Handle a Data Center Bridging update message from the firmware. */
623static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd)
624{
2b5fb1f2 625 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid));
688848b1
AB		 626 struct net_device *dev = adap->port[port];
627 int old_dcb_enabled = cxgb4_dcb_enabled(dev);
628 int new_dcb_enabled;
629
630 cxgb4_dcb_handle_fw_update(adap, pcmd);
631 new_dcb_enabled = cxgb4_dcb_enabled(dev);
632
633 /* If the DCB has become enabled or disabled on the port then we're
634 * going to need to set up/tear down DCB Priority parameters for the
635 * TX Queues associated with the port.
636 */
637 if (new_dcb_enabled != old_dcb_enabled)
638 dcb_tx_queue_prio_enable(dev, new_dcb_enabled);
639}
640#endif /* CONFIG_CHELSIO_T4_DCB */
641
f2b7e78d
VP		 642/* Clear a filter and release any of its resources that we own. This also
643 * clears the filter's "pending" status.
644 */
645static void clear_filter(struct adapter *adap, struct filter_entry *f)
646{
647 /* If the new or old filter have loopback rewriteing rules then we'll
648 * need to free any existing Layer Two Table (L2T) entries of the old
649 * filter rule. The firmware will handle freeing up any Source MAC
650 * Table (SMT) entries used for rewriting Source MAC Addresses in
651 * loopback rules.
652 */
653 if (f->l2t)
654 cxgb4_l2t_release(f->l2t);
655
656 /* The zeroing of the filter rule below clears the filter valid,
657 * pending, locked flags, l2t pointer, etc. so it's all we need for
658 * this operation.
659 */
660 memset(f, 0, sizeof(*f));
661}
662
663/* Handle a filter write/deletion reply.
664 */
665static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
666{
667 unsigned int idx = GET_TID(rpl);
668 unsigned int nidx = idx - adap->tids.ftid_base;
669 unsigned int ret;
670 struct filter_entry *f;
671
672 if (idx >= adap->tids.ftid_base && nidx <
673 (adap->tids.nftids + adap->tids.nsftids)) {
674 idx = nidx;
675 ret = GET_TCB_COOKIE(rpl->cookie);
676 f = &adap->tids.ftid_tab[idx];
677
678 if (ret == FW_FILTER_WR_FLT_DELETED) {
679 /* Clear the filter when we get confirmation from the
680 * hardware that the filter has been deleted.
681 */
682 clear_filter(adap, f);
683 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
684 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
685 idx);
686 clear_filter(adap, f);
687 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
688 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
689 f->pending = 0; /* asynchronous setup completed */
690 f->valid = 1;
691 } else {
692 /* Something went wrong. Issue a warning about the
693 * problem and clear everything out.
694 */
695 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
696 idx, ret);
697 clear_filter(adap, f);
698 }
699 }
700}
701
702/* Response queue handler for the FW event queue.
b8ff05a9
DM		 703 */
704static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
705 const struct pkt_gl *gl)
706{
707 u8 opcode = ((const struct rss_header *)rsp)->opcode;
708
709 rsp++; /* skip RSS header */
b407a4a9
VP		 710
711 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
712 */
713 if (unlikely(opcode == CPL_FW4_MSG &&
714 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
715 rsp++;
716 opcode = ((const struct rss_header *)rsp)->opcode;
717 rsp++;
718 if (opcode != CPL_SGE_EGR_UPDATE) {
719 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
720 , opcode);
721 goto out;
722 }
723 }
724
b8ff05a9
DM		 725 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
726 const struct cpl_sge_egr_update *p = (void *)rsp;
727 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
e46dab4d 728 struct sge_txq *txq;
b8ff05a9 729
e46dab4d 730 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
b8ff05a9 731 txq->restarts++;
e46dab4d 732 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
b8ff05a9
DM		 733 struct sge_eth_txq *eq;
734
735 eq = container_of(txq, struct sge_eth_txq, q);
736 netif_tx_wake_queue(eq->txq);
737 } else {
738 struct sge_ofld_txq *oq;
739
740 oq = container_of(txq, struct sge_ofld_txq, q);
741 tasklet_schedule(&oq->qresume_tsk);
742 }
743 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
744 const struct cpl_fw6_msg *p = (void *)rsp;
745
688848b1
AB		 746#ifdef CONFIG_CHELSIO_T4_DCB
747 const struct fw_port_cmd *pcmd = (const void *)p->data;
e2ac9628 748 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid));
688848b1 749 unsigned int action =
2b5fb1f2 750 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16));
688848b1
AB		 751
752 if (cmd == FW_PORT_CMD &&
753 action == FW_PORT_ACTION_GET_PORT_INFO) {
2b5fb1f2 754 int port = FW_PORT_CMD_PORTID_G(
688848b1
AB		 755 be32_to_cpu(pcmd->op_to_portid));
756 struct net_device *dev = q->adap->port[port];
757 int state_input = ((pcmd->u.info.dcbxdis_pkd &
2b5fb1f2 758 FW_PORT_CMD_DCBXDIS_F)
688848b1
AB		 759 ? CXGB4_DCB_INPUT_FW_DISABLED
760 : CXGB4_DCB_INPUT_FW_ENABLED);
761
762 cxgb4_dcb_state_fsm(dev, state_input);
763 }
764
765 if (cmd == FW_PORT_CMD &&
766 action == FW_PORT_ACTION_L2_DCB_CFG)
767 dcb_rpl(q->adap, pcmd);
768 else
769#endif
770 if (p->type == 0)
771 t4_handle_fw_rpl(q->adap, p->data);
b8ff05a9
DM		 772 } else if (opcode == CPL_L2T_WRITE_RPL) {
773 const struct cpl_l2t_write_rpl *p = (void *)rsp;
774
775 do_l2t_write_rpl(q->adap, p);
f2b7e78d
VP		 776 } else if (opcode == CPL_SET_TCB_RPL) {
777 const struct cpl_set_tcb_rpl *p = (void *)rsp;
778
779 filter_rpl(q->adap, p);
b8ff05a9
DM		 780 } else
781 dev_err(q->adap->pdev_dev,
782 "unexpected CPL %#x on FW event queue\n", opcode);
b407a4a9 783out:
b8ff05a9
DM		 784 return 0;
785}
786
787/**
788 * uldrx_handler - response queue handler for ULD queues
789 * @q: the response queue that received the packet
790 * @rsp: the response queue descriptor holding the offload message
791 * @gl: the gather list of packet fragments
792 *
793 * Deliver an ingress offload packet to a ULD. All processing is done by
794 * the ULD, we just maintain statistics.
795 */
796static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
797 const struct pkt_gl *gl)
798{
799 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
800
b407a4a9
VP		 801 /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
802 */
803 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
804 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
805 rsp += 2;
806
b8ff05a9
DM		 807 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
808 rxq->stats.nomem++;
809 return -1;
810 }
811 if (gl == NULL)
812 rxq->stats.imm++;
813 else if (gl == CXGB4_MSG_AN)
814 rxq->stats.an++;
815 else
816 rxq->stats.pkts++;
817 return 0;
818}
819
820static void disable_msi(struct adapter *adapter)
821{
822 if (adapter->flags & USING_MSIX) {
823 pci_disable_msix(adapter->pdev);
824 adapter->flags &= ~USING_MSIX;
825 } else if (adapter->flags & USING_MSI) {
826 pci_disable_msi(adapter->pdev);
827 adapter->flags &= ~USING_MSI;
828 }
829}
830
831/*
832 * Interrupt handler for non-data events used with MSI-X.
833 */
834static irqreturn_t t4_nondata_intr(int irq, void *cookie)
835{
836 struct adapter *adap = cookie;
0d804338 837 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A));
b8ff05a9 838
0d804338 839 if (v & PFSW_F) {
b8ff05a9 840 adap->swintr = 1;
0d804338 841 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v);
b8ff05a9
DM		 842 }
843 t4_slow_intr_handler(adap);
844 return IRQ_HANDLED;
845}
846
847/*
848 * Name the MSI-X interrupts.
849 */
850static void name_msix_vecs(struct adapter *adap)
851{
ba27816c 852 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
b8ff05a9
DM		 853
854 /* non-data interrupts */
b1a3c2b6 855 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
b8ff05a9
DM		 856
857 /* FW events */
b1a3c2b6
DM		 858 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
859 adap->port[0]->name);
b8ff05a9
DM		 860
861 /* Ethernet queues */
862 for_each_port(adap, j) {
863 struct net_device *d = adap->port[j];
864 const struct port_info *pi = netdev_priv(d);
865
ba27816c 866 for (i = 0; i < pi->nqsets; i++, msi_idx++)
b8ff05a9
DM		 867 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
868 d->name, i);
b8ff05a9
DM		 869 }
870
871 /* offload queues */
ba27816c
DM		 872 for_each_ofldrxq(&adap->sge, i)
873 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
b1a3c2b6 874 adap->port[0]->name, i);
ba27816c
DM		 875
876 for_each_rdmarxq(&adap->sge, i)
877 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
b1a3c2b6 878 adap->port[0]->name, i);
cf38be6d
HS		 879
880 for_each_rdmaciq(&adap->sge, i)
881 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d",
882 adap->port[0]->name, i);
b8ff05a9
DM		 883}
884
885static int request_msix_queue_irqs(struct adapter *adap)
886{
887 struct sge *s = &adap->sge;
cf38be6d
HS		 888 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0;
889 int msi_index = 2;
b8ff05a9
DM		 890
891 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
892 adap->msix_info[1].desc, &s->fw_evtq);
893 if (err)
894 return err;
895
896 for_each_ethrxq(s, ethqidx) {
404d9e3f
VP		 897 err = request_irq(adap->msix_info[msi_index].vec,
898 t4_sge_intr_msix, 0,
899 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 900 &s->ethrxq[ethqidx].rspq);
901 if (err)
902 goto unwind;
404d9e3f 903 msi_index++;
b8ff05a9
DM		 904 }
905 for_each_ofldrxq(s, ofldqidx) {
404d9e3f
VP		 906 err = request_irq(adap->msix_info[msi_index].vec,
907 t4_sge_intr_msix, 0,
908 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 909 &s->ofldrxq[ofldqidx].rspq);
910 if (err)
911 goto unwind;
404d9e3f 912 msi_index++;
b8ff05a9
DM		 913 }
914 for_each_rdmarxq(s, rdmaqidx) {
404d9e3f
VP		 915 err = request_irq(adap->msix_info[msi_index].vec,
916 t4_sge_intr_msix, 0,
917 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 918 &s->rdmarxq[rdmaqidx].rspq);
919 if (err)
920 goto unwind;
404d9e3f 921 msi_index++;
b8ff05a9 922 }
cf38be6d
HS		 923 for_each_rdmaciq(s, rdmaciqqidx) {
924 err = request_irq(adap->msix_info[msi_index].vec,
925 t4_sge_intr_msix, 0,
926 adap->msix_info[msi_index].desc,
927 &s->rdmaciq[rdmaciqqidx].rspq);
928 if (err)
929 goto unwind;
930 msi_index++;
931 }
b8ff05a9
DM		 932 return 0;
933
934unwind:
cf38be6d
HS		 935 while (--rdmaciqqidx >= 0)
936 free_irq(adap->msix_info[--msi_index].vec,
937 &s->rdmaciq[rdmaciqqidx].rspq);
b8ff05a9 938 while (--rdmaqidx >= 0)
404d9e3f 939 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 940 &s->rdmarxq[rdmaqidx].rspq);
941 while (--ofldqidx >= 0)
404d9e3f 942 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 943 &s->ofldrxq[ofldqidx].rspq);
944 while (--ethqidx >= 0)
404d9e3f
VP		 945 free_irq(adap->msix_info[--msi_index].vec,
946 &s->ethrxq[ethqidx].rspq);
b8ff05a9
DM		 947 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
948 return err;
949}
950
951static void free_msix_queue_irqs(struct adapter *adap)
952{
404d9e3f 953 int i, msi_index = 2;
b8ff05a9
DM		 954 struct sge *s = &adap->sge;
955
956 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
957 for_each_ethrxq(s, i)
404d9e3f 958 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
b8ff05a9 959 for_each_ofldrxq(s, i)
404d9e3f 960 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
b8ff05a9 961 for_each_rdmarxq(s, i)
404d9e3f 962 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
cf38be6d
HS		 963 for_each_rdmaciq(s, i)
964 free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq);
b8ff05a9
DM		 965}
966
671b0060
DM		 967/**
968 * write_rss - write the RSS table for a given port
969 * @pi: the port
970 * @queues: array of queue indices for RSS
971 *
972 * Sets up the portion of the HW RSS table for the port's VI to distribute
973 * packets to the Rx queues in @queues.
974 */
975static int write_rss(const struct port_info *pi, const u16 *queues)
976{
977 u16 *rss;
978 int i, err;
979 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
980
981 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
982 if (!rss)
983 return -ENOMEM;
984
985 /* map the queue indices to queue ids */
986 for (i = 0; i < pi->rss_size; i++, queues++)
987 rss[i] = q[*queues].rspq.abs_id;
988
060e0c75
DM		 989 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
990 pi->rss_size, rss, pi->rss_size);
671b0060
DM		 991 kfree(rss);
992 return err;
993}
994
b8ff05a9
DM		 995/**
996 * setup_rss - configure RSS
997 * @adap: the adapter
998 *
671b0060 999 * Sets up RSS for each port.
b8ff05a9
DM		 1000 */
1001static int setup_rss(struct adapter *adap)
1002{
671b0060 1003 int i, err;
b8ff05a9
DM		 1004
1005 for_each_port(adap, i) {
1006 const struct port_info *pi = adap2pinfo(adap, i);
b8ff05a9 1007
671b0060 1008 err = write_rss(pi, pi->rss);
b8ff05a9
DM		 1009 if (err)
1010 return err;
1011 }
1012 return 0;
1013}
1014
e46dab4d
DM		 1015/*
1016 * Return the channel of the ingress queue with the given qid.
1017 */
1018static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
1019{
1020 qid -= p->ingr_start;
1021 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
1022}
1023
b8ff05a9
DM		 1024/*
1025 * Wait until all NAPI handlers are descheduled.
1026 */
1027static void quiesce_rx(struct adapter *adap)
1028{
1029 int i;
1030
1031 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
1032 struct sge_rspq *q = adap->sge.ingr_map[i];
1033
1034 if (q && q->handler)
1035 napi_disable(&q->napi);
1036 }
1037}
1038
1039/*
1040 * Enable NAPI scheduling and interrupt generation for all Rx queues.
1041 */
1042static void enable_rx(struct adapter *adap)
1043{
1044 int i;
1045
1046 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
1047 struct sge_rspq *q = adap->sge.ingr_map[i];
1048
1049 if (!q)
1050 continue;
1051 if (q->handler)
1052 napi_enable(&q->napi);
1053 /* 0-increment GTS to start the timer and enable interrupts */
f612b815
HS		 1054 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
1055 SEINTARM_V(q->intr_params) |
1056 INGRESSQID_V(q->cntxt_id));
b8ff05a9
DM		 1057 }
1058}
1059
1060/**
1061 * setup_sge_queues - configure SGE Tx/Rx/response queues
1062 * @adap: the adapter
1063 *
1064 * Determines how many sets of SGE queues to use and initializes them.
1065 * We support multiple queue sets per port if we have MSI-X, otherwise
1066 * just one queue set per port.
1067 */
1068static int setup_sge_queues(struct adapter *adap)
1069{
1070 int err, msi_idx, i, j;
1071 struct sge *s = &adap->sge;
1072
1073 bitmap_zero(s->starving_fl, MAX_EGRQ);
1074 bitmap_zero(s->txq_maperr, MAX_EGRQ);
1075
1076 if (adap->flags & USING_MSIX)
1077 msi_idx = 1; /* vector 0 is for non-queue interrupts */
1078 else {
1079 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
1080 NULL, NULL);
1081 if (err)
1082 return err;
1083 msi_idx = -((int)s->intrq.abs_id + 1);
1084 }
1085
1086 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
1087 msi_idx, NULL, fwevtq_handler);
1088 if (err) {
1089freeout: t4_free_sge_resources(adap);
1090 return err;
1091 }
1092
1093 for_each_port(adap, i) {
1094 struct net_device *dev = adap->port[i];
1095 struct port_info *pi = netdev_priv(dev);
1096 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
1097 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
1098
1099 for (j = 0; j < pi->nqsets; j++, q++) {
1100 if (msi_idx > 0)
1101 msi_idx++;
1102 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1103 msi_idx, &q->fl,
1104 t4_ethrx_handler);
1105 if (err)
1106 goto freeout;
1107 q->rspq.idx = j;
1108 memset(&q->stats, 0, sizeof(q->stats));
1109 }
1110 for (j = 0; j < pi->nqsets; j++, t++) {
1111 err = t4_sge_alloc_eth_txq(adap, t, dev,
1112 netdev_get_tx_queue(dev, j),
1113 s->fw_evtq.cntxt_id);
1114 if (err)
1115 goto freeout;
1116 }
1117 }
1118
1119 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1120 for_each_ofldrxq(s, i) {
1121 struct sge_ofld_rxq *q = &s->ofldrxq[i];
1122 struct net_device *dev = adap->port[i / j];
1123
1124 if (msi_idx > 0)
1125 msi_idx++;
1126 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
cf38be6d
HS		 1127 q->fl.size ? &q->fl : NULL,
1128 uldrx_handler);
b8ff05a9
DM		 1129 if (err)
1130 goto freeout;
1131 memset(&q->stats, 0, sizeof(q->stats));
1132 s->ofld_rxq[i] = q->rspq.abs_id;
1133 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
1134 s->fw_evtq.cntxt_id);
1135 if (err)
1136 goto freeout;
1137 }
1138
1139 for_each_rdmarxq(s, i) {
1140 struct sge_ofld_rxq *q = &s->rdmarxq[i];
1141
1142 if (msi_idx > 0)
1143 msi_idx++;
1144 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
cf38be6d
HS		 1145 msi_idx, q->fl.size ? &q->fl : NULL,
1146 uldrx_handler);
b8ff05a9
DM		 1147 if (err)
1148 goto freeout;
1149 memset(&q->stats, 0, sizeof(q->stats));
1150 s->rdma_rxq[i] = q->rspq.abs_id;
1151 }
1152
cf38be6d
HS		 1153 for_each_rdmaciq(s, i) {
1154 struct sge_ofld_rxq *q = &s->rdmaciq[i];
1155
1156 if (msi_idx > 0)
1157 msi_idx++;
1158 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1159 msi_idx, q->fl.size ? &q->fl : NULL,
1160 uldrx_handler);
1161 if (err)
1162 goto freeout;
1163 memset(&q->stats, 0, sizeof(q->stats));
1164 s->rdma_ciq[i] = q->rspq.abs_id;
1165 }
1166
b8ff05a9
DM		 1167 for_each_port(adap, i) {
1168 /*
1169 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1170 * have RDMA queues, and that's the right value.
1171 */
1172 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1173 s->fw_evtq.cntxt_id,
1174 s->rdmarxq[i].rspq.cntxt_id);
1175 if (err)
1176 goto freeout;
1177 }
1178
9bb59b96 1179 t4_write_reg(adap, is_t4(adap->params.chip) ?
837e4a42
HS		 1180 MPS_TRC_RSS_CONTROL_A :
1181 MPS_T5_TRC_RSS_CONTROL_A,
1182 RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) |
1183 QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id));
b8ff05a9
DM		 1184 return 0;
1185}
1186
b8ff05a9
DM		 1187/*
1188 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1189 * The allocated memory is cleared.
1190 */
1191void *t4_alloc_mem(size_t size)
1192{
8be04b93 1193 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
b8ff05a9
DM		 1194
1195 if (!p)
89bf67f1 1196 p = vzalloc(size);
b8ff05a9
DM		 1197 return p;
1198}
1199
1200/*
1201 * Free memory allocated through alloc_mem().
1202 */
fd88b31a 1203void t4_free_mem(void *addr)
b8ff05a9
DM		 1204{
1205 if (is_vmalloc_addr(addr))
1206 vfree(addr);
1207 else
1208 kfree(addr);
1209}
1210
f2b7e78d
VP		 1211/* Send a Work Request to write the filter at a specified index. We construct
1212 * a Firmware Filter Work Request to have the work done and put the indicated
1213 * filter into "pending" mode which will prevent any further actions against
1214 * it till we get a reply from the firmware on the completion status of the
1215 * request.
1216 */
1217static int set_filter_wr(struct adapter *adapter, int fidx)
1218{
1219 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1220 struct sk_buff *skb;
1221 struct fw_filter_wr *fwr;
1222 unsigned int ftid;
1223
1224 /* If the new filter requires loopback Destination MAC and/or VLAN
1225 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1226 * the filter.
1227 */
1228 if (f->fs.newdmac || f->fs.newvlan) {
1229 /* allocate L2T entry for new filter */
1230 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1231 if (f->l2t == NULL)
1232 return -EAGAIN;
1233 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1234 f->fs.eport, f->fs.dmac)) {
1235 cxgb4_l2t_release(f->l2t);
1236 f->l2t = NULL;
1237 return -ENOMEM;
1238 }
1239 }
1240
1241 ftid = adapter->tids.ftid_base + fidx;
1242
1243 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1244 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1245 memset(fwr, 0, sizeof(*fwr));
1246
1247 /* It would be nice to put most of the following in t4_hw.c but most
1248 * of the work is translating the cxgbtool ch_filter_specification
1249 * into the Work Request and the definition of that structure is
1250 * currently in cxgbtool.h which isn't appropriate to pull into the
1251 * common code. We may eventually try to come up with a more neutral
1252 * filter specification structure but for now it's easiest to simply
1253 * put this fairly direct code in line ...
1254 */
e2ac9628
HS		 1255 fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR));
1256 fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr)/16));
f2b7e78d 1257 fwr->tid_to_iq =
77a80e23
HS		 1258 htonl(FW_FILTER_WR_TID_V(ftid) |
1259 FW_FILTER_WR_RQTYPE_V(f->fs.type) |
1260 FW_FILTER_WR_NOREPLY_V(0) |
1261 FW_FILTER_WR_IQ_V(f->fs.iq));
f2b7e78d 1262 fwr->del_filter_to_l2tix =
77a80e23
HS		 1263 htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) |
1264 FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) |
1265 FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) |
1266 FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) |
1267 FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) |
1268 FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) |
1269 FW_FILTER_WR_DMAC_V(f->fs.newdmac) |
1270 FW_FILTER_WR_SMAC_V(f->fs.newsmac) |
1271 FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT ||
f2b7e78d 1272 f->fs.newvlan == VLAN_REWRITE) |
77a80e23 1273 FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE ||
f2b7e78d 1274 f->fs.newvlan == VLAN_REWRITE) |
77a80e23
HS		 1275 FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) |
1276 FW_FILTER_WR_TXCHAN_V(f->fs.eport) |
1277 FW_FILTER_WR_PRIO_V(f->fs.prio) |
1278 FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0));
f2b7e78d
VP		 1279 fwr->ethtype = htons(f->fs.val.ethtype);
1280 fwr->ethtypem = htons(f->fs.mask.ethtype);
1281 fwr->frag_to_ovlan_vldm =
77a80e23
HS		 1282 (FW_FILTER_WR_FRAG_V(f->fs.val.frag) |
1283 FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) |
1284 FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) |
1285 FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) |
1286 FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) |
1287 FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld));
f2b7e78d
VP		 1288 fwr->smac_sel = 0;
1289 fwr->rx_chan_rx_rpl_iq =
77a80e23
HS		 1290 htons(FW_FILTER_WR_RX_CHAN_V(0) |
1291 FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id));
f2b7e78d 1292 fwr->maci_to_matchtypem =
77a80e23
HS		 1293 htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) |
1294 FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) |
1295 FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) |
1296 FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) |
1297 FW_FILTER_WR_PORT_V(f->fs.val.iport) |
1298 FW_FILTER_WR_PORTM_V(f->fs.mask.iport) |
1299 FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) |
1300 FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype));
f2b7e78d
VP		 1301 fwr->ptcl = f->fs.val.proto;
1302 fwr->ptclm = f->fs.mask.proto;
1303 fwr->ttyp = f->fs.val.tos;
1304 fwr->ttypm = f->fs.mask.tos;
1305 fwr->ivlan = htons(f->fs.val.ivlan);
1306 fwr->ivlanm = htons(f->fs.mask.ivlan);
1307 fwr->ovlan = htons(f->fs.val.ovlan);
1308 fwr->ovlanm = htons(f->fs.mask.ovlan);
1309 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1310 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1311 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1312 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1313 fwr->lp = htons(f->fs.val.lport);
1314 fwr->lpm = htons(f->fs.mask.lport);
1315 fwr->fp = htons(f->fs.val.fport);
1316 fwr->fpm = htons(f->fs.mask.fport);
1317 if (f->fs.newsmac)
1318 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1319
1320 /* Mark the filter as "pending" and ship off the Filter Work Request.
1321 * When we get the Work Request Reply we'll clear the pending status.
1322 */
1323 f->pending = 1;
1324 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1325 t4_ofld_send(adapter, skb);
1326 return 0;
1327}
1328
1329/* Delete the filter at a specified index.
1330 */
1331static int del_filter_wr(struct adapter *adapter, int fidx)
1332{
1333 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1334 struct sk_buff *skb;
1335 struct fw_filter_wr *fwr;
1336 unsigned int len, ftid;
1337
1338 len = sizeof(*fwr);
1339 ftid = adapter->tids.ftid_base + fidx;
1340
1341 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1342 fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1343 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1344
1345 /* Mark the filter as "pending" and ship off the Filter Work Request.
1346 * When we get the Work Request Reply we'll clear the pending status.
1347 */
1348 f->pending = 1;
1349 t4_mgmt_tx(adapter, skb);
1350 return 0;
1351}
1352
688848b1
AB		 1353static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
1354 void *accel_priv, select_queue_fallback_t fallback)
1355{
1356 int txq;
1357
1358#ifdef CONFIG_CHELSIO_T4_DCB
1359 /* If a Data Center Bridging has been successfully negotiated on this
1360 * link then we'll use the skb's priority to map it to a TX Queue.
1361 * The skb's priority is determined via the VLAN Tag Priority Code
1362 * Point field.
1363 */
1364 if (cxgb4_dcb_enabled(dev)) {
1365 u16 vlan_tci;
1366 int err;
1367
1368 err = vlan_get_tag(skb, &vlan_tci);
1369 if (unlikely(err)) {
1370 if (net_ratelimit())
1371 netdev_warn(dev,
1372 "TX Packet without VLAN Tag on DCB Link\n");
1373 txq = 0;
1374 } else {
1375 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1376 }
1377 return txq;
1378 }
1379#endif /* CONFIG_CHELSIO_T4_DCB */
1380
1381 if (select_queue) {
1382 txq = (skb_rx_queue_recorded(skb)
1383 ? skb_get_rx_queue(skb)
1384 : smp_processor_id());
1385
1386 while (unlikely(txq >= dev->real_num_tx_queues))
1387 txq -= dev->real_num_tx_queues;
1388
1389 return txq;
1390 }
1391
1392 return fallback(dev, skb) % dev->real_num_tx_queues;
1393}
1394
b8ff05a9
DM		 1395static inline int is_offload(const struct adapter *adap)
1396{
1397 return adap->params.offload;
1398}
1399
1400/*
1401 * Implementation of ethtool operations.
1402 */
1403
1404static u32 get_msglevel(struct net_device *dev)
1405{
1406 return netdev2adap(dev)->msg_enable;
1407}
1408
1409static void set_msglevel(struct net_device *dev, u32 val)
1410{
1411 netdev2adap(dev)->msg_enable = val;
1412}
1413
1414static char stats_strings[][ETH_GSTRING_LEN] = {
1415 "TxOctetsOK ",
1416 "TxFramesOK ",
1417 "TxBroadcastFrames ",
1418 "TxMulticastFrames ",
1419 "TxUnicastFrames ",
1420 "TxErrorFrames ",
1421
1422 "TxFrames64 ",
1423 "TxFrames65To127 ",
1424 "TxFrames128To255 ",
1425 "TxFrames256To511 ",
1426 "TxFrames512To1023 ",
1427 "TxFrames1024To1518 ",
1428 "TxFrames1519ToMax ",
1429
1430 "TxFramesDropped ",
1431 "TxPauseFrames ",
1432 "TxPPP0Frames ",
1433 "TxPPP1Frames ",
1434 "TxPPP2Frames ",
1435 "TxPPP3Frames ",
1436 "TxPPP4Frames ",
1437 "TxPPP5Frames ",
1438 "TxPPP6Frames ",
1439 "TxPPP7Frames ",
1440
1441 "RxOctetsOK ",
1442 "RxFramesOK ",
1443 "RxBroadcastFrames ",
1444 "RxMulticastFrames ",
1445 "RxUnicastFrames ",
1446
1447 "RxFramesTooLong ",
1448 "RxJabberErrors ",
1449 "RxFCSErrors ",
1450 "RxLengthErrors ",
1451 "RxSymbolErrors ",
1452 "RxRuntFrames ",
1453
1454 "RxFrames64 ",
1455 "RxFrames65To127 ",
1456 "RxFrames128To255 ",
1457 "RxFrames256To511 ",
1458 "RxFrames512To1023 ",
1459 "RxFrames1024To1518 ",
1460 "RxFrames1519ToMax ",
1461
1462 "RxPauseFrames ",
1463 "RxPPP0Frames ",
1464 "RxPPP1Frames ",
1465 "RxPPP2Frames ",
1466 "RxPPP3Frames ",
1467 "RxPPP4Frames ",
1468 "RxPPP5Frames ",
1469 "RxPPP6Frames ",
1470 "RxPPP7Frames ",
1471
1472 "RxBG0FramesDropped ",
1473 "RxBG1FramesDropped ",
1474 "RxBG2FramesDropped ",
1475 "RxBG3FramesDropped ",
1476 "RxBG0FramesTrunc ",
1477 "RxBG1FramesTrunc ",
1478 "RxBG2FramesTrunc ",
1479 "RxBG3FramesTrunc ",
1480
1481 "TSO ",
1482 "TxCsumOffload ",
1483 "RxCsumGood ",
1484 "VLANextractions ",
1485 "VLANinsertions ",
4a6346d4
DM		 1486 "GROpackets ",
1487 "GROmerged ",
22adfe0a
SR		 1488 "WriteCoalSuccess ",
1489 "WriteCoalFail ",
b8ff05a9
DM		 1490};
1491
1492static int get_sset_count(struct net_device *dev, int sset)
1493{
1494 switch (sset) {
1495 case ETH_SS_STATS:
1496 return ARRAY_SIZE(stats_strings);
1497 default:
1498 return -EOPNOTSUPP;
1499 }
1500}
1501
1502#define T4_REGMAP_SIZE (160 * 1024)
251f9e88 1503#define T5_REGMAP_SIZE (332 * 1024)
b8ff05a9
DM		 1504
1505static int get_regs_len(struct net_device *dev)
1506{
251f9e88 1507 struct adapter *adap = netdev2adap(dev);
d14807dd 1508 if (is_t4(adap->params.chip))
251f9e88
SR		 1509 return T4_REGMAP_SIZE;
1510 else
1511 return T5_REGMAP_SIZE;
b8ff05a9
DM		 1512}
1513
1514static int get_eeprom_len(struct net_device *dev)
1515{
1516 return EEPROMSIZE;
1517}
1518
1519static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1520{
1521 struct adapter *adapter = netdev2adap(dev);
1522
23020ab3
RJ		 1523 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1524 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1525 strlcpy(info->bus_info, pci_name(adapter->pdev),
1526 sizeof(info->bus_info));
b8ff05a9 1527
84b40501 1528 if (adapter->params.fw_vers)
b8ff05a9
DM		 1529 snprintf(info->fw_version, sizeof(info->fw_version),
1530 "%u.%u.%u.%u, TP %u.%u.%u.%u",
b2e1a3f0
HS		 1531 FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
1532 FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
1533 FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
1534 FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers),
1535 FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
1536 FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
1537 FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
1538 FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
b8ff05a9
DM		 1539}
1540
1541static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1542{
1543 if (stringset == ETH_SS_STATS)
1544 memcpy(data, stats_strings, sizeof(stats_strings));
1545}
1546
1547/*
1548 * port stats maintained per queue of the port. They should be in the same
1549 * order as in stats_strings above.
1550 */
1551struct queue_port_stats {
1552 u64 tso;
1553 u64 tx_csum;
1554 u64 rx_csum;
1555 u64 vlan_ex;
1556 u64 vlan_ins;
4a6346d4
DM		 1557 u64 gro_pkts;
1558 u64 gro_merged;
b8ff05a9
DM		 1559};
1560
1561static void collect_sge_port_stats(const struct adapter *adap,
1562 const struct port_info *p, struct queue_port_stats *s)
1563{
1564 int i;
1565 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1566 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1567
1568 memset(s, 0, sizeof(*s));
1569 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1570 s->tso += tx->tso;
1571 s->tx_csum += tx->tx_cso;
1572 s->rx_csum += rx->stats.rx_cso;
1573 s->vlan_ex += rx->stats.vlan_ex;
1574 s->vlan_ins += tx->vlan_ins;
4a6346d4
DM		 1575 s->gro_pkts += rx->stats.lro_pkts;
1576 s->gro_merged += rx->stats.lro_merged;
b8ff05a9
DM		 1577 }
1578}
1579
1580static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1581 u64 *data)
1582{
1583 struct port_info *pi = netdev_priv(dev);
1584 struct adapter *adapter = pi->adapter;
22adfe0a 1585 u32 val1, val2;
b8ff05a9
DM		 1586
1587 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1588
1589 data += sizeof(struct port_stats) / sizeof(u64);
1590 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
22adfe0a 1591 data += sizeof(struct queue_port_stats) / sizeof(u64);
d14807dd 1592 if (!is_t4(adapter->params.chip)) {
f061de42
HS		 1593 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7));
1594 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL_A);
1595 val2 = t4_read_reg(adapter, SGE_STAT_MATCH_A);
22adfe0a
SR		 1596 *data = val1 - val2;
1597 data++;
1598 *data = val2;
1599 data++;
1600 } else {
1601 memset(data, 0, 2 * sizeof(u64));
1602 *data += 2;
1603 }
b8ff05a9
DM		 1604}
1605
1606/*
1607 * Return a version number to identify the type of adapter. The scheme is:
1608 * - bits 0..9: chip version
1609 * - bits 10..15: chip revision
835bb606 1610 * - bits 16..23: register dump version
b8ff05a9
DM		 1611 */
1612static inline unsigned int mk_adap_vers(const struct adapter *ap)
1613{
d14807dd
HS		 1614 return CHELSIO_CHIP_VERSION(ap->params.chip) |
1615 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
b8ff05a9
DM		 1616}
1617
1618static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1619 unsigned int end)
1620{
1621 u32 *p = buf + start;
1622
1623 for ( ; start <= end; start += sizeof(u32))
1624 *p++ = t4_read_reg(ap, start);
1625}
1626
1627static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1628 void *buf)
1629{
251f9e88 1630 static const unsigned int t4_reg_ranges[] = {
b8ff05a9
DM		 1631 0x1008, 0x1108,
1632 0x1180, 0x11b4,
1633 0x11fc, 0x123c,
1634 0x1300, 0x173c,
1635 0x1800, 0x18fc,
1636 0x3000, 0x30d8,
1637 0x30e0, 0x5924,
1638 0x5960, 0x59d4,
1639 0x5a00, 0x5af8,
1640 0x6000, 0x6098,
1641 0x6100, 0x6150,
1642 0x6200, 0x6208,
1643 0x6240, 0x6248,
1644 0x6280, 0x6338,
1645 0x6370, 0x638c,
1646 0x6400, 0x643c,
1647 0x6500, 0x6524,
1648 0x6a00, 0x6a38,
1649 0x6a60, 0x6a78,
1650 0x6b00, 0x6b84,
1651 0x6bf0, 0x6c84,
1652 0x6cf0, 0x6d84,
1653 0x6df0, 0x6e84,
1654 0x6ef0, 0x6f84,
1655 0x6ff0, 0x7084,
1656 0x70f0, 0x7184,
1657 0x71f0, 0x7284,
1658 0x72f0, 0x7384,
1659 0x73f0, 0x7450,
1660 0x7500, 0x7530,
1661 0x7600, 0x761c,
1662 0x7680, 0x76cc,
1663 0x7700, 0x7798,
1664 0x77c0, 0x77fc,
1665 0x7900, 0x79fc,
1666 0x7b00, 0x7c38,
1667 0x7d00, 0x7efc,
1668 0x8dc0, 0x8e1c,
1669 0x8e30, 0x8e78,
1670 0x8ea0, 0x8f6c,
1671 0x8fc0, 0x9074,
1672 0x90fc, 0x90fc,
1673 0x9400, 0x9458,
1674 0x9600, 0x96bc,
1675 0x9800, 0x9808,
1676 0x9820, 0x983c,
1677 0x9850, 0x9864,
1678 0x9c00, 0x9c6c,
1679 0x9c80, 0x9cec,
1680 0x9d00, 0x9d6c,
1681 0x9d80, 0x9dec,
1682 0x9e00, 0x9e6c,
1683 0x9e80, 0x9eec,
1684 0x9f00, 0x9f6c,
1685 0x9f80, 0x9fec,
1686 0xd004, 0xd03c,
1687 0xdfc0, 0xdfe0,
1688 0xe000, 0xea7c,
3d9103f8
HS		 1689 0xf000, 0x11110,
1690 0x11118, 0x11190,
835bb606
DM		 1691 0x19040, 0x1906c,
1692 0x19078, 0x19080,
1693 0x1908c, 0x19124,
b8ff05a9
DM		 1694 0x19150, 0x191b0,
1695 0x191d0, 0x191e8,
1696 0x19238, 0x1924c,
1697 0x193f8, 0x19474,
1698 0x19490, 0x194f8,
1699 0x19800, 0x19f30,
1700 0x1a000, 0x1a06c,
1701 0x1a0b0, 0x1a120,
1702 0x1a128, 0x1a138,
1703 0x1a190, 0x1a1c4,
1704 0x1a1fc, 0x1a1fc,
1705 0x1e040, 0x1e04c,
835bb606 1706 0x1e284, 0x1e28c,
b8ff05a9
DM		 1707 0x1e2c0, 0x1e2c0,
1708 0x1e2e0, 0x1e2e0,
1709 0x1e300, 0x1e384,
1710 0x1e3c0, 0x1e3c8,
1711 0x1e440, 0x1e44c,
835bb606 1712 0x1e684, 0x1e68c,
b8ff05a9
DM		 1713 0x1e6c0, 0x1e6c0,
1714 0x1e6e0, 0x1e6e0,
1715 0x1e700, 0x1e784,
1716 0x1e7c0, 0x1e7c8,
1717 0x1e840, 0x1e84c,
835bb606 1718 0x1ea84, 0x1ea8c,
b8ff05a9
DM		 1719 0x1eac0, 0x1eac0,
1720 0x1eae0, 0x1eae0,
1721 0x1eb00, 0x1eb84,
1722 0x1ebc0, 0x1ebc8,
1723 0x1ec40, 0x1ec4c,
835bb606 1724 0x1ee84, 0x1ee8c,
b8ff05a9
DM		 1725 0x1eec0, 0x1eec0,
1726 0x1eee0, 0x1eee0,
1727 0x1ef00, 0x1ef84,
1728 0x1efc0, 0x1efc8,
1729 0x1f040, 0x1f04c,
835bb606 1730 0x1f284, 0x1f28c,
b8ff05a9
DM		 1731 0x1f2c0, 0x1f2c0,
1732 0x1f2e0, 0x1f2e0,
1733 0x1f300, 0x1f384,
1734 0x1f3c0, 0x1f3c8,
1735 0x1f440, 0x1f44c,
835bb606 1736 0x1f684, 0x1f68c,
b8ff05a9
DM		 1737 0x1f6c0, 0x1f6c0,
1738 0x1f6e0, 0x1f6e0,
1739 0x1f700, 0x1f784,
1740 0x1f7c0, 0x1f7c8,
1741 0x1f840, 0x1f84c,
835bb606 1742 0x1fa84, 0x1fa8c,
b8ff05a9
DM		 1743 0x1fac0, 0x1fac0,
1744 0x1fae0, 0x1fae0,
1745 0x1fb00, 0x1fb84,
1746 0x1fbc0, 0x1fbc8,
1747 0x1fc40, 0x1fc4c,
835bb606 1748 0x1fe84, 0x1fe8c,
b8ff05a9
DM		 1749 0x1fec0, 0x1fec0,
1750 0x1fee0, 0x1fee0,
1751 0x1ff00, 0x1ff84,
1752 0x1ffc0, 0x1ffc8,
1753 0x20000, 0x2002c,
1754 0x20100, 0x2013c,
1755 0x20190, 0x201c8,
1756 0x20200, 0x20318,
1757 0x20400, 0x20528,
1758 0x20540, 0x20614,
1759 0x21000, 0x21040,
1760 0x2104c, 0x21060,
1761 0x210c0, 0x210ec,
1762 0x21200, 0x21268,
1763 0x21270, 0x21284,
1764 0x212fc, 0x21388,
1765 0x21400, 0x21404,
1766 0x21500, 0x21518,
1767 0x2152c, 0x2153c,
1768 0x21550, 0x21554,
1769 0x21600, 0x21600,
1770 0x21608, 0x21628,
1771 0x21630, 0x2163c,
1772 0x21700, 0x2171c,
1773 0x21780, 0x2178c,
1774 0x21800, 0x21c38,
1775 0x21c80, 0x21d7c,
1776 0x21e00, 0x21e04,
1777 0x22000, 0x2202c,
1778 0x22100, 0x2213c,
1779 0x22190, 0x221c8,
1780 0x22200, 0x22318,
1781 0x22400, 0x22528,
1782 0x22540, 0x22614,
1783 0x23000, 0x23040,
1784 0x2304c, 0x23060,
1785 0x230c0, 0x230ec,
1786 0x23200, 0x23268,
1787 0x23270, 0x23284,
1788 0x232fc, 0x23388,
1789 0x23400, 0x23404,
1790 0x23500, 0x23518,
1791 0x2352c, 0x2353c,
1792 0x23550, 0x23554,
1793 0x23600, 0x23600,
1794 0x23608, 0x23628,
1795 0x23630, 0x2363c,
1796 0x23700, 0x2371c,
1797 0x23780, 0x2378c,
1798 0x23800, 0x23c38,
1799 0x23c80, 0x23d7c,
1800 0x23e00, 0x23e04,
1801 0x24000, 0x2402c,
1802 0x24100, 0x2413c,
1803 0x24190, 0x241c8,
1804 0x24200, 0x24318,
1805 0x24400, 0x24528,
1806 0x24540, 0x24614,
1807 0x25000, 0x25040,
1808 0x2504c, 0x25060,
1809 0x250c0, 0x250ec,
1810 0x25200, 0x25268,
1811 0x25270, 0x25284,
1812 0x252fc, 0x25388,
1813 0x25400, 0x25404,
1814 0x25500, 0x25518,
1815 0x2552c, 0x2553c,
1816 0x25550, 0x25554,
1817 0x25600, 0x25600,
1818 0x25608, 0x25628,
1819 0x25630, 0x2563c,
1820 0x25700, 0x2571c,
1821 0x25780, 0x2578c,
1822 0x25800, 0x25c38,
1823 0x25c80, 0x25d7c,
1824 0x25e00, 0x25e04,
1825 0x26000, 0x2602c,
1826 0x26100, 0x2613c,
1827 0x26190, 0x261c8,
1828 0x26200, 0x26318,
1829 0x26400, 0x26528,
1830 0x26540, 0x26614,
1831 0x27000, 0x27040,
1832 0x2704c, 0x27060,
1833 0x270c0, 0x270ec,
1834 0x27200, 0x27268,
1835 0x27270, 0x27284,
1836 0x272fc, 0x27388,
1837 0x27400, 0x27404,
1838 0x27500, 0x27518,
1839 0x2752c, 0x2753c,
1840 0x27550, 0x27554,
1841 0x27600, 0x27600,
1842 0x27608, 0x27628,
1843 0x27630, 0x2763c,
1844 0x27700, 0x2771c,
1845 0x27780, 0x2778c,
1846 0x27800, 0x27c38,
1847 0x27c80, 0x27d7c,
1848 0x27e00, 0x27e04
1849 };
1850
251f9e88
SR		 1851 static const unsigned int t5_reg_ranges[] = {
1852 0x1008, 0x1148,
1853 0x1180, 0x11b4,
1854 0x11fc, 0x123c,
1855 0x1280, 0x173c,
1856 0x1800, 0x18fc,
1857 0x3000, 0x3028,
1858 0x3060, 0x30d8,
1859 0x30e0, 0x30fc,
1860 0x3140, 0x357c,
1861 0x35a8, 0x35cc,
1862 0x35ec, 0x35ec,
1863 0x3600, 0x5624,
1864 0x56cc, 0x575c,
1865 0x580c, 0x5814,
1866 0x5890, 0x58bc,
1867 0x5940, 0x59dc,
1868 0x59fc, 0x5a18,
1869 0x5a60, 0x5a9c,
1870 0x5b9c, 0x5bfc,
1871 0x6000, 0x6040,
1872 0x6058, 0x614c,
1873 0x7700, 0x7798,
1874 0x77c0, 0x78fc,
1875 0x7b00, 0x7c54,
1876 0x7d00, 0x7efc,
1877 0x8dc0, 0x8de0,
1878 0x8df8, 0x8e84,
1879 0x8ea0, 0x8f84,
1880 0x8fc0, 0x90f8,
1881 0x9400, 0x9470,
1882 0x9600, 0x96f4,
1883 0x9800, 0x9808,
1884 0x9820, 0x983c,
1885 0x9850, 0x9864,
1886 0x9c00, 0x9c6c,
1887 0x9c80, 0x9cec,
1888 0x9d00, 0x9d6c,
1889 0x9d80, 0x9dec,
1890 0x9e00, 0x9e6c,
1891 0x9e80, 0x9eec,
1892 0x9f00, 0x9f6c,
1893 0x9f80, 0xa020,
1894 0xd004, 0xd03c,
1895 0xdfc0, 0xdfe0,
1896 0xe000, 0x11088,
3d9103f8
HS		 1897 0x1109c, 0x11110,
1898 0x11118, 0x1117c,
251f9e88
SR		 1899 0x11190, 0x11204,
1900 0x19040, 0x1906c,
1901 0x19078, 0x19080,
1902 0x1908c, 0x19124,
1903 0x19150, 0x191b0,
1904 0x191d0, 0x191e8,
1905 0x19238, 0x19290,
1906 0x193f8, 0x19474,
1907 0x19490, 0x194cc,
1908 0x194f0, 0x194f8,
1909 0x19c00, 0x19c60,
1910 0x19c94, 0x19e10,
1911 0x19e50, 0x19f34,
1912 0x19f40, 0x19f50,
1913 0x19f90, 0x19fe4,
1914 0x1a000, 0x1a06c,
1915 0x1a0b0, 0x1a120,
1916 0x1a128, 0x1a138,
1917 0x1a190, 0x1a1c4,
1918 0x1a1fc, 0x1a1fc,
1919 0x1e008, 0x1e00c,
1920 0x1e040, 0x1e04c,
1921 0x1e284, 0x1e290,
1922 0x1e2c0, 0x1e2c0,
1923 0x1e2e0, 0x1e2e0,
1924 0x1e300, 0x1e384,
1925 0x1e3c0, 0x1e3c8,
1926 0x1e408, 0x1e40c,
1927 0x1e440, 0x1e44c,
1928 0x1e684, 0x1e690,
1929 0x1e6c0, 0x1e6c0,
1930 0x1e6e0, 0x1e6e0,
1931 0x1e700, 0x1e784,
1932 0x1e7c0, 0x1e7c8,
1933 0x1e808, 0x1e80c,
1934 0x1e840, 0x1e84c,
1935 0x1ea84, 0x1ea90,
1936 0x1eac0, 0x1eac0,
1937 0x1eae0, 0x1eae0,
1938 0x1eb00, 0x1eb84,
1939 0x1ebc0, 0x1ebc8,
1940 0x1ec08, 0x1ec0c,
1941 0x1ec40, 0x1ec4c,
1942 0x1ee84, 0x1ee90,
1943 0x1eec0, 0x1eec0,
1944 0x1eee0, 0x1eee0,
1945 0x1ef00, 0x1ef84,
1946 0x1efc0, 0x1efc8,
1947 0x1f008, 0x1f00c,
1948 0x1f040, 0x1f04c,
1949 0x1f284, 0x1f290,
1950 0x1f2c0, 0x1f2c0,
1951 0x1f2e0, 0x1f2e0,
1952 0x1f300, 0x1f384,
1953 0x1f3c0, 0x1f3c8,
1954 0x1f408, 0x1f40c,
1955 0x1f440, 0x1f44c,
1956 0x1f684, 0x1f690,
1957 0x1f6c0, 0x1f6c0,
1958 0x1f6e0, 0x1f6e0,
1959 0x1f700, 0x1f784,
1960 0x1f7c0, 0x1f7c8,
1961 0x1f808, 0x1f80c,
1962 0x1f840, 0x1f84c,
1963 0x1fa84, 0x1fa90,
1964 0x1fac0, 0x1fac0,
1965 0x1fae0, 0x1fae0,
1966 0x1fb00, 0x1fb84,
1967 0x1fbc0, 0x1fbc8,
1968 0x1fc08, 0x1fc0c,
1969 0x1fc40, 0x1fc4c,
1970 0x1fe84, 0x1fe90,
1971 0x1fec0, 0x1fec0,
1972 0x1fee0, 0x1fee0,
1973 0x1ff00, 0x1ff84,
1974 0x1ffc0, 0x1ffc8,
1975 0x30000, 0x30030,
1976 0x30100, 0x30144,
1977 0x30190, 0x301d0,
1978 0x30200, 0x30318,
1979 0x30400, 0x3052c,
1980 0x30540, 0x3061c,
1981 0x30800, 0x30834,
1982 0x308c0, 0x30908,
1983 0x30910, 0x309ac,
1984 0x30a00, 0x30a04,
1985 0x30a0c, 0x30a2c,
1986 0x30a44, 0x30a50,
1987 0x30a74, 0x30c24,
1988 0x30d08, 0x30d14,
1989 0x30d1c, 0x30d20,
1990 0x30d3c, 0x30d50,
1991 0x31200, 0x3120c,
1992 0x31220, 0x31220,
1993 0x31240, 0x31240,
1994 0x31600, 0x31600,
1995 0x31608, 0x3160c,
1996 0x31a00, 0x31a1c,
1997 0x31e04, 0x31e20,
1998 0x31e38, 0x31e3c,
1999 0x31e80, 0x31e80,
2000 0x31e88, 0x31ea8,
2001 0x31eb0, 0x31eb4,
2002 0x31ec8, 0x31ed4,
2003 0x31fb8, 0x32004,
2004 0x32208, 0x3223c,
2005 0x32600, 0x32630,
2006 0x32a00, 0x32abc,
2007 0x32b00, 0x32b70,
2008 0x33000, 0x33048,
2009 0x33060, 0x3309c,
2010 0x330f0, 0x33148,
2011 0x33160, 0x3319c,
2012 0x331f0, 0x332e4,
2013 0x332f8, 0x333e4,
2014 0x333f8, 0x33448,
2015 0x33460, 0x3349c,
2016 0x334f0, 0x33548,
2017 0x33560, 0x3359c,
2018 0x335f0, 0x336e4,
2019 0x336f8, 0x337e4,
2020 0x337f8, 0x337fc,
2021 0x33814, 0x33814,
2022 0x3382c, 0x3382c,
2023 0x33880, 0x3388c,
2024 0x338e8, 0x338ec,
2025 0x33900, 0x33948,
2026 0x33960, 0x3399c,
2027 0x339f0, 0x33ae4,
2028 0x33af8, 0x33b10,
2029 0x33b28, 0x33b28,
2030 0x33b3c, 0x33b50,
2031 0x33bf0, 0x33c10,
2032 0x33c28, 0x33c28,
2033 0x33c3c, 0x33c50,
2034 0x33cf0, 0x33cfc,
2035 0x34000, 0x34030,
2036 0x34100, 0x34144,
2037 0x34190, 0x341d0,
2038 0x34200, 0x34318,
2039 0x34400, 0x3452c,
2040 0x34540, 0x3461c,
2041 0x34800, 0x34834,
2042 0x348c0, 0x34908,
2043 0x34910, 0x349ac,
2044 0x34a00, 0x34a04,
2045 0x34a0c, 0x34a2c,
2046 0x34a44, 0x34a50,
2047 0x34a74, 0x34c24,
2048 0x34d08, 0x34d14,
2049 0x34d1c, 0x34d20,
2050 0x34d3c, 0x34d50,
2051 0x35200, 0x3520c,
2052 0x35220, 0x35220,
2053 0x35240, 0x35240,
2054 0x35600, 0x35600,
2055 0x35608, 0x3560c,
2056 0x35a00, 0x35a1c,
2057 0x35e04, 0x35e20,
2058 0x35e38, 0x35e3c,
2059 0x35e80, 0x35e80,
2060 0x35e88, 0x35ea8,
2061 0x35eb0, 0x35eb4,
2062 0x35ec8, 0x35ed4,
2063 0x35fb8, 0x36004,
2064 0x36208, 0x3623c,
2065 0x36600, 0x36630,
2066 0x36a00, 0x36abc,
2067 0x36b00, 0x36b70,
2068 0x37000, 0x37048,
2069 0x37060, 0x3709c,
2070 0x370f0, 0x37148,
2071 0x37160, 0x3719c,
2072 0x371f0, 0x372e4,
2073 0x372f8, 0x373e4,
2074 0x373f8, 0x37448,
2075 0x37460, 0x3749c,
2076 0x374f0, 0x37548,
2077 0x37560, 0x3759c,
2078 0x375f0, 0x376e4,
2079 0x376f8, 0x377e4,
2080 0x377f8, 0x377fc,
2081 0x37814, 0x37814,
2082 0x3782c, 0x3782c,
2083 0x37880, 0x3788c,
2084 0x378e8, 0x378ec,
2085 0x37900, 0x37948,
2086 0x37960, 0x3799c,
2087 0x379f0, 0x37ae4,
2088 0x37af8, 0x37b10,
2089 0x37b28, 0x37b28,
2090 0x37b3c, 0x37b50,
2091 0x37bf0, 0x37c10,
2092 0x37c28, 0x37c28,
2093 0x37c3c, 0x37c50,
2094 0x37cf0, 0x37cfc,
2095 0x38000, 0x38030,
2096 0x38100, 0x38144,
2097 0x38190, 0x381d0,
2098 0x38200, 0x38318,
2099 0x38400, 0x3852c,
2100 0x38540, 0x3861c,
2101 0x38800, 0x38834,
2102 0x388c0, 0x38908,
2103 0x38910, 0x389ac,
2104 0x38a00, 0x38a04,
2105 0x38a0c, 0x38a2c,
2106 0x38a44, 0x38a50,
2107 0x38a74, 0x38c24,
2108 0x38d08, 0x38d14,
2109 0x38d1c, 0x38d20,
2110 0x38d3c, 0x38d50,
2111 0x39200, 0x3920c,
2112 0x39220, 0x39220,
2113 0x39240, 0x39240,
2114 0x39600, 0x39600,
2115 0x39608, 0x3960c,
2116 0x39a00, 0x39a1c,
2117 0x39e04, 0x39e20,
2118 0x39e38, 0x39e3c,
2119 0x39e80, 0x39e80,
2120 0x39e88, 0x39ea8,
2121 0x39eb0, 0x39eb4,
2122 0x39ec8, 0x39ed4,
2123 0x39fb8, 0x3a004,
2124 0x3a208, 0x3a23c,
2125 0x3a600, 0x3a630,
2126 0x3aa00, 0x3aabc,
2127 0x3ab00, 0x3ab70,
2128 0x3b000, 0x3b048,
2129 0x3b060, 0x3b09c,
2130 0x3b0f0, 0x3b148,
2131 0x3b160, 0x3b19c,
2132 0x3b1f0, 0x3b2e4,
2133 0x3b2f8, 0x3b3e4,
2134 0x3b3f8, 0x3b448,
2135 0x3b460, 0x3b49c,
2136 0x3b4f0, 0x3b548,
2137 0x3b560, 0x3b59c,
2138 0x3b5f0, 0x3b6e4,
2139 0x3b6f8, 0x3b7e4,
2140 0x3b7f8, 0x3b7fc,
2141 0x3b814, 0x3b814,
2142 0x3b82c, 0x3b82c,
2143 0x3b880, 0x3b88c,
2144 0x3b8e8, 0x3b8ec,
2145 0x3b900, 0x3b948,
2146 0x3b960, 0x3b99c,
2147 0x3b9f0, 0x3bae4,
2148 0x3baf8, 0x3bb10,
2149 0x3bb28, 0x3bb28,
2150 0x3bb3c, 0x3bb50,
2151 0x3bbf0, 0x3bc10,
2152 0x3bc28, 0x3bc28,
2153 0x3bc3c, 0x3bc50,
2154 0x3bcf0, 0x3bcfc,
2155 0x3c000, 0x3c030,
2156 0x3c100, 0x3c144,
2157 0x3c190, 0x3c1d0,
2158 0x3c200, 0x3c318,
2159 0x3c400, 0x3c52c,
2160 0x3c540, 0x3c61c,
2161 0x3c800, 0x3c834,
2162 0x3c8c0, 0x3c908,
2163 0x3c910, 0x3c9ac,
2164 0x3ca00, 0x3ca04,
2165 0x3ca0c, 0x3ca2c,
2166 0x3ca44, 0x3ca50,
2167 0x3ca74, 0x3cc24,
2168 0x3cd08, 0x3cd14,
2169 0x3cd1c, 0x3cd20,
2170 0x3cd3c, 0x3cd50,
2171 0x3d200, 0x3d20c,
2172 0x3d220, 0x3d220,
2173 0x3d240, 0x3d240,
2174 0x3d600, 0x3d600,
2175 0x3d608, 0x3d60c,
2176 0x3da00, 0x3da1c,
2177 0x3de04, 0x3de20,
2178 0x3de38, 0x3de3c,
2179 0x3de80, 0x3de80,
2180 0x3de88, 0x3dea8,
2181 0x3deb0, 0x3deb4,
2182 0x3dec8, 0x3ded4,
2183 0x3dfb8, 0x3e004,
2184 0x3e208, 0x3e23c,
2185 0x3e600, 0x3e630,
2186 0x3ea00, 0x3eabc,
2187 0x3eb00, 0x3eb70,
2188 0x3f000, 0x3f048,
2189 0x3f060, 0x3f09c,
2190 0x3f0f0, 0x3f148,
2191 0x3f160, 0x3f19c,
2192 0x3f1f0, 0x3f2e4,
2193 0x3f2f8, 0x3f3e4,
2194 0x3f3f8, 0x3f448,
2195 0x3f460, 0x3f49c,
2196 0x3f4f0, 0x3f548,
2197 0x3f560, 0x3f59c,
2198 0x3f5f0, 0x3f6e4,
2199 0x3f6f8, 0x3f7e4,
2200 0x3f7f8, 0x3f7fc,
2201 0x3f814, 0x3f814,
2202 0x3f82c, 0x3f82c,
2203 0x3f880, 0x3f88c,
2204 0x3f8e8, 0x3f8ec,
2205 0x3f900, 0x3f948,
2206 0x3f960, 0x3f99c,
2207 0x3f9f0, 0x3fae4,
2208 0x3faf8, 0x3fb10,
2209 0x3fb28, 0x3fb28,
2210 0x3fb3c, 0x3fb50,
2211 0x3fbf0, 0x3fc10,
2212 0x3fc28, 0x3fc28,
2213 0x3fc3c, 0x3fc50,
2214 0x3fcf0, 0x3fcfc,
2215 0x40000, 0x4000c,
2216 0x40040, 0x40068,
2217 0x40080, 0x40144,
2218 0x40180, 0x4018c,
2219 0x40200, 0x40298,
2220 0x402ac, 0x4033c,
2221 0x403f8, 0x403fc,
c1f49e3e 2222 0x41304, 0x413c4,
251f9e88
SR		 2223 0x41400, 0x4141c,
2224 0x41480, 0x414d0,
2225 0x44000, 0x44078,
2226 0x440c0, 0x44278,
2227 0x442c0, 0x44478,
2228 0x444c0, 0x44678,
2229 0x446c0, 0x44878,
2230 0x448c0, 0x449fc,
2231 0x45000, 0x45068,
2232 0x45080, 0x45084,
2233 0x450a0, 0x450b0,
2234 0x45200, 0x45268,
2235 0x45280, 0x45284,
2236 0x452a0, 0x452b0,
2237 0x460c0, 0x460e4,
2238 0x47000, 0x4708c,
2239 0x47200, 0x47250,
2240 0x47400, 0x47420,
2241 0x47600, 0x47618,
2242 0x47800, 0x47814,
2243 0x48000, 0x4800c,
2244 0x48040, 0x48068,
2245 0x48080, 0x48144,
2246 0x48180, 0x4818c,
2247 0x48200, 0x48298,
2248 0x482ac, 0x4833c,
2249 0x483f8, 0x483fc,
c1f49e3e 2250 0x49304, 0x493c4,
251f9e88
SR		 2251 0x49400, 0x4941c,
2252 0x49480, 0x494d0,
2253 0x4c000, 0x4c078,
2254 0x4c0c0, 0x4c278,
2255 0x4c2c0, 0x4c478,
2256 0x4c4c0, 0x4c678,
2257 0x4c6c0, 0x4c878,
2258 0x4c8c0, 0x4c9fc,
2259 0x4d000, 0x4d068,
2260 0x4d080, 0x4d084,
2261 0x4d0a0, 0x4d0b0,
2262 0x4d200, 0x4d268,
2263 0x4d280, 0x4d284,
2264 0x4d2a0, 0x4d2b0,
2265 0x4e0c0, 0x4e0e4,
2266 0x4f000, 0x4f08c,
2267 0x4f200, 0x4f250,
2268 0x4f400, 0x4f420,
2269 0x4f600, 0x4f618,
2270 0x4f800, 0x4f814,
2271 0x50000, 0x500cc,
2272 0x50400, 0x50400,
2273 0x50800, 0x508cc,
2274 0x50c00, 0x50c00,
2275 0x51000, 0x5101c,
2276 0x51300, 0x51308,
2277 };
2278
b8ff05a9
DM		 2279 int i;
2280 struct adapter *ap = netdev2adap(dev);
251f9e88
SR		 2281 static const unsigned int *reg_ranges;
2282 int arr_size = 0, buf_size = 0;
2283
d14807dd 2284 if (is_t4(ap->params.chip)) {
251f9e88
SR		 2285 reg_ranges = &t4_reg_ranges[0];
2286 arr_size = ARRAY_SIZE(t4_reg_ranges);
2287 buf_size = T4_REGMAP_SIZE;
2288 } else {
2289 reg_ranges = &t5_reg_ranges[0];
2290 arr_size = ARRAY_SIZE(t5_reg_ranges);
2291 buf_size = T5_REGMAP_SIZE;
2292 }
b8ff05a9
DM		 2293
2294 regs->version = mk_adap_vers(ap);
2295
251f9e88
SR		 2296 memset(buf, 0, buf_size);
2297 for (i = 0; i < arr_size; i += 2)
b8ff05a9
DM		 2298 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2299}
2300
2301static int restart_autoneg(struct net_device *dev)
2302{
2303 struct port_info *p = netdev_priv(dev);
2304
2305 if (!netif_running(dev))
2306 return -EAGAIN;
2307 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2308 return -EINVAL;
060e0c75 2309 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
b8ff05a9
DM		 2310 return 0;
2311}
2312
c5e06360
DM		 2313static int identify_port(struct net_device *dev,
2314 enum ethtool_phys_id_state state)
b8ff05a9 2315{
c5e06360 2316 unsigned int val;
060e0c75
DM		 2317 struct adapter *adap = netdev2adap(dev);
2318
c5e06360
DM		 2319 if (state == ETHTOOL_ID_ACTIVE)
2320 val = 0xffff;
2321 else if (state == ETHTOOL_ID_INACTIVE)
2322 val = 0;
2323 else
2324 return -EINVAL;
b8ff05a9 2325
c5e06360 2326 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
b8ff05a9
DM		 2327}
2328
40e9de4b 2329static unsigned int from_fw_linkcaps(enum fw_port_type type, unsigned int caps)
b8ff05a9
DM		 2330{
2331 unsigned int v = 0;
2332
a0881cab
DM		 2333 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2334 type == FW_PORT_TYPE_BT_XAUI) {
b8ff05a9
DM		 2335 v |= SUPPORTED_TP;
2336 if (caps & FW_PORT_CAP_SPEED_100M)
2337 v |= SUPPORTED_100baseT_Full;
2338 if (caps & FW_PORT_CAP_SPEED_1G)
2339 v |= SUPPORTED_1000baseT_Full;
2340 if (caps & FW_PORT_CAP_SPEED_10G)
2341 v |= SUPPORTED_10000baseT_Full;
2342 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2343 v |= SUPPORTED_Backplane;
2344 if (caps & FW_PORT_CAP_SPEED_1G)
2345 v |= SUPPORTED_1000baseKX_Full;
2346 if (caps & FW_PORT_CAP_SPEED_10G)
2347 v |= SUPPORTED_10000baseKX4_Full;
2348 } else if (type == FW_PORT_TYPE_KR)
2349 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
a0881cab 2350 else if (type == FW_PORT_TYPE_BP_AP)
7d5e77aa
DM		 2351 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2352 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2353 else if (type == FW_PORT_TYPE_BP4_AP)
2354 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2355 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2356 SUPPORTED_10000baseKX4_Full;
a0881cab 2357 else if (type == FW_PORT_TYPE_FIBER_XFI ||
40e9de4b
HS		 2358 type == FW_PORT_TYPE_FIBER_XAUI ||
2359 type == FW_PORT_TYPE_SFP ||
2360 type == FW_PORT_TYPE_QSFP_10G ||
2361 type == FW_PORT_TYPE_QSA) {
b8ff05a9 2362 v |= SUPPORTED_FIBRE;
4c2d5186
HS		 2363 if (caps & FW_PORT_CAP_SPEED_1G)
2364 v |= SUPPORTED_1000baseT_Full;
2365 if (caps & FW_PORT_CAP_SPEED_10G)
2366 v |= SUPPORTED_10000baseT_Full;
40e9de4b
HS		 2367 } else if (type == FW_PORT_TYPE_BP40_BA ||
2368 type == FW_PORT_TYPE_QSFP) {
72aca4bf 2369 v |= SUPPORTED_40000baseSR4_Full;
40e9de4b
HS		 2370 v |= SUPPORTED_FIBRE;
2371 }
b8ff05a9
DM		 2372
2373 if (caps & FW_PORT_CAP_ANEG)
2374 v |= SUPPORTED_Autoneg;
2375 return v;
2376}
2377
2378static unsigned int to_fw_linkcaps(unsigned int caps)
2379{
2380 unsigned int v = 0;
2381
2382 if (caps & ADVERTISED_100baseT_Full)
2383 v |= FW_PORT_CAP_SPEED_100M;
2384 if (caps & ADVERTISED_1000baseT_Full)
2385 v |= FW_PORT_CAP_SPEED_1G;
2386 if (caps & ADVERTISED_10000baseT_Full)
2387 v |= FW_PORT_CAP_SPEED_10G;
72aca4bf
KS		 2388 if (caps & ADVERTISED_40000baseSR4_Full)
2389 v |= FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2390 return v;
2391}
2392
2393static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2394{
2395 const struct port_info *p = netdev_priv(dev);
2396
2397 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
a0881cab 2398 p->port_type == FW_PORT_TYPE_BT_XFI ||
b8ff05a9
DM		 2399 p->port_type == FW_PORT_TYPE_BT_XAUI)
2400 cmd->port = PORT_TP;
a0881cab
DM		 2401 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2402 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
b8ff05a9 2403 cmd->port = PORT_FIBRE;
3e00a509
HS		 2404 else if (p->port_type == FW_PORT_TYPE_SFP ||
2405 p->port_type == FW_PORT_TYPE_QSFP_10G ||
40e9de4b 2406 p->port_type == FW_PORT_TYPE_QSA ||
3e00a509
HS		 2407 p->port_type == FW_PORT_TYPE_QSFP) {
2408 if (p->mod_type == FW_PORT_MOD_TYPE_LR ||
2409 p->mod_type == FW_PORT_MOD_TYPE_SR ||
2410 p->mod_type == FW_PORT_MOD_TYPE_ER ||
2411 p->mod_type == FW_PORT_MOD_TYPE_LRM)
2412 cmd->port = PORT_FIBRE;
2413 else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2414 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
a0881cab
DM		 2415 cmd->port = PORT_DA;
2416 else
3e00a509 2417 cmd->port = PORT_OTHER;
a0881cab 2418 } else
b8ff05a9
DM		 2419 cmd->port = PORT_OTHER;
2420
2421 if (p->mdio_addr >= 0) {
2422 cmd->phy_address = p->mdio_addr;
2423 cmd->transceiver = XCVR_EXTERNAL;
2424 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2425 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2426 } else {
2427 cmd->phy_address = 0; /* not really, but no better option */
2428 cmd->transceiver = XCVR_INTERNAL;
2429 cmd->mdio_support = 0;
2430 }
2431
2432 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2433 cmd->advertising = from_fw_linkcaps(p->port_type,
2434 p->link_cfg.advertising);
70739497
DD		 2435 ethtool_cmd_speed_set(cmd,
2436 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
b8ff05a9
DM		 2437 cmd->duplex = DUPLEX_FULL;
2438 cmd->autoneg = p->link_cfg.autoneg;
2439 cmd->maxtxpkt = 0;
2440 cmd->maxrxpkt = 0;
2441 return 0;
2442}
2443
2444static unsigned int speed_to_caps(int speed)
2445{
e8b39015 2446 if (speed == 100)
b8ff05a9 2447 return FW_PORT_CAP_SPEED_100M;
e8b39015 2448 if (speed == 1000)
b8ff05a9 2449 return FW_PORT_CAP_SPEED_1G;
e8b39015 2450 if (speed == 10000)
b8ff05a9 2451 return FW_PORT_CAP_SPEED_10G;
e8b39015 2452 if (speed == 40000)
72aca4bf 2453 return FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2454 return 0;
2455}
2456
2457static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2458{
2459 unsigned int cap;
2460 struct port_info *p = netdev_priv(dev);
2461 struct link_config *lc = &p->link_cfg;
25db0338 2462 u32 speed = ethtool_cmd_speed(cmd);
b8ff05a9
DM		 2463
2464 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
2465 return -EINVAL;
2466
2467 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2468 /*
2469 * PHY offers a single speed. See if that's what's
2470 * being requested.
2471 */
2472 if (cmd->autoneg == AUTONEG_DISABLE &&
25db0338
DD		 2473 (lc->supported & speed_to_caps(speed)))
2474 return 0;
b8ff05a9
DM		 2475 return -EINVAL;
2476 }
2477
2478 if (cmd->autoneg == AUTONEG_DISABLE) {
25db0338 2479 cap = speed_to_caps(speed);
b8ff05a9 2480
72aca4bf 2481 if (!(lc->supported & cap) ||
e8b39015
BH		 2482 (speed == 1000) ||
2483 (speed == 10000) ||
72aca4bf 2484 (speed == 40000))
b8ff05a9
DM		 2485 return -EINVAL;
2486 lc->requested_speed = cap;
2487 lc->advertising = 0;
2488 } else {
2489 cap = to_fw_linkcaps(cmd->advertising);
2490 if (!(lc->supported & cap))
2491 return -EINVAL;
2492 lc->requested_speed = 0;
2493 lc->advertising = cap | FW_PORT_CAP_ANEG;
2494 }
2495 lc->autoneg = cmd->autoneg;
2496
2497 if (netif_running(dev))
060e0c75
DM		 2498 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2499 lc);
b8ff05a9
DM		 2500 return 0;
2501}
2502
2503static void get_pauseparam(struct net_device *dev,
2504 struct ethtool_pauseparam *epause)
2505{
2506 struct port_info *p = netdev_priv(dev);
2507
2508 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2509 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2510 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2511}
2512
2513static int set_pauseparam(struct net_device *dev,
2514 struct ethtool_pauseparam *epause)
2515{
2516 struct port_info *p = netdev_priv(dev);
2517 struct link_config *lc = &p->link_cfg;
2518
2519 if (epause->autoneg == AUTONEG_DISABLE)
2520 lc->requested_fc = 0;
2521 else if (lc->supported & FW_PORT_CAP_ANEG)
2522 lc->requested_fc = PAUSE_AUTONEG;
2523 else
2524 return -EINVAL;
2525
2526 if (epause->rx_pause)
2527 lc->requested_fc |= PAUSE_RX;
2528 if (epause->tx_pause)
2529 lc->requested_fc |= PAUSE_TX;
2530 if (netif_running(dev))
060e0c75
DM		 2531 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2532 lc);
b8ff05a9
DM		 2533 return 0;
2534}
2535
b8ff05a9
DM		 2536static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2537{
2538 const struct port_info *pi = netdev_priv(dev);
2539 const struct sge *s = &pi->adapter->sge;
2540
2541 e->rx_max_pending = MAX_RX_BUFFERS;
2542 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2543 e->rx_jumbo_max_pending = 0;
2544 e->tx_max_pending = MAX_TXQ_ENTRIES;
2545
2546 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2547 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2548 e->rx_jumbo_pending = 0;
2549 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2550}
2551
2552static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2553{
2554 int i;
2555 const struct port_info *pi = netdev_priv(dev);
2556 struct adapter *adapter = pi->adapter;
2557 struct sge *s = &adapter->sge;
2558
2559 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2560 e->tx_pending > MAX_TXQ_ENTRIES ||
2561 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2562 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2563 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2564 return -EINVAL;
2565
2566 if (adapter->flags & FULL_INIT_DONE)
2567 return -EBUSY;
2568
2569 for (i = 0; i < pi->nqsets; ++i) {
2570 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2571 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2572 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2573 }
2574 return 0;
2575}
2576
2577static int closest_timer(const struct sge *s, int time)
2578{
2579 int i, delta, match = 0, min_delta = INT_MAX;
2580
2581 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2582 delta = time - s->timer_val[i];
2583 if (delta < 0)
2584 delta = -delta;
2585 if (delta < min_delta) {
2586 min_delta = delta;
2587 match = i;
2588 }
2589 }
2590 return match;
2591}
2592
2593static int closest_thres(const struct sge *s, int thres)
2594{
2595 int i, delta, match = 0, min_delta = INT_MAX;
2596
2597 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2598 delta = thres - s->counter_val[i];
2599 if (delta < 0)
2600 delta = -delta;
2601 if (delta < min_delta) {
2602 min_delta = delta;
2603 match = i;
2604 }
2605 }
2606 return match;
2607}
2608
2609/*
2610 * Return a queue's interrupt hold-off time in us. 0 means no timer.
2611 */
2612static unsigned int qtimer_val(const struct adapter *adap,
2613 const struct sge_rspq *q)
2614{
2615 unsigned int idx = q->intr_params >> 1;
2616
2617 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2618}
2619
2620/**
c887ad0e 2621 * set_rspq_intr_params - set a queue's interrupt holdoff parameters
b8ff05a9
DM		 2622 * @q: the Rx queue
2623 * @us: the hold-off time in us, or 0 to disable timer
2624 * @cnt: the hold-off packet count, or 0 to disable counter
2625 *
2626 * Sets an Rx queue's interrupt hold-off time and packet count. At least
2627 * one of the two needs to be enabled for the queue to generate interrupts.
2628 */
c887ad0e
HS		 2629static int set_rspq_intr_params(struct sge_rspq *q,
2630 unsigned int us, unsigned int cnt)
b8ff05a9 2631{
c887ad0e
HS		 2632 struct adapter *adap = q->adap;
2633
b8ff05a9
DM		 2634 if ((us | cnt) == 0)
2635 cnt = 1;
2636
2637 if (cnt) {
2638 int err;
2639 u32 v, new_idx;
2640
2641 new_idx = closest_thres(&adap->sge, cnt);
2642 if (q->desc && q->pktcnt_idx != new_idx) {
2643 /* the queue has already been created, update it */
5167865a
HS		 2644 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
2645 FW_PARAMS_PARAM_X_V(
2646 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2647 FW_PARAMS_PARAM_YZ_V(q->cntxt_id);
060e0c75
DM		 2648 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2649 &new_idx);
b8ff05a9
DM		 2650 if (err)
2651 return err;
2652 }
2653 q->pktcnt_idx = new_idx;
2654 }
2655
2656 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2657 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2658 return 0;
2659}
2660
c887ad0e
HS		 2661/**
2662 * set_rx_intr_params - set a net devices's RX interrupt holdoff paramete!
2663 * @dev: the network device
2664 * @us: the hold-off time in us, or 0 to disable timer
2665 * @cnt: the hold-off packet count, or 0 to disable counter
2666 *
2667 * Set the RX interrupt hold-off parameters for a network device.
2668 */
2669static int set_rx_intr_params(struct net_device *dev,
2670 unsigned int us, unsigned int cnt)
b8ff05a9 2671{
c887ad0e
HS		 2672 int i, err;
2673 struct port_info *pi = netdev_priv(dev);
b8ff05a9 2674 struct adapter *adap = pi->adapter;
c887ad0e
HS		 2675 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2676
2677 for (i = 0; i < pi->nqsets; i++, q++) {
2678 err = set_rspq_intr_params(&q->rspq, us, cnt);
2679 if (err)
2680 return err;
d4fc9dc2 2681 }
c887ad0e
HS		 2682 return 0;
2683}
2684
e553ec3f
HS		 2685static int set_adaptive_rx_setting(struct net_device *dev, int adaptive_rx)
2686{
2687 int i;
2688 struct port_info *pi = netdev_priv(dev);
2689 struct adapter *adap = pi->adapter;
2690 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2691
2692 for (i = 0; i < pi->nqsets; i++, q++)
2693 q->rspq.adaptive_rx = adaptive_rx;
2694
2695 return 0;
2696}
2697
2698static int get_adaptive_rx_setting(struct net_device *dev)
2699{
2700 struct port_info *pi = netdev_priv(dev);
2701 struct adapter *adap = pi->adapter;
2702 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2703
2704 return q->rspq.adaptive_rx;
2705}
2706
c887ad0e
HS		 2707static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2708{
e553ec3f 2709 set_adaptive_rx_setting(dev, c->use_adaptive_rx_coalesce);
c887ad0e
HS		 2710 return set_rx_intr_params(dev, c->rx_coalesce_usecs,
2711 c->rx_max_coalesced_frames);
b8ff05a9
DM		 2712}
2713
2714static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2715{
2716 const struct port_info *pi = netdev_priv(dev);
2717 const struct adapter *adap = pi->adapter;
2718 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2719
2720 c->rx_coalesce_usecs = qtimer_val(adap, rq);
2721 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2722 adap->sge.counter_val[rq->pktcnt_idx] : 0;
e553ec3f 2723 c->use_adaptive_rx_coalesce = get_adaptive_rx_setting(dev);
b8ff05a9
DM		 2724 return 0;
2725}
2726
1478b3ee
DM		 2727/**
2728 * eeprom_ptov - translate a physical EEPROM address to virtual
2729 * @phys_addr: the physical EEPROM address
2730 * @fn: the PCI function number
2731 * @sz: size of function-specific area
2732 *
2733 * Translate a physical EEPROM address to virtual. The first 1K is
2734 * accessed through virtual addresses starting at 31K, the rest is
2735 * accessed through virtual addresses starting at 0.
2736 *
2737 * The mapping is as follows:
2738 * [0..1K) -> [31K..32K)
2739 * [1K..1K+A) -> [31K-A..31K)
2740 * [1K+A..ES) -> [0..ES-A-1K)
2741 *
2742 * where A = @fn * @sz, and ES = EEPROM size.
b8ff05a9 2743 */
1478b3ee 2744static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
b8ff05a9 2745{
1478b3ee 2746 fn *= sz;
b8ff05a9
DM		 2747 if (phys_addr < 1024)
2748 return phys_addr + (31 << 10);
1478b3ee
DM		 2749 if (phys_addr < 1024 + fn)
2750 return 31744 - fn + phys_addr - 1024;
b8ff05a9 2751 if (phys_addr < EEPROMSIZE)
1478b3ee 2752 return phys_addr - 1024 - fn;
b8ff05a9
DM		 2753 return -EINVAL;
2754}
2755
2756/*
2757 * The next two routines implement eeprom read/write from physical addresses.
b8ff05a9
DM		 2758 */
2759static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2760{
1478b3ee 2761 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2762
2763 if (vaddr >= 0)
2764 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2765 return vaddr < 0 ? vaddr : 0;
2766}
2767
2768static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2769{
1478b3ee 2770 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2771
2772 if (vaddr >= 0)
2773 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2774 return vaddr < 0 ? vaddr : 0;
2775}
2776
2777#define EEPROM_MAGIC 0x38E2F10C
2778
2779static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2780 u8 *data)
2781{
2782 int i, err = 0;
2783 struct adapter *adapter = netdev2adap(dev);
2784
2785 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2786 if (!buf)
2787 return -ENOMEM;
2788
2789 e->magic = EEPROM_MAGIC;
2790 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2791 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2792
2793 if (!err)
2794 memcpy(data, buf + e->offset, e->len);
2795 kfree(buf);
2796 return err;
2797}
2798
2799static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2800 u8 *data)
2801{
2802 u8 *buf;
2803 int err = 0;
2804 u32 aligned_offset, aligned_len, *p;
2805 struct adapter *adapter = netdev2adap(dev);
2806
2807 if (eeprom->magic != EEPROM_MAGIC)
2808 return -EINVAL;
2809
2810 aligned_offset = eeprom->offset & ~3;
2811 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2812
1478b3ee
DM		 2813 if (adapter->fn > 0) {
2814 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2815
2816 if (aligned_offset < start ||
2817 aligned_offset + aligned_len > start + EEPROMPFSIZE)
2818 return -EPERM;
2819 }
2820
b8ff05a9
DM		 2821 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2822 /*
2823 * RMW possibly needed for first or last words.
2824 */
2825 buf = kmalloc(aligned_len, GFP_KERNEL);
2826 if (!buf)
2827 return -ENOMEM;
2828 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2829 if (!err && aligned_len > 4)
2830 err = eeprom_rd_phys(adapter,
2831 aligned_offset + aligned_len - 4,
2832 (u32 *)&buf[aligned_len - 4]);
2833 if (err)
2834 goto out;
2835 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2836 } else
2837 buf = data;
2838
2839 err = t4_seeprom_wp(adapter, false);
2840 if (err)
2841 goto out;
2842
2843 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2844 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2845 aligned_offset += 4;
2846 }
2847
2848 if (!err)
2849 err = t4_seeprom_wp(adapter, true);
2850out:
2851 if (buf != data)
2852 kfree(buf);
2853 return err;
2854}
2855
2856static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2857{
2858 int ret;
2859 const struct firmware *fw;
2860 struct adapter *adap = netdev2adap(netdev);
b2e1a3f0 2861 unsigned int mbox = PCIE_FW_MASTER_M + 1;
b8ff05a9
DM		 2862
2863 ef->data[sizeof(ef->data) - 1] = '\0';
2864 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2865 if (ret < 0)
2866 return ret;
2867
22c0b963
HS		 2868 /* If the adapter has been fully initialized then we'll go ahead and
2869 * try to get the firmware's cooperation in upgrading to the new
2870 * firmware image otherwise we'll try to do the entire job from the
2871 * host ... and we always "force" the operation in this path.
2872 */
2873 if (adap->flags & FULL_INIT_DONE)
2874 mbox = adap->mbox;
2875
2876 ret = t4_fw_upgrade(adap, mbox, fw->data, fw->size, 1);
b8ff05a9
DM		 2877 release_firmware(fw);
2878 if (!ret)
22c0b963
HS		 2879 dev_info(adap->pdev_dev, "loaded firmware %s,"
2880 " reload cxgb4 driver\n", ef->data);
b8ff05a9
DM		 2881 return ret;
2882}
2883
2884#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2885#define BCAST_CRC 0xa0ccc1a6
2886
2887static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2888{
2889 wol->supported = WAKE_BCAST | WAKE_MAGIC;
2890 wol->wolopts = netdev2adap(dev)->wol;
2891 memset(&wol->sopass, 0, sizeof(wol->sopass));
2892}
2893
2894static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2895{
2896 int err = 0;
2897 struct port_info *pi = netdev_priv(dev);
2898
2899 if (wol->wolopts & ~WOL_SUPPORTED)
2900 return -EINVAL;
2901 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2902 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2903 if (wol->wolopts & WAKE_BCAST) {
2904 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2905 ~0ULL, 0, false);
2906 if (!err)
2907 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2908 ~6ULL, ~0ULL, BCAST_CRC, true);
2909 } else
2910 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2911 return err;
2912}
2913
c8f44aff 2914static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
87b6cf51 2915{
2ed28baa 2916 const struct port_info *pi = netdev_priv(dev);
c8f44aff 2917 netdev_features_t changed = dev->features ^ features;
19ecae2c 2918 int err;
19ecae2c 2919
f646968f 2920 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2ed28baa 2921 return 0;
19ecae2c 2922
2ed28baa
MM		 2923 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2924 -1, -1, -1,
f646968f 2925 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2ed28baa 2926 if (unlikely(err))
f646968f 2927 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
19ecae2c 2928 return err;
87b6cf51
DM		 2929}
2930
7850f63f 2931static u32 get_rss_table_size(struct net_device *dev)
671b0060
DM		 2932{
2933 const struct port_info *pi = netdev_priv(dev);
671b0060 2934
7850f63f
BH		 2935 return pi->rss_size;
2936}
2937
892311f6 2938static int get_rss_table(struct net_device *dev, u32 *p, u8 *key, u8 *hfunc)
7850f63f
BH		 2939{
2940 const struct port_info *pi = netdev_priv(dev);
2941 unsigned int n = pi->rss_size;
2942
892311f6
EP		 2943 if (hfunc)
2944 *hfunc = ETH_RSS_HASH_TOP;
2945 if (!p)
2946 return 0;
671b0060 2947 while (n--)
7850f63f 2948 p[n] = pi->rss[n];
671b0060
DM		 2949 return 0;
2950}
2951
892311f6
EP		 2952static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key,
2953 const u8 hfunc)
671b0060
DM		 2954{
2955 unsigned int i;
2956 struct port_info *pi = netdev_priv(dev);
2957
892311f6
EP		 2958 /* We require at least one supported parameter to be changed and no
2959 * change in any of the unsupported parameters
2960 */
2961 if (key ||
2962 (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
2963 return -EOPNOTSUPP;
2964 if (!p)
2965 return 0;
2966
7850f63f
BH		 2967 for (i = 0; i < pi->rss_size; i++)
2968 pi->rss[i] = p[i];
671b0060
DM		 2969 if (pi->adapter->flags & FULL_INIT_DONE)
2970 return write_rss(pi, pi->rss);
2971 return 0;
2972}
2973
2974static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
815c7db5 2975 u32 *rules)
671b0060 2976{
f796564a
DM		 2977 const struct port_info *pi = netdev_priv(dev);
2978
671b0060 2979 switch (info->cmd) {
f796564a
DM		 2980 case ETHTOOL_GRXFH: {
2981 unsigned int v = pi->rss_mode;
2982
2983 info->data = 0;
2984 switch (info->flow_type) {
2985 case TCP_V4_FLOW:
b2e1a3f0 2986 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F)
f796564a
DM		 2987 info->data = RXH_IP_SRC | RXH_IP_DST |
2988 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2989 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
f796564a
DM		 2990 info->data = RXH_IP_SRC | RXH_IP_DST;
2991 break;
2992 case UDP_V4_FLOW:
b2e1a3f0
HS		 2993 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) &&
2994 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
f796564a
DM		 2995 info->data = RXH_IP_SRC | RXH_IP_DST |
2996 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 2997 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
f796564a
DM		 2998 info->data = RXH_IP_SRC | RXH_IP_DST;
2999 break;
3000 case SCTP_V4_FLOW:
3001 case AH_ESP_V4_FLOW:
3002 case IPV4_FLOW:
b2e1a3f0 3003 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
f796564a
DM		 3004 info->data = RXH_IP_SRC | RXH_IP_DST;
3005 break;
3006 case TCP_V6_FLOW:
b2e1a3f0 3007 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F)
f796564a
DM		 3008 info->data = RXH_IP_SRC | RXH_IP_DST |
3009 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 3010 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
f796564a
DM		 3011 info->data = RXH_IP_SRC | RXH_IP_DST;
3012 break;
3013 case UDP_V6_FLOW:
b2e1a3f0
HS		 3014 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) &&
3015 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
f796564a
DM		 3016 info->data = RXH_IP_SRC | RXH_IP_DST |
3017 RXH_L4_B_0_1 | RXH_L4_B_2_3;
b2e1a3f0 3018 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
f796564a
DM		 3019 info->data = RXH_IP_SRC | RXH_IP_DST;
3020 break;
3021 case SCTP_V6_FLOW:
3022 case AH_ESP_V6_FLOW:
3023 case IPV6_FLOW:
b2e1a3f0 3024 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
f796564a
DM		 3025 info->data = RXH_IP_SRC | RXH_IP_DST;
3026 break;
3027 }
3028 return 0;
3029 }
671b0060 3030 case ETHTOOL_GRXRINGS:
f796564a 3031 info->data = pi->nqsets;
671b0060
DM		 3032 return 0;
3033 }
3034 return -EOPNOTSUPP;
3035}
3036
9b07be4b 3037static const struct ethtool_ops cxgb_ethtool_ops = {
b8ff05a9
DM		 3038 .get_settings = get_settings,
3039 .set_settings = set_settings,
3040 .get_drvinfo = get_drvinfo,
3041 .get_msglevel = get_msglevel,
3042 .set_msglevel = set_msglevel,
3043 .get_ringparam = get_sge_param,
3044 .set_ringparam = set_sge_param,
3045 .get_coalesce = get_coalesce,
3046 .set_coalesce = set_coalesce,
3047 .get_eeprom_len = get_eeprom_len,
3048 .get_eeprom = get_eeprom,
3049 .set_eeprom = set_eeprom,
3050 .get_pauseparam = get_pauseparam,
3051 .set_pauseparam = set_pauseparam,
b8ff05a9
DM		 3052 .get_link = ethtool_op_get_link,
3053 .get_strings = get_strings,
c5e06360 3054 .set_phys_id = identify_port,
b8ff05a9
DM		 3055 .nway_reset = restart_autoneg,
3056 .get_sset_count = get_sset_count,
3057 .get_ethtool_stats = get_stats,
3058 .get_regs_len = get_regs_len,
3059 .get_regs = get_regs,
3060 .get_wol = get_wol,
3061 .set_wol = set_wol,
671b0060 3062 .get_rxnfc = get_rxnfc,
7850f63f 3063 .get_rxfh_indir_size = get_rss_table_size,
fe62d001
BH		 3064 .get_rxfh = get_rss_table,
3065 .set_rxfh = set_rss_table,
b8ff05a9
DM		 3066 .flash_device = set_flash,
3067};
3068
91744948 3069static int setup_debugfs(struct adapter *adap)
b8ff05a9 3070{
b8ff05a9
DM		 3071 if (IS_ERR_OR_NULL(adap->debugfs_root))
3072 return -1;
3073
fd88b31a
HS		 3074#ifdef CONFIG_DEBUG_FS
3075 t4_setup_debugfs(adap);
3076#endif
b8ff05a9
DM		 3077 return 0;
3078}
3079
3080/*
3081 * upper-layer driver support
3082 */
3083
3084/*
3085 * Allocate an active-open TID and set it to the supplied value.
3086 */
3087int cxgb4_alloc_atid(struct tid_info *t, void *data)
3088{
3089 int atid = -1;
3090
3091 spin_lock_bh(&t->atid_lock);
3092 if (t->afree) {
3093 union aopen_entry *p = t->afree;
3094
f2b7e78d 3095 atid = (p - t->atid_tab) + t->atid_base;
b8ff05a9
DM		 3096 t->afree = p->next;
3097 p->data = data;
3098 t->atids_in_use++;
3099 }
3100 spin_unlock_bh(&t->atid_lock);
3101 return atid;
3102}
3103EXPORT_SYMBOL(cxgb4_alloc_atid);
3104
3105/*
3106 * Release an active-open TID.
3107 */
3108void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
3109{
f2b7e78d 3110 union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
b8ff05a9
DM		 3111
3112 spin_lock_bh(&t->atid_lock);
3113 p->next = t->afree;
3114 t->afree = p;
3115 t->atids_in_use--;
3116 spin_unlock_bh(&t->atid_lock);
3117}
3118EXPORT_SYMBOL(cxgb4_free_atid);
3119
3120/*
3121 * Allocate a server TID and set it to the supplied value.
3122 */
3123int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
3124{
3125 int stid;
3126
3127 spin_lock_bh(&t->stid_lock);
3128 if (family == PF_INET) {
3129 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
3130 if (stid < t->nstids)
3131 __set_bit(stid, t->stid_bmap);
3132 else
3133 stid = -1;
3134 } else {
3135 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
3136 if (stid < 0)
3137 stid = -1;
3138 }
3139 if (stid >= 0) {
3140 t->stid_tab[stid].data = data;
3141 stid += t->stid_base;
15f63b74
KS		 3142 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3143 * This is equivalent to 4 TIDs. With CLIP enabled it
3144 * needs 2 TIDs.
3145 */
3146 if (family == PF_INET)
3147 t->stids_in_use++;
3148 else
3149 t->stids_in_use += 4;
b8ff05a9
DM		 3150 }
3151 spin_unlock_bh(&t->stid_lock);
3152 return stid;
3153}
3154EXPORT_SYMBOL(cxgb4_alloc_stid);
3155
dca4faeb
VP		 3156/* Allocate a server filter TID and set it to the supplied value.
3157 */
3158int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3159{
3160 int stid;
3161
3162 spin_lock_bh(&t->stid_lock);
3163 if (family == PF_INET) {
3164 stid = find_next_zero_bit(t->stid_bmap,
3165 t->nstids + t->nsftids, t->nstids);
3166 if (stid < (t->nstids + t->nsftids))
3167 __set_bit(stid, t->stid_bmap);
3168 else
3169 stid = -1;
3170 } else {
3171 stid = -1;
3172 }
3173 if (stid >= 0) {
3174 t->stid_tab[stid].data = data;
470c60c4
KS		 3175 stid -= t->nstids;
3176 stid += t->sftid_base;
dca4faeb
VP		 3177 t->stids_in_use++;
3178 }
3179 spin_unlock_bh(&t->stid_lock);
3180 return stid;
3181}
3182EXPORT_SYMBOL(cxgb4_alloc_sftid);
3183
3184/* Release a server TID.
b8ff05a9
DM		 3185 */
3186void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3187{
470c60c4
KS		 3188 /* Is it a server filter TID? */
3189 if (t->nsftids && (stid >= t->sftid_base)) {
3190 stid -= t->sftid_base;
3191 stid += t->nstids;
3192 } else {
3193 stid -= t->stid_base;
3194 }
3195
b8ff05a9
DM		 3196 spin_lock_bh(&t->stid_lock);
3197 if (family == PF_INET)
3198 __clear_bit(stid, t->stid_bmap);
3199 else
3200 bitmap_release_region(t->stid_bmap, stid, 2);
3201 t->stid_tab[stid].data = NULL;
15f63b74
KS		 3202 if (family == PF_INET)
3203 t->stids_in_use--;
3204 else
3205 t->stids_in_use -= 4;
b8ff05a9
DM		 3206 spin_unlock_bh(&t->stid_lock);
3207}
3208EXPORT_SYMBOL(cxgb4_free_stid);
3209
3210/*
3211 * Populate a TID_RELEASE WR. Caller must properly size the skb.
3212 */
3213static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3214 unsigned int tid)
3215{
3216 struct cpl_tid_release *req;
3217
3218 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3219 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3220 INIT_TP_WR(req, tid);
3221 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3222}
3223
3224/*
3225 * Queue a TID release request and if necessary schedule a work queue to
3226 * process it.
3227 */
31b9c19b 3228static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3229 unsigned int tid)
b8ff05a9
DM		 3230{
3231 void **p = &t->tid_tab[tid];
3232 struct adapter *adap = container_of(t, struct adapter, tids);
3233
3234 spin_lock_bh(&adap->tid_release_lock);
3235 *p = adap->tid_release_head;
3236 /* Low 2 bits encode the Tx channel number */
3237 adap->tid_release_head = (void **)((uintptr_t)p | chan);
3238 if (!adap->tid_release_task_busy) {
3239 adap->tid_release_task_busy = true;
29aaee65 3240 queue_work(adap->workq, &adap->tid_release_task);
b8ff05a9
DM		 3241 }
3242 spin_unlock_bh(&adap->tid_release_lock);
3243}
b8ff05a9
DM		 3244
3245/*
3246 * Process the list of pending TID release requests.
3247 */
3248static void process_tid_release_list(struct work_struct *work)
3249{
3250 struct sk_buff *skb;
3251 struct adapter *adap;
3252
3253 adap = container_of(work, struct adapter, tid_release_task);
3254
3255 spin_lock_bh(&adap->tid_release_lock);
3256 while (adap->tid_release_head) {
3257 void **p = adap->tid_release_head;
3258 unsigned int chan = (uintptr_t)p & 3;
3259 p = (void *)p - chan;
3260
3261 adap->tid_release_head = *p;
3262 *p = NULL;
3263 spin_unlock_bh(&adap->tid_release_lock);
3264
3265 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3266 GFP_KERNEL)))
3267 schedule_timeout_uninterruptible(1);
3268
3269 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3270 t4_ofld_send(adap, skb);
3271 spin_lock_bh(&adap->tid_release_lock);
3272 }
3273 adap->tid_release_task_busy = false;
3274 spin_unlock_bh(&adap->tid_release_lock);
3275}
3276
3277/*
3278 * Release a TID and inform HW. If we are unable to allocate the release
3279 * message we defer to a work queue.
3280 */
3281void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3282{
3283 void *old;
3284 struct sk_buff *skb;
3285 struct adapter *adap = container_of(t, struct adapter, tids);
3286
3287 old = t->tid_tab[tid];
3288 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3289 if (likely(skb)) {
3290 t->tid_tab[tid] = NULL;
3291 mk_tid_release(skb, chan, tid);
3292 t4_ofld_send(adap, skb);
3293 } else
3294 cxgb4_queue_tid_release(t, chan, tid);
3295 if (old)
3296 atomic_dec(&t->tids_in_use);
3297}
3298EXPORT_SYMBOL(cxgb4_remove_tid);
3299
3300/*
3301 * Allocate and initialize the TID tables. Returns 0 on success.
3302 */
3303static int tid_init(struct tid_info *t)
3304{
3305 size_t size;
f2b7e78d 3306 unsigned int stid_bmap_size;
b8ff05a9 3307 unsigned int natids = t->natids;
b6f8eaec 3308 struct adapter *adap = container_of(t, struct adapter, tids);
b8ff05a9 3309
dca4faeb 3310 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
f2b7e78d
VP		 3311 size = t->ntids * sizeof(*t->tid_tab) +
3312 natids * sizeof(*t->atid_tab) +
b8ff05a9 3313 t->nstids * sizeof(*t->stid_tab) +
dca4faeb 3314 t->nsftids * sizeof(*t->stid_tab) +
f2b7e78d 3315 stid_bmap_size * sizeof(long) +
dca4faeb
VP		 3316 t->nftids * sizeof(*t->ftid_tab) +
3317 t->nsftids * sizeof(*t->ftid_tab);
f2b7e78d 3318
b8ff05a9
DM		 3319 t->tid_tab = t4_alloc_mem(size);
3320 if (!t->tid_tab)
3321 return -ENOMEM;
3322
3323 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3324 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
dca4faeb 3325 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
f2b7e78d 3326 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
b8ff05a9
DM		 3327 spin_lock_init(&t->stid_lock);
3328 spin_lock_init(&t->atid_lock);
3329
3330 t->stids_in_use = 0;
3331 t->afree = NULL;
3332 t->atids_in_use = 0;
3333 atomic_set(&t->tids_in_use, 0);
3334
3335 /* Setup the free list for atid_tab and clear the stid bitmap. */
3336 if (natids) {
3337 while (--natids)
3338 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3339 t->afree = t->atid_tab;
3340 }
dca4faeb 3341 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
b6f8eaec
KS		 3342 /* Reserve stid 0 for T4/T5 adapters */
3343 if (!t->stid_base &&
3344 (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3345 __set_bit(0, t->stid_bmap);
3346
b8ff05a9
DM		 3347 return 0;
3348}
3349
a3e3b285
AB		 3350int cxgb4_clip_get(const struct net_device *dev,
3351 const struct in6_addr *lip)
01bcca68
VP		 3352{
3353 struct adapter *adap;
3354 struct fw_clip_cmd c;
3355
3356 adap = netdev2adap(dev);
3357 memset(&c, 0, sizeof(c));
e2ac9628
HS		 3358 c.op_to_write = htonl(FW_CMD_OP_V(FW_CLIP_CMD) |
3359 FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
b2e1a3f0 3360 c.alloc_to_len16 = htonl(FW_CLIP_CMD_ALLOC_F | FW_LEN16(c));
12f2a479
JP		 3361 c.ip_hi = *(__be64 *)(lip->s6_addr);
3362 c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
01bcca68
VP		 3363 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3364}
a3e3b285 3365EXPORT_SYMBOL(cxgb4_clip_get);
01bcca68 3366
a3e3b285
AB		 3367int cxgb4_clip_release(const struct net_device *dev,
3368 const struct in6_addr *lip)
01bcca68
VP		 3369{
3370 struct adapter *adap;
3371 struct fw_clip_cmd c;
3372
3373 adap = netdev2adap(dev);
3374 memset(&c, 0, sizeof(c));
e2ac9628
HS		 3375 c.op_to_write = htonl(FW_CMD_OP_V(FW_CLIP_CMD) |
3376 FW_CMD_REQUEST_F | FW_CMD_READ_F);
b2e1a3f0 3377 c.alloc_to_len16 = htonl(FW_CLIP_CMD_FREE_F | FW_LEN16(c));
12f2a479
JP		 3378 c.ip_hi = *(__be64 *)(lip->s6_addr);
3379 c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
01bcca68
VP		 3380 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3381}
a3e3b285 3382EXPORT_SYMBOL(cxgb4_clip_release);
01bcca68 3383
b8ff05a9
DM		 3384/**
3385 * cxgb4_create_server - create an IP server
3386 * @dev: the device
3387 * @stid: the server TID
3388 * @sip: local IP address to bind server to
3389 * @sport: the server's TCP port
3390 * @queue: queue to direct messages from this server to
3391 *
3392 * Create an IP server for the given port and address.
3393 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3394 */
3395int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
793dad94
VP		 3396 __be32 sip, __be16 sport, __be16 vlan,
3397 unsigned int queue)
b8ff05a9
DM		 3398{
3399 unsigned int chan;
3400 struct sk_buff *skb;
3401 struct adapter *adap;
3402 struct cpl_pass_open_req *req;
80f40c1f 3403 int ret;
b8ff05a9
DM		 3404
3405 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3406 if (!skb)
3407 return -ENOMEM;
3408
3409 adap = netdev2adap(dev);
3410 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3411 INIT_TP_WR(req, 0);
3412 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3413 req->local_port = sport;
3414 req->peer_port = htons(0);
3415 req->local_ip = sip;
3416 req->peer_ip = htonl(0);
e46dab4d 3417 chan = rxq_to_chan(&adap->sge, queue);
d7990b0c 3418 req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
b8ff05a9
DM		 3419 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3420 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
80f40c1f
VP		 3421 ret = t4_mgmt_tx(adap, skb);
3422 return net_xmit_eval(ret);
b8ff05a9
DM		 3423}
3424EXPORT_SYMBOL(cxgb4_create_server);
3425
80f40c1f
VP		 3426/* cxgb4_create_server6 - create an IPv6 server
3427 * @dev: the device
3428 * @stid: the server TID
3429 * @sip: local IPv6 address to bind server to
3430 * @sport: the server's TCP port
3431 * @queue: queue to direct messages from this server to
3432 *
3433 * Create an IPv6 server for the given port and address.
3434 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3435 */
3436int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3437 const struct in6_addr *sip, __be16 sport,
3438 unsigned int queue)
3439{
3440 unsigned int chan;
3441 struct sk_buff *skb;
3442 struct adapter *adap;
3443 struct cpl_pass_open_req6 *req;
3444 int ret;
3445
3446 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3447 if (!skb)
3448 return -ENOMEM;
3449
3450 adap = netdev2adap(dev);
3451 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3452 INIT_TP_WR(req, 0);
3453 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3454 req->local_port = sport;
3455 req->peer_port = htons(0);
3456 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3457 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3458 req->peer_ip_hi = cpu_to_be64(0);
3459 req->peer_ip_lo = cpu_to_be64(0);
3460 chan = rxq_to_chan(&adap->sge, queue);
d7990b0c 3461 req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
80f40c1f
VP		 3462 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3463 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
3464 ret = t4_mgmt_tx(adap, skb);
3465 return net_xmit_eval(ret);
3466}
3467EXPORT_SYMBOL(cxgb4_create_server6);
3468
3469int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3470 unsigned int queue, bool ipv6)
3471{
3472 struct sk_buff *skb;
3473 struct adapter *adap;
3474 struct cpl_close_listsvr_req *req;
3475 int ret;
3476
3477 adap = netdev2adap(dev);
3478
3479 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3480 if (!skb)
3481 return -ENOMEM;
3482
3483 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3484 INIT_TP_WR(req, 0);
3485 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
3486 req->reply_ctrl = htons(NO_REPLY(0) | (ipv6 ? LISTSVR_IPV6(1) :
3487 LISTSVR_IPV6(0)) | QUEUENO(queue));
3488 ret = t4_mgmt_tx(adap, skb);
3489 return net_xmit_eval(ret);
3490}
3491EXPORT_SYMBOL(cxgb4_remove_server);
3492
b8ff05a9
DM		 3493/**
3494 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3495 * @mtus: the HW MTU table
3496 * @mtu: the target MTU
3497 * @idx: index of selected entry in the MTU table
3498 *
3499 * Returns the index and the value in the HW MTU table that is closest to
3500 * but does not exceed @mtu, unless @mtu is smaller than any value in the
3501 * table, in which case that smallest available value is selected.
3502 */
3503unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3504 unsigned int *idx)
3505{
3506 unsigned int i = 0;
3507
3508 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3509 ++i;
3510 if (idx)
3511 *idx = i;
3512 return mtus[i];
3513}
3514EXPORT_SYMBOL(cxgb4_best_mtu);
3515
92e7ae71
HS		 3516/**
3517 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned
3518 * @mtus: the HW MTU table
3519 * @header_size: Header Size
3520 * @data_size_max: maximum Data Segment Size
3521 * @data_size_align: desired Data Segment Size Alignment (2^N)
3522 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL)
3523 *
3524 * Similar to cxgb4_best_mtu() but instead of searching the Hardware
3525 * MTU Table based solely on a Maximum MTU parameter, we break that
3526 * parameter up into a Header Size and Maximum Data Segment Size, and
3527 * provide a desired Data Segment Size Alignment. If we find an MTU in
3528 * the Hardware MTU Table which will result in a Data Segment Size with
3529 * the requested alignment _and_ that MTU isn't "too far" from the
3530 * closest MTU, then we'll return that rather than the closest MTU.
3531 */
3532unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus,
3533 unsigned short header_size,
3534 unsigned short data_size_max,
3535 unsigned short data_size_align,
3536 unsigned int *mtu_idxp)
3537{
3538 unsigned short max_mtu = header_size + data_size_max;
3539 unsigned short data_size_align_mask = data_size_align - 1;
3540 int mtu_idx, aligned_mtu_idx;
3541
3542 /* Scan the MTU Table till we find an MTU which is larger than our
3543 * Maximum MTU or we reach the end of the table. Along the way,
3544 * record the last MTU found, if any, which will result in a Data
3545 * Segment Length matching the requested alignment.
3546 */
3547 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) {
3548 unsigned short data_size = mtus[mtu_idx] - header_size;
3549
3550 /* If this MTU minus the Header Size would result in a
3551 * Data Segment Size of the desired alignment, remember it.
3552 */
3553 if ((data_size & data_size_align_mask) == 0)
3554 aligned_mtu_idx = mtu_idx;
3555
3556 /* If we're not at the end of the Hardware MTU Table and the
3557 * next element is larger than our Maximum MTU, drop out of
3558 * the loop.
3559 */
3560 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu)
3561 break;
3562 }
3563
3564 /* If we fell out of the loop because we ran to the end of the table,
3565 * then we just have to use the last [largest] entry.
3566 */
3567 if (mtu_idx == NMTUS)
3568 mtu_idx--;
3569
3570 /* If we found an MTU which resulted in the requested Data Segment
3571 * Length alignment and that's "not far" from the largest MTU which is
3572 * less than or equal to the maximum MTU, then use that.
3573 */
3574 if (aligned_mtu_idx >= 0 &&
3575 mtu_idx - aligned_mtu_idx <= 1)
3576 mtu_idx = aligned_mtu_idx;
3577
3578 /* If the caller has passed in an MTU Index pointer, pass the
3579 * MTU Index back. Return the MTU value.
3580 */
3581 if (mtu_idxp)
3582 *mtu_idxp = mtu_idx;
3583 return mtus[mtu_idx];
3584}
3585EXPORT_SYMBOL(cxgb4_best_aligned_mtu);
3586
b8ff05a9
DM		 3587/**
3588 * cxgb4_port_chan - get the HW channel of a port
3589 * @dev: the net device for the port
3590 *
3591 * Return the HW Tx channel of the given port.
3592 */
3593unsigned int cxgb4_port_chan(const struct net_device *dev)
3594{
3595 return netdev2pinfo(dev)->tx_chan;
3596}
3597EXPORT_SYMBOL(cxgb4_port_chan);
3598
881806bc
VP		 3599unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3600{
3601 struct adapter *adap = netdev2adap(dev);
2cc301d2 3602 u32 v1, v2, lp_count, hp_count;
881806bc 3603
f061de42
HS		 3604 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3605 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
d14807dd 3606 if (is_t4(adap->params.chip)) {
f061de42
HS		 3607 lp_count = LP_COUNT_G(v1);
3608 hp_count = HP_COUNT_G(v1);
2cc301d2 3609 } else {
f061de42
HS		 3610 lp_count = LP_COUNT_T5_G(v1);
3611 hp_count = HP_COUNT_T5_G(v2);
2cc301d2
SR		 3612 }
3613 return lpfifo ? lp_count : hp_count;
881806bc
VP		 3614}
3615EXPORT_SYMBOL(cxgb4_dbfifo_count);
3616
b8ff05a9
DM		 3617/**
3618 * cxgb4_port_viid - get the VI id of a port
3619 * @dev: the net device for the port
3620 *
3621 * Return the VI id of the given port.
3622 */
3623unsigned int cxgb4_port_viid(const struct net_device *dev)
3624{
3625 return netdev2pinfo(dev)->viid;
3626}
3627EXPORT_SYMBOL(cxgb4_port_viid);
3628
3629/**
3630 * cxgb4_port_idx - get the index of a port
3631 * @dev: the net device for the port
3632 *
3633 * Return the index of the given port.
3634 */
3635unsigned int cxgb4_port_idx(const struct net_device *dev)
3636{
3637 return netdev2pinfo(dev)->port_id;
3638}
3639EXPORT_SYMBOL(cxgb4_port_idx);
3640
b8ff05a9
DM		 3641void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3642 struct tp_tcp_stats *v6)
3643{
3644 struct adapter *adap = pci_get_drvdata(pdev);
3645
3646 spin_lock(&adap->stats_lock);
3647 t4_tp_get_tcp_stats(adap, v4, v6);
3648 spin_unlock(&adap->stats_lock);
3649}
3650EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3651
3652void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3653 const unsigned int *pgsz_order)
3654{
3655 struct adapter *adap = netdev2adap(dev);
3656
0d804338
HS		 3657 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask);
3658 t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) |
3659 HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) |
3660 HPZ3_V(pgsz_order[3]));
b8ff05a9
DM		 3661}
3662EXPORT_SYMBOL(cxgb4_iscsi_init);
3663
3069ee9b
VP		 3664int cxgb4_flush_eq_cache(struct net_device *dev)
3665{
3666 struct adapter *adap = netdev2adap(dev);
3667 int ret;
3668
3669 ret = t4_fwaddrspace_write(adap, adap->mbox,
f061de42 3670 0xe1000000 + SGE_CTXT_CMD_A, 0x20000000);
3069ee9b
VP		 3671 return ret;
3672}
3673EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3674
3675static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3676{
f061de42 3677 u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8;
3069ee9b
VP		 3678 __be64 indices;
3679 int ret;
3680
fc5ab020
HS		 3681 spin_lock(&adap->win0_lock);
3682 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr,
3683 sizeof(indices), (__be32 *)&indices,
3684 T4_MEMORY_READ);
3685 spin_unlock(&adap->win0_lock);
3069ee9b 3686 if (!ret) {
404d9e3f
VP		 3687 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3688 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3069ee9b
VP		 3689 }
3690 return ret;
3691}
3692
3693int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3694 u16 size)
3695{
3696 struct adapter *adap = netdev2adap(dev);
3697 u16 hw_pidx, hw_cidx;
3698 int ret;
3699
3700 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3701 if (ret)
3702 goto out;
3703
3704 if (pidx != hw_pidx) {
3705 u16 delta;
f612b815 3706 u32 val;
3069ee9b
VP		 3707
3708 if (pidx >= hw_pidx)
3709 delta = pidx - hw_pidx;
3710 else
3711 delta = size - hw_pidx + pidx;
f612b815
HS		 3712
3713 if (is_t4(adap->params.chip))
3714 val = PIDX_V(delta);
3715 else
3716 val = PIDX_T5_V(delta);
3069ee9b 3717 wmb();
f612b815
HS		 3718 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3719 QID_V(qid) | val);
3069ee9b
VP		 3720 }
3721out:
3722 return ret;
3723}
3724EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3725
3cbdb928
VP		 3726void cxgb4_disable_db_coalescing(struct net_device *dev)
3727{
3728 struct adapter *adap;
3729
3730 adap = netdev2adap(dev);
f061de42 3731 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F,
f612b815 3732 NOCOALESCE_F);
3cbdb928
VP		 3733}
3734EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3735
3736void cxgb4_enable_db_coalescing(struct net_device *dev)
3737{
3738 struct adapter *adap;
3739
3740 adap = netdev2adap(dev);
f061de42 3741 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F, 0);
3cbdb928
VP		 3742}
3743EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3744
031cf476
HS		 3745int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte)
3746{
3747 struct adapter *adap;
3748 u32 offset, memtype, memaddr;
6559a7e8 3749 u32 edc0_size, edc1_size, mc0_size, mc1_size, size;
031cf476
HS		 3750 u32 edc0_end, edc1_end, mc0_end, mc1_end;
3751 int ret;
3752
3753 adap = netdev2adap(dev);
3754
3755 offset = ((stag >> 8) * 32) + adap->vres.stag.start;
3756
3757 /* Figure out where the offset lands in the Memory Type/Address scheme.
3758 * This code assumes that the memory is laid out starting at offset 0
3759 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0
3760 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have
3761 * MC0, and some have both MC0 and MC1.
3762 */
6559a7e8
HS		 3763 size = t4_read_reg(adap, MA_EDRAM0_BAR_A);
3764 edc0_size = EDRAM0_SIZE_G(size) << 20;
3765 size = t4_read_reg(adap, MA_EDRAM1_BAR_A);
3766 edc1_size = EDRAM1_SIZE_G(size) << 20;
3767 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A);
3768 mc0_size = EXT_MEM0_SIZE_G(size) << 20;
031cf476
HS		 3769
3770 edc0_end = edc0_size;
3771 edc1_end = edc0_end + edc1_size;
3772 mc0_end = edc1_end + mc0_size;
3773
3774 if (offset < edc0_end) {
3775 memtype = MEM_EDC0;
3776 memaddr = offset;
3777 } else if (offset < edc1_end) {
3778 memtype = MEM_EDC1;
3779 memaddr = offset - edc0_end;
3780 } else {
3781 if (offset < mc0_end) {
3782 memtype = MEM_MC0;
3783 memaddr = offset - edc1_end;
3784 } else if (is_t4(adap->params.chip)) {
3785 /* T4 only has a single memory channel */
3786 goto err;
3787 } else {
6559a7e8
HS		 3788 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
3789 mc1_size = EXT_MEM1_SIZE_G(size) << 20;
031cf476
HS		 3790 mc1_end = mc0_end + mc1_size;
3791 if (offset < mc1_end) {
3792 memtype = MEM_MC1;
3793 memaddr = offset - mc0_end;
3794 } else {
3795 /* offset beyond the end of any memory */
3796 goto err;
3797 }
3798 }
3799 }
3800
3801 spin_lock(&adap->win0_lock);
3802 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ);
3803 spin_unlock(&adap->win0_lock);
3804 return ret;
3805
3806err:
3807 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n",
3808 stag, offset);
3809 return -EINVAL;
3810}
3811EXPORT_SYMBOL(cxgb4_read_tpte);
3812
7730b4c7
HS		 3813u64 cxgb4_read_sge_timestamp(struct net_device *dev)
3814{
3815 u32 hi, lo;
3816 struct adapter *adap;
3817
3818 adap = netdev2adap(dev);
f612b815
HS		 3819 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A);
3820 hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A));
7730b4c7
HS		 3821
3822 return ((u64)hi << 32) | (u64)lo;
3823}
3824EXPORT_SYMBOL(cxgb4_read_sge_timestamp);
3825
df64e4d3
HS		 3826int cxgb4_bar2_sge_qregs(struct net_device *dev,
3827 unsigned int qid,
3828 enum cxgb4_bar2_qtype qtype,
3829 u64 *pbar2_qoffset,
3830 unsigned int *pbar2_qid)
3831{
dd0bcc0b 3832 return cxgb4_t4_bar2_sge_qregs(netdev2adap(dev),
df64e4d3
HS		 3833 qid,
3834 (qtype == CXGB4_BAR2_QTYPE_EGRESS
3835 ? T4_BAR2_QTYPE_EGRESS
3836 : T4_BAR2_QTYPE_INGRESS),
3837 pbar2_qoffset,
3838 pbar2_qid);
3839}
3840EXPORT_SYMBOL(cxgb4_bar2_sge_qregs);
3841
b8ff05a9
DM		 3842static struct pci_driver cxgb4_driver;
3843
3844static void check_neigh_update(struct neighbour *neigh)
3845{
3846 const struct device *parent;
3847 const struct net_device *netdev = neigh->dev;
3848
3849 if (netdev->priv_flags & IFF_802_1Q_VLAN)
3850 netdev = vlan_dev_real_dev(netdev);
3851 parent = netdev->dev.parent;
3852 if (parent && parent->driver == &cxgb4_driver.driver)
3853 t4_l2t_update(dev_get_drvdata(parent), neigh);
3854}
3855
3856static int netevent_cb(struct notifier_block *nb, unsigned long event,
3857 void *data)
3858{
3859 switch (event) {
3860 case NETEVENT_NEIGH_UPDATE:
3861 check_neigh_update(data);
3862 break;
b8ff05a9
DM		 3863 case NETEVENT_REDIRECT:
3864 default:
3865 break;
3866 }
3867 return 0;
3868}
3869
3870static bool netevent_registered;
3871static struct notifier_block cxgb4_netevent_nb = {
3872 .notifier_call = netevent_cb
3873};
3874
3069ee9b
VP		 3875static void drain_db_fifo(struct adapter *adap, int usecs)
3876{
2cc301d2 3877 u32 v1, v2, lp_count, hp_count;
3069ee9b
VP		 3878
3879 do {
f061de42
HS		 3880 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3881 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
d14807dd 3882 if (is_t4(adap->params.chip)) {
f061de42
HS		 3883 lp_count = LP_COUNT_G(v1);
3884 hp_count = HP_COUNT_G(v1);
2cc301d2 3885 } else {
f061de42
HS		 3886 lp_count = LP_COUNT_T5_G(v1);
3887 hp_count = HP_COUNT_T5_G(v2);
2cc301d2
SR		 3888 }
3889
3890 if (lp_count == 0 && hp_count == 0)
3891 break;
3069ee9b
VP		 3892 set_current_state(TASK_UNINTERRUPTIBLE);
3893 schedule_timeout(usecs_to_jiffies(usecs));
3069ee9b
VP		 3894 } while (1);
3895}
3896
3897static void disable_txq_db(struct sge_txq *q)
3898{
05eb2389
SW		 3899 unsigned long flags;
3900
3901 spin_lock_irqsave(&q->db_lock, flags);
3069ee9b 3902 q->db_disabled = 1;
05eb2389 3903 spin_unlock_irqrestore(&q->db_lock, flags);
3069ee9b
VP		 3904}
3905
05eb2389 3906static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
3069ee9b
VP		 3907{
3908 spin_lock_irq(&q->db_lock);
05eb2389
SW		 3909 if (q->db_pidx_inc) {
3910 /* Make sure that all writes to the TX descriptors
3911 * are committed before we tell HW about them.
3912 */
3913 wmb();
f612b815
HS		 3914 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3915 QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc));
05eb2389
SW		 3916 q->db_pidx_inc = 0;
3917 }
3069ee9b
VP		 3918 q->db_disabled = 0;
3919 spin_unlock_irq(&q->db_lock);
3920}
3921
3922static void disable_dbs(struct adapter *adap)
3923{
3924 int i;
3925
3926 for_each_ethrxq(&adap->sge, i)
3927 disable_txq_db(&adap->sge.ethtxq[i].q);
3928 for_each_ofldrxq(&adap->sge, i)
3929 disable_txq_db(&adap->sge.ofldtxq[i].q);
3930 for_each_port(adap, i)
3931 disable_txq_db(&adap->sge.ctrlq[i].q);
3932}
3933
3934static void enable_dbs(struct adapter *adap)
3935{
3936 int i;
3937
3938 for_each_ethrxq(&adap->sge, i)
05eb2389 3939 enable_txq_db(adap, &adap->sge.ethtxq[i].q);
3069ee9b 3940 for_each_ofldrxq(&adap->sge, i)
05eb2389 3941 enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
3069ee9b 3942 for_each_port(adap, i)
05eb2389
SW		 3943 enable_txq_db(adap, &adap->sge.ctrlq[i].q);
3944}
3945
3946static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3947{
3948 if (adap->uld_handle[CXGB4_ULD_RDMA])
3949 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3950 cmd);
3951}
3952
3953static void process_db_full(struct work_struct *work)
3954{
3955 struct adapter *adap;
3956
3957 adap = container_of(work, struct adapter, db_full_task);
3958
3959 drain_db_fifo(adap, dbfifo_drain_delay);
3960 enable_dbs(adap);
3961 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
f612b815
HS		 3962 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3963 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F,
3964 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F);
3069ee9b
VP		 3965}
3966
3967static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3968{
3969 u16 hw_pidx, hw_cidx;
3970 int ret;
3971
05eb2389 3972 spin_lock_irq(&q->db_lock);
3069ee9b
VP		 3973 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3974 if (ret)
3975 goto out;
3976 if (q->db_pidx != hw_pidx) {
3977 u16 delta;
f612b815 3978 u32 val;
3069ee9b
VP		 3979
3980 if (q->db_pidx >= hw_pidx)
3981 delta = q->db_pidx - hw_pidx;
3982 else
3983 delta = q->size - hw_pidx + q->db_pidx;
f612b815
HS		 3984
3985 if (is_t4(adap->params.chip))
3986 val = PIDX_V(delta);
3987 else
3988 val = PIDX_T5_V(delta);
3069ee9b 3989 wmb();
f612b815
HS		 3990 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3991 QID_V(q->cntxt_id) | val);
3069ee9b
VP		 3992 }
3993out:
3994 q->db_disabled = 0;
05eb2389
SW		 3995 q->db_pidx_inc = 0;
3996 spin_unlock_irq(&q->db_lock);
3069ee9b
VP		 3997 if (ret)
3998 CH_WARN(adap, "DB drop recovery failed.\n");
3999}
4000static void recover_all_queues(struct adapter *adap)
4001{
4002 int i;
4003
4004 for_each_ethrxq(&adap->sge, i)
4005 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
4006 for_each_ofldrxq(&adap->sge, i)
4007 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
4008 for_each_port(adap, i)
4009 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
4010}
4011
881806bc
VP		 4012static void process_db_drop(struct work_struct *work)
4013{
4014 struct adapter *adap;
881806bc 4015
3069ee9b 4016 adap = container_of(work, struct adapter, db_drop_task);
881806bc 4017
d14807dd 4018 if (is_t4(adap->params.chip)) {
05eb2389 4019 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 4020 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
05eb2389 4021 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 4022 recover_all_queues(adap);
05eb2389 4023 drain_db_fifo(adap, dbfifo_drain_delay);
2cc301d2 4024 enable_dbs(adap);
05eb2389 4025 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
2cc301d2
SR		 4026 } else {
4027 u32 dropped_db = t4_read_reg(adap, 0x010ac);
4028 u16 qid = (dropped_db >> 15) & 0x1ffff;
4029 u16 pidx_inc = dropped_db & 0x1fff;
df64e4d3
HS		 4030 u64 bar2_qoffset;
4031 unsigned int bar2_qid;
4032 int ret;
2cc301d2 4033
dd0bcc0b 4034 ret = cxgb4_t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS,
df64e4d3
HS		 4035 &bar2_qoffset, &bar2_qid);
4036 if (ret)
4037 dev_err(adap->pdev_dev, "doorbell drop recovery: "
4038 "qid=%d, pidx_inc=%d\n", qid, pidx_inc);
4039 else
f612b815 4040 writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid),
df64e4d3 4041 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL);
2cc301d2
SR		 4042
4043 /* Re-enable BAR2 WC */
4044 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
4045 }
4046
f061de42 4047 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0);
881806bc
VP		 4048}
4049
4050void t4_db_full(struct adapter *adap)
4051{
d14807dd 4052 if (is_t4(adap->params.chip)) {
05eb2389
SW		 4053 disable_dbs(adap);
4054 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
f612b815
HS		 4055 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
4056 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0);
29aaee65 4057 queue_work(adap->workq, &adap->db_full_task);
2cc301d2 4058 }
881806bc
VP		 4059}
4060
4061void t4_db_dropped(struct adapter *adap)
4062{
05eb2389
SW		 4063 if (is_t4(adap->params.chip)) {
4064 disable_dbs(adap);
4065 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
4066 }
29aaee65 4067 queue_work(adap->workq, &adap->db_drop_task);
881806bc
VP		 4068}
4069
b8ff05a9
DM		 4070static void uld_attach(struct adapter *adap, unsigned int uld)
4071{
4072 void *handle;
4073 struct cxgb4_lld_info lli;
dca4faeb 4074 unsigned short i;
b8ff05a9
DM		 4075
4076 lli.pdev = adap->pdev;
35b1de55 4077 lli.pf = adap->fn;
b8ff05a9
DM		 4078 lli.l2t = adap->l2t;
4079 lli.tids = &adap->tids;
4080 lli.ports = adap->port;
4081 lli.vr = &adap->vres;
4082 lli.mtus = adap->params.mtus;
4083 if (uld == CXGB4_ULD_RDMA) {
4084 lli.rxq_ids = adap->sge.rdma_rxq;
cf38be6d 4085 lli.ciq_ids = adap->sge.rdma_ciq;
b8ff05a9 4086 lli.nrxq = adap->sge.rdmaqs;
cf38be6d 4087 lli.nciq = adap->sge.rdmaciqs;
b8ff05a9
DM		 4088 } else if (uld == CXGB4_ULD_ISCSI) {
4089 lli.rxq_ids = adap->sge.ofld_rxq;
4090 lli.nrxq = adap->sge.ofldqsets;
4091 }
4092 lli.ntxq = adap->sge.ofldqsets;
4093 lli.nchan = adap->params.nports;
4094 lli.nports = adap->params.nports;
4095 lli.wr_cred = adap->params.ofldq_wr_cred;
d14807dd 4096 lli.adapter_type = adap->params.chip;
837e4a42 4097 lli.iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
7730b4c7 4098 lli.cclk_ps = 1000000000 / adap->params.vpd.cclk;
df64e4d3
HS		 4099 lli.udb_density = 1 << adap->params.sge.eq_qpp;
4100 lli.ucq_density = 1 << adap->params.sge.iq_qpp;
dcf7b6f5 4101 lli.filt_mode = adap->params.tp.vlan_pri_map;
dca4faeb
VP		 4102 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
4103 for (i = 0; i < NCHAN; i++)
4104 lli.tx_modq[i] = i;
f612b815
HS		 4105 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
4106 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
b8ff05a9 4107 lli.fw_vers = adap->params.fw_vers;
3069ee9b 4108 lli.dbfifo_int_thresh = dbfifo_int_thresh;
04e10e21
HS		 4109 lli.sge_ingpadboundary = adap->sge.fl_align;
4110 lli.sge_egrstatuspagesize = adap->sge.stat_len;
dca4faeb
VP		 4111 lli.sge_pktshift = adap->sge.pktshift;
4112 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
4c2c5763
HS		 4113 lli.max_ordird_qp = adap->params.max_ordird_qp;
4114 lli.max_ird_adapter = adap->params.max_ird_adapter;
1ac0f095 4115 lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
b8ff05a9
DM		 4116
4117 handle = ulds[uld].add(&lli);
4118 if (IS_ERR(handle)) {
4119 dev_warn(adap->pdev_dev,
4120 "could not attach to the %s driver, error %ld\n",
4121 uld_str[uld], PTR_ERR(handle));
4122 return;
4123 }
4124
4125 adap->uld_handle[uld] = handle;
4126
4127 if (!netevent_registered) {
4128 register_netevent_notifier(&cxgb4_netevent_nb);
4129 netevent_registered = true;
4130 }
e29f5dbc
DM		 4131
4132 if (adap->flags & FULL_INIT_DONE)
4133 ulds[uld].state_change(handle, CXGB4_STATE_UP);
b8ff05a9
DM		 4134}
4135
4136static void attach_ulds(struct adapter *adap)
4137{
4138 unsigned int i;
4139
01bcca68
VP		 4140 spin_lock(&adap_rcu_lock);
4141 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
4142 spin_unlock(&adap_rcu_lock);
4143
b8ff05a9
DM		 4144 mutex_lock(&uld_mutex);
4145 list_add_tail(&adap->list_node, &adapter_list);
4146 for (i = 0; i < CXGB4_ULD_MAX; i++)
4147 if (ulds[i].add)
4148 uld_attach(adap, i);
4149 mutex_unlock(&uld_mutex);
4150}
4151
4152static void detach_ulds(struct adapter *adap)
4153{
4154 unsigned int i;
4155
4156 mutex_lock(&uld_mutex);
4157 list_del(&adap->list_node);
4158 for (i = 0; i < CXGB4_ULD_MAX; i++)
4159 if (adap->uld_handle[i]) {
4160 ulds[i].state_change(adap->uld_handle[i],
4161 CXGB4_STATE_DETACH);
4162 adap->uld_handle[i] = NULL;
4163 }
4164 if (netevent_registered && list_empty(&adapter_list)) {
4165 unregister_netevent_notifier(&cxgb4_netevent_nb);
4166 netevent_registered = false;
4167 }
4168 mutex_unlock(&uld_mutex);
01bcca68
VP		 4169
4170 spin_lock(&adap_rcu_lock);
4171 list_del_rcu(&adap->rcu_node);
4172 spin_unlock(&adap_rcu_lock);
b8ff05a9
DM		 4173}
4174
4175static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
4176{
4177 unsigned int i;
4178
4179 mutex_lock(&uld_mutex);
4180 for (i = 0; i < CXGB4_ULD_MAX; i++)
4181 if (adap->uld_handle[i])
4182 ulds[i].state_change(adap->uld_handle[i], new_state);
4183 mutex_unlock(&uld_mutex);
4184}
4185
4186/**
4187 * cxgb4_register_uld - register an upper-layer driver
4188 * @type: the ULD type
4189 * @p: the ULD methods
4190 *
4191 * Registers an upper-layer driver with this driver and notifies the ULD
4192 * about any presently available devices that support its type. Returns
4193 * %-EBUSY if a ULD of the same type is already registered.
4194 */
4195int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
4196{
4197 int ret = 0;
4198 struct adapter *adap;
4199
4200 if (type >= CXGB4_ULD_MAX)
4201 return -EINVAL;
4202 mutex_lock(&uld_mutex);
4203 if (ulds[type].add) {
4204 ret = -EBUSY;
4205 goto out;
4206 }
4207 ulds[type] = *p;
4208 list_for_each_entry(adap, &adapter_list, list_node)
4209 uld_attach(adap, type);
4210out: mutex_unlock(&uld_mutex);
4211 return ret;
4212}
4213EXPORT_SYMBOL(cxgb4_register_uld);
4214
4215/**
4216 * cxgb4_unregister_uld - unregister an upper-layer driver
4217 * @type: the ULD type
4218 *
4219 * Unregisters an existing upper-layer driver.
4220 */
4221int cxgb4_unregister_uld(enum cxgb4_uld type)
4222{
4223 struct adapter *adap;
4224
4225 if (type >= CXGB4_ULD_MAX)
4226 return -EINVAL;
4227 mutex_lock(&uld_mutex);
4228 list_for_each_entry(adap, &adapter_list, list_node)
4229 adap->uld_handle[type] = NULL;
4230 ulds[type].add = NULL;
4231 mutex_unlock(&uld_mutex);
4232 return 0;
4233}
4234EXPORT_SYMBOL(cxgb4_unregister_uld);
4235
01bcca68 4236/* Check if netdev on which event is occured belongs to us or not. Return
ee9a33b2
LR		 4237 * success (true) if it belongs otherwise failure (false).
4238 * Called with rcu_read_lock() held.
01bcca68 4239 */
1bb60376 4240#if IS_ENABLED(CONFIG_IPV6)
ee9a33b2 4241static bool cxgb4_netdev(const struct net_device *netdev)
01bcca68
VP		 4242{
4243 struct adapter *adap;
4244 int i;
4245
01bcca68
VP		 4246 list_for_each_entry_rcu(adap, &adap_rcu_list, rcu_node)
4247 for (i = 0; i < MAX_NPORTS; i++)
ee9a33b2
LR		 4248 if (adap->port[i] == netdev)
4249 return true;
4250 return false;
01bcca68
VP		 4251}
4252
4253static int clip_add(struct net_device *event_dev, struct inet6_ifaddr *ifa,
4254 unsigned long event)
4255{
4256 int ret = NOTIFY_DONE;
4257
4258 rcu_read_lock();
4259 if (cxgb4_netdev(event_dev)) {
4260 switch (event) {
4261 case NETDEV_UP:
4483589f 4262 ret = cxgb4_clip_get(event_dev, &ifa->addr);
01bcca68
VP		 4263 if (ret < 0) {
4264 rcu_read_unlock();
4265 return ret;
4266 }
4267 ret = NOTIFY_OK;
4268 break;
4269 case NETDEV_DOWN:
4483589f 4270 cxgb4_clip_release(event_dev, &ifa->addr);
01bcca68
VP		 4271 ret = NOTIFY_OK;
4272 break;
4273 default:
4274 break;
4275 }
4276 }
4277 rcu_read_unlock();
4278 return ret;
4279}
4280
4281static int cxgb4_inet6addr_handler(struct notifier_block *this,
4282 unsigned long event, void *data)
4283{
4284 struct inet6_ifaddr *ifa = data;
4285 struct net_device *event_dev;
4286 int ret = NOTIFY_DONE;
01bcca68 4287 struct bonding *bond = netdev_priv(ifa->idev->dev);
9caff1e7 4288 struct list_head *iter;
01bcca68
VP		 4289 struct slave *slave;
4290 struct pci_dev *first_pdev = NULL;
4291
4292 if (ifa->idev->dev->priv_flags & IFF_802_1Q_VLAN) {
4293 event_dev = vlan_dev_real_dev(ifa->idev->dev);
4294 ret = clip_add(event_dev, ifa, event);
4295 } else if (ifa->idev->dev->flags & IFF_MASTER) {
4296 /* It is possible that two different adapters are bonded in one
4297 * bond. We need to find such different adapters and add clip
4298 * in all of them only once.
4299 */
9caff1e7 4300 bond_for_each_slave(bond, slave, iter) {
01bcca68
VP		 4301 if (!first_pdev) {
4302 ret = clip_add(slave->dev, ifa, event);
4303 /* If clip_add is success then only initialize
4304 * first_pdev since it means it is our device
4305 */
4306 if (ret == NOTIFY_OK)
4307 first_pdev = to_pci_dev(
4308 slave->dev->dev.parent);
4309 } else if (first_pdev !=
4310 to_pci_dev(slave->dev->dev.parent))
4311 ret = clip_add(slave->dev, ifa, event);
4312 }
01bcca68
VP		 4313 } else
4314 ret = clip_add(ifa->idev->dev, ifa, event);
4315
4316 return ret;
4317}
4318
4319static struct notifier_block cxgb4_inet6addr_notifier = {
4320 .notifier_call = cxgb4_inet6addr_handler
4321};
4322
4323/* Retrieves IPv6 addresses from a root device (bond, vlan) associated with
4324 * a physical device.
4325 * The physical device reference is needed to send the actul CLIP command.
4326 */
4327static int update_dev_clip(struct net_device *root_dev, struct net_device *dev)
4328{
4329 struct inet6_dev *idev = NULL;
4330 struct inet6_ifaddr *ifa;
4331 int ret = 0;
4332
4333 idev = __in6_dev_get(root_dev);
4334 if (!idev)
4335 return ret;
4336
4337 read_lock_bh(&idev->lock);
4338 list_for_each_entry(ifa, &idev->addr_list, if_list) {
4483589f 4339 ret = cxgb4_clip_get(dev, &ifa->addr);
01bcca68
VP		 4340 if (ret < 0)
4341 break;
4342 }
4343 read_unlock_bh(&idev->lock);
4344
4345 return ret;
4346}
4347
4348static int update_root_dev_clip(struct net_device *dev)
4349{
4350 struct net_device *root_dev = NULL;
4351 int i, ret = 0;
4352
4353 /* First populate the real net device's IPv6 addresses */
4354 ret = update_dev_clip(dev, dev);
4355 if (ret)
4356 return ret;
4357
4358 /* Parse all bond and vlan devices layered on top of the physical dev */
587ddfe2
AB		 4359 root_dev = netdev_master_upper_dev_get_rcu(dev);
4360 if (root_dev) {
4361 ret = update_dev_clip(root_dev, dev);
4362 if (ret)
4363 return ret;
4364 }
4365
01bcca68 4366 for (i = 0; i < VLAN_N_VID; i++) {
f06c7f9f 4367 root_dev = __vlan_find_dev_deep_rcu(dev, htons(ETH_P_8021Q), i);
01bcca68
VP		 4368 if (!root_dev)
4369 continue;
4370
4371 ret = update_dev_clip(root_dev, dev);
4372 if (ret)
4373 break;
4374 }
4375 return ret;
4376}
4377
4378static void update_clip(const struct adapter *adap)
4379{
4380 int i;
4381 struct net_device *dev;
4382 int ret;
4383
4384 rcu_read_lock();
4385
4386 for (i = 0; i < MAX_NPORTS; i++) {
4387 dev = adap->port[i];
4388 ret = 0;
4389
4390 if (dev)
4391 ret = update_root_dev_clip(dev);
4392
4393 if (ret < 0)
4394 break;
4395 }
4396 rcu_read_unlock();
4397}
1bb60376 4398#endif /* IS_ENABLED(CONFIG_IPV6) */
01bcca68 4399
b8ff05a9
DM		 4400/**
4401 * cxgb_up - enable the adapter
4402 * @adap: adapter being enabled
4403 *
4404 * Called when the first port is enabled, this function performs the
4405 * actions necessary to make an adapter operational, such as completing
4406 * the initialization of HW modules, and enabling interrupts.
4407 *
4408 * Must be called with the rtnl lock held.
4409 */
4410static int cxgb_up(struct adapter *adap)
4411{
aaefae9b 4412 int err;
b8ff05a9 4413
aaefae9b
DM		 4414 err = setup_sge_queues(adap);
4415 if (err)
4416 goto out;
4417 err = setup_rss(adap);
4418 if (err)
4419 goto freeq;
b8ff05a9
DM		 4420
4421 if (adap->flags & USING_MSIX) {
aaefae9b 4422 name_msix_vecs(adap);
b8ff05a9
DM		 4423 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4424 adap->msix_info[0].desc, adap);
4425 if (err)
4426 goto irq_err;
4427
4428 err = request_msix_queue_irqs(adap);
4429 if (err) {
4430 free_irq(adap->msix_info[0].vec, adap);
4431 goto irq_err;
4432 }
4433 } else {
4434 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4435 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
b1a3c2b6 4436 adap->port[0]->name, adap);
b8ff05a9
DM		 4437 if (err)
4438 goto irq_err;
4439 }
4440 enable_rx(adap);
4441 t4_sge_start(adap);
4442 t4_intr_enable(adap);
aaefae9b 4443 adap->flags |= FULL_INIT_DONE;
b8ff05a9 4444 notify_ulds(adap, CXGB4_STATE_UP);
1bb60376 4445#if IS_ENABLED(CONFIG_IPV6)
01bcca68 4446 update_clip(adap);
1bb60376 4447#endif
b8ff05a9
DM		 4448 out:
4449 return err;
4450 irq_err:
4451 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
aaefae9b
DM		 4452 freeq:
4453 t4_free_sge_resources(adap);
b8ff05a9
DM		 4454 goto out;
4455}
4456
4457static void cxgb_down(struct adapter *adapter)
4458{
4459 t4_intr_disable(adapter);
4460 cancel_work_sync(&adapter->tid_release_task);
881806bc
VP		 4461 cancel_work_sync(&adapter->db_full_task);
4462 cancel_work_sync(&adapter->db_drop_task);
b8ff05a9 4463 adapter->tid_release_task_busy = false;
204dc3c0 4464 adapter->tid_release_head = NULL;
b8ff05a9
DM		 4465
4466 if (adapter->flags & USING_MSIX) {
4467 free_msix_queue_irqs(adapter);
4468 free_irq(adapter->msix_info[0].vec, adapter);
4469 } else
4470 free_irq(adapter->pdev->irq, adapter);
4471 quiesce_rx(adapter);
aaefae9b
DM		 4472 t4_sge_stop(adapter);
4473 t4_free_sge_resources(adapter);
4474 adapter->flags &= ~FULL_INIT_DONE;
b8ff05a9
DM		 4475}
4476
4477/*
4478 * net_device operations
4479 */
4480static int cxgb_open(struct net_device *dev)
4481{
4482 int err;
4483 struct port_info *pi = netdev_priv(dev);
4484 struct adapter *adapter = pi->adapter;
4485
6a3c869a
DM		 4486 netif_carrier_off(dev);
4487
aaefae9b
DM		 4488 if (!(adapter->flags & FULL_INIT_DONE)) {
4489 err = cxgb_up(adapter);
4490 if (err < 0)
4491 return err;
4492 }
b8ff05a9 4493
f68707b8
DM		 4494 err = link_start(dev);
4495 if (!err)
4496 netif_tx_start_all_queues(dev);
4497 return err;
b8ff05a9
DM		 4498}
4499
4500static int cxgb_close(struct net_device *dev)
4501{
b8ff05a9
DM		 4502 struct port_info *pi = netdev_priv(dev);
4503 struct adapter *adapter = pi->adapter;
4504
4505 netif_tx_stop_all_queues(dev);
4506 netif_carrier_off(dev);
060e0c75 4507 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
b8ff05a9
DM		 4508}
4509
f2b7e78d
VP		 4510/* Return an error number if the indicated filter isn't writable ...
4511 */
4512static int writable_filter(struct filter_entry *f)
4513{
4514 if (f->locked)
4515 return -EPERM;
4516 if (f->pending)
4517 return -EBUSY;
4518
4519 return 0;
4520}
4521
4522/* Delete the filter at the specified index (if valid). The checks for all
4523 * the common problems with doing this like the filter being locked, currently
4524 * pending in another operation, etc.
4525 */
4526static int delete_filter(struct adapter *adapter, unsigned int fidx)
4527{
4528 struct filter_entry *f;
4529 int ret;
4530
dca4faeb 4531 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
f2b7e78d
VP		 4532 return -EINVAL;
4533
4534 f = &adapter->tids.ftid_tab[fidx];
4535 ret = writable_filter(f);
4536 if (ret)
4537 return ret;
4538 if (f->valid)
4539 return del_filter_wr(adapter, fidx);
4540
4541 return 0;
4542}
4543
dca4faeb 4544int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
793dad94
VP		 4545 __be32 sip, __be16 sport, __be16 vlan,
4546 unsigned int queue, unsigned char port, unsigned char mask)
dca4faeb
VP		 4547{
4548 int ret;
4549 struct filter_entry *f;
4550 struct adapter *adap;
4551 int i;
4552 u8 *val;
4553
4554 adap = netdev2adap(dev);
4555
1cab775c 4556 /* Adjust stid to correct filter index */
470c60c4 4557 stid -= adap->tids.sftid_base;
1cab775c
VP		 4558 stid += adap->tids.nftids;
4559
dca4faeb
VP		 4560 /* Check to make sure the filter requested is writable ...
4561 */
4562 f = &adap->tids.ftid_tab[stid];
4563 ret = writable_filter(f);
4564 if (ret)
4565 return ret;
4566
4567 /* Clear out any old resources being used by the filter before
4568 * we start constructing the new filter.
4569 */
4570 if (f->valid)
4571 clear_filter(adap, f);
4572
4573 /* Clear out filter specifications */
4574 memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4575 f->fs.val.lport = cpu_to_be16(sport);
4576 f->fs.mask.lport = ~0;
4577 val = (u8 *)&sip;
793dad94 4578 if ((val[0] | val[1] | val[2] | val[3]) != 0) {
dca4faeb
VP		 4579 for (i = 0; i < 4; i++) {
4580 f->fs.val.lip[i] = val[i];
4581 f->fs.mask.lip[i] = ~0;
4582 }
0d804338 4583 if (adap->params.tp.vlan_pri_map & PORT_F) {
793dad94
VP		 4584 f->fs.val.iport = port;
4585 f->fs.mask.iport = mask;
4586 }
4587 }
dca4faeb 4588
0d804338 4589 if (adap->params.tp.vlan_pri_map & PROTOCOL_F) {
7c89e555
KS		 4590 f->fs.val.proto = IPPROTO_TCP;
4591 f->fs.mask.proto = ~0;
4592 }
4593
dca4faeb
VP		 4594 f->fs.dirsteer = 1;
4595 f->fs.iq = queue;
4596 /* Mark filter as locked */
4597 f->locked = 1;
4598 f->fs.rpttid = 1;
4599
4600 ret = set_filter_wr(adap, stid);
4601 if (ret) {
4602 clear_filter(adap, f);
4603 return ret;
4604 }
4605
4606 return 0;
4607}
4608EXPORT_SYMBOL(cxgb4_create_server_filter);
4609
4610int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4611 unsigned int queue, bool ipv6)
4612{
4613 int ret;
4614 struct filter_entry *f;
4615 struct adapter *adap;
4616
4617 adap = netdev2adap(dev);
1cab775c
VP		 4618
4619 /* Adjust stid to correct filter index */
470c60c4 4620 stid -= adap->tids.sftid_base;
1cab775c
VP		 4621 stid += adap->tids.nftids;
4622
dca4faeb
VP		 4623 f = &adap->tids.ftid_tab[stid];
4624 /* Unlock the filter */
4625 f->locked = 0;
4626
4627 ret = delete_filter(adap, stid);
4628 if (ret)
4629 return ret;
4630
4631 return 0;
4632}
4633EXPORT_SYMBOL(cxgb4_remove_server_filter);
4634
f5152c90
DM		 4635static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4636 struct rtnl_link_stats64 *ns)
b8ff05a9
DM		 4637{
4638 struct port_stats stats;
4639 struct port_info *p = netdev_priv(dev);
4640 struct adapter *adapter = p->adapter;
b8ff05a9 4641
9fe6cb58
GS		 4642 /* Block retrieving statistics during EEH error
4643 * recovery. Otherwise, the recovery might fail
4644 * and the PCI device will be removed permanently
4645 */
b8ff05a9 4646 spin_lock(&adapter->stats_lock);
9fe6cb58
GS		 4647 if (!netif_device_present(dev)) {
4648 spin_unlock(&adapter->stats_lock);
4649 return ns;
4650 }
b8ff05a9
DM		 4651 t4_get_port_stats(adapter, p->tx_chan, &stats);
4652 spin_unlock(&adapter->stats_lock);
4653
4654 ns->tx_bytes = stats.tx_octets;
4655 ns->tx_packets = stats.tx_frames;
4656 ns->rx_bytes = stats.rx_octets;
4657 ns->rx_packets = stats.rx_frames;
4658 ns->multicast = stats.rx_mcast_frames;
4659
4660 /* detailed rx_errors */
4661 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4662 stats.rx_runt;
4663 ns->rx_over_errors = 0;
4664 ns->rx_crc_errors = stats.rx_fcs_err;
4665 ns->rx_frame_errors = stats.rx_symbol_err;
4666 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
4667 stats.rx_ovflow2 + stats.rx_ovflow3 +
4668 stats.rx_trunc0 + stats.rx_trunc1 +
4669 stats.rx_trunc2 + stats.rx_trunc3;
4670 ns->rx_missed_errors = 0;
4671
4672 /* detailed tx_errors */
4673 ns->tx_aborted_errors = 0;
4674 ns->tx_carrier_errors = 0;
4675 ns->tx_fifo_errors = 0;
4676 ns->tx_heartbeat_errors = 0;
4677 ns->tx_window_errors = 0;
4678
4679 ns->tx_errors = stats.tx_error_frames;
4680 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4681 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4682 return ns;
4683}
4684
4685static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4686{
060e0c75 4687 unsigned int mbox;
b8ff05a9
DM		 4688 int ret = 0, prtad, devad;
4689 struct port_info *pi = netdev_priv(dev);
4690 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4691
4692 switch (cmd) {
4693 case SIOCGMIIPHY:
4694 if (pi->mdio_addr < 0)
4695 return -EOPNOTSUPP;
4696 data->phy_id = pi->mdio_addr;
4697 break;
4698 case SIOCGMIIREG:
4699 case SIOCSMIIREG:
4700 if (mdio_phy_id_is_c45(data->phy_id)) {
4701 prtad = mdio_phy_id_prtad(data->phy_id);
4702 devad = mdio_phy_id_devad(data->phy_id);
4703 } else if (data->phy_id < 32) {
4704 prtad = data->phy_id;
4705 devad = 0;
4706 data->reg_num &= 0x1f;
4707 } else
4708 return -EINVAL;
4709
060e0c75 4710 mbox = pi->adapter->fn;
b8ff05a9 4711 if (cmd == SIOCGMIIREG)
060e0c75 4712 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4713 data->reg_num, &data->val_out);
4714 else
060e0c75 4715 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4716 data->reg_num, data->val_in);
4717 break;
4718 default:
4719 return -EOPNOTSUPP;
4720 }
4721 return ret;
4722}
4723
4724static void cxgb_set_rxmode(struct net_device *dev)
4725{
4726 /* unfortunately we can't return errors to the stack */
4727 set_rxmode(dev, -1, false);
4728}
4729
4730static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4731{
4732 int ret;
4733 struct port_info *pi = netdev_priv(dev);
4734
4735 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
4736 return -EINVAL;
060e0c75
DM		 4737 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4738 -1, -1, -1, true);
b8ff05a9
DM		 4739 if (!ret)
4740 dev->mtu = new_mtu;
4741 return ret;
4742}
4743
4744static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4745{
4746 int ret;
4747 struct sockaddr *addr = p;
4748 struct port_info *pi = netdev_priv(dev);
4749
4750 if (!is_valid_ether_addr(addr->sa_data))
504f9b5a 4751 return -EADDRNOTAVAIL;
b8ff05a9 4752
060e0c75
DM		 4753 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4754 pi->xact_addr_filt, addr->sa_data, true, true);
b8ff05a9
DM		 4755 if (ret < 0)
4756 return ret;
4757
4758 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4759 pi->xact_addr_filt = ret;
4760 return 0;
4761}
4762
b8ff05a9
DM		 4763#ifdef CONFIG_NET_POLL_CONTROLLER
4764static void cxgb_netpoll(struct net_device *dev)
4765{
4766 struct port_info *pi = netdev_priv(dev);
4767 struct adapter *adap = pi->adapter;
4768
4769 if (adap->flags & USING_MSIX) {
4770 int i;
4771 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4772
4773 for (i = pi->nqsets; i; i--, rx++)
4774 t4_sge_intr_msix(0, &rx->rspq);
4775 } else
4776 t4_intr_handler(adap)(0, adap);
4777}
4778#endif
4779
4780static const struct net_device_ops cxgb4_netdev_ops = {
4781 .ndo_open = cxgb_open,
4782 .ndo_stop = cxgb_close,
4783 .ndo_start_xmit = t4_eth_xmit,
688848b1 4784 .ndo_select_queue = cxgb_select_queue,
9be793bf 4785 .ndo_get_stats64 = cxgb_get_stats,
b8ff05a9
DM		 4786 .ndo_set_rx_mode = cxgb_set_rxmode,
4787 .ndo_set_mac_address = cxgb_set_mac_addr,
2ed28baa 4788 .ndo_set_features = cxgb_set_features,
b8ff05a9
DM		 4789 .ndo_validate_addr = eth_validate_addr,
4790 .ndo_do_ioctl = cxgb_ioctl,
4791 .ndo_change_mtu = cxgb_change_mtu,
b8ff05a9
DM		 4792#ifdef CONFIG_NET_POLL_CONTROLLER
4793 .ndo_poll_controller = cxgb_netpoll,
4794#endif
4795};
4796
4797void t4_fatal_err(struct adapter *adap)
4798{
f612b815 4799 t4_set_reg_field(adap, SGE_CONTROL_A, GLOBALENABLE_F, 0);
b8ff05a9
DM		 4800 t4_intr_disable(adap);
4801 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4802}
4803
0abfd152
HS		 4804/* Return the specified PCI-E Configuration Space register from our Physical
4805 * Function. We try first via a Firmware LDST Command since we prefer to let
4806 * the firmware own all of these registers, but if that fails we go for it
4807 * directly ourselves.
4808 */
4809static u32 t4_read_pcie_cfg4(struct adapter *adap, int reg)
4810{
4811 struct fw_ldst_cmd ldst_cmd;
4812 u32 val;
4813 int ret;
4814
4815 /* Construct and send the Firmware LDST Command to retrieve the
4816 * specified PCI-E Configuration Space register.
4817 */
4818 memset(&ldst_cmd, 0, sizeof(ldst_cmd));
4819 ldst_cmd.op_to_addrspace =
e2ac9628
HS		 4820 htonl(FW_CMD_OP_V(FW_LDST_CMD) |
4821 FW_CMD_REQUEST_F |
4822 FW_CMD_READ_F |
5167865a 4823 FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE));
0abfd152 4824 ldst_cmd.cycles_to_len16 = htonl(FW_LEN16(ldst_cmd));
5167865a 4825 ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1);
0abfd152 4826 ldst_cmd.u.pcie.ctrl_to_fn =
5167865a 4827 (FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->fn));
0abfd152
HS		 4828 ldst_cmd.u.pcie.r = reg;
4829 ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
4830 &ldst_cmd);
4831
4832 /* If the LDST Command suucceeded, exctract the returned register
4833 * value. Otherwise read it directly ourself.
4834 */
4835 if (ret == 0)
4836 val = ntohl(ldst_cmd.u.pcie.data[0]);
4837 else
4838 t4_hw_pci_read_cfg4(adap, reg, &val);
4839
4840 return val;
4841}
4842
b8ff05a9
DM		 4843static void setup_memwin(struct adapter *adap)
4844{
0abfd152 4845 u32 mem_win0_base, mem_win1_base, mem_win2_base, mem_win2_aperture;
b8ff05a9 4846
d14807dd 4847 if (is_t4(adap->params.chip)) {
0abfd152
HS		 4848 u32 bar0;
4849
4850 /* Truncation intentional: we only read the bottom 32-bits of
4851 * the 64-bit BAR0/BAR1 ... We use the hardware backdoor
4852 * mechanism to read BAR0 instead of using
4853 * pci_resource_start() because we could be operating from
4854 * within a Virtual Machine which is trapping our accesses to
4855 * our Configuration Space and we need to set up the PCI-E
4856 * Memory Window decoders with the actual addresses which will
4857 * be coming across the PCI-E link.
4858 */
4859 bar0 = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_0);
4860 bar0 &= PCI_BASE_ADDRESS_MEM_MASK;
4861 adap->t4_bar0 = bar0;
4862
19dd37ba
SR		 4863 mem_win0_base = bar0 + MEMWIN0_BASE;
4864 mem_win1_base = bar0 + MEMWIN1_BASE;
4865 mem_win2_base = bar0 + MEMWIN2_BASE;
0abfd152 4866 mem_win2_aperture = MEMWIN2_APERTURE;
19dd37ba
SR		 4867 } else {
4868 /* For T5, only relative offset inside the PCIe BAR is passed */
4869 mem_win0_base = MEMWIN0_BASE;
0abfd152 4870 mem_win1_base = MEMWIN1_BASE;
19dd37ba 4871 mem_win2_base = MEMWIN2_BASE_T5;
0abfd152 4872 mem_win2_aperture = MEMWIN2_APERTURE_T5;
19dd37ba 4873 }
f061de42
HS		 4874 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 0),
4875 mem_win0_base | BIR_V(0) |
4876 WINDOW_V(ilog2(MEMWIN0_APERTURE) - 10));
4877 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 1),
4878 mem_win1_base | BIR_V(0) |
4879 WINDOW_V(ilog2(MEMWIN1_APERTURE) - 10));
4880 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2),
4881 mem_win2_base | BIR_V(0) |
4882 WINDOW_V(ilog2(mem_win2_aperture) - 10));
4883 t4_read_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2));
636f9d37
VP		 4884}
4885
4886static void setup_memwin_rdma(struct adapter *adap)
4887{
1ae970e0 4888 if (adap->vres.ocq.size) {
0abfd152
HS		 4889 u32 start;
4890 unsigned int sz_kb;
1ae970e0 4891
0abfd152
HS		 4892 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2);
4893 start &= PCI_BASE_ADDRESS_MEM_MASK;
4894 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
1ae970e0
DM		 4895 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4896 t4_write_reg(adap,
f061de42
HS		 4897 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3),
4898 start | BIR_V(1) | WINDOW_V(ilog2(sz_kb)));
1ae970e0 4899 t4_write_reg(adap,
f061de42 4900 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3),
1ae970e0
DM		 4901 adap->vres.ocq.start);
4902 t4_read_reg(adap,
f061de42 4903 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3));
1ae970e0 4904 }
b8ff05a9
DM		 4905}
4906
02b5fb8e
DM		 4907static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4908{
4909 u32 v;
4910 int ret;
4911
4912 /* get device capabilities */
4913 memset(c, 0, sizeof(*c));
e2ac9628
HS		 4914 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4915 FW_CMD_REQUEST_F | FW_CMD_READ_F);
ce91a923 4916 c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
060e0c75 4917 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
02b5fb8e
DM		 4918 if (ret < 0)
4919 return ret;
4920
4921 /* select capabilities we'll be using */
4922 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4923 if (!vf_acls)
4924 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4925 else
4926 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4927 } else if (vf_acls) {
4928 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4929 return ret;
4930 }
e2ac9628
HS		 4931 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4932 FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
060e0c75 4933 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
02b5fb8e
DM		 4934 if (ret < 0)
4935 return ret;
4936
060e0c75 4937 ret = t4_config_glbl_rss(adap, adap->fn,
02b5fb8e 4938 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
b2e1a3f0
HS		 4939 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F |
4940 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F);
02b5fb8e
DM		 4941 if (ret < 0)
4942 return ret;
4943
060e0c75
DM		 4944 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
4945 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
02b5fb8e
DM		 4946 if (ret < 0)
4947 return ret;
4948
4949 t4_sge_init(adap);
4950
02b5fb8e 4951 /* tweak some settings */
837e4a42 4952 t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849);
0d804338 4953 t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12));
837e4a42
HS		 4954 t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A);
4955 v = t4_read_reg(adap, TP_PIO_DATA_A);
4956 t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F);
060e0c75 4957
dca4faeb
VP		 4958 /* first 4 Tx modulation queues point to consecutive Tx channels */
4959 adap->params.tp.tx_modq_map = 0xE4;
0d804338
HS		 4960 t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A,
4961 TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map));
dca4faeb
VP		 4962
4963 /* associate each Tx modulation queue with consecutive Tx channels */
4964 v = 0x84218421;
837e4a42 4965 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
0d804338 4966 &v, 1, TP_TX_SCHED_HDR_A);
837e4a42 4967 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
0d804338 4968 &v, 1, TP_TX_SCHED_FIFO_A);
837e4a42 4969 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
0d804338 4970 &v, 1, TP_TX_SCHED_PCMD_A);
dca4faeb
VP		 4971
4972#define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4973 if (is_offload(adap)) {
0d804338
HS		 4974 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A,
4975 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4976 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4977 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4978 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4979 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A,
4980 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4981 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4982 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4983 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
dca4faeb
VP		 4984 }
4985
060e0c75
DM		 4986 /* get basic stuff going */
4987 return t4_early_init(adap, adap->fn);
02b5fb8e
DM		 4988}
4989
b8ff05a9
DM		 4990/*
4991 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
4992 */
4993#define MAX_ATIDS 8192U
4994
636f9d37
VP		 4995/*
4996 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4997 *
4998 * If the firmware we're dealing with has Configuration File support, then
4999 * we use that to perform all configuration
5000 */
5001
5002/*
5003 * Tweak configuration based on module parameters, etc. Most of these have
5004 * defaults assigned to them by Firmware Configuration Files (if we're using
5005 * them) but need to be explicitly set if we're using hard-coded
5006 * initialization. But even in the case of using Firmware Configuration
5007 * Files, we'd like to expose the ability to change these via module
5008 * parameters so these are essentially common tweaks/settings for
5009 * Configuration Files and hard-coded initialization ...
5010 */
5011static int adap_init0_tweaks(struct adapter *adapter)
5012{
5013 /*
5014 * Fix up various Host-Dependent Parameters like Page Size, Cache
5015 * Line Size, etc. The firmware default is for a 4KB Page Size and
5016 * 64B Cache Line Size ...
5017 */
5018 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
5019
5020 /*
5021 * Process module parameters which affect early initialization.
5022 */
5023 if (rx_dma_offset != 2 && rx_dma_offset != 0) {
5024 dev_err(&adapter->pdev->dev,
5025 "Ignoring illegal rx_dma_offset=%d, using 2\n",
5026 rx_dma_offset);
5027 rx_dma_offset = 2;
5028 }
f612b815
HS		 5029 t4_set_reg_field(adapter, SGE_CONTROL_A,
5030 PKTSHIFT_V(PKTSHIFT_M),
5031 PKTSHIFT_V(rx_dma_offset));
636f9d37
VP		 5032
5033 /*
5034 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
5035 * adds the pseudo header itself.
5036 */
837e4a42
HS		 5037 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A,
5038 CSUM_HAS_PSEUDO_HDR_F, 0);
636f9d37
VP		 5039
5040 return 0;
5041}
5042
5043/*
5044 * Attempt to initialize the adapter via a Firmware Configuration File.
5045 */
5046static int adap_init0_config(struct adapter *adapter, int reset)
5047{
5048 struct fw_caps_config_cmd caps_cmd;
5049 const struct firmware *cf;
5050 unsigned long mtype = 0, maddr = 0;
5051 u32 finiver, finicsum, cfcsum;
16e47624
HS		 5052 int ret;
5053 int config_issued = 0;
0a57a536 5054 char *fw_config_file, fw_config_file_path[256];
16e47624 5055 char *config_name = NULL;
636f9d37
VP		 5056
5057 /*
5058 * Reset device if necessary.
5059 */
5060 if (reset) {
5061 ret = t4_fw_reset(adapter, adapter->mbox,
0d804338 5062 PIORSTMODE_F | PIORST_F);
636f9d37
VP		 5063 if (ret < 0)
5064 goto bye;
5065 }
5066
5067 /*
5068 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
5069 * then use that. Otherwise, use the configuration file stored
5070 * in the adapter flash ...
5071 */
d14807dd 5072 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
0a57a536 5073 case CHELSIO_T4:
16e47624 5074 fw_config_file = FW4_CFNAME;
0a57a536
SR		 5075 break;
5076 case CHELSIO_T5:
5077 fw_config_file = FW5_CFNAME;
5078 break;
5079 default:
5080 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
5081 adapter->pdev->device);
5082 ret = -EINVAL;
5083 goto bye;
5084 }
5085
5086 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
636f9d37 5087 if (ret < 0) {
16e47624 5088 config_name = "On FLASH";
636f9d37
VP		 5089 mtype = FW_MEMTYPE_CF_FLASH;
5090 maddr = t4_flash_cfg_addr(adapter);
5091 } else {
5092 u32 params[7], val[7];
5093
16e47624
HS		 5094 sprintf(fw_config_file_path,
5095 "/lib/firmware/%s", fw_config_file);
5096 config_name = fw_config_file_path;
5097
636f9d37
VP		 5098 if (cf->size >= FLASH_CFG_MAX_SIZE)
5099 ret = -ENOMEM;
5100 else {
5167865a
HS		 5101 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5102 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
636f9d37
VP		 5103 ret = t4_query_params(adapter, adapter->mbox,
5104 adapter->fn, 0, 1, params, val);
5105 if (ret == 0) {
5106 /*
fc5ab020 5107 * For t4_memory_rw() below addresses and
636f9d37
VP		 5108 * sizes have to be in terms of multiples of 4
5109 * bytes. So, if the Configuration File isn't
5110 * a multiple of 4 bytes in length we'll have
5111 * to write that out separately since we can't
5112 * guarantee that the bytes following the
5113 * residual byte in the buffer returned by
5114 * request_firmware() are zeroed out ...
5115 */
5116 size_t resid = cf->size & 0x3;
5117 size_t size = cf->size & ~0x3;
5118 __be32 *data = (__be32 *)cf->data;
5119
5167865a
HS		 5120 mtype = FW_PARAMS_PARAM_Y_G(val[0]);
5121 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16;
636f9d37 5122
fc5ab020
HS		 5123 spin_lock(&adapter->win0_lock);
5124 ret = t4_memory_rw(adapter, 0, mtype, maddr,
5125 size, data, T4_MEMORY_WRITE);
636f9d37
VP		 5126 if (ret == 0 && resid != 0) {
5127 union {
5128 __be32 word;
5129 char buf[4];
5130 } last;
5131 int i;
5132
5133 last.word = data[size >> 2];
5134 for (i = resid; i < 4; i++)
5135 last.buf[i] = 0;
fc5ab020
HS		 5136 ret = t4_memory_rw(adapter, 0, mtype,
5137 maddr + size,
5138 4, &last.word,
5139 T4_MEMORY_WRITE);
636f9d37 5140 }
fc5ab020 5141 spin_unlock(&adapter->win0_lock);
636f9d37
VP		 5142 }
5143 }
5144
5145 release_firmware(cf);
5146 if (ret)
5147 goto bye;
5148 }
5149
5150 /*
5151 * Issue a Capability Configuration command to the firmware to get it
5152 * to parse the Configuration File. We don't use t4_fw_config_file()
5153 * because we want the ability to modify various features after we've
5154 * processed the configuration file ...
5155 */
5156 memset(&caps_cmd, 0, sizeof(caps_cmd));
5157 caps_cmd.op_to_write =
e2ac9628
HS		 5158 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5159 FW_CMD_REQUEST_F |
5160 FW_CMD_READ_F);
ce91a923 5161 caps_cmd.cfvalid_to_len16 =
5167865a
HS		 5162 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F |
5163 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) |
5164 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) |
636f9d37
VP		 5165 FW_LEN16(caps_cmd));
5166 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5167 &caps_cmd);
16e47624
HS		 5168
5169 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
5170 * Configuration File in FLASH), our last gasp effort is to use the
5171 * Firmware Configuration File which is embedded in the firmware. A
5172 * very few early versions of the firmware didn't have one embedded
5173 * but we can ignore those.
5174 */
5175 if (ret == -ENOENT) {
5176 memset(&caps_cmd, 0, sizeof(caps_cmd));
5177 caps_cmd.op_to_write =
e2ac9628
HS		 5178 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5179 FW_CMD_REQUEST_F |
5180 FW_CMD_READ_F);
16e47624
HS		 5181 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5182 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
5183 sizeof(caps_cmd), &caps_cmd);
5184 config_name = "Firmware Default";
5185 }
5186
5187 config_issued = 1;
636f9d37
VP		 5188 if (ret < 0)
5189 goto bye;
5190
5191 finiver = ntohl(caps_cmd.finiver);
5192 finicsum = ntohl(caps_cmd.finicsum);
5193 cfcsum = ntohl(caps_cmd.cfcsum);
5194 if (finicsum != cfcsum)
5195 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
5196 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
5197 finicsum, cfcsum);
5198
636f9d37
VP		 5199 /*
5200 * And now tell the firmware to use the configuration we just loaded.
5201 */
5202 caps_cmd.op_to_write =
e2ac9628
HS		 5203 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5204 FW_CMD_REQUEST_F |
5205 FW_CMD_WRITE_F);
ce91a923 5206 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 5207 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5208 NULL);
5209 if (ret < 0)
5210 goto bye;
5211
5212 /*
5213 * Tweak configuration based on system architecture, module
5214 * parameters, etc.
5215 */
5216 ret = adap_init0_tweaks(adapter);
5217 if (ret < 0)
5218 goto bye;
5219
5220 /*
5221 * And finally tell the firmware to initialize itself using the
5222 * parameters from the Configuration File.
5223 */
5224 ret = t4_fw_initialize(adapter, adapter->mbox);
5225 if (ret < 0)
5226 goto bye;
5227
5228 /*
5229 * Return successfully and note that we're operating with parameters
5230 * not supplied by the driver, rather than from hard-wired
5231 * initialization constants burried in the driver.
5232 */
5233 adapter->flags |= USING_SOFT_PARAMS;
5234 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
16e47624
HS		 5235 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
5236 config_name, finiver, cfcsum);
636f9d37
VP		 5237 return 0;
5238
5239 /*
5240 * Something bad happened. Return the error ... (If the "error"
5241 * is that there's no Configuration File on the adapter we don't
5242 * want to issue a warning since this is fairly common.)
5243 */
5244bye:
16e47624
HS		 5245 if (config_issued && ret != -ENOENT)
5246 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
5247 config_name, -ret);
636f9d37
VP		 5248 return ret;
5249}
5250
13ee15d3
VP		 5251/*
5252 * Attempt to initialize the adapter via hard-coded, driver supplied
5253 * parameters ...
5254 */
5255static int adap_init0_no_config(struct adapter *adapter, int reset)
5256{
5257 struct sge *s = &adapter->sge;
5258 struct fw_caps_config_cmd caps_cmd;
5259 u32 v;
5260 int i, ret;
5261
5262 /*
5263 * Reset device if necessary
5264 */
5265 if (reset) {
5266 ret = t4_fw_reset(adapter, adapter->mbox,
0d804338 5267 PIORSTMODE_F | PIORST_F);
13ee15d3
VP		 5268 if (ret < 0)
5269 goto bye;
5270 }
5271
5272 /*
5273 * Get device capabilities and select which we'll be using.
5274 */
5275 memset(&caps_cmd, 0, sizeof(caps_cmd));
e2ac9628
HS		 5276 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5277 FW_CMD_REQUEST_F | FW_CMD_READ_F);
ce91a923 5278 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
13ee15d3
VP		 5279 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5280 &caps_cmd);
5281 if (ret < 0)
5282 goto bye;
5283
13ee15d3
VP		 5284 if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
5285 if (!vf_acls)
5286 caps_cmd.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
5287 else
5288 caps_cmd.niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
5289 } else if (vf_acls) {
5290 dev_err(adapter->pdev_dev, "virtualization ACLs not supported");
5291 goto bye;
5292 }
e2ac9628
HS		 5293 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5294 FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
13ee15d3
VP		 5295 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5296 NULL);
5297 if (ret < 0)
5298 goto bye;
5299
5300 /*
5301 * Tweak configuration based on system architecture, module
5302 * parameters, etc.
5303 */
5304 ret = adap_init0_tweaks(adapter);
5305 if (ret < 0)
5306 goto bye;
5307
5308 /*
5309 * Select RSS Global Mode we want to use. We use "Basic Virtual"
5310 * mode which maps each Virtual Interface to its own section of
5311 * the RSS Table and we turn on all map and hash enables ...
5312 */
5313 adapter->flags |= RSS_TNLALLLOOKUP;
5314 ret = t4_config_glbl_rss(adapter, adapter->mbox,
5315 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
b2e1a3f0
HS		 5316 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F |
5317 FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ_F |
13ee15d3 5318 ((adapter->flags & RSS_TNLALLLOOKUP) ?
b2e1a3f0 5319 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F : 0));
13ee15d3
VP		 5320 if (ret < 0)
5321 goto bye;
5322
5323 /*
5324 * Set up our own fundamental resource provisioning ...
5325 */
5326 ret = t4_cfg_pfvf(adapter, adapter->mbox, adapter->fn, 0,
5327 PFRES_NEQ, PFRES_NETHCTRL,
5328 PFRES_NIQFLINT, PFRES_NIQ,
5329 PFRES_TC, PFRES_NVI,
5167865a 5330 FW_PFVF_CMD_CMASK_M,
13ee15d3
VP		 5331 pfvfres_pmask(adapter, adapter->fn, 0),
5332 PFRES_NEXACTF,
5333 PFRES_R_CAPS, PFRES_WX_CAPS);
5334 if (ret < 0)
5335 goto bye;
5336
5337 /*
5338 * Perform low level SGE initialization. We need to do this before we
5339 * send the firmware the INITIALIZE command because that will cause
5340 * any other PF Drivers which are waiting for the Master
5341 * Initialization to proceed forward.
5342 */
5343 for (i = 0; i < SGE_NTIMERS - 1; i++)
5344 s->timer_val[i] = min(intr_holdoff[i], MAX_SGE_TIMERVAL);
5345 s->timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
5346 s->counter_val[0] = 1;
5347 for (i = 1; i < SGE_NCOUNTERS; i++)
f612b815 5348 s->counter_val[i] = min(intr_cnt[i - 1], THRESHOLD_0_M);
13ee15d3
VP		 5349 t4_sge_init(adapter);
5350
5351#ifdef CONFIG_PCI_IOV
5352 /*
5353 * Provision resource limits for Virtual Functions. We currently
5354 * grant them all the same static resource limits except for the Port
5355 * Access Rights Mask which we're assigning based on the PF. All of
5356 * the static provisioning stuff for both the PF and VF really needs
5357 * to be managed in a persistent manner for each device which the
5358 * firmware controls.
5359 */
5360 {
5361 int pf, vf;
5362
7d6727cf 5363 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
13ee15d3
VP		 5364 if (num_vf[pf] <= 0)
5365 continue;
5366
5367 /* VF numbering starts at 1! */
5368 for (vf = 1; vf <= num_vf[pf]; vf++) {
5369 ret = t4_cfg_pfvf(adapter, adapter->mbox,
5370 pf, vf,
5371 VFRES_NEQ, VFRES_NETHCTRL,
5372 VFRES_NIQFLINT, VFRES_NIQ,
5373 VFRES_TC, VFRES_NVI,
5167865a 5374 FW_PFVF_CMD_CMASK_M,
13ee15d3
VP		 5375 pfvfres_pmask(
5376 adapter, pf, vf),
5377 VFRES_NEXACTF,
5378 VFRES_R_CAPS, VFRES_WX_CAPS);
5379 if (ret < 0)
5380 dev_warn(adapter->pdev_dev,
5381 "failed to "\
5382 "provision pf/vf=%d/%d; "
5383 "err=%d\n", pf, vf, ret);
5384 }
5385 }
5386 }
5387#endif
5388
5389 /*
5390 * Set up the default filter mode. Later we'll want to implement this
5391 * via a firmware command, etc. ... This needs to be done before the
5392 * firmare initialization command ... If the selected set of fields
5393 * isn't equal to the default value, we'll need to make sure that the
5394 * field selections will fit in the 36-bit budget.
5395 */
5396 if (tp_vlan_pri_map != TP_VLAN_PRI_MAP_DEFAULT) {
404d9e3f 5397 int j, bits = 0;
13ee15d3 5398
404d9e3f
VP		 5399 for (j = TP_VLAN_PRI_MAP_FIRST; j <= TP_VLAN_PRI_MAP_LAST; j++)
5400 switch (tp_vlan_pri_map & (1 << j)) {
13ee15d3
VP		 5401 case 0:
5402 /* compressed filter field not enabled */
5403 break;
837e4a42 5404 case FCOE_F:
13ee15d3
VP		 5405 bits += 1;
5406 break;
837e4a42 5407 case PORT_F:
13ee15d3
VP		 5408 bits += 3;
5409 break;
837e4a42 5410 case VNIC_F:
13ee15d3
VP		 5411 bits += 17;
5412 break;
837e4a42 5413 case VLAN_F:
13ee15d3
VP		 5414 bits += 17;
5415 break;
837e4a42 5416 case TOS_F:
13ee15d3
VP		 5417 bits += 8;
5418 break;
837e4a42 5419 case PROTOCOL_F:
13ee15d3
VP		 5420 bits += 8;
5421 break;
837e4a42 5422 case ETHERTYPE_F:
13ee15d3
VP		 5423 bits += 16;
5424 break;
837e4a42 5425 case MACMATCH_F:
13ee15d3
VP		 5426 bits += 9;
5427 break;
837e4a42 5428 case MPSHITTYPE_F:
13ee15d3
VP		 5429 bits += 3;
5430 break;
837e4a42 5431 case FRAGMENTATION_F:
13ee15d3
VP		 5432 bits += 1;
5433 break;
5434 }
5435
5436 if (bits > 36) {
5437 dev_err(adapter->pdev_dev,
5438 "tp_vlan_pri_map=%#x needs %d bits > 36;"\
5439 " using %#x\n", tp_vlan_pri_map, bits,
5440 TP_VLAN_PRI_MAP_DEFAULT);
5441 tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
5442 }
5443 }
5444 v = tp_vlan_pri_map;
837e4a42
HS		 5445 t4_write_indirect(adapter, TP_PIO_ADDR_A, TP_PIO_DATA_A,
5446 &v, 1, TP_VLAN_PRI_MAP_A);
13ee15d3
VP		 5447
5448 /*
5449 * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
5450 * to support any of the compressed filter fields above. Newer
5451 * versions of the firmware do this automatically but it doesn't hurt
5452 * to set it here. Meanwhile, we do _not_ need to set Lookup Every
5453 * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
5454 * since the firmware automatically turns this on and off when we have
5455 * a non-zero number of filters active (since it does have a
5456 * performance impact).
5457 */
5458 if (tp_vlan_pri_map)
837e4a42
HS		 5459 t4_set_reg_field(adapter, TP_GLOBAL_CONFIG_A,
5460 FIVETUPLELOOKUP_V(FIVETUPLELOOKUP_M),
5461 FIVETUPLELOOKUP_V(FIVETUPLELOOKUP_M));
13ee15d3
VP		 5462
5463 /*
5464 * Tweak some settings.
5465 */
837e4a42
HS		 5466 t4_write_reg(adapter, TP_SHIFT_CNT_A, SYNSHIFTMAX_V(6) |
5467 RXTSHIFTMAXR1_V(4) | RXTSHIFTMAXR2_V(15) |
5468 PERSHIFTBACKOFFMAX_V(8) | PERSHIFTMAX_V(8) |
5469 KEEPALIVEMAXR1_V(4) | KEEPALIVEMAXR2_V(9));
13ee15d3
VP		 5470
5471 /*
5472 * Get basic stuff going by issuing the Firmware Initialize command.
5473 * Note that this _must_ be after all PFVF commands ...
5474 */
5475 ret = t4_fw_initialize(adapter, adapter->mbox);
5476 if (ret < 0)
5477 goto bye;
5478
5479 /*
5480 * Return successfully!
5481 */
5482 dev_info(adapter->pdev_dev, "Successfully configured using built-in "\
5483 "driver parameters\n");
5484 return 0;
5485
5486 /*
5487 * Something bad happened. Return the error ...
5488 */
5489bye:
5490 return ret;
5491}
5492
16e47624
HS		 5493static struct fw_info fw_info_array[] = {
5494 {
5495 .chip = CHELSIO_T4,
5496 .fs_name = FW4_CFNAME,
5497 .fw_mod_name = FW4_FNAME,
5498 .fw_hdr = {
5499 .chip = FW_HDR_CHIP_T4,
5500 .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5501 .intfver_nic = FW_INTFVER(T4, NIC),
5502 .intfver_vnic = FW_INTFVER(T4, VNIC),
5503 .intfver_ri = FW_INTFVER(T4, RI),
5504 .intfver_iscsi = FW_INTFVER(T4, ISCSI),
5505 .intfver_fcoe = FW_INTFVER(T4, FCOE),
5506 },
5507 }, {
5508 .chip = CHELSIO_T5,
5509 .fs_name = FW5_CFNAME,
5510 .fw_mod_name = FW5_FNAME,
5511 .fw_hdr = {
5512 .chip = FW_HDR_CHIP_T5,
5513 .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5514 .intfver_nic = FW_INTFVER(T5, NIC),
5515 .intfver_vnic = FW_INTFVER(T5, VNIC),
5516 .intfver_ri = FW_INTFVER(T5, RI),
5517 .intfver_iscsi = FW_INTFVER(T5, ISCSI),
5518 .intfver_fcoe = FW_INTFVER(T5, FCOE),
5519 },
5520 }
5521};
5522
5523static struct fw_info *find_fw_info(int chip)
5524{
5525 int i;
5526
5527 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5528 if (fw_info_array[i].chip == chip)
5529 return &fw_info_array[i];
5530 }
5531 return NULL;
5532}
5533
b8ff05a9
DM		 5534/*
5535 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5536 */
5537static int adap_init0(struct adapter *adap)
5538{
5539 int ret;
5540 u32 v, port_vec;
5541 enum dev_state state;
5542 u32 params[7], val[7];
9a4da2cd 5543 struct fw_caps_config_cmd caps_cmd;
49aa284f
HS		 5544 struct fw_devlog_cmd devlog_cmd;
5545 u32 devlog_meminfo;
dcf7b6f5 5546 int reset = 1;
b8ff05a9 5547
666224d4
HS		 5548 /* Contact FW, advertising Master capability */
5549 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state);
b8ff05a9
DM		 5550 if (ret < 0) {
5551 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5552 ret);
5553 return ret;
5554 }
636f9d37
VP		 5555 if (ret == adap->mbox)
5556 adap->flags |= MASTER_PF;
b8ff05a9 5557
636f9d37
VP		 5558 /*
5559 * If we're the Master PF Driver and the device is uninitialized,
5560 * then let's consider upgrading the firmware ... (We always want
5561 * to check the firmware version number in order to A. get it for
5562 * later reporting and B. to warn if the currently loaded firmware
5563 * is excessively mismatched relative to the driver.)
5564 */
16e47624
HS		 5565 t4_get_fw_version(adap, &adap->params.fw_vers);
5566 t4_get_tp_version(adap, &adap->params.tp_vers);
636f9d37 5567 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
16e47624
HS		 5568 struct fw_info *fw_info;
5569 struct fw_hdr *card_fw;
5570 const struct firmware *fw;
5571 const u8 *fw_data = NULL;
5572 unsigned int fw_size = 0;
5573
5574 /* This is the firmware whose headers the driver was compiled
5575 * against
5576 */
5577 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5578 if (fw_info == NULL) {
5579 dev_err(adap->pdev_dev,
5580 "unable to get firmware info for chip %d.\n",
5581 CHELSIO_CHIP_VERSION(adap->params.chip));
5582 return -EINVAL;
636f9d37 5583 }
16e47624
HS		 5584
5585 /* allocate memory to read the header of the firmware on the
5586 * card
5587 */
5588 card_fw = t4_alloc_mem(sizeof(*card_fw));
5589
5590 /* Get FW from from /lib/firmware/ */
5591 ret = request_firmware(&fw, fw_info->fw_mod_name,
5592 adap->pdev_dev);
5593 if (ret < 0) {
5594 dev_err(adap->pdev_dev,
5595 "unable to load firmware image %s, error %d\n",
5596 fw_info->fw_mod_name, ret);
5597 } else {
5598 fw_data = fw->data;
5599 fw_size = fw->size;
5600 }
5601
5602 /* upgrade FW logic */
5603 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5604 state, &reset);
5605
5606 /* Cleaning up */
5607 if (fw != NULL)
5608 release_firmware(fw);
5609 t4_free_mem(card_fw);
5610
636f9d37 5611 if (ret < 0)
16e47624 5612 goto bye;
636f9d37 5613 }
b8ff05a9 5614
636f9d37
VP		 5615 /*
5616 * Grab VPD parameters. This should be done after we establish a
5617 * connection to the firmware since some of the VPD parameters
5618 * (notably the Core Clock frequency) are retrieved via requests to
5619 * the firmware. On the other hand, we need these fairly early on
5620 * so we do this right after getting ahold of the firmware.
5621 */
5622 ret = get_vpd_params(adap, &adap->params.vpd);
a0881cab
DM		 5623 if (ret < 0)
5624 goto bye;
a0881cab 5625
49aa284f
HS		 5626 /* Read firmware device log parameters. We really need to find a way
5627 * to get these parameters initialized with some default values (which
5628 * are likely to be correct) for the case where we either don't
5629 * attache to the firmware or it's crashed when we probe the adapter.
5630 * That way we'll still be able to perform early firmware startup
5631 * debugging ... If the request to get the Firmware's Device Log
5632 * parameters fails, we'll live so we don't make that a fatal error.
5633 */
5634 memset(&devlog_cmd, 0, sizeof(devlog_cmd));
5635 devlog_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_DEVLOG_CMD) |
5636 FW_CMD_REQUEST_F | FW_CMD_READ_F);
5637 devlog_cmd.retval_len16 = htonl(FW_LEN16(devlog_cmd));
5638 ret = t4_wr_mbox(adap, adap->mbox, &devlog_cmd, sizeof(devlog_cmd),
5639 &devlog_cmd);
5640 if (ret == 0) {
5641 devlog_meminfo =
5642 ntohl(devlog_cmd.memtype_devlog_memaddr16_devlog);
5643 adap->params.devlog.memtype =
5644 FW_DEVLOG_CMD_MEMTYPE_DEVLOG_G(devlog_meminfo);
5645 adap->params.devlog.start =
5646 FW_DEVLOG_CMD_MEMADDR16_DEVLOG_G(devlog_meminfo) << 4;
5647 adap->params.devlog.size = ntohl(devlog_cmd.memsize_devlog);
5648 }
5649
636f9d37 5650 /*
13ee15d3
VP		 5651 * Find out what ports are available to us. Note that we need to do
5652 * this before calling adap_init0_no_config() since it needs nports
5653 * and portvec ...
636f9d37
VP		 5654 */
5655 v =
5167865a
HS		 5656 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5657 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC);
636f9d37 5658 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
a0881cab
DM		 5659 if (ret < 0)
5660 goto bye;
5661
636f9d37
VP		 5662 adap->params.nports = hweight32(port_vec);
5663 adap->params.portvec = port_vec;
5664
5665 /*
5666 * If the firmware is initialized already (and we're not forcing a
5667 * master initialization), note that we're living with existing
5668 * adapter parameters. Otherwise, it's time to try initializing the
5669 * adapter ...
5670 */
5671 if (state == DEV_STATE_INIT) {
5672 dev_info(adap->pdev_dev, "Coming up as %s: "\
5673 "Adapter already initialized\n",
5674 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
5675 adap->flags |= USING_SOFT_PARAMS;
5676 } else {
5677 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5678 "Initializing adapter\n");
636f9d37
VP		 5679 /*
5680 * If the firmware doesn't support Configuration
5681 * Files warn user and exit,
5682 */
5683 if (ret < 0)
13ee15d3 5684 dev_warn(adap->pdev_dev, "Firmware doesn't support "
636f9d37 5685 "configuration file.\n");
13ee15d3
VP		 5686 if (force_old_init)
5687 ret = adap_init0_no_config(adap, reset);
636f9d37
VP		 5688 else {
5689 /*
13ee15d3
VP		 5690 * Find out whether we're dealing with a version of
5691 * the firmware which has configuration file support.
636f9d37 5692 */
5167865a
HS		 5693 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5694 FW_PARAMS_PARAM_X_V(
5695 FW_PARAMS_PARAM_DEV_CF));
13ee15d3
VP		 5696 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5697 params, val);
636f9d37 5698
13ee15d3
VP		 5699 /*
5700 * If the firmware doesn't support Configuration
5701 * Files, use the old Driver-based, hard-wired
5702 * initialization. Otherwise, try using the
5703 * Configuration File support and fall back to the
5704 * Driver-based initialization if there's no
5705 * Configuration File found.
5706 */
5707 if (ret < 0)
5708 ret = adap_init0_no_config(adap, reset);
5709 else {
5710 /*
5711 * The firmware provides us with a memory
5712 * buffer where we can load a Configuration
5713 * File from the host if we want to override
5714 * the Configuration File in flash.
5715 */
5716
5717 ret = adap_init0_config(adap, reset);
5718 if (ret == -ENOENT) {
5719 dev_info(adap->pdev_dev,
5720 "No Configuration File present "
16e47624 5721 "on adapter. Using hard-wired "
13ee15d3
VP		 5722 "configuration parameters.\n");
5723 ret = adap_init0_no_config(adap, reset);
5724 }
636f9d37
VP		 5725 }
5726 }
5727 if (ret < 0) {
5728 dev_err(adap->pdev_dev,
5729 "could not initialize adapter, error %d\n",
5730 -ret);
5731 goto bye;
5732 }
5733 }
5734
5735 /*
5736 * If we're living with non-hard-coded parameters (either from a
5737 * Firmware Configuration File or values programmed by a different PF
5738 * Driver), give the SGE code a chance to pull in anything that it
5739 * needs ... Note that this must be called after we retrieve our VPD
5740 * parameters in order to know how to convert core ticks to seconds.
5741 */
5742 if (adap->flags & USING_SOFT_PARAMS) {
5743 ret = t4_sge_init(adap);
5744 if (ret < 0)
5745 goto bye;
5746 }
5747
9a4da2cd
VP		 5748 if (is_bypass_device(adap->pdev->device))
5749 adap->params.bypass = 1;
5750
636f9d37
VP		 5751 /*
5752 * Grab some of our basic fundamental operating parameters.
5753 */
5754#define FW_PARAM_DEV(param) \
5167865a
HS		 5755 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \
5756 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param))
636f9d37 5757
b8ff05a9 5758#define FW_PARAM_PFVF(param) \
5167865a
HS		 5759 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \
5760 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \
5761 FW_PARAMS_PARAM_Y_V(0) | \
5762 FW_PARAMS_PARAM_Z_V(0)
b8ff05a9 5763
636f9d37 5764 params[0] = FW_PARAM_PFVF(EQ_START);
b8ff05a9
DM		 5765 params[1] = FW_PARAM_PFVF(L2T_START);
5766 params[2] = FW_PARAM_PFVF(L2T_END);
5767 params[3] = FW_PARAM_PFVF(FILTER_START);
5768 params[4] = FW_PARAM_PFVF(FILTER_END);
e46dab4d 5769 params[5] = FW_PARAM_PFVF(IQFLINT_START);
636f9d37 5770 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
b8ff05a9
DM		 5771 if (ret < 0)
5772 goto bye;
636f9d37
VP		 5773 adap->sge.egr_start = val[0];
5774 adap->l2t_start = val[1];
5775 adap->l2t_end = val[2];
b8ff05a9
DM		 5776 adap->tids.ftid_base = val[3];
5777 adap->tids.nftids = val[4] - val[3] + 1;
e46dab4d 5778 adap->sge.ingr_start = val[5];
b8ff05a9 5779
636f9d37
VP		 5780 /* query params related to active filter region */
5781 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5782 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5783 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5784 /* If Active filter size is set we enable establishing
5785 * offload connection through firmware work request
5786 */
5787 if ((val[0] != val[1]) && (ret >= 0)) {
5788 adap->flags |= FW_OFLD_CONN;
5789 adap->tids.aftid_base = val[0];
5790 adap->tids.aftid_end = val[1];
5791 }
5792
b407a4a9
VP		 5793 /* If we're running on newer firmware, let it know that we're
5794 * prepared to deal with encapsulated CPL messages. Older
5795 * firmware won't understand this and we'll just get
5796 * unencapsulated messages ...
5797 */
5798 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5799 val[0] = 1;
5800 (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5801
1ac0f095
KS		 5802 /*
5803 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5804 * capability. Earlier versions of the firmware didn't have the
5805 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5806 * permission to use ULPTX MEMWRITE DSGL.
5807 */
5808 if (is_t4(adap->params.chip)) {
5809 adap->params.ulptx_memwrite_dsgl = false;
5810 } else {
5811 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5812 ret = t4_query_params(adap, adap->mbox, adap->fn, 0,
5813 1, params, val);
5814 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
5815 }
5816
636f9d37
VP		 5817 /*
5818 * Get device capabilities so we can determine what resources we need
5819 * to manage.
5820 */
5821 memset(&caps_cmd, 0, sizeof(caps_cmd));
e2ac9628
HS		 5822 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5823 FW_CMD_REQUEST_F | FW_CMD_READ_F);
ce91a923 5824 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 5825 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5826 &caps_cmd);
5827 if (ret < 0)
5828 goto bye;
5829
13ee15d3 5830 if (caps_cmd.ofldcaps) {
b8ff05a9
DM		 5831 /* query offload-related parameters */
5832 params[0] = FW_PARAM_DEV(NTID);
5833 params[1] = FW_PARAM_PFVF(SERVER_START);
5834 params[2] = FW_PARAM_PFVF(SERVER_END);
5835 params[3] = FW_PARAM_PFVF(TDDP_START);
5836 params[4] = FW_PARAM_PFVF(TDDP_END);
5837 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
636f9d37
VP		 5838 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5839 params, val);
b8ff05a9
DM		 5840 if (ret < 0)
5841 goto bye;
5842 adap->tids.ntids = val[0];
5843 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5844 adap->tids.stid_base = val[1];
5845 adap->tids.nstids = val[2] - val[1] + 1;
636f9d37
VP		 5846 /*
5847 * Setup server filter region. Divide the availble filter
5848 * region into two parts. Regular filters get 1/3rd and server
5849 * filters get 2/3rd part. This is only enabled if workarond
5850 * path is enabled.
5851 * 1. For regular filters.
5852 * 2. Server filter: This are special filters which are used
5853 * to redirect SYN packets to offload queue.
5854 */
5855 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5856 adap->tids.sftid_base = adap->tids.ftid_base +
5857 DIV_ROUND_UP(adap->tids.nftids, 3);
5858 adap->tids.nsftids = adap->tids.nftids -
5859 DIV_ROUND_UP(adap->tids.nftids, 3);
5860 adap->tids.nftids = adap->tids.sftid_base -
5861 adap->tids.ftid_base;
5862 }
b8ff05a9
DM		 5863 adap->vres.ddp.start = val[3];
5864 adap->vres.ddp.size = val[4] - val[3] + 1;
5865 adap->params.ofldq_wr_cred = val[5];
636f9d37 5866
b8ff05a9
DM		 5867 adap->params.offload = 1;
5868 }
636f9d37 5869 if (caps_cmd.rdmacaps) {
b8ff05a9
DM		 5870 params[0] = FW_PARAM_PFVF(STAG_START);
5871 params[1] = FW_PARAM_PFVF(STAG_END);
5872 params[2] = FW_PARAM_PFVF(RQ_START);
5873 params[3] = FW_PARAM_PFVF(RQ_END);
5874 params[4] = FW_PARAM_PFVF(PBL_START);
5875 params[5] = FW_PARAM_PFVF(PBL_END);
636f9d37
VP		 5876 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5877 params, val);
b8ff05a9
DM		 5878 if (ret < 0)
5879 goto bye;
5880 adap->vres.stag.start = val[0];
5881 adap->vres.stag.size = val[1] - val[0] + 1;
5882 adap->vres.rq.start = val[2];
5883 adap->vres.rq.size = val[3] - val[2] + 1;
5884 adap->vres.pbl.start = val[4];
5885 adap->vres.pbl.size = val[5] - val[4] + 1;
a0881cab
DM		 5886
5887 params[0] = FW_PARAM_PFVF(SQRQ_START);
5888 params[1] = FW_PARAM_PFVF(SQRQ_END);
5889 params[2] = FW_PARAM_PFVF(CQ_START);
5890 params[3] = FW_PARAM_PFVF(CQ_END);
1ae970e0
DM		 5891 params[4] = FW_PARAM_PFVF(OCQ_START);
5892 params[5] = FW_PARAM_PFVF(OCQ_END);
5c937dd3
HS		 5893 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params,
5894 val);
a0881cab
DM		 5895 if (ret < 0)
5896 goto bye;
5897 adap->vres.qp.start = val[0];
5898 adap->vres.qp.size = val[1] - val[0] + 1;
5899 adap->vres.cq.start = val[2];
5900 adap->vres.cq.size = val[3] - val[2] + 1;
1ae970e0
DM		 5901 adap->vres.ocq.start = val[4];
5902 adap->vres.ocq.size = val[5] - val[4] + 1;
4c2c5763
HS		 5903
5904 params[0] = FW_PARAM_DEV(MAXORDIRD_QP);
5905 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER);
5c937dd3
HS		 5906 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params,
5907 val);
4c2c5763
HS		 5908 if (ret < 0) {
5909 adap->params.max_ordird_qp = 8;
5910 adap->params.max_ird_adapter = 32 * adap->tids.ntids;
5911 ret = 0;
5912 } else {
5913 adap->params.max_ordird_qp = val[0];
5914 adap->params.max_ird_adapter = val[1];
5915 }
5916 dev_info(adap->pdev_dev,
5917 "max_ordird_qp %d max_ird_adapter %d\n",
5918 adap->params.max_ordird_qp,
5919 adap->params.max_ird_adapter);
b8ff05a9 5920 }
636f9d37 5921 if (caps_cmd.iscsicaps) {
b8ff05a9
DM		 5922 params[0] = FW_PARAM_PFVF(ISCSI_START);
5923 params[1] = FW_PARAM_PFVF(ISCSI_END);
636f9d37
VP		 5924 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5925 params, val);
b8ff05a9
DM		 5926 if (ret < 0)
5927 goto bye;
5928 adap->vres.iscsi.start = val[0];
5929 adap->vres.iscsi.size = val[1] - val[0] + 1;
5930 }
5931#undef FW_PARAM_PFVF
5932#undef FW_PARAM_DEV
5933
92e7ae71
HS		 5934 /* The MTU/MSS Table is initialized by now, so load their values. If
5935 * we're initializing the adapter, then we'll make any modifications
5936 * we want to the MTU/MSS Table and also initialize the congestion
5937 * parameters.
636f9d37 5938 */
b8ff05a9 5939 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
92e7ae71
HS		 5940 if (state != DEV_STATE_INIT) {
5941 int i;
5942
5943 /* The default MTU Table contains values 1492 and 1500.
5944 * However, for TCP, it's better to have two values which are
5945 * a multiple of 8 +/- 4 bytes apart near this popular MTU.
5946 * This allows us to have a TCP Data Payload which is a
5947 * multiple of 8 regardless of what combination of TCP Options
5948 * are in use (always a multiple of 4 bytes) which is
5949 * important for performance reasons. For instance, if no
5950 * options are in use, then we have a 20-byte IP header and a
5951 * 20-byte TCP header. In this case, a 1500-byte MSS would
5952 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
5953 * which is not a multiple of 8. So using an MSS of 1488 in
5954 * this case results in a TCP Data Payload of 1448 bytes which
5955 * is a multiple of 8. On the other hand, if 12-byte TCP Time
5956 * Stamps have been negotiated, then an MTU of 1500 bytes
5957 * results in a TCP Data Payload of 1448 bytes which, as
5958 * above, is a multiple of 8 bytes ...
5959 */
5960 for (i = 0; i < NMTUS; i++)
5961 if (adap->params.mtus[i] == 1492) {
5962 adap->params.mtus[i] = 1488;
5963 break;
5964 }
7ee9ff94 5965
92e7ae71
HS		 5966 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5967 adap->params.b_wnd);
5968 }
df64e4d3 5969 t4_init_sge_params(adap);
dcf7b6f5 5970 t4_init_tp_params(adap);
636f9d37 5971 adap->flags |= FW_OK;
b8ff05a9
DM		 5972 return 0;
5973
5974 /*
636f9d37
VP		 5975 * Something bad happened. If a command timed out or failed with EIO
5976 * FW does not operate within its spec or something catastrophic
5977 * happened to HW/FW, stop issuing commands.
b8ff05a9 5978 */
636f9d37
VP		 5979bye:
5980 if (ret != -ETIMEDOUT && ret != -EIO)
5981 t4_fw_bye(adap, adap->mbox);
b8ff05a9
DM		 5982 return ret;
5983}
5984
204dc3c0
DM		 5985/* EEH callbacks */
5986
5987static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5988 pci_channel_state_t state)
5989{
5990 int i;
5991 struct adapter *adap = pci_get_drvdata(pdev);
5992
5993 if (!adap)
5994 goto out;
5995
5996 rtnl_lock();
5997 adap->flags &= ~FW_OK;
5998 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
9fe6cb58 5999 spin_lock(&adap->stats_lock);
204dc3c0
DM		 6000 for_each_port(adap, i) {
6001 struct net_device *dev = adap->port[i];
6002
6003 netif_device_detach(dev);
6004 netif_carrier_off(dev);
6005 }
9fe6cb58 6006 spin_unlock(&adap->stats_lock);
204dc3c0
DM		 6007 if (adap->flags & FULL_INIT_DONE)
6008 cxgb_down(adap);
6009 rtnl_unlock();
144be3d9
GS		 6010 if ((adap->flags & DEV_ENABLED)) {
6011 pci_disable_device(pdev);
6012 adap->flags &= ~DEV_ENABLED;
6013 }
204dc3c0
DM		 6014out: return state == pci_channel_io_perm_failure ?
6015 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
6016}
6017
6018static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
6019{
6020 int i, ret;
6021 struct fw_caps_config_cmd c;
6022 struct adapter *adap = pci_get_drvdata(pdev);
6023
6024 if (!adap) {
6025 pci_restore_state(pdev);
6026 pci_save_state(pdev);
6027 return PCI_ERS_RESULT_RECOVERED;
6028 }
6029
144be3d9
GS		 6030 if (!(adap->flags & DEV_ENABLED)) {
6031 if (pci_enable_device(pdev)) {
6032 dev_err(&pdev->dev, "Cannot reenable PCI "
6033 "device after reset\n");
6034 return PCI_ERS_RESULT_DISCONNECT;
6035 }
6036 adap->flags |= DEV_ENABLED;
204dc3c0
DM		 6037 }
6038
6039 pci_set_master(pdev);
6040 pci_restore_state(pdev);
6041 pci_save_state(pdev);
6042 pci_cleanup_aer_uncorrect_error_status(pdev);
6043
8203b509 6044 if (t4_wait_dev_ready(adap->regs) < 0)
204dc3c0 6045 return PCI_ERS_RESULT_DISCONNECT;
777c2300 6046 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
204dc3c0
DM		 6047 return PCI_ERS_RESULT_DISCONNECT;
6048 adap->flags |= FW_OK;
6049 if (adap_init1(adap, &c))
6050 return PCI_ERS_RESULT_DISCONNECT;
6051
6052 for_each_port(adap, i) {
6053 struct port_info *p = adap2pinfo(adap, i);
6054
060e0c75
DM		 6055 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
6056 NULL, NULL);
204dc3c0
DM		 6057 if (ret < 0)
6058 return PCI_ERS_RESULT_DISCONNECT;
6059 p->viid = ret;
6060 p->xact_addr_filt = -1;
6061 }
6062
6063 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
6064 adap->params.b_wnd);
1ae970e0 6065 setup_memwin(adap);
204dc3c0
DM		 6066 if (cxgb_up(adap))
6067 return PCI_ERS_RESULT_DISCONNECT;
6068 return PCI_ERS_RESULT_RECOVERED;
6069}
6070
6071static void eeh_resume(struct pci_dev *pdev)
6072{
6073 int i;
6074 struct adapter *adap = pci_get_drvdata(pdev);
6075
6076 if (!adap)
6077 return;
6078
6079 rtnl_lock();
6080 for_each_port(adap, i) {
6081 struct net_device *dev = adap->port[i];
6082
6083 if (netif_running(dev)) {
6084 link_start(dev);
6085 cxgb_set_rxmode(dev);
6086 }
6087 netif_device_attach(dev);
6088 }
6089 rtnl_unlock();
6090}
6091
3646f0e5 6092static const struct pci_error_handlers cxgb4_eeh = {
204dc3c0
DM		 6093 .error_detected = eeh_err_detected,
6094 .slot_reset = eeh_slot_reset,
6095 .resume = eeh_resume,
6096};
6097
57d8b764 6098static inline bool is_x_10g_port(const struct link_config *lc)
b8ff05a9 6099{
57d8b764
KS		 6100 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
6101 (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
b8ff05a9
DM		 6102}
6103
c887ad0e
HS		 6104static inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
6105 unsigned int us, unsigned int cnt,
b8ff05a9
DM		 6106 unsigned int size, unsigned int iqe_size)
6107{
c887ad0e
HS		 6108 q->adap = adap;
6109 set_rspq_intr_params(q, us, cnt);
b8ff05a9
DM		 6110 q->iqe_len = iqe_size;
6111 q->size = size;
6112}
6113
6114/*
6115 * Perform default configuration of DMA queues depending on the number and type
6116 * of ports we found and the number of available CPUs. Most settings can be
6117 * modified by the admin prior to actual use.
6118 */
91744948 6119static void cfg_queues(struct adapter *adap)
b8ff05a9
DM		 6120{
6121 struct sge *s = &adap->sge;
688848b1
AB		 6122 int i, n10g = 0, qidx = 0;
6123#ifndef CONFIG_CHELSIO_T4_DCB
6124 int q10g = 0;
6125#endif
cf38be6d 6126 int ciq_size;
b8ff05a9
DM		 6127
6128 for_each_port(adap, i)
57d8b764 6129 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
688848b1
AB		 6130#ifdef CONFIG_CHELSIO_T4_DCB
6131 /* For Data Center Bridging support we need to be able to support up
6132 * to 8 Traffic Priorities; each of which will be assigned to its
6133 * own TX Queue in order to prevent Head-Of-Line Blocking.
6134 */
6135 if (adap->params.nports * 8 > MAX_ETH_QSETS) {
6136 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n",
6137 MAX_ETH_QSETS, adap->params.nports * 8);
6138 BUG_ON(1);
6139 }
b8ff05a9 6140
688848b1
AB		 6141 for_each_port(adap, i) {
6142 struct port_info *pi = adap2pinfo(adap, i);
6143
6144 pi->first_qset = qidx;
6145 pi->nqsets = 8;
6146 qidx += pi->nqsets;
6147 }
6148#else /* !CONFIG_CHELSIO_T4_DCB */
b8ff05a9
DM		 6149 /*
6150 * We default to 1 queue per non-10G port and up to # of cores queues
6151 * per 10G port.
6152 */
6153 if (n10g)
6154 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5952dde7
YM		 6155 if (q10g > netif_get_num_default_rss_queues())
6156 q10g = netif_get_num_default_rss_queues();
b8ff05a9
DM		 6157
6158 for_each_port(adap, i) {
6159 struct port_info *pi = adap2pinfo(adap, i);
6160
6161 pi->first_qset = qidx;
57d8b764 6162 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
b8ff05a9
DM		 6163 qidx += pi->nqsets;
6164 }
688848b1 6165#endif /* !CONFIG_CHELSIO_T4_DCB */
b8ff05a9
DM		 6166
6167 s->ethqsets = qidx;
6168 s->max_ethqsets = qidx; /* MSI-X may lower it later */
6169
6170 if (is_offload(adap)) {
6171 /*
6172 * For offload we use 1 queue/channel if all ports are up to 1G,
6173 * otherwise we divide all available queues amongst the channels
6174 * capped by the number of available cores.
6175 */
6176 if (n10g) {
6177 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
6178 num_online_cpus());
6179 s->ofldqsets = roundup(i, adap->params.nports);
6180 } else
6181 s->ofldqsets = adap->params.nports;
6182 /* For RDMA one Rx queue per channel suffices */
6183 s->rdmaqs = adap->params.nports;
cf38be6d 6184 s->rdmaciqs = adap->params.nports;
b8ff05a9
DM		 6185 }
6186
6187 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
6188 struct sge_eth_rxq *r = &s->ethrxq[i];
6189
c887ad0e 6190 init_rspq(adap, &r->rspq, 5, 10, 1024, 64);
b8ff05a9
DM		 6191 r->fl.size = 72;
6192 }
6193
6194 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
6195 s->ethtxq[i].q.size = 1024;
6196
6197 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
6198 s->ctrlq[i].q.size = 512;
6199
6200 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
6201 s->ofldtxq[i].q.size = 1024;
6202
6203 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
6204 struct sge_ofld_rxq *r = &s->ofldrxq[i];
6205
c887ad0e 6206 init_rspq(adap, &r->rspq, 5, 1, 1024, 64);
b8ff05a9
DM		 6207 r->rspq.uld = CXGB4_ULD_ISCSI;
6208 r->fl.size = 72;
6209 }
6210
6211 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
6212 struct sge_ofld_rxq *r = &s->rdmarxq[i];
6213
c887ad0e 6214 init_rspq(adap, &r->rspq, 5, 1, 511, 64);
b8ff05a9
DM		 6215 r->rspq.uld = CXGB4_ULD_RDMA;
6216 r->fl.size = 72;
6217 }
6218
cf38be6d
HS		 6219 ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
6220 if (ciq_size > SGE_MAX_IQ_SIZE) {
6221 CH_WARN(adap, "CIQ size too small for available IQs\n");
6222 ciq_size = SGE_MAX_IQ_SIZE;
6223 }
6224
6225 for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) {
6226 struct sge_ofld_rxq *r = &s->rdmaciq[i];
6227
c887ad0e 6228 init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64);
cf38be6d
HS		 6229 r->rspq.uld = CXGB4_ULD_RDMA;
6230 }
6231
c887ad0e
HS		 6232 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64);
6233 init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64);
b8ff05a9
DM		 6234}
6235
6236/*
6237 * Reduce the number of Ethernet queues across all ports to at most n.
6238 * n provides at least one queue per port.
6239 */
91744948 6240static void reduce_ethqs(struct adapter *adap, int n)
b8ff05a9
DM		 6241{
6242 int i;
6243 struct port_info *pi;
6244
6245 while (n < adap->sge.ethqsets)
6246 for_each_port(adap, i) {
6247 pi = adap2pinfo(adap, i);
6248 if (pi->nqsets > 1) {
6249 pi->nqsets--;
6250 adap->sge.ethqsets--;
6251 if (adap->sge.ethqsets <= n)
6252 break;
6253 }
6254 }
6255
6256 n = 0;
6257 for_each_port(adap, i) {
6258 pi = adap2pinfo(adap, i);
6259 pi->first_qset = n;
6260 n += pi->nqsets;
6261 }
6262}
6263
6264/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
6265#define EXTRA_VECS 2
6266
91744948 6267static int enable_msix(struct adapter *adap)
b8ff05a9
DM		 6268{
6269 int ofld_need = 0;
c32ad224 6270 int i, want, need;
b8ff05a9
DM		 6271 struct sge *s = &adap->sge;
6272 unsigned int nchan = adap->params.nports;
6273 struct msix_entry entries[MAX_INGQ + 1];
6274
6275 for (i = 0; i < ARRAY_SIZE(entries); ++i)
6276 entries[i].entry = i;
6277
6278 want = s->max_ethqsets + EXTRA_VECS;
6279 if (is_offload(adap)) {
cf38be6d 6280 want += s->rdmaqs + s->rdmaciqs + s->ofldqsets;
b8ff05a9 6281 /* need nchan for each possible ULD */
cf38be6d 6282 ofld_need = 3 * nchan;
b8ff05a9 6283 }
688848b1
AB		 6284#ifdef CONFIG_CHELSIO_T4_DCB
6285 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
6286 * each port.
6287 */
6288 need = 8 * adap->params.nports + EXTRA_VECS + ofld_need;
6289#else
b8ff05a9 6290 need = adap->params.nports + EXTRA_VECS + ofld_need;
688848b1 6291#endif
c32ad224
AG		 6292 want = pci_enable_msix_range(adap->pdev, entries, need, want);
6293 if (want < 0)
6294 return want;
b8ff05a9 6295
c32ad224
AG		 6296 /*
6297 * Distribute available vectors to the various queue groups.
6298 * Every group gets its minimum requirement and NIC gets top
6299 * priority for leftovers.
6300 */
6301 i = want - EXTRA_VECS - ofld_need;
6302 if (i < s->max_ethqsets) {
6303 s->max_ethqsets = i;
6304 if (i < s->ethqsets)
6305 reduce_ethqs(adap, i);
6306 }
6307 if (is_offload(adap)) {
6308 i = want - EXTRA_VECS - s->max_ethqsets;
6309 i -= ofld_need - nchan;
6310 s->ofldqsets = (i / nchan) * nchan; /* round down */
6311 }
6312 for (i = 0; i < want; ++i)
6313 adap->msix_info[i].vec = entries[i].vector;
6314
6315 return 0;
b8ff05a9
DM		 6316}
6317
6318#undef EXTRA_VECS
6319
91744948 6320static int init_rss(struct adapter *adap)
671b0060
DM		 6321{
6322 unsigned int i, j;
6323
6324 for_each_port(adap, i) {
6325 struct port_info *pi = adap2pinfo(adap, i);
6326
6327 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
6328 if (!pi->rss)
6329 return -ENOMEM;
6330 for (j = 0; j < pi->rss_size; j++)
278bc429 6331 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
671b0060
DM		 6332 }
6333 return 0;
6334}
6335
91744948 6336static void print_port_info(const struct net_device *dev)
b8ff05a9 6337{
b8ff05a9 6338 char buf[80];
118969ed 6339 char *bufp = buf;
f1a051b9 6340 const char *spd = "";
118969ed
DM		 6341 const struct port_info *pi = netdev_priv(dev);
6342 const struct adapter *adap = pi->adapter;
f1a051b9
DM		 6343
6344 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
6345 spd = " 2.5 GT/s";
6346 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
6347 spd = " 5 GT/s";
d2e752db
RD		 6348 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB)
6349 spd = " 8 GT/s";
b8ff05a9 6350
118969ed
DM		 6351 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
6352 bufp += sprintf(bufp, "100/");
6353 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
6354 bufp += sprintf(bufp, "1000/");
6355 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
6356 bufp += sprintf(bufp, "10G/");
72aca4bf
KS		 6357 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
6358 bufp += sprintf(bufp, "40G/");
118969ed
DM		 6359 if (bufp != buf)
6360 --bufp;
72aca4bf 6361 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
118969ed
DM		 6362
6363 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
0a57a536 6364 adap->params.vpd.id,
d14807dd 6365 CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
118969ed
DM		 6366 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
6367 (adap->flags & USING_MSIX) ? " MSI-X" :
6368 (adap->flags & USING_MSI) ? " MSI" : "");
a94cd705
KS		 6369 netdev_info(dev, "S/N: %s, P/N: %s\n",
6370 adap->params.vpd.sn, adap->params.vpd.pn);
b8ff05a9
DM		 6371}
6372
91744948 6373static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
ef306b50 6374{
e5c8ae5f 6375 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
ef306b50
DM		 6376}
6377
06546391
DM		 6378/*
6379 * Free the following resources:
6380 * - memory used for tables
6381 * - MSI/MSI-X
6382 * - net devices
6383 * - resources FW is holding for us
6384 */
6385static void free_some_resources(struct adapter *adapter)
6386{
6387 unsigned int i;
6388
6389 t4_free_mem(adapter->l2t);
6390 t4_free_mem(adapter->tids.tid_tab);
6391 disable_msi(adapter);
6392
6393 for_each_port(adapter, i)
671b0060
DM		 6394 if (adapter->port[i]) {
6395 kfree(adap2pinfo(adapter, i)->rss);
06546391 6396 free_netdev(adapter->port[i]);
671b0060 6397 }
06546391 6398 if (adapter->flags & FW_OK)
060e0c75 6399 t4_fw_bye(adapter, adapter->fn);
06546391
DM		 6400}
6401
2ed28baa 6402#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
35d35682 6403#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
b8ff05a9 6404 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
22adfe0a 6405#define SEGMENT_SIZE 128
b8ff05a9 6406
1dd06ae8 6407static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
b8ff05a9 6408{
22adfe0a 6409 int func, i, err, s_qpp, qpp, num_seg;
b8ff05a9 6410 struct port_info *pi;
c8f44aff 6411 bool highdma = false;
b8ff05a9 6412 struct adapter *adapter = NULL;
d6ce2628 6413 void __iomem *regs;
b8ff05a9
DM		 6414
6415 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
6416
6417 err = pci_request_regions(pdev, KBUILD_MODNAME);
6418 if (err) {
6419 /* Just info, some other driver may have claimed the device. */
6420 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
6421 return err;
6422 }
6423
b8ff05a9
DM		 6424 err = pci_enable_device(pdev);
6425 if (err) {
6426 dev_err(&pdev->dev, "cannot enable PCI device\n");
6427 goto out_release_regions;
6428 }
6429
d6ce2628
HS		 6430 regs = pci_ioremap_bar(pdev, 0);
6431 if (!regs) {
6432 dev_err(&pdev->dev, "cannot map device registers\n");
6433 err = -ENOMEM;
6434 goto out_disable_device;
6435 }
6436
8203b509
HS		 6437 err = t4_wait_dev_ready(regs);
6438 if (err < 0)
6439 goto out_unmap_bar0;
6440
d6ce2628 6441 /* We control everything through one PF */
0d804338 6442 func = SOURCEPF_G(readl(regs + PL_WHOAMI_A));
d6ce2628
HS		 6443 if (func != ent->driver_data) {
6444 iounmap(regs);
6445 pci_disable_device(pdev);
6446 pci_save_state(pdev); /* to restore SR-IOV later */
6447 goto sriov;
6448 }
6449
b8ff05a9 6450 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
c8f44aff 6451 highdma = true;
b8ff05a9
DM		 6452 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
6453 if (err) {
6454 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
6455 "coherent allocations\n");
d6ce2628 6456 goto out_unmap_bar0;
b8ff05a9
DM		 6457 }
6458 } else {
6459 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
6460 if (err) {
6461 dev_err(&pdev->dev, "no usable DMA configuration\n");
d6ce2628 6462 goto out_unmap_bar0;
b8ff05a9
DM		 6463 }
6464 }
6465
6466 pci_enable_pcie_error_reporting(pdev);
ef306b50 6467 enable_pcie_relaxed_ordering(pdev);
b8ff05a9
DM		 6468 pci_set_master(pdev);
6469 pci_save_state(pdev);
6470
6471 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
6472 if (!adapter) {
6473 err = -ENOMEM;
d6ce2628 6474 goto out_unmap_bar0;
b8ff05a9
DM		 6475 }
6476
29aaee65
AB		 6477 adapter->workq = create_singlethread_workqueue("cxgb4");
6478 if (!adapter->workq) {
6479 err = -ENOMEM;
6480 goto out_free_adapter;
6481 }
6482
144be3d9
GS		 6483 /* PCI device has been enabled */
6484 adapter->flags |= DEV_ENABLED;
6485
d6ce2628 6486 adapter->regs = regs;
b8ff05a9
DM		 6487 adapter->pdev = pdev;
6488 adapter->pdev_dev = &pdev->dev;
3069ee9b 6489 adapter->mbox = func;
060e0c75 6490 adapter->fn = func;
b8ff05a9
DM		 6491 adapter->msg_enable = dflt_msg_enable;
6492 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
6493
6494 spin_lock_init(&adapter->stats_lock);
6495 spin_lock_init(&adapter->tid_release_lock);
e327c225 6496 spin_lock_init(&adapter->win0_lock);
b8ff05a9
DM		 6497
6498 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
881806bc
VP		 6499 INIT_WORK(&adapter->db_full_task, process_db_full);
6500 INIT_WORK(&adapter->db_drop_task, process_db_drop);
b8ff05a9
DM		 6501
6502 err = t4_prep_adapter(adapter);
6503 if (err)
d6ce2628
HS		 6504 goto out_free_adapter;
6505
22adfe0a 6506
d14807dd 6507 if (!is_t4(adapter->params.chip)) {
f612b815
HS		 6508 s_qpp = (QUEUESPERPAGEPF0_S +
6509 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) *
6510 adapter->fn);
6511 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter,
6512 SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp);
22adfe0a
SR		 6513 num_seg = PAGE_SIZE / SEGMENT_SIZE;
6514
6515 /* Each segment size is 128B. Write coalescing is enabled only
6516 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
6517 * queue is less no of segments that can be accommodated in
6518 * a page size.
6519 */
6520 if (qpp > num_seg) {
6521 dev_err(&pdev->dev,
6522 "Incorrect number of egress queues per page\n");
6523 err = -EINVAL;
d6ce2628 6524 goto out_free_adapter;
22adfe0a
SR		 6525 }
6526 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
6527 pci_resource_len(pdev, 2));
6528 if (!adapter->bar2) {
6529 dev_err(&pdev->dev, "cannot map device bar2 region\n");
6530 err = -ENOMEM;
d6ce2628 6531 goto out_free_adapter;
22adfe0a
SR		 6532 }
6533 }
6534
636f9d37 6535 setup_memwin(adapter);
b8ff05a9 6536 err = adap_init0(adapter);
636f9d37 6537 setup_memwin_rdma(adapter);
b8ff05a9
DM		 6538 if (err)
6539 goto out_unmap_bar;
6540
6541 for_each_port(adapter, i) {
6542 struct net_device *netdev;
6543
6544 netdev = alloc_etherdev_mq(sizeof(struct port_info),
6545 MAX_ETH_QSETS);
6546 if (!netdev) {
6547 err = -ENOMEM;
6548 goto out_free_dev;
6549 }
6550
6551 SET_NETDEV_DEV(netdev, &pdev->dev);
6552
6553 adapter->port[i] = netdev;
6554 pi = netdev_priv(netdev);
6555 pi->adapter = adapter;
6556 pi->xact_addr_filt = -1;
b8ff05a9 6557 pi->port_id = i;
b8ff05a9
DM		 6558 netdev->irq = pdev->irq;
6559
2ed28baa
MM		 6560 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6561 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6562 NETIF_F_RXCSUM | NETIF_F_RXHASH |
f646968f 6563 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
c8f44aff
MM		 6564 if (highdma)
6565 netdev->hw_features |= NETIF_F_HIGHDMA;
6566 netdev->features |= netdev->hw_features;
b8ff05a9
DM		 6567 netdev->vlan_features = netdev->features & VLAN_FEAT;
6568
01789349
JP		 6569 netdev->priv_flags |= IFF_UNICAST_FLT;
6570
b8ff05a9 6571 netdev->netdev_ops = &cxgb4_netdev_ops;
688848b1
AB		 6572#ifdef CONFIG_CHELSIO_T4_DCB
6573 netdev->dcbnl_ops = &cxgb4_dcb_ops;
6574 cxgb4_dcb_state_init(netdev);
6575#endif
7ad24ea4 6576 netdev->ethtool_ops = &cxgb_ethtool_ops;
b8ff05a9
DM		 6577 }
6578
6579 pci_set_drvdata(pdev, adapter);
6580
6581 if (adapter->flags & FW_OK) {
060e0c75 6582 err = t4_port_init(adapter, func, func, 0);
b8ff05a9
DM		 6583 if (err)
6584 goto out_free_dev;
6585 }
6586
6587 /*
6588 * Configure queues and allocate tables now, they can be needed as
6589 * soon as the first register_netdev completes.
6590 */
6591 cfg_queues(adapter);
6592
6593 adapter->l2t = t4_init_l2t();
6594 if (!adapter->l2t) {
6595 /* We tolerate a lack of L2T, giving up some functionality */
6596 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6597 adapter->params.offload = 0;
6598 }
6599
6600 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6601 dev_warn(&pdev->dev, "could not allocate TID table, "
6602 "continuing\n");
6603 adapter->params.offload = 0;
6604 }
6605
f7cabcdd
DM		 6606 /* See what interrupts we'll be using */
6607 if (msi > 1 && enable_msix(adapter) == 0)
6608 adapter->flags |= USING_MSIX;
6609 else if (msi > 0 && pci_enable_msi(pdev) == 0)
6610 adapter->flags |= USING_MSI;
6611
671b0060
DM		 6612 err = init_rss(adapter);
6613 if (err)
6614 goto out_free_dev;
6615
b8ff05a9
DM		 6616 /*
6617 * The card is now ready to go. If any errors occur during device
6618 * registration we do not fail the whole card but rather proceed only
6619 * with the ports we manage to register successfully. However we must
6620 * register at least one net device.
6621 */
6622 for_each_port(adapter, i) {
a57cabe0
DM		 6623 pi = adap2pinfo(adapter, i);
6624 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6625 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6626
b8ff05a9
DM		 6627 err = register_netdev(adapter->port[i]);
6628 if (err)
b1a3c2b6 6629 break;
b1a3c2b6
DM		 6630 adapter->chan_map[pi->tx_chan] = i;
6631 print_port_info(adapter->port[i]);
b8ff05a9 6632 }
b1a3c2b6 6633 if (i == 0) {
b8ff05a9
DM		 6634 dev_err(&pdev->dev, "could not register any net devices\n");
6635 goto out_free_dev;
6636 }
b1a3c2b6
DM		 6637 if (err) {
6638 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6639 err = 0;
6403eab1 6640 }
b8ff05a9
DM		 6641
6642 if (cxgb4_debugfs_root) {
6643 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6644 cxgb4_debugfs_root);
6645 setup_debugfs(adapter);
6646 }
6647
6482aa7c
DLR		 6648 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6649 pdev->needs_freset = 1;
6650
b8ff05a9
DM		 6651 if (is_offload(adapter))
6652 attach_ulds(adapter);
6653
8e1e6059 6654sriov:
b8ff05a9 6655#ifdef CONFIG_PCI_IOV
7d6727cf 6656 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
b8ff05a9
DM		 6657 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6658 dev_info(&pdev->dev,
6659 "instantiated %u virtual functions\n",
6660 num_vf[func]);
6661#endif
6662 return 0;
6663
6664 out_free_dev:
06546391 6665 free_some_resources(adapter);
b8ff05a9 6666 out_unmap_bar:
d14807dd 6667 if (!is_t4(adapter->params.chip))
22adfe0a 6668 iounmap(adapter->bar2);
b8ff05a9 6669 out_free_adapter:
29aaee65
AB		 6670 if (adapter->workq)
6671 destroy_workqueue(adapter->workq);
6672
b8ff05a9 6673 kfree(adapter);
d6ce2628
HS		 6674 out_unmap_bar0:
6675 iounmap(regs);
b8ff05a9
DM		 6676 out_disable_device:
6677 pci_disable_pcie_error_reporting(pdev);
6678 pci_disable_device(pdev);
6679 out_release_regions:
6680 pci_release_regions(pdev);
b8ff05a9
DM		 6681 return err;
6682}
6683
91744948 6684static void remove_one(struct pci_dev *pdev)
b8ff05a9
DM		 6685{
6686 struct adapter *adapter = pci_get_drvdata(pdev);
6687
636f9d37 6688#ifdef CONFIG_PCI_IOV
b8ff05a9
DM		 6689 pci_disable_sriov(pdev);
6690
636f9d37
VP		 6691#endif
6692
b8ff05a9
DM		 6693 if (adapter) {
6694 int i;
6695
29aaee65
AB		 6696 /* Tear down per-adapter Work Queue first since it can contain
6697 * references to our adapter data structure.
6698 */
6699 destroy_workqueue(adapter->workq);
6700
b8ff05a9
DM		 6701 if (is_offload(adapter))
6702 detach_ulds(adapter);
6703
6704 for_each_port(adapter, i)
8f3a7676 6705 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
b8ff05a9
DM		 6706 unregister_netdev(adapter->port[i]);
6707
9f16dc2e 6708 debugfs_remove_recursive(adapter->debugfs_root);
b8ff05a9 6709
f2b7e78d
VP		 6710 /* If we allocated filters, free up state associated with any
6711 * valid filters ...
6712 */
6713 if (adapter->tids.ftid_tab) {
6714 struct filter_entry *f = &adapter->tids.ftid_tab[0];
dca4faeb
VP		 6715 for (i = 0; i < (adapter->tids.nftids +
6716 adapter->tids.nsftids); i++, f++)
f2b7e78d
VP		 6717 if (f->valid)
6718 clear_filter(adapter, f);
6719 }
6720
aaefae9b
DM		 6721 if (adapter->flags & FULL_INIT_DONE)
6722 cxgb_down(adapter);
b8ff05a9 6723
06546391 6724 free_some_resources(adapter);
b8ff05a9 6725 iounmap(adapter->regs);
d14807dd 6726 if (!is_t4(adapter->params.chip))
22adfe0a 6727 iounmap(adapter->bar2);
b8ff05a9 6728 pci_disable_pcie_error_reporting(pdev);
144be3d9
GS		 6729 if ((adapter->flags & DEV_ENABLED)) {
6730 pci_disable_device(pdev);
6731 adapter->flags &= ~DEV_ENABLED;
6732 }
b8ff05a9 6733 pci_release_regions(pdev);
ee9a33b2 6734 synchronize_rcu();
8b662fe7 6735 kfree(adapter);
a069ec91 6736 } else
b8ff05a9
DM		 6737 pci_release_regions(pdev);
6738}
6739
6740static struct pci_driver cxgb4_driver = {
6741 .name = KBUILD_MODNAME,
6742 .id_table = cxgb4_pci_tbl,
6743 .probe = init_one,
91744948 6744 .remove = remove_one,
687d705c 6745 .shutdown = remove_one,
204dc3c0 6746 .err_handler = &cxgb4_eeh,
b8ff05a9
DM		 6747};
6748
6749static int __init cxgb4_init_module(void)
6750{
6751 int ret;
6752
6753 /* Debugfs support is optional, just warn if this fails */
6754 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6755 if (!cxgb4_debugfs_root)
428ac43f 6756 pr_warn("could not create debugfs entry, continuing\n");
b8ff05a9
DM		 6757
6758 ret = pci_register_driver(&cxgb4_driver);
29aaee65 6759 if (ret < 0)
b8ff05a9 6760 debugfs_remove(cxgb4_debugfs_root);
01bcca68 6761
1bb60376 6762#if IS_ENABLED(CONFIG_IPV6)
01bcca68 6763 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
1bb60376 6764#endif
01bcca68 6765
b8ff05a9
DM		 6766 return ret;
6767}
6768
6769static void __exit cxgb4_cleanup_module(void)
6770{
1bb60376 6771#if IS_ENABLED(CONFIG_IPV6)
01bcca68 6772 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
1bb60376 6773#endif
b8ff05a9
DM		 6774 pci_unregister_driver(&cxgb4_driver);
6775 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
6776}
6777
6778module_init(cxgb4_init_module);
6779module_exit(cxgb4_cleanup_module);
A mirror of Linus' kernel repository