]> git.ipfire.org Git - people/arne_f/kernel.git/blame - drivers/scsi/be2iscsi/be_main.c
projects / people / arne_f / kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[SCSI] be2iscsi: Removing the iscsi_data_pdu setting.
[people/arne_f/kernel.git] / drivers / scsi / be2iscsi / be_main.c
CommitLineData
6733b39a 1/**
255fa9a3 2 * Copyright (C) 2005 - 2011 Emulex
6733b39a
JK		 3 * All rights reserved.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version 2
7 * as published by the Free Software Foundation. The full GNU General
8 * Public License is included in this distribution in the file called COPYING.
9 *
255fa9a3 10 * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
6733b39a
JK		 11 *
12 * Contact Information:
255fa9a3 13 * linux-drivers@emulex.com
6733b39a 14 *
255fa9a3
JK		 15 * Emulex
16 * 3333 Susan Street
17 * Costa Mesa, CA 92626
6733b39a 18 */
255fa9a3 19
6733b39a
JK		 20#include <linux/reboot.h>
21#include <linux/delay.h>
5a0e3ad6 22#include <linux/slab.h>
6733b39a
JK		 23#include <linux/interrupt.h>
24#include <linux/blkdev.h>
25#include <linux/pci.h>
26#include <linux/string.h>
27#include <linux/kernel.h>
28#include <linux/semaphore.h>
c7acc5b8 29#include <linux/iscsi_boot_sysfs.h>
acf3368f 30#include <linux/module.h>
ffce3e2e 31#include <linux/bsg-lib.h>
6733b39a
JK		 32
33#include <scsi/libiscsi.h>
ffce3e2e
JK		 34#include <scsi/scsi_bsg_iscsi.h>
35#include <scsi/scsi_netlink.h>
6733b39a
JK		 36#include <scsi/scsi_transport_iscsi.h>
37#include <scsi/scsi_transport.h>
38#include <scsi/scsi_cmnd.h>
39#include <scsi/scsi_device.h>
40#include <scsi/scsi_host.h>
41#include <scsi/scsi.h>
42#include "be_main.h"
43#include "be_iscsi.h"
44#include "be_mgmt.h"
45
46static unsigned int be_iopoll_budget = 10;
47static unsigned int be_max_phys_size = 64;
bfead3b2 48static unsigned int enable_msix = 1;
e9b91193
JK		 49static unsigned int gcrashmode = 0;
50static unsigned int num_hba = 0;
6733b39a
JK		 51
52MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
53MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
76d15dbd 54MODULE_VERSION(BUILD_STR);
2f635883 55MODULE_AUTHOR("Emulex Corporation");
6733b39a
JK		 56MODULE_LICENSE("GPL");
57module_param(be_iopoll_budget, int, 0);
58module_param(enable_msix, int, 0);
59module_param(be_max_phys_size, uint, S_IRUGO);
60MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
61 "contiguous memory that can be allocated."
62 "Range is 16 - 128");
63
64static int beiscsi_slave_configure(struct scsi_device *sdev)
65{
66 blk_queue_max_segment_size(sdev->request_queue, 65536);
67 return 0;
68}
69
4183122d
JK		 70static int beiscsi_eh_abort(struct scsi_cmnd *sc)
71{
72 struct iscsi_cls_session *cls_session;
73 struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
74 struct beiscsi_io_task *aborted_io_task;
75 struct iscsi_conn *conn;
76 struct beiscsi_conn *beiscsi_conn;
77 struct beiscsi_hba *phba;
78 struct iscsi_session *session;
79 struct invalidate_command_table *inv_tbl;
3cbb7a74 80 struct be_dma_mem nonemb_cmd;
4183122d
JK		 81 unsigned int cid, tag, num_invalidate;
82
83 cls_session = starget_to_session(scsi_target(sc->device));
84 session = cls_session->dd_data;
85
86 spin_lock_bh(&session->lock);
87 if (!aborted_task || !aborted_task->sc) {
88 /* we raced */
89 spin_unlock_bh(&session->lock);
90 return SUCCESS;
91 }
92
93 aborted_io_task = aborted_task->dd_data;
94 if (!aborted_io_task->scsi_cmnd) {
95 /* raced or invalid command */
96 spin_unlock_bh(&session->lock);
97 return SUCCESS;
98 }
99 spin_unlock_bh(&session->lock);
100 conn = aborted_task->conn;
101 beiscsi_conn = conn->dd_data;
102 phba = beiscsi_conn->phba;
103
104 /* invalidate iocb */
105 cid = beiscsi_conn->beiscsi_conn_cid;
106 inv_tbl = phba->inv_tbl;
107 memset(inv_tbl, 0x0, sizeof(*inv_tbl));
108 inv_tbl->cid = cid;
109 inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
110 num_invalidate = 1;
3cbb7a74
JK		 111 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
112 sizeof(struct invalidate_commands_params_in),
113 &nonemb_cmd.dma);
114 if (nonemb_cmd.va == NULL) {
115 SE_DEBUG(DBG_LVL_1,
116 "Failed to allocate memory for"
117 "mgmt_invalidate_icds\n");
118 return FAILED;
119 }
120 nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
121
122 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
123 cid, &nonemb_cmd);
4183122d
JK		 124 if (!tag) {
125 shost_printk(KERN_WARNING, phba->shost,
126 "mgmt_invalidate_icds could not be"
127 " submitted\n");
3cbb7a74
JK		 128 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
129 nonemb_cmd.va, nonemb_cmd.dma);
130
4183122d
JK		 131 return FAILED;
132 } else {
133 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
134 phba->ctrl.mcc_numtag[tag]);
135 free_mcc_tag(&phba->ctrl, tag);
136 }
3cbb7a74
JK		 137 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
138 nonemb_cmd.va, nonemb_cmd.dma);
4183122d
JK		 139 return iscsi_eh_abort(sc);
140}
141
142static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
143{
144 struct iscsi_task *abrt_task;
145 struct beiscsi_io_task *abrt_io_task;
146 struct iscsi_conn *conn;
147 struct beiscsi_conn *beiscsi_conn;
148 struct beiscsi_hba *phba;
149 struct iscsi_session *session;
150 struct iscsi_cls_session *cls_session;
151 struct invalidate_command_table *inv_tbl;
3cbb7a74 152 struct be_dma_mem nonemb_cmd;
4183122d 153 unsigned int cid, tag, i, num_invalidate;
4183122d
JK		 154
155 /* invalidate iocbs */
156 cls_session = starget_to_session(scsi_target(sc->device));
157 session = cls_session->dd_data;
158 spin_lock_bh(&session->lock);
db7f7709
JK		 159 if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {
160 spin_unlock_bh(&session->lock);
161 return FAILED;
162 }
4183122d
JK		 163 conn = session->leadconn;
164 beiscsi_conn = conn->dd_data;
165 phba = beiscsi_conn->phba;
166 cid = beiscsi_conn->beiscsi_conn_cid;
167 inv_tbl = phba->inv_tbl;
168 memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
169 num_invalidate = 0;
170 for (i = 0; i < conn->session->cmds_max; i++) {
171 abrt_task = conn->session->cmds[i];
172 abrt_io_task = abrt_task->dd_data;
173 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
174 continue;
175
176 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
177 continue;
178
179 inv_tbl->cid = cid;
180 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
181 num_invalidate++;
182 inv_tbl++;
183 }
184 spin_unlock_bh(&session->lock);
185 inv_tbl = phba->inv_tbl;
186
3cbb7a74
JK		 187 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
188 sizeof(struct invalidate_commands_params_in),
189 &nonemb_cmd.dma);
190 if (nonemb_cmd.va == NULL) {
191 SE_DEBUG(DBG_LVL_1,
192 "Failed to allocate memory for"
193 "mgmt_invalidate_icds\n");
194 return FAILED;
195 }
196 nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
197 memset(nonemb_cmd.va, 0, nonemb_cmd.size);
198 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
199 cid, &nonemb_cmd);
4183122d
JK		 200 if (!tag) {
201 shost_printk(KERN_WARNING, phba->shost,
202 "mgmt_invalidate_icds could not be"
203 " submitted\n");
3cbb7a74
JK		 204 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
205 nonemb_cmd.va, nonemb_cmd.dma);
4183122d
JK		 206 return FAILED;
207 } else {
208 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
209 phba->ctrl.mcc_numtag[tag]);
210 free_mcc_tag(&phba->ctrl, tag);
211 }
3cbb7a74
JK		 212 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
213 nonemb_cmd.va, nonemb_cmd.dma);
4183122d 214 return iscsi_eh_device_reset(sc);
4183122d
JK		 215}
216
c7acc5b8
JK		 217static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
218{
219 struct beiscsi_hba *phba = data;
f457a46f
MC		 220 struct mgmt_session_info *boot_sess = &phba->boot_sess;
221 struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
c7acc5b8
JK		 222 char *str = buf;
223 int rc;
224
225 switch (type) {
226 case ISCSI_BOOT_TGT_NAME:
227 rc = sprintf(buf, "%.*s\n",
f457a46f
MC		 228 (int)strlen(boot_sess->target_name),
229 (char *)&boot_sess->target_name);
c7acc5b8
JK		 230 break;
231 case ISCSI_BOOT_TGT_IP_ADDR:
f457a46f 232 if (boot_conn->dest_ipaddr.ip_type == 0x1)
c7acc5b8 233 rc = sprintf(buf, "%pI4\n",
0e43895e 234 (char *)&boot_conn->dest_ipaddr.addr);
c7acc5b8
JK		 235 else
236 rc = sprintf(str, "%pI6\n",
0e43895e 237 (char *)&boot_conn->dest_ipaddr.addr);
c7acc5b8
JK		 238 break;
239 case ISCSI_BOOT_TGT_PORT:
f457a46f 240 rc = sprintf(str, "%d\n", boot_conn->dest_port);
c7acc5b8
JK		 241 break;
242
243 case ISCSI_BOOT_TGT_CHAP_NAME:
244 rc = sprintf(str, "%.*s\n",
f457a46f
MC		 245 boot_conn->negotiated_login_options.auth_data.chap.
246 target_chap_name_length,
247 (char *)&boot_conn->negotiated_login_options.
248 auth_data.chap.target_chap_name);
c7acc5b8
JK		 249 break;
250 case ISCSI_BOOT_TGT_CHAP_SECRET:
251 rc = sprintf(str, "%.*s\n",
f457a46f
MC		 252 boot_conn->negotiated_login_options.auth_data.chap.
253 target_secret_length,
254 (char *)&boot_conn->negotiated_login_options.
255 auth_data.chap.target_secret);
c7acc5b8
JK		 256 break;
257 case ISCSI_BOOT_TGT_REV_CHAP_NAME:
258 rc = sprintf(str, "%.*s\n",
f457a46f
MC		 259 boot_conn->negotiated_login_options.auth_data.chap.
260 intr_chap_name_length,
261 (char *)&boot_conn->negotiated_login_options.
262 auth_data.chap.intr_chap_name);
c7acc5b8
JK		 263 break;
264 case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
f457a46f
MC		 265 rc = sprintf(str, "%.*s\n",
266 boot_conn->negotiated_login_options.auth_data.chap.
267 intr_secret_length,
268 (char *)&boot_conn->negotiated_login_options.
269 auth_data.chap.intr_secret);
c7acc5b8
JK		 270 break;
271 case ISCSI_BOOT_TGT_FLAGS:
f457a46f 272 rc = sprintf(str, "2\n");
c7acc5b8
JK		 273 break;
274 case ISCSI_BOOT_TGT_NIC_ASSOC:
f457a46f 275 rc = sprintf(str, "0\n");
c7acc5b8
JK		 276 break;
277 default:
278 rc = -ENOSYS;
279 break;
280 }
281 return rc;
282}
283
284static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
285{
286 struct beiscsi_hba *phba = data;
287 char *str = buf;
288 int rc;
289
290 switch (type) {
291 case ISCSI_BOOT_INI_INITIATOR_NAME:
292 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
293 break;
294 default:
295 rc = -ENOSYS;
296 break;
297 }
298 return rc;
299}
300
301static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
302{
303 struct beiscsi_hba *phba = data;
304 char *str = buf;
305 int rc;
306
307 switch (type) {
308 case ISCSI_BOOT_ETH_FLAGS:
f457a46f 309 rc = sprintf(str, "2\n");
c7acc5b8
JK		 310 break;
311 case ISCSI_BOOT_ETH_INDEX:
f457a46f 312 rc = sprintf(str, "0\n");
c7acc5b8
JK		 313 break;
314 case ISCSI_BOOT_ETH_MAC:
0e43895e
MC		 315 rc = beiscsi_get_macaddr(str, phba);
316 break;
c7acc5b8
JK		 317 default:
318 rc = -ENOSYS;
319 break;
320 }
321 return rc;
322}
323
324
587a1f16 325static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
c7acc5b8 326{
587a1f16 327 umode_t rc;
c7acc5b8
JK		 328
329 switch (type) {
330 case ISCSI_BOOT_TGT_NAME:
331 case ISCSI_BOOT_TGT_IP_ADDR:
332 case ISCSI_BOOT_TGT_PORT:
333 case ISCSI_BOOT_TGT_CHAP_NAME:
334 case ISCSI_BOOT_TGT_CHAP_SECRET:
335 case ISCSI_BOOT_TGT_REV_CHAP_NAME:
336 case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
337 case ISCSI_BOOT_TGT_NIC_ASSOC:
338 case ISCSI_BOOT_TGT_FLAGS:
339 rc = S_IRUGO;
340 break;
341 default:
342 rc = 0;
343 break;
344 }
345 return rc;
346}
347
587a1f16 348static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
c7acc5b8 349{
587a1f16 350 umode_t rc;
c7acc5b8
JK		 351
352 switch (type) {
353 case ISCSI_BOOT_INI_INITIATOR_NAME:
354 rc = S_IRUGO;
355 break;
356 default:
357 rc = 0;
358 break;
359 }
360 return rc;
361}
362
363
587a1f16 364static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
c7acc5b8 365{
587a1f16 366 umode_t rc;
c7acc5b8
JK		 367
368 switch (type) {
369 case ISCSI_BOOT_ETH_FLAGS:
370 case ISCSI_BOOT_ETH_MAC:
371 case ISCSI_BOOT_ETH_INDEX:
372 rc = S_IRUGO;
373 break;
374 default:
375 rc = 0;
376 break;
377 }
378 return rc;
379}
380
bfead3b2
JK		 381/*------------------- PCI Driver operations and data ----------------- */
382static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
383 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
f98c96b0 384 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
bfead3b2
JK		 385 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
386 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
387 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
bfead3b2
JK		 388 { 0 }
389};
390MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
391
6733b39a
JK		 392static struct scsi_host_template beiscsi_sht = {
393 .module = THIS_MODULE,
2f635883 394 .name = "Emulex 10Gbe open-iscsi Initiator Driver",
6733b39a
JK		 395 .proc_name = DRV_NAME,
396 .queuecommand = iscsi_queuecommand,
6733b39a
JK		 397 .change_queue_depth = iscsi_change_queue_depth,
398 .slave_configure = beiscsi_slave_configure,
399 .target_alloc = iscsi_target_alloc,
4183122d
JK		 400 .eh_abort_handler = beiscsi_eh_abort,
401 .eh_device_reset_handler = beiscsi_eh_device_reset,
309ce156 402 .eh_target_reset_handler = iscsi_eh_session_reset,
6733b39a
JK		 403 .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
404 .can_queue = BE2_IO_DEPTH,
405 .this_id = -1,
406 .max_sectors = BEISCSI_MAX_SECTORS,
407 .cmd_per_lun = BEISCSI_CMD_PER_LUN,
408 .use_clustering = ENABLE_CLUSTERING,
ffce3e2e
JK		 409 .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID,
410
6733b39a 411};
6733b39a 412
bfead3b2 413static struct scsi_transport_template *beiscsi_scsi_transport;
6733b39a
JK		 414
415static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
416{
417 struct beiscsi_hba *phba;
418 struct Scsi_Host *shost;
419
420 shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
421 if (!shost) {
422 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
457ff3b7 423 "iscsi_host_alloc failed\n");
6733b39a
JK		 424 return NULL;
425 }
426 shost->dma_boundary = pcidev->dma_mask;
427 shost->max_id = BE2_MAX_SESSIONS;
428 shost->max_channel = 0;
429 shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
430 shost->max_lun = BEISCSI_NUM_MAX_LUN;
431 shost->transportt = beiscsi_scsi_transport;
6733b39a
JK		 432 phba = iscsi_host_priv(shost);
433 memset(phba, 0, sizeof(*phba));
434 phba->shost = shost;
435 phba->pcidev = pci_dev_get(pcidev);
2807afb7 436 pci_set_drvdata(pcidev, phba);
0e43895e 437 phba->interface_handle = 0xFFFFFFFF;
6733b39a
JK		 438
439 if (iscsi_host_add(shost, &phba->pcidev->dev))
440 goto free_devices;
c7acc5b8 441
6733b39a
JK		 442 return phba;
443
444free_devices:
445 pci_dev_put(phba->pcidev);
446 iscsi_host_free(phba->shost);
447 return NULL;
448}
449
450static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
451{
452 if (phba->csr_va) {
453 iounmap(phba->csr_va);
454 phba->csr_va = NULL;
455 }
456 if (phba->db_va) {
457 iounmap(phba->db_va);
458 phba->db_va = NULL;
459 }
460 if (phba->pci_va) {
461 iounmap(phba->pci_va);
462 phba->pci_va = NULL;
463 }
464}
465
466static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
467 struct pci_dev *pcidev)
468{
469 u8 __iomem *addr;
f98c96b0 470 int pcicfg_reg;
6733b39a
JK		 471
472 addr = ioremap_nocache(pci_resource_start(pcidev, 2),
473 pci_resource_len(pcidev, 2));
474 if (addr == NULL)
475 return -ENOMEM;
476 phba->ctrl.csr = addr;
477 phba->csr_va = addr;
478 phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
479
480 addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
481 if (addr == NULL)
482 goto pci_map_err;
483 phba->ctrl.db = addr;
484 phba->db_va = addr;
485 phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4);
486
f98c96b0
JK		 487 if (phba->generation == BE_GEN2)
488 pcicfg_reg = 1;
489 else
490 pcicfg_reg = 0;
491
492 addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
493 pci_resource_len(pcidev, pcicfg_reg));
494
6733b39a
JK		 495 if (addr == NULL)
496 goto pci_map_err;
497 phba->ctrl.pcicfg = addr;
498 phba->pci_va = addr;
f98c96b0 499 phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
6733b39a
JK		 500 return 0;
501
502pci_map_err:
503 beiscsi_unmap_pci_function(phba);
504 return -ENOMEM;
505}
506
507static int beiscsi_enable_pci(struct pci_dev *pcidev)
508{
509 int ret;
510
511 ret = pci_enable_device(pcidev);
512 if (ret) {
513 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
514 "failed. Returning -ENODEV\n");
515 return ret;
516 }
517
bfead3b2 518 pci_set_master(pcidev);
6733b39a
JK		 519 if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
520 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
521 if (ret) {
522 dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
523 pci_disable_device(pcidev);
524 return ret;
525 }
526 }
527 return 0;
528}
529
530static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
531{
532 struct be_ctrl_info *ctrl = &phba->ctrl;
533 struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
534 struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
535 int status = 0;
536
537 ctrl->pdev = pdev;
538 status = beiscsi_map_pci_bars(phba, pdev);
539 if (status)
540 return status;
6733b39a
JK		 541 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
542 mbox_mem_alloc->va = pci_alloc_consistent(pdev,
543 mbox_mem_alloc->size,
544 &mbox_mem_alloc->dma);
545 if (!mbox_mem_alloc->va) {
546 beiscsi_unmap_pci_function(phba);
a49e06d5 547 return -ENOMEM;
6733b39a
JK		 548 }
549
550 mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
551 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
552 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
553 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
554 spin_lock_init(&ctrl->mbox_lock);
bfead3b2
JK		 555 spin_lock_init(&phba->ctrl.mcc_lock);
556 spin_lock_init(&phba->ctrl.mcc_cq_lock);
557
6733b39a
JK		 558 return status;
559}
560
561static void beiscsi_get_params(struct beiscsi_hba *phba)
562{
7da50879
JK		 563 phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
564 - (phba->fw_config.iscsi_cid_count
565 + BE2_TMFS
566 + BE2_NOPOUT_REQ));
567 phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
ed58ea2a 568 phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
6eab04a8 569 phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
6733b39a
JK		 570 phba->params.num_sge_per_io = BE2_SGE;
571 phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
572 phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
573 phba->params.eq_timer = 64;
574 phba->params.num_eq_entries =
7da50879
JK		 575 (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
576 + BE2_TMFS) / 512) + 1) * 512;
6733b39a
JK		 577 phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
578 ? 1024 : phba->params.num_eq_entries;
457ff3b7 579 SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d\n",
7da50879 580 phba->params.num_eq_entries);
6733b39a 581 phba->params.num_cq_entries =
7da50879
JK		 582 (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
583 + BE2_TMFS) / 512) + 1) * 512;
6733b39a
JK		 584 phba->params.wrbs_per_cxn = 256;
585}
586
587static void hwi_ring_eq_db(struct beiscsi_hba *phba,
588 unsigned int id, unsigned int clr_interrupt,
589 unsigned int num_processed,
590 unsigned char rearm, unsigned char event)
591{
592 u32 val = 0;
593 val |= id & DB_EQ_RING_ID_MASK;
594 if (rearm)
595 val |= 1 << DB_EQ_REARM_SHIFT;
596 if (clr_interrupt)
597 val |= 1 << DB_EQ_CLR_SHIFT;
598 if (event)
599 val |= 1 << DB_EQ_EVNT_SHIFT;
600 val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
601 iowrite32(val, phba->db_va + DB_EQ_OFFSET);
602}
603
bfead3b2
JK		 604/**
605 * be_isr_mcc - The isr routine of the driver.
606 * @irq: Not used
607 * @dev_id: Pointer to host adapter structure
608 */
609static irqreturn_t be_isr_mcc(int irq, void *dev_id)
610{
611 struct beiscsi_hba *phba;
612 struct be_eq_entry *eqe = NULL;
613 struct be_queue_info *eq;
614 struct be_queue_info *mcc;
615 unsigned int num_eq_processed;
616 struct be_eq_obj *pbe_eq;
617 unsigned long flags;
618
619 pbe_eq = dev_id;
620 eq = &pbe_eq->q;
621 phba = pbe_eq->phba;
622 mcc = &phba->ctrl.mcc_obj.cq;
623 eqe = queue_tail_node(eq);
624 if (!eqe)
625 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
626
627 num_eq_processed = 0;
628
629 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
630 & EQE_VALID_MASK) {
631 if (((eqe->dw[offsetof(struct amap_eq_entry,
632 resource_id) / 32] &
633 EQE_RESID_MASK) >> 16) == mcc->id) {
634 spin_lock_irqsave(&phba->isr_lock, flags);
635 phba->todo_mcc_cq = 1;
636 spin_unlock_irqrestore(&phba->isr_lock, flags);
637 }
638 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
639 queue_tail_inc(eq);
640 eqe = queue_tail_node(eq);
641 num_eq_processed++;
642 }
643 if (phba->todo_mcc_cq)
644 queue_work(phba->wq, &phba->work_cqs);
645 if (num_eq_processed)
646 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
647
648 return IRQ_HANDLED;
649}
650
651/**
652 * be_isr_msix - The isr routine of the driver.
653 * @irq: Not used
654 * @dev_id: Pointer to host adapter structure
655 */
656static irqreturn_t be_isr_msix(int irq, void *dev_id)
657{
658 struct beiscsi_hba *phba;
659 struct be_eq_entry *eqe = NULL;
660 struct be_queue_info *eq;
661 struct be_queue_info *cq;
662 unsigned int num_eq_processed;
663 struct be_eq_obj *pbe_eq;
664 unsigned long flags;
665
666 pbe_eq = dev_id;
667 eq = &pbe_eq->q;
668 cq = pbe_eq->cq;
669 eqe = queue_tail_node(eq);
670 if (!eqe)
671 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
672
673 phba = pbe_eq->phba;
674 num_eq_processed = 0;
675 if (blk_iopoll_enabled) {
676 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
677 & EQE_VALID_MASK) {
678 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
679 blk_iopoll_sched(&pbe_eq->iopoll);
680
681 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
682 queue_tail_inc(eq);
683 eqe = queue_tail_node(eq);
684 num_eq_processed++;
685 }
686 if (num_eq_processed)
687 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
688
689 return IRQ_HANDLED;
690 } else {
691 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
692 & EQE_VALID_MASK) {
693 spin_lock_irqsave(&phba->isr_lock, flags);
694 phba->todo_cq = 1;
695 spin_unlock_irqrestore(&phba->isr_lock, flags);
696 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
697 queue_tail_inc(eq);
698 eqe = queue_tail_node(eq);
699 num_eq_processed++;
700 }
701 if (phba->todo_cq)
702 queue_work(phba->wq, &phba->work_cqs);
703
704 if (num_eq_processed)
705 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
706
707 return IRQ_HANDLED;
708 }
709}
710
6733b39a
JK		 711/**
712 * be_isr - The isr routine of the driver.
713 * @irq: Not used
714 * @dev_id: Pointer to host adapter structure
715 */
716static irqreturn_t be_isr(int irq, void *dev_id)
717{
718 struct beiscsi_hba *phba;
719 struct hwi_controller *phwi_ctrlr;
720 struct hwi_context_memory *phwi_context;
721 struct be_eq_entry *eqe = NULL;
722 struct be_queue_info *eq;
723 struct be_queue_info *cq;
bfead3b2 724 struct be_queue_info *mcc;
6733b39a 725 unsigned long flags, index;
bfead3b2 726 unsigned int num_mcceq_processed, num_ioeq_processed;
6733b39a 727 struct be_ctrl_info *ctrl;
bfead3b2 728 struct be_eq_obj *pbe_eq;
6733b39a
JK		 729 int isr;
730
731 phba = dev_id;
6eab04a8 732 ctrl = &phba->ctrl;
bfead3b2
JK		 733 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
734 (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
735 if (!isr)
736 return IRQ_NONE;
6733b39a
JK		 737
738 phwi_ctrlr = phba->phwi_ctrlr;
739 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 740 pbe_eq = &phwi_context->be_eq[0];
741
742 eq = &phwi_context->be_eq[0].q;
743 mcc = &phba->ctrl.mcc_obj.cq;
6733b39a
JK		 744 index = 0;
745 eqe = queue_tail_node(eq);
746 if (!eqe)
747 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
748
bfead3b2
JK		 749 num_ioeq_processed = 0;
750 num_mcceq_processed = 0;
6733b39a
JK		 751 if (blk_iopoll_enabled) {
752 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
753 & EQE_VALID_MASK) {
bfead3b2
JK		 754 if (((eqe->dw[offsetof(struct amap_eq_entry,
755 resource_id) / 32] &
756 EQE_RESID_MASK) >> 16) == mcc->id) {
757 spin_lock_irqsave(&phba->isr_lock, flags);
758 phba->todo_mcc_cq = 1;
759 spin_unlock_irqrestore(&phba->isr_lock, flags);
760 num_mcceq_processed++;
761 } else {
762 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
763 blk_iopoll_sched(&pbe_eq->iopoll);
764 num_ioeq_processed++;
765 }
6733b39a
JK		 766 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
767 queue_tail_inc(eq);
768 eqe = queue_tail_node(eq);
6733b39a 769 }
bfead3b2
JK		 770 if (num_ioeq_processed || num_mcceq_processed) {
771 if (phba->todo_mcc_cq)
772 queue_work(phba->wq, &phba->work_cqs);
773
756d29c8 774 if ((num_mcceq_processed) && (!num_ioeq_processed))
bfead3b2
JK		 775 hwi_ring_eq_db(phba, eq->id, 0,
776 (num_ioeq_processed +
777 num_mcceq_processed) , 1, 1);
778 else
779 hwi_ring_eq_db(phba, eq->id, 0,
780 (num_ioeq_processed +
781 num_mcceq_processed), 0, 1);
782
6733b39a
JK		 783 return IRQ_HANDLED;
784 } else
785 return IRQ_NONE;
786 } else {
bfead3b2 787 cq = &phwi_context->be_cq[0];
6733b39a
JK		 788 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
789 & EQE_VALID_MASK) {
790
791 if (((eqe->dw[offsetof(struct amap_eq_entry,
792 resource_id) / 32] &
793 EQE_RESID_MASK) >> 16) != cq->id) {
794 spin_lock_irqsave(&phba->isr_lock, flags);
795 phba->todo_mcc_cq = 1;
796 spin_unlock_irqrestore(&phba->isr_lock, flags);
797 } else {
798 spin_lock_irqsave(&phba->isr_lock, flags);
799 phba->todo_cq = 1;
800 spin_unlock_irqrestore(&phba->isr_lock, flags);
801 }
802 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
803 queue_tail_inc(eq);
804 eqe = queue_tail_node(eq);
bfead3b2 805 num_ioeq_processed++;
6733b39a
JK		 806 }
807 if (phba->todo_cq || phba->todo_mcc_cq)
808 queue_work(phba->wq, &phba->work_cqs);
809
bfead3b2
JK		 810 if (num_ioeq_processed) {
811 hwi_ring_eq_db(phba, eq->id, 0,
812 num_ioeq_processed, 1, 1);
6733b39a
JK		 813 return IRQ_HANDLED;
814 } else
815 return IRQ_NONE;
816 }
817}
818
819static int beiscsi_init_irqs(struct beiscsi_hba *phba)
820{
821 struct pci_dev *pcidev = phba->pcidev;
bfead3b2
JK		 822 struct hwi_controller *phwi_ctrlr;
823 struct hwi_context_memory *phwi_context;
4f5af07e 824 int ret, msix_vec, i, j;
6733b39a 825
bfead3b2
JK		 826 phwi_ctrlr = phba->phwi_ctrlr;
827 phwi_context = phwi_ctrlr->phwi_ctxt;
828
829 if (phba->msix_enabled) {
830 for (i = 0; i < phba->num_cpus; i++) {
8fcfb210
JK		 831 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
832 GFP_KERNEL);
833 if (!phba->msi_name[i]) {
834 ret = -ENOMEM;
835 goto free_msix_irqs;
836 }
837
838 sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
839 phba->shost->host_no, i);
bfead3b2 840 msix_vec = phba->msix_entries[i].vector;
8fcfb210
JK		 841 ret = request_irq(msix_vec, be_isr_msix, 0,
842 phba->msi_name[i],
bfead3b2 843 &phwi_context->be_eq[i]);
4f5af07e
JK		 844 if (ret) {
845 shost_printk(KERN_ERR, phba->shost,
846 "beiscsi_init_irqs-Failed to"
847 "register msix for i = %d\n", i);
8fcfb210 848 kfree(phba->msi_name[i]);
4f5af07e
JK		 849 goto free_msix_irqs;
850 }
bfead3b2 851 }
8fcfb210
JK		 852 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
853 if (!phba->msi_name[i]) {
854 ret = -ENOMEM;
855 goto free_msix_irqs;
856 }
857 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
858 phba->shost->host_no);
bfead3b2 859 msix_vec = phba->msix_entries[i].vector;
8fcfb210 860 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
bfead3b2 861 &phwi_context->be_eq[i]);
4f5af07e
JK		 862 if (ret) {
863 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
864 "Failed to register beiscsi_msix_mcc\n");
8fcfb210 865 kfree(phba->msi_name[i]);
4f5af07e
JK		 866 goto free_msix_irqs;
867 }
868
bfead3b2
JK		 869 } else {
870 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
871 "beiscsi", phba);
872 if (ret) {
873 shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
874 "Failed to register irq\\n");
875 return ret;
876 }
6733b39a
JK		 877 }
878 return 0;
4f5af07e 879free_msix_irqs:
8fcfb210
JK		 880 for (j = i - 1; j >= 0; j--) {
881 kfree(phba->msi_name[j]);
882 msix_vec = phba->msix_entries[j].vector;
4f5af07e 883 free_irq(msix_vec, &phwi_context->be_eq[j]);
8fcfb210 884 }
4f5af07e 885 return ret;
6733b39a
JK		 886}
887
888static void hwi_ring_cq_db(struct beiscsi_hba *phba,
889 unsigned int id, unsigned int num_processed,
890 unsigned char rearm, unsigned char event)
891{
892 u32 val = 0;
893 val |= id & DB_CQ_RING_ID_MASK;
894 if (rearm)
895 val |= 1 << DB_CQ_REARM_SHIFT;
896 val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
897 iowrite32(val, phba->db_va + DB_CQ_OFFSET);
898}
899
6733b39a
JK		 900static unsigned int
901beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
902 struct beiscsi_hba *phba,
903 unsigned short cid,
904 struct pdu_base *ppdu,
905 unsigned long pdu_len,
906 void *pbuffer, unsigned long buf_len)
907{
908 struct iscsi_conn *conn = beiscsi_conn->conn;
909 struct iscsi_session *session = conn->session;
bfead3b2
JK		 910 struct iscsi_task *task;
911 struct beiscsi_io_task *io_task;
912 struct iscsi_hdr *login_hdr;
6733b39a
JK		 913
914 switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
915 PDUBASE_OPCODE_MASK) {
916 case ISCSI_OP_NOOP_IN:
917 pbuffer = NULL;
918 buf_len = 0;
919 break;
920 case ISCSI_OP_ASYNC_EVENT:
921 break;
922 case ISCSI_OP_REJECT:
923 WARN_ON(!pbuffer);
924 WARN_ON(!(buf_len == 48));
925 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
926 break;
927 case ISCSI_OP_LOGIN_RSP:
7bd6e25c 928 case ISCSI_OP_TEXT_RSP:
bfead3b2
JK		 929 task = conn->login_task;
930 io_task = task->dd_data;
931 login_hdr = (struct iscsi_hdr *)ppdu;
932 login_hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 933 break;
934 default:
935 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 936 "Unrecognized opcode 0x%x in async msg\n",
6733b39a
JK		 937 (ppdu->
938 dw[offsetof(struct amap_pdu_base, opcode) / 32]
939 & PDUBASE_OPCODE_MASK));
940 return 1;
941 }
942
943 spin_lock_bh(&session->lock);
944 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
945 spin_unlock_bh(&session->lock);
946 return 0;
947}
948
949static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
950{
951 struct sgl_handle *psgl_handle;
952
953 if (phba->io_sgl_hndl_avbl) {
954 SE_DEBUG(DBG_LVL_8,
457ff3b7 955 "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n",
6733b39a
JK		 956 phba->io_sgl_alloc_index);
957 psgl_handle = phba->io_sgl_hndl_base[phba->
958 io_sgl_alloc_index];
959 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
960 phba->io_sgl_hndl_avbl--;
bfead3b2
JK		 961 if (phba->io_sgl_alloc_index == (phba->params.
962 ios_per_ctrl - 1))
6733b39a
JK		 963 phba->io_sgl_alloc_index = 0;
964 else
965 phba->io_sgl_alloc_index++;
966 } else
967 psgl_handle = NULL;
968 return psgl_handle;
969}
970
971static void
972free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
973{
457ff3b7 974 SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d\n",
6733b39a
JK		 975 phba->io_sgl_free_index);
976 if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
977 /*
978 * this can happen if clean_task is called on a task that
979 * failed in xmit_task or alloc_pdu.
980 */
981 SE_DEBUG(DBG_LVL_8,
982 "Double Free in IO SGL io_sgl_free_index=%d,"
457ff3b7 983 "value there=%p\n", phba->io_sgl_free_index,
6733b39a
JK		 984 phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
985 return;
986 }
987 phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
988 phba->io_sgl_hndl_avbl++;
989 if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
990 phba->io_sgl_free_index = 0;
991 else
992 phba->io_sgl_free_index++;
993}
994
995/**
996 * alloc_wrb_handle - To allocate a wrb handle
997 * @phba: The hba pointer
998 * @cid: The cid to use for allocation
6733b39a
JK		 999 *
1000 * This happens under session_lock until submission to chip
1001 */
d5431488 1002struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
6733b39a
JK		 1003{
1004 struct hwi_wrb_context *pwrb_context;
1005 struct hwi_controller *phwi_ctrlr;
d5431488 1006 struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
6733b39a
JK		 1007
1008 phwi_ctrlr = phba->phwi_ctrlr;
1009 pwrb_context = &phwi_ctrlr->wrb_context[cid];
d5431488 1010 if (pwrb_context->wrb_handles_available >= 2) {
bfead3b2
JK		 1011 pwrb_handle = pwrb_context->pwrb_handle_base[
1012 pwrb_context->alloc_index];
1013 pwrb_context->wrb_handles_available--;
bfead3b2
JK		 1014 if (pwrb_context->alloc_index ==
1015 (phba->params.wrbs_per_cxn - 1))
1016 pwrb_context->alloc_index = 0;
1017 else
1018 pwrb_context->alloc_index++;
d5431488
JK		 1019 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1020 pwrb_context->alloc_index];
1021 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
bfead3b2
JK		 1022 } else
1023 pwrb_handle = NULL;
6733b39a
JK		 1024 return pwrb_handle;
1025}
1026
1027/**
1028 * free_wrb_handle - To free the wrb handle back to pool
1029 * @phba: The hba pointer
1030 * @pwrb_context: The context to free from
1031 * @pwrb_handle: The wrb_handle to free
1032 *
1033 * This happens under session_lock until submission to chip
1034 */
1035static void
1036free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1037 struct wrb_handle *pwrb_handle)
1038{
32951dd8 1039 pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
bfead3b2
JK		 1040 pwrb_context->wrb_handles_available++;
1041 if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1042 pwrb_context->free_index = 0;
1043 else
1044 pwrb_context->free_index++;
1045
6733b39a 1046 SE_DEBUG(DBG_LVL_8,
bfead3b2 1047 "FREE WRB: pwrb_handle=%p free_index=0x%x"
457ff3b7 1048 "wrb_handles_available=%d\n",
6733b39a 1049 pwrb_handle, pwrb_context->free_index,
bfead3b2 1050 pwrb_context->wrb_handles_available);
6733b39a
JK		 1051}
1052
1053static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1054{
1055 struct sgl_handle *psgl_handle;
1056
1057 if (phba->eh_sgl_hndl_avbl) {
1058 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1059 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
457ff3b7 1060 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x\n",
6733b39a
JK		 1061 phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
1062 phba->eh_sgl_hndl_avbl--;
1063 if (phba->eh_sgl_alloc_index ==
1064 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1065 1))
1066 phba->eh_sgl_alloc_index = 0;
1067 else
1068 phba->eh_sgl_alloc_index++;
1069 } else
1070 psgl_handle = NULL;
1071 return psgl_handle;
1072}
1073
1074void
1075free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1076{
1077
457ff3b7 1078 SE_DEBUG(DBG_LVL_8, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d\n",
bfead3b2 1079 phba->eh_sgl_free_index);
6733b39a
JK		 1080 if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1081 /*
1082 * this can happen if clean_task is called on a task that
1083 * failed in xmit_task or alloc_pdu.
1084 */
1085 SE_DEBUG(DBG_LVL_8,
457ff3b7 1086 "Double Free in eh SGL ,eh_sgl_free_index=%d\n",
6733b39a
JK		 1087 phba->eh_sgl_free_index);
1088 return;
1089 }
1090 phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1091 phba->eh_sgl_hndl_avbl++;
1092 if (phba->eh_sgl_free_index ==
1093 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1094 phba->eh_sgl_free_index = 0;
1095 else
1096 phba->eh_sgl_free_index++;
1097}
1098
1099static void
1100be_complete_io(struct beiscsi_conn *beiscsi_conn,
1101 struct iscsi_task *task, struct sol_cqe *psol)
1102{
1103 struct beiscsi_io_task *io_task = task->dd_data;
1104 struct be_status_bhs *sts_bhs =
1105 (struct be_status_bhs *)io_task->cmd_bhs;
1106 struct iscsi_conn *conn = beiscsi_conn->conn;
6733b39a
JK		 1107 unsigned char *sense;
1108 u32 resid = 0, exp_cmdsn, max_cmdsn;
1109 u8 rsp, status, flags;
1110
bfead3b2 1111 exp_cmdsn = (psol->
6733b39a
JK		 1112 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1113 & SOL_EXP_CMD_SN_MASK);
bfead3b2 1114 max_cmdsn = ((psol->
6733b39a
JK		 1115 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1116 & SOL_EXP_CMD_SN_MASK) +
1117 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1118 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1119 rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
1120 & SOL_RESP_MASK) >> 16);
1121 status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
1122 & SOL_STS_MASK) >> 8);
1123 flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1124 & SOL_FLAGS_MASK) >> 24) | 0x80;
bd535451
JK		 1125 if (!task->sc) {
1126 if (io_task->scsi_cmnd)
1127 scsi_dma_unmap(io_task->scsi_cmnd);
6733b39a 1128
bd535451
JK		 1129 return;
1130 }
6733b39a
JK		 1131 task->sc->result = (DID_OK << 16) | status;
1132 if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1133 task->sc->result = DID_ERROR << 16;
1134 goto unmap;
1135 }
1136
1137 /* bidi not initially supported */
1138 if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1139 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
1140 32] & SOL_RES_CNT_MASK);
1141
1142 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1143 task->sc->result = DID_ERROR << 16;
1144
1145 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1146 scsi_set_resid(task->sc, resid);
1147 if (!status && (scsi_bufflen(task->sc) - resid <
1148 task->sc->underflow))
1149 task->sc->result = DID_ERROR << 16;
1150 }
1151 }
1152
1153 if (status == SAM_STAT_CHECK_CONDITION) {
4053a4be 1154 u16 sense_len;
bfead3b2 1155 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
4053a4be 1156
6733b39a 1157 sense = sts_bhs->sense_info + sizeof(unsigned short);
4053a4be 1158 sense_len = be16_to_cpu(*slen);
6733b39a
JK		 1159 memcpy(task->sc->sense_buffer, sense,
1160 min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1161 }
756d29c8 1162
6733b39a
JK		 1163 if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
1164 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1165 & SOL_RES_CNT_MASK)
1166 conn->rxdata_octets += (psol->
bfead3b2
JK		 1167 dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1168 & SOL_RES_CNT_MASK);
6733b39a
JK		 1169 }
1170unmap:
1171 scsi_dma_unmap(io_task->scsi_cmnd);
1172 iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1173}
1174
1175static void
1176be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1177 struct iscsi_task *task, struct sol_cqe *psol)
1178{
1179 struct iscsi_logout_rsp *hdr;
bfead3b2 1180 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 1181 struct iscsi_conn *conn = beiscsi_conn->conn;
1182
1183 hdr = (struct iscsi_logout_rsp *)task->hdr;
7bd6e25c 1184 hdr->opcode = ISCSI_OP_LOGOUT_RSP;
6733b39a
JK		 1185 hdr->t2wait = 5;
1186 hdr->t2retain = 0;
1187 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1188 & SOL_FLAGS_MASK) >> 24) | 0x80;
1189 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1190 32] & SOL_RESP_MASK);
1191 hdr->exp_cmdsn = cpu_to_be32(psol->
1192 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1193 & SOL_EXP_CMD_SN_MASK);
1194 hdr->max_cmdsn = be32_to_cpu((psol->
1195 dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1196 & SOL_EXP_CMD_SN_MASK) +
1197 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1198 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
7bd6e25c
JK		 1199 hdr->dlength[0] = 0;
1200 hdr->dlength[1] = 0;
1201 hdr->dlength[2] = 0;
6733b39a 1202 hdr->hlength = 0;
bfead3b2 1203 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1204 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1205}
1206
1207static void
1208be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1209 struct iscsi_task *task, struct sol_cqe *psol)
1210{
1211 struct iscsi_tm_rsp *hdr;
1212 struct iscsi_conn *conn = beiscsi_conn->conn;
bfead3b2 1213 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 1214
1215 hdr = (struct iscsi_tm_rsp *)task->hdr;
7bd6e25c 1216 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
6733b39a
JK		 1217 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1218 & SOL_FLAGS_MASK) >> 24) | 0x80;
1219 hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1220 32] & SOL_RESP_MASK);
1221 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
bfead3b2 1222 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
6733b39a
JK		 1223 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1224 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1225 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1226 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
bfead3b2 1227 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1228 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1229}
1230
1231static void
1232hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1233 struct beiscsi_hba *phba, struct sol_cqe *psol)
1234{
1235 struct hwi_wrb_context *pwrb_context;
bfead3b2 1236 struct wrb_handle *pwrb_handle = NULL;
6733b39a 1237 struct hwi_controller *phwi_ctrlr;
bfead3b2
JK		 1238 struct iscsi_task *task;
1239 struct beiscsi_io_task *io_task;
6733b39a
JK		 1240 struct iscsi_conn *conn = beiscsi_conn->conn;
1241 struct iscsi_session *session = conn->session;
1242
1243 phwi_ctrlr = phba->phwi_ctrlr;
32951dd8 1244 pwrb_context = &phwi_ctrlr->wrb_context[((psol->
35e66019 1245 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
7da50879
JK		 1246 SOL_CID_MASK) >> 6) -
1247 phba->fw_config.iscsi_cid_start];
32951dd8 1248 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
35e66019
JK		 1249 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1250 32] & SOL_WRB_INDEX_MASK) >> 16)];
32951dd8 1251 task = pwrb_handle->pio_handle;
35e66019 1252
bfead3b2 1253 io_task = task->dd_data;
1282ab76 1254 spin_lock_bh(&phba->mgmt_sgl_lock);
bfead3b2 1255 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1282ab76 1256 spin_unlock_bh(&phba->mgmt_sgl_lock);
6733b39a
JK		 1257 spin_lock_bh(&session->lock);
1258 free_wrb_handle(phba, pwrb_context, pwrb_handle);
1259 spin_unlock_bh(&session->lock);
1260}
1261
1262static void
1263be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1264 struct iscsi_task *task, struct sol_cqe *psol)
1265{
1266 struct iscsi_nopin *hdr;
1267 struct iscsi_conn *conn = beiscsi_conn->conn;
bfead3b2 1268 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 1269
1270 hdr = (struct iscsi_nopin *)task->hdr;
1271 hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1272 & SOL_FLAGS_MASK) >> 24) | 0x80;
1273 hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1274 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1275 hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1276 i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1277 ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1278 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1279 hdr->opcode = ISCSI_OP_NOOP_IN;
bfead3b2 1280 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1281 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1282}
1283
1284static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1285 struct beiscsi_hba *phba, struct sol_cqe *psol)
1286{
1287 struct hwi_wrb_context *pwrb_context;
1288 struct wrb_handle *pwrb_handle;
1289 struct iscsi_wrb *pwrb = NULL;
1290 struct hwi_controller *phwi_ctrlr;
1291 struct iscsi_task *task;
bfead3b2 1292 unsigned int type;
6733b39a
JK		 1293 struct iscsi_conn *conn = beiscsi_conn->conn;
1294 struct iscsi_session *session = conn->session;
1295
1296 phwi_ctrlr = phba->phwi_ctrlr;
32951dd8 1297 pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
35e66019 1298 (struct amap_sol_cqe, cid) / 32]
7da50879
JK		 1299 & SOL_CID_MASK) >> 6) -
1300 phba->fw_config.iscsi_cid_start];
32951dd8 1301 pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
35e66019
JK		 1302 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1303 32] & SOL_WRB_INDEX_MASK) >> 16)];
32951dd8
JK		 1304 task = pwrb_handle->pio_handle;
1305 pwrb = pwrb_handle->pwrb;
1306 type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
1307 WRB_TYPE_MASK) >> 28;
1308
bfead3b2
JK		 1309 spin_lock_bh(&session->lock);
1310 switch (type) {
6733b39a
JK		 1311 case HWH_TYPE_IO:
1312 case HWH_TYPE_IO_RD:
1313 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
dafab8e0 1314 ISCSI_OP_NOOP_OUT)
6733b39a 1315 be_complete_nopin_resp(beiscsi_conn, task, psol);
dafab8e0 1316 else
6733b39a
JK		 1317 be_complete_io(beiscsi_conn, task, psol);
1318 break;
1319
1320 case HWH_TYPE_LOGOUT:
dafab8e0
JK		 1321 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1322 be_complete_logout(beiscsi_conn, task, psol);
1323 else
1324 be_complete_tmf(beiscsi_conn, task, psol);
1325
6733b39a
JK		 1326 break;
1327
1328 case HWH_TYPE_LOGIN:
1329 SE_DEBUG(DBG_LVL_1,
1330 "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
457ff3b7 1331 "- Solicited path\n");
6733b39a
JK		 1332 break;
1333
6733b39a
JK		 1334 case HWH_TYPE_NOP:
1335 be_complete_nopin_resp(beiscsi_conn, task, psol);
1336 break;
1337
1338 default:
32951dd8 1339 shost_printk(KERN_WARNING, phba->shost,
35e66019
JK		 1340 "In hwi_complete_cmd, unknown type = %d"
1341 "wrb_index 0x%x CID 0x%x\n", type,
1342 ((psol->dw[offsetof(struct amap_iscsi_wrb,
1343 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
1344 ((psol->dw[offsetof(struct amap_sol_cqe,
1345 cid) / 32] & SOL_CID_MASK) >> 6));
6733b39a
JK		 1346 break;
1347 }
35e66019 1348
6733b39a
JK		 1349 spin_unlock_bh(&session->lock);
1350}
1351
1352static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1353 *pasync_ctx, unsigned int is_header,
1354 unsigned int host_write_ptr)
1355{
1356 if (is_header)
1357 return &pasync_ctx->async_entry[host_write_ptr].
1358 header_busy_list;
1359 else
1360 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1361}
1362
1363static struct async_pdu_handle *
1364hwi_get_async_handle(struct beiscsi_hba *phba,
1365 struct beiscsi_conn *beiscsi_conn,
1366 struct hwi_async_pdu_context *pasync_ctx,
1367 struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1368{
1369 struct be_bus_address phys_addr;
1370 struct list_head *pbusy_list;
1371 struct async_pdu_handle *pasync_handle = NULL;
6733b39a
JK		 1372 unsigned char is_header = 0;
1373
1374 phys_addr.u.a32.address_lo =
1375 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1376 ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1377 & PDUCQE_DPL_MASK) >> 16);
1378 phys_addr.u.a32.address_hi =
1379 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1380
1381 phys_addr.u.a64.address =
1382 *((unsigned long long *)(&phys_addr.u.a64.address));
1383
1384 switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1385 & PDUCQE_CODE_MASK) {
1386 case UNSOL_HDR_NOTIFY:
1387 is_header = 1;
1388
1389 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1390 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1391 index) / 32] & PDUCQE_INDEX_MASK));
6733b39a
JK		 1392 break;
1393 case UNSOL_DATA_NOTIFY:
1394 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1395 dw[offsetof(struct amap_i_t_dpdu_cqe,
1396 index) / 32] & PDUCQE_INDEX_MASK));
6733b39a
JK		 1397 break;
1398 default:
1399 pbusy_list = NULL;
1400 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 1401 "Unexpected code=%d\n",
6733b39a
JK		 1402 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1403 code) / 32] & PDUCQE_CODE_MASK);
1404 return NULL;
1405 }
1406
6733b39a
JK		 1407 WARN_ON(list_empty(pbusy_list));
1408 list_for_each_entry(pasync_handle, pbusy_list, link) {
dc63aac6 1409 if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address)
6733b39a
JK		 1410 break;
1411 }
1412
1413 WARN_ON(!pasync_handle);
1414
7da50879
JK		 1415 pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1416 phba->fw_config.iscsi_cid_start;
6733b39a
JK		 1417 pasync_handle->is_header = is_header;
1418 pasync_handle->buffer_len = ((pdpdu_cqe->
1419 dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1420 & PDUCQE_DPL_MASK) >> 16);
1421
1422 *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1423 index) / 32] & PDUCQE_INDEX_MASK);
1424 return pasync_handle;
1425}
1426
1427static unsigned int
1428hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1429 unsigned int is_header, unsigned int cq_index)
1430{
1431 struct list_head *pbusy_list;
1432 struct async_pdu_handle *pasync_handle;
1433 unsigned int num_entries, writables = 0;
1434 unsigned int *pep_read_ptr, *pwritables;
1435
dc63aac6 1436 num_entries = pasync_ctx->num_entries;
6733b39a
JK		 1437 if (is_header) {
1438 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1439 pwritables = &pasync_ctx->async_header.writables;
6733b39a
JK		 1440 } else {
1441 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1442 pwritables = &pasync_ctx->async_data.writables;
6733b39a
JK		 1443 }
1444
1445 while ((*pep_read_ptr) != cq_index) {
1446 (*pep_read_ptr)++;
1447 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1448
1449 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1450 *pep_read_ptr);
1451 if (writables == 0)
1452 WARN_ON(list_empty(pbusy_list));
1453
1454 if (!list_empty(pbusy_list)) {
1455 pasync_handle = list_entry(pbusy_list->next,
1456 struct async_pdu_handle,
1457 link);
1458 WARN_ON(!pasync_handle);
1459 pasync_handle->consumed = 1;
1460 }
1461
1462 writables++;
1463 }
1464
1465 if (!writables) {
1466 SE_DEBUG(DBG_LVL_1,
1467 "Duplicate notification received - index 0x%x!!\n",
1468 cq_index);
1469 WARN_ON(1);
1470 }
1471
1472 *pwritables = *pwritables + writables;
1473 return 0;
1474}
1475
9728d8d0 1476static void hwi_free_async_msg(struct beiscsi_hba *phba,
6733b39a
JK		 1477 unsigned int cri)
1478{
1479 struct hwi_controller *phwi_ctrlr;
1480 struct hwi_async_pdu_context *pasync_ctx;
1481 struct async_pdu_handle *pasync_handle, *tmp_handle;
1482 struct list_head *plist;
6733b39a
JK		 1483
1484 phwi_ctrlr = phba->phwi_ctrlr;
1485 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1486
1487 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1488
1489 list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1490 list_del(&pasync_handle->link);
1491
9728d8d0 1492 if (pasync_handle->is_header) {
6733b39a
JK		 1493 list_add_tail(&pasync_handle->link,
1494 &pasync_ctx->async_header.free_list);
1495 pasync_ctx->async_header.free_entries++;
6733b39a
JK		 1496 } else {
1497 list_add_tail(&pasync_handle->link,
1498 &pasync_ctx->async_data.free_list);
1499 pasync_ctx->async_data.free_entries++;
6733b39a
JK		 1500 }
1501 }
1502
1503 INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1504 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1505 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
6733b39a
JK		 1506}
1507
1508static struct phys_addr *
1509hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1510 unsigned int is_header, unsigned int host_write_ptr)
1511{
1512 struct phys_addr *pasync_sge = NULL;
1513
1514 if (is_header)
1515 pasync_sge = pasync_ctx->async_header.ring_base;
1516 else
1517 pasync_sge = pasync_ctx->async_data.ring_base;
1518
1519 return pasync_sge + host_write_ptr;
1520}
1521
1522static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1523 unsigned int is_header)
1524{
1525 struct hwi_controller *phwi_ctrlr;
1526 struct hwi_async_pdu_context *pasync_ctx;
1527 struct async_pdu_handle *pasync_handle;
1528 struct list_head *pfree_link, *pbusy_list;
1529 struct phys_addr *pasync_sge;
1530 unsigned int ring_id, num_entries;
1531 unsigned int host_write_num;
1532 unsigned int writables;
1533 unsigned int i = 0;
1534 u32 doorbell = 0;
1535
1536 phwi_ctrlr = phba->phwi_ctrlr;
1537 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
dc63aac6 1538 num_entries = pasync_ctx->num_entries;
6733b39a
JK		 1539
1540 if (is_header) {
6733b39a
JK		 1541 writables = min(pasync_ctx->async_header.writables,
1542 pasync_ctx->async_header.free_entries);
1543 pfree_link = pasync_ctx->async_header.free_list.next;
1544 host_write_num = pasync_ctx->async_header.host_write_ptr;
1545 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1546 } else {
6733b39a
JK		 1547 writables = min(pasync_ctx->async_data.writables,
1548 pasync_ctx->async_data.free_entries);
1549 pfree_link = pasync_ctx->async_data.free_list.next;
1550 host_write_num = pasync_ctx->async_data.host_write_ptr;
1551 ring_id = phwi_ctrlr->default_pdu_data.id;
1552 }
1553
1554 writables = (writables / 8) * 8;
1555 if (writables) {
1556 for (i = 0; i < writables; i++) {
1557 pbusy_list =
1558 hwi_get_async_busy_list(pasync_ctx, is_header,
1559 host_write_num);
1560 pasync_handle =
1561 list_entry(pfree_link, struct async_pdu_handle,
1562 link);
1563 WARN_ON(!pasync_handle);
1564 pasync_handle->consumed = 0;
1565
1566 pfree_link = pfree_link->next;
1567
1568 pasync_sge = hwi_get_ring_address(pasync_ctx,
1569 is_header, host_write_num);
1570
1571 pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1572 pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1573
1574 list_move(&pasync_handle->link, pbusy_list);
1575
1576 host_write_num++;
1577 host_write_num = host_write_num % num_entries;
1578 }
1579
1580 if (is_header) {
1581 pasync_ctx->async_header.host_write_ptr =
1582 host_write_num;
1583 pasync_ctx->async_header.free_entries -= writables;
1584 pasync_ctx->async_header.writables -= writables;
1585 pasync_ctx->async_header.busy_entries += writables;
1586 } else {
1587 pasync_ctx->async_data.host_write_ptr = host_write_num;
1588 pasync_ctx->async_data.free_entries -= writables;
1589 pasync_ctx->async_data.writables -= writables;
1590 pasync_ctx->async_data.busy_entries += writables;
1591 }
1592
1593 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1594 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1595 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1596 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1597 << DB_DEF_PDU_CQPROC_SHIFT;
1598
1599 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1600 }
1601}
1602
1603static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1604 struct beiscsi_conn *beiscsi_conn,
1605 struct i_t_dpdu_cqe *pdpdu_cqe)
1606{
1607 struct hwi_controller *phwi_ctrlr;
1608 struct hwi_async_pdu_context *pasync_ctx;
1609 struct async_pdu_handle *pasync_handle = NULL;
1610 unsigned int cq_index = -1;
1611
1612 phwi_ctrlr = phba->phwi_ctrlr;
1613 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1614
1615 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1616 pdpdu_cqe, &cq_index);
1617 BUG_ON(pasync_handle->is_header != 0);
1618 if (pasync_handle->consumed == 0)
1619 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1620 cq_index);
1621
1622 hwi_free_async_msg(phba, pasync_handle->cri);
1623 hwi_post_async_buffers(phba, pasync_handle->is_header);
1624}
1625
1626static unsigned int
1627hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1628 struct beiscsi_hba *phba,
1629 struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1630{
1631 struct list_head *plist;
1632 struct async_pdu_handle *pasync_handle;
1633 void *phdr = NULL;
1634 unsigned int hdr_len = 0, buf_len = 0;
1635 unsigned int status, index = 0, offset = 0;
1636 void *pfirst_buffer = NULL;
1637 unsigned int num_buf = 0;
1638
1639 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1640
1641 list_for_each_entry(pasync_handle, plist, link) {
1642 if (index == 0) {
1643 phdr = pasync_handle->pbuffer;
1644 hdr_len = pasync_handle->buffer_len;
1645 } else {
1646 buf_len = pasync_handle->buffer_len;
1647 if (!num_buf) {
1648 pfirst_buffer = pasync_handle->pbuffer;
1649 num_buf++;
1650 }
1651 memcpy(pfirst_buffer + offset,
1652 pasync_handle->pbuffer, buf_len);
f2ba02b8 1653 offset += buf_len;
6733b39a
JK		 1654 }
1655 index++;
1656 }
1657
1658 status = beiscsi_process_async_pdu(beiscsi_conn, phba,
7da50879
JK		 1659 (beiscsi_conn->beiscsi_conn_cid -
1660 phba->fw_config.iscsi_cid_start),
1661 phdr, hdr_len, pfirst_buffer,
f2ba02b8 1662 offset);
6733b39a 1663
605c6cd2 1664 hwi_free_async_msg(phba, cri);
6733b39a
JK		 1665 return 0;
1666}
1667
1668static unsigned int
1669hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1670 struct beiscsi_hba *phba,
1671 struct async_pdu_handle *pasync_handle)
1672{
1673 struct hwi_async_pdu_context *pasync_ctx;
1674 struct hwi_controller *phwi_ctrlr;
1675 unsigned int bytes_needed = 0, status = 0;
1676 unsigned short cri = pasync_handle->cri;
1677 struct pdu_base *ppdu;
1678
1679 phwi_ctrlr = phba->phwi_ctrlr;
1680 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1681
1682 list_del(&pasync_handle->link);
1683 if (pasync_handle->is_header) {
1684 pasync_ctx->async_header.busy_entries--;
1685 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1686 hwi_free_async_msg(phba, cri);
1687 BUG();
1688 }
1689
1690 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1691 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1692 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1693 (unsigned short)pasync_handle->buffer_len;
1694 list_add_tail(&pasync_handle->link,
1695 &pasync_ctx->async_entry[cri].wait_queue.list);
1696
1697 ppdu = pasync_handle->pbuffer;
1698 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1699 data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1700 0xFFFF0000) | ((be16_to_cpu((ppdu->
1701 dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1702 & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1703
1704 if (status == 0) {
1705 pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1706 bytes_needed;
1707
1708 if (bytes_needed == 0)
1709 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1710 pasync_ctx, cri);
1711 }
1712 } else {
1713 pasync_ctx->async_data.busy_entries--;
1714 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1715 list_add_tail(&pasync_handle->link,
1716 &pasync_ctx->async_entry[cri].wait_queue.
1717 list);
1718 pasync_ctx->async_entry[cri].wait_queue.
1719 bytes_received +=
1720 (unsigned short)pasync_handle->buffer_len;
1721
1722 if (pasync_ctx->async_entry[cri].wait_queue.
1723 bytes_received >=
1724 pasync_ctx->async_entry[cri].wait_queue.
1725 bytes_needed)
1726 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1727 pasync_ctx, cri);
1728 }
1729 }
1730 return status;
1731}
1732
1733static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1734 struct beiscsi_hba *phba,
1735 struct i_t_dpdu_cqe *pdpdu_cqe)
1736{
1737 struct hwi_controller *phwi_ctrlr;
1738 struct hwi_async_pdu_context *pasync_ctx;
1739 struct async_pdu_handle *pasync_handle = NULL;
1740 unsigned int cq_index = -1;
1741
1742 phwi_ctrlr = phba->phwi_ctrlr;
1743 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1744 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1745 pdpdu_cqe, &cq_index);
1746
1747 if (pasync_handle->consumed == 0)
1748 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1749 cq_index);
1750 hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1751 hwi_post_async_buffers(phba, pasync_handle->is_header);
1752}
1753
756d29c8
JK		 1754static void beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1755{
1756 struct be_queue_info *mcc_cq;
1757 struct be_mcc_compl *mcc_compl;
1758 unsigned int num_processed = 0;
1759
1760 mcc_cq = &phba->ctrl.mcc_obj.cq;
1761 mcc_compl = queue_tail_node(mcc_cq);
1762 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1763 while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1764
1765 if (num_processed >= 32) {
1766 hwi_ring_cq_db(phba, mcc_cq->id,
1767 num_processed, 0, 0);
1768 num_processed = 0;
1769 }
1770 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1771 /* Interpret flags as an async trailer */
1772 if (is_link_state_evt(mcc_compl->flags))
1773 /* Interpret compl as a async link evt */
1774 beiscsi_async_link_state_process(phba,
1775 (struct be_async_event_link_state *) mcc_compl);
1776 else
1777 SE_DEBUG(DBG_LVL_1,
1778 " Unsupported Async Event, flags"
457ff3b7 1779 " = 0x%08x\n", mcc_compl->flags);
756d29c8
JK		 1780 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1781 be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1782 atomic_dec(&phba->ctrl.mcc_obj.q.used);
1783 }
1784
1785 mcc_compl->flags = 0;
1786 queue_tail_inc(mcc_cq);
1787 mcc_compl = queue_tail_node(mcc_cq);
1788 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1789 num_processed++;
1790 }
1791
1792 if (num_processed > 0)
1793 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1794
1795}
bfead3b2
JK		 1796
1797static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
6733b39a 1798{
6733b39a
JK		 1799 struct be_queue_info *cq;
1800 struct sol_cqe *sol;
1801 struct dmsg_cqe *dmsg;
1802 unsigned int num_processed = 0;
1803 unsigned int tot_nump = 0;
1804 struct beiscsi_conn *beiscsi_conn;
c2462288
JK		 1805 struct beiscsi_endpoint *beiscsi_ep;
1806 struct iscsi_endpoint *ep;
bfead3b2 1807 struct beiscsi_hba *phba;
6733b39a 1808
bfead3b2 1809 cq = pbe_eq->cq;
6733b39a 1810 sol = queue_tail_node(cq);
bfead3b2 1811 phba = pbe_eq->phba;
6733b39a
JK		 1812
1813 while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1814 CQE_VALID_MASK) {
1815 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1816
32951dd8 1817 ep = phba->ep_array[(u32) ((sol->
c2462288
JK		 1818 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1819 SOL_CID_MASK) >> 6) -
7da50879 1820 phba->fw_config.iscsi_cid_start];
32951dd8 1821
c2462288
JK		 1822 beiscsi_ep = ep->dd_data;
1823 beiscsi_conn = beiscsi_ep->conn;
756d29c8 1824
6733b39a 1825 if (num_processed >= 32) {
bfead3b2 1826 hwi_ring_cq_db(phba, cq->id,
6733b39a
JK		 1827 num_processed, 0, 0);
1828 tot_nump += num_processed;
1829 num_processed = 0;
1830 }
1831
1832 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1833 32] & CQE_CODE_MASK) {
1834 case SOL_CMD_COMPLETE:
1835 hwi_complete_cmd(beiscsi_conn, phba, sol);
1836 break;
1837 case DRIVERMSG_NOTIFY:
457ff3b7 1838 SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY\n");
6733b39a
JK		 1839 dmsg = (struct dmsg_cqe *)sol;
1840 hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1841 break;
1842 case UNSOL_HDR_NOTIFY:
bfead3b2
JK		 1843 SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1844 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1845 (struct i_t_dpdu_cqe *)sol);
1846 break;
6733b39a 1847 case UNSOL_DATA_NOTIFY:
bfead3b2 1848 SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
6733b39a
JK		 1849 hwi_process_default_pdu_ring(beiscsi_conn, phba,
1850 (struct i_t_dpdu_cqe *)sol);
1851 break;
1852 case CXN_INVALIDATE_INDEX_NOTIFY:
1853 case CMD_INVALIDATED_NOTIFY:
1854 case CXN_INVALIDATE_NOTIFY:
1855 SE_DEBUG(DBG_LVL_1,
1856 "Ignoring CQ Error notification for cmd/cxn"
1857 "invalidate\n");
1858 break;
1859 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1860 case CMD_KILLED_INVALID_STATSN_RCVD:
1861 case CMD_KILLED_INVALID_R2T_RCVD:
1862 case CMD_CXN_KILLED_LUN_INVALID:
1863 case CMD_CXN_KILLED_ICD_INVALID:
1864 case CMD_CXN_KILLED_ITT_INVALID:
1865 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1866 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
32951dd8 1867 SE_DEBUG(DBG_LVL_1,
6733b39a
JK		 1868 "CQ Error notification for cmd.. "
1869 "code %d cid 0x%x\n",
1870 sol->dw[offsetof(struct amap_sol_cqe, code) /
1871 32] & CQE_CODE_MASK,
1872 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1873 32] & SOL_CID_MASK));
1874 break;
1875 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1876 SE_DEBUG(DBG_LVL_1,
1877 "Digest error on def pdu ring, dropping..\n");
1878 hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1879 (struct i_t_dpdu_cqe *) sol);
1880 break;
1881 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1882 case CXN_KILLED_BURST_LEN_MISMATCH:
1883 case CXN_KILLED_AHS_RCVD:
1884 case CXN_KILLED_HDR_DIGEST_ERR:
1885 case CXN_KILLED_UNKNOWN_HDR:
1886 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1887 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1888 case CXN_KILLED_TIMED_OUT:
1889 case CXN_KILLED_FIN_RCVD:
1890 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1891 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1892 case CXN_KILLED_OVER_RUN_RESIDUAL:
1893 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1894 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
32951dd8 1895 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
6733b39a
JK		 1896 "0x%x...\n",
1897 sol->dw[offsetof(struct amap_sol_cqe, code) /
1898 32] & CQE_CODE_MASK,
7da50879
JK		 1899 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1900 32] & CQE_CID_MASK));
6733b39a
JK		 1901 iscsi_conn_failure(beiscsi_conn->conn,
1902 ISCSI_ERR_CONN_FAILED);
1903 break;
1904 case CXN_KILLED_RST_SENT:
1905 case CXN_KILLED_RST_RCVD:
32951dd8 1906 SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
bfead3b2 1907 "received/sent on CID 0x%x...\n",
6733b39a
JK		 1908 sol->dw[offsetof(struct amap_sol_cqe, code) /
1909 32] & CQE_CODE_MASK,
7da50879
JK		 1910 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1911 32] & CQE_CID_MASK));
6733b39a
JK		 1912 iscsi_conn_failure(beiscsi_conn->conn,
1913 ISCSI_ERR_CONN_FAILED);
1914 break;
1915 default:
1916 SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1917 "received on CID 0x%x...\n",
1918 sol->dw[offsetof(struct amap_sol_cqe, code) /
1919 32] & CQE_CODE_MASK,
7da50879
JK		 1920 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1921 32] & CQE_CID_MASK));
6733b39a
JK		 1922 break;
1923 }
1924
1925 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1926 queue_tail_inc(cq);
1927 sol = queue_tail_node(cq);
1928 num_processed++;
1929 }
1930
1931 if (num_processed > 0) {
1932 tot_nump += num_processed;
bfead3b2 1933 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
6733b39a
JK		 1934 }
1935 return tot_nump;
1936}
1937
756d29c8 1938void beiscsi_process_all_cqs(struct work_struct *work)
6733b39a
JK		 1939{
1940 unsigned long flags;
bfead3b2
JK		 1941 struct hwi_controller *phwi_ctrlr;
1942 struct hwi_context_memory *phwi_context;
1943 struct be_eq_obj *pbe_eq;
6733b39a
JK		 1944 struct beiscsi_hba *phba =
1945 container_of(work, struct beiscsi_hba, work_cqs);
1946
bfead3b2
JK		 1947 phwi_ctrlr = phba->phwi_ctrlr;
1948 phwi_context = phwi_ctrlr->phwi_ctxt;
1949 if (phba->msix_enabled)
1950 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1951 else
1952 pbe_eq = &phwi_context->be_eq[0];
1953
6733b39a
JK		 1954 if (phba->todo_mcc_cq) {
1955 spin_lock_irqsave(&phba->isr_lock, flags);
1956 phba->todo_mcc_cq = 0;
1957 spin_unlock_irqrestore(&phba->isr_lock, flags);
756d29c8 1958 beiscsi_process_mcc_isr(phba);
6733b39a
JK		 1959 }
1960
1961 if (phba->todo_cq) {
1962 spin_lock_irqsave(&phba->isr_lock, flags);
1963 phba->todo_cq = 0;
1964 spin_unlock_irqrestore(&phba->isr_lock, flags);
bfead3b2 1965 beiscsi_process_cq(pbe_eq);
6733b39a
JK		 1966 }
1967}
1968
1969static int be_iopoll(struct blk_iopoll *iop, int budget)
1970{
1971 static unsigned int ret;
1972 struct beiscsi_hba *phba;
bfead3b2 1973 struct be_eq_obj *pbe_eq;
6733b39a 1974
bfead3b2
JK		 1975 pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1976 ret = beiscsi_process_cq(pbe_eq);
6733b39a 1977 if (ret < budget) {
bfead3b2 1978 phba = pbe_eq->phba;
6733b39a 1979 blk_iopoll_complete(iop);
bfead3b2
JK		 1980 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
1981 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
6733b39a
JK		 1982 }
1983 return ret;
1984}
1985
1986static void
1987hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
1988 unsigned int num_sg, struct beiscsi_io_task *io_task)
1989{
1990 struct iscsi_sge *psgl;
58ff4bd0 1991 unsigned int sg_len, index;
6733b39a
JK		 1992 unsigned int sge_len = 0;
1993 unsigned long long addr;
1994 struct scatterlist *l_sg;
1995 unsigned int offset;
1996
1997 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
1998 io_task->bhs_pa.u.a32.address_lo);
1999 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2000 io_task->bhs_pa.u.a32.address_hi);
2001
2002 l_sg = sg;
48bd86cf
JK		 2003 for (index = 0; (index < num_sg) && (index < 2); index++,
2004 sg = sg_next(sg)) {
6733b39a
JK		 2005 if (index == 0) {
2006 sg_len = sg_dma_len(sg);
2007 addr = (u64) sg_dma_address(sg);
2008 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
457ff3b7 2009 ((u32)(addr & 0xFFFFFFFF)));
6733b39a 2010 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
457ff3b7 2011 ((u32)(addr >> 32)));
6733b39a
JK		 2012 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2013 sg_len);
2014 sge_len = sg_len;
6733b39a 2015 } else {
6733b39a
JK		 2016 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2017 pwrb, sge_len);
2018 sg_len = sg_dma_len(sg);
2019 addr = (u64) sg_dma_address(sg);
2020 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
457ff3b7 2021 ((u32)(addr & 0xFFFFFFFF)));
6733b39a 2022 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
457ff3b7 2023 ((u32)(addr >> 32)));
6733b39a
JK		 2024 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2025 sg_len);
2026 }
2027 }
2028 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2029 memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2030
2031 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2032
2033 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2034 io_task->bhs_pa.u.a32.address_hi);
2035 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2036 io_task->bhs_pa.u.a32.address_lo);
2037
caf818f1
JK		 2038 if (num_sg == 1) {
2039 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2040 1);
2041 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2042 0);
2043 } else if (num_sg == 2) {
2044 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2045 0);
2046 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2047 1);
2048 } else {
2049 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2050 0);
2051 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2052 0);
2053 }
6733b39a
JK		 2054 sg = l_sg;
2055 psgl++;
2056 psgl++;
2057 offset = 0;
48bd86cf 2058 for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
6733b39a
JK		 2059 sg_len = sg_dma_len(sg);
2060 addr = (u64) sg_dma_address(sg);
2061 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2062 (addr & 0xFFFFFFFF));
2063 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2064 (addr >> 32));
2065 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2066 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2067 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2068 offset += sg_len;
2069 }
2070 psgl--;
2071 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2072}
2073
2074static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2075{
2076 struct iscsi_sge *psgl;
2077 unsigned long long addr;
2078 struct beiscsi_io_task *io_task = task->dd_data;
2079 struct beiscsi_conn *beiscsi_conn = io_task->conn;
2080 struct beiscsi_hba *phba = beiscsi_conn->phba;
2081
2082 io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2083 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2084 io_task->bhs_pa.u.a32.address_lo);
2085 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2086 io_task->bhs_pa.u.a32.address_hi);
2087
2088 if (task->data) {
2089 if (task->data_count) {
2090 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
2091 addr = (u64) pci_map_single(phba->pcidev,
2092 task->data,
2093 task->data_count, 1);
2094 } else {
2095 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2096 addr = 0;
2097 }
2098 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
457ff3b7 2099 ((u32)(addr & 0xFFFFFFFF)));
6733b39a 2100 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
457ff3b7 2101 ((u32)(addr >> 32)));
6733b39a
JK		 2102 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2103 task->data_count);
2104
2105 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2106 } else {
2107 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2108 addr = 0;
2109 }
2110
2111 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2112
2113 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2114
2115 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2116 io_task->bhs_pa.u.a32.address_hi);
2117 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2118 io_task->bhs_pa.u.a32.address_lo);
2119 if (task->data) {
2120 psgl++;
2121 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2122 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2123 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2124 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2125 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2126 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2127
2128 psgl++;
2129 if (task->data) {
2130 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
457ff3b7 2131 ((u32)(addr & 0xFFFFFFFF)));
6733b39a 2132 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
457ff3b7 2133 ((u32)(addr >> 32)));
6733b39a
JK		 2134 }
2135 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2136 }
2137 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2138}
2139
2140static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2141{
bfead3b2 2142 unsigned int num_cq_pages, num_async_pdu_buf_pages;
6733b39a
JK		 2143 unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2144 unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2145
2146 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2147 sizeof(struct sol_cqe));
6733b39a
JK		 2148 num_async_pdu_buf_pages =
2149 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2150 phba->params.defpdu_hdr_sz);
2151 num_async_pdu_buf_sgl_pages =
2152 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2153 sizeof(struct phys_addr));
2154 num_async_pdu_data_pages =
2155 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2156 phba->params.defpdu_data_sz);
2157 num_async_pdu_data_sgl_pages =
2158 PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2159 sizeof(struct phys_addr));
2160
2161 phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2162
2163 phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2164 BE_ISCSI_PDU_HEADER_SIZE;
2165 phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2166 sizeof(struct hwi_context_memory);
2167
6733b39a
JK		 2168
2169 phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2170 * (phba->params.wrbs_per_cxn)
2171 * phba->params.cxns_per_ctrl;
2172 wrb_sz_per_cxn = sizeof(struct wrb_handle) *
2173 (phba->params.wrbs_per_cxn);
2174 phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2175 phba->params.cxns_per_ctrl);
2176
2177 phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2178 phba->params.icds_per_ctrl;
2179 phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2180 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2181
2182 phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
2183 num_async_pdu_buf_pages * PAGE_SIZE;
2184 phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
2185 num_async_pdu_data_pages * PAGE_SIZE;
2186 phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
2187 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
2188 phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
2189 num_async_pdu_data_sgl_pages * PAGE_SIZE;
2190 phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
2191 phba->params.asyncpdus_per_ctrl *
2192 sizeof(struct async_pdu_handle);
2193 phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
2194 phba->params.asyncpdus_per_ctrl *
2195 sizeof(struct async_pdu_handle);
2196 phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
2197 sizeof(struct hwi_async_pdu_context) +
2198 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
2199}
2200
2201static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2202{
2203 struct be_mem_descriptor *mem_descr;
2204 dma_addr_t bus_add;
2205 struct mem_array *mem_arr, *mem_arr_orig;
2206 unsigned int i, j, alloc_size, curr_alloc_size;
2207
3ec78271 2208 phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
6733b39a
JK		 2209 if (!phba->phwi_ctrlr)
2210 return -ENOMEM;
2211
2212 phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2213 GFP_KERNEL);
2214 if (!phba->init_mem) {
2215 kfree(phba->phwi_ctrlr);
2216 return -ENOMEM;
2217 }
2218
2219 mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2220 GFP_KERNEL);
2221 if (!mem_arr_orig) {
2222 kfree(phba->init_mem);
2223 kfree(phba->phwi_ctrlr);
2224 return -ENOMEM;
2225 }
2226
2227 mem_descr = phba->init_mem;
2228 for (i = 0; i < SE_MEM_MAX; i++) {
2229 j = 0;
2230 mem_arr = mem_arr_orig;
2231 alloc_size = phba->mem_req[i];
2232 memset(mem_arr, 0, sizeof(struct mem_array) *
2233 BEISCSI_MAX_FRAGS_INIT);
2234 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2235 do {
2236 mem_arr->virtual_address = pci_alloc_consistent(
2237 phba->pcidev,
2238 curr_alloc_size,
2239 &bus_add);
2240 if (!mem_arr->virtual_address) {
2241 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2242 goto free_mem;
2243 if (curr_alloc_size -
2244 rounddown_pow_of_two(curr_alloc_size))
2245 curr_alloc_size = rounddown_pow_of_two
2246 (curr_alloc_size);
2247 else
2248 curr_alloc_size = curr_alloc_size / 2;
2249 } else {
2250 mem_arr->bus_address.u.
2251 a64.address = (__u64) bus_add;
2252 mem_arr->size = curr_alloc_size;
2253 alloc_size -= curr_alloc_size;
2254 curr_alloc_size = min(be_max_phys_size *
2255 1024, alloc_size);
2256 j++;
2257 mem_arr++;
2258 }
2259 } while (alloc_size);
2260 mem_descr->num_elements = j;
2261 mem_descr->size_in_bytes = phba->mem_req[i];
2262 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2263 GFP_KERNEL);
2264 if (!mem_descr->mem_array)
2265 goto free_mem;
2266
2267 memcpy(mem_descr->mem_array, mem_arr_orig,
2268 sizeof(struct mem_array) * j);
2269 mem_descr++;
2270 }
2271 kfree(mem_arr_orig);
2272 return 0;
2273free_mem:
2274 mem_descr->num_elements = j;
2275 while ((i) || (j)) {
2276 for (j = mem_descr->num_elements; j > 0; j--) {
2277 pci_free_consistent(phba->pcidev,
2278 mem_descr->mem_array[j - 1].size,
2279 mem_descr->mem_array[j - 1].
2280 virtual_address,
457ff3b7
JK		 2281 (unsigned long)mem_descr->
2282 mem_array[j - 1].
6733b39a
JK		 2283 bus_address.u.a64.address);
2284 }
2285 if (i) {
2286 i--;
2287 kfree(mem_descr->mem_array);
2288 mem_descr--;
2289 }
2290 }
2291 kfree(mem_arr_orig);
2292 kfree(phba->init_mem);
2293 kfree(phba->phwi_ctrlr);
2294 return -ENOMEM;
2295}
2296
2297static int beiscsi_get_memory(struct beiscsi_hba *phba)
2298{
2299 beiscsi_find_mem_req(phba);
2300 return beiscsi_alloc_mem(phba);
2301}
2302
2303static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2304{
2305 struct pdu_data_out *pdata_out;
2306 struct pdu_nop_out *pnop_out;
2307 struct be_mem_descriptor *mem_descr;
2308
2309 mem_descr = phba->init_mem;
2310 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2311 pdata_out =
2312 (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2313 memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2314
2315 AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2316 IIOC_SCSI_DATA);
2317
2318 pnop_out =
2319 (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2320 virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2321
2322 memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2323 AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2324 AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2325 AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2326}
2327
3ec78271 2328static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
6733b39a
JK		 2329{
2330 struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
3ec78271 2331 struct wrb_handle *pwrb_handle = NULL;
6733b39a
JK		 2332 struct hwi_controller *phwi_ctrlr;
2333 struct hwi_wrb_context *pwrb_context;
3ec78271
JK		 2334 struct iscsi_wrb *pwrb = NULL;
2335 unsigned int num_cxn_wrbh = 0;
2336 unsigned int num_cxn_wrb = 0, j, idx = 0, index;
6733b39a
JK		 2337
2338 mem_descr_wrbh = phba->init_mem;
2339 mem_descr_wrbh += HWI_MEM_WRBH;
2340
2341 mem_descr_wrb = phba->init_mem;
2342 mem_descr_wrb += HWI_MEM_WRB;
6733b39a
JK		 2343 phwi_ctrlr = phba->phwi_ctrlr;
2344
2345 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2346 pwrb_context = &phwi_ctrlr->wrb_context[index];
6733b39a
JK		 2347 pwrb_context->pwrb_handle_base =
2348 kzalloc(sizeof(struct wrb_handle *) *
2349 phba->params.wrbs_per_cxn, GFP_KERNEL);
3ec78271
JK		 2350 if (!pwrb_context->pwrb_handle_base) {
2351 shost_printk(KERN_ERR, phba->shost,
2352 "Mem Alloc Failed. Failing to load\n");
2353 goto init_wrb_hndl_failed;
2354 }
6733b39a
JK		 2355 pwrb_context->pwrb_handle_basestd =
2356 kzalloc(sizeof(struct wrb_handle *) *
2357 phba->params.wrbs_per_cxn, GFP_KERNEL);
3ec78271
JK		 2358 if (!pwrb_context->pwrb_handle_basestd) {
2359 shost_printk(KERN_ERR, phba->shost,
2360 "Mem Alloc Failed. Failing to load\n");
2361 goto init_wrb_hndl_failed;
2362 }
2363 if (!num_cxn_wrbh) {
2364 pwrb_handle =
2365 mem_descr_wrbh->mem_array[idx].virtual_address;
2366 num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2367 ((sizeof(struct wrb_handle)) *
2368 phba->params.wrbs_per_cxn));
2369 idx++;
2370 }
2371 pwrb_context->alloc_index = 0;
2372 pwrb_context->wrb_handles_available = 0;
2373 pwrb_context->free_index = 0;
2374
6733b39a 2375 if (num_cxn_wrbh) {
6733b39a
JK		 2376 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2377 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2378 pwrb_context->pwrb_handle_basestd[j] =
2379 pwrb_handle;
2380 pwrb_context->wrb_handles_available++;
bfead3b2 2381 pwrb_handle->wrb_index = j;
6733b39a
JK		 2382 pwrb_handle++;
2383 }
6733b39a
JK		 2384 num_cxn_wrbh--;
2385 }
2386 }
2387 idx = 0;
ed58ea2a 2388 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
6733b39a 2389 pwrb_context = &phwi_ctrlr->wrb_context[index];
3ec78271 2390 if (!num_cxn_wrb) {
6733b39a 2391 pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
7c56533c 2392 num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
3ec78271
JK		 2393 ((sizeof(struct iscsi_wrb) *
2394 phba->params.wrbs_per_cxn));
2395 idx++;
2396 }
2397
2398 if (num_cxn_wrb) {
6733b39a
JK		 2399 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2400 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2401 pwrb_handle->pwrb = pwrb;
2402 pwrb++;
2403 }
2404 num_cxn_wrb--;
2405 }
2406 }
3ec78271
JK		 2407 return 0;
2408init_wrb_hndl_failed:
2409 for (j = index; j > 0; j--) {
2410 pwrb_context = &phwi_ctrlr->wrb_context[j];
2411 kfree(pwrb_context->pwrb_handle_base);
2412 kfree(pwrb_context->pwrb_handle_basestd);
2413 }
2414 return -ENOMEM;
6733b39a
JK		 2415}
2416
2417static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2418{
2419 struct hwi_controller *phwi_ctrlr;
2420 struct hba_parameters *p = &phba->params;
2421 struct hwi_async_pdu_context *pasync_ctx;
2422 struct async_pdu_handle *pasync_header_h, *pasync_data_h;
dc63aac6 2423 unsigned int index, idx, num_per_mem, num_async_data;
6733b39a
JK		 2424 struct be_mem_descriptor *mem_descr;
2425
2426 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2427 mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2428
2429 phwi_ctrlr = phba->phwi_ctrlr;
2430 phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2431 mem_descr->mem_array[0].virtual_address;
2432 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2433 memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2434
dc63aac6
JK		 2435 pasync_ctx->num_entries = p->asyncpdus_per_ctrl;
2436 pasync_ctx->buffer_size = p->defpdu_hdr_sz;
6733b39a
JK		 2437
2438 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2439 mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2440 if (mem_descr->mem_array[0].virtual_address) {
2441 SE_DEBUG(DBG_LVL_8,
2442 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
457ff3b7 2443 "va=%p\n", mem_descr->mem_array[0].virtual_address);
6733b39a
JK		 2444 } else
2445 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 2446 "No Virtual address\n");
6733b39a
JK		 2447
2448 pasync_ctx->async_header.va_base =
2449 mem_descr->mem_array[0].virtual_address;
2450
2451 pasync_ctx->async_header.pa_base.u.a64.address =
2452 mem_descr->mem_array[0].bus_address.u.a64.address;
2453
2454 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2455 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2456 if (mem_descr->mem_array[0].virtual_address) {
2457 SE_DEBUG(DBG_LVL_8,
2458 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
457ff3b7 2459 "va=%p\n", mem_descr->mem_array[0].virtual_address);
6733b39a
JK		 2460 } else
2461 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 2462 "No Virtual address\n");
6733b39a
JK		 2463 pasync_ctx->async_header.ring_base =
2464 mem_descr->mem_array[0].virtual_address;
2465
2466 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2467 mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2468 if (mem_descr->mem_array[0].virtual_address) {
2469 SE_DEBUG(DBG_LVL_8,
2470 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
457ff3b7 2471 "va=%p\n", mem_descr->mem_array[0].virtual_address);
6733b39a
JK		 2472 } else
2473 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 2474 "No Virtual address\n");
6733b39a
JK		 2475
2476 pasync_ctx->async_header.handle_base =
2477 mem_descr->mem_array[0].virtual_address;
2478 pasync_ctx->async_header.writables = 0;
2479 INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2480
6733b39a
JK		 2481
2482 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2483 mem_descr += HWI_MEM_ASYNC_DATA_RING;
2484 if (mem_descr->mem_array[0].virtual_address) {
2485 SE_DEBUG(DBG_LVL_8,
2486 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
457ff3b7 2487 "va=%p\n", mem_descr->mem_array[0].virtual_address);
6733b39a
JK		 2488 } else
2489 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 2490 "No Virtual address\n");
6733b39a
JK		 2491
2492 pasync_ctx->async_data.ring_base =
2493 mem_descr->mem_array[0].virtual_address;
2494
2495 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2496 mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2497 if (!mem_descr->mem_array[0].virtual_address)
2498 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 2499 "No Virtual address\n");
6733b39a
JK		 2500
2501 pasync_ctx->async_data.handle_base =
2502 mem_descr->mem_array[0].virtual_address;
2503 pasync_ctx->async_data.writables = 0;
2504 INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2505
2506 pasync_header_h =
2507 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2508 pasync_data_h =
2509 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2510
dc63aac6
JK		 2511 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2512 mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2513 if (mem_descr->mem_array[0].virtual_address) {
2514 SE_DEBUG(DBG_LVL_8,
2515 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2516 "va=%p\n", mem_descr->mem_array[0].virtual_address);
2517 } else
2518 shost_printk(KERN_WARNING, phba->shost,
2519 "No Virtual address\n");
2520 idx = 0;
2521 pasync_ctx->async_data.va_base =
2522 mem_descr->mem_array[idx].virtual_address;
2523 pasync_ctx->async_data.pa_base.u.a64.address =
2524 mem_descr->mem_array[idx].bus_address.u.a64.address;
2525
2526 num_async_data = ((mem_descr->mem_array[idx].size) /
2527 phba->params.defpdu_data_sz);
2528 num_per_mem = 0;
2529
6733b39a
JK		 2530 for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2531 pasync_header_h->cri = -1;
2532 pasync_header_h->index = (char)index;
2533 INIT_LIST_HEAD(&pasync_header_h->link);
2534 pasync_header_h->pbuffer =
2535 (void *)((unsigned long)
2536 (pasync_ctx->async_header.va_base) +
2537 (p->defpdu_hdr_sz * index));
2538
2539 pasync_header_h->pa.u.a64.address =
2540 pasync_ctx->async_header.pa_base.u.a64.address +
2541 (p->defpdu_hdr_sz * index);
2542
2543 list_add_tail(&pasync_header_h->link,
2544 &pasync_ctx->async_header.free_list);
2545 pasync_header_h++;
2546 pasync_ctx->async_header.free_entries++;
2547 pasync_ctx->async_header.writables++;
2548
2549 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2550 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2551 header_busy_list);
2552 pasync_data_h->cri = -1;
2553 pasync_data_h->index = (char)index;
2554 INIT_LIST_HEAD(&pasync_data_h->link);
dc63aac6
JK		 2555
2556 if (!num_async_data) {
2557 num_per_mem = 0;
2558 idx++;
2559 pasync_ctx->async_data.va_base =
2560 mem_descr->mem_array[idx].virtual_address;
2561 pasync_ctx->async_data.pa_base.u.a64.address =
2562 mem_descr->mem_array[idx].
2563 bus_address.u.a64.address;
2564
2565 num_async_data = ((mem_descr->mem_array[idx].size) /
2566 phba->params.defpdu_data_sz);
2567 }
6733b39a
JK		 2568 pasync_data_h->pbuffer =
2569 (void *)((unsigned long)
2570 (pasync_ctx->async_data.va_base) +
dc63aac6 2571 (p->defpdu_data_sz * num_per_mem));
6733b39a
JK		 2572
2573 pasync_data_h->pa.u.a64.address =
2574 pasync_ctx->async_data.pa_base.u.a64.address +
dc63aac6
JK		 2575 (p->defpdu_data_sz * num_per_mem);
2576 num_per_mem++;
2577 num_async_data--;
6733b39a
JK		 2578
2579 list_add_tail(&pasync_data_h->link,
2580 &pasync_ctx->async_data.free_list);
2581 pasync_data_h++;
2582 pasync_ctx->async_data.free_entries++;
2583 pasync_ctx->async_data.writables++;
2584
2585 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2586 }
2587
2588 pasync_ctx->async_header.host_write_ptr = 0;
2589 pasync_ctx->async_header.ep_read_ptr = -1;
2590 pasync_ctx->async_data.host_write_ptr = 0;
2591 pasync_ctx->async_data.ep_read_ptr = -1;
2592}
2593
2594static int
2595be_sgl_create_contiguous(void *virtual_address,
2596 u64 physical_address, u32 length,
2597 struct be_dma_mem *sgl)
2598{
2599 WARN_ON(!virtual_address);
2600 WARN_ON(!physical_address);
2601 WARN_ON(!length > 0);
2602 WARN_ON(!sgl);
2603
2604 sgl->va = virtual_address;
457ff3b7 2605 sgl->dma = (unsigned long)physical_address;
6733b39a
JK		 2606 sgl->size = length;
2607
2608 return 0;
2609}
2610
2611static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2612{
2613 memset(sgl, 0, sizeof(*sgl));
2614}
2615
2616static void
2617hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2618 struct mem_array *pmem, struct be_dma_mem *sgl)
2619{
2620 if (sgl->va)
2621 be_sgl_destroy_contiguous(sgl);
2622
2623 be_sgl_create_contiguous(pmem->virtual_address,
2624 pmem->bus_address.u.a64.address,
2625 pmem->size, sgl);
2626}
2627
2628static void
2629hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2630 struct mem_array *pmem, struct be_dma_mem *sgl)
2631{
2632 if (sgl->va)
2633 be_sgl_destroy_contiguous(sgl);
2634
2635 be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2636 pmem->bus_address.u.a64.address,
2637 pmem->size, sgl);
2638}
2639
2640static int be_fill_queue(struct be_queue_info *q,
2641 u16 len, u16 entry_size, void *vaddress)
2642{
2643 struct be_dma_mem *mem = &q->dma_mem;
2644
2645 memset(q, 0, sizeof(*q));
2646 q->len = len;
2647 q->entry_size = entry_size;
2648 mem->size = len * entry_size;
2649 mem->va = vaddress;
2650 if (!mem->va)
2651 return -ENOMEM;
2652 memset(mem->va, 0, mem->size);
2653 return 0;
2654}
2655
bfead3b2 2656static int beiscsi_create_eqs(struct beiscsi_hba *phba,
6733b39a
JK		 2657 struct hwi_context_memory *phwi_context)
2658{
bfead3b2
JK		 2659 unsigned int i, num_eq_pages;
2660 int ret, eq_for_mcc;
6733b39a
JK		 2661 struct be_queue_info *eq;
2662 struct be_dma_mem *mem;
6733b39a 2663 void *eq_vaddress;
bfead3b2 2664 dma_addr_t paddr;
6733b39a 2665
bfead3b2
JK		 2666 num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2667 sizeof(struct be_eq_entry));
6733b39a 2668
bfead3b2
JK		 2669 if (phba->msix_enabled)
2670 eq_for_mcc = 1;
2671 else
2672 eq_for_mcc = 0;
2673 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2674 eq = &phwi_context->be_eq[i].q;
2675 mem = &eq->dma_mem;
2676 phwi_context->be_eq[i].phba = phba;
2677 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2678 num_eq_pages * PAGE_SIZE,
2679 &paddr);
2680 if (!eq_vaddress)
2681 goto create_eq_error;
2682
2683 mem->va = eq_vaddress;
2684 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2685 sizeof(struct be_eq_entry), eq_vaddress);
2686 if (ret) {
2687 shost_printk(KERN_ERR, phba->shost,
457ff3b7 2688 "be_fill_queue Failed for EQ\n");
bfead3b2
JK		 2689 goto create_eq_error;
2690 }
6733b39a 2691
bfead3b2
JK		 2692 mem->dma = paddr;
2693 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2694 phwi_context->cur_eqd);
2695 if (ret) {
2696 shost_printk(KERN_ERR, phba->shost,
2697 "beiscsi_cmd_eq_create"
457ff3b7 2698 "Failedfor EQ\n");
bfead3b2
JK		 2699 goto create_eq_error;
2700 }
2701 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
6733b39a 2702 }
6733b39a 2703 return 0;
bfead3b2
JK		 2704create_eq_error:
2705 for (i = 0; i < (phba->num_cpus + 1); i++) {
2706 eq = &phwi_context->be_eq[i].q;
2707 mem = &eq->dma_mem;
2708 if (mem->va)
2709 pci_free_consistent(phba->pcidev, num_eq_pages
2710 * PAGE_SIZE,
2711 mem->va, mem->dma);
2712 }
2713 return ret;
6733b39a
JK		 2714}
2715
bfead3b2 2716static int beiscsi_create_cqs(struct beiscsi_hba *phba,
6733b39a
JK		 2717 struct hwi_context_memory *phwi_context)
2718{
bfead3b2 2719 unsigned int i, num_cq_pages;
6733b39a
JK		 2720 int ret;
2721 struct be_queue_info *cq, *eq;
2722 struct be_dma_mem *mem;
bfead3b2 2723 struct be_eq_obj *pbe_eq;
6733b39a 2724 void *cq_vaddress;
bfead3b2 2725 dma_addr_t paddr;
6733b39a 2726
bfead3b2
JK		 2727 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2728 sizeof(struct sol_cqe));
6733b39a 2729
bfead3b2
JK		 2730 for (i = 0; i < phba->num_cpus; i++) {
2731 cq = &phwi_context->be_cq[i];
2732 eq = &phwi_context->be_eq[i].q;
2733 pbe_eq = &phwi_context->be_eq[i];
2734 pbe_eq->cq = cq;
2735 pbe_eq->phba = phba;
2736 mem = &cq->dma_mem;
2737 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2738 num_cq_pages * PAGE_SIZE,
2739 &paddr);
2740 if (!cq_vaddress)
2741 goto create_cq_error;
7da50879 2742 ret = be_fill_queue(cq, phba->params.num_cq_entries,
bfead3b2
JK		 2743 sizeof(struct sol_cqe), cq_vaddress);
2744 if (ret) {
2745 shost_printk(KERN_ERR, phba->shost,
457ff3b7 2746 "be_fill_queue Failed for ISCSI CQ\n");
bfead3b2
JK		 2747 goto create_cq_error;
2748 }
2749
2750 mem->dma = paddr;
2751 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2752 false, 0);
2753 if (ret) {
2754 shost_printk(KERN_ERR, phba->shost,
2755 "beiscsi_cmd_eq_create"
457ff3b7 2756 "Failed for ISCSI CQ\n");
bfead3b2
JK		 2757 goto create_cq_error;
2758 }
2759 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2760 cq->id, eq->id);
2761 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
6733b39a 2762 }
6733b39a 2763 return 0;
bfead3b2
JK		 2764
2765create_cq_error:
2766 for (i = 0; i < phba->num_cpus; i++) {
2767 cq = &phwi_context->be_cq[i];
2768 mem = &cq->dma_mem;
2769 if (mem->va)
2770 pci_free_consistent(phba->pcidev, num_cq_pages
2771 * PAGE_SIZE,
2772 mem->va, mem->dma);
2773 }
2774 return ret;
2775
6733b39a
JK		 2776}
2777
2778static int
2779beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2780 struct hwi_context_memory *phwi_context,
2781 struct hwi_controller *phwi_ctrlr,
2782 unsigned int def_pdu_ring_sz)
2783{
2784 unsigned int idx;
2785 int ret;
2786 struct be_queue_info *dq, *cq;
2787 struct be_dma_mem *mem;
2788 struct be_mem_descriptor *mem_descr;
2789 void *dq_vaddress;
2790
2791 idx = 0;
2792 dq = &phwi_context->be_def_hdrq;
bfead3b2 2793 cq = &phwi_context->be_cq[0];
6733b39a
JK		 2794 mem = &dq->dma_mem;
2795 mem_descr = phba->init_mem;
2796 mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2797 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2798 ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2799 sizeof(struct phys_addr),
2800 sizeof(struct phys_addr), dq_vaddress);
2801 if (ret) {
2802 shost_printk(KERN_ERR, phba->shost,
2803 "be_fill_queue Failed for DEF PDU HDR\n");
2804 return ret;
2805 }
457ff3b7
JK		 2806 mem->dma = (unsigned long)mem_descr->mem_array[idx].
2807 bus_address.u.a64.address;
6733b39a
JK		 2808 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2809 def_pdu_ring_sz,
2810 phba->params.defpdu_hdr_sz);
2811 if (ret) {
2812 shost_printk(KERN_ERR, phba->shost,
2813 "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2814 return ret;
2815 }
2816 phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2817 SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2818 phwi_context->be_def_hdrq.id);
2819 hwi_post_async_buffers(phba, 1);
2820 return 0;
2821}
2822
2823static int
2824beiscsi_create_def_data(struct beiscsi_hba *phba,
2825 struct hwi_context_memory *phwi_context,
2826 struct hwi_controller *phwi_ctrlr,
2827 unsigned int def_pdu_ring_sz)
2828{
2829 unsigned int idx;
2830 int ret;
2831 struct be_queue_info *dataq, *cq;
2832 struct be_dma_mem *mem;
2833 struct be_mem_descriptor *mem_descr;
2834 void *dq_vaddress;
2835
2836 idx = 0;
2837 dataq = &phwi_context->be_def_dataq;
bfead3b2 2838 cq = &phwi_context->be_cq[0];
6733b39a
JK		 2839 mem = &dataq->dma_mem;
2840 mem_descr = phba->init_mem;
2841 mem_descr += HWI_MEM_ASYNC_DATA_RING;
2842 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2843 ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2844 sizeof(struct phys_addr),
2845 sizeof(struct phys_addr), dq_vaddress);
2846 if (ret) {
2847 shost_printk(KERN_ERR, phba->shost,
2848 "be_fill_queue Failed for DEF PDU DATA\n");
2849 return ret;
2850 }
457ff3b7
JK		 2851 mem->dma = (unsigned long)mem_descr->mem_array[idx].
2852 bus_address.u.a64.address;
6733b39a
JK		 2853 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2854 def_pdu_ring_sz,
2855 phba->params.defpdu_data_sz);
2856 if (ret) {
2857 shost_printk(KERN_ERR, phba->shost,
2858 "be_cmd_create_default_pdu_queue Failed"
2859 " for DEF PDU DATA\n");
2860 return ret;
2861 }
2862 phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2863 SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2864 phwi_context->be_def_dataq.id);
2865 hwi_post_async_buffers(phba, 0);
457ff3b7 2866 SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED\n");
6733b39a
JK		 2867 return 0;
2868}
2869
2870static int
2871beiscsi_post_pages(struct beiscsi_hba *phba)
2872{
2873 struct be_mem_descriptor *mem_descr;
2874 struct mem_array *pm_arr;
2875 unsigned int page_offset, i;
2876 struct be_dma_mem sgl;
2877 int status;
2878
2879 mem_descr = phba->init_mem;
2880 mem_descr += HWI_MEM_SGE;
2881 pm_arr = mem_descr->mem_array;
2882
2883 page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2884 phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2885 for (i = 0; i < mem_descr->num_elements; i++) {
2886 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2887 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2888 page_offset,
2889 (pm_arr->size / PAGE_SIZE));
2890 page_offset += pm_arr->size / PAGE_SIZE;
2891 if (status != 0) {
2892 shost_printk(KERN_ERR, phba->shost,
2893 "post sgl failed.\n");
2894 return status;
2895 }
2896 pm_arr++;
2897 }
457ff3b7 2898 SE_DEBUG(DBG_LVL_8, "POSTED PAGES\n");
6733b39a
JK		 2899 return 0;
2900}
2901
bfead3b2
JK		 2902static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2903{
2904 struct be_dma_mem *mem = &q->dma_mem;
c8b25598 2905 if (mem->va) {
bfead3b2
JK		 2906 pci_free_consistent(phba->pcidev, mem->size,
2907 mem->va, mem->dma);
c8b25598
JK		 2908 mem->va = NULL;
2909 }
bfead3b2
JK		 2910}
2911
2912static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2913 u16 len, u16 entry_size)
2914{
2915 struct be_dma_mem *mem = &q->dma_mem;
2916
2917 memset(q, 0, sizeof(*q));
2918 q->len = len;
2919 q->entry_size = entry_size;
2920 mem->size = len * entry_size;
2921 mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2922 if (!mem->va)
d3ad2bb3 2923 return -ENOMEM;
bfead3b2
JK		 2924 memset(mem->va, 0, mem->size);
2925 return 0;
2926}
2927
6733b39a
JK		 2928static int
2929beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2930 struct hwi_context_memory *phwi_context,
2931 struct hwi_controller *phwi_ctrlr)
2932{
2933 unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2934 u64 pa_addr_lo;
2935 unsigned int idx, num, i;
2936 struct mem_array *pwrb_arr;
2937 void *wrb_vaddr;
2938 struct be_dma_mem sgl;
2939 struct be_mem_descriptor *mem_descr;
2940 int status;
2941
2942 idx = 0;
2943 mem_descr = phba->init_mem;
2944 mem_descr += HWI_MEM_WRB;
2945 pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2946 GFP_KERNEL);
2947 if (!pwrb_arr) {
2948 shost_printk(KERN_ERR, phba->shost,
2949 "Memory alloc failed in create wrb ring.\n");
2950 return -ENOMEM;
2951 }
2952 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2953 pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2954 num_wrb_rings = mem_descr->mem_array[idx].size /
2955 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2956
2957 for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2958 if (num_wrb_rings) {
2959 pwrb_arr[num].virtual_address = wrb_vaddr;
2960 pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2961 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2962 sizeof(struct iscsi_wrb);
2963 wrb_vaddr += pwrb_arr[num].size;
2964 pa_addr_lo += pwrb_arr[num].size;
2965 num_wrb_rings--;
2966 } else {
2967 idx++;
2968 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2969 pa_addr_lo = mem_descr->mem_array[idx].\
2970 bus_address.u.a64.address;
2971 num_wrb_rings = mem_descr->mem_array[idx].size /
2972 (phba->params.wrbs_per_cxn *
2973 sizeof(struct iscsi_wrb));
2974 pwrb_arr[num].virtual_address = wrb_vaddr;
2975 pwrb_arr[num].bus_address.u.a64.address\
2976 = pa_addr_lo;
2977 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2978 sizeof(struct iscsi_wrb);
2979 wrb_vaddr += pwrb_arr[num].size;
2980 pa_addr_lo += pwrb_arr[num].size;
2981 num_wrb_rings--;
2982 }
2983 }
2984 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2985 wrb_mem_index = 0;
2986 offset = 0;
2987 size = 0;
2988
2989 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2990 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2991 &phwi_context->be_wrbq[i]);
2992 if (status != 0) {
2993 shost_printk(KERN_ERR, phba->shost,
2994 "wrbq create failed.");
1462b8ff 2995 kfree(pwrb_arr);
6733b39a
JK		 2996 return status;
2997 }
7da50879
JK		 2998 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
2999 id;
6733b39a
JK		 3000 }
3001 kfree(pwrb_arr);
3002 return 0;
3003}
3004
3005static void free_wrb_handles(struct beiscsi_hba *phba)
3006{
3007 unsigned int index;
3008 struct hwi_controller *phwi_ctrlr;
3009 struct hwi_wrb_context *pwrb_context;
3010
3011 phwi_ctrlr = phba->phwi_ctrlr;
3012 for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
3013 pwrb_context = &phwi_ctrlr->wrb_context[index];
3014 kfree(pwrb_context->pwrb_handle_base);
3015 kfree(pwrb_context->pwrb_handle_basestd);
3016 }
3017}
3018
bfead3b2
JK		 3019static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3020{
3021 struct be_queue_info *q;
3022 struct be_ctrl_info *ctrl = &phba->ctrl;
3023
3024 q = &phba->ctrl.mcc_obj.q;
3025 if (q->created)
3026 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3027 be_queue_free(phba, q);
3028
3029 q = &phba->ctrl.mcc_obj.cq;
3030 if (q->created)
3031 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3032 be_queue_free(phba, q);
3033}
3034
6733b39a
JK		 3035static void hwi_cleanup(struct beiscsi_hba *phba)
3036{
3037 struct be_queue_info *q;
3038 struct be_ctrl_info *ctrl = &phba->ctrl;
3039 struct hwi_controller *phwi_ctrlr;
3040 struct hwi_context_memory *phwi_context;
bfead3b2 3041 int i, eq_num;
6733b39a
JK		 3042
3043 phwi_ctrlr = phba->phwi_ctrlr;
3044 phwi_context = phwi_ctrlr->phwi_ctxt;
3045 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3046 q = &phwi_context->be_wrbq[i];
3047 if (q->created)
3048 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3049 }
6733b39a
JK		 3050 free_wrb_handles(phba);
3051
3052 q = &phwi_context->be_def_hdrq;
3053 if (q->created)
3054 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3055
3056 q = &phwi_context->be_def_dataq;
3057 if (q->created)
3058 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3059
3060 beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3061
bfead3b2
JK		 3062 for (i = 0; i < (phba->num_cpus); i++) {
3063 q = &phwi_context->be_cq[i];
3064 if (q->created)
3065 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3066 }
3067 if (phba->msix_enabled)
3068 eq_num = 1;
3069 else
3070 eq_num = 0;
3071 for (i = 0; i < (phba->num_cpus + eq_num); i++) {
3072 q = &phwi_context->be_eq[i].q;
3073 if (q->created)
3074 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3075 }
3076 be_mcc_queues_destroy(phba);
3077}
6733b39a 3078
bfead3b2
JK		 3079static int be_mcc_queues_create(struct beiscsi_hba *phba,
3080 struct hwi_context_memory *phwi_context)
3081{
3082 struct be_queue_info *q, *cq;
3083 struct be_ctrl_info *ctrl = &phba->ctrl;
3084
3085 /* Alloc MCC compl queue */
3086 cq = &phba->ctrl.mcc_obj.cq;
3087 if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3088 sizeof(struct be_mcc_compl)))
3089 goto err;
3090 /* Ask BE to create MCC compl queue; */
3091 if (phba->msix_enabled) {
3092 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3093 [phba->num_cpus].q, false, true, 0))
3094 goto mcc_cq_free;
3095 } else {
3096 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3097 false, true, 0))
3098 goto mcc_cq_free;
3099 }
3100
3101 /* Alloc MCC queue */
3102 q = &phba->ctrl.mcc_obj.q;
3103 if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3104 goto mcc_cq_destroy;
3105
3106 /* Ask BE to create MCC queue */
35e66019 3107 if (beiscsi_cmd_mccq_create(phba, q, cq))
bfead3b2
JK		 3108 goto mcc_q_free;
3109
3110 return 0;
3111
3112mcc_q_free:
3113 be_queue_free(phba, q);
3114mcc_cq_destroy:
3115 beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3116mcc_cq_free:
3117 be_queue_free(phba, cq);
3118err:
d3ad2bb3 3119 return -ENOMEM;
bfead3b2
JK		 3120}
3121
3122static int find_num_cpus(void)
3123{
3124 int num_cpus = 0;
3125
3126 num_cpus = num_online_cpus();
3127 if (num_cpus >= MAX_CPUS)
3128 num_cpus = MAX_CPUS - 1;
3129
457ff3b7 3130 SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", num_cpus);
bfead3b2 3131 return num_cpus;
6733b39a
JK		 3132}
3133
3134static int hwi_init_port(struct beiscsi_hba *phba)
3135{
3136 struct hwi_controller *phwi_ctrlr;
3137 struct hwi_context_memory *phwi_context;
3138 unsigned int def_pdu_ring_sz;
3139 struct be_ctrl_info *ctrl = &phba->ctrl;
3140 int status;
3141
3142 def_pdu_ring_sz =
3143 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
3144 phwi_ctrlr = phba->phwi_ctrlr;
6733b39a 3145 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 3146 phwi_context->max_eqd = 0;
3147 phwi_context->min_eqd = 0;
3148 phwi_context->cur_eqd = 64;
6733b39a 3149 be_cmd_fw_initialize(&phba->ctrl);
bfead3b2
JK		 3150
3151 status = beiscsi_create_eqs(phba, phwi_context);
6733b39a 3152 if (status != 0) {
457ff3b7 3153 shost_printk(KERN_ERR, phba->shost, "EQ not created\n");
6733b39a
JK		 3154 goto error;
3155 }
3156
bfead3b2
JK		 3157 status = be_mcc_queues_create(phba, phwi_context);
3158 if (status != 0)
3159 goto error;
3160
3161 status = mgmt_check_supported_fw(ctrl, phba);
6733b39a
JK		 3162 if (status != 0) {
3163 shost_printk(KERN_ERR, phba->shost,
457ff3b7 3164 "Unsupported fw version\n");
6733b39a
JK		 3165 goto error;
3166 }
3167
bfead3b2 3168 status = beiscsi_create_cqs(phba, phwi_context);
6733b39a
JK		 3169 if (status != 0) {
3170 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
3171 goto error;
3172 }
3173
3174 status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
3175 def_pdu_ring_sz);
3176 if (status != 0) {
3177 shost_printk(KERN_ERR, phba->shost,
3178 "Default Header not created\n");
3179 goto error;
3180 }
3181
3182 status = beiscsi_create_def_data(phba, phwi_context,
3183 phwi_ctrlr, def_pdu_ring_sz);
3184 if (status != 0) {
3185 shost_printk(KERN_ERR, phba->shost,
3186 "Default Data not created\n");
3187 goto error;
3188 }
3189
3190 status = beiscsi_post_pages(phba);
3191 if (status != 0) {
3192 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
3193 goto error;
3194 }
3195
3196 status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3197 if (status != 0) {
3198 shost_printk(KERN_ERR, phba->shost,
3199 "WRB Rings not created\n");
3200 goto error;
3201 }
3202
3203 SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
3204 return 0;
3205
3206error:
3207 shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
3208 hwi_cleanup(phba);
a49e06d5 3209 return status;
6733b39a
JK		 3210}
3211
6733b39a
JK		 3212static int hwi_init_controller(struct beiscsi_hba *phba)
3213{
3214 struct hwi_controller *phwi_ctrlr;
3215
3216 phwi_ctrlr = phba->phwi_ctrlr;
3217 if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3218 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3219 init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
457ff3b7 3220 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p\n",
6733b39a
JK		 3221 phwi_ctrlr->phwi_ctxt);
3222 } else {
3223 shost_printk(KERN_ERR, phba->shost,
3224 "HWI_MEM_ADDN_CONTEXT is more than one element."
3225 "Failing to load\n");
3226 return -ENOMEM;
3227 }
3228
3229 iscsi_init_global_templates(phba);
3ec78271
JK		 3230 if (beiscsi_init_wrb_handle(phba))
3231 return -ENOMEM;
3232
6733b39a
JK		 3233 hwi_init_async_pdu_ctx(phba);
3234 if (hwi_init_port(phba) != 0) {
3235 shost_printk(KERN_ERR, phba->shost,
3236 "hwi_init_controller failed\n");
3237 return -ENOMEM;
3238 }
3239 return 0;
3240}
3241
3242static void beiscsi_free_mem(struct beiscsi_hba *phba)
3243{
3244 struct be_mem_descriptor *mem_descr;
3245 int i, j;
3246
3247 mem_descr = phba->init_mem;
3248 i = 0;
3249 j = 0;
3250 for (i = 0; i < SE_MEM_MAX; i++) {
3251 for (j = mem_descr->num_elements; j > 0; j--) {
3252 pci_free_consistent(phba->pcidev,
3253 mem_descr->mem_array[j - 1].size,
3254 mem_descr->mem_array[j - 1].virtual_address,
457ff3b7
JK		 3255 (unsigned long)mem_descr->mem_array[j - 1].
3256 bus_address.u.a64.address);
6733b39a
JK		 3257 }
3258 kfree(mem_descr->mem_array);
3259 mem_descr++;
3260 }
3261 kfree(phba->init_mem);
3262 kfree(phba->phwi_ctrlr);
3263}
3264
3265static int beiscsi_init_controller(struct beiscsi_hba *phba)
3266{
3267 int ret = -ENOMEM;
3268
3269 ret = beiscsi_get_memory(phba);
3270 if (ret < 0) {
3271 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
457ff3b7 3272 "Failed in beiscsi_alloc_memory\n");
6733b39a
JK		 3273 return ret;
3274 }
3275
3276 ret = hwi_init_controller(phba);
3277 if (ret)
3278 goto free_init;
3279 SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
3280 return 0;
3281
3282free_init:
3283 beiscsi_free_mem(phba);
a49e06d5 3284 return ret;
6733b39a
JK		 3285}
3286
3287static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3288{
3289 struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3290 struct sgl_handle *psgl_handle;
3291 struct iscsi_sge *pfrag;
3292 unsigned int arr_index, i, idx;
3293
3294 phba->io_sgl_hndl_avbl = 0;
3295 phba->eh_sgl_hndl_avbl = 0;
bfead3b2 3296
6733b39a
JK		 3297 mem_descr_sglh = phba->init_mem;
3298 mem_descr_sglh += HWI_MEM_SGLH;
3299 if (1 == mem_descr_sglh->num_elements) {
3300 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3301 phba->params.ios_per_ctrl,
3302 GFP_KERNEL);
3303 if (!phba->io_sgl_hndl_base) {
3304 shost_printk(KERN_ERR, phba->shost,
3305 "Mem Alloc Failed. Failing to load\n");
3306 return -ENOMEM;
3307 }
3308 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3309 (phba->params.icds_per_ctrl -
3310 phba->params.ios_per_ctrl),
3311 GFP_KERNEL);
3312 if (!phba->eh_sgl_hndl_base) {
3313 kfree(phba->io_sgl_hndl_base);
3314 shost_printk(KERN_ERR, phba->shost,
3315 "Mem Alloc Failed. Failing to load\n");
3316 return -ENOMEM;
3317 }
3318 } else {
3319 shost_printk(KERN_ERR, phba->shost,
3320 "HWI_MEM_SGLH is more than one element."
3321 "Failing to load\n");
3322 return -ENOMEM;
3323 }
3324
3325 arr_index = 0;
3326 idx = 0;
3327 while (idx < mem_descr_sglh->num_elements) {
3328 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3329
3330 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3331 sizeof(struct sgl_handle)); i++) {
3332 if (arr_index < phba->params.ios_per_ctrl) {
3333 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3334 phba->io_sgl_hndl_avbl++;
3335 arr_index++;
3336 } else {
3337 phba->eh_sgl_hndl_base[arr_index -
3338 phba->params.ios_per_ctrl] =
3339 psgl_handle;
3340 arr_index++;
3341 phba->eh_sgl_hndl_avbl++;
3342 }
3343 psgl_handle++;
3344 }
3345 idx++;
3346 }
3347 SE_DEBUG(DBG_LVL_8,
3348 "phba->io_sgl_hndl_avbl=%d"
457ff3b7 3349 "phba->eh_sgl_hndl_avbl=%d\n",
6733b39a
JK		 3350 phba->io_sgl_hndl_avbl,
3351 phba->eh_sgl_hndl_avbl);
3352 mem_descr_sg = phba->init_mem;
3353 mem_descr_sg += HWI_MEM_SGE;
457ff3b7 3354 SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d\n",
6733b39a
JK		 3355 mem_descr_sg->num_elements);
3356 arr_index = 0;
3357 idx = 0;
3358 while (idx < mem_descr_sg->num_elements) {
3359 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3360
3361 for (i = 0;
3362 i < (mem_descr_sg->mem_array[idx].size) /
3363 (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3364 i++) {
3365 if (arr_index < phba->params.ios_per_ctrl)
3366 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3367 else
3368 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3369 phba->params.ios_per_ctrl];
3370 psgl_handle->pfrag = pfrag;
3371 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3372 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3373 pfrag += phba->params.num_sge_per_io;
3374 psgl_handle->sgl_index =
7da50879 3375 phba->fw_config.iscsi_icd_start + arr_index++;
6733b39a
JK		 3376 }
3377 idx++;
3378 }
3379 phba->io_sgl_free_index = 0;
3380 phba->io_sgl_alloc_index = 0;
3381 phba->eh_sgl_free_index = 0;
3382 phba->eh_sgl_alloc_index = 0;
3383 return 0;
3384}
3385
3386static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3387{
3388 int i, new_cid;
3389
c2462288 3390 phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
6733b39a
JK		 3391 GFP_KERNEL);
3392 if (!phba->cid_array) {
3393 shost_printk(KERN_ERR, phba->shost,
3394 "Failed to allocate memory in "
3395 "hba_setup_cid_tbls\n");
3396 return -ENOMEM;
3397 }
c2462288 3398 phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
6733b39a
JK		 3399 phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3400 if (!phba->ep_array) {
3401 shost_printk(KERN_ERR, phba->shost,
3402 "Failed to allocate memory in "
457ff3b7 3403 "hba_setup_cid_tbls\n");
6733b39a
JK		 3404 kfree(phba->cid_array);
3405 return -ENOMEM;
3406 }
7da50879 3407 new_cid = phba->fw_config.iscsi_cid_start;
6733b39a
JK		 3408 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3409 phba->cid_array[i] = new_cid;
3410 new_cid += 2;
3411 }
3412 phba->avlbl_cids = phba->params.cxns_per_ctrl;
3413 return 0;
3414}
3415
238f6b72 3416static void hwi_enable_intr(struct beiscsi_hba *phba)
6733b39a
JK		 3417{
3418 struct be_ctrl_info *ctrl = &phba->ctrl;
3419 struct hwi_controller *phwi_ctrlr;
3420 struct hwi_context_memory *phwi_context;
3421 struct be_queue_info *eq;
3422 u8 __iomem *addr;
bfead3b2 3423 u32 reg, i;
6733b39a
JK		 3424 u32 enabled;
3425
3426 phwi_ctrlr = phba->phwi_ctrlr;
3427 phwi_context = phwi_ctrlr->phwi_ctxt;
3428
6733b39a
JK		 3429 addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3430 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3431 reg = ioread32(addr);
6733b39a
JK		 3432
3433 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3434 if (!enabled) {
3435 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
457ff3b7 3436 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p\n", reg, addr);
6733b39a 3437 iowrite32(reg, addr);
665d6d94
JK		 3438 }
3439
3440 if (!phba->msix_enabled) {
3441 eq = &phwi_context->be_eq[0].q;
3442 SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
3443 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3444 } else {
3445 for (i = 0; i <= phba->num_cpus; i++) {
3446 eq = &phwi_context->be_eq[i].q;
457ff3b7 3447 SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
bfead3b2
JK		 3448 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3449 }
c03af1ae 3450 }
6733b39a
JK		 3451}
3452
3453static void hwi_disable_intr(struct beiscsi_hba *phba)
3454{
3455 struct be_ctrl_info *ctrl = &phba->ctrl;
3456
3457 u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3458 u32 reg = ioread32(addr);
3459
3460 u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3461 if (enabled) {
3462 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3463 iowrite32(reg, addr);
3464 } else
3465 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 3466 "In hwi_disable_intr, Already Disabled\n");
6733b39a
JK		 3467}
3468
c7acc5b8
JK		 3469static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
3470{
0e43895e
MC		 3471 struct be_cmd_get_boot_target_resp *boot_resp;
3472 struct be_cmd_get_session_resp *session_resp;
c7acc5b8
JK		 3473 struct be_mcc_wrb *wrb;
3474 struct be_dma_mem nonemb_cmd;
3475 unsigned int tag, wrb_num;
3476 unsigned short status, extd_status;
3477 struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q;
f457a46f 3478 int ret = -ENOMEM;
c7acc5b8 3479
0e43895e 3480 tag = mgmt_get_boot_target(phba);
c7acc5b8 3481 if (!tag) {
0e43895e 3482 SE_DEBUG(DBG_LVL_1, "beiscsi_get_boot_info Failed\n");
c7acc5b8
JK		 3483 return -EAGAIN;
3484 } else
3485 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3486 phba->ctrl.mcc_numtag[tag]);
3487
3488 wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3489 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3490 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3491 if (status || extd_status) {
0e43895e 3492 SE_DEBUG(DBG_LVL_1, "beiscsi_get_boot_info Failed"
c7acc5b8
JK		 3493 " status = %d extd_status = %d\n",
3494 status, extd_status);
3495 free_mcc_tag(&phba->ctrl, tag);
3496 return -EBUSY;
3497 }
3498 wrb = queue_get_wrb(mccq, wrb_num);
3499 free_mcc_tag(&phba->ctrl, tag);
3500 boot_resp = embedded_payload(wrb);
3501
3502 if (boot_resp->boot_session_handle < 0) {
f457a46f 3503 shost_printk(KERN_INFO, phba->shost, "No Boot Session.\n");
c7acc5b8
JK		 3504 return -ENXIO;
3505 }
3506
3507 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
3508 sizeof(*session_resp),
3509 &nonemb_cmd.dma);
3510 if (nonemb_cmd.va == NULL) {
3511 SE_DEBUG(DBG_LVL_1,
3512 "Failed to allocate memory for"
3513 "beiscsi_get_session_info\n");
3514 return -ENOMEM;
3515 }
3516
3517 memset(nonemb_cmd.va, 0, sizeof(*session_resp));
0e43895e
MC		 3518 tag = mgmt_get_session_info(phba, boot_resp->boot_session_handle,
3519 &nonemb_cmd);
c7acc5b8
JK		 3520 if (!tag) {
3521 SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info"
3522 " Failed\n");
3523 goto boot_freemem;
3524 } else
3525 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3526 phba->ctrl.mcc_numtag[tag]);
3527
3528 wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3529 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3530 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3531 if (status || extd_status) {
3532 SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info Failed"
3533 " status = %d extd_status = %d\n",
3534 status, extd_status);
3535 free_mcc_tag(&phba->ctrl, tag);
3536 goto boot_freemem;
3537 }
3538 wrb = queue_get_wrb(mccq, wrb_num);
3539 free_mcc_tag(&phba->ctrl, tag);
3540 session_resp = nonemb_cmd.va ;
f457a46f 3541
c7acc5b8
JK		 3542 memcpy(&phba->boot_sess, &session_resp->session_info,
3543 sizeof(struct mgmt_session_info));
f457a46f
MC		 3544 ret = 0;
3545
c7acc5b8
JK		 3546boot_freemem:
3547 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
3548 nonemb_cmd.va, nonemb_cmd.dma);
f457a46f
MC		 3549 return ret;
3550}
3551
3552static void beiscsi_boot_release(void *data)
3553{
3554 struct beiscsi_hba *phba = data;
3555
3556 scsi_host_put(phba->shost);
3557}
3558
3559static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
3560{
3561 struct iscsi_boot_kobj *boot_kobj;
3562
3563 /* get boot info using mgmt cmd */
3564 if (beiscsi_get_boot_info(phba))
3565 /* Try to see if we can carry on without this */
3566 return 0;
3567
3568 phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
3569 if (!phba->boot_kset)
3570 return -ENOMEM;
3571
3572 /* get a ref because the show function will ref the phba */
3573 if (!scsi_host_get(phba->shost))
3574 goto free_kset;
3575 boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
3576 beiscsi_show_boot_tgt_info,
3577 beiscsi_tgt_get_attr_visibility,
3578 beiscsi_boot_release);
3579 if (!boot_kobj)
3580 goto put_shost;
3581
3582 if (!scsi_host_get(phba->shost))
3583 goto free_kset;
3584 boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
3585 beiscsi_show_boot_ini_info,
3586 beiscsi_ini_get_attr_visibility,
3587 beiscsi_boot_release);
3588 if (!boot_kobj)
3589 goto put_shost;
3590
3591 if (!scsi_host_get(phba->shost))
3592 goto free_kset;
3593 boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
3594 beiscsi_show_boot_eth_info,
3595 beiscsi_eth_get_attr_visibility,
3596 beiscsi_boot_release);
3597 if (!boot_kobj)
3598 goto put_shost;
3599 return 0;
3600
3601put_shost:
3602 scsi_host_put(phba->shost);
3603free_kset:
3604 iscsi_boot_destroy_kset(phba->boot_kset);
c7acc5b8
JK		 3605 return -ENOMEM;
3606}
3607
6733b39a
JK		 3608static int beiscsi_init_port(struct beiscsi_hba *phba)
3609{
3610 int ret;
3611
3612 ret = beiscsi_init_controller(phba);
3613 if (ret < 0) {
3614 shost_printk(KERN_ERR, phba->shost,
3615 "beiscsi_dev_probe - Failed in"
457ff3b7 3616 "beiscsi_init_controller\n");
6733b39a
JK		 3617 return ret;
3618 }
3619 ret = beiscsi_init_sgl_handle(phba);
3620 if (ret < 0) {
3621 shost_printk(KERN_ERR, phba->shost,
3622 "beiscsi_dev_probe - Failed in"
457ff3b7 3623 "beiscsi_init_sgl_handle\n");
6733b39a
JK		 3624 goto do_cleanup_ctrlr;
3625 }
3626
3627 if (hba_setup_cid_tbls(phba)) {
3628 shost_printk(KERN_ERR, phba->shost,
3629 "Failed in hba_setup_cid_tbls\n");
3630 kfree(phba->io_sgl_hndl_base);
3631 kfree(phba->eh_sgl_hndl_base);
3632 goto do_cleanup_ctrlr;
3633 }
3634
3635 return ret;
3636
3637do_cleanup_ctrlr:
3638 hwi_cleanup(phba);
3639 return ret;
3640}
3641
3642static void hwi_purge_eq(struct beiscsi_hba *phba)
3643{
3644 struct hwi_controller *phwi_ctrlr;
3645 struct hwi_context_memory *phwi_context;
3646 struct be_queue_info *eq;
3647 struct be_eq_entry *eqe = NULL;
bfead3b2 3648 int i, eq_msix;
756d29c8 3649 unsigned int num_processed;
6733b39a
JK		 3650
3651 phwi_ctrlr = phba->phwi_ctrlr;
3652 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 3653 if (phba->msix_enabled)
3654 eq_msix = 1;
3655 else
3656 eq_msix = 0;
6733b39a 3657
bfead3b2
JK		 3658 for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3659 eq = &phwi_context->be_eq[i].q;
6733b39a 3660 eqe = queue_tail_node(eq);
756d29c8 3661 num_processed = 0;
bfead3b2
JK		 3662 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3663 & EQE_VALID_MASK) {
3664 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3665 queue_tail_inc(eq);
3666 eqe = queue_tail_node(eq);
756d29c8 3667 num_processed++;
bfead3b2 3668 }
756d29c8
JK		 3669
3670 if (num_processed)
3671 hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
6733b39a
JK		 3672 }
3673}
3674
3675static void beiscsi_clean_port(struct beiscsi_hba *phba)
3676{
03a12310 3677 int mgmt_status;
6733b39a
JK		 3678
3679 mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3680 if (mgmt_status)
3681 shost_printk(KERN_WARNING, phba->shost,
457ff3b7 3682 "mgmt_epfw_cleanup FAILED\n");
756d29c8 3683
6733b39a 3684 hwi_purge_eq(phba);
756d29c8 3685 hwi_cleanup(phba);
6733b39a
JK		 3686 kfree(phba->io_sgl_hndl_base);
3687 kfree(phba->eh_sgl_hndl_base);
3688 kfree(phba->cid_array);
3689 kfree(phba->ep_array);
3690}
3691
1282ab76
MC		 3692static void beiscsi_cleanup_task(struct iscsi_task *task)
3693{
3694 struct beiscsi_io_task *io_task = task->dd_data;
3695 struct iscsi_conn *conn = task->conn;
3696 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3697 struct beiscsi_hba *phba = beiscsi_conn->phba;
3698 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3699 struct hwi_wrb_context *pwrb_context;
3700 struct hwi_controller *phwi_ctrlr;
3701
3702 phwi_ctrlr = phba->phwi_ctrlr;
3703 pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
3704 - phba->fw_config.iscsi_cid_start];
3705
3706 if (io_task->cmd_bhs) {
3707 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3708 io_task->bhs_pa.u.a64.address);
3709 io_task->cmd_bhs = NULL;
3710 }
3711
3712 if (task->sc) {
3713 if (io_task->pwrb_handle) {
3714 free_wrb_handle(phba, pwrb_context,
3715 io_task->pwrb_handle);
3716 io_task->pwrb_handle = NULL;
3717 }
3718
3719 if (io_task->psgl_handle) {
3720 spin_lock(&phba->io_sgl_lock);
3721 free_io_sgl_handle(phba, io_task->psgl_handle);
3722 spin_unlock(&phba->io_sgl_lock);
3723 io_task->psgl_handle = NULL;
3724 }
3725 } else {
3726 if (!beiscsi_conn->login_in_progress) {
3727 if (io_task->pwrb_handle) {
3728 free_wrb_handle(phba, pwrb_context,
3729 io_task->pwrb_handle);
3730 io_task->pwrb_handle = NULL;
3731 }
3732 if (io_task->psgl_handle) {
3733 spin_lock(&phba->mgmt_sgl_lock);
3734 free_mgmt_sgl_handle(phba,
3735 io_task->psgl_handle);
3736 spin_unlock(&phba->mgmt_sgl_lock);
3737 io_task->psgl_handle = NULL;
3738 }
3739 }
3740 }
3741}
3742
6733b39a
JK		 3743void
3744beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3745 struct beiscsi_offload_params *params)
3746{
3747 struct wrb_handle *pwrb_handle;
3748 struct iscsi_target_context_update_wrb *pwrb = NULL;
3749 struct be_mem_descriptor *mem_descr;
3750 struct beiscsi_hba *phba = beiscsi_conn->phba;
1282ab76
MC		 3751 struct iscsi_task *task = beiscsi_conn->task;
3752 struct iscsi_session *session = task->conn->session;
6733b39a
JK		 3753 u32 doorbell = 0;
3754
3755 /*
3756 * We can always use 0 here because it is reserved by libiscsi for
3757 * login/startup related tasks.
3758 */
1282ab76
MC		 3759 beiscsi_conn->login_in_progress = 0;
3760 spin_lock_bh(&session->lock);
3761 beiscsi_cleanup_task(task);
3762 spin_unlock_bh(&session->lock);
3763
7da50879 3764 pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
d5431488 3765 phba->fw_config.iscsi_cid_start));
6733b39a
JK		 3766 pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
3767 memset(pwrb, 0, sizeof(*pwrb));
3768 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3769 max_burst_length, pwrb, params->dw[offsetof
3770 (struct amap_beiscsi_offload_params,
3771 max_burst_length) / 32]);
3772 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3773 max_send_data_segment_length, pwrb,
3774 params->dw[offsetof(struct amap_beiscsi_offload_params,
3775 max_send_data_segment_length) / 32]);
3776 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3777 first_burst_length,
3778 pwrb,
3779 params->dw[offsetof(struct amap_beiscsi_offload_params,
3780 first_burst_length) / 32]);
3781
3782 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
3783 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3784 erl) / 32] & OFFLD_PARAMS_ERL));
3785 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
3786 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3787 dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
3788 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
3789 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3790 hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
3791 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
3792 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3793 ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
3794 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
3795 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3796 imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
3797 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
3798 pwrb,
3799 (params->dw[offsetof(struct amap_beiscsi_offload_params,
3800 exp_statsn) / 32] + 1));
3801 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
3802 0x7);
3803 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
3804 pwrb, pwrb_handle->wrb_index);
3805 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
3806 pwrb, pwrb_handle->nxt_wrb_index);
3807 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3808 session_state, pwrb, 0);
3809 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
3810 pwrb, 1);
3811 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
3812 pwrb, 0);
3813 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
3814 0);
3815
3816 mem_descr = phba->init_mem;
3817 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
3818
3819 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3820 pad_buffer_addr_hi, pwrb,
3821 mem_descr->mem_array[0].bus_address.u.a32.address_hi);
3822 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3823 pad_buffer_addr_lo, pwrb,
3824 mem_descr->mem_array[0].bus_address.u.a32.address_lo);
3825
3826 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
3827
3828 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
32951dd8 3829 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
bfead3b2 3830 << DB_DEF_PDU_WRB_INDEX_SHIFT;
6733b39a
JK		 3831 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3832
3833 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3834}
3835
3836static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
3837 int *index, int *age)
3838{
bfead3b2 3839 *index = (int)itt;
6733b39a
JK		 3840 if (age)
3841 *age = conn->session->age;
3842}
3843
3844/**
3845 * beiscsi_alloc_pdu - allocates pdu and related resources
3846 * @task: libiscsi task
3847 * @opcode: opcode of pdu for task
3848 *
3849 * This is called with the session lock held. It will allocate
3850 * the wrb and sgl if needed for the command. And it will prep
3851 * the pdu's itt. beiscsi_parse_pdu will later translate
3852 * the pdu itt to the libiscsi task itt.
3853 */
3854static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
3855{
3856 struct beiscsi_io_task *io_task = task->dd_data;
3857 struct iscsi_conn *conn = task->conn;
3858 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3859 struct beiscsi_hba *phba = beiscsi_conn->phba;
3860 struct hwi_wrb_context *pwrb_context;
3861 struct hwi_controller *phwi_ctrlr;
3862 itt_t itt;
2afc95bf
JK		 3863 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3864 dma_addr_t paddr;
6733b39a 3865
2afc95bf 3866 io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
bc7accec 3867 GFP_ATOMIC, &paddr);
2afc95bf
JK		 3868 if (!io_task->cmd_bhs)
3869 return -ENOMEM;
2afc95bf 3870 io_task->bhs_pa.u.a64.address = paddr;
bfead3b2 3871 io_task->libiscsi_itt = (itt_t)task->itt;
6733b39a
JK		 3872 io_task->conn = beiscsi_conn;
3873
3874 task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
3875 task->hdr_max = sizeof(struct be_cmd_bhs);
d2cecf0d 3876 io_task->psgl_handle = NULL;
3ec78271 3877 io_task->pwrb_handle = NULL;
6733b39a
JK		 3878
3879 if (task->sc) {
3880 spin_lock(&phba->io_sgl_lock);
3881 io_task->psgl_handle = alloc_io_sgl_handle(phba);
3882 spin_unlock(&phba->io_sgl_lock);
2afc95bf
JK		 3883 if (!io_task->psgl_handle)
3884 goto free_hndls;
d2cecf0d
JK		 3885 io_task->pwrb_handle = alloc_wrb_handle(phba,
3886 beiscsi_conn->beiscsi_conn_cid -
3887 phba->fw_config.iscsi_cid_start);
3888 if (!io_task->pwrb_handle)
3889 goto free_io_hndls;
6733b39a
JK		 3890 } else {
3891 io_task->scsi_cmnd = NULL;
d7aea67b 3892 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
6733b39a
JK		 3893 if (!beiscsi_conn->login_in_progress) {
3894 spin_lock(&phba->mgmt_sgl_lock);
3895 io_task->psgl_handle = (struct sgl_handle *)
3896 alloc_mgmt_sgl_handle(phba);
3897 spin_unlock(&phba->mgmt_sgl_lock);
2afc95bf
JK		 3898 if (!io_task->psgl_handle)
3899 goto free_hndls;
3900
6733b39a
JK		 3901 beiscsi_conn->login_in_progress = 1;
3902 beiscsi_conn->plogin_sgl_handle =
3903 io_task->psgl_handle;
d2cecf0d
JK		 3904 io_task->pwrb_handle =
3905 alloc_wrb_handle(phba,
3906 beiscsi_conn->beiscsi_conn_cid -
3907 phba->fw_config.iscsi_cid_start);
3908 if (!io_task->pwrb_handle)
3909 goto free_io_hndls;
3910 beiscsi_conn->plogin_wrb_handle =
3911 io_task->pwrb_handle;
3912
6733b39a
JK		 3913 } else {
3914 io_task->psgl_handle =
3915 beiscsi_conn->plogin_sgl_handle;
d2cecf0d
JK		 3916 io_task->pwrb_handle =
3917 beiscsi_conn->plogin_wrb_handle;
6733b39a 3918 }
1282ab76 3919 beiscsi_conn->task = task;
6733b39a
JK		 3920 } else {
3921 spin_lock(&phba->mgmt_sgl_lock);
3922 io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
3923 spin_unlock(&phba->mgmt_sgl_lock);
2afc95bf
JK		 3924 if (!io_task->psgl_handle)
3925 goto free_hndls;
d2cecf0d
JK		 3926 io_task->pwrb_handle =
3927 alloc_wrb_handle(phba,
3928 beiscsi_conn->beiscsi_conn_cid -
3929 phba->fw_config.iscsi_cid_start);
3930 if (!io_task->pwrb_handle)
3931 goto free_mgmt_hndls;
3932
6733b39a
JK		 3933 }
3934 }
bfead3b2
JK		 3935 itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
3936 wrb_index << 16) | (unsigned int)
3937 (io_task->psgl_handle->sgl_index));
32951dd8 3938 io_task->pwrb_handle->pio_handle = task;
bfead3b2 3939
6733b39a
JK		 3940 io_task->cmd_bhs->iscsi_hdr.itt = itt;
3941 return 0;
2afc95bf 3942
d2cecf0d
JK		 3943free_io_hndls:
3944 spin_lock(&phba->io_sgl_lock);
3945 free_io_sgl_handle(phba, io_task->psgl_handle);
3946 spin_unlock(&phba->io_sgl_lock);
3947 goto free_hndls;
3948free_mgmt_hndls:
3949 spin_lock(&phba->mgmt_sgl_lock);
3950 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3951 spin_unlock(&phba->mgmt_sgl_lock);
2afc95bf
JK		 3952free_hndls:
3953 phwi_ctrlr = phba->phwi_ctrlr;
7da50879
JK		 3954 pwrb_context = &phwi_ctrlr->wrb_context[
3955 beiscsi_conn->beiscsi_conn_cid -
3956 phba->fw_config.iscsi_cid_start];
d2cecf0d
JK		 3957 if (io_task->pwrb_handle)
3958 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
2afc95bf
JK		 3959 io_task->pwrb_handle = NULL;
3960 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3961 io_task->bhs_pa.u.a64.address);
1282ab76 3962 io_task->cmd_bhs = NULL;
457ff3b7 3963 SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed\n");
2afc95bf 3964 return -ENOMEM;
6733b39a
JK		 3965}
3966
6733b39a
JK		 3967static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3968 unsigned int num_sg, unsigned int xferlen,
3969 unsigned int writedir)
3970{
3971
3972 struct beiscsi_io_task *io_task = task->dd_data;
3973 struct iscsi_conn *conn = task->conn;
3974 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3975 struct beiscsi_hba *phba = beiscsi_conn->phba;
3976 struct iscsi_wrb *pwrb = NULL;
3977 unsigned int doorbell = 0;
3978
3979 pwrb = io_task->pwrb_handle->pwrb;
6733b39a
JK		 3980 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3981 io_task->bhs_len = sizeof(struct be_cmd_bhs);
3982
3983 if (writedir) {
32951dd8
JK		 3984 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3985 INI_WR_CMD);
6733b39a 3986 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
6733b39a 3987 } else {
32951dd8
JK		 3988 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3989 INI_RD_CMD);
6733b39a
JK		 3990 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3991 }
6733b39a
JK		 3992
3993 AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
dc63aac6
JK		 3994 cpu_to_be16(*(unsigned short *)
3995 &io_task->cmd_bhs->iscsi_hdr.lun));
6733b39a
JK		 3996 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3997 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3998 io_task->pwrb_handle->wrb_index);
3999 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4000 be32_to_cpu(task->cmdsn));
4001 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4002 io_task->psgl_handle->sgl_index);
4003
4004 hwi_write_sgl(pwrb, sg, num_sg, io_task);
4005
4006 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4007 io_task->pwrb_handle->nxt_wrb_index);
4008 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4009
4010 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
32951dd8 4011 doorbell |= (io_task->pwrb_handle->wrb_index &
6733b39a
JK		 4012 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4013 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4014
4015 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4016 return 0;
4017}
4018
4019static int beiscsi_mtask(struct iscsi_task *task)
4020{
dafab8e0 4021 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 4022 struct iscsi_conn *conn = task->conn;
4023 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4024 struct beiscsi_hba *phba = beiscsi_conn->phba;
4025 struct iscsi_wrb *pwrb = NULL;
4026 unsigned int doorbell = 0;
dafab8e0 4027 unsigned int cid;
6733b39a 4028
bfead3b2 4029 cid = beiscsi_conn->beiscsi_conn_cid;
6733b39a 4030 pwrb = io_task->pwrb_handle->pwrb;
caf818f1 4031 memset(pwrb, 0, sizeof(*pwrb));
6733b39a
JK		 4032 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4033 be32_to_cpu(task->cmdsn));
4034 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4035 io_task->pwrb_handle->wrb_index);
4036 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4037 io_task->psgl_handle->sgl_index);
dafab8e0 4038
6733b39a
JK		 4039 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4040 case ISCSI_OP_LOGIN:
32951dd8
JK		 4041 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4042 TGT_DM_CMD);
6733b39a
JK		 4043 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4044 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
4045 hwi_write_buffer(pwrb, task);
4046 break;
4047 case ISCSI_OP_NOOP_OUT:
1390b01b
JK		 4048 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
4049 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4050 TGT_DM_CMD);
4051 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt,
4052 pwrb, 0);
685e16fd 4053 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1);
1390b01b
JK		 4054 } else {
4055 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4056 INI_RD_CMD);
685e16fd 4057 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
1390b01b 4058 }
6733b39a
JK		 4059 hwi_write_buffer(pwrb, task);
4060 break;
4061 case ISCSI_OP_TEXT:
32951dd8 4062 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
b30c6dab 4063 TGT_DM_CMD);
0ecb0b45 4064 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
6733b39a
JK		 4065 hwi_write_buffer(pwrb, task);
4066 break;
4067 case ISCSI_OP_SCSI_TMFUNC:
32951dd8
JK		 4068 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4069 INI_TMF_CMD);
6733b39a
JK		 4070 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4071 hwi_write_buffer(pwrb, task);
4072 break;
4073 case ISCSI_OP_LOGOUT:
4074 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4075 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
dafab8e0 4076 HWH_TYPE_LOGOUT);
6733b39a
JK		 4077 hwi_write_buffer(pwrb, task);
4078 break;
4079
4080 default:
457ff3b7 4081 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported\n",
6733b39a
JK		 4082 task->hdr->opcode & ISCSI_OPCODE_MASK);
4083 return -EINVAL;
4084 }
4085
4086 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
51a46250 4087 task->data_count);
6733b39a
JK		 4088 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4089 io_task->pwrb_handle->nxt_wrb_index);
4090 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4091
bfead3b2 4092 doorbell |= cid & DB_WRB_POST_CID_MASK;
32951dd8 4093 doorbell |= (io_task->pwrb_handle->wrb_index &
6733b39a
JK		 4094 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4095 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4096 iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4097 return 0;
4098}
4099
4100static int beiscsi_task_xmit(struct iscsi_task *task)
4101{
6733b39a
JK		 4102 struct beiscsi_io_task *io_task = task->dd_data;
4103 struct scsi_cmnd *sc = task->sc;
6733b39a
JK		 4104 struct scatterlist *sg;
4105 int num_sg;
4106 unsigned int writedir = 0, xferlen = 0;
4107
6733b39a
JK		 4108 if (!sc)
4109 return beiscsi_mtask(task);
4110
4111 io_task->scsi_cmnd = sc;
4112 num_sg = scsi_dma_map(sc);
4113 if (num_sg < 0) {
4114 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
4115 return num_sg;
4116 }
6733b39a
JK		 4117 xferlen = scsi_bufflen(sc);
4118 sg = scsi_sglist(sc);
4119 if (sc->sc_data_direction == DMA_TO_DEVICE) {
4120 writedir = 1;
457ff3b7 4121 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x\n",
6733b39a
JK		 4122 task->imm_count);
4123 } else
4124 writedir = 0;
4125 return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
4126}
4127
ffce3e2e
JK		 4128/**
4129 * beiscsi_bsg_request - handle bsg request from ISCSI transport
4130 * @job: job to handle
4131 */
4132static int beiscsi_bsg_request(struct bsg_job *job)
4133{
4134 struct Scsi_Host *shost;
4135 struct beiscsi_hba *phba;
4136 struct iscsi_bsg_request *bsg_req = job->request;
4137 int rc = -EINVAL;
4138 unsigned int tag;
4139 struct be_dma_mem nonemb_cmd;
4140 struct be_cmd_resp_hdr *resp;
4141 struct iscsi_bsg_reply *bsg_reply = job->reply;
4142 unsigned short status, extd_status;
4143
4144 shost = iscsi_job_to_shost(job);
4145 phba = iscsi_host_priv(shost);
4146
4147 switch (bsg_req->msgcode) {
4148 case ISCSI_BSG_HST_VENDOR:
4149 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
4150 job->request_payload.payload_len,
4151 &nonemb_cmd.dma);
4152 if (nonemb_cmd.va == NULL) {
4153 SE_DEBUG(DBG_LVL_1, "Failed to allocate memory for "
4154 "beiscsi_bsg_request\n");
4155 return -EIO;
4156 }
4157 tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,
4158 &nonemb_cmd);
4159 if (!tag) {
4160 SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed\n");
4161 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4162 nonemb_cmd.va, nonemb_cmd.dma);
4163 return -EAGAIN;
4164 } else
4165 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
4166 phba->ctrl.mcc_numtag[tag]);
4167 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
4168 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
4169 free_mcc_tag(&phba->ctrl, tag);
4170 resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va;
4171 sg_copy_from_buffer(job->reply_payload.sg_list,
4172 job->reply_payload.sg_cnt,
4173 nonemb_cmd.va, (resp->response_length
4174 + sizeof(*resp)));
4175 bsg_reply->reply_payload_rcv_len = resp->response_length;
4176 bsg_reply->result = status;
4177 bsg_job_done(job, bsg_reply->result,
4178 bsg_reply->reply_payload_rcv_len);
4179 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4180 nonemb_cmd.va, nonemb_cmd.dma);
4181 if (status || extd_status) {
4182 SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed"
4183 " status = %d extd_status = %d\n",
4184 status, extd_status);
4185 return -EIO;
4186 }
4187 break;
4188
4189 default:
4190 SE_DEBUG(DBG_LVL_1, "Unsupported bsg command: 0x%x\n",
4191 bsg_req->msgcode);
4192 break;
4193 }
4194
4195 return rc;
4196}
4197
25602c97 4198static void beiscsi_quiesce(struct beiscsi_hba *phba)
6733b39a 4199{
bfead3b2
JK		 4200 struct hwi_controller *phwi_ctrlr;
4201 struct hwi_context_memory *phwi_context;
4202 struct be_eq_obj *pbe_eq;
4203 unsigned int i, msix_vec;
e9b91193
JK		 4204 u8 *real_offset = 0;
4205 u32 value = 0;
6733b39a 4206
bfead3b2
JK		 4207 phwi_ctrlr = phba->phwi_ctrlr;
4208 phwi_context = phwi_ctrlr->phwi_ctxt;
6733b39a 4209 hwi_disable_intr(phba);
bfead3b2
JK		 4210 if (phba->msix_enabled) {
4211 for (i = 0; i <= phba->num_cpus; i++) {
4212 msix_vec = phba->msix_entries[i].vector;
4213 free_irq(msix_vec, &phwi_context->be_eq[i]);
8fcfb210 4214 kfree(phba->msi_name[i]);
bfead3b2
JK		 4215 }
4216 } else
4217 if (phba->pcidev->irq)
4218 free_irq(phba->pcidev->irq, phba);
4219 pci_disable_msix(phba->pcidev);
6733b39a
JK		 4220 destroy_workqueue(phba->wq);
4221 if (blk_iopoll_enabled)
bfead3b2
JK		 4222 for (i = 0; i < phba->num_cpus; i++) {
4223 pbe_eq = &phwi_context->be_eq[i];
4224 blk_iopoll_disable(&pbe_eq->iopoll);
4225 }
6733b39a
JK		 4226
4227 beiscsi_clean_port(phba);
4228 beiscsi_free_mem(phba);
e9b91193
JK		 4229 real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4230
4231 value = readl((void *)real_offset);
4232
4233 if (value & 0x00010000) {
4234 value &= 0xfffeffff;
4235 writel(value, (void *)real_offset);
4236 }
6733b39a
JK		 4237 beiscsi_unmap_pci_function(phba);
4238 pci_free_consistent(phba->pcidev,
4239 phba->ctrl.mbox_mem_alloced.size,
4240 phba->ctrl.mbox_mem_alloced.va,
4241 phba->ctrl.mbox_mem_alloced.dma);
25602c97
JK		 4242}
4243
4244static void beiscsi_remove(struct pci_dev *pcidev)
4245{
4246
4247 struct beiscsi_hba *phba = NULL;
4248
4249 phba = pci_get_drvdata(pcidev);
4250 if (!phba) {
4251 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
4252 return;
4253 }
4254
0e43895e 4255 beiscsi_destroy_def_ifaces(phba);
25602c97 4256 beiscsi_quiesce(phba);
9d045163 4257 iscsi_boot_destroy_kset(phba->boot_kset);
6733b39a
JK		 4258 iscsi_host_remove(phba->shost);
4259 pci_dev_put(phba->pcidev);
4260 iscsi_host_free(phba->shost);
8dce69ff 4261 pci_disable_device(pcidev);
6733b39a
JK		 4262}
4263
25602c97
JK		 4264static void beiscsi_shutdown(struct pci_dev *pcidev)
4265{
4266
4267 struct beiscsi_hba *phba = NULL;
4268
4269 phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
4270 if (!phba) {
4271 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n");
4272 return;
4273 }
4274
4275 beiscsi_quiesce(phba);
8dce69ff 4276 pci_disable_device(pcidev);
25602c97
JK		 4277}
4278
bfead3b2
JK		 4279static void beiscsi_msix_enable(struct beiscsi_hba *phba)
4280{
4281 int i, status;
4282
4283 for (i = 0; i <= phba->num_cpus; i++)
4284 phba->msix_entries[i].entry = i;
4285
4286 status = pci_enable_msix(phba->pcidev, phba->msix_entries,
4287 (phba->num_cpus + 1));
4288 if (!status)
4289 phba->msix_enabled = true;
4290
4291 return;
4292}
4293
6733b39a
JK		 4294static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
4295 const struct pci_device_id *id)
4296{
4297 struct beiscsi_hba *phba = NULL;
bfead3b2
JK		 4298 struct hwi_controller *phwi_ctrlr;
4299 struct hwi_context_memory *phwi_context;
4300 struct be_eq_obj *pbe_eq;
238f6b72 4301 int ret, num_cpus, i;
e9b91193
JK		 4302 u8 *real_offset = 0;
4303 u32 value = 0;
6733b39a
JK		 4304
4305 ret = beiscsi_enable_pci(pcidev);
4306 if (ret < 0) {
82284c09
DC		 4307 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
4308 " Failed to enable pci device\n");
6733b39a
JK		 4309 return ret;
4310 }
4311
4312 phba = beiscsi_hba_alloc(pcidev);
4313 if (!phba) {
4314 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
457ff3b7 4315 " Failed in beiscsi_hba_alloc\n");
6733b39a
JK		 4316 goto disable_pci;
4317 }
4318
f98c96b0
JK		 4319 switch (pcidev->device) {
4320 case BE_DEVICE_ID1:
4321 case OC_DEVICE_ID1:
4322 case OC_DEVICE_ID2:
4323 phba->generation = BE_GEN2;
4324 break;
4325 case BE_DEVICE_ID2:
4326 case OC_DEVICE_ID3:
4327 phba->generation = BE_GEN3;
4328 break;
4329 default:
4330 phba->generation = 0;
4331 }
4332
bfead3b2
JK		 4333 if (enable_msix)
4334 num_cpus = find_num_cpus();
4335 else
4336 num_cpus = 1;
4337 phba->num_cpus = num_cpus;
457ff3b7 4338 SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", phba->num_cpus);
bfead3b2 4339
b547f2d6 4340 if (enable_msix) {
bfead3b2 4341 beiscsi_msix_enable(phba);
b547f2d6
JK		 4342 if (!phba->msix_enabled)
4343 phba->num_cpus = 1;
4344 }
6733b39a
JK		 4345 ret = be_ctrl_init(phba, pcidev);
4346 if (ret) {
4347 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4348 "Failed in be_ctrl_init\n");
4349 goto hba_free;
4350 }
4351
e9b91193
JK		 4352 if (!num_hba) {
4353 real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4354 value = readl((void *)real_offset);
4355 if (value & 0x00010000) {
4356 gcrashmode++;
4357 shost_printk(KERN_ERR, phba->shost,
4358 "Loading Driver in crashdump mode\n");
e5285860 4359 ret = beiscsi_cmd_reset_function(phba);
e9b91193
JK		 4360 if (ret) {
4361 shost_printk(KERN_ERR, phba->shost,
4362 "Reset Failed. Aborting Crashdump\n");
4363 goto hba_free;
4364 }
4365 ret = be_chk_reset_complete(phba);
4366 if (ret) {
4367 shost_printk(KERN_ERR, phba->shost,
4368 "Failed to get out of reset."
4369 "Aborting Crashdump\n");
4370 goto hba_free;
4371 }
4372 } else {
4373 value |= 0x00010000;
4374 writel(value, (void *)real_offset);
4375 num_hba++;
4376 }
4377 }
4378
6733b39a
JK		 4379 spin_lock_init(&phba->io_sgl_lock);
4380 spin_lock_init(&phba->mgmt_sgl_lock);
4381 spin_lock_init(&phba->isr_lock);
7da50879
JK		 4382 ret = mgmt_get_fw_config(&phba->ctrl, phba);
4383 if (ret != 0) {
4384 shost_printk(KERN_ERR, phba->shost,
4385 "Error getting fw config\n");
4386 goto free_port;
4387 }
4388 phba->shost->max_id = phba->fw_config.iscsi_cid_count;
6733b39a 4389 beiscsi_get_params(phba);
aa874f07 4390 phba->shost->can_queue = phba->params.ios_per_ctrl;
6733b39a
JK		 4391 ret = beiscsi_init_port(phba);
4392 if (ret < 0) {
4393 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4394 "Failed in beiscsi_init_port\n");
4395 goto free_port;
4396 }
4397
756d29c8
JK		 4398 for (i = 0; i < MAX_MCC_CMD ; i++) {
4399 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
4400 phba->ctrl.mcc_tag[i] = i + 1;
4401 phba->ctrl.mcc_numtag[i + 1] = 0;
4402 phba->ctrl.mcc_tag_available++;
4403 }
4404
4405 phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
4406
6733b39a
JK		 4407 snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
4408 phba->shost->host_no);
278274d5 4409 phba->wq = alloc_workqueue(phba->wq_name, WQ_MEM_RECLAIM, 1);
6733b39a
JK		 4410 if (!phba->wq) {
4411 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4412 "Failed to allocate work queue\n");
4413 goto free_twq;
4414 }
4415
4416 INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
4417
bfead3b2
JK		 4418 phwi_ctrlr = phba->phwi_ctrlr;
4419 phwi_context = phwi_ctrlr->phwi_ctxt;
6733b39a 4420 if (blk_iopoll_enabled) {
bfead3b2
JK		 4421 for (i = 0; i < phba->num_cpus; i++) {
4422 pbe_eq = &phwi_context->be_eq[i];
4423 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
4424 be_iopoll);
4425 blk_iopoll_enable(&pbe_eq->iopoll);
4426 }
6733b39a 4427 }
6733b39a
JK		 4428 ret = beiscsi_init_irqs(phba);
4429 if (ret < 0) {
4430 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4431 "Failed to beiscsi_init_irqs\n");
4432 goto free_blkenbld;
4433 }
238f6b72 4434 hwi_enable_intr(phba);
f457a46f
MC		 4435
4436 if (beiscsi_setup_boot_info(phba))
4437 /*
4438 * log error but continue, because we may not be using
4439 * iscsi boot.
4440 */
4441 shost_printk(KERN_ERR, phba->shost, "Could not set up "
0e43895e 4442 "iSCSI boot info.\n");
f457a46f 4443
0e43895e 4444 beiscsi_create_def_ifaces(phba);
457ff3b7 4445 SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED\n\n\n");
6733b39a
JK		 4446 return 0;
4447
6733b39a
JK		 4448free_blkenbld:
4449 destroy_workqueue(phba->wq);
4450 if (blk_iopoll_enabled)
bfead3b2
JK		 4451 for (i = 0; i < phba->num_cpus; i++) {
4452 pbe_eq = &phwi_context->be_eq[i];
4453 blk_iopoll_disable(&pbe_eq->iopoll);
4454 }
6733b39a
JK		 4455free_twq:
4456 beiscsi_clean_port(phba);
4457 beiscsi_free_mem(phba);
4458free_port:
e9b91193
JK		 4459 real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4460
4461 value = readl((void *)real_offset);
4462
4463 if (value & 0x00010000) {
4464 value &= 0xfffeffff;
4465 writel(value, (void *)real_offset);
4466 }
4467
6733b39a
JK		 4468 pci_free_consistent(phba->pcidev,
4469 phba->ctrl.mbox_mem_alloced.size,
4470 phba->ctrl.mbox_mem_alloced.va,
4471 phba->ctrl.mbox_mem_alloced.dma);
4472 beiscsi_unmap_pci_function(phba);
4473hba_free:
238f6b72
JK		 4474 if (phba->msix_enabled)
4475 pci_disable_msix(phba->pcidev);
6733b39a
JK		 4476 iscsi_host_remove(phba->shost);
4477 pci_dev_put(phba->pcidev);
4478 iscsi_host_free(phba->shost);
4479disable_pci:
4480 pci_disable_device(pcidev);
4481 return ret;
4482}
4483
4484struct iscsi_transport beiscsi_iscsi_transport = {
4485 .owner = THIS_MODULE,
4486 .name = DRV_NAME,
9db0fb3a 4487 .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
6733b39a 4488 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
6733b39a
JK		 4489 .create_session = beiscsi_session_create,
4490 .destroy_session = beiscsi_session_destroy,
4491 .create_conn = beiscsi_conn_create,
4492 .bind_conn = beiscsi_conn_bind,
4493 .destroy_conn = iscsi_conn_teardown,
3128c6c7 4494 .attr_is_visible = be2iscsi_attr_is_visible,
0e43895e
MC		 4495 .set_iface_param = be2iscsi_iface_set_param,
4496 .get_iface_param = be2iscsi_iface_get_param,
6733b39a 4497 .set_param = beiscsi_set_param,
c7f7fd5b 4498 .get_conn_param = iscsi_conn_get_param,
6733b39a
JK		 4499 .get_session_param = iscsi_session_get_param,
4500 .get_host_param = beiscsi_get_host_param,
4501 .start_conn = beiscsi_conn_start,
fa95d206 4502 .stop_conn = iscsi_conn_stop,
6733b39a
JK		 4503 .send_pdu = iscsi_conn_send_pdu,
4504 .xmit_task = beiscsi_task_xmit,
4505 .cleanup_task = beiscsi_cleanup_task,
4506 .alloc_pdu = beiscsi_alloc_pdu,
4507 .parse_pdu_itt = beiscsi_parse_pdu,
4508 .get_stats = beiscsi_conn_get_stats,
c7f7fd5b 4509 .get_ep_param = beiscsi_ep_get_param,
6733b39a
JK		 4510 .ep_connect = beiscsi_ep_connect,
4511 .ep_poll = beiscsi_ep_poll,
4512 .ep_disconnect = beiscsi_ep_disconnect,
4513 .session_recovery_timedout = iscsi_session_recovery_timedout,
ffce3e2e 4514 .bsg_request = beiscsi_bsg_request,
6733b39a
JK		 4515};
4516
4517static struct pci_driver beiscsi_pci_driver = {
4518 .name = DRV_NAME,
4519 .probe = beiscsi_dev_probe,
4520 .remove = beiscsi_remove,
25602c97 4521 .shutdown = beiscsi_shutdown,
6733b39a
JK		 4522 .id_table = beiscsi_pci_id_table
4523};
4524
bfead3b2 4525
6733b39a
JK		 4526static int __init beiscsi_module_init(void)
4527{
4528 int ret;
4529
4530 beiscsi_scsi_transport =
4531 iscsi_register_transport(&beiscsi_iscsi_transport);
4532 if (!beiscsi_scsi_transport) {
4533 SE_DEBUG(DBG_LVL_1,
4534 "beiscsi_module_init - Unable to register beiscsi"
4535 "transport.\n");
f55a24f2 4536 return -ENOMEM;
6733b39a 4537 }
457ff3b7 4538 SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p\n",
6733b39a
JK		 4539 &beiscsi_iscsi_transport);
4540
4541 ret = pci_register_driver(&beiscsi_pci_driver);
4542 if (ret) {
4543 SE_DEBUG(DBG_LVL_1,
4544 "beiscsi_module_init - Unable to register"
4545 "beiscsi pci driver.\n");
4546 goto unregister_iscsi_transport;
4547 }
4548 return 0;
4549
4550unregister_iscsi_transport:
4551 iscsi_unregister_transport(&beiscsi_iscsi_transport);
4552 return ret;
4553}
4554
4555static void __exit beiscsi_module_exit(void)
4556{
4557 pci_unregister_driver(&beiscsi_pci_driver);
4558 iscsi_unregister_transport(&beiscsi_iscsi_transport);
4559}
4560
4561module_init(beiscsi_module_init);
4562module_exit(beiscsi_module_exit);
Arne's tree of linux kernel.