]> git.ipfire.org Git - people/arne_f/kernel.git/blame - drivers/scsi/be2iscsi/be_main.c
projects / people / arne_f / kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
be2iscsi: Fix bogus WARN_ON length check
[people/arne_f/kernel.git] / drivers / scsi / be2iscsi / be_main.c
CommitLineData
6733b39a 1/**
533c165f 2 * Copyright (C) 2005 - 2013 Emulex
6733b39a
JK		 3 * All rights reserved.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version 2
7 * as published by the Free Software Foundation. The full GNU General
8 * Public License is included in this distribution in the file called COPYING.
9 *
255fa9a3 10 * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
6733b39a
JK		 11 *
12 * Contact Information:
255fa9a3 13 * linux-drivers@emulex.com
6733b39a 14 *
255fa9a3
JK		 15 * Emulex
16 * 3333 Susan Street
17 * Costa Mesa, CA 92626
6733b39a 18 */
255fa9a3 19
6733b39a
JK		 20#include <linux/reboot.h>
21#include <linux/delay.h>
5a0e3ad6 22#include <linux/slab.h>
6733b39a
JK		 23#include <linux/interrupt.h>
24#include <linux/blkdev.h>
25#include <linux/pci.h>
26#include <linux/string.h>
27#include <linux/kernel.h>
28#include <linux/semaphore.h>
c7acc5b8 29#include <linux/iscsi_boot_sysfs.h>
acf3368f 30#include <linux/module.h>
ffce3e2e 31#include <linux/bsg-lib.h>
6733b39a
JK		 32
33#include <scsi/libiscsi.h>
ffce3e2e
JK		 34#include <scsi/scsi_bsg_iscsi.h>
35#include <scsi/scsi_netlink.h>
6733b39a
JK		 36#include <scsi/scsi_transport_iscsi.h>
37#include <scsi/scsi_transport.h>
38#include <scsi/scsi_cmnd.h>
39#include <scsi/scsi_device.h>
40#include <scsi/scsi_host.h>
41#include <scsi/scsi.h>
42#include "be_main.h"
43#include "be_iscsi.h"
44#include "be_mgmt.h"
0a513dd8 45#include "be_cmds.h"
6733b39a
JK		 46
47static unsigned int be_iopoll_budget = 10;
48static unsigned int be_max_phys_size = 64;
bfead3b2 49static unsigned int enable_msix = 1;
6733b39a
JK		 50
51MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
52MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
76d15dbd 53MODULE_VERSION(BUILD_STR);
2f635883 54MODULE_AUTHOR("Emulex Corporation");
6733b39a
JK		 55MODULE_LICENSE("GPL");
56module_param(be_iopoll_budget, int, 0);
57module_param(enable_msix, int, 0);
58module_param(be_max_phys_size, uint, S_IRUGO);
99bc5d55
JSJ		 59MODULE_PARM_DESC(be_max_phys_size,
60 "Maximum Size (In Kilobytes) of physically contiguous "
61 "memory that can be allocated. Range is 16 - 128");
62
63#define beiscsi_disp_param(_name)\
64ssize_t \
65beiscsi_##_name##_disp(struct device *dev,\
66 struct device_attribute *attrib, char *buf) \
67{ \
68 struct Scsi_Host *shost = class_to_shost(dev);\
69 struct beiscsi_hba *phba = iscsi_host_priv(shost); \
70 uint32_t param_val = 0; \
71 param_val = phba->attr_##_name;\
72 return snprintf(buf, PAGE_SIZE, "%d\n",\
73 phba->attr_##_name);\
74}
75
76#define beiscsi_change_param(_name, _minval, _maxval, _defaval)\
77int \
78beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\
79{\
80 if (val >= _minval && val <= _maxval) {\
81 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
82 "BA_%d : beiscsi_"#_name" updated "\
83 "from 0x%x ==> 0x%x\n",\
84 phba->attr_##_name, val); \
85 phba->attr_##_name = val;\
86 return 0;\
87 } \
88 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \
89 "BA_%d beiscsi_"#_name" attribute "\
90 "cannot be updated to 0x%x, "\
91 "range allowed is ["#_minval" - "#_maxval"]\n", val);\
92 return -EINVAL;\
93}
94
95#define beiscsi_store_param(_name) \
96ssize_t \
97beiscsi_##_name##_store(struct device *dev,\
98 struct device_attribute *attr, const char *buf,\
99 size_t count) \
100{ \
101 struct Scsi_Host *shost = class_to_shost(dev);\
102 struct beiscsi_hba *phba = iscsi_host_priv(shost);\
103 uint32_t param_val = 0;\
104 if (!isdigit(buf[0]))\
105 return -EINVAL;\
106 if (sscanf(buf, "%i", &param_val) != 1)\
107 return -EINVAL;\
108 if (beiscsi_##_name##_change(phba, param_val) == 0) \
109 return strlen(buf);\
110 else \
111 return -EINVAL;\
112}
113
114#define beiscsi_init_param(_name, _minval, _maxval, _defval) \
115int \
116beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \
117{ \
118 if (val >= _minval && val <= _maxval) {\
119 phba->attr_##_name = val;\
120 return 0;\
121 } \
122 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
123 "BA_%d beiscsi_"#_name" attribute " \
124 "cannot be updated to 0x%x, "\
125 "range allowed is ["#_minval" - "#_maxval"]\n", val);\
126 phba->attr_##_name = _defval;\
127 return -EINVAL;\
128}
129
130#define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \
131static uint beiscsi_##_name = _defval;\
132module_param(beiscsi_##_name, uint, S_IRUGO);\
133MODULE_PARM_DESC(beiscsi_##_name, _descp);\
134beiscsi_disp_param(_name)\
135beiscsi_change_param(_name, _minval, _maxval, _defval)\
136beiscsi_store_param(_name)\
137beiscsi_init_param(_name, _minval, _maxval, _defval)\
138DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
139 beiscsi_##_name##_disp, beiscsi_##_name##_store)
140
141/*
142 * When new log level added update the
143 * the MAX allowed value for log_enable
144 */
145BEISCSI_RW_ATTR(log_enable, 0x00,
146 0xFF, 0x00, "Enable logging Bit Mask\n"
147 "\t\t\t\tInitialization Events : 0x01\n"
148 "\t\t\t\tMailbox Events : 0x02\n"
149 "\t\t\t\tMiscellaneous Events : 0x04\n"
150 "\t\t\t\tError Handling : 0x08\n"
151 "\t\t\t\tIO Path Events : 0x10\n"
afb96058
JK		 152 "\t\t\t\tConfiguration Path : 0x20\n"
153 "\t\t\t\tiSCSI Protocol : 0x40\n");
99bc5d55 154
5cac7596 155DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL);
26000db7 156DEVICE_ATTR(beiscsi_adapter_family, S_IRUGO, beiscsi_adap_family_disp, NULL);
22661e25 157DEVICE_ATTR(beiscsi_fw_ver, S_IRUGO, beiscsi_fw_ver_disp, NULL);
d3fea9af 158DEVICE_ATTR(beiscsi_phys_port, S_IRUGO, beiscsi_phys_port_disp, NULL);
6103c1f7
JK		 159DEVICE_ATTR(beiscsi_active_session_count, S_IRUGO,
160 beiscsi_active_session_disp, NULL);
161DEVICE_ATTR(beiscsi_free_session_count, S_IRUGO,
162 beiscsi_free_session_disp, NULL);
99bc5d55
JSJ		 163struct device_attribute *beiscsi_attrs[] = {
164 &dev_attr_beiscsi_log_enable,
5cac7596 165 &dev_attr_beiscsi_drvr_ver,
26000db7 166 &dev_attr_beiscsi_adapter_family,
22661e25 167 &dev_attr_beiscsi_fw_ver,
6103c1f7
JK		 168 &dev_attr_beiscsi_active_session_count,
169 &dev_attr_beiscsi_free_session_count,
d3fea9af 170 &dev_attr_beiscsi_phys_port,
99bc5d55
JSJ		 171 NULL,
172};
6733b39a 173
6763daae
JSJ		 174static char const *cqe_desc[] = {
175 "RESERVED_DESC",
176 "SOL_CMD_COMPLETE",
177 "SOL_CMD_KILLED_DATA_DIGEST_ERR",
178 "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL",
179 "CXN_KILLED_BURST_LEN_MISMATCH",
180 "CXN_KILLED_AHS_RCVD",
181 "CXN_KILLED_HDR_DIGEST_ERR",
182 "CXN_KILLED_UNKNOWN_HDR",
183 "CXN_KILLED_STALE_ITT_TTT_RCVD",
184 "CXN_KILLED_INVALID_ITT_TTT_RCVD",
185 "CXN_KILLED_RST_RCVD",
186 "CXN_KILLED_TIMED_OUT",
187 "CXN_KILLED_RST_SENT",
188 "CXN_KILLED_FIN_RCVD",
189 "CXN_KILLED_BAD_UNSOL_PDU_RCVD",
190 "CXN_KILLED_BAD_WRB_INDEX_ERROR",
191 "CXN_KILLED_OVER_RUN_RESIDUAL",
192 "CXN_KILLED_UNDER_RUN_RESIDUAL",
193 "CMD_KILLED_INVALID_STATSN_RCVD",
194 "CMD_KILLED_INVALID_R2T_RCVD",
195 "CMD_CXN_KILLED_LUN_INVALID",
196 "CMD_CXN_KILLED_ICD_INVALID",
197 "CMD_CXN_KILLED_ITT_INVALID",
198 "CMD_CXN_KILLED_SEQ_OUTOFORDER",
199 "CMD_CXN_KILLED_INVALID_DATASN_RCVD",
200 "CXN_INVALIDATE_NOTIFY",
201 "CXN_INVALIDATE_INDEX_NOTIFY",
202 "CMD_INVALIDATED_NOTIFY",
203 "UNSOL_HDR_NOTIFY",
204 "UNSOL_DATA_NOTIFY",
205 "UNSOL_DATA_DIGEST_ERROR_NOTIFY",
206 "DRIVERMSG_NOTIFY",
207 "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN",
208 "SOL_CMD_KILLED_DIF_ERR",
209 "CXN_KILLED_SYN_RCVD",
210 "CXN_KILLED_IMM_DATA_RCVD"
211};
212
6733b39a
JK		 213static int beiscsi_slave_configure(struct scsi_device *sdev)
214{
215 blk_queue_max_segment_size(sdev->request_queue, 65536);
216 return 0;
217}
218
4183122d
JK		 219static int beiscsi_eh_abort(struct scsi_cmnd *sc)
220{
221 struct iscsi_cls_session *cls_session;
222 struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
223 struct beiscsi_io_task *aborted_io_task;
224 struct iscsi_conn *conn;
225 struct beiscsi_conn *beiscsi_conn;
226 struct beiscsi_hba *phba;
227 struct iscsi_session *session;
228 struct invalidate_command_table *inv_tbl;
3cbb7a74 229 struct be_dma_mem nonemb_cmd;
4183122d
JK		 230 unsigned int cid, tag, num_invalidate;
231
232 cls_session = starget_to_session(scsi_target(sc->device));
233 session = cls_session->dd_data;
234
235 spin_lock_bh(&session->lock);
236 if (!aborted_task || !aborted_task->sc) {
237 /* we raced */
238 spin_unlock_bh(&session->lock);
239 return SUCCESS;
240 }
241
242 aborted_io_task = aborted_task->dd_data;
243 if (!aborted_io_task->scsi_cmnd) {
244 /* raced or invalid command */
245 spin_unlock_bh(&session->lock);
246 return SUCCESS;
247 }
248 spin_unlock_bh(&session->lock);
7626c06b
JK		 249 /* Invalidate WRB Posted for this Task */
250 AMAP_SET_BITS(struct amap_iscsi_wrb, invld,
251 aborted_io_task->pwrb_handle->pwrb,
252 1);
253
4183122d
JK		 254 conn = aborted_task->conn;
255 beiscsi_conn = conn->dd_data;
256 phba = beiscsi_conn->phba;
257
258 /* invalidate iocb */
259 cid = beiscsi_conn->beiscsi_conn_cid;
260 inv_tbl = phba->inv_tbl;
261 memset(inv_tbl, 0x0, sizeof(*inv_tbl));
262 inv_tbl->cid = cid;
263 inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
264 num_invalidate = 1;
3cbb7a74
JK		 265 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
266 sizeof(struct invalidate_commands_params_in),
267 &nonemb_cmd.dma);
268 if (nonemb_cmd.va == NULL) {
99bc5d55
JSJ		 269 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
270 "BM_%d : Failed to allocate memory for"
271 "mgmt_invalidate_icds\n");
3cbb7a74
JK		 272 return FAILED;
273 }
274 nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
275
276 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
277 cid, &nonemb_cmd);
4183122d 278 if (!tag) {
99bc5d55
JSJ		 279 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
280 "BM_%d : mgmt_invalidate_icds could not be"
281 "submitted\n");
3cbb7a74
JK		 282 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
283 nonemb_cmd.va, nonemb_cmd.dma);
284
4183122d 285 return FAILED;
4183122d 286 }
e175defe
JSJ		 287
288 beiscsi_mccq_compl(phba, tag, NULL, nonemb_cmd.va);
3cbb7a74
JK		 289 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
290 nonemb_cmd.va, nonemb_cmd.dma);
4183122d
JK		 291 return iscsi_eh_abort(sc);
292}
293
294static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
295{
296 struct iscsi_task *abrt_task;
297 struct beiscsi_io_task *abrt_io_task;
298 struct iscsi_conn *conn;
299 struct beiscsi_conn *beiscsi_conn;
300 struct beiscsi_hba *phba;
301 struct iscsi_session *session;
302 struct iscsi_cls_session *cls_session;
303 struct invalidate_command_table *inv_tbl;
3cbb7a74 304 struct be_dma_mem nonemb_cmd;
4183122d 305 unsigned int cid, tag, i, num_invalidate;
4183122d
JK		 306
307 /* invalidate iocbs */
308 cls_session = starget_to_session(scsi_target(sc->device));
309 session = cls_session->dd_data;
310 spin_lock_bh(&session->lock);
db7f7709
JK		 311 if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {
312 spin_unlock_bh(&session->lock);
313 return FAILED;
314 }
4183122d
JK		 315 conn = session->leadconn;
316 beiscsi_conn = conn->dd_data;
317 phba = beiscsi_conn->phba;
318 cid = beiscsi_conn->beiscsi_conn_cid;
319 inv_tbl = phba->inv_tbl;
320 memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
321 num_invalidate = 0;
322 for (i = 0; i < conn->session->cmds_max; i++) {
323 abrt_task = conn->session->cmds[i];
324 abrt_io_task = abrt_task->dd_data;
325 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
326 continue;
327
126e964a 328 if (sc->device->lun != abrt_task->sc->device->lun)
4183122d
JK		 329 continue;
330
7626c06b
JK		 331 /* Invalidate WRB Posted for this Task */
332 AMAP_SET_BITS(struct amap_iscsi_wrb, invld,
333 abrt_io_task->pwrb_handle->pwrb,
334 1);
335
4183122d
JK		 336 inv_tbl->cid = cid;
337 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
338 num_invalidate++;
339 inv_tbl++;
340 }
341 spin_unlock_bh(&session->lock);
342 inv_tbl = phba->inv_tbl;
343
3cbb7a74
JK		 344 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
345 sizeof(struct invalidate_commands_params_in),
346 &nonemb_cmd.dma);
347 if (nonemb_cmd.va == NULL) {
99bc5d55
JSJ		 348 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
349 "BM_%d : Failed to allocate memory for"
350 "mgmt_invalidate_icds\n");
3cbb7a74
JK		 351 return FAILED;
352 }
353 nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
354 memset(nonemb_cmd.va, 0, nonemb_cmd.size);
355 tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
356 cid, &nonemb_cmd);
4183122d 357 if (!tag) {
99bc5d55
JSJ		 358 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
359 "BM_%d : mgmt_invalidate_icds could not be"
360 " submitted\n");
3cbb7a74
JK		 361 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
362 nonemb_cmd.va, nonemb_cmd.dma);
4183122d 363 return FAILED;
4183122d 364 }
e175defe
JSJ		 365
366 beiscsi_mccq_compl(phba, tag, NULL, nonemb_cmd.va);
3cbb7a74
JK		 367 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
368 nonemb_cmd.va, nonemb_cmd.dma);
4183122d 369 return iscsi_eh_device_reset(sc);
4183122d
JK		 370}
371
c7acc5b8
JK		 372static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
373{
374 struct beiscsi_hba *phba = data;
f457a46f
MC		 375 struct mgmt_session_info *boot_sess = &phba->boot_sess;
376 struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
c7acc5b8
JK		 377 char *str = buf;
378 int rc;
379
380 switch (type) {
381 case ISCSI_BOOT_TGT_NAME:
382 rc = sprintf(buf, "%.*s\n",
f457a46f
MC		 383 (int)strlen(boot_sess->target_name),
384 (char *)&boot_sess->target_name);
c7acc5b8
JK		 385 break;
386 case ISCSI_BOOT_TGT_IP_ADDR:
f457a46f 387 if (boot_conn->dest_ipaddr.ip_type == 0x1)
c7acc5b8 388 rc = sprintf(buf, "%pI4\n",
0e43895e 389 (char *)&boot_conn->dest_ipaddr.addr);
c7acc5b8
JK		 390 else
391 rc = sprintf(str, "%pI6\n",
0e43895e 392 (char *)&boot_conn->dest_ipaddr.addr);
c7acc5b8
JK		 393 break;
394 case ISCSI_BOOT_TGT_PORT:
f457a46f 395 rc = sprintf(str, "%d\n", boot_conn->dest_port);
c7acc5b8
JK		 396 break;
397
398 case ISCSI_BOOT_TGT_CHAP_NAME:
399 rc = sprintf(str, "%.*s\n",
f457a46f
MC		 400 boot_conn->negotiated_login_options.auth_data.chap.
401 target_chap_name_length,
402 (char *)&boot_conn->negotiated_login_options.
403 auth_data.chap.target_chap_name);
c7acc5b8
JK		 404 break;
405 case ISCSI_BOOT_TGT_CHAP_SECRET:
406 rc = sprintf(str, "%.*s\n",
f457a46f
MC		 407 boot_conn->negotiated_login_options.auth_data.chap.
408 target_secret_length,
409 (char *)&boot_conn->negotiated_login_options.
410 auth_data.chap.target_secret);
c7acc5b8
JK		 411 break;
412 case ISCSI_BOOT_TGT_REV_CHAP_NAME:
413 rc = sprintf(str, "%.*s\n",
f457a46f
MC		 414 boot_conn->negotiated_login_options.auth_data.chap.
415 intr_chap_name_length,
416 (char *)&boot_conn->negotiated_login_options.
417 auth_data.chap.intr_chap_name);
c7acc5b8
JK		 418 break;
419 case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
f457a46f
MC		 420 rc = sprintf(str, "%.*s\n",
421 boot_conn->negotiated_login_options.auth_data.chap.
422 intr_secret_length,
423 (char *)&boot_conn->negotiated_login_options.
424 auth_data.chap.intr_secret);
c7acc5b8
JK		 425 break;
426 case ISCSI_BOOT_TGT_FLAGS:
f457a46f 427 rc = sprintf(str, "2\n");
c7acc5b8
JK		 428 break;
429 case ISCSI_BOOT_TGT_NIC_ASSOC:
f457a46f 430 rc = sprintf(str, "0\n");
c7acc5b8
JK		 431 break;
432 default:
433 rc = -ENOSYS;
434 break;
435 }
436 return rc;
437}
438
439static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
440{
441 struct beiscsi_hba *phba = data;
442 char *str = buf;
443 int rc;
444
445 switch (type) {
446 case ISCSI_BOOT_INI_INITIATOR_NAME:
447 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
448 break;
449 default:
450 rc = -ENOSYS;
451 break;
452 }
453 return rc;
454}
455
456static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
457{
458 struct beiscsi_hba *phba = data;
459 char *str = buf;
460 int rc;
461
462 switch (type) {
463 case ISCSI_BOOT_ETH_FLAGS:
f457a46f 464 rc = sprintf(str, "2\n");
c7acc5b8
JK		 465 break;
466 case ISCSI_BOOT_ETH_INDEX:
f457a46f 467 rc = sprintf(str, "0\n");
c7acc5b8
JK		 468 break;
469 case ISCSI_BOOT_ETH_MAC:
0e43895e
MC		 470 rc = beiscsi_get_macaddr(str, phba);
471 break;
c7acc5b8
JK		 472 default:
473 rc = -ENOSYS;
474 break;
475 }
476 return rc;
477}
478
479
587a1f16 480static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
c7acc5b8 481{
587a1f16 482 umode_t rc;
c7acc5b8
JK		 483
484 switch (type) {
485 case ISCSI_BOOT_TGT_NAME:
486 case ISCSI_BOOT_TGT_IP_ADDR:
487 case ISCSI_BOOT_TGT_PORT:
488 case ISCSI_BOOT_TGT_CHAP_NAME:
489 case ISCSI_BOOT_TGT_CHAP_SECRET:
490 case ISCSI_BOOT_TGT_REV_CHAP_NAME:
491 case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
492 case ISCSI_BOOT_TGT_NIC_ASSOC:
493 case ISCSI_BOOT_TGT_FLAGS:
494 rc = S_IRUGO;
495 break;
496 default:
497 rc = 0;
498 break;
499 }
500 return rc;
501}
502
587a1f16 503static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
c7acc5b8 504{
587a1f16 505 umode_t rc;
c7acc5b8
JK		 506
507 switch (type) {
508 case ISCSI_BOOT_INI_INITIATOR_NAME:
509 rc = S_IRUGO;
510 break;
511 default:
512 rc = 0;
513 break;
514 }
515 return rc;
516}
517
518
587a1f16 519static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
c7acc5b8 520{
587a1f16 521 umode_t rc;
c7acc5b8
JK		 522
523 switch (type) {
524 case ISCSI_BOOT_ETH_FLAGS:
525 case ISCSI_BOOT_ETH_MAC:
526 case ISCSI_BOOT_ETH_INDEX:
527 rc = S_IRUGO;
528 break;
529 default:
530 rc = 0;
531 break;
532 }
533 return rc;
534}
535
bfead3b2
JK		 536/*------------------- PCI Driver operations and data ----------------- */
537static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
538 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
f98c96b0 539 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
bfead3b2
JK		 540 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
541 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
542 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
139a1b1e 543 { PCI_DEVICE(ELX_VENDOR_ID, OC_SKH_ID1) },
bfead3b2
JK		 544 { 0 }
545};
546MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
547
99bc5d55 548
6733b39a
JK		 549static struct scsi_host_template beiscsi_sht = {
550 .module = THIS_MODULE,
2f635883 551 .name = "Emulex 10Gbe open-iscsi Initiator Driver",
6733b39a
JK		 552 .proc_name = DRV_NAME,
553 .queuecommand = iscsi_queuecommand,
6733b39a
JK		 554 .change_queue_depth = iscsi_change_queue_depth,
555 .slave_configure = beiscsi_slave_configure,
556 .target_alloc = iscsi_target_alloc,
4183122d
JK		 557 .eh_abort_handler = beiscsi_eh_abort,
558 .eh_device_reset_handler = beiscsi_eh_device_reset,
309ce156 559 .eh_target_reset_handler = iscsi_eh_session_reset,
99bc5d55 560 .shost_attrs = beiscsi_attrs,
6733b39a
JK		 561 .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
562 .can_queue = BE2_IO_DEPTH,
563 .this_id = -1,
564 .max_sectors = BEISCSI_MAX_SECTORS,
565 .cmd_per_lun = BEISCSI_CMD_PER_LUN,
566 .use_clustering = ENABLE_CLUSTERING,
ffce3e2e
JK		 567 .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID,
568
6733b39a 569};
6733b39a 570
bfead3b2 571static struct scsi_transport_template *beiscsi_scsi_transport;
6733b39a
JK		 572
573static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
574{
575 struct beiscsi_hba *phba;
576 struct Scsi_Host *shost;
577
578 shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
579 if (!shost) {
99bc5d55
JSJ		 580 dev_err(&pcidev->dev,
581 "beiscsi_hba_alloc - iscsi_host_alloc failed\n");
6733b39a
JK		 582 return NULL;
583 }
6733b39a
JK		 584 shost->max_id = BE2_MAX_SESSIONS;
585 shost->max_channel = 0;
586 shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
587 shost->max_lun = BEISCSI_NUM_MAX_LUN;
588 shost->transportt = beiscsi_scsi_transport;
6733b39a
JK		 589 phba = iscsi_host_priv(shost);
590 memset(phba, 0, sizeof(*phba));
591 phba->shost = shost;
592 phba->pcidev = pci_dev_get(pcidev);
2807afb7 593 pci_set_drvdata(pcidev, phba);
0e43895e 594 phba->interface_handle = 0xFFFFFFFF;
6733b39a
JK		 595
596 if (iscsi_host_add(shost, &phba->pcidev->dev))
597 goto free_devices;
c7acc5b8 598
6733b39a
JK		 599 return phba;
600
601free_devices:
602 pci_dev_put(phba->pcidev);
603 iscsi_host_free(phba->shost);
604 return NULL;
605}
606
607static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
608{
609 if (phba->csr_va) {
610 iounmap(phba->csr_va);
611 phba->csr_va = NULL;
612 }
613 if (phba->db_va) {
614 iounmap(phba->db_va);
615 phba->db_va = NULL;
616 }
617 if (phba->pci_va) {
618 iounmap(phba->pci_va);
619 phba->pci_va = NULL;
620 }
621}
622
623static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
624 struct pci_dev *pcidev)
625{
626 u8 __iomem *addr;
f98c96b0 627 int pcicfg_reg;
6733b39a
JK		 628
629 addr = ioremap_nocache(pci_resource_start(pcidev, 2),
630 pci_resource_len(pcidev, 2));
631 if (addr == NULL)
632 return -ENOMEM;
633 phba->ctrl.csr = addr;
634 phba->csr_va = addr;
635 phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
636
637 addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
638 if (addr == NULL)
639 goto pci_map_err;
640 phba->ctrl.db = addr;
641 phba->db_va = addr;
642 phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4);
643
f98c96b0
JK		 644 if (phba->generation == BE_GEN2)
645 pcicfg_reg = 1;
646 else
647 pcicfg_reg = 0;
648
649 addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
650 pci_resource_len(pcidev, pcicfg_reg));
651
6733b39a
JK		 652 if (addr == NULL)
653 goto pci_map_err;
654 phba->ctrl.pcicfg = addr;
655 phba->pci_va = addr;
f98c96b0 656 phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
6733b39a
JK		 657 return 0;
658
659pci_map_err:
660 beiscsi_unmap_pci_function(phba);
661 return -ENOMEM;
662}
663
664static int beiscsi_enable_pci(struct pci_dev *pcidev)
665{
666 int ret;
667
668 ret = pci_enable_device(pcidev);
669 if (ret) {
99bc5d55
JSJ		 670 dev_err(&pcidev->dev,
671 "beiscsi_enable_pci - enable device failed\n");
6733b39a
JK		 672 return ret;
673 }
674
bfead3b2 675 pci_set_master(pcidev);
6733b39a
JK		 676 if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
677 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
678 if (ret) {
679 dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
680 pci_disable_device(pcidev);
681 return ret;
682 }
683 }
684 return 0;
685}
686
687static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
688{
689 struct be_ctrl_info *ctrl = &phba->ctrl;
690 struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
691 struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
692 int status = 0;
693
694 ctrl->pdev = pdev;
695 status = beiscsi_map_pci_bars(phba, pdev);
696 if (status)
697 return status;
6733b39a
JK		 698 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
699 mbox_mem_alloc->va = pci_alloc_consistent(pdev,
700 mbox_mem_alloc->size,
701 &mbox_mem_alloc->dma);
702 if (!mbox_mem_alloc->va) {
703 beiscsi_unmap_pci_function(phba);
a49e06d5 704 return -ENOMEM;
6733b39a
JK		 705 }
706
707 mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
708 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
709 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
710 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
711 spin_lock_init(&ctrl->mbox_lock);
bfead3b2
JK		 712 spin_lock_init(&phba->ctrl.mcc_lock);
713 spin_lock_init(&phba->ctrl.mcc_cq_lock);
714
6733b39a
JK		 715 return status;
716}
717
843ae752
JK		 718/**
719 * beiscsi_get_params()- Set the config paramters
720 * @phba: ptr device priv structure
721 **/
6733b39a
JK		 722static void beiscsi_get_params(struct beiscsi_hba *phba)
723{
843ae752
JK		 724 uint32_t total_cid_count = 0;
725 uint32_t total_icd_count = 0;
726 uint8_t ulp_num = 0;
727
728 total_cid_count = BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP0) +
729 BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP1);
730
cf987b79
JK		 731 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
732 uint32_t align_mask = 0;
733 uint32_t icd_post_per_page = 0;
734 uint32_t icd_count_unavailable = 0;
735 uint32_t icd_start = 0, icd_count = 0;
736 uint32_t icd_start_align = 0, icd_count_align = 0;
737
843ae752 738 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
cf987b79
JK		 739 icd_start = phba->fw_config.iscsi_icd_start[ulp_num];
740 icd_count = phba->fw_config.iscsi_icd_count[ulp_num];
741
742 /* Get ICD count that can be posted on each page */
743 icd_post_per_page = (PAGE_SIZE / (BE2_SGE *
744 sizeof(struct iscsi_sge)));
745 align_mask = (icd_post_per_page - 1);
746
747 /* Check if icd_start is aligned ICD per page posting */
748 if (icd_start % icd_post_per_page) {
749 icd_start_align = ((icd_start +
750 icd_post_per_page) &
751 ~(align_mask));
752 phba->fw_config.
753 iscsi_icd_start[ulp_num] =
754 icd_start_align;
755 }
756
757 icd_count_align = (icd_count & ~align_mask);
758
759 /* ICD discarded in the process of alignment */
760 if (icd_start_align)
761 icd_count_unavailable = ((icd_start_align -
762 icd_start) +
763 (icd_count -
764 icd_count_align));
765
766 /* Updated ICD count available */
767 phba->fw_config.iscsi_icd_count[ulp_num] = (icd_count -
768 icd_count_unavailable);
769
770 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
771 "BM_%d : Aligned ICD values\n"
772 "\t ICD Start : %d\n"
773 "\t ICD Count : %d\n"
774 "\t ICD Discarded : %d\n",
775 phba->fw_config.
776 iscsi_icd_start[ulp_num],
777 phba->fw_config.
778 iscsi_icd_count[ulp_num],
779 icd_count_unavailable);
843ae752
JK		 780 break;
781 }
cf987b79 782 }
843ae752 783
cf987b79 784 total_icd_count = phba->fw_config.iscsi_icd_count[ulp_num];
843ae752
JK		 785 phba->params.ios_per_ctrl = (total_icd_count -
786 (total_cid_count +
787 BE2_TMFS + BE2_NOPOUT_REQ));
788 phba->params.cxns_per_ctrl = total_cid_count;
789 phba->params.asyncpdus_per_ctrl = total_cid_count;
790 phba->params.icds_per_ctrl = total_icd_count;
6733b39a
JK		 791 phba->params.num_sge_per_io = BE2_SGE;
792 phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
793 phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
794 phba->params.eq_timer = 64;
843ae752
JK		 795 phba->params.num_eq_entries = 1024;
796 phba->params.num_cq_entries = 1024;
6733b39a
JK		 797 phba->params.wrbs_per_cxn = 256;
798}
799
800static void hwi_ring_eq_db(struct beiscsi_hba *phba,
801 unsigned int id, unsigned int clr_interrupt,
802 unsigned int num_processed,
803 unsigned char rearm, unsigned char event)
804{
805 u32 val = 0;
806 val |= id & DB_EQ_RING_ID_MASK;
807 if (rearm)
808 val |= 1 << DB_EQ_REARM_SHIFT;
809 if (clr_interrupt)
810 val |= 1 << DB_EQ_CLR_SHIFT;
811 if (event)
812 val |= 1 << DB_EQ_EVNT_SHIFT;
813 val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
814 iowrite32(val, phba->db_va + DB_EQ_OFFSET);
815}
816
bfead3b2
JK		 817/**
818 * be_isr_mcc - The isr routine of the driver.
819 * @irq: Not used
820 * @dev_id: Pointer to host adapter structure
821 */
822static irqreturn_t be_isr_mcc(int irq, void *dev_id)
823{
824 struct beiscsi_hba *phba;
825 struct be_eq_entry *eqe = NULL;
826 struct be_queue_info *eq;
827 struct be_queue_info *mcc;
828 unsigned int num_eq_processed;
829 struct be_eq_obj *pbe_eq;
830 unsigned long flags;
831
832 pbe_eq = dev_id;
833 eq = &pbe_eq->q;
834 phba = pbe_eq->phba;
835 mcc = &phba->ctrl.mcc_obj.cq;
836 eqe = queue_tail_node(eq);
bfead3b2
JK		 837
838 num_eq_processed = 0;
839
840 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
841 & EQE_VALID_MASK) {
842 if (((eqe->dw[offsetof(struct amap_eq_entry,
843 resource_id) / 32] &
844 EQE_RESID_MASK) >> 16) == mcc->id) {
845 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 846 pbe_eq->todo_mcc_cq = true;
bfead3b2
JK		 847 spin_unlock_irqrestore(&phba->isr_lock, flags);
848 }
849 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
850 queue_tail_inc(eq);
851 eqe = queue_tail_node(eq);
852 num_eq_processed++;
853 }
72fb46a9
JSJ		 854 if (pbe_eq->todo_mcc_cq)
855 queue_work(phba->wq, &pbe_eq->work_cqs);
bfead3b2
JK		 856 if (num_eq_processed)
857 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
858
859 return IRQ_HANDLED;
860}
861
862/**
863 * be_isr_msix - The isr routine of the driver.
864 * @irq: Not used
865 * @dev_id: Pointer to host adapter structure
866 */
867static irqreturn_t be_isr_msix(int irq, void *dev_id)
868{
869 struct beiscsi_hba *phba;
870 struct be_eq_entry *eqe = NULL;
871 struct be_queue_info *eq;
872 struct be_queue_info *cq;
873 unsigned int num_eq_processed;
874 struct be_eq_obj *pbe_eq;
875 unsigned long flags;
876
877 pbe_eq = dev_id;
878 eq = &pbe_eq->q;
879 cq = pbe_eq->cq;
880 eqe = queue_tail_node(eq);
bfead3b2
JK		 881
882 phba = pbe_eq->phba;
883 num_eq_processed = 0;
884 if (blk_iopoll_enabled) {
885 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
886 & EQE_VALID_MASK) {
887 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
888 blk_iopoll_sched(&pbe_eq->iopoll);
889
890 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
891 queue_tail_inc(eq);
892 eqe = queue_tail_node(eq);
893 num_eq_processed++;
894 }
bfead3b2
JK		 895 } else {
896 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
897 & EQE_VALID_MASK) {
898 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 899 pbe_eq->todo_cq = true;
bfead3b2
JK		 900 spin_unlock_irqrestore(&phba->isr_lock, flags);
901 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
902 queue_tail_inc(eq);
903 eqe = queue_tail_node(eq);
904 num_eq_processed++;
905 }
bfead3b2 906
72fb46a9
JSJ		 907 if (pbe_eq->todo_cq)
908 queue_work(phba->wq, &pbe_eq->work_cqs);
bfead3b2 909 }
72fb46a9
JSJ		 910
911 if (num_eq_processed)
912 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
913
914 return IRQ_HANDLED;
bfead3b2
JK		 915}
916
6733b39a
JK		 917/**
918 * be_isr - The isr routine of the driver.
919 * @irq: Not used
920 * @dev_id: Pointer to host adapter structure
921 */
922static irqreturn_t be_isr(int irq, void *dev_id)
923{
924 struct beiscsi_hba *phba;
925 struct hwi_controller *phwi_ctrlr;
926 struct hwi_context_memory *phwi_context;
927 struct be_eq_entry *eqe = NULL;
928 struct be_queue_info *eq;
929 struct be_queue_info *cq;
bfead3b2 930 struct be_queue_info *mcc;
6733b39a 931 unsigned long flags, index;
bfead3b2 932 unsigned int num_mcceq_processed, num_ioeq_processed;
6733b39a 933 struct be_ctrl_info *ctrl;
bfead3b2 934 struct be_eq_obj *pbe_eq;
6733b39a
JK		 935 int isr;
936
937 phba = dev_id;
6eab04a8 938 ctrl = &phba->ctrl;
bfead3b2
JK		 939 isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
940 (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
941 if (!isr)
942 return IRQ_NONE;
6733b39a
JK		 943
944 phwi_ctrlr = phba->phwi_ctrlr;
945 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 946 pbe_eq = &phwi_context->be_eq[0];
947
948 eq = &phwi_context->be_eq[0].q;
949 mcc = &phba->ctrl.mcc_obj.cq;
6733b39a
JK		 950 index = 0;
951 eqe = queue_tail_node(eq);
6733b39a 952
bfead3b2
JK		 953 num_ioeq_processed = 0;
954 num_mcceq_processed = 0;
6733b39a
JK		 955 if (blk_iopoll_enabled) {
956 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
957 & EQE_VALID_MASK) {
bfead3b2
JK		 958 if (((eqe->dw[offsetof(struct amap_eq_entry,
959 resource_id) / 32] &
960 EQE_RESID_MASK) >> 16) == mcc->id) {
961 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 962 pbe_eq->todo_mcc_cq = true;
bfead3b2
JK		 963 spin_unlock_irqrestore(&phba->isr_lock, flags);
964 num_mcceq_processed++;
965 } else {
966 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
967 blk_iopoll_sched(&pbe_eq->iopoll);
968 num_ioeq_processed++;
969 }
6733b39a
JK		 970 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
971 queue_tail_inc(eq);
972 eqe = queue_tail_node(eq);
6733b39a 973 }
bfead3b2 974 if (num_ioeq_processed || num_mcceq_processed) {
72fb46a9
JSJ		 975 if (pbe_eq->todo_mcc_cq)
976 queue_work(phba->wq, &pbe_eq->work_cqs);
bfead3b2 977
756d29c8 978 if ((num_mcceq_processed) && (!num_ioeq_processed))
bfead3b2
JK		 979 hwi_ring_eq_db(phba, eq->id, 0,
980 (num_ioeq_processed +
981 num_mcceq_processed) , 1, 1);
982 else
983 hwi_ring_eq_db(phba, eq->id, 0,
984 (num_ioeq_processed +
985 num_mcceq_processed), 0, 1);
986
6733b39a
JK		 987 return IRQ_HANDLED;
988 } else
989 return IRQ_NONE;
990 } else {
bfead3b2 991 cq = &phwi_context->be_cq[0];
6733b39a
JK		 992 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
993 & EQE_VALID_MASK) {
994
995 if (((eqe->dw[offsetof(struct amap_eq_entry,
996 resource_id) / 32] &
997 EQE_RESID_MASK) >> 16) != cq->id) {
998 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 999 pbe_eq->todo_mcc_cq = true;
6733b39a
JK		 1000 spin_unlock_irqrestore(&phba->isr_lock, flags);
1001 } else {
1002 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 1003 pbe_eq->todo_cq = true;
6733b39a
JK		 1004 spin_unlock_irqrestore(&phba->isr_lock, flags);
1005 }
1006 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
1007 queue_tail_inc(eq);
1008 eqe = queue_tail_node(eq);
bfead3b2 1009 num_ioeq_processed++;
6733b39a 1010 }
72fb46a9
JSJ		 1011 if (pbe_eq->todo_cq || pbe_eq->todo_mcc_cq)
1012 queue_work(phba->wq, &pbe_eq->work_cqs);
6733b39a 1013
bfead3b2
JK		 1014 if (num_ioeq_processed) {
1015 hwi_ring_eq_db(phba, eq->id, 0,
1016 num_ioeq_processed, 1, 1);
6733b39a
JK		 1017 return IRQ_HANDLED;
1018 } else
1019 return IRQ_NONE;
1020 }
1021}
1022
1023static int beiscsi_init_irqs(struct beiscsi_hba *phba)
1024{
1025 struct pci_dev *pcidev = phba->pcidev;
bfead3b2
JK		 1026 struct hwi_controller *phwi_ctrlr;
1027 struct hwi_context_memory *phwi_context;
4f5af07e 1028 int ret, msix_vec, i, j;
6733b39a 1029
bfead3b2
JK		 1030 phwi_ctrlr = phba->phwi_ctrlr;
1031 phwi_context = phwi_ctrlr->phwi_ctxt;
1032
1033 if (phba->msix_enabled) {
1034 for (i = 0; i < phba->num_cpus; i++) {
8fcfb210
JK		 1035 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
1036 GFP_KERNEL);
1037 if (!phba->msi_name[i]) {
1038 ret = -ENOMEM;
1039 goto free_msix_irqs;
1040 }
1041
1042 sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
1043 phba->shost->host_no, i);
bfead3b2 1044 msix_vec = phba->msix_entries[i].vector;
8fcfb210
JK		 1045 ret = request_irq(msix_vec, be_isr_msix, 0,
1046 phba->msi_name[i],
bfead3b2 1047 &phwi_context->be_eq[i]);
4f5af07e 1048 if (ret) {
99bc5d55
JSJ		 1049 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
1050 "BM_%d : beiscsi_init_irqs-Failed to"
1051 "register msix for i = %d\n",
1052 i);
8fcfb210 1053 kfree(phba->msi_name[i]);
4f5af07e
JK		 1054 goto free_msix_irqs;
1055 }
bfead3b2 1056 }
8fcfb210
JK		 1057 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
1058 if (!phba->msi_name[i]) {
1059 ret = -ENOMEM;
1060 goto free_msix_irqs;
1061 }
1062 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
1063 phba->shost->host_no);
bfead3b2 1064 msix_vec = phba->msix_entries[i].vector;
8fcfb210 1065 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
bfead3b2 1066 &phwi_context->be_eq[i]);
4f5af07e 1067 if (ret) {
99bc5d55
JSJ		 1068 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT ,
1069 "BM_%d : beiscsi_init_irqs-"
1070 "Failed to register beiscsi_msix_mcc\n");
8fcfb210 1071 kfree(phba->msi_name[i]);
4f5af07e
JK		 1072 goto free_msix_irqs;
1073 }
1074
bfead3b2
JK		 1075 } else {
1076 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
1077 "beiscsi", phba);
1078 if (ret) {
99bc5d55
JSJ		 1079 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
1080 "BM_%d : beiscsi_init_irqs-"
1081 "Failed to register irq\\n");
bfead3b2
JK		 1082 return ret;
1083 }
6733b39a
JK		 1084 }
1085 return 0;
4f5af07e 1086free_msix_irqs:
8fcfb210
JK		 1087 for (j = i - 1; j >= 0; j--) {
1088 kfree(phba->msi_name[j]);
1089 msix_vec = phba->msix_entries[j].vector;
4f5af07e 1090 free_irq(msix_vec, &phwi_context->be_eq[j]);
8fcfb210 1091 }
4f5af07e 1092 return ret;
6733b39a
JK		 1093}
1094
1095static void hwi_ring_cq_db(struct beiscsi_hba *phba,
1096 unsigned int id, unsigned int num_processed,
1097 unsigned char rearm, unsigned char event)
1098{
1099 u32 val = 0;
1100 val |= id & DB_CQ_RING_ID_MASK;
1101 if (rearm)
1102 val |= 1 << DB_CQ_REARM_SHIFT;
1103 val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
1104 iowrite32(val, phba->db_va + DB_CQ_OFFSET);
1105}
1106
6733b39a
JK		 1107static unsigned int
1108beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
1109 struct beiscsi_hba *phba,
6733b39a
JK		 1110 struct pdu_base *ppdu,
1111 unsigned long pdu_len,
1112 void *pbuffer, unsigned long buf_len)
1113{
1114 struct iscsi_conn *conn = beiscsi_conn->conn;
1115 struct iscsi_session *session = conn->session;
bfead3b2
JK		 1116 struct iscsi_task *task;
1117 struct beiscsi_io_task *io_task;
1118 struct iscsi_hdr *login_hdr;
6733b39a
JK		 1119
1120 switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
1121 PDUBASE_OPCODE_MASK) {
1122 case ISCSI_OP_NOOP_IN:
1123 pbuffer = NULL;
1124 buf_len = 0;
1125 break;
1126 case ISCSI_OP_ASYNC_EVENT:
1127 break;
1128 case ISCSI_OP_REJECT:
1129 WARN_ON(!pbuffer);
1130 WARN_ON(!(buf_len == 48));
99bc5d55
JSJ		 1131 beiscsi_log(phba, KERN_ERR,
1132 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1133 "BM_%d : In ISCSI_OP_REJECT\n");
6733b39a
JK		 1134 break;
1135 case ISCSI_OP_LOGIN_RSP:
7bd6e25c 1136 case ISCSI_OP_TEXT_RSP:
bfead3b2
JK		 1137 task = conn->login_task;
1138 io_task = task->dd_data;
1139 login_hdr = (struct iscsi_hdr *)ppdu;
1140 login_hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1141 break;
1142 default:
99bc5d55
JSJ		 1143 beiscsi_log(phba, KERN_WARNING,
1144 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1145 "BM_%d : Unrecognized opcode 0x%x in async msg\n",
1146 (ppdu->
6733b39a 1147 dw[offsetof(struct amap_pdu_base, opcode) / 32]
99bc5d55 1148 & PDUBASE_OPCODE_MASK));
6733b39a
JK		 1149 return 1;
1150 }
1151
1152 spin_lock_bh(&session->lock);
1153 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
1154 spin_unlock_bh(&session->lock);
1155 return 0;
1156}
1157
1158static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
1159{
1160 struct sgl_handle *psgl_handle;
1161
1162 if (phba->io_sgl_hndl_avbl) {
99bc5d55
JSJ		 1163 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1164 "BM_%d : In alloc_io_sgl_handle,"
1165 " io_sgl_alloc_index=%d\n",
1166 phba->io_sgl_alloc_index);
1167
6733b39a
JK		 1168 psgl_handle = phba->io_sgl_hndl_base[phba->
1169 io_sgl_alloc_index];
1170 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
1171 phba->io_sgl_hndl_avbl--;
bfead3b2
JK		 1172 if (phba->io_sgl_alloc_index == (phba->params.
1173 ios_per_ctrl - 1))
6733b39a
JK		 1174 phba->io_sgl_alloc_index = 0;
1175 else
1176 phba->io_sgl_alloc_index++;
1177 } else
1178 psgl_handle = NULL;
1179 return psgl_handle;
1180}
1181
1182static void
1183free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1184{
99bc5d55
JSJ		 1185 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1186 "BM_%d : In free_,io_sgl_free_index=%d\n",
1187 phba->io_sgl_free_index);
1188
6733b39a
JK		 1189 if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
1190 /*
1191 * this can happen if clean_task is called on a task that
1192 * failed in xmit_task or alloc_pdu.
1193 */
99bc5d55
JSJ		 1194 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1195 "BM_%d : Double Free in IO SGL io_sgl_free_index=%d,"
1196 "value there=%p\n", phba->io_sgl_free_index,
1197 phba->io_sgl_hndl_base
1198 [phba->io_sgl_free_index]);
6733b39a
JK		 1199 return;
1200 }
1201 phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
1202 phba->io_sgl_hndl_avbl++;
1203 if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
1204 phba->io_sgl_free_index = 0;
1205 else
1206 phba->io_sgl_free_index++;
1207}
1208
1209/**
1210 * alloc_wrb_handle - To allocate a wrb handle
1211 * @phba: The hba pointer
1212 * @cid: The cid to use for allocation
6733b39a
JK		 1213 *
1214 * This happens under session_lock until submission to chip
1215 */
d5431488 1216struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
6733b39a
JK		 1217{
1218 struct hwi_wrb_context *pwrb_context;
1219 struct hwi_controller *phwi_ctrlr;
d5431488 1220 struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
a7909b39 1221 uint16_t cri_index = BE_GET_CRI_FROM_CID(cid);
6733b39a
JK		 1222
1223 phwi_ctrlr = phba->phwi_ctrlr;
a7909b39 1224 pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
d5431488 1225 if (pwrb_context->wrb_handles_available >= 2) {
bfead3b2
JK		 1226 pwrb_handle = pwrb_context->pwrb_handle_base[
1227 pwrb_context->alloc_index];
1228 pwrb_context->wrb_handles_available--;
bfead3b2
JK		 1229 if (pwrb_context->alloc_index ==
1230 (phba->params.wrbs_per_cxn - 1))
1231 pwrb_context->alloc_index = 0;
1232 else
1233 pwrb_context->alloc_index++;
d5431488
JK		 1234 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1235 pwrb_context->alloc_index];
1236 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
bfead3b2
JK		 1237 } else
1238 pwrb_handle = NULL;
6733b39a
JK		 1239 return pwrb_handle;
1240}
1241
1242/**
1243 * free_wrb_handle - To free the wrb handle back to pool
1244 * @phba: The hba pointer
1245 * @pwrb_context: The context to free from
1246 * @pwrb_handle: The wrb_handle to free
1247 *
1248 * This happens under session_lock until submission to chip
1249 */
1250static void
1251free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1252 struct wrb_handle *pwrb_handle)
1253{
32951dd8 1254 pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
bfead3b2
JK		 1255 pwrb_context->wrb_handles_available++;
1256 if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1257 pwrb_context->free_index = 0;
1258 else
1259 pwrb_context->free_index++;
1260
99bc5d55
JSJ		 1261 beiscsi_log(phba, KERN_INFO,
1262 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1263 "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x"
1264 "wrb_handles_available=%d\n",
1265 pwrb_handle, pwrb_context->free_index,
1266 pwrb_context->wrb_handles_available);
6733b39a
JK		 1267}
1268
1269static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1270{
1271 struct sgl_handle *psgl_handle;
1272
1273 if (phba->eh_sgl_hndl_avbl) {
1274 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1275 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
99bc5d55
JSJ		 1276 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1277 "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n",
1278 phba->eh_sgl_alloc_index,
1279 phba->eh_sgl_alloc_index);
1280
6733b39a
JK		 1281 phba->eh_sgl_hndl_avbl--;
1282 if (phba->eh_sgl_alloc_index ==
1283 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1284 1))
1285 phba->eh_sgl_alloc_index = 0;
1286 else
1287 phba->eh_sgl_alloc_index++;
1288 } else
1289 psgl_handle = NULL;
1290 return psgl_handle;
1291}
1292
1293void
1294free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1295{
1296
99bc5d55
JSJ		 1297 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1298 "BM_%d : In free_mgmt_sgl_handle,"
1299 "eh_sgl_free_index=%d\n",
1300 phba->eh_sgl_free_index);
1301
6733b39a
JK		 1302 if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1303 /*
1304 * this can happen if clean_task is called on a task that
1305 * failed in xmit_task or alloc_pdu.
1306 */
99bc5d55
JSJ		 1307 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
1308 "BM_%d : Double Free in eh SGL ,"
1309 "eh_sgl_free_index=%d\n",
1310 phba->eh_sgl_free_index);
6733b39a
JK		 1311 return;
1312 }
1313 phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1314 phba->eh_sgl_hndl_avbl++;
1315 if (phba->eh_sgl_free_index ==
1316 (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1317 phba->eh_sgl_free_index = 0;
1318 else
1319 phba->eh_sgl_free_index++;
1320}
1321
1322static void
1323be_complete_io(struct beiscsi_conn *beiscsi_conn,
73133261
JSJ		 1324 struct iscsi_task *task,
1325 struct common_sol_cqe *csol_cqe)
6733b39a
JK		 1326{
1327 struct beiscsi_io_task *io_task = task->dd_data;
1328 struct be_status_bhs *sts_bhs =
1329 (struct be_status_bhs *)io_task->cmd_bhs;
1330 struct iscsi_conn *conn = beiscsi_conn->conn;
6733b39a
JK		 1331 unsigned char *sense;
1332 u32 resid = 0, exp_cmdsn, max_cmdsn;
1333 u8 rsp, status, flags;
1334
73133261
JSJ		 1335 exp_cmdsn = csol_cqe->exp_cmdsn;
1336 max_cmdsn = (csol_cqe->exp_cmdsn +
1337 csol_cqe->cmd_wnd - 1);
1338 rsp = csol_cqe->i_resp;
1339 status = csol_cqe->i_sts;
1340 flags = csol_cqe->i_flags;
1341 resid = csol_cqe->res_cnt;
1342
bd535451
JK		 1343 if (!task->sc) {
1344 if (io_task->scsi_cmnd)
1345 scsi_dma_unmap(io_task->scsi_cmnd);
6733b39a 1346
bd535451
JK		 1347 return;
1348 }
6733b39a
JK		 1349 task->sc->result = (DID_OK << 16) | status;
1350 if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1351 task->sc->result = DID_ERROR << 16;
1352 goto unmap;
1353 }
1354
1355 /* bidi not initially supported */
1356 if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
6733b39a
JK		 1357 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1358 task->sc->result = DID_ERROR << 16;
1359
1360 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1361 scsi_set_resid(task->sc, resid);
1362 if (!status && (scsi_bufflen(task->sc) - resid <
1363 task->sc->underflow))
1364 task->sc->result = DID_ERROR << 16;
1365 }
1366 }
1367
1368 if (status == SAM_STAT_CHECK_CONDITION) {
4053a4be 1369 u16 sense_len;
bfead3b2 1370 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
4053a4be 1371
6733b39a 1372 sense = sts_bhs->sense_info + sizeof(unsigned short);
4053a4be 1373 sense_len = be16_to_cpu(*slen);
6733b39a
JK		 1374 memcpy(task->sc->sense_buffer, sense,
1375 min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1376 }
756d29c8 1377
73133261
JSJ		 1378 if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ)
1379 conn->rxdata_octets += resid;
6733b39a
JK		 1380unmap:
1381 scsi_dma_unmap(io_task->scsi_cmnd);
1382 iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1383}
1384
1385static void
1386be_complete_logout(struct beiscsi_conn *beiscsi_conn,
73133261
JSJ		 1387 struct iscsi_task *task,
1388 struct common_sol_cqe *csol_cqe)
6733b39a
JK		 1389{
1390 struct iscsi_logout_rsp *hdr;
bfead3b2 1391 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 1392 struct iscsi_conn *conn = beiscsi_conn->conn;
1393
1394 hdr = (struct iscsi_logout_rsp *)task->hdr;
7bd6e25c 1395 hdr->opcode = ISCSI_OP_LOGOUT_RSP;
6733b39a
JK		 1396 hdr->t2wait = 5;
1397 hdr->t2retain = 0;
73133261
JSJ		 1398 hdr->flags = csol_cqe->i_flags;
1399 hdr->response = csol_cqe->i_resp;
702dc5e8
JK		 1400 hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1401 hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1402 csol_cqe->cmd_wnd - 1);
73133261 1403
7bd6e25c
JK		 1404 hdr->dlength[0] = 0;
1405 hdr->dlength[1] = 0;
1406 hdr->dlength[2] = 0;
6733b39a 1407 hdr->hlength = 0;
bfead3b2 1408 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1409 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1410}
1411
1412static void
1413be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
73133261
JSJ		 1414 struct iscsi_task *task,
1415 struct common_sol_cqe *csol_cqe)
6733b39a
JK		 1416{
1417 struct iscsi_tm_rsp *hdr;
1418 struct iscsi_conn *conn = beiscsi_conn->conn;
bfead3b2 1419 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 1420
1421 hdr = (struct iscsi_tm_rsp *)task->hdr;
7bd6e25c 1422 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
73133261
JSJ		 1423 hdr->flags = csol_cqe->i_flags;
1424 hdr->response = csol_cqe->i_resp;
702dc5e8
JK		 1425 hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1426 hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1427 csol_cqe->cmd_wnd - 1);
73133261 1428
bfead3b2 1429 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1430 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1431}
1432
1433static void
1434hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1435 struct beiscsi_hba *phba, struct sol_cqe *psol)
1436{
1437 struct hwi_wrb_context *pwrb_context;
bfead3b2 1438 struct wrb_handle *pwrb_handle = NULL;
6733b39a 1439 struct hwi_controller *phwi_ctrlr;
bfead3b2
JK		 1440 struct iscsi_task *task;
1441 struct beiscsi_io_task *io_task;
a7909b39 1442 uint16_t wrb_index, cid, cri_index;
6733b39a
JK		 1443
1444 phwi_ctrlr = phba->phwi_ctrlr;
2c9dfd36
JK		 1445 if (is_chip_be2_be3r(phba)) {
1446 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
73133261 1447 wrb_idx, psol);
2c9dfd36 1448 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
73133261
JSJ		 1449 cid, psol);
1450 } else {
2c9dfd36 1451 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
73133261 1452 wrb_idx, psol);
2c9dfd36 1453 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
73133261
JSJ		 1454 cid, psol);
1455 }
1456
a7909b39
JK		 1457 cri_index = BE_GET_CRI_FROM_CID(cid);
1458 pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
73133261 1459 pwrb_handle = pwrb_context->pwrb_handle_basestd[wrb_index];
32951dd8 1460 task = pwrb_handle->pio_handle;
35e66019 1461
bfead3b2 1462 io_task = task->dd_data;
4a4a11b9
JK		 1463 memset(io_task->pwrb_handle->pwrb, 0, sizeof(struct iscsi_wrb));
1464 iscsi_put_task(task);
6733b39a
JK		 1465}
1466
1467static void
1468be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
73133261
JSJ		 1469 struct iscsi_task *task,
1470 struct common_sol_cqe *csol_cqe)
6733b39a
JK		 1471{
1472 struct iscsi_nopin *hdr;
1473 struct iscsi_conn *conn = beiscsi_conn->conn;
bfead3b2 1474 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 1475
1476 hdr = (struct iscsi_nopin *)task->hdr;
73133261
JSJ		 1477 hdr->flags = csol_cqe->i_flags;
1478 hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
702dc5e8
JK		 1479 hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1480 csol_cqe->cmd_wnd - 1);
73133261 1481
6733b39a 1482 hdr->opcode = ISCSI_OP_NOOP_IN;
bfead3b2 1483 hdr->itt = io_task->libiscsi_itt;
6733b39a
JK		 1484 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1485}
1486
73133261
JSJ		 1487static void adapter_get_sol_cqe(struct beiscsi_hba *phba,
1488 struct sol_cqe *psol,
1489 struct common_sol_cqe *csol_cqe)
1490{
2c9dfd36
JK		 1491 if (is_chip_be2_be3r(phba)) {
1492 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe,
1493 i_exp_cmd_sn, psol);
1494 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe,
1495 i_res_cnt, psol);
1496 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe,
1497 i_cmd_wnd, psol);
1498 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe,
1499 wrb_index, psol);
1500 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe,
1501 cid, psol);
1502 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1503 hw_sts, psol);
1504 csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe,
1505 i_resp, psol);
1506 csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1507 i_sts, psol);
1508 csol_cqe->i_flags = AMAP_GET_BITS(struct amap_sol_cqe,
1509 i_flags, psol);
1510 } else {
73133261
JSJ		 1511 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1512 i_exp_cmd_sn, psol);
1513 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1514 i_res_cnt, psol);
1515 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1516 wrb_index, psol);
1517 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1518 cid, psol);
1519 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1520 hw_sts, psol);
702dc5e8 1521 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe_v2,
73133261
JSJ		 1522 i_cmd_wnd, psol);
1523 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1524 cmd_cmpl, psol))
1525 csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1526 i_sts, psol);
1527 else
1528 csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1529 i_sts, psol);
1530 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1531 u, psol))
1532 csol_cqe->i_flags = ISCSI_FLAG_CMD_UNDERFLOW;
1533
1534 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1535 o, psol))
1536 csol_cqe->i_flags |= ISCSI_FLAG_CMD_OVERFLOW;
73133261
JSJ		 1537 }
1538}
1539
1540
6733b39a
JK		 1541static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1542 struct beiscsi_hba *phba, struct sol_cqe *psol)
1543{
1544 struct hwi_wrb_context *pwrb_context;
1545 struct wrb_handle *pwrb_handle;
1546 struct iscsi_wrb *pwrb = NULL;
1547 struct hwi_controller *phwi_ctrlr;
1548 struct iscsi_task *task;
bfead3b2 1549 unsigned int type;
6733b39a
JK		 1550 struct iscsi_conn *conn = beiscsi_conn->conn;
1551 struct iscsi_session *session = conn->session;
73133261 1552 struct common_sol_cqe csol_cqe = {0};
a7909b39 1553 uint16_t cri_index = 0;
6733b39a
JK		 1554
1555 phwi_ctrlr = phba->phwi_ctrlr;
73133261
JSJ		 1556
1557 /* Copy the elements to a common structure */
1558 adapter_get_sol_cqe(phba, psol, &csol_cqe);
1559
a7909b39
JK		 1560 cri_index = BE_GET_CRI_FROM_CID(csol_cqe.cid);
1561 pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
73133261
JSJ		 1562
1563 pwrb_handle = pwrb_context->pwrb_handle_basestd[
1564 csol_cqe.wrb_index];
1565
32951dd8
JK		 1566 task = pwrb_handle->pio_handle;
1567 pwrb = pwrb_handle->pwrb;
73133261 1568 type = ((struct beiscsi_io_task *)task->dd_data)->wrb_type;
32951dd8 1569
bfead3b2
JK		 1570 spin_lock_bh(&session->lock);
1571 switch (type) {
6733b39a
JK		 1572 case HWH_TYPE_IO:
1573 case HWH_TYPE_IO_RD:
1574 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
dafab8e0 1575 ISCSI_OP_NOOP_OUT)
73133261 1576 be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
dafab8e0 1577 else
73133261 1578 be_complete_io(beiscsi_conn, task, &csol_cqe);
6733b39a
JK		 1579 break;
1580
1581 case HWH_TYPE_LOGOUT:
dafab8e0 1582 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
73133261 1583 be_complete_logout(beiscsi_conn, task, &csol_cqe);
dafab8e0 1584 else
73133261 1585 be_complete_tmf(beiscsi_conn, task, &csol_cqe);
6733b39a
JK		 1586 break;
1587
1588 case HWH_TYPE_LOGIN:
99bc5d55
JSJ		 1589 beiscsi_log(phba, KERN_ERR,
1590 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1591 "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in"
1592 " hwi_complete_cmd- Solicited path\n");
6733b39a
JK		 1593 break;
1594
6733b39a 1595 case HWH_TYPE_NOP:
73133261 1596 be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
6733b39a
JK		 1597 break;
1598
1599 default:
99bc5d55
JSJ		 1600 beiscsi_log(phba, KERN_WARNING,
1601 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1602 "BM_%d : In hwi_complete_cmd, unknown type = %d"
1603 "wrb_index 0x%x CID 0x%x\n", type,
73133261
JSJ		 1604 csol_cqe.wrb_index,
1605 csol_cqe.cid);
6733b39a
JK		 1606 break;
1607 }
35e66019 1608
6733b39a
JK		 1609 spin_unlock_bh(&session->lock);
1610}
1611
1612static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1613 *pasync_ctx, unsigned int is_header,
1614 unsigned int host_write_ptr)
1615{
1616 if (is_header)
1617 return &pasync_ctx->async_entry[host_write_ptr].
1618 header_busy_list;
1619 else
1620 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1621}
1622
1623static struct async_pdu_handle *
1624hwi_get_async_handle(struct beiscsi_hba *phba,
1625 struct beiscsi_conn *beiscsi_conn,
1626 struct hwi_async_pdu_context *pasync_ctx,
1627 struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1628{
1629 struct be_bus_address phys_addr;
1630 struct list_head *pbusy_list;
1631 struct async_pdu_handle *pasync_handle = NULL;
6733b39a 1632 unsigned char is_header = 0;
73133261
JSJ		 1633 unsigned int index, dpl;
1634
2c9dfd36
JK		 1635 if (is_chip_be2_be3r(phba)) {
1636 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
73133261 1637 dpl, pdpdu_cqe);
2c9dfd36 1638 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
73133261
JSJ		 1639 index, pdpdu_cqe);
1640 } else {
2c9dfd36 1641 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
73133261 1642 dpl, pdpdu_cqe);
2c9dfd36 1643 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
73133261
JSJ		 1644 index, pdpdu_cqe);
1645 }
6733b39a
JK		 1646
1647 phys_addr.u.a32.address_lo =
73133261
JSJ		 1648 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1649 db_addr_lo) / 32] - dpl);
6733b39a 1650 phys_addr.u.a32.address_hi =
73133261
JSJ		 1651 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1652 db_addr_hi) / 32];
6733b39a
JK		 1653
1654 phys_addr.u.a64.address =
1655 *((unsigned long long *)(&phys_addr.u.a64.address));
1656
1657 switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1658 & PDUCQE_CODE_MASK) {
1659 case UNSOL_HDR_NOTIFY:
1660 is_header = 1;
1661
73133261
JSJ		 1662 pbusy_list = hwi_get_async_busy_list(pasync_ctx,
1663 is_header, index);
6733b39a
JK		 1664 break;
1665 case UNSOL_DATA_NOTIFY:
73133261
JSJ		 1666 pbusy_list = hwi_get_async_busy_list(pasync_ctx,
1667 is_header, index);
6733b39a
JK		 1668 break;
1669 default:
1670 pbusy_list = NULL;
99bc5d55
JSJ		 1671 beiscsi_log(phba, KERN_WARNING,
1672 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1673 "BM_%d : Unexpected code=%d\n",
1674 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1675 code) / 32] & PDUCQE_CODE_MASK);
6733b39a
JK		 1676 return NULL;
1677 }
1678
6733b39a
JK		 1679 WARN_ON(list_empty(pbusy_list));
1680 list_for_each_entry(pasync_handle, pbusy_list, link) {
dc63aac6 1681 if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address)
6733b39a
JK		 1682 break;
1683 }
1684
1685 WARN_ON(!pasync_handle);
1686
8a86e833
JK		 1687 pasync_handle->cri = BE_GET_ASYNC_CRI_FROM_CID(
1688 beiscsi_conn->beiscsi_conn_cid);
6733b39a 1689 pasync_handle->is_header = is_header;
73133261
JSJ		 1690 pasync_handle->buffer_len = dpl;
1691 *pcq_index = index;
6733b39a 1692
6733b39a
JK		 1693 return pasync_handle;
1694}
1695
1696static unsigned int
99bc5d55
JSJ		 1697hwi_update_async_writables(struct beiscsi_hba *phba,
1698 struct hwi_async_pdu_context *pasync_ctx,
1699 unsigned int is_header, unsigned int cq_index)
6733b39a
JK		 1700{
1701 struct list_head *pbusy_list;
1702 struct async_pdu_handle *pasync_handle;
1703 unsigned int num_entries, writables = 0;
1704 unsigned int *pep_read_ptr, *pwritables;
1705
dc63aac6 1706 num_entries = pasync_ctx->num_entries;
6733b39a
JK		 1707 if (is_header) {
1708 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1709 pwritables = &pasync_ctx->async_header.writables;
6733b39a
JK		 1710 } else {
1711 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1712 pwritables = &pasync_ctx->async_data.writables;
6733b39a
JK		 1713 }
1714
1715 while ((*pep_read_ptr) != cq_index) {
1716 (*pep_read_ptr)++;
1717 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1718
1719 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1720 *pep_read_ptr);
1721 if (writables == 0)
1722 WARN_ON(list_empty(pbusy_list));
1723
1724 if (!list_empty(pbusy_list)) {
1725 pasync_handle = list_entry(pbusy_list->next,
1726 struct async_pdu_handle,
1727 link);
1728 WARN_ON(!pasync_handle);
1729 pasync_handle->consumed = 1;
1730 }
1731
1732 writables++;
1733 }
1734
1735 if (!writables) {
99bc5d55
JSJ		 1736 beiscsi_log(phba, KERN_ERR,
1737 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1738 "BM_%d : Duplicate notification received - index 0x%x!!\n",
1739 cq_index);
6733b39a
JK		 1740 WARN_ON(1);
1741 }
1742
1743 *pwritables = *pwritables + writables;
1744 return 0;
1745}
1746
9728d8d0 1747static void hwi_free_async_msg(struct beiscsi_hba *phba,
8a86e833
JK		 1748 struct hwi_async_pdu_context *pasync_ctx,
1749 unsigned int cri)
6733b39a 1750{
6733b39a
JK		 1751 struct async_pdu_handle *pasync_handle, *tmp_handle;
1752 struct list_head *plist;
6733b39a 1753
6733b39a 1754 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
6733b39a
JK		 1755 list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1756 list_del(&pasync_handle->link);
1757
9728d8d0 1758 if (pasync_handle->is_header) {
6733b39a
JK		 1759 list_add_tail(&pasync_handle->link,
1760 &pasync_ctx->async_header.free_list);
1761 pasync_ctx->async_header.free_entries++;
6733b39a
JK		 1762 } else {
1763 list_add_tail(&pasync_handle->link,
1764 &pasync_ctx->async_data.free_list);
1765 pasync_ctx->async_data.free_entries++;
6733b39a
JK		 1766 }
1767 }
1768
1769 INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1770 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1771 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
6733b39a
JK		 1772}
1773
1774static struct phys_addr *
1775hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1776 unsigned int is_header, unsigned int host_write_ptr)
1777{
1778 struct phys_addr *pasync_sge = NULL;
1779
1780 if (is_header)
1781 pasync_sge = pasync_ctx->async_header.ring_base;
1782 else
1783 pasync_sge = pasync_ctx->async_data.ring_base;
1784
1785 return pasync_sge + host_write_ptr;
1786}
1787
1788static void hwi_post_async_buffers(struct beiscsi_hba *phba,
8a86e833 1789 unsigned int is_header, uint8_t ulp_num)
6733b39a
JK		 1790{
1791 struct hwi_controller *phwi_ctrlr;
1792 struct hwi_async_pdu_context *pasync_ctx;
1793 struct async_pdu_handle *pasync_handle;
1794 struct list_head *pfree_link, *pbusy_list;
1795 struct phys_addr *pasync_sge;
1796 unsigned int ring_id, num_entries;
8a86e833 1797 unsigned int host_write_num, doorbell_offset;
6733b39a
JK		 1798 unsigned int writables;
1799 unsigned int i = 0;
1800 u32 doorbell = 0;
1801
1802 phwi_ctrlr = phba->phwi_ctrlr;
8a86e833 1803 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, ulp_num);
dc63aac6 1804 num_entries = pasync_ctx->num_entries;
6733b39a
JK		 1805
1806 if (is_header) {
6733b39a
JK		 1807 writables = min(pasync_ctx->async_header.writables,
1808 pasync_ctx->async_header.free_entries);
1809 pfree_link = pasync_ctx->async_header.free_list.next;
1810 host_write_num = pasync_ctx->async_header.host_write_ptr;
8a86e833
JK		 1811 ring_id = phwi_ctrlr->default_pdu_hdr[ulp_num].id;
1812 doorbell_offset = phwi_ctrlr->default_pdu_hdr[ulp_num].
1813 doorbell_offset;
6733b39a 1814 } else {
6733b39a
JK		 1815 writables = min(pasync_ctx->async_data.writables,
1816 pasync_ctx->async_data.free_entries);
1817 pfree_link = pasync_ctx->async_data.free_list.next;
1818 host_write_num = pasync_ctx->async_data.host_write_ptr;
8a86e833
JK		 1819 ring_id = phwi_ctrlr->default_pdu_data[ulp_num].id;
1820 doorbell_offset = phwi_ctrlr->default_pdu_data[ulp_num].
1821 doorbell_offset;
6733b39a
JK		 1822 }
1823
1824 writables = (writables / 8) * 8;
1825 if (writables) {
1826 for (i = 0; i < writables; i++) {
1827 pbusy_list =
1828 hwi_get_async_busy_list(pasync_ctx, is_header,
1829 host_write_num);
1830 pasync_handle =
1831 list_entry(pfree_link, struct async_pdu_handle,
1832 link);
1833 WARN_ON(!pasync_handle);
1834 pasync_handle->consumed = 0;
1835
1836 pfree_link = pfree_link->next;
1837
1838 pasync_sge = hwi_get_ring_address(pasync_ctx,
1839 is_header, host_write_num);
1840
1841 pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1842 pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1843
1844 list_move(&pasync_handle->link, pbusy_list);
1845
1846 host_write_num++;
1847 host_write_num = host_write_num % num_entries;
1848 }
1849
1850 if (is_header) {
1851 pasync_ctx->async_header.host_write_ptr =
1852 host_write_num;
1853 pasync_ctx->async_header.free_entries -= writables;
1854 pasync_ctx->async_header.writables -= writables;
1855 pasync_ctx->async_header.busy_entries += writables;
1856 } else {
1857 pasync_ctx->async_data.host_write_ptr = host_write_num;
1858 pasync_ctx->async_data.free_entries -= writables;
1859 pasync_ctx->async_data.writables -= writables;
1860 pasync_ctx->async_data.busy_entries += writables;
1861 }
1862
1863 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1864 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1865 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1866 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1867 << DB_DEF_PDU_CQPROC_SHIFT;
1868
8a86e833 1869 iowrite32(doorbell, phba->db_va + doorbell_offset);
6733b39a
JK		 1870 }
1871}
1872
1873static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1874 struct beiscsi_conn *beiscsi_conn,
1875 struct i_t_dpdu_cqe *pdpdu_cqe)
1876{
1877 struct hwi_controller *phwi_ctrlr;
1878 struct hwi_async_pdu_context *pasync_ctx;
1879 struct async_pdu_handle *pasync_handle = NULL;
1880 unsigned int cq_index = -1;
8a86e833
JK		 1881 uint16_t cri_index = BE_GET_CRI_FROM_CID(
1882 beiscsi_conn->beiscsi_conn_cid);
6733b39a
JK		 1883
1884 phwi_ctrlr = phba->phwi_ctrlr;
8a86e833
JK		 1885 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr,
1886 BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
1887 cri_index));
6733b39a
JK		 1888
1889 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1890 pdpdu_cqe, &cq_index);
1891 BUG_ON(pasync_handle->is_header != 0);
1892 if (pasync_handle->consumed == 0)
99bc5d55
JSJ		 1893 hwi_update_async_writables(phba, pasync_ctx,
1894 pasync_handle->is_header, cq_index);
6733b39a 1895
8a86e833
JK		 1896 hwi_free_async_msg(phba, pasync_ctx, pasync_handle->cri);
1897 hwi_post_async_buffers(phba, pasync_handle->is_header,
1898 BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
1899 cri_index));
6733b39a
JK		 1900}
1901
1902static unsigned int
1903hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1904 struct beiscsi_hba *phba,
1905 struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1906{
1907 struct list_head *plist;
1908 struct async_pdu_handle *pasync_handle;
1909 void *phdr = NULL;
1910 unsigned int hdr_len = 0, buf_len = 0;
1911 unsigned int status, index = 0, offset = 0;
1912 void *pfirst_buffer = NULL;
1913 unsigned int num_buf = 0;
1914
1915 plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1916
1917 list_for_each_entry(pasync_handle, plist, link) {
1918 if (index == 0) {
1919 phdr = pasync_handle->pbuffer;
1920 hdr_len = pasync_handle->buffer_len;
1921 } else {
1922 buf_len = pasync_handle->buffer_len;
1923 if (!num_buf) {
1924 pfirst_buffer = pasync_handle->pbuffer;
1925 num_buf++;
1926 }
1927 memcpy(pfirst_buffer + offset,
1928 pasync_handle->pbuffer, buf_len);
f2ba02b8 1929 offset += buf_len;
6733b39a
JK		 1930 }
1931 index++;
1932 }
1933
1934 status = beiscsi_process_async_pdu(beiscsi_conn, phba,
7da50879 1935 phdr, hdr_len, pfirst_buffer,
f2ba02b8 1936 offset);
6733b39a 1937
8a86e833 1938 hwi_free_async_msg(phba, pasync_ctx, cri);
6733b39a
JK		 1939 return 0;
1940}
1941
1942static unsigned int
1943hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1944 struct beiscsi_hba *phba,
1945 struct async_pdu_handle *pasync_handle)
1946{
1947 struct hwi_async_pdu_context *pasync_ctx;
1948 struct hwi_controller *phwi_ctrlr;
1949 unsigned int bytes_needed = 0, status = 0;
1950 unsigned short cri = pasync_handle->cri;
1951 struct pdu_base *ppdu;
1952
1953 phwi_ctrlr = phba->phwi_ctrlr;
8a86e833
JK		 1954 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr,
1955 BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
1956 BE_GET_CRI_FROM_CID(beiscsi_conn->
1957 beiscsi_conn_cid)));
6733b39a
JK		 1958
1959 list_del(&pasync_handle->link);
1960 if (pasync_handle->is_header) {
1961 pasync_ctx->async_header.busy_entries--;
1962 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
8a86e833 1963 hwi_free_async_msg(phba, pasync_ctx, cri);
6733b39a
JK		 1964 BUG();
1965 }
1966
1967 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1968 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1969 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1970 (unsigned short)pasync_handle->buffer_len;
1971 list_add_tail(&pasync_handle->link,
1972 &pasync_ctx->async_entry[cri].wait_queue.list);
1973
1974 ppdu = pasync_handle->pbuffer;
1975 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1976 data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1977 0xFFFF0000) | ((be16_to_cpu((ppdu->
1978 dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1979 & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1980
1981 if (status == 0) {
1982 pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1983 bytes_needed;
1984
1985 if (bytes_needed == 0)
1986 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1987 pasync_ctx, cri);
1988 }
1989 } else {
1990 pasync_ctx->async_data.busy_entries--;
1991 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1992 list_add_tail(&pasync_handle->link,
1993 &pasync_ctx->async_entry[cri].wait_queue.
1994 list);
1995 pasync_ctx->async_entry[cri].wait_queue.
1996 bytes_received +=
1997 (unsigned short)pasync_handle->buffer_len;
1998
1999 if (pasync_ctx->async_entry[cri].wait_queue.
2000 bytes_received >=
2001 pasync_ctx->async_entry[cri].wait_queue.
2002 bytes_needed)
2003 status = hwi_fwd_async_msg(beiscsi_conn, phba,
2004 pasync_ctx, cri);
2005 }
2006 }
2007 return status;
2008}
2009
2010static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
2011 struct beiscsi_hba *phba,
2012 struct i_t_dpdu_cqe *pdpdu_cqe)
2013{
2014 struct hwi_controller *phwi_ctrlr;
2015 struct hwi_async_pdu_context *pasync_ctx;
2016 struct async_pdu_handle *pasync_handle = NULL;
2017 unsigned int cq_index = -1;
8a86e833
JK		 2018 uint16_t cri_index = BE_GET_CRI_FROM_CID(
2019 beiscsi_conn->beiscsi_conn_cid);
6733b39a
JK		 2020
2021 phwi_ctrlr = phba->phwi_ctrlr;
8a86e833
JK		 2022 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr,
2023 BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
2024 cri_index));
2025
6733b39a
JK		 2026 pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
2027 pdpdu_cqe, &cq_index);
2028
2029 if (pasync_handle->consumed == 0)
99bc5d55
JSJ		 2030 hwi_update_async_writables(phba, pasync_ctx,
2031 pasync_handle->is_header, cq_index);
2032
6733b39a 2033 hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
8a86e833
JK		 2034 hwi_post_async_buffers(phba, pasync_handle->is_header,
2035 BEISCSI_GET_ULP_FROM_CRI(
2036 phwi_ctrlr, cri_index));
6733b39a
JK		 2037}
2038
756d29c8
JK		 2039static void beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
2040{
2041 struct be_queue_info *mcc_cq;
2042 struct be_mcc_compl *mcc_compl;
2043 unsigned int num_processed = 0;
2044
2045 mcc_cq = &phba->ctrl.mcc_obj.cq;
2046 mcc_compl = queue_tail_node(mcc_cq);
2047 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
2048 while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
2049
2050 if (num_processed >= 32) {
2051 hwi_ring_cq_db(phba, mcc_cq->id,
2052 num_processed, 0, 0);
2053 num_processed = 0;
2054 }
2055 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
2056 /* Interpret flags as an async trailer */
2057 if (is_link_state_evt(mcc_compl->flags))
2058 /* Interpret compl as a async link evt */
2059 beiscsi_async_link_state_process(phba,
2060 (struct be_async_event_link_state *) mcc_compl);
2061 else
99bc5d55
JSJ		 2062 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_MBOX,
2063 "BM_%d : Unsupported Async Event, flags"
2064 " = 0x%08x\n",
2065 mcc_compl->flags);
756d29c8
JK		 2066 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
2067 be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
2068 atomic_dec(&phba->ctrl.mcc_obj.q.used);
2069 }
2070
2071 mcc_compl->flags = 0;
2072 queue_tail_inc(mcc_cq);
2073 mcc_compl = queue_tail_node(mcc_cq);
2074 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
2075 num_processed++;
2076 }
2077
2078 if (num_processed > 0)
2079 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
2080
2081}
bfead3b2 2082
6763daae
JSJ		 2083/**
2084 * beiscsi_process_cq()- Process the Completion Queue
2085 * @pbe_eq: Event Q on which the Completion has come
2086 *
2087 * return
2088 * Number of Completion Entries processed.
2089 **/
bfead3b2 2090static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
6733b39a 2091{
6733b39a
JK		 2092 struct be_queue_info *cq;
2093 struct sol_cqe *sol;
2094 struct dmsg_cqe *dmsg;
2095 unsigned int num_processed = 0;
2096 unsigned int tot_nump = 0;
0a513dd8 2097 unsigned short code = 0, cid = 0;
a7909b39 2098 uint16_t cri_index = 0;
6733b39a 2099 struct beiscsi_conn *beiscsi_conn;
c2462288
JK		 2100 struct beiscsi_endpoint *beiscsi_ep;
2101 struct iscsi_endpoint *ep;
bfead3b2 2102 struct beiscsi_hba *phba;
6733b39a 2103
bfead3b2 2104 cq = pbe_eq->cq;
6733b39a 2105 sol = queue_tail_node(cq);
bfead3b2 2106 phba = pbe_eq->phba;
6733b39a
JK		 2107
2108 while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
2109 CQE_VALID_MASK) {
2110 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
2111
73133261
JSJ		 2112 code = (sol->dw[offsetof(struct amap_sol_cqe, code) /
2113 32] & CQE_CODE_MASK);
2114
2115 /* Get the CID */
2c9dfd36
JK		 2116 if (is_chip_be2_be3r(phba)) {
2117 cid = AMAP_GET_BITS(struct amap_sol_cqe, cid, sol);
2118 } else {
73133261
JSJ		 2119 if ((code == DRIVERMSG_NOTIFY) ||
2120 (code == UNSOL_HDR_NOTIFY) ||
2121 (code == UNSOL_DATA_NOTIFY))
2122 cid = AMAP_GET_BITS(
2123 struct amap_i_t_dpdu_cqe_v2,
2124 cid, sol);
2125 else
2126 cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
2127 cid, sol);
2c9dfd36 2128 }
32951dd8 2129
a7909b39
JK		 2130 cri_index = BE_GET_CRI_FROM_CID(cid);
2131 ep = phba->ep_array[cri_index];
c2462288
JK		 2132 beiscsi_ep = ep->dd_data;
2133 beiscsi_conn = beiscsi_ep->conn;
756d29c8 2134
6733b39a 2135 if (num_processed >= 32) {
bfead3b2 2136 hwi_ring_cq_db(phba, cq->id,
6733b39a
JK		 2137 num_processed, 0, 0);
2138 tot_nump += num_processed;
2139 num_processed = 0;
2140 }
2141
0a513dd8 2142 switch (code) {
6733b39a
JK		 2143 case SOL_CMD_COMPLETE:
2144 hwi_complete_cmd(beiscsi_conn, phba, sol);
2145 break;
2146 case DRIVERMSG_NOTIFY:
99bc5d55
JSJ		 2147 beiscsi_log(phba, KERN_INFO,
2148 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
6763daae
JSJ		 2149 "BM_%d : Received %s[%d] on CID : %d\n",
2150 cqe_desc[code], code, cid);
99bc5d55 2151
6733b39a
JK		 2152 dmsg = (struct dmsg_cqe *)sol;
2153 hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
2154 break;
2155 case UNSOL_HDR_NOTIFY:
99bc5d55
JSJ		 2156 beiscsi_log(phba, KERN_INFO,
2157 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
6763daae
JSJ		 2158 "BM_%d : Received %s[%d] on CID : %d\n",
2159 cqe_desc[code], code, cid);
99bc5d55 2160
8f09a3b9 2161 spin_lock_bh(&phba->async_pdu_lock);
bfead3b2
JK		 2162 hwi_process_default_pdu_ring(beiscsi_conn, phba,
2163 (struct i_t_dpdu_cqe *)sol);
8f09a3b9 2164 spin_unlock_bh(&phba->async_pdu_lock);
bfead3b2 2165 break;
6733b39a 2166 case UNSOL_DATA_NOTIFY:
99bc5d55
JSJ		 2167 beiscsi_log(phba, KERN_INFO,
2168 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
6763daae
JSJ		 2169 "BM_%d : Received %s[%d] on CID : %d\n",
2170 cqe_desc[code], code, cid);
99bc5d55 2171
8f09a3b9 2172 spin_lock_bh(&phba->async_pdu_lock);
6733b39a
JK		 2173 hwi_process_default_pdu_ring(beiscsi_conn, phba,
2174 (struct i_t_dpdu_cqe *)sol);
8f09a3b9 2175 spin_unlock_bh(&phba->async_pdu_lock);
6733b39a
JK		 2176 break;
2177 case CXN_INVALIDATE_INDEX_NOTIFY:
2178 case CMD_INVALIDATED_NOTIFY:
2179 case CXN_INVALIDATE_NOTIFY:
99bc5d55
JSJ		 2180 beiscsi_log(phba, KERN_ERR,
2181 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
6763daae
JSJ		 2182 "BM_%d : Ignoring %s[%d] on CID : %d\n",
2183 cqe_desc[code], code, cid);
6733b39a
JK		 2184 break;
2185 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
2186 case CMD_KILLED_INVALID_STATSN_RCVD:
2187 case CMD_KILLED_INVALID_R2T_RCVD:
2188 case CMD_CXN_KILLED_LUN_INVALID:
2189 case CMD_CXN_KILLED_ICD_INVALID:
2190 case CMD_CXN_KILLED_ITT_INVALID:
2191 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
2192 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
99bc5d55
JSJ		 2193 beiscsi_log(phba, KERN_ERR,
2194 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
6763daae
JSJ		 2195 "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
2196 cqe_desc[code], code, cid);
6733b39a
JK		 2197 break;
2198 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
99bc5d55
JSJ		 2199 beiscsi_log(phba, KERN_ERR,
2200 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
6763daae
JSJ		 2201 "BM_%d : Dropping %s[%d] on DPDU ring on CID : %d\n",
2202 cqe_desc[code], code, cid);
8f09a3b9 2203 spin_lock_bh(&phba->async_pdu_lock);
6733b39a
JK		 2204 hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
2205 (struct i_t_dpdu_cqe *) sol);
8f09a3b9 2206 spin_unlock_bh(&phba->async_pdu_lock);
6733b39a
JK		 2207 break;
2208 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
2209 case CXN_KILLED_BURST_LEN_MISMATCH:
2210 case CXN_KILLED_AHS_RCVD:
2211 case CXN_KILLED_HDR_DIGEST_ERR:
2212 case CXN_KILLED_UNKNOWN_HDR:
2213 case CXN_KILLED_STALE_ITT_TTT_RCVD:
2214 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
2215 case CXN_KILLED_TIMED_OUT:
2216 case CXN_KILLED_FIN_RCVD:
6763daae
JSJ		 2217 case CXN_KILLED_RST_SENT:
2218 case CXN_KILLED_RST_RCVD:
6733b39a
JK		 2219 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
2220 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
2221 case CXN_KILLED_OVER_RUN_RESIDUAL:
2222 case CXN_KILLED_UNDER_RUN_RESIDUAL:
2223 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
99bc5d55
JSJ		 2224 beiscsi_log(phba, KERN_ERR,
2225 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
6763daae
JSJ		 2226 "BM_%d : Event %s[%d] received on CID : %d\n",
2227 cqe_desc[code], code, cid);
0a513dd8
JSJ		 2228 if (beiscsi_conn)
2229 iscsi_conn_failure(beiscsi_conn->conn,
2230 ISCSI_ERR_CONN_FAILED);
6733b39a
JK		 2231 break;
2232 default:
99bc5d55
JSJ		 2233 beiscsi_log(phba, KERN_ERR,
2234 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
6763daae
JSJ		 2235 "BM_%d : Invalid CQE Event Received Code : %d"
2236 "CID 0x%x...\n",
0a513dd8 2237 code, cid);
6733b39a
JK		 2238 break;
2239 }
2240
2241 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
2242 queue_tail_inc(cq);
2243 sol = queue_tail_node(cq);
2244 num_processed++;
2245 }
2246
2247 if (num_processed > 0) {
2248 tot_nump += num_processed;
bfead3b2 2249 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
6733b39a
JK		 2250 }
2251 return tot_nump;
2252}
2253
756d29c8 2254void beiscsi_process_all_cqs(struct work_struct *work)
6733b39a
JK		 2255{
2256 unsigned long flags;
bfead3b2
JK		 2257 struct hwi_controller *phwi_ctrlr;
2258 struct hwi_context_memory *phwi_context;
72fb46a9
JSJ		 2259 struct beiscsi_hba *phba;
2260 struct be_eq_obj *pbe_eq =
2261 container_of(work, struct be_eq_obj, work_cqs);
6733b39a 2262
72fb46a9 2263 phba = pbe_eq->phba;
bfead3b2
JK		 2264 phwi_ctrlr = phba->phwi_ctrlr;
2265 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2 2266
72fb46a9 2267 if (pbe_eq->todo_mcc_cq) {
6733b39a 2268 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 2269 pbe_eq->todo_mcc_cq = false;
6733b39a 2270 spin_unlock_irqrestore(&phba->isr_lock, flags);
756d29c8 2271 beiscsi_process_mcc_isr(phba);
6733b39a
JK		 2272 }
2273
72fb46a9 2274 if (pbe_eq->todo_cq) {
6733b39a 2275 spin_lock_irqsave(&phba->isr_lock, flags);
72fb46a9 2276 pbe_eq->todo_cq = false;
6733b39a 2277 spin_unlock_irqrestore(&phba->isr_lock, flags);
bfead3b2 2278 beiscsi_process_cq(pbe_eq);
6733b39a 2279 }
72fb46a9
JSJ		 2280
2281 /* rearm EQ for further interrupts */
2282 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
6733b39a
JK		 2283}
2284
2285static int be_iopoll(struct blk_iopoll *iop, int budget)
2286{
ad3f428e 2287 unsigned int ret;
6733b39a 2288 struct beiscsi_hba *phba;
bfead3b2 2289 struct be_eq_obj *pbe_eq;
6733b39a 2290
bfead3b2
JK		 2291 pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
2292 ret = beiscsi_process_cq(pbe_eq);
6733b39a 2293 if (ret < budget) {
bfead3b2 2294 phba = pbe_eq->phba;
6733b39a 2295 blk_iopoll_complete(iop);
99bc5d55
JSJ		 2296 beiscsi_log(phba, KERN_INFO,
2297 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2298 "BM_%d : rearm pbe_eq->q.id =%d\n",
2299 pbe_eq->q.id);
bfead3b2 2300 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
6733b39a
JK		 2301 }
2302 return ret;
2303}
2304
09a1093a
JSJ		 2305static void
2306hwi_write_sgl_v2(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2307 unsigned int num_sg, struct beiscsi_io_task *io_task)
2308{
2309 struct iscsi_sge *psgl;
2310 unsigned int sg_len, index;
2311 unsigned int sge_len = 0;
2312 unsigned long long addr;
2313 struct scatterlist *l_sg;
2314 unsigned int offset;
2315
2316 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_lo, pwrb,
2317 io_task->bhs_pa.u.a32.address_lo);
2318 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_hi, pwrb,
2319 io_task->bhs_pa.u.a32.address_hi);
2320
2321 l_sg = sg;
2322 for (index = 0; (index < num_sg) && (index < 2); index++,
2323 sg = sg_next(sg)) {
2324 if (index == 0) {
2325 sg_len = sg_dma_len(sg);
2326 addr = (u64) sg_dma_address(sg);
2327 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2328 sge0_addr_lo, pwrb,
2329 lower_32_bits(addr));
2330 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2331 sge0_addr_hi, pwrb,
2332 upper_32_bits(addr));
2333 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2334 sge0_len, pwrb,
2335 sg_len);
2336 sge_len = sg_len;
2337 } else {
2338 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_r2t_offset,
2339 pwrb, sge_len);
2340 sg_len = sg_dma_len(sg);
2341 addr = (u64) sg_dma_address(sg);
2342 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2343 sge1_addr_lo, pwrb,
2344 lower_32_bits(addr));
2345 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2346 sge1_addr_hi, pwrb,
2347 upper_32_bits(addr));
2348 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2349 sge1_len, pwrb,
2350 sg_len);
2351 }
2352 }
2353 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2354 memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2355
2356 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2357
2358 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2359 io_task->bhs_pa.u.a32.address_hi);
2360 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2361 io_task->bhs_pa.u.a32.address_lo);
2362
2363 if (num_sg == 1) {
2364 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2365 1);
2366 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2367 0);
2368 } else if (num_sg == 2) {
2369 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2370 0);
2371 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2372 1);
2373 } else {
2374 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2375 0);
2376 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2377 0);
2378 }
2379
2380 sg = l_sg;
2381 psgl++;
2382 psgl++;
2383 offset = 0;
2384 for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2385 sg_len = sg_dma_len(sg);
2386 addr = (u64) sg_dma_address(sg);
2387 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2388 lower_32_bits(addr));
2389 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2390 upper_32_bits(addr));
2391 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2392 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2393 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2394 offset += sg_len;
2395 }
2396 psgl--;
2397 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2398}
2399
6733b39a
JK		 2400static void
2401hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2402 unsigned int num_sg, struct beiscsi_io_task *io_task)
2403{
2404 struct iscsi_sge *psgl;
58ff4bd0 2405 unsigned int sg_len, index;
6733b39a
JK		 2406 unsigned int sge_len = 0;
2407 unsigned long long addr;
2408 struct scatterlist *l_sg;
2409 unsigned int offset;
2410
2411 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2412 io_task->bhs_pa.u.a32.address_lo);
2413 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2414 io_task->bhs_pa.u.a32.address_hi);
2415
2416 l_sg = sg;
48bd86cf
JK		 2417 for (index = 0; (index < num_sg) && (index < 2); index++,
2418 sg = sg_next(sg)) {
6733b39a
JK		 2419 if (index == 0) {
2420 sg_len = sg_dma_len(sg);
2421 addr = (u64) sg_dma_address(sg);
2422 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
457ff3b7 2423 ((u32)(addr & 0xFFFFFFFF)));
6733b39a 2424 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
457ff3b7 2425 ((u32)(addr >> 32)));
6733b39a
JK		 2426 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2427 sg_len);
2428 sge_len = sg_len;
6733b39a 2429 } else {
6733b39a
JK		 2430 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2431 pwrb, sge_len);
2432 sg_len = sg_dma_len(sg);
2433 addr = (u64) sg_dma_address(sg);
2434 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
457ff3b7 2435 ((u32)(addr & 0xFFFFFFFF)));
6733b39a 2436 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
457ff3b7 2437 ((u32)(addr >> 32)));
6733b39a
JK		 2438 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2439 sg_len);
2440 }
2441 }
2442 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2443 memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2444
2445 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2446
2447 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2448 io_task->bhs_pa.u.a32.address_hi);
2449 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2450 io_task->bhs_pa.u.a32.address_lo);
2451
caf818f1
JK		 2452 if (num_sg == 1) {
2453 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2454 1);
2455 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2456 0);
2457 } else if (num_sg == 2) {
2458 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2459 0);
2460 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2461 1);
2462 } else {
2463 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2464 0);
2465 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2466 0);
2467 }
6733b39a
JK		 2468 sg = l_sg;
2469 psgl++;
2470 psgl++;
2471 offset = 0;
48bd86cf 2472 for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
6733b39a
JK		 2473 sg_len = sg_dma_len(sg);
2474 addr = (u64) sg_dma_address(sg);
2475 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2476 (addr & 0xFFFFFFFF));
2477 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2478 (addr >> 32));
2479 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2480 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2481 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2482 offset += sg_len;
2483 }
2484 psgl--;
2485 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2486}
2487
d629c471
JSJ		 2488/**
2489 * hwi_write_buffer()- Populate the WRB with task info
2490 * @pwrb: ptr to the WRB entry
2491 * @task: iscsi task which is to be executed
2492 **/
6733b39a
JK		 2493static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2494{
2495 struct iscsi_sge *psgl;
6733b39a
JK		 2496 struct beiscsi_io_task *io_task = task->dd_data;
2497 struct beiscsi_conn *beiscsi_conn = io_task->conn;
2498 struct beiscsi_hba *phba = beiscsi_conn->phba;
09a1093a 2499 uint8_t dsp_value = 0;
6733b39a
JK		 2500
2501 io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2502 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2503 io_task->bhs_pa.u.a32.address_lo);
2504 AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2505 io_task->bhs_pa.u.a32.address_hi);
2506
2507 if (task->data) {
09a1093a
JSJ		 2508
2509 /* Check for the data_count */
2510 dsp_value = (task->data_count) ? 1 : 0;
2511
2c9dfd36
JK		 2512 if (is_chip_be2_be3r(phba))
2513 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp,
09a1093a
JSJ		 2514 pwrb, dsp_value);
2515 else
2c9dfd36 2516 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp,
09a1093a
JSJ		 2517 pwrb, dsp_value);
2518
2519 /* Map addr only if there is data_count */
2520 if (dsp_value) {
d629c471
JSJ		 2521 io_task->mtask_addr = pci_map_single(phba->pcidev,
2522 task->data,
2523 task->data_count,
2524 PCI_DMA_TODEVICE);
d629c471 2525 io_task->mtask_data_count = task->data_count;
09a1093a 2526 } else
d629c471 2527 io_task->mtask_addr = 0;
09a1093a 2528
6733b39a 2529 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
d629c471 2530 lower_32_bits(io_task->mtask_addr));
6733b39a 2531 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
d629c471 2532 upper_32_bits(io_task->mtask_addr));
6733b39a
JK		 2533 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2534 task->data_count);
2535
2536 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2537 } else {
2538 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
d629c471 2539 io_task->mtask_addr = 0;
6733b39a
JK		 2540 }
2541
2542 psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2543
2544 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2545
2546 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2547 io_task->bhs_pa.u.a32.address_hi);
2548 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2549 io_task->bhs_pa.u.a32.address_lo);
2550 if (task->data) {
2551 psgl++;
2552 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2553 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2554 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2555 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2556 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2557 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2558
2559 psgl++;
2560 if (task->data) {
2561 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
d629c471 2562 lower_32_bits(io_task->mtask_addr));
6733b39a 2563 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
d629c471 2564 upper_32_bits(io_task->mtask_addr));
6733b39a
JK		 2565 }
2566 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2567 }
2568 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2569}
2570
843ae752
JK		 2571/**
2572 * beiscsi_find_mem_req()- Find mem needed
2573 * @phba: ptr to HBA struct
2574 **/
6733b39a
JK		 2575static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2576{
8a86e833 2577 uint8_t mem_descr_index, ulp_num;
bfead3b2 2578 unsigned int num_cq_pages, num_async_pdu_buf_pages;
6733b39a
JK		 2579 unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2580 unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2581
2582 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2583 sizeof(struct sol_cqe));
6733b39a
JK		 2584
2585 phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2586
2587 phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2588 BE_ISCSI_PDU_HEADER_SIZE;
2589 phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2590 sizeof(struct hwi_context_memory);
2591
6733b39a
JK		 2592
2593 phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2594 * (phba->params.wrbs_per_cxn)
2595 * phba->params.cxns_per_ctrl;
2596 wrb_sz_per_cxn = sizeof(struct wrb_handle) *
2597 (phba->params.wrbs_per_cxn);
2598 phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2599 phba->params.cxns_per_ctrl);
2600
2601 phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2602 phba->params.icds_per_ctrl;
2603 phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2604 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
8a86e833
JK		 2605 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
2606 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
6733b39a 2607
8a86e833
JK		 2608 num_async_pdu_buf_sgl_pages =
2609 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2610 phba, ulp_num) *
2611 sizeof(struct phys_addr));
2612
2613 num_async_pdu_buf_pages =
2614 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2615 phba, ulp_num) *
2616 phba->params.defpdu_hdr_sz);
2617
2618 num_async_pdu_data_pages =
2619 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2620 phba, ulp_num) *
2621 phba->params.defpdu_data_sz);
2622
2623 num_async_pdu_data_sgl_pages =
2624 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2625 phba, ulp_num) *
2626 sizeof(struct phys_addr));
2627
a129d92f
JK		 2628 mem_descr_index = (HWI_MEM_TEMPLATE_HDR_ULP0 +
2629 (ulp_num * MEM_DESCR_OFFSET));
2630 phba->mem_req[mem_descr_index] =
2631 BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2632 BEISCSI_TEMPLATE_HDR_PER_CXN_SIZE;
2633
8a86e833
JK		 2634 mem_descr_index = (HWI_MEM_ASYNC_HEADER_BUF_ULP0 +
2635 (ulp_num * MEM_DESCR_OFFSET));
2636 phba->mem_req[mem_descr_index] =
2637 num_async_pdu_buf_pages *
2638 PAGE_SIZE;
2639
2640 mem_descr_index = (HWI_MEM_ASYNC_DATA_BUF_ULP0 +
2641 (ulp_num * MEM_DESCR_OFFSET));
2642 phba->mem_req[mem_descr_index] =
2643 num_async_pdu_data_pages *
2644 PAGE_SIZE;
2645
2646 mem_descr_index = (HWI_MEM_ASYNC_HEADER_RING_ULP0 +
2647 (ulp_num * MEM_DESCR_OFFSET));
2648 phba->mem_req[mem_descr_index] =
2649 num_async_pdu_buf_sgl_pages *
2650 PAGE_SIZE;
2651
2652 mem_descr_index = (HWI_MEM_ASYNC_DATA_RING_ULP0 +
2653 (ulp_num * MEM_DESCR_OFFSET));
2654 phba->mem_req[mem_descr_index] =
2655 num_async_pdu_data_sgl_pages *
2656 PAGE_SIZE;
2657
2658 mem_descr_index = (HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 +
2659 (ulp_num * MEM_DESCR_OFFSET));
2660 phba->mem_req[mem_descr_index] =
2661 BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2662 sizeof(struct async_pdu_handle);
2663
2664 mem_descr_index = (HWI_MEM_ASYNC_DATA_HANDLE_ULP0 +
2665 (ulp_num * MEM_DESCR_OFFSET));
2666 phba->mem_req[mem_descr_index] =
2667 BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2668 sizeof(struct async_pdu_handle);
2669
2670 mem_descr_index = (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 +
2671 (ulp_num * MEM_DESCR_OFFSET));
2672 phba->mem_req[mem_descr_index] =
2673 sizeof(struct hwi_async_pdu_context) +
2674 (BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2675 sizeof(struct hwi_async_entry));
2676 }
2677 }
6733b39a
JK		 2678}
2679
2680static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2681{
6733b39a 2682 dma_addr_t bus_add;
a7909b39
JK		 2683 struct hwi_controller *phwi_ctrlr;
2684 struct be_mem_descriptor *mem_descr;
6733b39a
JK		 2685 struct mem_array *mem_arr, *mem_arr_orig;
2686 unsigned int i, j, alloc_size, curr_alloc_size;
2687
3ec78271 2688 phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
6733b39a
JK		 2689 if (!phba->phwi_ctrlr)
2690 return -ENOMEM;
2691
a7909b39
JK		 2692 /* Allocate memory for wrb_context */
2693 phwi_ctrlr = phba->phwi_ctrlr;
2694 phwi_ctrlr->wrb_context = kzalloc(sizeof(struct hwi_wrb_context) *
2695 phba->params.cxns_per_ctrl,
2696 GFP_KERNEL);
2697 if (!phwi_ctrlr->wrb_context)
2698 return -ENOMEM;
2699
6733b39a
JK		 2700 phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2701 GFP_KERNEL);
2702 if (!phba->init_mem) {
a7909b39 2703 kfree(phwi_ctrlr->wrb_context);
6733b39a
JK		 2704 kfree(phba->phwi_ctrlr);
2705 return -ENOMEM;
2706 }
2707
2708 mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2709 GFP_KERNEL);
2710 if (!mem_arr_orig) {
2711 kfree(phba->init_mem);
a7909b39 2712 kfree(phwi_ctrlr->wrb_context);
6733b39a
JK		 2713 kfree(phba->phwi_ctrlr);
2714 return -ENOMEM;
2715 }
2716
2717 mem_descr = phba->init_mem;
2718 for (i = 0; i < SE_MEM_MAX; i++) {
8a86e833
JK		 2719 if (!phba->mem_req[i]) {
2720 mem_descr->mem_array = NULL;
2721 mem_descr++;
2722 continue;
2723 }
2724
6733b39a
JK		 2725 j = 0;
2726 mem_arr = mem_arr_orig;
2727 alloc_size = phba->mem_req[i];
2728 memset(mem_arr, 0, sizeof(struct mem_array) *
2729 BEISCSI_MAX_FRAGS_INIT);
2730 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2731 do {
2732 mem_arr->virtual_address = pci_alloc_consistent(
2733 phba->pcidev,
2734 curr_alloc_size,
2735 &bus_add);
2736 if (!mem_arr->virtual_address) {
2737 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2738 goto free_mem;
2739 if (curr_alloc_size -
2740 rounddown_pow_of_two(curr_alloc_size))
2741 curr_alloc_size = rounddown_pow_of_two
2742 (curr_alloc_size);
2743 else
2744 curr_alloc_size = curr_alloc_size / 2;
2745 } else {
2746 mem_arr->bus_address.u.
2747 a64.address = (__u64) bus_add;
2748 mem_arr->size = curr_alloc_size;
2749 alloc_size -= curr_alloc_size;
2750 curr_alloc_size = min(be_max_phys_size *
2751 1024, alloc_size);
2752 j++;
2753 mem_arr++;
2754 }
2755 } while (alloc_size);
2756 mem_descr->num_elements = j;
2757 mem_descr->size_in_bytes = phba->mem_req[i];
2758 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2759 GFP_KERNEL);
2760 if (!mem_descr->mem_array)
2761 goto free_mem;
2762
2763 memcpy(mem_descr->mem_array, mem_arr_orig,
2764 sizeof(struct mem_array) * j);
2765 mem_descr++;
2766 }
2767 kfree(mem_arr_orig);
2768 return 0;
2769free_mem:
2770 mem_descr->num_elements = j;
2771 while ((i) || (j)) {
2772 for (j = mem_descr->num_elements; j > 0; j--) {
2773 pci_free_consistent(phba->pcidev,
2774 mem_descr->mem_array[j - 1].size,
2775 mem_descr->mem_array[j - 1].
2776 virtual_address,
457ff3b7
JK		 2777 (unsigned long)mem_descr->
2778 mem_array[j - 1].
6733b39a
JK		 2779 bus_address.u.a64.address);
2780 }
2781 if (i) {
2782 i--;
2783 kfree(mem_descr->mem_array);
2784 mem_descr--;
2785 }
2786 }
2787 kfree(mem_arr_orig);
2788 kfree(phba->init_mem);
a7909b39 2789 kfree(phba->phwi_ctrlr->wrb_context);
6733b39a
JK		 2790 kfree(phba->phwi_ctrlr);
2791 return -ENOMEM;
2792}
2793
2794static int beiscsi_get_memory(struct beiscsi_hba *phba)
2795{
2796 beiscsi_find_mem_req(phba);
2797 return beiscsi_alloc_mem(phba);
2798}
2799
2800static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2801{
2802 struct pdu_data_out *pdata_out;
2803 struct pdu_nop_out *pnop_out;
2804 struct be_mem_descriptor *mem_descr;
2805
2806 mem_descr = phba->init_mem;
2807 mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2808 pdata_out =
2809 (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2810 memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2811
2812 AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2813 IIOC_SCSI_DATA);
2814
2815 pnop_out =
2816 (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2817 virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2818
2819 memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2820 AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2821 AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2822 AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2823}
2824
3ec78271 2825static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
6733b39a
JK		 2826{
2827 struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
a7909b39 2828 struct hwi_context_memory *phwi_ctxt;
3ec78271 2829 struct wrb_handle *pwrb_handle = NULL;
6733b39a
JK		 2830 struct hwi_controller *phwi_ctrlr;
2831 struct hwi_wrb_context *pwrb_context;
3ec78271
JK		 2832 struct iscsi_wrb *pwrb = NULL;
2833 unsigned int num_cxn_wrbh = 0;
2834 unsigned int num_cxn_wrb = 0, j, idx = 0, index;
6733b39a
JK		 2835
2836 mem_descr_wrbh = phba->init_mem;
2837 mem_descr_wrbh += HWI_MEM_WRBH;
2838
2839 mem_descr_wrb = phba->init_mem;
2840 mem_descr_wrb += HWI_MEM_WRB;
6733b39a
JK		 2841 phwi_ctrlr = phba->phwi_ctrlr;
2842
a7909b39
JK		 2843 /* Allocate memory for WRBQ */
2844 phwi_ctxt = phwi_ctrlr->phwi_ctxt;
2845 phwi_ctxt->be_wrbq = kzalloc(sizeof(struct be_queue_info) *
843ae752 2846 phba->params.cxns_per_ctrl,
a7909b39
JK		 2847 GFP_KERNEL);
2848 if (!phwi_ctxt->be_wrbq) {
2849 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2850 "BM_%d : WRBQ Mem Alloc Failed\n");
2851 return -ENOMEM;
2852 }
2853
2854 for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
6733b39a 2855 pwrb_context = &phwi_ctrlr->wrb_context[index];
6733b39a
JK		 2856 pwrb_context->pwrb_handle_base =
2857 kzalloc(sizeof(struct wrb_handle *) *
2858 phba->params.wrbs_per_cxn, GFP_KERNEL);
3ec78271 2859 if (!pwrb_context->pwrb_handle_base) {
99bc5d55
JSJ		 2860 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2861 "BM_%d : Mem Alloc Failed. Failing to load\n");
3ec78271
JK		 2862 goto init_wrb_hndl_failed;
2863 }
6733b39a
JK		 2864 pwrb_context->pwrb_handle_basestd =
2865 kzalloc(sizeof(struct wrb_handle *) *
2866 phba->params.wrbs_per_cxn, GFP_KERNEL);
3ec78271 2867 if (!pwrb_context->pwrb_handle_basestd) {
99bc5d55
JSJ		 2868 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2869 "BM_%d : Mem Alloc Failed. Failing to load\n");
3ec78271
JK		 2870 goto init_wrb_hndl_failed;
2871 }
2872 if (!num_cxn_wrbh) {
2873 pwrb_handle =
2874 mem_descr_wrbh->mem_array[idx].virtual_address;
2875 num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2876 ((sizeof(struct wrb_handle)) *
2877 phba->params.wrbs_per_cxn));
2878 idx++;
2879 }
2880 pwrb_context->alloc_index = 0;
2881 pwrb_context->wrb_handles_available = 0;
2882 pwrb_context->free_index = 0;
2883
6733b39a 2884 if (num_cxn_wrbh) {
6733b39a
JK		 2885 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2886 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2887 pwrb_context->pwrb_handle_basestd[j] =
2888 pwrb_handle;
2889 pwrb_context->wrb_handles_available++;
bfead3b2 2890 pwrb_handle->wrb_index = j;
6733b39a
JK		 2891 pwrb_handle++;
2892 }
6733b39a
JK		 2893 num_cxn_wrbh--;
2894 }
2895 }
2896 idx = 0;
a7909b39 2897 for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
6733b39a 2898 pwrb_context = &phwi_ctrlr->wrb_context[index];
3ec78271 2899 if (!num_cxn_wrb) {
6733b39a 2900 pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
7c56533c 2901 num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
3ec78271
JK		 2902 ((sizeof(struct iscsi_wrb) *
2903 phba->params.wrbs_per_cxn));
2904 idx++;
2905 }
2906
2907 if (num_cxn_wrb) {
6733b39a
JK		 2908 for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2909 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2910 pwrb_handle->pwrb = pwrb;
2911 pwrb++;
2912 }
2913 num_cxn_wrb--;
2914 }
2915 }
3ec78271
JK		 2916 return 0;
2917init_wrb_hndl_failed:
2918 for (j = index; j > 0; j--) {
2919 pwrb_context = &phwi_ctrlr->wrb_context[j];
2920 kfree(pwrb_context->pwrb_handle_base);
2921 kfree(pwrb_context->pwrb_handle_basestd);
2922 }
2923 return -ENOMEM;
6733b39a
JK		 2924}
2925
a7909b39 2926static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
6733b39a 2927{
8a86e833 2928 uint8_t ulp_num;
6733b39a
JK		 2929 struct hwi_controller *phwi_ctrlr;
2930 struct hba_parameters *p = &phba->params;
2931 struct hwi_async_pdu_context *pasync_ctx;
2932 struct async_pdu_handle *pasync_header_h, *pasync_data_h;
dc63aac6 2933 unsigned int index, idx, num_per_mem, num_async_data;
6733b39a
JK		 2934 struct be_mem_descriptor *mem_descr;
2935
8a86e833
JK		 2936 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
2937 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
6733b39a 2938
8a86e833
JK		 2939 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2940 mem_descr += (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 +
2941 (ulp_num * MEM_DESCR_OFFSET));
2942
2943 phwi_ctrlr = phba->phwi_ctrlr;
2944 phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num] =
2945 (struct hwi_async_pdu_context *)
2946 mem_descr->mem_array[0].virtual_address;
2947
2948 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];
2949 memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2950
2951 pasync_ctx->async_entry =
2952 (struct hwi_async_entry *)
2953 ((long unsigned int)pasync_ctx +
2954 sizeof(struct hwi_async_pdu_context));
2955
2956 pasync_ctx->num_entries = BEISCSI_GET_CID_COUNT(phba,
2957 ulp_num);
2958 pasync_ctx->buffer_size = p->defpdu_hdr_sz;
2959
2960 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2961 mem_descr += HWI_MEM_ASYNC_HEADER_BUF_ULP0 +
2962 (ulp_num * MEM_DESCR_OFFSET);
2963 if (mem_descr->mem_array[0].virtual_address) {
2964 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2965 "BM_%d : hwi_init_async_pdu_ctx"
2966 " HWI_MEM_ASYNC_HEADER_BUF_ULP%d va=%p\n",
2967 ulp_num,
2968 mem_descr->mem_array[0].
2969 virtual_address);
2970 } else
2971 beiscsi_log(phba, KERN_WARNING,
2972 BEISCSI_LOG_INIT,
2973 "BM_%d : No Virtual address for ULP : %d\n",
2974 ulp_num);
2975
2976 pasync_ctx->async_header.va_base =
6733b39a 2977 mem_descr->mem_array[0].virtual_address;
6733b39a 2978
8a86e833
JK		 2979 pasync_ctx->async_header.pa_base.u.a64.address =
2980 mem_descr->mem_array[0].
2981 bus_address.u.a64.address;
6733b39a 2982
8a86e833
JK		 2983 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2984 mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 +
2985 (ulp_num * MEM_DESCR_OFFSET);
2986 if (mem_descr->mem_array[0].virtual_address) {
2987 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2988 "BM_%d : hwi_init_async_pdu_ctx"
2989 " HWI_MEM_ASYNC_HEADER_RING_ULP%d va=%p\n",
2990 ulp_num,
2991 mem_descr->mem_array[0].
2992 virtual_address);
2993 } else
2994 beiscsi_log(phba, KERN_WARNING,
2995 BEISCSI_LOG_INIT,
2996 "BM_%d : No Virtual address for ULP : %d\n",
2997 ulp_num);
2998
2999 pasync_ctx->async_header.ring_base =
3000 mem_descr->mem_array[0].virtual_address;
6733b39a 3001
8a86e833
JK		 3002 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3003 mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 +
3004 (ulp_num * MEM_DESCR_OFFSET);
3005 if (mem_descr->mem_array[0].virtual_address) {
3006 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3007 "BM_%d : hwi_init_async_pdu_ctx"
3008 " HWI_MEM_ASYNC_HEADER_HANDLE_ULP%d va=%p\n",
3009 ulp_num,
3010 mem_descr->mem_array[0].
3011 virtual_address);
3012 } else
3013 beiscsi_log(phba, KERN_WARNING,
3014 BEISCSI_LOG_INIT,
3015 "BM_%d : No Virtual address for ULP : %d\n",
3016 ulp_num);
3017
3018 pasync_ctx->async_header.handle_base =
3019 mem_descr->mem_array[0].virtual_address;
3020 pasync_ctx->async_header.writables = 0;
3021 INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
3022
3023 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3024 mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 +
3025 (ulp_num * MEM_DESCR_OFFSET);
3026 if (mem_descr->mem_array[0].virtual_address) {
3027 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3028 "BM_%d : hwi_init_async_pdu_ctx"
3029 " HWI_MEM_ASYNC_DATA_RING_ULP%d va=%p\n",
3030 ulp_num,
3031 mem_descr->mem_array[0].
3032 virtual_address);
3033 } else
3034 beiscsi_log(phba, KERN_WARNING,
3035 BEISCSI_LOG_INIT,
3036 "BM_%d : No Virtual address for ULP : %d\n",
3037 ulp_num);
3038
3039 pasync_ctx->async_data.ring_base =
3040 mem_descr->mem_array[0].virtual_address;
6733b39a 3041
8a86e833
JK		 3042 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3043 mem_descr += HWI_MEM_ASYNC_DATA_HANDLE_ULP0 +
3044 (ulp_num * MEM_DESCR_OFFSET);
3045 if (!mem_descr->mem_array[0].virtual_address)
3046 beiscsi_log(phba, KERN_WARNING,
3047 BEISCSI_LOG_INIT,
3048 "BM_%d : No Virtual address for ULP : %d\n",
3049 ulp_num);
99bc5d55 3050
8a86e833
JK		 3051 pasync_ctx->async_data.handle_base =
3052 mem_descr->mem_array[0].virtual_address;
3053 pasync_ctx->async_data.writables = 0;
3054 INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
3055
3056 pasync_header_h =
3057 (struct async_pdu_handle *)
3058 pasync_ctx->async_header.handle_base;
3059 pasync_data_h =
3060 (struct async_pdu_handle *)
3061 pasync_ctx->async_data.handle_base;
3062
3063 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3064 mem_descr += HWI_MEM_ASYNC_DATA_BUF_ULP0 +
3065 (ulp_num * MEM_DESCR_OFFSET);
3066 if (mem_descr->mem_array[0].virtual_address) {
3067 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3068 "BM_%d : hwi_init_async_pdu_ctx"
3069 " HWI_MEM_ASYNC_DATA_BUF_ULP%d va=%p\n",
3070 ulp_num,
3071 mem_descr->mem_array[0].
3072 virtual_address);
3073 } else
3074 beiscsi_log(phba, KERN_WARNING,
3075 BEISCSI_LOG_INIT,
3076 "BM_%d : No Virtual address for ULP : %d\n",
3077 ulp_num);
3078
3079 idx = 0;
dc63aac6
JK		 3080 pasync_ctx->async_data.va_base =
3081 mem_descr->mem_array[idx].virtual_address;
3082 pasync_ctx->async_data.pa_base.u.a64.address =
3083 mem_descr->mem_array[idx].
3084 bus_address.u.a64.address;
3085
3086 num_async_data = ((mem_descr->mem_array[idx].size) /
3087 phba->params.defpdu_data_sz);
8a86e833 3088 num_per_mem = 0;
6733b39a 3089
8a86e833
JK		 3090 for (index = 0; index < BEISCSI_GET_CID_COUNT
3091 (phba, ulp_num); index++) {
3092 pasync_header_h->cri = -1;
3093 pasync_header_h->index = (char)index;
3094 INIT_LIST_HEAD(&pasync_header_h->link);
3095 pasync_header_h->pbuffer =
3096 (void *)((unsigned long)
3097 (pasync_ctx->
3098 async_header.va_base) +
3099 (p->defpdu_hdr_sz * index));
3100
3101 pasync_header_h->pa.u.a64.address =
3102 pasync_ctx->async_header.pa_base.u.a64.
3103 address + (p->defpdu_hdr_sz * index);
3104
3105 list_add_tail(&pasync_header_h->link,
3106 &pasync_ctx->async_header.
3107 free_list);
3108 pasync_header_h++;
3109 pasync_ctx->async_header.free_entries++;
3110 pasync_ctx->async_header.writables++;
3111
3112 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
3113 wait_queue.list);
3114 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
3115 header_busy_list);
3116 pasync_data_h->cri = -1;
3117 pasync_data_h->index = (char)index;
3118 INIT_LIST_HEAD(&pasync_data_h->link);
3119
3120 if (!num_async_data) {
3121 num_per_mem = 0;
3122 idx++;
3123 pasync_ctx->async_data.va_base =
3124 mem_descr->mem_array[idx].
3125 virtual_address;
3126 pasync_ctx->async_data.pa_base.u.
3127 a64.address =
3128 mem_descr->mem_array[idx].
3129 bus_address.u.a64.address;
3130 num_async_data =
3131 ((mem_descr->mem_array[idx].
3132 size) /
3133 phba->params.defpdu_data_sz);
3134 }
3135 pasync_data_h->pbuffer =
3136 (void *)((unsigned long)
3137 (pasync_ctx->async_data.va_base) +
3138 (p->defpdu_data_sz * num_per_mem));
3139
3140 pasync_data_h->pa.u.a64.address =
3141 pasync_ctx->async_data.pa_base.u.a64.
3142 address + (p->defpdu_data_sz *
3143 num_per_mem);
3144 num_per_mem++;
3145 num_async_data--;
3146
3147 list_add_tail(&pasync_data_h->link,
3148 &pasync_ctx->async_data.
3149 free_list);
3150 pasync_data_h++;
3151 pasync_ctx->async_data.free_entries++;
3152 pasync_ctx->async_data.writables++;
3153
3154 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
3155 data_busy_list);
3156 }
6733b39a 3157
8a86e833
JK		 3158 pasync_ctx->async_header.host_write_ptr = 0;
3159 pasync_ctx->async_header.ep_read_ptr = -1;
3160 pasync_ctx->async_data.host_write_ptr = 0;
3161 pasync_ctx->async_data.ep_read_ptr = -1;
3162 }
6733b39a
JK		 3163 }
3164
a7909b39 3165 return 0;
6733b39a
JK		 3166}
3167
3168static int
3169be_sgl_create_contiguous(void *virtual_address,
3170 u64 physical_address, u32 length,
3171 struct be_dma_mem *sgl)
3172{
3173 WARN_ON(!virtual_address);
3174 WARN_ON(!physical_address);
399a9503 3175 WARN_ON(!length);
6733b39a
JK		 3176 WARN_ON(!sgl);
3177
3178 sgl->va = virtual_address;
457ff3b7 3179 sgl->dma = (unsigned long)physical_address;
6733b39a
JK		 3180 sgl->size = length;
3181
3182 return 0;
3183}
3184
3185static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
3186{
3187 memset(sgl, 0, sizeof(*sgl));
3188}
3189
3190static void
3191hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
3192 struct mem_array *pmem, struct be_dma_mem *sgl)
3193{
3194 if (sgl->va)
3195 be_sgl_destroy_contiguous(sgl);
3196
3197 be_sgl_create_contiguous(pmem->virtual_address,
3198 pmem->bus_address.u.a64.address,
3199 pmem->size, sgl);
3200}
3201
3202static void
3203hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
3204 struct mem_array *pmem, struct be_dma_mem *sgl)
3205{
3206 if (sgl->va)
3207 be_sgl_destroy_contiguous(sgl);
3208
3209 be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
3210 pmem->bus_address.u.a64.address,
3211 pmem->size, sgl);
3212}
3213
3214static int be_fill_queue(struct be_queue_info *q,
3215 u16 len, u16 entry_size, void *vaddress)
3216{
3217 struct be_dma_mem *mem = &q->dma_mem;
3218
3219 memset(q, 0, sizeof(*q));
3220 q->len = len;
3221 q->entry_size = entry_size;
3222 mem->size = len * entry_size;
3223 mem->va = vaddress;
3224 if (!mem->va)
3225 return -ENOMEM;
3226 memset(mem->va, 0, mem->size);
3227 return 0;
3228}
3229
bfead3b2 3230static int beiscsi_create_eqs(struct beiscsi_hba *phba,
6733b39a
JK		 3231 struct hwi_context_memory *phwi_context)
3232{
bfead3b2 3233 unsigned int i, num_eq_pages;
99bc5d55 3234 int ret = 0, eq_for_mcc;
6733b39a
JK		 3235 struct be_queue_info *eq;
3236 struct be_dma_mem *mem;
6733b39a 3237 void *eq_vaddress;
bfead3b2 3238 dma_addr_t paddr;
6733b39a 3239
bfead3b2
JK		 3240 num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
3241 sizeof(struct be_eq_entry));
6733b39a 3242
bfead3b2
JK		 3243 if (phba->msix_enabled)
3244 eq_for_mcc = 1;
3245 else
3246 eq_for_mcc = 0;
3247 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3248 eq = &phwi_context->be_eq[i].q;
3249 mem = &eq->dma_mem;
3250 phwi_context->be_eq[i].phba = phba;
3251 eq_vaddress = pci_alloc_consistent(phba->pcidev,
3252 num_eq_pages * PAGE_SIZE,
3253 &paddr);
3254 if (!eq_vaddress)
3255 goto create_eq_error;
3256
3257 mem->va = eq_vaddress;
3258 ret = be_fill_queue(eq, phba->params.num_eq_entries,
3259 sizeof(struct be_eq_entry), eq_vaddress);
3260 if (ret) {
99bc5d55
JSJ		 3261 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3262 "BM_%d : be_fill_queue Failed for EQ\n");
bfead3b2
JK		 3263 goto create_eq_error;
3264 }
6733b39a 3265
bfead3b2
JK		 3266 mem->dma = paddr;
3267 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
3268 phwi_context->cur_eqd);
3269 if (ret) {
99bc5d55
JSJ		 3270 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3271 "BM_%d : beiscsi_cmd_eq_create"
3272 "Failed for EQ\n");
bfead3b2
JK		 3273 goto create_eq_error;
3274 }
99bc5d55
JSJ		 3275
3276 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3277 "BM_%d : eqid = %d\n",
3278 phwi_context->be_eq[i].q.id);
6733b39a 3279 }
6733b39a 3280 return 0;
bfead3b2 3281create_eq_error:
107dfcba 3282 for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
bfead3b2
JK		 3283 eq = &phwi_context->be_eq[i].q;
3284 mem = &eq->dma_mem;
3285 if (mem->va)
3286 pci_free_consistent(phba->pcidev, num_eq_pages
3287 * PAGE_SIZE,
3288 mem->va, mem->dma);
3289 }
3290 return ret;
6733b39a
JK		 3291}
3292
bfead3b2 3293static int beiscsi_create_cqs(struct beiscsi_hba *phba,
6733b39a
JK		 3294 struct hwi_context_memory *phwi_context)
3295{
bfead3b2 3296 unsigned int i, num_cq_pages;
99bc5d55 3297 int ret = 0;
6733b39a
JK		 3298 struct be_queue_info *cq, *eq;
3299 struct be_dma_mem *mem;
bfead3b2 3300 struct be_eq_obj *pbe_eq;
6733b39a 3301 void *cq_vaddress;
bfead3b2 3302 dma_addr_t paddr;
6733b39a 3303
bfead3b2
JK		 3304 num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
3305 sizeof(struct sol_cqe));
6733b39a 3306
bfead3b2
JK		 3307 for (i = 0; i < phba->num_cpus; i++) {
3308 cq = &phwi_context->be_cq[i];
3309 eq = &phwi_context->be_eq[i].q;
3310 pbe_eq = &phwi_context->be_eq[i];
3311 pbe_eq->cq = cq;
3312 pbe_eq->phba = phba;
3313 mem = &cq->dma_mem;
3314 cq_vaddress = pci_alloc_consistent(phba->pcidev,
3315 num_cq_pages * PAGE_SIZE,
3316 &paddr);
3317 if (!cq_vaddress)
3318 goto create_cq_error;
7da50879 3319 ret = be_fill_queue(cq, phba->params.num_cq_entries,
bfead3b2
JK		 3320 sizeof(struct sol_cqe), cq_vaddress);
3321 if (ret) {
99bc5d55
JSJ		 3322 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3323 "BM_%d : be_fill_queue Failed "
3324 "for ISCSI CQ\n");
bfead3b2
JK		 3325 goto create_cq_error;
3326 }
3327
3328 mem->dma = paddr;
3329 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
3330 false, 0);
3331 if (ret) {
99bc5d55
JSJ		 3332 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3333 "BM_%d : beiscsi_cmd_eq_create"
3334 "Failed for ISCSI CQ\n");
bfead3b2
JK		 3335 goto create_cq_error;
3336 }
99bc5d55
JSJ		 3337 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3338 "BM_%d : iscsi cq_id is %d for eq_id %d\n"
3339 "iSCSI CQ CREATED\n", cq->id, eq->id);
6733b39a 3340 }
6733b39a 3341 return 0;
bfead3b2
JK		 3342
3343create_cq_error:
3344 for (i = 0; i < phba->num_cpus; i++) {
3345 cq = &phwi_context->be_cq[i];
3346 mem = &cq->dma_mem;
3347 if (mem->va)
3348 pci_free_consistent(phba->pcidev, num_cq_pages
3349 * PAGE_SIZE,
3350 mem->va, mem->dma);
3351 }
3352 return ret;
3353
6733b39a
JK		 3354}
3355
3356static int
3357beiscsi_create_def_hdr(struct beiscsi_hba *phba,
3358 struct hwi_context_memory *phwi_context,
3359 struct hwi_controller *phwi_ctrlr,
8a86e833 3360 unsigned int def_pdu_ring_sz, uint8_t ulp_num)
6733b39a
JK		 3361{
3362 unsigned int idx;
3363 int ret;
3364 struct be_queue_info *dq, *cq;
3365 struct be_dma_mem *mem;
3366 struct be_mem_descriptor *mem_descr;
3367 void *dq_vaddress;
3368
3369 idx = 0;
8a86e833 3370 dq = &phwi_context->be_def_hdrq[ulp_num];
bfead3b2 3371 cq = &phwi_context->be_cq[0];
6733b39a
JK		 3372 mem = &dq->dma_mem;
3373 mem_descr = phba->init_mem;
8a86e833
JK		 3374 mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 +
3375 (ulp_num * MEM_DESCR_OFFSET);
6733b39a
JK		 3376 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3377 ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
3378 sizeof(struct phys_addr),
3379 sizeof(struct phys_addr), dq_vaddress);
3380 if (ret) {
99bc5d55 3381 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
8a86e833
JK		 3382 "BM_%d : be_fill_queue Failed for DEF PDU HDR on ULP : %d\n",
3383 ulp_num);
3384
6733b39a
JK		 3385 return ret;
3386 }
457ff3b7
JK		 3387 mem->dma = (unsigned long)mem_descr->mem_array[idx].
3388 bus_address.u.a64.address;
6733b39a
JK		 3389 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
3390 def_pdu_ring_sz,
8a86e833
JK		 3391 phba->params.defpdu_hdr_sz,
3392 BEISCSI_DEFQ_HDR, ulp_num);
6733b39a 3393 if (ret) {
99bc5d55 3394 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
8a86e833
JK		 3395 "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR on ULP : %d\n",
3396 ulp_num);
3397
6733b39a
JK		 3398 return ret;
3399 }
99bc5d55 3400
8a86e833
JK		 3401 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3402 "BM_%d : iscsi hdr def pdu id for ULP : %d is %d\n",
3403 ulp_num,
3404 phwi_context->be_def_hdrq[ulp_num].id);
3405 hwi_post_async_buffers(phba, BEISCSI_DEFQ_HDR, ulp_num);
6733b39a
JK		 3406 return 0;
3407}
3408
3409static int
3410beiscsi_create_def_data(struct beiscsi_hba *phba,
3411 struct hwi_context_memory *phwi_context,
3412 struct hwi_controller *phwi_ctrlr,
8a86e833 3413 unsigned int def_pdu_ring_sz, uint8_t ulp_num)
6733b39a
JK		 3414{
3415 unsigned int idx;
3416 int ret;
3417 struct be_queue_info *dataq, *cq;
3418 struct be_dma_mem *mem;
3419 struct be_mem_descriptor *mem_descr;
3420 void *dq_vaddress;
3421
3422 idx = 0;
8a86e833 3423 dataq = &phwi_context->be_def_dataq[ulp_num];
bfead3b2 3424 cq = &phwi_context->be_cq[0];
6733b39a
JK		 3425 mem = &dataq->dma_mem;
3426 mem_descr = phba->init_mem;
8a86e833
JK		 3427 mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 +
3428 (ulp_num * MEM_DESCR_OFFSET);
6733b39a
JK		 3429 dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3430 ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
3431 sizeof(struct phys_addr),
3432 sizeof(struct phys_addr), dq_vaddress);
3433 if (ret) {
99bc5d55 3434 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
8a86e833
JK		 3435 "BM_%d : be_fill_queue Failed for DEF PDU "
3436 "DATA on ULP : %d\n",
3437 ulp_num);
3438
6733b39a
JK		 3439 return ret;
3440 }
457ff3b7
JK		 3441 mem->dma = (unsigned long)mem_descr->mem_array[idx].
3442 bus_address.u.a64.address;
6733b39a
JK		 3443 ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
3444 def_pdu_ring_sz,
8a86e833
JK		 3445 phba->params.defpdu_data_sz,
3446 BEISCSI_DEFQ_DATA, ulp_num);
6733b39a 3447 if (ret) {
99bc5d55
JSJ		 3448 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3449 "BM_%d be_cmd_create_default_pdu_queue"
8a86e833
JK		 3450 " Failed for DEF PDU DATA on ULP : %d\n",
3451 ulp_num);
6733b39a
JK		 3452 return ret;
3453 }
8a86e833 3454
99bc5d55 3455 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
8a86e833
JK		 3456 "BM_%d : iscsi def data id on ULP : %d is %d\n",
3457 ulp_num,
3458 phwi_context->be_def_dataq[ulp_num].id);
99bc5d55 3459
8a86e833 3460 hwi_post_async_buffers(phba, BEISCSI_DEFQ_DATA, ulp_num);
99bc5d55 3461 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
8a86e833
JK		 3462 "BM_%d : DEFAULT PDU DATA RING CREATED"
3463 "on ULP : %d\n", ulp_num);
99bc5d55 3464
6733b39a
JK		 3465 return 0;
3466}
3467
15a90fe0
JK		 3468
3469static int
3470beiscsi_post_template_hdr(struct beiscsi_hba *phba)
3471{
3472 struct be_mem_descriptor *mem_descr;
3473 struct mem_array *pm_arr;
3474 struct be_dma_mem sgl;
a129d92f 3475 int status, ulp_num;
15a90fe0 3476
a129d92f
JK		 3477 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3478 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3479 mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3480 mem_descr += HWI_MEM_TEMPLATE_HDR_ULP0 +
3481 (ulp_num * MEM_DESCR_OFFSET);
3482 pm_arr = mem_descr->mem_array;
15a90fe0 3483
a129d92f
JK		 3484 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3485 status = be_cmd_iscsi_post_template_hdr(
3486 &phba->ctrl, &sgl);
15a90fe0 3487
a129d92f
JK		 3488 if (status != 0) {
3489 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3490 "BM_%d : Post Template HDR Failed for"
3491 "ULP_%d\n", ulp_num);
3492 return status;
3493 }
3494
3495 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3496 "BM_%d : Template HDR Pages Posted for"
3497 "ULP_%d\n", ulp_num);
15a90fe0
JK		 3498 }
3499 }
15a90fe0
JK		 3500 return 0;
3501}
3502
6733b39a
JK		 3503static int
3504beiscsi_post_pages(struct beiscsi_hba *phba)
3505{
3506 struct be_mem_descriptor *mem_descr;
3507 struct mem_array *pm_arr;
3508 unsigned int page_offset, i;
3509 struct be_dma_mem sgl;
843ae752 3510 int status, ulp_num = 0;
6733b39a
JK		 3511
3512 mem_descr = phba->init_mem;
3513 mem_descr += HWI_MEM_SGE;
3514 pm_arr = mem_descr->mem_array;
3515
90622db3
JK		 3516 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3517 if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
3518 break;
3519
6733b39a 3520 page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
843ae752 3521 phba->fw_config.iscsi_icd_start[ulp_num]) / PAGE_SIZE;
6733b39a
JK		 3522 for (i = 0; i < mem_descr->num_elements; i++) {
3523 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3524 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
3525 page_offset,
3526 (pm_arr->size / PAGE_SIZE));
3527 page_offset += pm_arr->size / PAGE_SIZE;
3528 if (status != 0) {
99bc5d55
JSJ		 3529 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3530 "BM_%d : post sgl failed.\n");
6733b39a
JK		 3531 return status;
3532 }
3533 pm_arr++;
3534 }
99bc5d55
JSJ		 3535 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3536 "BM_%d : POSTED PAGES\n");
6733b39a
JK		 3537 return 0;
3538}
3539
bfead3b2
JK		 3540static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
3541{
3542 struct be_dma_mem *mem = &q->dma_mem;
c8b25598 3543 if (mem->va) {
bfead3b2
JK		 3544 pci_free_consistent(phba->pcidev, mem->size,
3545 mem->va, mem->dma);
c8b25598
JK		 3546 mem->va = NULL;
3547 }
bfead3b2
JK		 3548}
3549
3550static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
3551 u16 len, u16 entry_size)
3552{
3553 struct be_dma_mem *mem = &q->dma_mem;
3554
3555 memset(q, 0, sizeof(*q));
3556 q->len = len;
3557 q->entry_size = entry_size;
3558 mem->size = len * entry_size;
3559 mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
3560 if (!mem->va)
d3ad2bb3 3561 return -ENOMEM;
bfead3b2
JK		 3562 memset(mem->va, 0, mem->size);
3563 return 0;
3564}
3565
6733b39a
JK		 3566static int
3567beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
3568 struct hwi_context_memory *phwi_context,
3569 struct hwi_controller *phwi_ctrlr)
3570{
3571 unsigned int wrb_mem_index, offset, size, num_wrb_rings;
3572 u64 pa_addr_lo;
4eea99d5 3573 unsigned int idx, num, i, ulp_num;
6733b39a
JK		 3574 struct mem_array *pwrb_arr;
3575 void *wrb_vaddr;
3576 struct be_dma_mem sgl;
3577 struct be_mem_descriptor *mem_descr;
a7909b39 3578 struct hwi_wrb_context *pwrb_context;
6733b39a 3579 int status;
4eea99d5
JK		 3580 uint8_t ulp_count = 0, ulp_base_num = 0;
3581 uint16_t cid_count_ulp[BEISCSI_ULP_COUNT] = { 0 };
6733b39a
JK		 3582
3583 idx = 0;
3584 mem_descr = phba->init_mem;
3585 mem_descr += HWI_MEM_WRB;
3586 pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
3587 GFP_KERNEL);
3588 if (!pwrb_arr) {
99bc5d55
JSJ		 3589 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3590 "BM_%d : Memory alloc failed in create wrb ring.\n");
6733b39a
JK		 3591 return -ENOMEM;
3592 }
3593 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3594 pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
3595 num_wrb_rings = mem_descr->mem_array[idx].size /
3596 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
3597
3598 for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
3599 if (num_wrb_rings) {
3600 pwrb_arr[num].virtual_address = wrb_vaddr;
3601 pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
3602 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3603 sizeof(struct iscsi_wrb);
3604 wrb_vaddr += pwrb_arr[num].size;
3605 pa_addr_lo += pwrb_arr[num].size;
3606 num_wrb_rings--;
3607 } else {
3608 idx++;
3609 wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3610 pa_addr_lo = mem_descr->mem_array[idx].\
3611 bus_address.u.a64.address;
3612 num_wrb_rings = mem_descr->mem_array[idx].size /
3613 (phba->params.wrbs_per_cxn *
3614 sizeof(struct iscsi_wrb));
3615 pwrb_arr[num].virtual_address = wrb_vaddr;
3616 pwrb_arr[num].bus_address.u.a64.address\
3617 = pa_addr_lo;
3618 pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3619 sizeof(struct iscsi_wrb);
3620 wrb_vaddr += pwrb_arr[num].size;
3621 pa_addr_lo += pwrb_arr[num].size;
3622 num_wrb_rings--;
3623 }
3624 }
4eea99d5
JK		 3625
3626 /* Get the ULP Count */
3627 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3628 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3629 ulp_count++;
3630 ulp_base_num = ulp_num;
3631 cid_count_ulp[ulp_num] =
3632 BEISCSI_GET_CID_COUNT(phba, ulp_num);
3633 }
3634
6733b39a
JK		 3635 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3636 wrb_mem_index = 0;
3637 offset = 0;
3638 size = 0;
3639
4eea99d5
JK		 3640 if (ulp_count > 1) {
3641 ulp_base_num = (ulp_base_num + 1) % BEISCSI_ULP_COUNT;
3642
3643 if (!cid_count_ulp[ulp_base_num])
3644 ulp_base_num = (ulp_base_num + 1) %
3645 BEISCSI_ULP_COUNT;
3646
3647 cid_count_ulp[ulp_base_num]--;
3648 }
3649
3650
6733b39a
JK		 3651 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
3652 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
4eea99d5
JK		 3653 &phwi_context->be_wrbq[i],
3654 &phwi_ctrlr->wrb_context[i],
3655 ulp_base_num);
6733b39a 3656 if (status != 0) {
99bc5d55
JSJ		 3657 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3658 "BM_%d : wrbq create failed.");
1462b8ff 3659 kfree(pwrb_arr);
6733b39a
JK		 3660 return status;
3661 }
a7909b39 3662 pwrb_context = &phwi_ctrlr->wrb_context[i];
a7909b39 3663 BE_SET_CID_TO_CRI(i, pwrb_context->cid);
6733b39a
JK		 3664 }
3665 kfree(pwrb_arr);
3666 return 0;
3667}
3668
3669static void free_wrb_handles(struct beiscsi_hba *phba)
3670{
3671 unsigned int index;
3672 struct hwi_controller *phwi_ctrlr;
3673 struct hwi_wrb_context *pwrb_context;
3674
3675 phwi_ctrlr = phba->phwi_ctrlr;
a7909b39 3676 for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
6733b39a
JK		 3677 pwrb_context = &phwi_ctrlr->wrb_context[index];
3678 kfree(pwrb_context->pwrb_handle_base);
3679 kfree(pwrb_context->pwrb_handle_basestd);
3680 }
3681}
3682
bfead3b2
JK		 3683static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3684{
3685 struct be_queue_info *q;
3686 struct be_ctrl_info *ctrl = &phba->ctrl;
3687
3688 q = &phba->ctrl.mcc_obj.q;
3689 if (q->created)
3690 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3691 be_queue_free(phba, q);
3692
3693 q = &phba->ctrl.mcc_obj.cq;
3694 if (q->created)
3695 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3696 be_queue_free(phba, q);
3697}
3698
6733b39a
JK		 3699static void hwi_cleanup(struct beiscsi_hba *phba)
3700{
3701 struct be_queue_info *q;
3702 struct be_ctrl_info *ctrl = &phba->ctrl;
3703 struct hwi_controller *phwi_ctrlr;
3704 struct hwi_context_memory *phwi_context;
a7909b39 3705 struct hwi_async_pdu_context *pasync_ctx;
8a86e833 3706 int i, eq_num, ulp_num;
6733b39a
JK		 3707
3708 phwi_ctrlr = phba->phwi_ctrlr;
3709 phwi_context = phwi_ctrlr->phwi_ctxt;
15a90fe0
JK		 3710
3711 be_cmd_iscsi_remove_template_hdr(ctrl);
3712
6733b39a
JK		 3713 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3714 q = &phwi_context->be_wrbq[i];
3715 if (q->created)
3716 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3717 }
a7909b39 3718 kfree(phwi_context->be_wrbq);
6733b39a
JK		 3719 free_wrb_handles(phba);
3720
8a86e833
JK		 3721 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3722 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
6733b39a 3723
8a86e833
JK		 3724 q = &phwi_context->be_def_hdrq[ulp_num];
3725 if (q->created)
3726 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3727
3728 q = &phwi_context->be_def_dataq[ulp_num];
3729 if (q->created)
3730 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3731
3732 pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];
3733 }
3734 }
6733b39a
JK		 3735
3736 beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3737
bfead3b2
JK		 3738 for (i = 0; i < (phba->num_cpus); i++) {
3739 q = &phwi_context->be_cq[i];
3740 if (q->created)
3741 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3742 }
3743 if (phba->msix_enabled)
3744 eq_num = 1;
3745 else
3746 eq_num = 0;
3747 for (i = 0; i < (phba->num_cpus + eq_num); i++) {
3748 q = &phwi_context->be_eq[i].q;
3749 if (q->created)
3750 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3751 }
3752 be_mcc_queues_destroy(phba);
0283fbb1 3753 be_cmd_fw_uninit(ctrl);
bfead3b2 3754}
6733b39a 3755
bfead3b2
JK		 3756static int be_mcc_queues_create(struct beiscsi_hba *phba,
3757 struct hwi_context_memory *phwi_context)
3758{
3759 struct be_queue_info *q, *cq;
3760 struct be_ctrl_info *ctrl = &phba->ctrl;
3761
3762 /* Alloc MCC compl queue */
3763 cq = &phba->ctrl.mcc_obj.cq;
3764 if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3765 sizeof(struct be_mcc_compl)))
3766 goto err;
3767 /* Ask BE to create MCC compl queue; */
3768 if (phba->msix_enabled) {
3769 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3770 [phba->num_cpus].q, false, true, 0))
3771 goto mcc_cq_free;
3772 } else {
3773 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3774 false, true, 0))
3775 goto mcc_cq_free;
3776 }
3777
3778 /* Alloc MCC queue */
3779 q = &phba->ctrl.mcc_obj.q;
3780 if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3781 goto mcc_cq_destroy;
3782
3783 /* Ask BE to create MCC queue */
35e66019 3784 if (beiscsi_cmd_mccq_create(phba, q, cq))
bfead3b2
JK		 3785 goto mcc_q_free;
3786
3787 return 0;
3788
3789mcc_q_free:
3790 be_queue_free(phba, q);
3791mcc_cq_destroy:
3792 beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3793mcc_cq_free:
3794 be_queue_free(phba, cq);
3795err:
d3ad2bb3 3796 return -ENOMEM;
bfead3b2
JK		 3797}
3798
107dfcba
JSJ		 3799/**
3800 * find_num_cpus()- Get the CPU online count
3801 * @phba: ptr to priv structure
3802 *
3803 * CPU count is used for creating EQ.
3804 **/
3805static void find_num_cpus(struct beiscsi_hba *phba)
bfead3b2
JK		 3806{
3807 int num_cpus = 0;
3808
3809 num_cpus = num_online_cpus();
bfead3b2 3810
22abeef0
JSJ		 3811 switch (phba->generation) {
3812 case BE_GEN2:
3813 case BE_GEN3:
3814 phba->num_cpus = (num_cpus > BEISCSI_MAX_NUM_CPUS) ?
3815 BEISCSI_MAX_NUM_CPUS : num_cpus;
3816 break;
3817 case BE_GEN4:
68c26a3a
JK		 3818 /*
3819 * If eqid_count == 1 fall back to
3820 * INTX mechanism
3821 **/
3822 if (phba->fw_config.eqid_count == 1) {
3823 enable_msix = 0;
3824 phba->num_cpus = 1;
3825 return;
3826 }
3827
3828 phba->num_cpus =
3829 (num_cpus > (phba->fw_config.eqid_count - 1)) ?
3830 (phba->fw_config.eqid_count - 1) : num_cpus;
22abeef0
JSJ		 3831 break;
3832 default:
3833 phba->num_cpus = 1;
3834 }
6733b39a
JK		 3835}
3836
3837static int hwi_init_port(struct beiscsi_hba *phba)
3838{
3839 struct hwi_controller *phwi_ctrlr;
3840 struct hwi_context_memory *phwi_context;
3841 unsigned int def_pdu_ring_sz;
3842 struct be_ctrl_info *ctrl = &phba->ctrl;
8a86e833 3843 int status, ulp_num;
6733b39a 3844
6733b39a 3845 phwi_ctrlr = phba->phwi_ctrlr;
6733b39a 3846 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 3847 phwi_context->max_eqd = 0;
3848 phwi_context->min_eqd = 0;
3849 phwi_context->cur_eqd = 64;
6733b39a 3850 be_cmd_fw_initialize(&phba->ctrl);
bfead3b2
JK		 3851
3852 status = beiscsi_create_eqs(phba, phwi_context);
6733b39a 3853 if (status != 0) {
99bc5d55
JSJ		 3854 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3855 "BM_%d : EQ not created\n");
6733b39a
JK		 3856 goto error;
3857 }
3858
bfead3b2
JK		 3859 status = be_mcc_queues_create(phba, phwi_context);
3860 if (status != 0)
3861 goto error;
3862
3863 status = mgmt_check_supported_fw(ctrl, phba);
6733b39a 3864 if (status != 0) {
99bc5d55
JSJ		 3865 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3866 "BM_%d : Unsupported fw version\n");
6733b39a
JK		 3867 goto error;
3868 }
3869
bfead3b2 3870 status = beiscsi_create_cqs(phba, phwi_context);
6733b39a 3871 if (status != 0) {
99bc5d55
JSJ		 3872 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3873 "BM_%d : CQ not created\n");
6733b39a
JK		 3874 goto error;
3875 }
3876
8a86e833
JK		 3877 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3878 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
6733b39a 3879
8a86e833
JK		 3880 def_pdu_ring_sz =
3881 BEISCSI_GET_CID_COUNT(phba, ulp_num) *
3882 sizeof(struct phys_addr);
3883
3884 status = beiscsi_create_def_hdr(phba, phwi_context,
3885 phwi_ctrlr,
3886 def_pdu_ring_sz,
3887 ulp_num);
3888 if (status != 0) {
3889 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3890 "BM_%d : Default Header not created for ULP : %d\n",
3891 ulp_num);
3892 goto error;
3893 }
3894
3895 status = beiscsi_create_def_data(phba, phwi_context,
3896 phwi_ctrlr,
3897 def_pdu_ring_sz,
3898 ulp_num);
3899 if (status != 0) {
3900 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3901 "BM_%d : Default Data not created for ULP : %d\n",
3902 ulp_num);
3903 goto error;
3904 }
3905 }
6733b39a
JK		 3906 }
3907
3908 status = beiscsi_post_pages(phba);
3909 if (status != 0) {
99bc5d55
JSJ		 3910 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3911 "BM_%d : Post SGL Pages Failed\n");
6733b39a
JK		 3912 goto error;
3913 }
3914
15a90fe0
JK		 3915 status = beiscsi_post_template_hdr(phba);
3916 if (status != 0) {
3917 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3918 "BM_%d : Template HDR Posting for CXN Failed\n");
3919 }
3920
6733b39a
JK		 3921 status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3922 if (status != 0) {
99bc5d55
JSJ		 3923 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3924 "BM_%d : WRB Rings not created\n");
6733b39a
JK		 3925 goto error;
3926 }
3927
8a86e833
JK		 3928 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3929 uint16_t async_arr_idx = 0;
3930
3931 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3932 uint16_t cri = 0;
3933 struct hwi_async_pdu_context *pasync_ctx;
3934
3935 pasync_ctx = HWI_GET_ASYNC_PDU_CTX(
3936 phwi_ctrlr, ulp_num);
3937 for (cri = 0; cri <
3938 phba->params.cxns_per_ctrl; cri++) {
3939 if (ulp_num == BEISCSI_GET_ULP_FROM_CRI
3940 (phwi_ctrlr, cri))
3941 pasync_ctx->cid_to_async_cri_map[
3942 phwi_ctrlr->wrb_context[cri].cid] =
3943 async_arr_idx++;
3944 }
3945 }
3946 }
3947
99bc5d55
JSJ		 3948 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3949 "BM_%d : hwi_init_port success\n");
6733b39a
JK		 3950 return 0;
3951
3952error:
99bc5d55
JSJ		 3953 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3954 "BM_%d : hwi_init_port failed");
6733b39a 3955 hwi_cleanup(phba);
a49e06d5 3956 return status;
6733b39a
JK		 3957}
3958
6733b39a
JK		 3959static int hwi_init_controller(struct beiscsi_hba *phba)
3960{
3961 struct hwi_controller *phwi_ctrlr;
3962
3963 phwi_ctrlr = phba->phwi_ctrlr;
3964 if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3965 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3966 init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
99bc5d55
JSJ		 3967 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3968 "BM_%d : phwi_ctrlr->phwi_ctxt=%p\n",
3969 phwi_ctrlr->phwi_ctxt);
6733b39a 3970 } else {
99bc5d55
JSJ		 3971 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3972 "BM_%d : HWI_MEM_ADDN_CONTEXT is more "
3973 "than one element.Failing to load\n");
6733b39a
JK		 3974 return -ENOMEM;
3975 }
3976
3977 iscsi_init_global_templates(phba);
3ec78271
JK		 3978 if (beiscsi_init_wrb_handle(phba))
3979 return -ENOMEM;
3980
a7909b39
JK		 3981 if (hwi_init_async_pdu_ctx(phba)) {
3982 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3983 "BM_%d : hwi_init_async_pdu_ctx failed\n");
3984 return -ENOMEM;
3985 }
3986
6733b39a 3987 if (hwi_init_port(phba) != 0) {
99bc5d55
JSJ		 3988 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3989 "BM_%d : hwi_init_controller failed\n");
3990
6733b39a
JK		 3991 return -ENOMEM;
3992 }
3993 return 0;
3994}
3995
3996static void beiscsi_free_mem(struct beiscsi_hba *phba)
3997{
3998 struct be_mem_descriptor *mem_descr;
3999 int i, j;
4000
4001 mem_descr = phba->init_mem;
4002 i = 0;
4003 j = 0;
4004 for (i = 0; i < SE_MEM_MAX; i++) {
4005 for (j = mem_descr->num_elements; j > 0; j--) {
4006 pci_free_consistent(phba->pcidev,
4007 mem_descr->mem_array[j - 1].size,
4008 mem_descr->mem_array[j - 1].virtual_address,
457ff3b7
JK		 4009 (unsigned long)mem_descr->mem_array[j - 1].
4010 bus_address.u.a64.address);
6733b39a 4011 }
8a86e833 4012
6733b39a
JK		 4013 kfree(mem_descr->mem_array);
4014 mem_descr++;
4015 }
4016 kfree(phba->init_mem);
a7909b39 4017 kfree(phba->phwi_ctrlr->wrb_context);
6733b39a
JK		 4018 kfree(phba->phwi_ctrlr);
4019}
4020
4021static int beiscsi_init_controller(struct beiscsi_hba *phba)
4022{
4023 int ret = -ENOMEM;
4024
4025 ret = beiscsi_get_memory(phba);
4026 if (ret < 0) {
99bc5d55
JSJ		 4027 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4028 "BM_%d : beiscsi_dev_probe -"
4029 "Failed in beiscsi_alloc_memory\n");
6733b39a
JK		 4030 return ret;
4031 }
4032
4033 ret = hwi_init_controller(phba);
4034 if (ret)
4035 goto free_init;
99bc5d55
JSJ		 4036 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4037 "BM_%d : Return success from beiscsi_init_controller");
4038
6733b39a
JK		 4039 return 0;
4040
4041free_init:
4042 beiscsi_free_mem(phba);
a49e06d5 4043 return ret;
6733b39a
JK		 4044}
4045
4046static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
4047{
4048 struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
4049 struct sgl_handle *psgl_handle;
4050 struct iscsi_sge *pfrag;
90622db3
JK		 4051 unsigned int arr_index, i, idx;
4052 unsigned int ulp_icd_start, ulp_num = 0;
6733b39a
JK		 4053
4054 phba->io_sgl_hndl_avbl = 0;
4055 phba->eh_sgl_hndl_avbl = 0;
bfead3b2 4056
6733b39a
JK		 4057 mem_descr_sglh = phba->init_mem;
4058 mem_descr_sglh += HWI_MEM_SGLH;
4059 if (1 == mem_descr_sglh->num_elements) {
4060 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
4061 phba->params.ios_per_ctrl,
4062 GFP_KERNEL);
4063 if (!phba->io_sgl_hndl_base) {
99bc5d55
JSJ		 4064 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4065 "BM_%d : Mem Alloc Failed. Failing to load\n");
6733b39a
JK		 4066 return -ENOMEM;
4067 }
4068 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
4069 (phba->params.icds_per_ctrl -
4070 phba->params.ios_per_ctrl),
4071 GFP_KERNEL);
4072 if (!phba->eh_sgl_hndl_base) {
4073 kfree(phba->io_sgl_hndl_base);
99bc5d55
JSJ		 4074 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4075 "BM_%d : Mem Alloc Failed. Failing to load\n");
6733b39a
JK		 4076 return -ENOMEM;
4077 }
4078 } else {
99bc5d55
JSJ		 4079 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4080 "BM_%d : HWI_MEM_SGLH is more than one element."
4081 "Failing to load\n");
6733b39a
JK		 4082 return -ENOMEM;
4083 }
4084
4085 arr_index = 0;
4086 idx = 0;
4087 while (idx < mem_descr_sglh->num_elements) {
4088 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
4089
4090 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
4091 sizeof(struct sgl_handle)); i++) {
4092 if (arr_index < phba->params.ios_per_ctrl) {
4093 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
4094 phba->io_sgl_hndl_avbl++;
4095 arr_index++;
4096 } else {
4097 phba->eh_sgl_hndl_base[arr_index -
4098 phba->params.ios_per_ctrl] =
4099 psgl_handle;
4100 arr_index++;
4101 phba->eh_sgl_hndl_avbl++;
4102 }
4103 psgl_handle++;
4104 }
4105 idx++;
4106 }
99bc5d55
JSJ		 4107 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4108 "BM_%d : phba->io_sgl_hndl_avbl=%d"
4109 "phba->eh_sgl_hndl_avbl=%d\n",
4110 phba->io_sgl_hndl_avbl,
4111 phba->eh_sgl_hndl_avbl);
4112
6733b39a
JK		 4113 mem_descr_sg = phba->init_mem;
4114 mem_descr_sg += HWI_MEM_SGE;
99bc5d55
JSJ		 4115 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4116 "\n BM_%d : mem_descr_sg->num_elements=%d\n",
4117 mem_descr_sg->num_elements);
4118
90622db3
JK		 4119 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
4120 if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
4121 break;
4122
4123 ulp_icd_start = phba->fw_config.iscsi_icd_start[ulp_num];
4124
6733b39a
JK		 4125 arr_index = 0;
4126 idx = 0;
4127 while (idx < mem_descr_sg->num_elements) {
4128 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
4129
4130 for (i = 0;
4131 i < (mem_descr_sg->mem_array[idx].size) /
4132 (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
4133 i++) {
4134 if (arr_index < phba->params.ios_per_ctrl)
4135 psgl_handle = phba->io_sgl_hndl_base[arr_index];
4136 else
4137 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
4138 phba->params.ios_per_ctrl];
4139 psgl_handle->pfrag = pfrag;
4140 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
4141 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
4142 pfrag += phba->params.num_sge_per_io;
90622db3 4143 psgl_handle->sgl_index = ulp_icd_start + arr_index++;
6733b39a
JK		 4144 }
4145 idx++;
4146 }
4147 phba->io_sgl_free_index = 0;
4148 phba->io_sgl_alloc_index = 0;
4149 phba->eh_sgl_free_index = 0;
4150 phba->eh_sgl_alloc_index = 0;
4151 return 0;
4152}
4153
4154static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
4155{
0a3db7c0
JK		 4156 int ret;
4157 uint16_t i, ulp_num;
4158 struct ulp_cid_info *ptr_cid_info = NULL;
6733b39a 4159
0a3db7c0
JK		 4160 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4161 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4162 ptr_cid_info = kzalloc(sizeof(struct ulp_cid_info),
4163 GFP_KERNEL);
4164
4165 if (!ptr_cid_info) {
4166 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4167 "BM_%d : Failed to allocate memory"
4168 "for ULP_CID_INFO for ULP : %d\n",
4169 ulp_num);
4170 ret = -ENOMEM;
4171 goto free_memory;
4172
4173 }
4174
4175 /* Allocate memory for CID array */
4176 ptr_cid_info->cid_array = kzalloc(sizeof(void *) *
4177 BEISCSI_GET_CID_COUNT(phba,
4178 ulp_num), GFP_KERNEL);
4179 if (!ptr_cid_info->cid_array) {
4180 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4181 "BM_%d : Failed to allocate memory"
4182 "for CID_ARRAY for ULP : %d\n",
4183 ulp_num);
4184 kfree(ptr_cid_info);
4185 ptr_cid_info = NULL;
4186 ret = -ENOMEM;
4187
4188 goto free_memory;
4189 }
4190 ptr_cid_info->avlbl_cids = BEISCSI_GET_CID_COUNT(
4191 phba, ulp_num);
4192
4193 /* Save the cid_info_array ptr */
4194 phba->cid_array_info[ulp_num] = ptr_cid_info;
4195 }
6733b39a 4196 }
c2462288 4197 phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
a7909b39 4198 phba->params.cxns_per_ctrl, GFP_KERNEL);
6733b39a 4199 if (!phba->ep_array) {
99bc5d55
JSJ		 4200 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4201 "BM_%d : Failed to allocate memory in "
4202 "hba_setup_cid_tbls\n");
0a3db7c0
JK		 4203 ret = -ENOMEM;
4204
4205 goto free_memory;
6733b39a 4206 }
a7909b39
JK		 4207
4208 phba->conn_table = kzalloc(sizeof(struct beiscsi_conn *) *
4209 phba->params.cxns_per_ctrl, GFP_KERNEL);
4210 if (!phba->conn_table) {
4211 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4212 "BM_%d : Failed to allocate memory in"
4213 "hba_setup_cid_tbls\n");
4214
a7909b39 4215 kfree(phba->ep_array);
a7909b39 4216 phba->ep_array = NULL;
0a3db7c0 4217 ret = -ENOMEM;
6733b39a 4218 }
a7909b39 4219
0a3db7c0
JK		 4220 for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
4221 ulp_num = phba->phwi_ctrlr->wrb_context[i].ulp_num;
4222
4223 ptr_cid_info = phba->cid_array_info[ulp_num];
4224 ptr_cid_info->cid_array[ptr_cid_info->cid_alloc++] =
4225 phba->phwi_ctrlr->wrb_context[i].cid;
4226
4227 }
4228
4229 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4230 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4231 ptr_cid_info = phba->cid_array_info[ulp_num];
a7909b39 4232
0a3db7c0
JK		 4233 ptr_cid_info->cid_alloc = 0;
4234 ptr_cid_info->cid_free = 0;
4235 }
4236 }
6733b39a 4237 return 0;
0a3db7c0
JK		 4238
4239free_memory:
4240 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4241 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4242 ptr_cid_info = phba->cid_array_info[ulp_num];
4243
4244 if (ptr_cid_info) {
4245 kfree(ptr_cid_info->cid_array);
4246 kfree(ptr_cid_info);
4247 phba->cid_array_info[ulp_num] = NULL;
4248 }
4249 }
4250 }
4251
4252 return ret;
6733b39a
JK		 4253}
4254
238f6b72 4255static void hwi_enable_intr(struct beiscsi_hba *phba)
6733b39a
JK		 4256{
4257 struct be_ctrl_info *ctrl = &phba->ctrl;
4258 struct hwi_controller *phwi_ctrlr;
4259 struct hwi_context_memory *phwi_context;
4260 struct be_queue_info *eq;
4261 u8 __iomem *addr;
bfead3b2 4262 u32 reg, i;
6733b39a
JK		 4263 u32 enabled;
4264
4265 phwi_ctrlr = phba->phwi_ctrlr;
4266 phwi_context = phwi_ctrlr->phwi_ctxt;
4267
6733b39a
JK		 4268 addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
4269 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
4270 reg = ioread32(addr);
6733b39a
JK		 4271
4272 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4273 if (!enabled) {
4274 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
99bc5d55
JSJ		 4275 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4276 "BM_%d : reg =x%08x addr=%p\n", reg, addr);
6733b39a 4277 iowrite32(reg, addr);
665d6d94
JK		 4278 }
4279
4280 if (!phba->msix_enabled) {
4281 eq = &phwi_context->be_eq[0].q;
99bc5d55
JSJ		 4282 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4283 "BM_%d : eq->id=%d\n", eq->id);
4284
665d6d94
JK		 4285 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
4286 } else {
4287 for (i = 0; i <= phba->num_cpus; i++) {
4288 eq = &phwi_context->be_eq[i].q;
99bc5d55
JSJ		 4289 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4290 "BM_%d : eq->id=%d\n", eq->id);
bfead3b2
JK		 4291 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
4292 }
c03af1ae 4293 }
6733b39a
JK		 4294}
4295
4296static void hwi_disable_intr(struct beiscsi_hba *phba)
4297{
4298 struct be_ctrl_info *ctrl = &phba->ctrl;
4299
4300 u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
4301 u32 reg = ioread32(addr);
4302
4303 u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4304 if (enabled) {
4305 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4306 iowrite32(reg, addr);
4307 } else
99bc5d55
JSJ		 4308 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
4309 "BM_%d : In hwi_disable_intr, Already Disabled\n");
6733b39a
JK		 4310}
4311
9aef4200
JSJ		 4312/**
4313 * beiscsi_get_boot_info()- Get the boot session info
4314 * @phba: The device priv structure instance
4315 *
4316 * Get the boot target info and store in driver priv structure
4317 *
4318 * return values
4319 * Success: 0
4320 * Failure: Non-Zero Value
4321 **/
c7acc5b8
JK		 4322static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
4323{
0e43895e 4324 struct be_cmd_get_session_resp *session_resp;
c7acc5b8 4325 struct be_dma_mem nonemb_cmd;
e175defe 4326 unsigned int tag;
9aef4200 4327 unsigned int s_handle;
f457a46f 4328 int ret = -ENOMEM;
c7acc5b8 4329
9aef4200
JSJ		 4330 /* Get the session handle of the boot target */
4331 ret = be_mgmt_get_boot_shandle(phba, &s_handle);
4332 if (ret) {
99bc5d55
JSJ		 4333 beiscsi_log(phba, KERN_ERR,
4334 BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
4335 "BM_%d : No boot session\n");
9aef4200 4336 return ret;
c7acc5b8 4337 }
c7acc5b8
JK		 4338 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
4339 sizeof(*session_resp),
4340 &nonemb_cmd.dma);
4341 if (nonemb_cmd.va == NULL) {
99bc5d55
JSJ		 4342 beiscsi_log(phba, KERN_ERR,
4343 BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
4344 "BM_%d : Failed to allocate memory for"
4345 "beiscsi_get_session_info\n");
4346
c7acc5b8
JK		 4347 return -ENOMEM;
4348 }
4349
4350 memset(nonemb_cmd.va, 0, sizeof(*session_resp));
9aef4200 4351 tag = mgmt_get_session_info(phba, s_handle,
0e43895e 4352 &nonemb_cmd);
c7acc5b8 4353 if (!tag) {
99bc5d55
JSJ		 4354 beiscsi_log(phba, KERN_ERR,
4355 BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
4356 "BM_%d : beiscsi_get_session_info"
4357 " Failed\n");
4358
c7acc5b8 4359 goto boot_freemem;
e175defe 4360 }
c7acc5b8 4361
e175defe
JSJ		 4362 ret = beiscsi_mccq_compl(phba, tag, NULL, nonemb_cmd.va);
4363 if (ret) {
99bc5d55
JSJ		 4364 beiscsi_log(phba, KERN_ERR,
4365 BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
e175defe 4366 "BM_%d : beiscsi_get_session_info Failed");
c7acc5b8
JK		 4367 goto boot_freemem;
4368 }
e175defe 4369
c7acc5b8 4370 session_resp = nonemb_cmd.va ;
f457a46f 4371
c7acc5b8
JK		 4372 memcpy(&phba->boot_sess, &session_resp->session_info,
4373 sizeof(struct mgmt_session_info));
f457a46f
MC		 4374 ret = 0;
4375
c7acc5b8
JK		 4376boot_freemem:
4377 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4378 nonemb_cmd.va, nonemb_cmd.dma);
f457a46f
MC		 4379 return ret;
4380}
4381
4382static void beiscsi_boot_release(void *data)
4383{
4384 struct beiscsi_hba *phba = data;
4385
4386 scsi_host_put(phba->shost);
4387}
4388
4389static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
4390{
4391 struct iscsi_boot_kobj *boot_kobj;
4392
4393 /* get boot info using mgmt cmd */
4394 if (beiscsi_get_boot_info(phba))
4395 /* Try to see if we can carry on without this */
4396 return 0;
4397
4398 phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
4399 if (!phba->boot_kset)
4400 return -ENOMEM;
4401
4402 /* get a ref because the show function will ref the phba */
4403 if (!scsi_host_get(phba->shost))
4404 goto free_kset;
4405 boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
4406 beiscsi_show_boot_tgt_info,
4407 beiscsi_tgt_get_attr_visibility,
4408 beiscsi_boot_release);
4409 if (!boot_kobj)
4410 goto put_shost;
4411
4412 if (!scsi_host_get(phba->shost))
4413 goto free_kset;
4414 boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
4415 beiscsi_show_boot_ini_info,
4416 beiscsi_ini_get_attr_visibility,
4417 beiscsi_boot_release);
4418 if (!boot_kobj)
4419 goto put_shost;
4420
4421 if (!scsi_host_get(phba->shost))
4422 goto free_kset;
4423 boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
4424 beiscsi_show_boot_eth_info,
4425 beiscsi_eth_get_attr_visibility,
4426 beiscsi_boot_release);
4427 if (!boot_kobj)
4428 goto put_shost;
4429 return 0;
4430
4431put_shost:
4432 scsi_host_put(phba->shost);
4433free_kset:
4434 iscsi_boot_destroy_kset(phba->boot_kset);
53e7ab66 4435 phba->boot_kset = NULL;
c7acc5b8
JK		 4436 return -ENOMEM;
4437}
4438
6733b39a
JK		 4439static int beiscsi_init_port(struct beiscsi_hba *phba)
4440{
4441 int ret;
4442
4443 ret = beiscsi_init_controller(phba);
4444 if (ret < 0) {
99bc5d55
JSJ		 4445 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4446 "BM_%d : beiscsi_dev_probe - Failed in"
4447 "beiscsi_init_controller\n");
6733b39a
JK		 4448 return ret;
4449 }
4450 ret = beiscsi_init_sgl_handle(phba);
4451 if (ret < 0) {
99bc5d55
JSJ		 4452 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4453 "BM_%d : beiscsi_dev_probe - Failed in"
4454 "beiscsi_init_sgl_handle\n");
6733b39a
JK		 4455 goto do_cleanup_ctrlr;
4456 }
4457
4458 if (hba_setup_cid_tbls(phba)) {
99bc5d55
JSJ		 4459 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4460 "BM_%d : Failed in hba_setup_cid_tbls\n");
6733b39a
JK		 4461 kfree(phba->io_sgl_hndl_base);
4462 kfree(phba->eh_sgl_hndl_base);
4463 goto do_cleanup_ctrlr;
4464 }
4465
4466 return ret;
4467
4468do_cleanup_ctrlr:
4469 hwi_cleanup(phba);
4470 return ret;
4471}
4472
4473static void hwi_purge_eq(struct beiscsi_hba *phba)
4474{
4475 struct hwi_controller *phwi_ctrlr;
4476 struct hwi_context_memory *phwi_context;
4477 struct be_queue_info *eq;
4478 struct be_eq_entry *eqe = NULL;
bfead3b2 4479 int i, eq_msix;
756d29c8 4480 unsigned int num_processed;
6733b39a
JK		 4481
4482 phwi_ctrlr = phba->phwi_ctrlr;
4483 phwi_context = phwi_ctrlr->phwi_ctxt;
bfead3b2
JK		 4484 if (phba->msix_enabled)
4485 eq_msix = 1;
4486 else
4487 eq_msix = 0;
6733b39a 4488
bfead3b2
JK		 4489 for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
4490 eq = &phwi_context->be_eq[i].q;
6733b39a 4491 eqe = queue_tail_node(eq);
756d29c8 4492 num_processed = 0;
bfead3b2
JK		 4493 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
4494 & EQE_VALID_MASK) {
4495 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
4496 queue_tail_inc(eq);
4497 eqe = queue_tail_node(eq);
756d29c8 4498 num_processed++;
bfead3b2 4499 }
756d29c8
JK		 4500
4501 if (num_processed)
4502 hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
6733b39a
JK		 4503 }
4504}
4505
4506static void beiscsi_clean_port(struct beiscsi_hba *phba)
4507{
0a3db7c0
JK		 4508 int mgmt_status, ulp_num;
4509 struct ulp_cid_info *ptr_cid_info = NULL;
6733b39a 4510
bd41c2bd
JK		 4511 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4512 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4513 mgmt_status = mgmt_epfw_cleanup(phba, ulp_num);
4514 if (mgmt_status)
4515 beiscsi_log(phba, KERN_WARNING,
4516 BEISCSI_LOG_INIT,
4517 "BM_%d : mgmt_epfw_cleanup FAILED"
4518 " for ULP_%d\n", ulp_num);
4519 }
4520 }
756d29c8 4521
6733b39a 4522 hwi_purge_eq(phba);
756d29c8 4523 hwi_cleanup(phba);
6733b39a
JK		 4524 kfree(phba->io_sgl_hndl_base);
4525 kfree(phba->eh_sgl_hndl_base);
6733b39a 4526 kfree(phba->ep_array);
a7909b39 4527 kfree(phba->conn_table);
0a3db7c0
JK		 4528
4529 for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4530 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4531 ptr_cid_info = phba->cid_array_info[ulp_num];
4532
4533 if (ptr_cid_info) {
4534 kfree(ptr_cid_info->cid_array);
4535 kfree(ptr_cid_info);
4536 phba->cid_array_info[ulp_num] = NULL;
4537 }
4538 }
4539 }
4540
6733b39a
JK		 4541}
4542
43f388b0
JK		 4543/**
4544 * beiscsi_free_mgmt_task_handles()- Free driver CXN resources
4545 * @beiscsi_conn: ptr to the conn to be cleaned up
4a4a11b9 4546 * @task: ptr to iscsi_task resource to be freed.
43f388b0
JK		 4547 *
4548 * Free driver mgmt resources binded to CXN.
4549 **/
4550void
4a4a11b9
JK		 4551beiscsi_free_mgmt_task_handles(struct beiscsi_conn *beiscsi_conn,
4552 struct iscsi_task *task)
43f388b0
JK		 4553{
4554 struct beiscsi_io_task *io_task;
4555 struct beiscsi_hba *phba = beiscsi_conn->phba;
4556 struct hwi_wrb_context *pwrb_context;
4557 struct hwi_controller *phwi_ctrlr;
a7909b39
JK		 4558 uint16_t cri_index = BE_GET_CRI_FROM_CID(
4559 beiscsi_conn->beiscsi_conn_cid);
43f388b0
JK		 4560
4561 phwi_ctrlr = phba->phwi_ctrlr;
a7909b39
JK		 4562 pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
4563
4a4a11b9 4564 io_task = task->dd_data;
43f388b0
JK		 4565
4566 if (io_task->pwrb_handle) {
4567 memset(io_task->pwrb_handle->pwrb, 0,
4568 sizeof(struct iscsi_wrb));
4569 free_wrb_handle(phba, pwrb_context,
4570 io_task->pwrb_handle);
4571 io_task->pwrb_handle = NULL;
4572 }
4573
4574 if (io_task->psgl_handle) {
4575 spin_lock_bh(&phba->mgmt_sgl_lock);
4576 free_mgmt_sgl_handle(phba,
4577 io_task->psgl_handle);
43f388b0 4578 io_task->psgl_handle = NULL;
4a4a11b9 4579 spin_unlock_bh(&phba->mgmt_sgl_lock);
43f388b0
JK		 4580 }
4581
4582 if (io_task->mtask_addr)
4583 pci_unmap_single(phba->pcidev,
4584 io_task->mtask_addr,
4585 io_task->mtask_data_count,
4586 PCI_DMA_TODEVICE);
4587}
4588
d629c471
JSJ		 4589/**
4590 * beiscsi_cleanup_task()- Free driver resources of the task
4591 * @task: ptr to the iscsi task
4592 *
4593 **/
1282ab76
MC		 4594static void beiscsi_cleanup_task(struct iscsi_task *task)
4595{
4596 struct beiscsi_io_task *io_task = task->dd_data;
4597 struct iscsi_conn *conn = task->conn;
4598 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4599 struct beiscsi_hba *phba = beiscsi_conn->phba;
4600 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4601 struct hwi_wrb_context *pwrb_context;
4602 struct hwi_controller *phwi_ctrlr;
a7909b39
JK		 4603 uint16_t cri_index = BE_GET_CRI_FROM_CID(
4604 beiscsi_conn->beiscsi_conn_cid);
1282ab76
MC		 4605
4606 phwi_ctrlr = phba->phwi_ctrlr;
a7909b39 4607 pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
1282ab76
MC		 4608
4609 if (io_task->cmd_bhs) {
4610 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4611 io_task->bhs_pa.u.a64.address);
4612 io_task->cmd_bhs = NULL;
4613 }
4614
4615 if (task->sc) {
4616 if (io_task->pwrb_handle) {
4617 free_wrb_handle(phba, pwrb_context,
4618 io_task->pwrb_handle);
4619 io_task->pwrb_handle = NULL;
4620 }
4621
4622 if (io_task->psgl_handle) {
4623 spin_lock(&phba->io_sgl_lock);
4624 free_io_sgl_handle(phba, io_task->psgl_handle);
4625 spin_unlock(&phba->io_sgl_lock);
4626 io_task->psgl_handle = NULL;
4627 }
4628 } else {
43f388b0 4629 if (!beiscsi_conn->login_in_progress)
4a4a11b9 4630 beiscsi_free_mgmt_task_handles(beiscsi_conn, task);
1282ab76
MC		 4631 }
4632}
4633
6733b39a
JK		 4634void
4635beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
4636 struct beiscsi_offload_params *params)
4637{
4638 struct wrb_handle *pwrb_handle;
6733b39a 4639 struct beiscsi_hba *phba = beiscsi_conn->phba;
1282ab76
MC		 4640 struct iscsi_task *task = beiscsi_conn->task;
4641 struct iscsi_session *session = task->conn->session;
6733b39a
JK		 4642 u32 doorbell = 0;
4643
4644 /*
4645 * We can always use 0 here because it is reserved by libiscsi for
4646 * login/startup related tasks.
4647 */
1282ab76
MC		 4648 beiscsi_conn->login_in_progress = 0;
4649 spin_lock_bh(&session->lock);
4650 beiscsi_cleanup_task(task);
4651 spin_unlock_bh(&session->lock);
4652
a7909b39 4653 pwrb_handle = alloc_wrb_handle(phba, beiscsi_conn->beiscsi_conn_cid);
6733b39a 4654
acb9693c 4655 /* Check for the adapter family */
2c9dfd36 4656 if (is_chip_be2_be3r(phba))
acb9693c
JSJ		 4657 beiscsi_offload_cxn_v0(params, pwrb_handle,
4658 phba->init_mem);
2c9dfd36
JK		 4659 else
4660 beiscsi_offload_cxn_v2(params, pwrb_handle);
6733b39a 4661
acb9693c
JSJ		 4662 be_dws_le_to_cpu(pwrb_handle->pwrb,
4663 sizeof(struct iscsi_target_context_update_wrb));
6733b39a
JK		 4664
4665 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
32951dd8 4666 doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
bfead3b2 4667 << DB_DEF_PDU_WRB_INDEX_SHIFT;
6733b39a 4668 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
1e4be6ff
JK		 4669 iowrite32(doorbell, phba->db_va +
4670 beiscsi_conn->doorbell_offset);
6733b39a
JK		 4671}
4672
4673static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
4674 int *index, int *age)
4675{
bfead3b2 4676 *index = (int)itt;
6733b39a
JK		 4677 if (age)
4678 *age = conn->session->age;
4679}
4680
4681/**
4682 * beiscsi_alloc_pdu - allocates pdu and related resources
4683 * @task: libiscsi task
4684 * @opcode: opcode of pdu for task
4685 *
4686 * This is called with the session lock held. It will allocate
4687 * the wrb and sgl if needed for the command. And it will prep
4688 * the pdu's itt. beiscsi_parse_pdu will later translate
4689 * the pdu itt to the libiscsi task itt.
4690 */
4691static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
4692{
4693 struct beiscsi_io_task *io_task = task->dd_data;
4694 struct iscsi_conn *conn = task->conn;
4695 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4696 struct beiscsi_hba *phba = beiscsi_conn->phba;
4697 struct hwi_wrb_context *pwrb_context;
4698 struct hwi_controller *phwi_ctrlr;
4699 itt_t itt;
a7909b39 4700 uint16_t cri_index = 0;
2afc95bf
JK		 4701 struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4702 dma_addr_t paddr;
6733b39a 4703
2afc95bf 4704 io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
bc7accec 4705 GFP_ATOMIC, &paddr);
2afc95bf
JK		 4706 if (!io_task->cmd_bhs)
4707 return -ENOMEM;
2afc95bf 4708 io_task->bhs_pa.u.a64.address = paddr;
bfead3b2 4709 io_task->libiscsi_itt = (itt_t)task->itt;
6733b39a
JK		 4710 io_task->conn = beiscsi_conn;
4711
4712 task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
4713 task->hdr_max = sizeof(struct be_cmd_bhs);
d2cecf0d 4714 io_task->psgl_handle = NULL;
3ec78271 4715 io_task->pwrb_handle = NULL;
6733b39a
JK		 4716
4717 if (task->sc) {
4718 spin_lock(&phba->io_sgl_lock);
4719 io_task->psgl_handle = alloc_io_sgl_handle(phba);
4720 spin_unlock(&phba->io_sgl_lock);
8359c79b
JSJ		 4721 if (!io_task->psgl_handle) {
4722 beiscsi_log(phba, KERN_ERR,
4723 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4724 "BM_%d : Alloc of IO_SGL_ICD Failed"
4725 "for the CID : %d\n",
4726 beiscsi_conn->beiscsi_conn_cid);
2afc95bf 4727 goto free_hndls;
8359c79b 4728 }
d2cecf0d 4729 io_task->pwrb_handle = alloc_wrb_handle(phba,
a7909b39 4730 beiscsi_conn->beiscsi_conn_cid);
8359c79b
JSJ		 4731 if (!io_task->pwrb_handle) {
4732 beiscsi_log(phba, KERN_ERR,
4733 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4734 "BM_%d : Alloc of WRB_HANDLE Failed"
4735 "for the CID : %d\n",
4736 beiscsi_conn->beiscsi_conn_cid);
d2cecf0d 4737 goto free_io_hndls;
8359c79b 4738 }
6733b39a
JK		 4739 } else {
4740 io_task->scsi_cmnd = NULL;
d7aea67b 4741 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
43f388b0 4742 beiscsi_conn->task = task;
6733b39a
JK		 4743 if (!beiscsi_conn->login_in_progress) {
4744 spin_lock(&phba->mgmt_sgl_lock);
4745 io_task->psgl_handle = (struct sgl_handle *)
4746 alloc_mgmt_sgl_handle(phba);
4747 spin_unlock(&phba->mgmt_sgl_lock);
8359c79b
JSJ		 4748 if (!io_task->psgl_handle) {
4749 beiscsi_log(phba, KERN_ERR,
4750 BEISCSI_LOG_IO |
4751 BEISCSI_LOG_CONFIG,
4752 "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4753 "for the CID : %d\n",
4754 beiscsi_conn->
4755 beiscsi_conn_cid);
2afc95bf 4756 goto free_hndls;
8359c79b 4757 }
2afc95bf 4758
6733b39a
JK		 4759 beiscsi_conn->login_in_progress = 1;
4760 beiscsi_conn->plogin_sgl_handle =
4761 io_task->psgl_handle;
d2cecf0d
JK		 4762 io_task->pwrb_handle =
4763 alloc_wrb_handle(phba,
a7909b39 4764 beiscsi_conn->beiscsi_conn_cid);
8359c79b
JSJ		 4765 if (!io_task->pwrb_handle) {
4766 beiscsi_log(phba, KERN_ERR,
4767 BEISCSI_LOG_IO |
4768 BEISCSI_LOG_CONFIG,
4769 "BM_%d : Alloc of WRB_HANDLE Failed"
4770 "for the CID : %d\n",
4771 beiscsi_conn->
4772 beiscsi_conn_cid);
4773 goto free_mgmt_hndls;
4774 }
d2cecf0d
JK		 4775 beiscsi_conn->plogin_wrb_handle =
4776 io_task->pwrb_handle;
4777
6733b39a
JK		 4778 } else {
4779 io_task->psgl_handle =
4780 beiscsi_conn->plogin_sgl_handle;
d2cecf0d
JK		 4781 io_task->pwrb_handle =
4782 beiscsi_conn->plogin_wrb_handle;
6733b39a
JK		 4783 }
4784 } else {
4785 spin_lock(&phba->mgmt_sgl_lock);
4786 io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
4787 spin_unlock(&phba->mgmt_sgl_lock);
8359c79b
JSJ		 4788 if (!io_task->psgl_handle) {
4789 beiscsi_log(phba, KERN_ERR,
4790 BEISCSI_LOG_IO |
4791 BEISCSI_LOG_CONFIG,
4792 "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4793 "for the CID : %d\n",
4794 beiscsi_conn->
4795 beiscsi_conn_cid);
2afc95bf 4796 goto free_hndls;
8359c79b 4797 }
d2cecf0d
JK		 4798 io_task->pwrb_handle =
4799 alloc_wrb_handle(phba,
a7909b39 4800 beiscsi_conn->beiscsi_conn_cid);
8359c79b
JSJ		 4801 if (!io_task->pwrb_handle) {
4802 beiscsi_log(phba, KERN_ERR,
4803 BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4804 "BM_%d : Alloc of WRB_HANDLE Failed"
4805 "for the CID : %d\n",
4806 beiscsi_conn->beiscsi_conn_cid);
d2cecf0d 4807 goto free_mgmt_hndls;
8359c79b 4808 }
d2cecf0d 4809
6733b39a
JK		 4810 }
4811 }
bfead3b2
JK		 4812 itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
4813 wrb_index << 16) | (unsigned int)
4814 (io_task->psgl_handle->sgl_index));
32951dd8 4815 io_task->pwrb_handle->pio_handle = task;
bfead3b2 4816
6733b39a
JK		 4817 io_task->cmd_bhs->iscsi_hdr.itt = itt;
4818 return 0;
2afc95bf 4819
d2cecf0d
JK		 4820free_io_hndls:
4821 spin_lock(&phba->io_sgl_lock);
4822 free_io_sgl_handle(phba, io_task->psgl_handle);
4823 spin_unlock(&phba->io_sgl_lock);
4824 goto free_hndls;
4825free_mgmt_hndls:
4826 spin_lock(&phba->mgmt_sgl_lock);
4827 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
a7909b39 4828 io_task->psgl_handle = NULL;
d2cecf0d 4829 spin_unlock(&phba->mgmt_sgl_lock);
2afc95bf
JK		 4830free_hndls:
4831 phwi_ctrlr = phba->phwi_ctrlr;
a7909b39
JK		 4832 cri_index = BE_GET_CRI_FROM_CID(
4833 beiscsi_conn->beiscsi_conn_cid);
4834 pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
d2cecf0d
JK		 4835 if (io_task->pwrb_handle)
4836 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
2afc95bf
JK		 4837 io_task->pwrb_handle = NULL;
4838 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4839 io_task->bhs_pa.u.a64.address);
1282ab76 4840 io_task->cmd_bhs = NULL;
2afc95bf 4841 return -ENOMEM;
6733b39a 4842}
09a1093a
JSJ		 4843int beiscsi_iotask_v2(struct iscsi_task *task, struct scatterlist *sg,
4844 unsigned int num_sg, unsigned int xferlen,
4845 unsigned int writedir)
4846{
4847
4848 struct beiscsi_io_task *io_task = task->dd_data;
4849 struct iscsi_conn *conn = task->conn;
4850 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4851 struct beiscsi_hba *phba = beiscsi_conn->phba;
4852 struct iscsi_wrb *pwrb = NULL;
4853 unsigned int doorbell = 0;
4854
4855 pwrb = io_task->pwrb_handle->pwrb;
09a1093a
JSJ		 4856
4857 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4858 io_task->bhs_len = sizeof(struct be_cmd_bhs);
4859
4860 if (writedir) {
4861 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4862 INI_WR_CMD);
4863 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 1);
4864 } else {
4865 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4866 INI_RD_CMD);
4867 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 0);
4868 }
4869
4870 io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb_v2,
4871 type, pwrb);
4872
4873 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, lun, pwrb,
4874 cpu_to_be16(*(unsigned short *)
4875 &io_task->cmd_bhs->iscsi_hdr.lun));
4876 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb, xferlen);
4877 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4878 io_task->pwrb_handle->wrb_index);
4879 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4880 be32_to_cpu(task->cmdsn));
4881 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4882 io_task->psgl_handle->sgl_index);
4883
4884 hwi_write_sgl_v2(pwrb, sg, num_sg, io_task);
4885 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4886 io_task->pwrb_handle->nxt_wrb_index);
4887
4888 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4889
4890 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4891 doorbell |= (io_task->pwrb_handle->wrb_index &
4892 DB_DEF_PDU_WRB_INDEX_MASK) <<
4893 DB_DEF_PDU_WRB_INDEX_SHIFT;
4894 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
1e4be6ff
JK		 4895 iowrite32(doorbell, phba->db_va +
4896 beiscsi_conn->doorbell_offset);
09a1093a
JSJ		 4897 return 0;
4898}
6733b39a 4899
6733b39a
JK		 4900static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
4901 unsigned int num_sg, unsigned int xferlen,
4902 unsigned int writedir)
4903{
4904
4905 struct beiscsi_io_task *io_task = task->dd_data;
4906 struct iscsi_conn *conn = task->conn;
4907 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4908 struct beiscsi_hba *phba = beiscsi_conn->phba;
4909 struct iscsi_wrb *pwrb = NULL;
4910 unsigned int doorbell = 0;
4911
4912 pwrb = io_task->pwrb_handle->pwrb;
6733b39a
JK		 4913 io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4914 io_task->bhs_len = sizeof(struct be_cmd_bhs);
4915
4916 if (writedir) {
32951dd8
JK		 4917 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4918 INI_WR_CMD);
6733b39a 4919 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
6733b39a 4920 } else {
32951dd8
JK		 4921 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4922 INI_RD_CMD);
6733b39a
JK		 4923 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
4924 }
6733b39a 4925
09a1093a
JSJ		 4926 io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb,
4927 type, pwrb);
4928
6733b39a 4929 AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
dc63aac6
JK		 4930 cpu_to_be16(*(unsigned short *)
4931 &io_task->cmd_bhs->iscsi_hdr.lun));
6733b39a
JK		 4932 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
4933 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4934 io_task->pwrb_handle->wrb_index);
4935 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4936 be32_to_cpu(task->cmdsn));
4937 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4938 io_task->psgl_handle->sgl_index);
4939
4940 hwi_write_sgl(pwrb, sg, num_sg, io_task);
4941
4942 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4943 io_task->pwrb_handle->nxt_wrb_index);
4944 be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4945
4946 doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
32951dd8 4947 doorbell |= (io_task->pwrb_handle->wrb_index &
6733b39a
JK		 4948 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4949 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4950
1e4be6ff
JK		 4951 iowrite32(doorbell, phba->db_va +
4952 beiscsi_conn->doorbell_offset);
6733b39a
JK		 4953 return 0;
4954}
4955
4956static int beiscsi_mtask(struct iscsi_task *task)
4957{
dafab8e0 4958 struct beiscsi_io_task *io_task = task->dd_data;
6733b39a
JK		 4959 struct iscsi_conn *conn = task->conn;
4960 struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4961 struct beiscsi_hba *phba = beiscsi_conn->phba;
4962 struct iscsi_wrb *pwrb = NULL;
4963 unsigned int doorbell = 0;
dafab8e0 4964 unsigned int cid;
09a1093a 4965 unsigned int pwrb_typeoffset = 0;
6733b39a 4966
bfead3b2 4967 cid = beiscsi_conn->beiscsi_conn_cid;
6733b39a 4968 pwrb = io_task->pwrb_handle->pwrb;
caf818f1 4969 memset(pwrb, 0, sizeof(*pwrb));
09a1093a 4970
2c9dfd36 4971 if (is_chip_be2_be3r(phba)) {
09a1093a
JSJ		 4972 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4973 be32_to_cpu(task->cmdsn));
4974 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4975 io_task->pwrb_handle->wrb_index);
4976 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4977 io_task->psgl_handle->sgl_index);
4978 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4979 task->data_count);
4980 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4981 io_task->pwrb_handle->nxt_wrb_index);
4982 pwrb_typeoffset = BE_WRB_TYPE_OFFSET;
2c9dfd36
JK		 4983 } else {
4984 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4985 be32_to_cpu(task->cmdsn));
4986 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4987 io_task->pwrb_handle->wrb_index);
4988 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4989 io_task->psgl_handle->sgl_index);
4990 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb,
4991 task->data_count);
4992 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4993 io_task->pwrb_handle->nxt_wrb_index);
4994 pwrb_typeoffset = SKH_WRB_TYPE_OFFSET;
09a1093a
JSJ		 4995 }
4996
dafab8e0 4997
6733b39a
JK		 4998 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4999 case ISCSI_OP_LOGIN:
6733b39a 5000 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
09a1093a 5001 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
6733b39a
JK		 5002 hwi_write_buffer(pwrb, task);
5003 break;
5004 case ISCSI_OP_NOOP_OUT:
1390b01b 5005 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
09a1093a 5006 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
2c9dfd36
JK		 5007 if (is_chip_be2_be3r(phba))
5008 AMAP_SET_BITS(struct amap_iscsi_wrb,
09a1093a
JSJ		 5009 dmsg, pwrb, 1);
5010 else
2c9dfd36 5011 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
09a1093a 5012 dmsg, pwrb, 1);
1390b01b 5013 } else {
09a1093a 5014 ADAPTER_SET_WRB_TYPE(pwrb, INI_RD_CMD, pwrb_typeoffset);
2c9dfd36
JK		 5015 if (is_chip_be2_be3r(phba))
5016 AMAP_SET_BITS(struct amap_iscsi_wrb,
09a1093a
JSJ		 5017 dmsg, pwrb, 0);
5018 else
2c9dfd36 5019 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
09a1093a 5020 dmsg, pwrb, 0);
1390b01b 5021 }
6733b39a
JK		 5022 hwi_write_buffer(pwrb, task);
5023 break;
5024 case ISCSI_OP_TEXT:
09a1093a 5025 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
6733b39a
JK		 5026 hwi_write_buffer(pwrb, task);
5027 break;
5028 case ISCSI_OP_SCSI_TMFUNC:
09a1093a 5029 ADAPTER_SET_WRB_TYPE(pwrb, INI_TMF_CMD, pwrb_typeoffset);
6733b39a
JK		 5030 hwi_write_buffer(pwrb, task);
5031 break;
5032 case ISCSI_OP_LOGOUT:
09a1093a 5033 ADAPTER_SET_WRB_TYPE(pwrb, HWH_TYPE_LOGOUT, pwrb_typeoffset);
6733b39a
JK		 5034 hwi_write_buffer(pwrb, task);
5035 break;
5036
5037 default:
99bc5d55
JSJ		 5038 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
5039 "BM_%d : opcode =%d Not supported\n",
5040 task->hdr->opcode & ISCSI_OPCODE_MASK);
5041
6733b39a
JK		 5042 return -EINVAL;
5043 }
5044
09a1093a 5045 /* Set the task type */
2c9dfd36
JK		 5046 io_task->wrb_type = (is_chip_be2_be3r(phba)) ?
5047 AMAP_GET_BITS(struct amap_iscsi_wrb, type, pwrb) :
5048 AMAP_GET_BITS(struct amap_iscsi_wrb_v2, type, pwrb);
6733b39a 5049
bfead3b2 5050 doorbell |= cid & DB_WRB_POST_CID_MASK;
32951dd8 5051 doorbell |= (io_task->pwrb_handle->wrb_index &
6733b39a
JK		 5052 DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
5053 doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
1e4be6ff
JK		 5054 iowrite32(doorbell, phba->db_va +
5055 beiscsi_conn->doorbell_offset);
6733b39a
JK		 5056 return 0;
5057}
5058
5059static int beiscsi_task_xmit(struct iscsi_task *task)
5060{
6733b39a
JK		 5061 struct beiscsi_io_task *io_task = task->dd_data;
5062 struct scsi_cmnd *sc = task->sc;
09a1093a 5063 struct beiscsi_hba *phba = NULL;
6733b39a
JK		 5064 struct scatterlist *sg;
5065 int num_sg;
5066 unsigned int writedir = 0, xferlen = 0;
5067
09a1093a
JSJ		 5068 phba = ((struct beiscsi_conn *)task->conn->dd_data)->phba;
5069
6733b39a
JK		 5070 if (!sc)
5071 return beiscsi_mtask(task);
5072
5073 io_task->scsi_cmnd = sc;
5074 num_sg = scsi_dma_map(sc);
5075 if (num_sg < 0) {
99bc5d55
JSJ		 5076 struct iscsi_conn *conn = task->conn;
5077 struct beiscsi_hba *phba = NULL;
5078
5079 phba = ((struct beiscsi_conn *)conn->dd_data)->phba;
afb96058
JK		 5080 beiscsi_log(phba, KERN_ERR,
5081 BEISCSI_LOG_IO | BEISCSI_LOG_ISCSI,
5082 "BM_%d : scsi_dma_map Failed "
5083 "Driver_ITT : 0x%x ITT : 0x%x Xferlen : 0x%x\n",
5084 be32_to_cpu(io_task->cmd_bhs->iscsi_hdr.itt),
5085 io_task->libiscsi_itt, scsi_bufflen(sc));
99bc5d55 5086
6733b39a
JK		 5087 return num_sg;
5088 }
6733b39a
JK		 5089 xferlen = scsi_bufflen(sc);
5090 sg = scsi_sglist(sc);
99bc5d55 5091 if (sc->sc_data_direction == DMA_TO_DEVICE)
6733b39a 5092 writedir = 1;
99bc5d55 5093 else
6733b39a 5094 writedir = 0;
99bc5d55 5095
09a1093a 5096 return phba->iotask_fn(task, sg, num_sg, xferlen, writedir);
6733b39a
JK		 5097}
5098
ffce3e2e
JK		 5099/**
5100 * beiscsi_bsg_request - handle bsg request from ISCSI transport
5101 * @job: job to handle
5102 */
5103static int beiscsi_bsg_request(struct bsg_job *job)
5104{
5105 struct Scsi_Host *shost;
5106 struct beiscsi_hba *phba;
5107 struct iscsi_bsg_request *bsg_req = job->request;
5108 int rc = -EINVAL;
5109 unsigned int tag;
5110 struct be_dma_mem nonemb_cmd;
5111 struct be_cmd_resp_hdr *resp;
5112 struct iscsi_bsg_reply *bsg_reply = job->reply;
5113 unsigned short status, extd_status;
5114
5115 shost = iscsi_job_to_shost(job);
5116 phba = iscsi_host_priv(shost);
5117
5118 switch (bsg_req->msgcode) {
5119 case ISCSI_BSG_HST_VENDOR:
5120 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
5121 job->request_payload.payload_len,
5122 &nonemb_cmd.dma);
5123 if (nonemb_cmd.va == NULL) {
99bc5d55
JSJ		 5124 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
5125 "BM_%d : Failed to allocate memory for "
5126 "beiscsi_bsg_request\n");
8359c79b 5127 return -ENOMEM;
ffce3e2e
JK		 5128 }
5129 tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,
5130 &nonemb_cmd);
5131 if (!tag) {
99bc5d55 5132 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
8359c79b 5133 "BM_%d : MBX Tag Allocation Failed\n");
99bc5d55 5134
ffce3e2e
JK		 5135 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
5136 nonemb_cmd.va, nonemb_cmd.dma);
5137 return -EAGAIN;
e175defe
JSJ		 5138 }
5139
5140 rc = wait_event_interruptible_timeout(
5141 phba->ctrl.mcc_wait[tag],
5142 phba->ctrl.mcc_numtag[tag],
5143 msecs_to_jiffies(
5144 BEISCSI_HOST_MBX_TIMEOUT));
ffce3e2e
JK		 5145 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
5146 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
5147 free_mcc_tag(&phba->ctrl, tag);
5148 resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va;
5149 sg_copy_from_buffer(job->reply_payload.sg_list,
5150 job->reply_payload.sg_cnt,
5151 nonemb_cmd.va, (resp->response_length
5152 + sizeof(*resp)));
5153 bsg_reply->reply_payload_rcv_len = resp->response_length;
5154 bsg_reply->result = status;
5155 bsg_job_done(job, bsg_reply->result,
5156 bsg_reply->reply_payload_rcv_len);
5157 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
5158 nonemb_cmd.va, nonemb_cmd.dma);
5159 if (status || extd_status) {
99bc5d55 5160 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
8359c79b 5161 "BM_%d : MBX Cmd Failed"
99bc5d55
JSJ		 5162 " status = %d extd_status = %d\n",
5163 status, extd_status);
5164
ffce3e2e 5165 return -EIO;
8359c79b
JSJ		 5166 } else {
5167 rc = 0;
ffce3e2e
JK		 5168 }
5169 break;
5170
5171 default:
99bc5d55
JSJ		 5172 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
5173 "BM_%d : Unsupported bsg command: 0x%x\n",
5174 bsg_req->msgcode);
ffce3e2e
JK		 5175 break;
5176 }
5177
5178 return rc;
5179}
5180
99bc5d55
JSJ		 5181void beiscsi_hba_attrs_init(struct beiscsi_hba *phba)
5182{
5183 /* Set the logging parameter */
5184 beiscsi_log_enable_init(phba, beiscsi_log_enable);
5185}
5186
4d4d1ef8
JSJ		 5187/*
5188 * beiscsi_quiesce()- Cleanup Driver resources
5189 * @phba: Instance Priv structure
3567f36a 5190 * @unload_state:i Clean or EEH unload state
4d4d1ef8
JSJ		 5191 *
5192 * Free the OS and HW resources held by the driver
5193 **/
3567f36a
JK		 5194static void beiscsi_quiesce(struct beiscsi_hba *phba,
5195 uint32_t unload_state)
6733b39a 5196{
bfead3b2
JK		 5197 struct hwi_controller *phwi_ctrlr;
5198 struct hwi_context_memory *phwi_context;
5199 struct be_eq_obj *pbe_eq;
5200 unsigned int i, msix_vec;
6733b39a 5201
bfead3b2
JK		 5202 phwi_ctrlr = phba->phwi_ctrlr;
5203 phwi_context = phwi_ctrlr->phwi_ctxt;
6733b39a 5204 hwi_disable_intr(phba);
bfead3b2
JK		 5205 if (phba->msix_enabled) {
5206 for (i = 0; i <= phba->num_cpus; i++) {
5207 msix_vec = phba->msix_entries[i].vector;
3567f36a 5208 synchronize_irq(msix_vec);
bfead3b2 5209 free_irq(msix_vec, &phwi_context->be_eq[i]);
8fcfb210 5210 kfree(phba->msi_name[i]);
bfead3b2
JK		 5211 }
5212 } else
3567f36a
JK		 5213 if (phba->pcidev->irq) {
5214 synchronize_irq(phba->pcidev->irq);
bfead3b2 5215 free_irq(phba->pcidev->irq, phba);
3567f36a 5216 }
bfead3b2 5217 pci_disable_msix(phba->pcidev);
3567f36a 5218
6733b39a 5219 if (blk_iopoll_enabled)
bfead3b2
JK		 5220 for (i = 0; i < phba->num_cpus; i++) {
5221 pbe_eq = &phwi_context->be_eq[i];
5222 blk_iopoll_disable(&pbe_eq->iopoll);
5223 }
6733b39a 5224
3567f36a
JK		 5225 if (unload_state == BEISCSI_CLEAN_UNLOAD) {
5226 destroy_workqueue(phba->wq);
5227 beiscsi_clean_port(phba);
5228 beiscsi_free_mem(phba);
e9b91193 5229
3567f36a
JK		 5230 beiscsi_unmap_pci_function(phba);
5231 pci_free_consistent(phba->pcidev,
5232 phba->ctrl.mbox_mem_alloced.size,
5233 phba->ctrl.mbox_mem_alloced.va,
5234 phba->ctrl.mbox_mem_alloced.dma);
5235 } else {
5236 hwi_purge_eq(phba);
5237 hwi_cleanup(phba);
5238 }
7a158003
JSJ		 5239
5240 cancel_delayed_work_sync(&phba->beiscsi_hw_check_task);
25602c97
JK		 5241}
5242
5243static void beiscsi_remove(struct pci_dev *pcidev)
5244{
5245
5246 struct beiscsi_hba *phba = NULL;
5247
5248 phba = pci_get_drvdata(pcidev);
5249 if (!phba) {
5250 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
5251 return;
5252 }
5253
0e43895e 5254 beiscsi_destroy_def_ifaces(phba);
3567f36a 5255 beiscsi_quiesce(phba, BEISCSI_CLEAN_UNLOAD);
9d045163 5256 iscsi_boot_destroy_kset(phba->boot_kset);
6733b39a
JK		 5257 iscsi_host_remove(phba->shost);
5258 pci_dev_put(phba->pcidev);
5259 iscsi_host_free(phba->shost);
3567f36a
JK		 5260 pci_disable_pcie_error_reporting(pcidev);
5261 pci_set_drvdata(pcidev, NULL);
8dce69ff 5262 pci_disable_device(pcidev);
6733b39a
JK		 5263}
5264
25602c97
JK		 5265static void beiscsi_shutdown(struct pci_dev *pcidev)
5266{
5267
5268 struct beiscsi_hba *phba = NULL;
5269
5270 phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
5271 if (!phba) {
5272 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n");
5273 return;
5274 }
5275
3567f36a 5276 beiscsi_quiesce(phba, BEISCSI_CLEAN_UNLOAD);
8dce69ff 5277 pci_disable_device(pcidev);
25602c97
JK		 5278}
5279
bfead3b2
JK		 5280static void beiscsi_msix_enable(struct beiscsi_hba *phba)
5281{
5282 int i, status;
5283
5284 for (i = 0; i <= phba->num_cpus; i++)
5285 phba->msix_entries[i].entry = i;
5286
5287 status = pci_enable_msix(phba->pcidev, phba->msix_entries,
5288 (phba->num_cpus + 1));
5289 if (!status)
5290 phba->msix_enabled = true;
5291
5292 return;
5293}
5294
7a158003
JSJ		 5295/*
5296 * beiscsi_hw_health_check()- Check adapter health
5297 * @work: work item to check HW health
5298 *
5299 * Check if adapter in an unrecoverable state or not.
5300 **/
5301static void
5302beiscsi_hw_health_check(struct work_struct *work)
5303{
5304 struct beiscsi_hba *phba =
5305 container_of(work, struct beiscsi_hba,
5306 beiscsi_hw_check_task.work);
5307
5308 beiscsi_ue_detect(phba);
5309
5310 schedule_delayed_work(&phba->beiscsi_hw_check_task,
5311 msecs_to_jiffies(1000));
5312}
5313
3567f36a
JK		 5314
5315static pci_ers_result_t beiscsi_eeh_err_detected(struct pci_dev *pdev,
5316 pci_channel_state_t state)
5317{
5318 struct beiscsi_hba *phba = NULL;
5319
5320 phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5321 phba->state |= BE_ADAPTER_PCI_ERR;
5322
5323 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5324 "BM_%d : EEH error detected\n");
5325
5326 beiscsi_quiesce(phba, BEISCSI_EEH_UNLOAD);
5327
5328 if (state == pci_channel_io_perm_failure) {
5329 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5330 "BM_%d : EEH : State PERM Failure");
5331 return PCI_ERS_RESULT_DISCONNECT;
5332 }
5333
5334 pci_disable_device(pdev);
5335
5336 /* The error could cause the FW to trigger a flash debug dump.
5337 * Resetting the card while flash dump is in progress
5338 * can cause it not to recover; wait for it to finish.
5339 * Wait only for first function as it is needed only once per
5340 * adapter.
5341 **/
5342 if (pdev->devfn == 0)
5343 ssleep(30);
5344
5345 return PCI_ERS_RESULT_NEED_RESET;
5346}
5347
5348static pci_ers_result_t beiscsi_eeh_reset(struct pci_dev *pdev)
5349{
5350 struct beiscsi_hba *phba = NULL;
5351 int status = 0;
5352
5353 phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5354
5355 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5356 "BM_%d : EEH Reset\n");
5357
5358 status = pci_enable_device(pdev);
5359 if (status)
5360 return PCI_ERS_RESULT_DISCONNECT;
5361
5362 pci_set_master(pdev);
5363 pci_set_power_state(pdev, PCI_D0);
5364 pci_restore_state(pdev);
5365
5366 /* Wait for the CHIP Reset to complete */
5367 status = be_chk_reset_complete(phba);
5368 if (!status) {
5369 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5370 "BM_%d : EEH Reset Completed\n");
5371 } else {
5372 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5373 "BM_%d : EEH Reset Completion Failure\n");
5374 return PCI_ERS_RESULT_DISCONNECT;
5375 }
5376
5377 pci_cleanup_aer_uncorrect_error_status(pdev);
5378 return PCI_ERS_RESULT_RECOVERED;
5379}
5380
5381static void beiscsi_eeh_resume(struct pci_dev *pdev)
5382{
5383 int ret = 0, i;
5384 struct be_eq_obj *pbe_eq;
5385 struct beiscsi_hba *phba = NULL;
5386 struct hwi_controller *phwi_ctrlr;
5387 struct hwi_context_memory *phwi_context;
5388
5389 phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5390 pci_save_state(pdev);
5391
5392 if (enable_msix)
5393 find_num_cpus(phba);
5394 else
5395 phba->num_cpus = 1;
5396
5397 if (enable_msix) {
5398 beiscsi_msix_enable(phba);
5399 if (!phba->msix_enabled)
5400 phba->num_cpus = 1;
5401 }
5402
5403 ret = beiscsi_cmd_reset_function(phba);
5404 if (ret) {
5405 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5406 "BM_%d : Reset Failed\n");
5407 goto ret_err;
5408 }
5409
5410 ret = be_chk_reset_complete(phba);
5411 if (ret) {
5412 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5413 "BM_%d : Failed to get out of reset.\n");
5414 goto ret_err;
5415 }
5416
5417 beiscsi_get_params(phba);
5418 phba->shost->max_id = phba->params.cxns_per_ctrl;
5419 phba->shost->can_queue = phba->params.ios_per_ctrl;
5420 ret = hwi_init_controller(phba);
5421
5422 for (i = 0; i < MAX_MCC_CMD; i++) {
5423 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
5424 phba->ctrl.mcc_tag[i] = i + 1;
5425 phba->ctrl.mcc_numtag[i + 1] = 0;
5426 phba->ctrl.mcc_tag_available++;
5427 }
5428
5429 phwi_ctrlr = phba->phwi_ctrlr;
5430 phwi_context = phwi_ctrlr->phwi_ctxt;
5431
5432 if (blk_iopoll_enabled) {
5433 for (i = 0; i < phba->num_cpus; i++) {
5434 pbe_eq = &phwi_context->be_eq[i];
5435 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
5436 be_iopoll);
5437 blk_iopoll_enable(&pbe_eq->iopoll);
5438 }
5439
5440 i = (phba->msix_enabled) ? i : 0;
5441 /* Work item for MCC handling */
5442 pbe_eq = &phwi_context->be_eq[i];
5443 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
5444 } else {
5445 if (phba->msix_enabled) {
5446 for (i = 0; i <= phba->num_cpus; i++) {
5447 pbe_eq = &phwi_context->be_eq[i];
5448 INIT_WORK(&pbe_eq->work_cqs,
5449 beiscsi_process_all_cqs);
5450 }
5451 } else {
5452 pbe_eq = &phwi_context->be_eq[0];
5453 INIT_WORK(&pbe_eq->work_cqs,
5454 beiscsi_process_all_cqs);
5455 }
5456 }
5457
5458 ret = beiscsi_init_irqs(phba);
5459 if (ret < 0) {
5460 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5461 "BM_%d : beiscsi_eeh_resume - "
5462 "Failed to beiscsi_init_irqs\n");
5463 goto ret_err;
5464 }
5465
5466 hwi_enable_intr(phba);
5467 phba->state &= ~BE_ADAPTER_PCI_ERR;
5468
5469 return;
5470ret_err:
5471 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5472 "BM_%d : AER EEH Resume Failed\n");
5473}
5474
6f039790
GKH		 5475static int beiscsi_dev_probe(struct pci_dev *pcidev,
5476 const struct pci_device_id *id)
6733b39a
JK		 5477{
5478 struct beiscsi_hba *phba = NULL;
bfead3b2
JK		 5479 struct hwi_controller *phwi_ctrlr;
5480 struct hwi_context_memory *phwi_context;
5481 struct be_eq_obj *pbe_eq;
3567f36a 5482 int ret = 0, i;
6733b39a
JK		 5483
5484 ret = beiscsi_enable_pci(pcidev);
5485 if (ret < 0) {
99bc5d55
JSJ		 5486 dev_err(&pcidev->dev,
5487 "beiscsi_dev_probe - Failed to enable pci device\n");
6733b39a
JK		 5488 return ret;
5489 }
5490
5491 phba = beiscsi_hba_alloc(pcidev);
5492 if (!phba) {
99bc5d55
JSJ		 5493 dev_err(&pcidev->dev,
5494 "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");
6733b39a
JK		 5495 goto disable_pci;
5496 }
5497
3567f36a
JK		 5498 /* Enable EEH reporting */
5499 ret = pci_enable_pcie_error_reporting(pcidev);
5500 if (ret)
5501 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5502 "BM_%d : PCIe Error Reporting "
5503 "Enabling Failed\n");
5504
5505 pci_save_state(pcidev);
5506
99bc5d55
JSJ		 5507 /* Initialize Driver configuration Paramters */
5508 beiscsi_hba_attrs_init(phba);
5509
e175defe 5510 phba->fw_timeout = false;
6c83185a 5511 phba->mac_addr_set = false;
e175defe
JSJ		 5512
5513
f98c96b0
JK		 5514 switch (pcidev->device) {
5515 case BE_DEVICE_ID1:
5516 case OC_DEVICE_ID1:
5517 case OC_DEVICE_ID2:
5518 phba->generation = BE_GEN2;
09a1093a 5519 phba->iotask_fn = beiscsi_iotask;
f98c96b0
JK		 5520 break;
5521 case BE_DEVICE_ID2:
5522 case OC_DEVICE_ID3:
5523 phba->generation = BE_GEN3;
09a1093a 5524 phba->iotask_fn = beiscsi_iotask;
f98c96b0 5525 break;
139a1b1e
JSJ		 5526 case OC_SKH_ID1:
5527 phba->generation = BE_GEN4;
09a1093a 5528 phba->iotask_fn = beiscsi_iotask_v2;
bf9131cb 5529 break;
f98c96b0
JK		 5530 default:
5531 phba->generation = 0;
5532 }
5533
6733b39a
JK		 5534 ret = be_ctrl_init(phba, pcidev);
5535 if (ret) {
99bc5d55
JSJ		 5536 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5537 "BM_%d : beiscsi_dev_probe-"
5538 "Failed in be_ctrl_init\n");
6733b39a
JK		 5539 goto hba_free;
5540 }
5541
4d4d1ef8
JSJ		 5542 ret = beiscsi_cmd_reset_function(phba);
5543 if (ret) {
5544 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
92665a66 5545 "BM_%d : Reset Failed\n");
4d4d1ef8
JSJ		 5546 goto hba_free;
5547 }
5548 ret = be_chk_reset_complete(phba);
5549 if (ret) {
5550 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
92665a66 5551 "BM_%d : Failed to get out of reset.\n");
4d4d1ef8 5552 goto hba_free;
e9b91193
JK		 5553 }
5554
6733b39a
JK		 5555 spin_lock_init(&phba->io_sgl_lock);
5556 spin_lock_init(&phba->mgmt_sgl_lock);
5557 spin_lock_init(&phba->isr_lock);
8f09a3b9 5558 spin_lock_init(&phba->async_pdu_lock);
7da50879
JK		 5559 ret = mgmt_get_fw_config(&phba->ctrl, phba);
5560 if (ret != 0) {
99bc5d55
JSJ		 5561 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5562 "BM_%d : Error getting fw config\n");
7da50879
JK		 5563 goto free_port;
5564 }
68c26a3a
JK		 5565
5566 if (enable_msix)
5567 find_num_cpus(phba);
5568 else
5569 phba->num_cpus = 1;
5570
5571 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
5572 "BM_%d : num_cpus = %d\n",
5573 phba->num_cpus);
5574
5575 if (enable_msix) {
5576 beiscsi_msix_enable(phba);
5577 if (!phba->msix_enabled)
5578 phba->num_cpus = 1;
5579 }
5580
843ae752 5581 phba->shost->max_id = phba->params.cxns_per_ctrl;
6733b39a 5582 beiscsi_get_params(phba);
aa874f07 5583 phba->shost->can_queue = phba->params.ios_per_ctrl;
6733b39a
JK		 5584 ret = beiscsi_init_port(phba);
5585 if (ret < 0) {
99bc5d55
JSJ		 5586 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5587 "BM_%d : beiscsi_dev_probe-"
5588 "Failed in beiscsi_init_port\n");
6733b39a
JK		 5589 goto free_port;
5590 }
5591
3567f36a 5592 for (i = 0; i < MAX_MCC_CMD; i++) {
756d29c8
JK		 5593 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
5594 phba->ctrl.mcc_tag[i] = i + 1;
5595 phba->ctrl.mcc_numtag[i + 1] = 0;
5596 phba->ctrl.mcc_tag_available++;
5597 }
5598
5599 phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
5600
72fb46a9 5601 snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_%02x_wq",
6733b39a 5602 phba->shost->host_no);
d8537548 5603 phba->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 1, phba->wq_name);
6733b39a 5604 if (!phba->wq) {
99bc5d55
JSJ		 5605 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5606 "BM_%d : beiscsi_dev_probe-"
5607 "Failed to allocate work queue\n");
6733b39a
JK		 5608 goto free_twq;
5609 }
5610
7a158003
JSJ		 5611 INIT_DELAYED_WORK(&phba->beiscsi_hw_check_task,
5612 beiscsi_hw_health_check);
6733b39a 5613
bfead3b2
JK		 5614 phwi_ctrlr = phba->phwi_ctrlr;
5615 phwi_context = phwi_ctrlr->phwi_ctxt;
72fb46a9 5616
6733b39a 5617 if (blk_iopoll_enabled) {
bfead3b2
JK		 5618 for (i = 0; i < phba->num_cpus; i++) {
5619 pbe_eq = &phwi_context->be_eq[i];
5620 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
5621 be_iopoll);
5622 blk_iopoll_enable(&pbe_eq->iopoll);
5623 }
72fb46a9
JSJ		 5624
5625 i = (phba->msix_enabled) ? i : 0;
5626 /* Work item for MCC handling */
5627 pbe_eq = &phwi_context->be_eq[i];
5628 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
5629 } else {
5630 if (phba->msix_enabled) {
5631 for (i = 0; i <= phba->num_cpus; i++) {
5632 pbe_eq = &phwi_context->be_eq[i];
5633 INIT_WORK(&pbe_eq->work_cqs,
5634 beiscsi_process_all_cqs);
5635 }
5636 } else {
5637 pbe_eq = &phwi_context->be_eq[0];
5638 INIT_WORK(&pbe_eq->work_cqs,
5639 beiscsi_process_all_cqs);
5640 }
6733b39a 5641 }
72fb46a9 5642
6733b39a
JK		 5643 ret = beiscsi_init_irqs(phba);
5644 if (ret < 0) {
99bc5d55
JSJ		 5645 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5646 "BM_%d : beiscsi_dev_probe-"
5647 "Failed to beiscsi_init_irqs\n");
6733b39a
JK		 5648 goto free_blkenbld;
5649 }
238f6b72 5650 hwi_enable_intr(phba);
f457a46f
MC		 5651
5652 if (beiscsi_setup_boot_info(phba))
5653 /*
5654 * log error but continue, because we may not be using
5655 * iscsi boot.
5656 */
99bc5d55
JSJ		 5657 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5658 "BM_%d : Could not set up "
5659 "iSCSI boot info.\n");
f457a46f 5660
0e43895e 5661 beiscsi_create_def_ifaces(phba);
7a158003
JSJ		 5662 schedule_delayed_work(&phba->beiscsi_hw_check_task,
5663 msecs_to_jiffies(1000));
5664
99bc5d55
JSJ		 5665 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
5666 "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");
6733b39a
JK		 5667 return 0;
5668
6733b39a
JK		 5669free_blkenbld:
5670 destroy_workqueue(phba->wq);
5671 if (blk_iopoll_enabled)
bfead3b2
JK		 5672 for (i = 0; i < phba->num_cpus; i++) {
5673 pbe_eq = &phwi_context->be_eq[i];
5674 blk_iopoll_disable(&pbe_eq->iopoll);
5675 }
6733b39a
JK		 5676free_twq:
5677 beiscsi_clean_port(phba);
5678 beiscsi_free_mem(phba);
5679free_port:
5680 pci_free_consistent(phba->pcidev,
5681 phba->ctrl.mbox_mem_alloced.size,
5682 phba->ctrl.mbox_mem_alloced.va,
5683 phba->ctrl.mbox_mem_alloced.dma);
5684 beiscsi_unmap_pci_function(phba);
5685hba_free:
238f6b72
JK		 5686 if (phba->msix_enabled)
5687 pci_disable_msix(phba->pcidev);
6733b39a
JK		 5688 pci_dev_put(phba->pcidev);
5689 iscsi_host_free(phba->shost);
c9fda3f2 5690 pci_set_drvdata(pcidev, NULL);
6733b39a
JK		 5691disable_pci:
5692 pci_disable_device(pcidev);
5693 return ret;
5694}
5695
3567f36a
JK		 5696static struct pci_error_handlers beiscsi_eeh_handlers = {
5697 .error_detected = beiscsi_eeh_err_detected,
5698 .slot_reset = beiscsi_eeh_reset,
5699 .resume = beiscsi_eeh_resume,
5700};
5701
6733b39a
JK		 5702struct iscsi_transport beiscsi_iscsi_transport = {
5703 .owner = THIS_MODULE,
5704 .name = DRV_NAME,
9db0fb3a 5705 .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
6733b39a 5706 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
6733b39a
JK		 5707 .create_session = beiscsi_session_create,
5708 .destroy_session = beiscsi_session_destroy,
5709 .create_conn = beiscsi_conn_create,
5710 .bind_conn = beiscsi_conn_bind,
5711 .destroy_conn = iscsi_conn_teardown,
3128c6c7 5712 .attr_is_visible = be2iscsi_attr_is_visible,
0e43895e
MC		 5713 .set_iface_param = be2iscsi_iface_set_param,
5714 .get_iface_param = be2iscsi_iface_get_param,
6733b39a 5715 .set_param = beiscsi_set_param,
c7f7fd5b 5716 .get_conn_param = iscsi_conn_get_param,
6733b39a
JK		 5717 .get_session_param = iscsi_session_get_param,
5718 .get_host_param = beiscsi_get_host_param,
5719 .start_conn = beiscsi_conn_start,
fa95d206 5720 .stop_conn = iscsi_conn_stop,
6733b39a
JK		 5721 .send_pdu = iscsi_conn_send_pdu,
5722 .xmit_task = beiscsi_task_xmit,
5723 .cleanup_task = beiscsi_cleanup_task,
5724 .alloc_pdu = beiscsi_alloc_pdu,
5725 .parse_pdu_itt = beiscsi_parse_pdu,
5726 .get_stats = beiscsi_conn_get_stats,
c7f7fd5b 5727 .get_ep_param = beiscsi_ep_get_param,
6733b39a
JK		 5728 .ep_connect = beiscsi_ep_connect,
5729 .ep_poll = beiscsi_ep_poll,
5730 .ep_disconnect = beiscsi_ep_disconnect,
5731 .session_recovery_timedout = iscsi_session_recovery_timedout,
ffce3e2e 5732 .bsg_request = beiscsi_bsg_request,
6733b39a
JK		 5733};
5734
5735static struct pci_driver beiscsi_pci_driver = {
5736 .name = DRV_NAME,
5737 .probe = beiscsi_dev_probe,
5738 .remove = beiscsi_remove,
25602c97 5739 .shutdown = beiscsi_shutdown,
3567f36a
JK		 5740 .id_table = beiscsi_pci_id_table,
5741 .err_handler = &beiscsi_eeh_handlers
6733b39a
JK		 5742};
5743
bfead3b2 5744
6733b39a
JK		 5745static int __init beiscsi_module_init(void)
5746{
5747 int ret;
5748
5749 beiscsi_scsi_transport =
5750 iscsi_register_transport(&beiscsi_iscsi_transport);
5751 if (!beiscsi_scsi_transport) {
99bc5d55
JSJ		 5752 printk(KERN_ERR
5753 "beiscsi_module_init - Unable to register beiscsi transport.\n");
f55a24f2 5754 return -ENOMEM;
6733b39a 5755 }
99bc5d55
JSJ		 5756 printk(KERN_INFO "In beiscsi_module_init, tt=%p\n",
5757 &beiscsi_iscsi_transport);
6733b39a
JK		 5758
5759 ret = pci_register_driver(&beiscsi_pci_driver);
5760 if (ret) {
99bc5d55
JSJ		 5761 printk(KERN_ERR
5762 "beiscsi_module_init - Unable to register beiscsi pci driver.\n");
6733b39a
JK		 5763 goto unregister_iscsi_transport;
5764 }
5765 return 0;
5766
5767unregister_iscsi_transport:
5768 iscsi_unregister_transport(&beiscsi_iscsi_transport);
5769 return ret;
5770}
5771
5772static void __exit beiscsi_module_exit(void)
5773{
5774 pci_unregister_driver(&beiscsi_pci_driver);
5775 iscsi_unregister_transport(&beiscsi_iscsi_transport);
5776}
5777
5778module_init(beiscsi_module_init);
5779module_exit(beiscsi_module_exit);
Arne's tree of linux kernel.