1 /*******************************************************************
2 * This file is part of the Emulex Linux Device Driver for *
3 * Fibre Channel Host Bus Adapters. *
4 * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
5 * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
6 * Copyright (C) 2004-2016 Emulex. All rights reserved. *
7 * EMULEX and SLI are trademarks of Emulex. *
9 * Portions Copyright (C) 2004-2005 Christoph Hellwig *
11 * This program is free software; you can redistribute it and/or *
12 * modify it under the terms of version 2 of the GNU General *
13 * Public License as published by the Free Software Foundation. *
14 * This program is distributed in the hope that it will be useful. *
15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19 * TO BE LEGALLY INVALID. See the GNU General Public License for *
20 * more details, a copy of which can be found in the file COPYING *
21 * included with this package. *
22 ********************************************************************/
23 #include <linux/pci.h>
24 #include <linux/slab.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <asm/unaligned.h>
28 #include <linux/crc-t10dif.h>
29 #include <net/checksum.h>
31 #include <scsi/scsi.h>
32 #include <scsi/scsi_device.h>
33 #include <scsi/scsi_eh.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_tcq.h>
36 #include <scsi/scsi_transport_fc.h>
37 #include <scsi/fc/fc_fs.h>
39 #include <linux/nvme.h>
40 #include <linux/nvme-fc-driver.h>
41 #include <linux/nvme-fc.h>
42 #include "lpfc_version.h"
46 #include "lpfc_sli4.h"
48 #include "lpfc_disc.h"
50 #include "lpfc_nvme.h"
51 #include "lpfc_scsi.h"
52 #include "lpfc_logmsg.h"
53 #include "lpfc_crtn.h"
54 #include "lpfc_vport.h"
55 #include "lpfc_debugfs.h"
57 /* NVME initiator-based functions */
59 static struct lpfc_nvme_buf
*
60 lpfc_get_nvme_buf(struct lpfc_hba
*phba
, struct lpfc_nodelist
*ndlp
);
63 lpfc_release_nvme_buf(struct lpfc_hba
*, struct lpfc_nvme_buf
*);
67 * lpfc_nvme_create_queue -
68 * @lpfc_pnvme: Pointer to the driver's nvme instance data
69 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
70 * @handle: An opaque driver handle used in follow-up calls.
72 * Driver registers this routine to preallocate and initialize any
73 * internal data structures to bind the @qidx to its internal IO queues.
74 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
78 * -EINVAL - Unsupported input value.
79 * -ENOMEM - Could not alloc necessary memory
82 lpfc_nvme_create_queue(struct nvme_fc_local_port
*pnvme_lport
,
83 unsigned int qidx
, u16 qsize
,
86 struct lpfc_nvme_lport
*lport
;
87 struct lpfc_vport
*vport
;
88 struct lpfc_nvme_qhandle
*qhandle
;
91 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
93 qhandle
= kzalloc(sizeof(struct lpfc_nvme_qhandle
), GFP_KERNEL
);
97 qhandle
->cpu_id
= smp_processor_id();
100 * NVME qidx == 0 is the admin queue, so both admin queue
101 * and first IO queue will use MSI-X vector and associated
102 * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
105 str
= "IO "; /* IO queue */
106 qhandle
->index
= ((qidx
- 1) %
107 vport
->phba
->cfg_nvme_io_channel
);
109 str
= "ADM"; /* Admin queue */
110 qhandle
->index
= qidx
;
113 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME
,
114 "6073 Binding %s HdwQueue %d (cpu %d) to "
115 "io_channel %d qhandle %p\n", str
,
116 qidx
, qhandle
->cpu_id
, qhandle
->index
, qhandle
);
117 *handle
= (void *)qhandle
;
122 * lpfc_nvme_delete_queue -
123 * @lpfc_pnvme: Pointer to the driver's nvme instance data
124 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
125 * @handle: An opaque driver handle from lpfc_nvme_create_queue
127 * Driver registers this routine to free
128 * any internal data structures to bind the @qidx to its internal
133 * TODO: What are the failure codes.
136 lpfc_nvme_delete_queue(struct nvme_fc_local_port
*pnvme_lport
,
140 struct lpfc_nvme_lport
*lport
;
141 struct lpfc_vport
*vport
;
143 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
144 vport
= lport
->vport
;
146 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
147 "6001 ENTER. lpfc_pnvme %p, qidx x%xi qhandle %p\n",
148 lport
, qidx
, handle
);
153 lpfc_nvme_localport_delete(struct nvme_fc_local_port
*localport
)
155 struct lpfc_nvme_lport
*lport
= localport
->private;
157 /* release any threads waiting for the unreg to complete */
158 complete(&lport
->lport_unreg_done
);
161 /* lpfc_nvme_remoteport_delete
163 * @remoteport: Pointer to an nvme transport remoteport instance.
165 * This is a template downcall. NVME transport calls this function
166 * when it has completed the unregistration of a previously
167 * registered remoteport.
173 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port
*remoteport
)
175 struct lpfc_nvme_rport
*rport
= remoteport
->private;
176 struct lpfc_vport
*vport
;
177 struct lpfc_nodelist
*ndlp
;
187 /* Remove this rport from the lport's list - memory is owned by the
188 * transport. Remove the ndlp reference for the NVME transport before
189 * calling state machine to remove the node, this is devloss = 0
192 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
193 "6146 remoteport delete complete %p\n",
195 list_del(&rport
->list
);
199 /* This call has to execute as long as the rport is valid.
200 * Release any threads waiting for the unreg to complete.
202 complete(&rport
->rport_unreg_done
);
206 lpfc_nvme_cmpl_gen_req(struct lpfc_hba
*phba
, struct lpfc_iocbq
*cmdwqe
,
207 struct lpfc_wcqe_complete
*wcqe
)
209 struct lpfc_vport
*vport
= cmdwqe
->vport
;
211 struct nvmefc_ls_req
*pnvme_lsreq
;
212 struct lpfc_dmabuf
*buf_ptr
;
213 struct lpfc_nodelist
*ndlp
;
215 vport
->phba
->fc4NvmeLsCmpls
++;
217 pnvme_lsreq
= (struct nvmefc_ls_req
*)cmdwqe
->context2
;
218 status
= bf_get(lpfc_wcqe_c_status
, wcqe
) & LPFC_IOCB_STATUS_MASK
;
219 ndlp
= (struct lpfc_nodelist
*)cmdwqe
->context1
;
220 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
221 "6047 nvme cmpl Enter "
222 "Data %p DID %x Xri: %x status %x cmd:%p lsreg:%p "
224 pnvme_lsreq
, ndlp
? ndlp
->nlp_DID
: 0,
225 cmdwqe
->sli4_xritag
, status
,
226 cmdwqe
, pnvme_lsreq
, cmdwqe
->context3
, ndlp
);
228 lpfc_nvmeio_data(phba
, "NVME LS CMPL: xri x%x stat x%x parm x%x\n",
229 cmdwqe
->sli4_xritag
, status
, wcqe
->parameter
);
231 if (cmdwqe
->context3
) {
232 buf_ptr
= (struct lpfc_dmabuf
*)cmdwqe
->context3
;
233 lpfc_mbuf_free(phba
, buf_ptr
->virt
, buf_ptr
->phys
);
235 cmdwqe
->context3
= NULL
;
237 if (pnvme_lsreq
->done
)
238 pnvme_lsreq
->done(pnvme_lsreq
, status
);
240 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
241 "6046 nvme cmpl without done call back? "
242 "Data %p DID %x Xri: %x status %x\n",
243 pnvme_lsreq
, ndlp
? ndlp
->nlp_DID
: 0,
244 cmdwqe
->sli4_xritag
, status
);
247 cmdwqe
->context1
= NULL
;
249 lpfc_sli_release_iocbq(phba
, cmdwqe
);
253 lpfc_nvme_gen_req(struct lpfc_vport
*vport
, struct lpfc_dmabuf
*bmp
,
254 struct lpfc_dmabuf
*inp
,
255 struct nvmefc_ls_req
*pnvme_lsreq
,
256 void (*cmpl
)(struct lpfc_hba
*, struct lpfc_iocbq
*,
257 struct lpfc_wcqe_complete
*),
258 struct lpfc_nodelist
*ndlp
, uint32_t num_entry
,
259 uint32_t tmo
, uint8_t retry
)
261 struct lpfc_hba
*phba
= vport
->phba
;
263 struct lpfc_iocbq
*genwqe
;
264 struct ulp_bde64
*bpl
;
265 struct ulp_bde64 bde
;
266 int i
, rc
, xmit_len
, first_len
;
268 /* Allocate buffer for command WQE */
269 genwqe
= lpfc_sli_get_iocbq(phba
);
274 memset(wqe
, 0, sizeof(union lpfc_wqe
));
276 genwqe
->context3
= (uint8_t *)bmp
;
277 genwqe
->iocb_flag
|= LPFC_IO_NVME_LS
;
279 /* Save for completion so we can release these resources */
280 genwqe
->context1
= lpfc_nlp_get(ndlp
);
281 genwqe
->context2
= (uint8_t *)pnvme_lsreq
;
282 /* Fill in payload, bp points to frame payload */
285 /* FC spec states we need 3 * ratov for CT requests */
286 tmo
= (3 * phba
->fc_ratov
);
288 /* For this command calculate the xmit length of the request bde. */
291 bpl
= (struct ulp_bde64
*)bmp
->virt
;
292 for (i
= 0; i
< num_entry
; i
++) {
293 bde
.tus
.w
= bpl
[i
].tus
.w
;
294 if (bde
.tus
.f
.bdeFlags
!= BUFF_TYPE_BDE_64
)
296 xmit_len
+= bde
.tus
.f
.bdeSize
;
298 first_len
= xmit_len
;
301 genwqe
->rsvd2
= num_entry
;
302 genwqe
->hba_wqidx
= 0;
305 wqe
->generic
.bde
.tus
.f
.bdeFlags
= BUFF_TYPE_BDE_64
;
306 wqe
->generic
.bde
.tus
.f
.bdeSize
= first_len
;
307 wqe
->generic
.bde
.addrLow
= bpl
[0].addrLow
;
308 wqe
->generic
.bde
.addrHigh
= bpl
[0].addrHigh
;
311 wqe
->gen_req
.request_payload_len
= first_len
;
316 bf_set(wqe_dfctl
, &wqe
->gen_req
.wge_ctl
, 0);
317 bf_set(wqe_si
, &wqe
->gen_req
.wge_ctl
, 1);
318 bf_set(wqe_la
, &wqe
->gen_req
.wge_ctl
, 1);
319 bf_set(wqe_rctl
, &wqe
->gen_req
.wge_ctl
, FC_RCTL_ELS4_REQ
);
320 bf_set(wqe_type
, &wqe
->gen_req
.wge_ctl
, FC_TYPE_NVME
);
323 bf_set(wqe_ctxt_tag
, &wqe
->gen_req
.wqe_com
,
324 phba
->sli4_hba
.rpi_ids
[ndlp
->nlp_rpi
]);
325 bf_set(wqe_xri_tag
, &wqe
->gen_req
.wqe_com
, genwqe
->sli4_xritag
);
328 bf_set(wqe_tmo
, &wqe
->gen_req
.wqe_com
, (vport
->phba
->fc_ratov
-1));
329 bf_set(wqe_class
, &wqe
->gen_req
.wqe_com
, CLASS3
);
330 bf_set(wqe_cmnd
, &wqe
->gen_req
.wqe_com
, CMD_GEN_REQUEST64_WQE
);
331 bf_set(wqe_ct
, &wqe
->gen_req
.wqe_com
, SLI4_CT_RPI
);
334 wqe
->gen_req
.wqe_com
.abort_tag
= genwqe
->iotag
;
337 bf_set(wqe_reqtag
, &wqe
->gen_req
.wqe_com
, genwqe
->iotag
);
340 bf_set(wqe_dbde
, &wqe
->gen_req
.wqe_com
, 1);
341 bf_set(wqe_iod
, &wqe
->gen_req
.wqe_com
, LPFC_WQE_IOD_READ
);
342 bf_set(wqe_qosd
, &wqe
->gen_req
.wqe_com
, 1);
343 bf_set(wqe_lenloc
, &wqe
->gen_req
.wqe_com
, LPFC_WQE_LENLOC_NONE
);
344 bf_set(wqe_ebde_cnt
, &wqe
->gen_req
.wqe_com
, 0);
347 bf_set(wqe_cqid
, &wqe
->gen_req
.wqe_com
, LPFC_WQE_CQ_ID_DEFAULT
);
348 bf_set(wqe_cmd_type
, &wqe
->gen_req
.wqe_com
, OTHER_COMMAND
);
351 /* Issue GEN REQ WQE for NPORT <did> */
352 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_ELS
,
353 "6050 Issue GEN REQ WQE to NPORT x%x "
354 "Data: x%x x%x wq:%p lsreq:%p bmp:%p xmit:%d 1st:%d\n",
355 ndlp
->nlp_DID
, genwqe
->iotag
,
357 genwqe
, pnvme_lsreq
, bmp
, xmit_len
, first_len
);
358 genwqe
->wqe_cmpl
= cmpl
;
359 genwqe
->iocb_cmpl
= NULL
;
360 genwqe
->drvrTimeout
= tmo
+ LPFC_DRVR_TIMEOUT
;
361 genwqe
->vport
= vport
;
362 genwqe
->retry
= retry
;
364 lpfc_nvmeio_data(phba
, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n",
365 genwqe
->sli4_xritag
, genwqe
->iotag
, ndlp
->nlp_DID
);
367 rc
= lpfc_sli4_issue_wqe(phba
, LPFC_ELS_RING
, genwqe
);
368 if (rc
== WQE_ERROR
) {
369 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_ELS
,
370 "6045 Issue GEN REQ WQE to NPORT x%x "
372 ndlp
->nlp_DID
, genwqe
->iotag
,
374 lpfc_sli_release_iocbq(phba
, genwqe
);
381 * lpfc_nvme_ls_req - Issue an Link Service request
382 * @lpfc_pnvme: Pointer to the driver's nvme instance data
383 * @lpfc_nvme_lport: Pointer to the driver's local port data
384 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
386 * Driver registers this routine to handle any link service request
387 * from the nvme_fc transport to a remote nvme-aware port.
391 * TODO: What are the failure codes.
394 lpfc_nvme_ls_req(struct nvme_fc_local_port
*pnvme_lport
,
395 struct nvme_fc_remote_port
*pnvme_rport
,
396 struct nvmefc_ls_req
*pnvme_lsreq
)
399 struct lpfc_nvme_lport
*lport
;
400 struct lpfc_vport
*vport
;
401 struct lpfc_nodelist
*ndlp
;
402 struct ulp_bde64
*bpl
;
403 struct lpfc_dmabuf
*bmp
;
405 /* there are two dma buf in the request, actually there is one and
406 * the second one is just the start address + cmd size.
407 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
408 * in a lpfc_dmabuf struct. When freeing we just free the wrapper
409 * because the nvem layer owns the data bufs.
410 * We do not have to break these packets open, we don't care what is in
411 * them. And we do not have to look at the resonse data, we only care
412 * that we got a response. All of the caring is going to happen in the
416 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
417 vport
= lport
->vport
;
419 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
421 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
422 "6043 Could not find node for DID %x\n",
423 pnvme_rport
->port_id
);
426 bmp
= kmalloc(sizeof(struct lpfc_dmabuf
), GFP_KERNEL
);
429 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
430 "6044 Could not find node for DID %x\n",
431 pnvme_rport
->port_id
);
434 INIT_LIST_HEAD(&bmp
->list
);
435 bmp
->virt
= lpfc_mbuf_alloc(vport
->phba
, MEM_PRI
, &(bmp
->phys
));
437 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
438 "6042 Could not find node for DID %x\n",
439 pnvme_rport
->port_id
);
443 bpl
= (struct ulp_bde64
*)bmp
->virt
;
444 bpl
->addrHigh
= le32_to_cpu(putPaddrHigh(pnvme_lsreq
->rqstdma
));
445 bpl
->addrLow
= le32_to_cpu(putPaddrLow(pnvme_lsreq
->rqstdma
));
446 bpl
->tus
.f
.bdeFlags
= 0;
447 bpl
->tus
.f
.bdeSize
= pnvme_lsreq
->rqstlen
;
448 bpl
->tus
.w
= le32_to_cpu(bpl
->tus
.w
);
451 bpl
->addrHigh
= le32_to_cpu(putPaddrHigh(pnvme_lsreq
->rspdma
));
452 bpl
->addrLow
= le32_to_cpu(putPaddrLow(pnvme_lsreq
->rspdma
));
453 bpl
->tus
.f
.bdeFlags
= BUFF_TYPE_BDE_64I
;
454 bpl
->tus
.f
.bdeSize
= pnvme_lsreq
->rsplen
;
455 bpl
->tus
.w
= le32_to_cpu(bpl
->tus
.w
);
457 /* Expand print to include key fields. */
458 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
459 "6051 ENTER. lport %p, rport %p lsreq%p rqstlen:%d "
460 "rsplen:%d %pad %pad\n",
461 pnvme_lport
, pnvme_rport
,
462 pnvme_lsreq
, pnvme_lsreq
->rqstlen
,
463 pnvme_lsreq
->rsplen
, &pnvme_lsreq
->rqstdma
,
464 &pnvme_lsreq
->rspdma
);
466 vport
->phba
->fc4NvmeLsRequests
++;
468 /* Hardcode the wait to 30 seconds. Connections are failing otherwise.
469 * This code allows it all to work.
471 ret
= lpfc_nvme_gen_req(vport
, bmp
, pnvme_lsreq
->rqstaddr
,
472 pnvme_lsreq
, lpfc_nvme_cmpl_gen_req
,
474 if (ret
!= WQE_SUCCESS
) {
475 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
476 "6052 EXIT. issue ls wqe failed lport %p, "
477 "rport %p lsreq%p Status %x DID %x\n",
478 pnvme_lport
, pnvme_rport
, pnvme_lsreq
,
480 lpfc_mbuf_free(vport
->phba
, bmp
->virt
, bmp
->phys
);
485 /* Stub in routine and return 0 for now. */
490 * lpfc_nvme_ls_abort - Issue an Link Service request
491 * @lpfc_pnvme: Pointer to the driver's nvme instance data
492 * @lpfc_nvme_lport: Pointer to the driver's local port data
493 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
495 * Driver registers this routine to handle any link service request
496 * from the nvme_fc transport to a remote nvme-aware port.
500 * TODO: What are the failure codes.
503 lpfc_nvme_ls_abort(struct nvme_fc_local_port
*pnvme_lport
,
504 struct nvme_fc_remote_port
*pnvme_rport
,
505 struct nvmefc_ls_req
*pnvme_lsreq
)
507 struct lpfc_nvme_lport
*lport
;
508 struct lpfc_vport
*vport
;
509 struct lpfc_hba
*phba
;
510 struct lpfc_nodelist
*ndlp
;
511 LIST_HEAD(abort_list
);
512 struct lpfc_sli_ring
*pring
;
513 struct lpfc_iocbq
*wqe
, *next_wqe
;
515 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
516 vport
= lport
->vport
;
519 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
521 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
522 "6049 Could not find node for DID %x\n",
523 pnvme_rport
->port_id
);
527 /* Expand print to include key fields. */
528 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_ABTS
,
529 "6040 ENTER. lport %p, rport %p lsreq %p rqstlen:%d "
530 "rsplen:%d %pad %pad\n",
531 pnvme_lport
, pnvme_rport
,
532 pnvme_lsreq
, pnvme_lsreq
->rqstlen
,
533 pnvme_lsreq
->rsplen
, &pnvme_lsreq
->rqstdma
,
534 &pnvme_lsreq
->rspdma
);
537 * Lock the ELS ring txcmplq and build a local list of all ELS IOs
538 * that need an ABTS. The IOs need to stay on the txcmplq so that
539 * the abort operation completes them successfully.
541 pring
= phba
->sli4_hba
.nvmels_wq
->pring
;
542 spin_lock_irq(&phba
->hbalock
);
543 spin_lock(&pring
->ring_lock
);
544 list_for_each_entry_safe(wqe
, next_wqe
, &pring
->txcmplq
, list
) {
545 /* Add to abort_list on on NDLP match. */
546 if (lpfc_check_sli_ndlp(phba
, pring
, wqe
, ndlp
)) {
547 wqe
->iocb_flag
|= LPFC_DRIVER_ABORTED
;
548 list_add_tail(&wqe
->dlist
, &abort_list
);
551 spin_unlock(&pring
->ring_lock
);
552 spin_unlock_irq(&phba
->hbalock
);
554 /* Abort the targeted IOs and remove them from the abort list. */
555 list_for_each_entry_safe(wqe
, next_wqe
, &abort_list
, dlist
) {
556 spin_lock_irq(&phba
->hbalock
);
557 list_del_init(&wqe
->dlist
);
558 lpfc_sli_issue_abort_iotag(phba
, pring
, wqe
);
559 spin_unlock_irq(&phba
->hbalock
);
563 /* Fix up the existing sgls for NVME IO. */
565 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport
*vport
,
566 struct lpfc_nvme_buf
*lpfc_ncmd
,
567 struct nvmefc_fcp_req
*nCmd
)
569 struct sli4_sge
*sgl
;
570 union lpfc_wqe128
*wqe
;
571 uint32_t *wptr
, *dptr
;
574 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
575 * match NVME. NVME sends 96 bytes. Also, use the
576 * nvme commands command and response dma addresses
577 * rather than the virtual memory to ease the restore
580 sgl
= lpfc_ncmd
->nvme_sgl
;
581 sgl
->sge_len
= cpu_to_le32(nCmd
->cmdlen
);
585 /* Setup the physical region for the FCP RSP */
586 sgl
->addr_hi
= cpu_to_le32(putPaddrHigh(nCmd
->rspdma
));
587 sgl
->addr_lo
= cpu_to_le32(putPaddrLow(nCmd
->rspdma
));
588 sgl
->word2
= le32_to_cpu(sgl
->word2
);
590 bf_set(lpfc_sli4_sge_last
, sgl
, 0);
592 bf_set(lpfc_sli4_sge_last
, sgl
, 1);
593 sgl
->word2
= cpu_to_le32(sgl
->word2
);
594 sgl
->sge_len
= cpu_to_le32(nCmd
->rsplen
);
597 * Get a local pointer to the built-in wqe and correct
598 * the cmd size to match NVME's 96 bytes and fix
602 /* 128 byte wqe support here */
603 wqe
= (union lpfc_wqe128
*)&lpfc_ncmd
->cur_iocbq
.wqe
;
605 /* Word 0-2 - NVME CMND IU (embedded payload) */
606 wqe
->generic
.bde
.tus
.f
.bdeFlags
= BUFF_TYPE_BDE_IMMED
;
607 wqe
->generic
.bde
.tus
.f
.bdeSize
= 60;
608 wqe
->generic
.bde
.addrHigh
= 0;
609 wqe
->generic
.bde
.addrLow
= 64; /* Word 16 */
612 bf_set(payload_offset_len
, &wqe
->fcp_icmd
,
613 (nCmd
->rsplen
+ nCmd
->cmdlen
));
616 bf_set(wqe_nvme
, &wqe
->fcp_icmd
.wqe_com
, 1);
617 bf_set(wqe_wqes
, &wqe
->fcp_icmd
.wqe_com
, 1);
620 * Embed the payload in the last half of the WQE
621 * WQE words 16-30 get the NVME CMD IU payload
623 * WQE words 16-19 get payload Words 1-4
624 * WQE words 20-21 get payload Words 6-7
625 * WQE words 22-29 get payload Words 16-23
627 wptr
= &wqe
->words
[16]; /* WQE ptr */
628 dptr
= (uint32_t *)nCmd
->cmdaddr
; /* payload ptr */
629 dptr
++; /* Skip Word 0 in payload */
631 *wptr
++ = *dptr
++; /* Word 1 */
632 *wptr
++ = *dptr
++; /* Word 2 */
633 *wptr
++ = *dptr
++; /* Word 3 */
634 *wptr
++ = *dptr
++; /* Word 4 */
635 dptr
++; /* Skip Word 5 in payload */
636 *wptr
++ = *dptr
++; /* Word 6 */
637 *wptr
++ = *dptr
++; /* Word 7 */
638 dptr
+= 8; /* Skip Words 8-15 in payload */
639 *wptr
++ = *dptr
++; /* Word 16 */
640 *wptr
++ = *dptr
++; /* Word 17 */
641 *wptr
++ = *dptr
++; /* Word 18 */
642 *wptr
++ = *dptr
++; /* Word 19 */
643 *wptr
++ = *dptr
++; /* Word 20 */
644 *wptr
++ = *dptr
++; /* Word 21 */
645 *wptr
++ = *dptr
++; /* Word 22 */
646 *wptr
= *dptr
; /* Word 23 */
649 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
651 lpfc_nvme_ktime(struct lpfc_hba
*phba
,
652 struct lpfc_nvme_buf
*lpfc_ncmd
)
654 uint64_t seg1
, seg2
, seg3
, seg4
;
658 if (!lpfc_ncmd
->ts_last_cmd
||
659 !lpfc_ncmd
->ts_cmd_start
||
660 !lpfc_ncmd
->ts_cmd_wqput
||
661 !lpfc_ncmd
->ts_isr_cmpl
||
662 !lpfc_ncmd
->ts_data_nvme
)
664 if (lpfc_ncmd
->ts_cmd_start
< lpfc_ncmd
->ts_last_cmd
)
666 if (lpfc_ncmd
->ts_cmd_wqput
< lpfc_ncmd
->ts_cmd_start
)
668 if (lpfc_ncmd
->ts_isr_cmpl
< lpfc_ncmd
->ts_cmd_wqput
)
670 if (lpfc_ncmd
->ts_data_nvme
< lpfc_ncmd
->ts_isr_cmpl
)
673 * Segment 1 - Time from Last FCP command cmpl is handed
674 * off to NVME Layer to start of next command.
675 * Segment 2 - Time from Driver receives a IO cmd start
676 * from NVME Layer to WQ put is done on IO cmd.
677 * Segment 3 - Time from Driver WQ put is done on IO cmd
678 * to MSI-X ISR for IO cmpl.
679 * Segment 4 - Time from MSI-X ISR for IO cmpl to when
680 * cmpl is handled off to the NVME Layer.
682 seg1
= lpfc_ncmd
->ts_cmd_start
- lpfc_ncmd
->ts_last_cmd
;
683 if (seg1
> 5000000) /* 5 ms - for sequential IOs */
686 /* Calculate times relative to start of IO */
687 seg2
= (lpfc_ncmd
->ts_cmd_wqput
- lpfc_ncmd
->ts_cmd_start
);
688 seg3
= (lpfc_ncmd
->ts_isr_cmpl
-
689 lpfc_ncmd
->ts_cmd_start
) - seg2
;
690 seg4
= (lpfc_ncmd
->ts_data_nvme
-
691 lpfc_ncmd
->ts_cmd_start
) - seg2
- seg3
;
692 phba
->ktime_data_samples
++;
693 phba
->ktime_seg1_total
+= seg1
;
694 if (seg1
< phba
->ktime_seg1_min
)
695 phba
->ktime_seg1_min
= seg1
;
696 else if (seg1
> phba
->ktime_seg1_max
)
697 phba
->ktime_seg1_max
= seg1
;
698 phba
->ktime_seg2_total
+= seg2
;
699 if (seg2
< phba
->ktime_seg2_min
)
700 phba
->ktime_seg2_min
= seg2
;
701 else if (seg2
> phba
->ktime_seg2_max
)
702 phba
->ktime_seg2_max
= seg2
;
703 phba
->ktime_seg3_total
+= seg3
;
704 if (seg3
< phba
->ktime_seg3_min
)
705 phba
->ktime_seg3_min
= seg3
;
706 else if (seg3
> phba
->ktime_seg3_max
)
707 phba
->ktime_seg3_max
= seg3
;
708 phba
->ktime_seg4_total
+= seg4
;
709 if (seg4
< phba
->ktime_seg4_min
)
710 phba
->ktime_seg4_min
= seg4
;
711 else if (seg4
> phba
->ktime_seg4_max
)
712 phba
->ktime_seg4_max
= seg4
;
714 lpfc_ncmd
->ts_last_cmd
= 0;
715 lpfc_ncmd
->ts_cmd_start
= 0;
716 lpfc_ncmd
->ts_cmd_wqput
= 0;
717 lpfc_ncmd
->ts_isr_cmpl
= 0;
718 lpfc_ncmd
->ts_data_nvme
= 0;
723 * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO
724 * @lpfc_pnvme: Pointer to the driver's nvme instance data
725 * @lpfc_nvme_lport: Pointer to the driver's local port data
726 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
728 * Driver registers this routine as it io request handler. This
729 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
730 * data structure to the rport indicated in @lpfc_nvme_rport.
734 * TODO: What are the failure codes.
737 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba
*phba
, struct lpfc_iocbq
*pwqeIn
,
738 struct lpfc_wcqe_complete
*wcqe
)
740 struct lpfc_nvme_buf
*lpfc_ncmd
=
741 (struct lpfc_nvme_buf
*)pwqeIn
->context1
;
742 struct lpfc_vport
*vport
= pwqeIn
->vport
;
743 struct nvmefc_fcp_req
*nCmd
;
744 struct nvme_fc_ersp_iu
*ep
;
745 struct nvme_fc_cmd_iu
*cp
;
746 struct lpfc_nvme_rport
*rport
;
747 struct lpfc_nodelist
*ndlp
;
750 uint16_t cid
, sqhd
, data
;
753 /* Sanity check on return of outstanding command */
754 if (!lpfc_ncmd
|| !lpfc_ncmd
->nvmeCmd
|| !lpfc_ncmd
->nrport
) {
755 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
756 "6071 Completion pointers bad on wqe %p.\n",
760 phba
->fc4NvmeIoCmpls
++;
762 nCmd
= lpfc_ncmd
->nvmeCmd
;
763 rport
= lpfc_ncmd
->nrport
;
765 lpfc_nvmeio_data(phba
, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
766 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
767 bf_get(lpfc_wcqe_c_status
, wcqe
), wcqe
->parameter
);
769 * Catch race where our node has transitioned, but the
770 * transport is still transitioning.
773 if (!ndlp
|| !NLP_CHK_NODE_ACT(ndlp
)) {
774 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
775 "6061 rport %p, ndlp %p, DID x%06x ndlp "
777 rport
, ndlp
, rport
->remoteport
->port_id
);
779 ndlp
= lpfc_findnode_did(vport
, rport
->remoteport
->port_id
);
781 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
782 "6062 Ignoring NVME cmpl. No ndlp\n");
787 code
= bf_get(lpfc_wcqe_c_code
, wcqe
);
788 if (code
== CQE_CODE_NVME_ERSP
) {
789 /* For this type of CQE, we need to rebuild the rsp */
790 ep
= (struct nvme_fc_ersp_iu
*)nCmd
->rspaddr
;
793 * Get Command Id from cmd to plug into response. This
794 * code is not needed in the next NVME Transport drop.
796 cp
= (struct nvme_fc_cmd_iu
*)nCmd
->cmdaddr
;
797 cid
= cp
->sqe
.common
.command_id
;
800 * RSN is in CQE word 2
801 * SQHD is in CQE Word 3 bits 15:0
802 * Cmd Specific info is in CQE Word 1
803 * and in CQE Word 0 bits 15:0
805 sqhd
= bf_get(lpfc_wcqe_c_sqhead
, wcqe
);
807 /* Now lets build the NVME ERSP IU */
808 ep
->iu_len
= cpu_to_be16(8);
809 ep
->rsn
= wcqe
->parameter
;
810 ep
->xfrd_len
= cpu_to_be32(nCmd
->payload_length
);
812 ptr
= (uint32_t *)&ep
->cqe
.result
.u64
;
813 *ptr
++ = wcqe
->total_data_placed
;
814 data
= bf_get(lpfc_wcqe_c_ersp0
, wcqe
);
815 *ptr
= (uint32_t)data
;
816 ep
->cqe
.sq_head
= sqhd
;
817 ep
->cqe
.sq_id
= nCmd
->sqid
;
818 ep
->cqe
.command_id
= cid
;
821 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
822 lpfc_ncmd
->result
= 0;
823 nCmd
->rcv_rsplen
= LPFC_NVME_ERSP_LEN
;
824 nCmd
->transferred_length
= nCmd
->payload_length
;
826 lpfc_ncmd
->status
= (bf_get(lpfc_wcqe_c_status
, wcqe
) &
827 LPFC_IOCB_STATUS_MASK
);
828 lpfc_ncmd
->result
= wcqe
->parameter
;
830 /* For NVME, the only failure path that results in an
831 * IO error is when the adapter rejects it. All other
832 * conditions are a success case and resolved by the
834 * IOSTAT_FCP_RSP_ERROR means:
835 * 1. Length of data received doesn't match total
836 * transfer length in WQE
837 * 2. If the RSP payload does NOT match these cases:
838 * a. RSP length 12/24 bytes and all zeros
841 switch (lpfc_ncmd
->status
) {
843 nCmd
->transferred_length
= wcqe
->total_data_placed
;
844 nCmd
->rcv_rsplen
= 0;
847 case IOSTAT_FCP_RSP_ERROR
:
848 nCmd
->transferred_length
= wcqe
->total_data_placed
;
849 nCmd
->rcv_rsplen
= wcqe
->parameter
;
852 if (nCmd
->rcv_rsplen
== LPFC_NVME_ERSP_LEN
)
854 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
855 "6081 NVME Completion Protocol Error: "
856 "status x%x result x%x placed x%x\n",
857 lpfc_ncmd
->status
, lpfc_ncmd
->result
,
858 wcqe
->total_data_placed
);
862 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
863 "6072 NVME Completion Error: "
864 "status x%x result x%x placed x%x\n",
865 lpfc_ncmd
->status
, lpfc_ncmd
->result
,
866 wcqe
->total_data_placed
);
867 nCmd
->transferred_length
= 0;
868 nCmd
->rcv_rsplen
= 0;
869 nCmd
->status
= NVME_SC_FC_TRANSPORT_ERROR
;
873 /* pick up SLI4 exhange busy condition */
874 if (bf_get(lpfc_wcqe_c_xb
, wcqe
))
875 lpfc_ncmd
->flags
|= LPFC_SBUF_XBUSY
;
877 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
879 if (ndlp
&& NLP_CHK_NODE_ACT(ndlp
))
880 atomic_dec(&ndlp
->cmd_pending
);
882 /* Update stats and complete the IO. There is
883 * no need for dma unprep because the nvme_transport
884 * owns the dma address.
886 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
887 if (phba
->ktime_on
) {
888 lpfc_ncmd
->ts_isr_cmpl
= pwqeIn
->isr_timestamp
;
889 lpfc_ncmd
->ts_data_nvme
= ktime_get_ns();
890 phba
->ktime_last_cmd
= lpfc_ncmd
->ts_data_nvme
;
891 lpfc_nvme_ktime(phba
, lpfc_ncmd
);
893 if (phba
->cpucheck_on
& LPFC_CHECK_NVME_IO
) {
894 if (lpfc_ncmd
->cpu
!= smp_processor_id())
895 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
896 "6701 CPU Check cmpl: "
897 "cpu %d expect %d\n",
898 smp_processor_id(), lpfc_ncmd
->cpu
);
899 if (lpfc_ncmd
->cpu
< LPFC_CHECK_CPU_CNT
)
900 phba
->cpucheck_cmpl_io
[lpfc_ncmd
->cpu
]++;
905 spin_lock_irqsave(&phba
->hbalock
, flags
);
906 lpfc_ncmd
->nrport
= NULL
;
907 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
909 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
914 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
915 * @lpfc_pnvme: Pointer to the driver's nvme instance data
916 * @lpfc_nvme_lport: Pointer to the driver's local port data
917 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
918 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
919 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
921 * Driver registers this routine as it io request handler. This
922 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
923 * data structure to the rport indicated in @lpfc_nvme_rport.
927 * TODO: What are the failure codes.
930 lpfc_nvme_prep_io_cmd(struct lpfc_vport
*vport
,
931 struct lpfc_nvme_buf
*lpfc_ncmd
,
932 struct lpfc_nodelist
*pnode
)
934 struct lpfc_hba
*phba
= vport
->phba
;
935 struct nvmefc_fcp_req
*nCmd
= lpfc_ncmd
->nvmeCmd
;
936 struct lpfc_iocbq
*pwqeq
= &(lpfc_ncmd
->cur_iocbq
);
937 union lpfc_wqe128
*wqe
= (union lpfc_wqe128
*)&pwqeq
->wqe
;
940 if (!pnode
|| !NLP_CHK_NODE_ACT(pnode
))
944 * There are three possibilities here - use scatter-gather segment, use
945 * the single mapping, or neither.
947 wqe
->fcp_iwrite
.initial_xfer_len
= 0;
949 if (nCmd
->io_dir
== NVMEFC_FCP_WRITE
) {
951 if ((phba
->cfg_nvme_enable_fb
) &&
952 (pnode
->nlp_flag
& NLP_FIRSTBURST
)) {
953 req_len
= lpfc_ncmd
->nvmeCmd
->payload_length
;
954 if (req_len
< pnode
->nvme_fb_size
)
955 wqe
->fcp_iwrite
.initial_xfer_len
=
958 wqe
->fcp_iwrite
.initial_xfer_len
=
963 bf_set(wqe_cmnd
, &wqe
->generic
.wqe_com
,
964 CMD_FCP_IWRITE64_WQE
);
965 bf_set(wqe_pu
, &wqe
->generic
.wqe_com
,
969 bf_set(wqe_qosd
, &wqe
->fcp_iwrite
.wqe_com
, 0);
970 bf_set(wqe_iod
, &wqe
->fcp_iwrite
.wqe_com
,
972 bf_set(wqe_lenloc
, &wqe
->fcp_iwrite
.wqe_com
,
973 LPFC_WQE_LENLOC_WORD4
);
974 if (phba
->cfg_nvme_oas
)
975 bf_set(wqe_oas
, &wqe
->fcp_iwrite
.wqe_com
, 1);
978 bf_set(wqe_cmd_type
, &wqe
->generic
.wqe_com
,
981 phba
->fc4NvmeOutputRequests
++;
984 bf_set(wqe_cmnd
, &wqe
->generic
.wqe_com
,
985 CMD_FCP_IREAD64_WQE
);
986 bf_set(wqe_pu
, &wqe
->generic
.wqe_com
,
990 bf_set(wqe_qosd
, &wqe
->fcp_iread
.wqe_com
, 0);
991 bf_set(wqe_iod
, &wqe
->fcp_iread
.wqe_com
,
993 bf_set(wqe_lenloc
, &wqe
->fcp_iread
.wqe_com
,
994 LPFC_WQE_LENLOC_WORD4
);
995 if (phba
->cfg_nvme_oas
)
996 bf_set(wqe_oas
, &wqe
->fcp_iread
.wqe_com
, 1);
999 bf_set(wqe_cmd_type
, &wqe
->generic
.wqe_com
,
1002 phba
->fc4NvmeInputRequests
++;
1006 wqe
->fcp_icmd
.rsrvd4
= 0;
1009 bf_set(wqe_cmnd
, &wqe
->generic
.wqe_com
, CMD_FCP_ICMND64_WQE
);
1010 bf_set(wqe_pu
, &wqe
->generic
.wqe_com
, 0);
1013 bf_set(wqe_qosd
, &wqe
->fcp_icmd
.wqe_com
, 1);
1014 bf_set(wqe_iod
, &wqe
->fcp_icmd
.wqe_com
, LPFC_WQE_IOD_WRITE
);
1015 bf_set(wqe_lenloc
, &wqe
->fcp_icmd
.wqe_com
,
1016 LPFC_WQE_LENLOC_NONE
);
1017 if (phba
->cfg_nvme_oas
)
1018 bf_set(wqe_oas
, &wqe
->fcp_icmd
.wqe_com
, 1);
1021 bf_set(wqe_cmd_type
, &wqe
->generic
.wqe_com
, NVME_READ_CMD
);
1023 phba
->fc4NvmeControlRequests
++;
1026 * Finish initializing those WQE fields that are independent
1027 * of the nvme_cmnd request_buffer
1031 bf_set(wqe_ctxt_tag
, &wqe
->generic
.wqe_com
,
1032 phba
->sli4_hba
.rpi_ids
[pnode
->nlp_rpi
]);
1033 bf_set(wqe_xri_tag
, &wqe
->generic
.wqe_com
, pwqeq
->sli4_xritag
);
1036 /* Preserve Class data in the ndlp. */
1037 bf_set(wqe_class
, &wqe
->generic
.wqe_com
,
1038 (pnode
->nlp_fcp_info
& 0x0f));
1041 wqe
->generic
.wqe_com
.abort_tag
= pwqeq
->iotag
;
1044 bf_set(wqe_reqtag
, &wqe
->generic
.wqe_com
, pwqeq
->iotag
);
1047 bf_set(wqe_cqid
, &wqe
->generic
.wqe_com
, LPFC_WQE_CQ_ID_DEFAULT
);
1049 pwqeq
->vport
= vport
;
1055 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
1056 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1057 * @lpfc_nvme_lport: Pointer to the driver's local port data
1058 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1059 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1060 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1062 * Driver registers this routine as it io request handler. This
1063 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1064 * data structure to the rport indicated in @lpfc_nvme_rport.
1068 * TODO: What are the failure codes.
1071 lpfc_nvme_prep_io_dma(struct lpfc_vport
*vport
,
1072 struct lpfc_nvme_buf
*lpfc_ncmd
)
1074 struct lpfc_hba
*phba
= vport
->phba
;
1075 struct nvmefc_fcp_req
*nCmd
= lpfc_ncmd
->nvmeCmd
;
1076 union lpfc_wqe128
*wqe
= (union lpfc_wqe128
*)&lpfc_ncmd
->cur_iocbq
.wqe
;
1077 struct sli4_sge
*sgl
= lpfc_ncmd
->nvme_sgl
;
1078 struct scatterlist
*data_sg
;
1079 struct sli4_sge
*first_data_sgl
;
1080 dma_addr_t physaddr
;
1081 uint32_t num_bde
= 0;
1083 uint32_t dma_offset
= 0;
1086 /* Fix up the command and response DMA stuff. */
1087 lpfc_nvme_adj_fcp_sgls(vport
, lpfc_ncmd
, nCmd
);
1090 * There are three possibilities here - use scatter-gather segment, use
1091 * the single mapping, or neither.
1095 * Jump over the cmd and rsp SGEs. The fix routine
1096 * has already adjusted for this.
1100 first_data_sgl
= sgl
;
1101 lpfc_ncmd
->seg_cnt
= nCmd
->sg_cnt
;
1102 if (lpfc_ncmd
->seg_cnt
> phba
->cfg_sg_seg_cnt
) {
1103 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1104 "6058 Too many sg segments from "
1105 "NVME Transport. Max %d, "
1106 "nvmeIO sg_cnt %d\n",
1107 phba
->cfg_sg_seg_cnt
,
1108 lpfc_ncmd
->seg_cnt
);
1109 lpfc_ncmd
->seg_cnt
= 0;
1114 * The driver established a maximum scatter-gather segment count
1115 * during probe that limits the number of sg elements in any
1116 * single nvme command. Just run through the seg_cnt and format
1119 nseg
= nCmd
->sg_cnt
;
1120 data_sg
= nCmd
->first_sgl
;
1121 for (i
= 0; i
< nseg
; i
++) {
1122 if (data_sg
== NULL
) {
1123 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1124 "6059 dptr err %d, nseg %d\n",
1126 lpfc_ncmd
->seg_cnt
= 0;
1129 physaddr
= data_sg
->dma_address
;
1130 dma_len
= data_sg
->length
;
1131 sgl
->addr_lo
= cpu_to_le32(putPaddrLow(physaddr
));
1132 sgl
->addr_hi
= cpu_to_le32(putPaddrHigh(physaddr
));
1133 sgl
->word2
= le32_to_cpu(sgl
->word2
);
1134 if ((num_bde
+ 1) == nseg
)
1135 bf_set(lpfc_sli4_sge_last
, sgl
, 1);
1137 bf_set(lpfc_sli4_sge_last
, sgl
, 0);
1138 bf_set(lpfc_sli4_sge_offset
, sgl
, dma_offset
);
1139 bf_set(lpfc_sli4_sge_type
, sgl
, LPFC_SGE_TYPE_DATA
);
1140 sgl
->word2
= cpu_to_le32(sgl
->word2
);
1141 sgl
->sge_len
= cpu_to_le32(dma_len
);
1143 dma_offset
+= dma_len
;
1144 data_sg
= sg_next(data_sg
);
1148 /* For this clause to be valid, the payload_length
1149 * and sg_cnt must zero.
1151 if (nCmd
->payload_length
!= 0) {
1152 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1153 "6063 NVME DMA Prep Err: sg_cnt %d "
1154 "payload_length x%x\n",
1155 nCmd
->sg_cnt
, nCmd
->payload_length
);
1161 * Due to difference in data length between DIF/non-DIF paths,
1162 * we need to set word 4 of WQE here
1164 wqe
->fcp_iread
.total_xfer_len
= nCmd
->payload_length
;
1169 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
1170 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1171 * @lpfc_nvme_lport: Pointer to the driver's local port data
1172 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1173 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1174 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1176 * Driver registers this routine as it io request handler. This
1177 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1178 * data structure to the rport
1179 indicated in @lpfc_nvme_rport.
1183 * TODO: What are the failure codes.
1186 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port
*pnvme_lport
,
1187 struct nvme_fc_remote_port
*pnvme_rport
,
1188 void *hw_queue_handle
,
1189 struct nvmefc_fcp_req
*pnvme_fcreq
)
1192 struct lpfc_nvme_lport
*lport
;
1193 struct lpfc_vport
*vport
;
1194 struct lpfc_hba
*phba
;
1195 struct lpfc_nodelist
*ndlp
;
1196 struct lpfc_nvme_buf
*lpfc_ncmd
;
1197 struct lpfc_nvme_rport
*rport
;
1198 struct lpfc_nvme_qhandle
*lpfc_queue_info
;
1199 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1203 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
1204 vport
= lport
->vport
;
1207 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1209 start
= ktime_get_ns();
1211 rport
= (struct lpfc_nvme_rport
*)pnvme_rport
->private;
1212 lpfc_queue_info
= (struct lpfc_nvme_qhandle
*)hw_queue_handle
;
1215 * Catch race where our node has transitioned, but the
1216 * transport is still transitioning.
1219 if (!ndlp
|| !NLP_CHK_NODE_ACT(ndlp
)) {
1220 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
1221 "6053 rport %p, ndlp %p, DID x%06x "
1222 "ndlp not ready.\n",
1223 rport
, ndlp
, pnvme_rport
->port_id
);
1225 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
1227 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
1228 "6066 Missing node for DID %x\n",
1229 pnvme_rport
->port_id
);
1235 /* The remote node has to be a mapped target or it's an error. */
1236 if ((ndlp
->nlp_type
& NLP_NVME_TARGET
) &&
1237 (ndlp
->nlp_state
!= NLP_STE_MAPPED_NODE
)) {
1238 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
1239 "6036 rport %p, DID x%06x not ready for "
1240 "IO. State x%x, Type x%x\n",
1241 rport
, pnvme_rport
->port_id
,
1242 ndlp
->nlp_state
, ndlp
->nlp_type
);
1248 /* The node is shared with FCP IO, make sure the IO pending count does
1249 * not exceed the programmed depth.
1251 if (atomic_read(&ndlp
->cmd_pending
) >= ndlp
->cmd_qdepth
) {
1256 lpfc_ncmd
= lpfc_get_nvme_buf(phba
, ndlp
);
1257 if (lpfc_ncmd
== NULL
) {
1258 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_IOERR
,
1259 "6065 driver's buffer pool is empty, "
1264 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1265 if (phba
->ktime_on
) {
1266 lpfc_ncmd
->ts_cmd_start
= start
;
1267 lpfc_ncmd
->ts_last_cmd
= phba
->ktime_last_cmd
;
1272 * Store the data needed by the driver to issue, abort, and complete
1274 * Do not let the IO hang out forever. There is no midlayer issuing
1275 * an abort so inform the FW of the maximum IO pending time.
1277 pnvme_fcreq
->private = (void *)lpfc_ncmd
;
1278 lpfc_ncmd
->nvmeCmd
= pnvme_fcreq
;
1279 lpfc_ncmd
->nrport
= rport
;
1280 lpfc_ncmd
->ndlp
= ndlp
;
1281 lpfc_ncmd
->start_time
= jiffies
;
1283 lpfc_nvme_prep_io_cmd(vport
, lpfc_ncmd
, ndlp
);
1284 ret
= lpfc_nvme_prep_io_dma(vport
, lpfc_ncmd
);
1287 goto out_free_nvme_buf
;
1290 atomic_inc(&ndlp
->cmd_pending
);
1293 * Issue the IO on the WQ indicated by index in the hw_queue_handle.
1294 * This identfier was create in our hardware queue create callback
1295 * routine. The driver now is dependent on the IO queue steering from
1296 * the transport. We are trusting the upper NVME layers know which
1297 * index to use and that they have affinitized a CPU to this hardware
1298 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
1300 lpfc_ncmd
->cur_iocbq
.hba_wqidx
= lpfc_queue_info
->index
;
1302 lpfc_nvmeio_data(phba
, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1303 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
1304 lpfc_queue_info
->index
, ndlp
->nlp_DID
);
1306 ret
= lpfc_sli4_issue_wqe(phba
, LPFC_FCP_RING
, &lpfc_ncmd
->cur_iocbq
);
1308 atomic_dec(&ndlp
->cmd_pending
);
1309 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
1310 "6113 FCP could not issue WQE err %x "
1311 "sid: x%x did: x%x oxid: x%x\n",
1312 ret
, vport
->fc_myDID
, ndlp
->nlp_DID
,
1313 lpfc_ncmd
->cur_iocbq
.sli4_xritag
);
1315 goto out_free_nvme_buf
;
1318 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1320 lpfc_ncmd
->ts_cmd_wqput
= ktime_get_ns();
1322 if (phba
->cpucheck_on
& LPFC_CHECK_NVME_IO
) {
1323 lpfc_ncmd
->cpu
= smp_processor_id();
1324 if (lpfc_ncmd
->cpu
!= lpfc_queue_info
->index
) {
1325 /* Check for admin queue */
1326 if (lpfc_queue_info
->qidx
) {
1327 lpfc_printf_vlog(vport
,
1328 KERN_ERR
, LOG_NVME_IOERR
,
1329 "6702 CPU Check cmd: "
1332 lpfc_queue_info
->index
);
1334 lpfc_ncmd
->cpu
= lpfc_queue_info
->index
;
1336 if (lpfc_ncmd
->cpu
< LPFC_CHECK_CPU_CNT
)
1337 phba
->cpucheck_xmt_io
[lpfc_ncmd
->cpu
]++;
1343 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
1349 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
1350 * @phba: Pointer to HBA context object
1351 * @cmdiocb: Pointer to command iocb object.
1352 * @rspiocb: Pointer to response iocb object.
1354 * This is the callback function for any NVME FCP IO that was aborted.
1360 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba
*phba
, struct lpfc_iocbq
*cmdiocb
,
1361 struct lpfc_wcqe_complete
*abts_cmpl
)
1363 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME
,
1364 "6145 ABORT_XRI_CN completing on rpi x%x "
1365 "original iotag x%x, abort cmd iotag x%x "
1366 "req_tag x%x, status x%x, hwstatus x%x\n",
1367 cmdiocb
->iocb
.un
.acxri
.abortContextTag
,
1368 cmdiocb
->iocb
.un
.acxri
.abortIoTag
,
1370 bf_get(lpfc_wcqe_c_request_tag
, abts_cmpl
),
1371 bf_get(lpfc_wcqe_c_status
, abts_cmpl
),
1372 bf_get(lpfc_wcqe_c_hw_status
, abts_cmpl
));
1373 lpfc_sli_release_iocbq(phba
, cmdiocb
);
1377 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
1378 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1379 * @lpfc_nvme_lport: Pointer to the driver's local port data
1380 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1381 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1382 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1384 * Driver registers this routine as its nvme request io abort handler. This
1385 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
1386 * data structure to the rport indicated in @lpfc_nvme_rport. This routine
1387 * is executed asynchronously - one the target is validated as "MAPPED" and
1388 * ready for IO, the driver issues the abort request and returns.
1394 lpfc_nvme_fcp_abort(struct nvme_fc_local_port
*pnvme_lport
,
1395 struct nvme_fc_remote_port
*pnvme_rport
,
1396 void *hw_queue_handle
,
1397 struct nvmefc_fcp_req
*pnvme_fcreq
)
1399 struct lpfc_nvme_lport
*lport
;
1400 struct lpfc_vport
*vport
;
1401 struct lpfc_hba
*phba
;
1402 struct lpfc_nodelist
*ndlp
;
1403 struct lpfc_nvme_rport
*rport
;
1404 struct lpfc_nvme_buf
*lpfc_nbuf
;
1405 struct lpfc_iocbq
*abts_buf
;
1406 struct lpfc_iocbq
*nvmereq_wqe
;
1407 union lpfc_wqe
*abts_wqe
;
1408 unsigned long flags
;
1411 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
1412 rport
= (struct lpfc_nvme_rport
*)pnvme_rport
->private;
1413 vport
= lport
->vport
;
1416 /* Announce entry to new IO submit field. */
1417 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1418 "6002 Abort Request to rport DID x%06x "
1419 "for nvme_fc_req %p\n",
1420 pnvme_rport
->port_id
,
1424 * Catch race where our node has transitioned, but the
1425 * transport is still transitioning.
1428 if (!ndlp
|| !NLP_CHK_NODE_ACT(ndlp
)) {
1429 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_ABTS
,
1430 "6054 rport %p, ndlp %p, DID x%06x ndlp "
1432 rport
, ndlp
, pnvme_rport
->port_id
);
1434 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
1436 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1437 "6055 Could not find node for "
1439 pnvme_rport
->port_id
);
1444 /* The remote node has to be ready to send an abort. */
1445 if ((ndlp
->nlp_state
!= NLP_STE_MAPPED_NODE
) &&
1446 !(ndlp
->nlp_type
& NLP_NVME_TARGET
)) {
1447 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_ABTS
,
1448 "6048 rport %p, DID x%06x not ready for "
1449 "IO. State x%x, Type x%x\n",
1450 rport
, pnvme_rport
->port_id
,
1451 ndlp
->nlp_state
, ndlp
->nlp_type
);
1455 /* If the hba is getting reset, this flag is set. It is
1456 * cleared when the reset is complete and rings reestablished.
1458 spin_lock_irqsave(&phba
->hbalock
, flags
);
1459 /* driver queued commands are in process of being flushed */
1460 if (phba
->hba_flag
& HBA_NVME_IOQ_FLUSH
) {
1461 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1462 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME
,
1463 "6139 Driver in reset cleanup - flushing "
1464 "NVME Req now. hba_flag x%x\n",
1469 lpfc_nbuf
= (struct lpfc_nvme_buf
*)pnvme_fcreq
->private;
1471 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1472 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME
,
1473 "6140 NVME IO req has no matching lpfc nvme "
1474 "io buffer. Skipping abort req.\n");
1476 } else if (!lpfc_nbuf
->nvmeCmd
) {
1477 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1478 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME
,
1479 "6141 lpfc NVME IO req has no nvme_fcreq "
1480 "io buffer. Skipping abort req.\n");
1485 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
1486 * state must match the nvme_fcreq passed by the nvme
1487 * transport. If they don't match, it is likely the driver
1488 * has already completed the NVME IO and the nvme transport
1489 * has not seen it yet.
1491 if (lpfc_nbuf
->nvmeCmd
!= pnvme_fcreq
) {
1492 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1493 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME
,
1494 "6143 NVME req mismatch: "
1495 "lpfc_nbuf %p nvmeCmd %p, "
1496 "pnvme_fcreq %p. Skipping Abort\n",
1497 lpfc_nbuf
, lpfc_nbuf
->nvmeCmd
,
1502 /* Don't abort IOs no longer on the pending queue. */
1503 nvmereq_wqe
= &lpfc_nbuf
->cur_iocbq
;
1504 if (!(nvmereq_wqe
->iocb_flag
& LPFC_IO_ON_TXCMPLQ
)) {
1505 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1506 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME
,
1507 "6142 NVME IO req %p not queued - skipping "
1513 lpfc_nvmeio_data(phba
, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1514 nvmereq_wqe
->sli4_xritag
,
1515 nvmereq_wqe
->hba_wqidx
, ndlp
->nlp_DID
);
1517 /* Outstanding abort is in progress */
1518 if (nvmereq_wqe
->iocb_flag
& LPFC_DRIVER_ABORTED
) {
1519 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1520 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME
,
1521 "6144 Outstanding NVME I/O Abort Request "
1522 "still pending on nvme_fcreq %p, "
1524 pnvme_fcreq
, lpfc_nbuf
);
1528 abts_buf
= __lpfc_sli_get_iocbq(phba
);
1530 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1531 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME
,
1532 "6136 No available abort wqes. Skipping "
1533 "Abts req for nvme_fcreq %p.\n",
1538 /* Ready - mark outstanding as aborted by driver. */
1539 nvmereq_wqe
->iocb_flag
|= LPFC_DRIVER_ABORTED
;
1541 /* Complete prepping the abort wqe and issue to the FW. */
1542 abts_wqe
= &abts_buf
->wqe
;
1544 /* WQEs are reused. Clear stale data and set key fields to
1545 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
1547 memset(abts_wqe
, 0, sizeof(union lpfc_wqe
));
1548 bf_set(abort_cmd_criteria
, &abts_wqe
->abort_cmd
, T_XRI_TAG
);
1551 bf_set(wqe_ct
, &abts_wqe
->abort_cmd
.wqe_com
, 0);
1552 bf_set(wqe_cmnd
, &abts_wqe
->abort_cmd
.wqe_com
, CMD_ABORT_XRI_CX
);
1553 bf_set(wqe_class
, &abts_wqe
->abort_cmd
.wqe_com
,
1554 nvmereq_wqe
->iocb
.ulpClass
);
1556 /* word 8 - tell the FW to abort the IO associated with this
1557 * outstanding exchange ID.
1559 abts_wqe
->abort_cmd
.wqe_com
.abort_tag
= nvmereq_wqe
->sli4_xritag
;
1561 /* word 9 - this is the iotag for the abts_wqe completion. */
1562 bf_set(wqe_reqtag
, &abts_wqe
->abort_cmd
.wqe_com
,
1566 bf_set(wqe_wqid
, &abts_wqe
->abort_cmd
.wqe_com
, nvmereq_wqe
->hba_wqidx
);
1567 bf_set(wqe_qosd
, &abts_wqe
->abort_cmd
.wqe_com
, 1);
1568 bf_set(wqe_lenloc
, &abts_wqe
->abort_cmd
.wqe_com
, LPFC_WQE_LENLOC_NONE
);
1571 bf_set(wqe_cmd_type
, &abts_wqe
->abort_cmd
.wqe_com
, OTHER_COMMAND
);
1572 bf_set(wqe_wqec
, &abts_wqe
->abort_cmd
.wqe_com
, 1);
1573 bf_set(wqe_cqid
, &abts_wqe
->abort_cmd
.wqe_com
, LPFC_WQE_CQ_ID_DEFAULT
);
1575 /* ABTS WQE must go to the same WQ as the WQE to be aborted */
1576 abts_buf
->iocb_flag
|= LPFC_IO_NVME
;
1577 abts_buf
->hba_wqidx
= nvmereq_wqe
->hba_wqidx
;
1578 abts_buf
->vport
= vport
;
1579 abts_buf
->wqe_cmpl
= lpfc_nvme_abort_fcreq_cmpl
;
1580 ret_val
= lpfc_sli4_issue_wqe(phba
, LPFC_FCP_RING
, abts_buf
);
1581 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1582 if (ret_val
== IOCB_ERROR
) {
1583 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME
,
1584 "6137 Failed abts issue_wqe with status x%x "
1585 "for nvme_fcreq %p.\n",
1586 ret_val
, pnvme_fcreq
);
1587 lpfc_sli_release_iocbq(phba
, abts_buf
);
1591 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME
,
1592 "6138 Transport Abort NVME Request Issued for\n"
1593 "ox_id x%x on reqtag x%x\n",
1594 nvmereq_wqe
->sli4_xritag
,
1598 /* Declare and initialization an instance of the FC NVME template. */
1599 static struct nvme_fc_port_template lpfc_nvme_template
= {
1600 /* initiator-based functions */
1601 .localport_delete
= lpfc_nvme_localport_delete
,
1602 .remoteport_delete
= lpfc_nvme_remoteport_delete
,
1603 .create_queue
= lpfc_nvme_create_queue
,
1604 .delete_queue
= lpfc_nvme_delete_queue
,
1605 .ls_req
= lpfc_nvme_ls_req
,
1606 .fcp_io
= lpfc_nvme_fcp_io_submit
,
1607 .ls_abort
= lpfc_nvme_ls_abort
,
1608 .fcp_abort
= lpfc_nvme_fcp_abort
,
1611 .max_sgl_segments
= LPFC_NVME_DEFAULT_SEGS
,
1612 .max_dif_sgl_segments
= LPFC_NVME_DEFAULT_SEGS
,
1613 .dma_boundary
= 0xFFFFFFFF,
1615 /* Sizes of additional private data for data structures.
1616 * No use for the last two sizes at this time.
1618 .local_priv_sz
= sizeof(struct lpfc_nvme_lport
),
1619 .remote_priv_sz
= sizeof(struct lpfc_nvme_rport
),
1620 .lsrqst_priv_sz
= 0,
1621 .fcprqst_priv_sz
= 0,
1625 * lpfc_sli4_post_nvme_sgl_block - post a block of nvme sgl list to firmware
1626 * @phba: pointer to lpfc hba data structure.
1627 * @nblist: pointer to nvme buffer list.
1628 * @count: number of scsi buffers on the list.
1630 * This routine is invoked to post a block of @count scsi sgl pages from a
1631 * SCSI buffer list @nblist to the HBA using non-embedded mailbox command.
1636 lpfc_sli4_post_nvme_sgl_block(struct lpfc_hba
*phba
,
1637 struct list_head
*nblist
,
1640 struct lpfc_nvme_buf
*lpfc_ncmd
;
1641 struct lpfc_mbx_post_uembed_sgl_page1
*sgl
;
1642 struct sgl_page_pairs
*sgl_pg_pairs
;
1645 uint32_t reqlen
, alloclen
, pg_pairs
;
1647 uint16_t xritag_start
= 0;
1649 uint32_t shdr_status
, shdr_add_status
;
1650 dma_addr_t pdma_phys_bpl1
;
1651 union lpfc_sli4_cfg_shdr
*shdr
;
1653 /* Calculate the requested length of the dma memory */
1654 reqlen
= count
* sizeof(struct sgl_page_pairs
) +
1655 sizeof(union lpfc_sli4_cfg_shdr
) + sizeof(uint32_t);
1656 if (reqlen
> SLI4_PAGE_SIZE
) {
1657 lpfc_printf_log(phba
, KERN_WARNING
, LOG_INIT
,
1658 "6118 Block sgl registration required DMA "
1659 "size (%d) great than a page\n", reqlen
);
1662 mbox
= mempool_alloc(phba
->mbox_mem_pool
, GFP_KERNEL
);
1664 lpfc_printf_log(phba
, KERN_ERR
, LOG_INIT
,
1665 "6119 Failed to allocate mbox cmd memory\n");
1669 /* Allocate DMA memory and set up the non-embedded mailbox command */
1670 alloclen
= lpfc_sli4_config(phba
, mbox
, LPFC_MBOX_SUBSYSTEM_FCOE
,
1671 LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES
, reqlen
,
1672 LPFC_SLI4_MBX_NEMBED
);
1674 if (alloclen
< reqlen
) {
1675 lpfc_printf_log(phba
, KERN_ERR
, LOG_INIT
,
1676 "6120 Allocated DMA memory size (%d) is "
1677 "less than the requested DMA memory "
1678 "size (%d)\n", alloclen
, reqlen
);
1679 lpfc_sli4_mbox_cmd_free(phba
, mbox
);
1683 /* Get the first SGE entry from the non-embedded DMA memory */
1684 viraddr
= mbox
->sge_array
->addr
[0];
1686 /* Set up the SGL pages in the non-embedded DMA pages */
1687 sgl
= (struct lpfc_mbx_post_uembed_sgl_page1
*)viraddr
;
1688 sgl_pg_pairs
= &sgl
->sgl_pg_pairs
;
1691 list_for_each_entry(lpfc_ncmd
, nblist
, list
) {
1692 /* Set up the sge entry */
1693 sgl_pg_pairs
->sgl_pg0_addr_lo
=
1694 cpu_to_le32(putPaddrLow(lpfc_ncmd
->dma_phys_sgl
));
1695 sgl_pg_pairs
->sgl_pg0_addr_hi
=
1696 cpu_to_le32(putPaddrHigh(lpfc_ncmd
->dma_phys_sgl
));
1697 if (phba
->cfg_sg_dma_buf_size
> SGL_PAGE_SIZE
)
1698 pdma_phys_bpl1
= lpfc_ncmd
->dma_phys_sgl
+
1702 sgl_pg_pairs
->sgl_pg1_addr_lo
=
1703 cpu_to_le32(putPaddrLow(pdma_phys_bpl1
));
1704 sgl_pg_pairs
->sgl_pg1_addr_hi
=
1705 cpu_to_le32(putPaddrHigh(pdma_phys_bpl1
));
1706 /* Keep the first xritag on the list */
1708 xritag_start
= lpfc_ncmd
->cur_iocbq
.sli4_xritag
;
1712 bf_set(lpfc_post_sgl_pages_xri
, sgl
, xritag_start
);
1713 bf_set(lpfc_post_sgl_pages_xricnt
, sgl
, pg_pairs
);
1714 /* Perform endian conversion if necessary */
1715 sgl
->word0
= cpu_to_le32(sgl
->word0
);
1717 if (!phba
->sli4_hba
.intr_enable
)
1718 rc
= lpfc_sli_issue_mbox(phba
, mbox
, MBX_POLL
);
1720 mbox_tmo
= lpfc_mbox_tmo_val(phba
, mbox
);
1721 rc
= lpfc_sli_issue_mbox_wait(phba
, mbox
, mbox_tmo
);
1723 shdr
= (union lpfc_sli4_cfg_shdr
*)&sgl
->cfg_shdr
;
1724 shdr_status
= bf_get(lpfc_mbox_hdr_status
, &shdr
->response
);
1725 shdr_add_status
= bf_get(lpfc_mbox_hdr_add_status
, &shdr
->response
);
1726 if (rc
!= MBX_TIMEOUT
)
1727 lpfc_sli4_mbox_cmd_free(phba
, mbox
);
1728 if (shdr_status
|| shdr_add_status
|| rc
) {
1729 lpfc_printf_log(phba
, KERN_ERR
, LOG_SLI
,
1730 "6125 POST_SGL_BLOCK mailbox command failed "
1731 "status x%x add_status x%x mbx status x%x\n",
1732 shdr_status
, shdr_add_status
, rc
);
1739 * lpfc_post_nvme_sgl_list - Post blocks of nvme buffer sgls from a list
1740 * @phba: pointer to lpfc hba data structure.
1741 * @post_nblist: pointer to the nvme buffer list.
1743 * This routine walks a list of nvme buffers that was passed in. It attempts
1744 * to construct blocks of nvme buffer sgls which contains contiguous xris and
1745 * uses the non-embedded SGL block post mailbox commands to post to the port.
1746 * For single NVME buffer sgl with non-contiguous xri, if any, it shall use
1747 * embedded SGL post mailbox command for posting. The @post_nblist passed in
1748 * must be local list, thus no lock is needed when manipulate the list.
1750 * Returns: 0 = failure, non-zero number of successfully posted buffers.
1753 lpfc_post_nvme_sgl_list(struct lpfc_hba
*phba
,
1754 struct list_head
*post_nblist
, int sb_count
)
1756 struct lpfc_nvme_buf
*lpfc_ncmd
, *lpfc_ncmd_next
;
1757 int status
, sgl_size
;
1758 int post_cnt
= 0, block_cnt
= 0, num_posting
= 0, num_posted
= 0;
1759 dma_addr_t pdma_phys_sgl1
;
1760 int last_xritag
= NO_XRI
;
1762 LIST_HEAD(prep_nblist
);
1763 LIST_HEAD(blck_nblist
);
1764 LIST_HEAD(nvme_nblist
);
1770 sgl_size
= phba
->cfg_sg_dma_buf_size
;
1772 list_for_each_entry_safe(lpfc_ncmd
, lpfc_ncmd_next
, post_nblist
, list
) {
1773 list_del_init(&lpfc_ncmd
->list
);
1775 if ((last_xritag
!= NO_XRI
) &&
1776 (lpfc_ncmd
->cur_iocbq
.sli4_xritag
!= last_xritag
+ 1)) {
1777 /* a hole in xri block, form a sgl posting block */
1778 list_splice_init(&prep_nblist
, &blck_nblist
);
1779 post_cnt
= block_cnt
- 1;
1780 /* prepare list for next posting block */
1781 list_add_tail(&lpfc_ncmd
->list
, &prep_nblist
);
1784 /* prepare list for next posting block */
1785 list_add_tail(&lpfc_ncmd
->list
, &prep_nblist
);
1786 /* enough sgls for non-embed sgl mbox command */
1787 if (block_cnt
== LPFC_NEMBED_MBOX_SGL_CNT
) {
1788 list_splice_init(&prep_nblist
, &blck_nblist
);
1789 post_cnt
= block_cnt
;
1794 last_xritag
= lpfc_ncmd
->cur_iocbq
.sli4_xritag
;
1796 /* end of repost sgl list condition for NVME buffers */
1797 if (num_posting
== sb_count
) {
1798 if (post_cnt
== 0) {
1799 /* last sgl posting block */
1800 list_splice_init(&prep_nblist
, &blck_nblist
);
1801 post_cnt
= block_cnt
;
1802 } else if (block_cnt
== 1) {
1803 /* last single sgl with non-contiguous xri */
1804 if (sgl_size
> SGL_PAGE_SIZE
)
1806 lpfc_ncmd
->dma_phys_sgl
+
1810 cur_xritag
= lpfc_ncmd
->cur_iocbq
.sli4_xritag
;
1811 status
= lpfc_sli4_post_sgl(phba
,
1812 lpfc_ncmd
->dma_phys_sgl
,
1813 pdma_phys_sgl1
, cur_xritag
);
1815 /* failure, put on abort nvme list */
1816 lpfc_ncmd
->flags
|= LPFC_SBUF_XBUSY
;
1818 /* success, put on NVME buffer list */
1819 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
1820 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
1823 /* success, put on NVME buffer sgl list */
1824 list_add_tail(&lpfc_ncmd
->list
, &nvme_nblist
);
1828 /* continue until a nembed page worth of sgls */
1832 /* post block of NVME buffer list sgls */
1833 status
= lpfc_sli4_post_nvme_sgl_block(phba
, &blck_nblist
,
1836 /* don't reset xirtag due to hole in xri block */
1838 last_xritag
= NO_XRI
;
1840 /* reset NVME buffer post count for next round of posting */
1843 /* put posted NVME buffer-sgl posted on NVME buffer sgl list */
1844 while (!list_empty(&blck_nblist
)) {
1845 list_remove_head(&blck_nblist
, lpfc_ncmd
,
1846 struct lpfc_nvme_buf
, list
);
1848 /* failure, put on abort nvme list */
1849 lpfc_ncmd
->flags
|= LPFC_SBUF_XBUSY
;
1851 /* success, put on NVME buffer list */
1852 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
1853 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
1856 list_add_tail(&lpfc_ncmd
->list
, &nvme_nblist
);
1859 /* Push NVME buffers with sgl posted to the available list */
1860 while (!list_empty(&nvme_nblist
)) {
1861 list_remove_head(&nvme_nblist
, lpfc_ncmd
,
1862 struct lpfc_nvme_buf
, list
);
1863 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
1869 * lpfc_repost_nvme_sgl_list - Repost all the allocated nvme buffer sgls
1870 * @phba: pointer to lpfc hba data structure.
1872 * This routine walks the list of nvme buffers that have been allocated and
1873 * repost them to the port by using SGL block post. This is needed after a
1874 * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine
1875 * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list
1876 * to the lpfc_nvme_buf_list. If the repost fails, reject all nvme buffers.
1878 * Returns: 0 = success, non-zero failure.
1881 lpfc_repost_nvme_sgl_list(struct lpfc_hba
*phba
)
1883 LIST_HEAD(post_nblist
);
1884 int num_posted
, rc
= 0;
1886 /* get all NVME buffers need to repost to a local list */
1887 spin_lock_irq(&phba
->nvme_buf_list_get_lock
);
1888 spin_lock(&phba
->nvme_buf_list_put_lock
);
1889 list_splice_init(&phba
->lpfc_nvme_buf_list_get
, &post_nblist
);
1890 list_splice(&phba
->lpfc_nvme_buf_list_put
, &post_nblist
);
1891 spin_unlock(&phba
->nvme_buf_list_put_lock
);
1892 spin_unlock_irq(&phba
->nvme_buf_list_get_lock
);
1894 /* post the list of nvme buffer sgls to port if available */
1895 if (!list_empty(&post_nblist
)) {
1896 num_posted
= lpfc_post_nvme_sgl_list(phba
, &post_nblist
,
1897 phba
->sli4_hba
.nvme_xri_cnt
);
1898 /* failed to post any nvme buffer, return error */
1899 if (num_posted
== 0)
1906 * lpfc_new_nvme_buf - Scsi buffer allocator for HBA with SLI4 IF spec
1907 * @vport: The virtual port for which this call being executed.
1908 * @num_to_allocate: The requested number of buffers to allocate.
1910 * This routine allocates nvme buffers for device with SLI-4 interface spec,
1911 * the nvme buffer contains all the necessary information needed to initiate
1912 * a NVME I/O. After allocating up to @num_to_allocate NVME buffers and put
1913 * them on a list, it post them to the port by using SGL block post.
1916 * int - number of nvme buffers that were allocated and posted.
1917 * 0 = failure, less than num_to_alloc is a partial failure.
1920 lpfc_new_nvme_buf(struct lpfc_vport
*vport
, int num_to_alloc
)
1922 struct lpfc_hba
*phba
= vport
->phba
;
1923 struct lpfc_nvme_buf
*lpfc_ncmd
;
1924 struct lpfc_iocbq
*pwqeq
;
1925 union lpfc_wqe128
*wqe
;
1926 struct sli4_sge
*sgl
;
1927 dma_addr_t pdma_phys_sgl
;
1928 uint16_t iotag
, lxri
= 0;
1929 int bcnt
, num_posted
, sgl_size
;
1930 LIST_HEAD(prep_nblist
);
1931 LIST_HEAD(post_nblist
);
1932 LIST_HEAD(nvme_nblist
);
1934 sgl_size
= phba
->cfg_sg_dma_buf_size
;
1936 for (bcnt
= 0; bcnt
< num_to_alloc
; bcnt
++) {
1937 lpfc_ncmd
= kzalloc(sizeof(struct lpfc_nvme_buf
), GFP_KERNEL
);
1941 * Get memory from the pci pool to map the virt space to
1942 * pci bus space for an I/O. The DMA buffer includes the
1943 * number of SGE's necessary to support the sg_tablesize.
1945 lpfc_ncmd
->data
= pci_pool_alloc(phba
->lpfc_sg_dma_buf_pool
,
1947 &lpfc_ncmd
->dma_handle
);
1948 if (!lpfc_ncmd
->data
) {
1952 memset(lpfc_ncmd
->data
, 0, phba
->cfg_sg_dma_buf_size
);
1954 lxri
= lpfc_sli4_next_xritag(phba
);
1955 if (lxri
== NO_XRI
) {
1956 pci_pool_free(phba
->lpfc_sg_dma_buf_pool
,
1957 lpfc_ncmd
->data
, lpfc_ncmd
->dma_handle
);
1961 pwqeq
= &(lpfc_ncmd
->cur_iocbq
);
1962 wqe
= (union lpfc_wqe128
*)&pwqeq
->wqe
;
1964 /* Allocate iotag for lpfc_ncmd->cur_iocbq. */
1965 iotag
= lpfc_sli_next_iotag(phba
, pwqeq
);
1967 pci_pool_free(phba
->lpfc_sg_dma_buf_pool
,
1968 lpfc_ncmd
->data
, lpfc_ncmd
->dma_handle
);
1970 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1971 "6121 Failed to allocated IOTAG for"
1972 " XRI:0x%x\n", lxri
);
1973 lpfc_sli4_free_xri(phba
, lxri
);
1976 pwqeq
->sli4_lxritag
= lxri
;
1977 pwqeq
->sli4_xritag
= phba
->sli4_hba
.xri_ids
[lxri
];
1978 pwqeq
->iocb_flag
|= LPFC_IO_NVME
;
1979 pwqeq
->context1
= lpfc_ncmd
;
1980 pwqeq
->wqe_cmpl
= lpfc_nvme_io_cmd_wqe_cmpl
;
1982 /* Initialize local short-hand pointers. */
1983 lpfc_ncmd
->nvme_sgl
= lpfc_ncmd
->data
;
1984 sgl
= lpfc_ncmd
->nvme_sgl
;
1985 pdma_phys_sgl
= lpfc_ncmd
->dma_handle
;
1986 lpfc_ncmd
->dma_phys_sgl
= pdma_phys_sgl
;
1988 /* Rsp SGE will be filled in when we rcv an IO
1989 * from the NVME Layer to be sent.
1990 * The cmd is going to be embedded so we need a SKIP SGE.
1992 bf_set(lpfc_sli4_sge_type
, sgl
, LPFC_SGE_TYPE_SKIP
);
1993 bf_set(lpfc_sli4_sge_last
, sgl
, 0);
1994 sgl
->word2
= cpu_to_le32(sgl
->word2
);
1995 /* Fill in word 3 / sgl_len during cmd submission */
1997 lpfc_ncmd
->cur_iocbq
.context1
= lpfc_ncmd
;
2000 bf_set(wqe_erp
, &wqe
->generic
.wqe_com
, 0);
2001 /* NVME upper layers will time things out, if needed */
2002 bf_set(wqe_tmo
, &wqe
->generic
.wqe_com
, 0);
2005 bf_set(wqe_ebde_cnt
, &wqe
->generic
.wqe_com
, 0);
2006 bf_set(wqe_dbde
, &wqe
->generic
.wqe_com
, 1);
2008 /* add the nvme buffer to a post list */
2009 list_add_tail(&lpfc_ncmd
->list
, &post_nblist
);
2010 spin_lock_irq(&phba
->nvme_buf_list_get_lock
);
2011 phba
->sli4_hba
.nvme_xri_cnt
++;
2012 spin_unlock_irq(&phba
->nvme_buf_list_get_lock
);
2014 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME
,
2015 "6114 Allocate %d out of %d requested new NVME "
2016 "buffers\n", bcnt
, num_to_alloc
);
2018 /* post the list of nvme buffer sgls to port if available */
2019 if (!list_empty(&post_nblist
))
2020 num_posted
= lpfc_post_nvme_sgl_list(phba
,
2021 &post_nblist
, bcnt
);
2029 * lpfc_get_nvme_buf - Get a nvme buffer from lpfc_nvme_buf_list of the HBA
2030 * @phba: The HBA for which this call is being executed.
2032 * This routine removes a nvme buffer from head of @phba lpfc_nvme_buf_list list
2033 * and returns to caller.
2037 * Pointer to lpfc_nvme_buf - Success
2039 static struct lpfc_nvme_buf
*
2040 lpfc_get_nvme_buf(struct lpfc_hba
*phba
, struct lpfc_nodelist
*ndlp
)
2042 struct lpfc_nvme_buf
*lpfc_ncmd
, *lpfc_ncmd_next
;
2043 unsigned long iflag
= 0;
2046 spin_lock_irqsave(&phba
->nvme_buf_list_get_lock
, iflag
);
2047 list_for_each_entry_safe(lpfc_ncmd
, lpfc_ncmd_next
,
2048 &phba
->lpfc_nvme_buf_list_get
, list
) {
2049 if (lpfc_test_rrq_active(phba
, ndlp
,
2050 lpfc_ncmd
->cur_iocbq
.sli4_lxritag
))
2052 list_del(&lpfc_ncmd
->list
);
2057 spin_lock(&phba
->nvme_buf_list_put_lock
);
2058 list_splice(&phba
->lpfc_nvme_buf_list_put
,
2059 &phba
->lpfc_nvme_buf_list_get
);
2060 INIT_LIST_HEAD(&phba
->lpfc_nvme_buf_list_put
);
2061 spin_unlock(&phba
->nvme_buf_list_put_lock
);
2062 list_for_each_entry_safe(lpfc_ncmd
, lpfc_ncmd_next
,
2063 &phba
->lpfc_nvme_buf_list_get
, list
) {
2064 if (lpfc_test_rrq_active(
2065 phba
, ndlp
, lpfc_ncmd
->cur_iocbq
.sli4_lxritag
))
2067 list_del(&lpfc_ncmd
->list
);
2072 spin_unlock_irqrestore(&phba
->nvme_buf_list_get_lock
, iflag
);
2079 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
2080 * @phba: The Hba for which this call is being executed.
2081 * @lpfc_ncmd: The nvme buffer which is being released.
2083 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
2084 * lpfc_nvme_buf_list list. For SLI4 XRI's are tied to the nvme buffer
2085 * and cannot be reused for at least RA_TOV amount of time if it was
2089 lpfc_release_nvme_buf(struct lpfc_hba
*phba
, struct lpfc_nvme_buf
*lpfc_ncmd
)
2091 unsigned long iflag
= 0;
2093 lpfc_ncmd
->nonsg_phys
= 0;
2094 if (lpfc_ncmd
->flags
& LPFC_SBUF_XBUSY
) {
2095 spin_lock_irqsave(&phba
->sli4_hba
.abts_nvme_buf_list_lock
,
2097 lpfc_ncmd
->nvmeCmd
= NULL
;
2098 list_add_tail(&lpfc_ncmd
->list
,
2099 &phba
->sli4_hba
.lpfc_abts_nvme_buf_list
);
2100 spin_unlock_irqrestore(&phba
->sli4_hba
.abts_nvme_buf_list_lock
,
2103 lpfc_ncmd
->nvmeCmd
= NULL
;
2104 lpfc_ncmd
->cur_iocbq
.iocb_flag
= LPFC_IO_NVME
;
2105 spin_lock_irqsave(&phba
->nvme_buf_list_put_lock
, iflag
);
2106 list_add_tail(&lpfc_ncmd
->list
, &phba
->lpfc_nvme_buf_list_put
);
2107 spin_unlock_irqrestore(&phba
->nvme_buf_list_put_lock
, iflag
);
2112 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2113 * @pvport - the lpfc_vport instance requesting a localport.
2115 * This routine is invoked to create an nvme localport instance to bind
2116 * to the nvme_fc_transport. It is called once during driver load
2117 * like lpfc_create_shost after all other services are initialized.
2118 * It requires a vport, vpi, and wwns at call time. Other localport
2119 * parameters are modified as the driver's FCID and the Fabric WWN
2124 * -ENOMEM - no heap memory available
2125 * other values - from nvme registration upcall
2128 lpfc_nvme_create_localport(struct lpfc_vport
*vport
)
2131 struct lpfc_hba
*phba
= vport
->phba
;
2132 struct nvme_fc_port_info nfcp_info
;
2133 struct nvme_fc_local_port
*localport
;
2134 struct lpfc_nvme_lport
*lport
;
2137 /* Initialize this localport instance. The vport wwn usage ensures
2138 * that NPIV is accounted for.
2140 memset(&nfcp_info
, 0, sizeof(struct nvme_fc_port_info
));
2141 nfcp_info
.port_role
= FC_PORT_ROLE_NVME_INITIATOR
;
2142 nfcp_info
.node_name
= wwn_to_u64(vport
->fc_nodename
.u
.wwn
);
2143 nfcp_info
.port_name
= wwn_to_u64(vport
->fc_portname
.u
.wwn
);
2145 /* For now need + 1 to get around NVME transport logic */
2146 lpfc_nvme_template
.max_sgl_segments
= phba
->cfg_sg_seg_cnt
+ 1;
2147 lpfc_nvme_template
.max_hw_queues
= phba
->cfg_nvme_io_channel
;
2149 /* localport is allocated from the stack, but the registration
2150 * call allocates heap memory as well as the private area.
2152 #if (IS_ENABLED(CONFIG_NVME_FC))
2153 ret
= nvme_fc_register_localport(&nfcp_info
, &lpfc_nvme_template
,
2154 &vport
->phba
->pcidev
->dev
, &localport
);
2159 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
| LOG_NVME_DISC
,
2160 "6005 Successfully registered local "
2161 "NVME port num %d, localP %p, private %p, "
2163 localport
->port_num
, localport
,
2165 lpfc_nvme_template
.max_sgl_segments
);
2167 /* Private is our lport size declared in the template. */
2168 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2169 vport
->localport
= localport
;
2170 lport
->vport
= vport
;
2171 INIT_LIST_HEAD(&lport
->rport_list
);
2172 vport
->nvmei_support
= 1;
2173 len
= lpfc_new_nvme_buf(vport
, phba
->sli4_hba
.nvme_xri_max
);
2174 vport
->phba
->total_nvme_bufs
+= len
;
2181 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
2182 * @pnvme: pointer to lpfc nvme data structure.
2184 * This routine is invoked to destroy all lports bound to the phba.
2185 * The lport memory was allocated by the nvme fc transport and is
2186 * released there. This routine ensures all rports bound to the
2187 * lport have been disconnected.
2191 lpfc_nvme_destroy_localport(struct lpfc_vport
*vport
)
2193 #if (IS_ENABLED(CONFIG_NVME_FC))
2194 struct nvme_fc_local_port
*localport
;
2195 struct lpfc_nvme_lport
*lport
;
2196 struct lpfc_nvme_rport
*rport
= NULL
, *rport_next
= NULL
;
2199 if (vport
->nvmei_support
== 0)
2202 localport
= vport
->localport
;
2203 vport
->localport
= NULL
;
2204 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2206 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
2207 "6011 Destroying NVME localport %p\n",
2209 list_for_each_entry_safe(rport
, rport_next
, &lport
->rport_list
, list
) {
2210 /* The last node ref has to get released now before the rport
2211 * private memory area is released by the transport.
2213 list_del(&rport
->list
);
2215 init_completion(&rport
->rport_unreg_done
);
2216 ret
= nvme_fc_unregister_remoteport(rport
->remoteport
);
2218 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
2219 "6008 rport fail destroy %x\n", ret
);
2220 wait_for_completion_timeout(&rport
->rport_unreg_done
, 5);
2223 /* lport's rport list is clear. Unregister
2224 * lport and release resources.
2226 init_completion(&lport
->lport_unreg_done
);
2227 ret
= nvme_fc_unregister_localport(localport
);
2228 wait_for_completion_timeout(&lport
->lport_unreg_done
, 5);
2230 /* Regardless of the unregister upcall response, clear
2231 * nvmei_support. All rports are unregistered and the
2232 * driver will clean up.
2234 vport
->nvmei_support
= 0;
2236 lpfc_printf_vlog(vport
,
2237 KERN_INFO
, LOG_NVME_DISC
,
2238 "6009 Unregistered lport Success\n");
2240 lpfc_printf_vlog(vport
,
2241 KERN_INFO
, LOG_NVME_DISC
,
2242 "6010 Unregistered lport "
2243 "Failed, status x%x\n",
2250 lpfc_nvme_update_localport(struct lpfc_vport
*vport
)
2252 struct nvme_fc_local_port
*localport
;
2253 struct lpfc_nvme_lport
*lport
;
2255 localport
= vport
->localport
;
2256 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2258 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
2259 "6012 Update NVME lport %p did x%x\n",
2260 localport
, vport
->fc_myDID
);
2262 localport
->port_id
= vport
->fc_myDID
;
2263 if (localport
->port_id
== 0)
2264 localport
->port_role
= FC_PORT_ROLE_NVME_DISCOVERY
;
2266 localport
->port_role
= FC_PORT_ROLE_NVME_INITIATOR
;
2268 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
2269 "6030 bound lport %p to DID x%06x\n",
2270 lport
, localport
->port_id
);
2275 lpfc_nvme_register_port(struct lpfc_vport
*vport
, struct lpfc_nodelist
*ndlp
)
2277 #if (IS_ENABLED(CONFIG_NVME_FC))
2279 struct nvme_fc_local_port
*localport
;
2280 struct lpfc_nvme_lport
*lport
;
2281 struct lpfc_nvme_rport
*rport
;
2282 struct nvme_fc_remote_port
*remote_port
;
2283 struct nvme_fc_port_info rpinfo
;
2285 lpfc_printf_vlog(ndlp
->vport
, KERN_INFO
, LOG_NVME_DISC
,
2286 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2287 ndlp
->nlp_DID
, ndlp
->nlp_type
);
2289 localport
= vport
->localport
;
2290 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2292 if (ndlp
->nlp_type
& (NLP_NVME_TARGET
| NLP_NVME_INITIATOR
)) {
2294 /* The driver isn't expecting the rport wwn to change
2295 * but it might get a different DID on a different
2298 list_for_each_entry(rport
, &lport
->rport_list
, list
) {
2299 if (rport
->remoteport
->port_name
!=
2300 wwn_to_u64(ndlp
->nlp_portname
.u
.wwn
))
2302 lpfc_printf_vlog(ndlp
->vport
, KERN_INFO
, LOG_NVME_DISC
,
2303 "6035 lport %p, found matching rport "
2304 "at wwpn 0x%llx, Data: x%x x%x x%x "
2307 rport
->remoteport
->port_name
,
2308 rport
->remoteport
->port_id
,
2309 rport
->remoteport
->port_role
,
2312 remote_port
= rport
->remoteport
;
2313 if ((remote_port
->port_id
== 0) &&
2314 (remote_port
->port_role
==
2315 FC_PORT_ROLE_NVME_DISCOVERY
)) {
2316 remote_port
->port_id
= ndlp
->nlp_DID
;
2317 remote_port
->port_role
&=
2318 ~FC_PORT_ROLE_NVME_DISCOVERY
;
2319 if (ndlp
->nlp_type
& NLP_NVME_TARGET
)
2320 remote_port
->port_role
|=
2321 FC_PORT_ROLE_NVME_TARGET
;
2322 if (ndlp
->nlp_type
& NLP_NVME_INITIATOR
)
2323 remote_port
->port_role
|=
2324 FC_PORT_ROLE_NVME_INITIATOR
;
2326 lpfc_printf_vlog(ndlp
->vport
, KERN_INFO
,
2328 "6014 Rebinding lport to "
2329 "rport wwpn 0x%llx, "
2330 "Data: x%x x%x x%x x%06x\n",
2331 remote_port
->port_name
,
2332 remote_port
->port_id
,
2333 remote_port
->port_role
,
2340 /* NVME rports are not preserved across devloss.
2341 * Just register this instance.
2343 rpinfo
.port_id
= ndlp
->nlp_DID
;
2344 rpinfo
.port_role
= 0;
2345 if (ndlp
->nlp_type
& NLP_NVME_TARGET
)
2346 rpinfo
.port_role
|= FC_PORT_ROLE_NVME_TARGET
;
2347 if (ndlp
->nlp_type
& NLP_NVME_INITIATOR
)
2348 rpinfo
.port_role
|= FC_PORT_ROLE_NVME_INITIATOR
;
2349 rpinfo
.port_name
= wwn_to_u64(ndlp
->nlp_portname
.u
.wwn
);
2350 rpinfo
.node_name
= wwn_to_u64(ndlp
->nlp_nodename
.u
.wwn
);
2351 ret
= nvme_fc_register_remoteport(localport
, &rpinfo
,
2354 rport
= remote_port
->private;
2355 rport
->remoteport
= remote_port
;
2356 rport
->lport
= lport
;
2357 rport
->ndlp
= lpfc_nlp_get(ndlp
);
2360 ndlp
->nrport
= rport
;
2361 INIT_LIST_HEAD(&rport
->list
);
2362 list_add_tail(&rport
->list
, &lport
->rport_list
);
2363 lpfc_printf_vlog(vport
, KERN_INFO
,
2364 LOG_NVME_DISC
| LOG_NODE
,
2365 "6022 Binding new rport to lport %p "
2366 "Rport WWNN 0x%llx, Rport WWPN 0x%llx "
2367 "DID x%06x Role x%x\n",
2369 rpinfo
.node_name
, rpinfo
.port_name
,
2370 rpinfo
.port_id
, rpinfo
.port_role
);
2372 lpfc_printf_vlog(vport
, KERN_ERR
,
2373 LOG_NVME_DISC
| LOG_NODE
,
2374 "6031 RemotePort Registration failed "
2375 "err: %d, DID x%06x\n",
2376 ret
, ndlp
->nlp_DID
);
2380 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
2381 "6027 Unknown nlp_type x%x on DID x%06x "
2382 "ndlp %p. Not Registering nvme rport\n",
2383 ndlp
->nlp_type
, ndlp
->nlp_DID
, ndlp
);
2391 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
2393 * There is no notion of Devloss or rport recovery from the current
2394 * nvme_transport perspective. Loss of an rport just means IO cannot
2395 * be sent and recovery is completely up to the initator.
2396 * For now, the driver just unbinds the DID and port_role so that
2397 * no further IO can be issued. Changes are planned for later.
2399 * Notes - the ndlp reference count is not decremented here since
2400 * since there is no nvme_transport api for devloss. Node ref count
2401 * is only adjusted in driver unload.
2404 lpfc_nvme_unregister_port(struct lpfc_vport
*vport
, struct lpfc_nodelist
*ndlp
)
2406 #if (IS_ENABLED(CONFIG_NVME_FC))
2408 struct nvme_fc_local_port
*localport
;
2409 struct lpfc_nvme_lport
*lport
;
2410 struct lpfc_nvme_rport
*rport
;
2411 struct nvme_fc_remote_port
*remoteport
;
2413 localport
= vport
->localport
;
2415 /* This is fundamental error. The localport is always
2416 * available until driver unload. Just exit.
2421 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2425 rport
= ndlp
->nrport
;
2429 remoteport
= rport
->remoteport
;
2430 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
2431 "6033 Unreg nvme remoteport %p, portname x%llx, "
2432 "port_id x%06x, portstate x%x port type x%x\n",
2433 remoteport
, remoteport
->port_name
,
2434 remoteport
->port_id
, remoteport
->port_state
,
2437 /* Sanity check ndlp type. Only call for NVME ports. Don't
2438 * clear any rport state until the transport calls back.
2440 if (ndlp
->nlp_type
& (NLP_NVME_TARGET
| NLP_NVME_INITIATOR
)) {
2441 init_completion(&rport
->rport_unreg_done
);
2442 ret
= nvme_fc_unregister_remoteport(remoteport
);
2444 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
2445 "6167 NVME unregister failed %d "
2447 ret
, remoteport
->port_state
);
2450 /* Wait for the driver's delete completion routine to finish
2451 * before proceeding. This guarantees the transport and driver
2452 * have completed the unreg process.
2454 ret
= wait_for_completion_timeout(&rport
->rport_unreg_done
, 5);
2456 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
2457 "6169 Unreg nvme wait failed %d\n",
2465 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
2466 "6168: State error: lport %p, rport%p FCID x%06x\n",
2467 vport
->localport
, ndlp
->rport
, ndlp
->nlp_DID
);
2471 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2472 * @phba: pointer to lpfc hba data structure.
2473 * @axri: pointer to the fcp xri abort wcqe structure.
2475 * This routine is invoked by the worker thread to process a SLI4 fast-path
2479 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba
*phba
,
2480 struct sli4_wcqe_xri_aborted
*axri
)
2482 uint16_t xri
= bf_get(lpfc_wcqe_xa_xri
, axri
);
2483 uint16_t rxid
= bf_get(lpfc_wcqe_xa_remote_xid
, axri
);
2484 struct lpfc_nvme_buf
*lpfc_ncmd
, *next_lpfc_ncmd
;
2485 struct lpfc_nodelist
*ndlp
;
2486 unsigned long iflag
= 0;
2489 if (!(phba
->cfg_enable_fc4_type
& LPFC_ENABLE_NVME
))
2491 spin_lock_irqsave(&phba
->hbalock
, iflag
);
2492 spin_lock(&phba
->sli4_hba
.abts_nvme_buf_list_lock
);
2493 list_for_each_entry_safe(lpfc_ncmd
, next_lpfc_ncmd
,
2494 &phba
->sli4_hba
.lpfc_abts_nvme_buf_list
,
2496 if (lpfc_ncmd
->cur_iocbq
.sli4_xritag
== xri
) {
2497 list_del(&lpfc_ncmd
->list
);
2498 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
2499 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
2501 &phba
->sli4_hba
.abts_nvme_buf_list_lock
);
2503 rrq_empty
= list_empty(&phba
->active_rrq_list
);
2504 spin_unlock_irqrestore(&phba
->hbalock
, iflag
);
2505 ndlp
= lpfc_ncmd
->ndlp
;
2507 lpfc_set_rrq_active(
2509 lpfc_ncmd
->cur_iocbq
.sli4_lxritag
,
2511 lpfc_sli4_abts_err_handler(phba
, ndlp
, axri
);
2513 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
2515 lpfc_worker_wake_up(phba
);
2519 spin_unlock(&phba
->sli4_hba
.abts_nvme_buf_list_lock
);
2520 spin_unlock_irqrestore(&phba
->hbalock
, iflag
);