2 * Driver for sTec s1120 PCIe SSDs. sTec was acquired in 2013 by HGST and HGST
3 * was acquired by Western Digital in 2012.
5 * Copyright 2012 sTec, Inc.
6 * Copyright (c) 2017 Western Digital Corporation or its affiliates.
8 * This file is part of the Linux kernel, and is made available under
9 * the terms of the GNU General Public License version 2.
12 #include <linux/kernel.h>
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/pci.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 #include <linux/blkdev.h>
19 #include <linux/blk-mq.h>
20 #include <linux/sched.h>
21 #include <linux/interrupt.h>
22 #include <linux/compiler.h>
23 #include <linux/workqueue.h>
24 #include <linux/delay.h>
25 #include <linux/time.h>
26 #include <linux/hdreg.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/completion.h>
29 #include <linux/scatterlist.h>
30 #include <linux/version.h>
31 #include <linux/err.h>
32 #include <linux/aer.h>
33 #include <linux/wait.h>
34 #include <linux/stringify.h>
35 #include <linux/slab_def.h>
36 #include <scsi/scsi.h>
39 #include <linux/uaccess.h>
40 #include <asm/unaligned.h>
42 #include "skd_s1120.h"
44 static int skd_dbg_level
;
45 static int skd_isr_comp_limit
= 4;
47 #define SKD_ASSERT(expr) \
49 if (unlikely(!(expr))) { \
50 pr_err("Assertion failed! %s,%s,%s,line=%d\n", \
51 # expr, __FILE__, __func__, __LINE__); \
55 #define DRV_NAME "skd"
56 #define PFX DRV_NAME ": "
58 MODULE_LICENSE("GPL");
60 MODULE_DESCRIPTION("STEC s1120 PCIe SSD block driver");
62 #define PCI_VENDOR_ID_STEC 0x1B39
63 #define PCI_DEVICE_ID_S1120 0x0001
65 #define SKD_FUA_NV (1 << 1)
66 #define SKD_MINORS_PER_DEVICE 16
68 #define SKD_MAX_QUEUE_DEPTH 200u
70 #define SKD_PAUSE_TIMEOUT (5 * 1000)
72 #define SKD_N_FITMSG_BYTES (512u)
73 #define SKD_MAX_REQ_PER_MSG 14
75 #define SKD_N_SPECIAL_FITMSG_BYTES (128u)
77 /* SG elements are 32 bytes, so we can make this 4096 and still be under the
78 * 128KB limit. That allows 4096*4K = 16M xfer size
80 #define SKD_N_SG_PER_REQ_DEFAULT 256u
82 #define SKD_N_COMPLETION_ENTRY 256u
83 #define SKD_N_READ_CAP_BYTES (8u)
85 #define SKD_N_INTERNAL_BYTES (512u)
87 #define SKD_SKCOMP_SIZE \
88 ((sizeof(struct fit_completion_entry_v1) + \
89 sizeof(struct fit_comp_error_info)) * SKD_N_COMPLETION_ENTRY)
91 /* 5 bits of uniqifier, 0xF800 */
92 #define SKD_ID_TABLE_MASK (3u << 8u)
93 #define SKD_ID_RW_REQUEST (0u << 8u)
94 #define SKD_ID_INTERNAL (1u << 8u)
95 #define SKD_ID_FIT_MSG (3u << 8u)
96 #define SKD_ID_SLOT_MASK 0x00FFu
97 #define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
99 #define SKD_N_MAX_SECTORS 2048u
101 #define SKD_MAX_RETRIES 2u
103 #define SKD_TIMER_SECONDS(seconds) (seconds)
104 #define SKD_TIMER_MINUTES(minutes) ((minutes) * (60))
106 #define INQ_STD_NBYTES 36
108 enum skd_drvr_state
{
112 SKD_DRVR_STATE_STARTING
,
113 SKD_DRVR_STATE_ONLINE
,
114 SKD_DRVR_STATE_PAUSING
,
115 SKD_DRVR_STATE_PAUSED
,
116 SKD_DRVR_STATE_RESTARTING
,
117 SKD_DRVR_STATE_RESUMING
,
118 SKD_DRVR_STATE_STOPPING
,
119 SKD_DRVR_STATE_FAULT
,
120 SKD_DRVR_STATE_DISAPPEARED
,
121 SKD_DRVR_STATE_PROTOCOL_MISMATCH
,
122 SKD_DRVR_STATE_BUSY_ERASE
,
123 SKD_DRVR_STATE_BUSY_SANITIZE
,
124 SKD_DRVR_STATE_BUSY_IMMINENT
,
125 SKD_DRVR_STATE_WAIT_BOOT
,
126 SKD_DRVR_STATE_SYNCING
,
129 #define SKD_WAIT_BOOT_TIMO SKD_TIMER_SECONDS(90u)
130 #define SKD_STARTING_TIMO SKD_TIMER_SECONDS(8u)
131 #define SKD_RESTARTING_TIMO SKD_TIMER_MINUTES(4u)
132 #define SKD_BUSY_TIMO SKD_TIMER_MINUTES(20u)
133 #define SKD_STARTED_BUSY_TIMO SKD_TIMER_SECONDS(60u)
134 #define SKD_START_WAIT_SECONDS 90u
140 SKD_REQ_STATE_COMPLETED
,
141 SKD_REQ_STATE_TIMEOUT
,
144 enum skd_check_status_action
{
145 SKD_CHECK_STATUS_REPORT_GOOD
,
146 SKD_CHECK_STATUS_REPORT_SMART_ALERT
,
147 SKD_CHECK_STATUS_REQUEUE_REQUEST
,
148 SKD_CHECK_STATUS_REPORT_ERROR
,
149 SKD_CHECK_STATUS_BUSY_IMMINENT
,
153 struct fit_msg_hdr fmh
;
154 struct skd_scsi_request scsi
[SKD_MAX_REQ_PER_MSG
];
157 struct skd_fitmsg_context
{
162 struct skd_msg_buf
*msg_buf
;
163 dma_addr_t mb_dma_address
;
166 struct skd_request_context
{
167 enum skd_req_state state
;
174 enum dma_data_direction data_dir
;
175 struct scatterlist
*sg
;
179 struct fit_sg_descriptor
*sksg_list
;
180 dma_addr_t sksg_dma_address
;
182 struct fit_completion_entry_v1 completion
;
184 struct fit_comp_error_info err_info
;
189 struct skd_special_context
{
190 struct skd_request_context req
;
193 dma_addr_t db_dma_address
;
195 struct skd_msg_buf
*msg_buf
;
196 dma_addr_t mb_dma_address
;
199 typedef enum skd_irq_type
{
205 #define SKD_MAX_BARS 2
208 void __iomem
*mem_map
[SKD_MAX_BARS
];
209 resource_size_t mem_phys
[SKD_MAX_BARS
];
210 u32 mem_size
[SKD_MAX_BARS
];
212 struct skd_msix_entry
*msix_entries
;
214 struct pci_dev
*pdev
;
215 int pcie_error_reporting_is_enabled
;
218 struct gendisk
*disk
;
219 struct blk_mq_tag_set tag_set
;
220 struct request_queue
*queue
;
221 struct skd_fitmsg_context
*skmsg
;
222 struct device
*class_dev
;
230 enum skd_drvr_state state
;
233 u32 cur_max_queue_depth
;
234 u32 queue_low_water_mark
;
235 u32 dev_max_queue_depth
;
237 u32 num_fitmsg_context
;
240 struct skd_fitmsg_context
*skmsg_table
;
242 struct skd_special_context internal_skspcl
;
243 u32 read_cap_blocksize
;
244 u32 read_cap_last_lba
;
245 int read_cap_is_valid
;
246 int inquiry_is_valid
;
247 u8 inq_serial_num
[13]; /*12 chars plus null term */
251 struct kmem_cache
*msgbuf_cache
;
252 struct kmem_cache
*sglist_cache
;
253 struct kmem_cache
*databuf_cache
;
254 struct fit_completion_entry_v1
*skcomp_table
;
255 struct fit_comp_error_info
*skerr_table
;
256 dma_addr_t cq_dma_address
;
258 wait_queue_head_t waitq
;
260 struct timer_list timer
;
270 u32 connect_time_stamp
;
272 #define SKD_MAX_CONNECT_RETRIES 16
277 struct work_struct start_queue
;
278 struct work_struct completion_worker
;
281 #define SKD_WRITEL(DEV, VAL, OFF) skd_reg_write32(DEV, VAL, OFF)
282 #define SKD_READL(DEV, OFF) skd_reg_read32(DEV, OFF)
283 #define SKD_WRITEQ(DEV, VAL, OFF) skd_reg_write64(DEV, VAL, OFF)
285 static inline u32
skd_reg_read32(struct skd_device
*skdev
, u32 offset
)
287 u32 val
= readl(skdev
->mem_map
[1] + offset
);
289 if (unlikely(skdev
->dbg_level
>= 2))
290 dev_dbg(&skdev
->pdev
->dev
, "offset %x = %x\n", offset
, val
);
294 static inline void skd_reg_write32(struct skd_device
*skdev
, u32 val
,
297 writel(val
, skdev
->mem_map
[1] + offset
);
298 if (unlikely(skdev
->dbg_level
>= 2))
299 dev_dbg(&skdev
->pdev
->dev
, "offset %x = %x\n", offset
, val
);
302 static inline void skd_reg_write64(struct skd_device
*skdev
, u64 val
,
305 writeq(val
, skdev
->mem_map
[1] + offset
);
306 if (unlikely(skdev
->dbg_level
>= 2))
307 dev_dbg(&skdev
->pdev
->dev
, "offset %x = %016llx\n", offset
,
312 #define SKD_IRQ_DEFAULT SKD_IRQ_MSIX
313 static int skd_isr_type
= SKD_IRQ_DEFAULT
;
315 module_param(skd_isr_type
, int, 0444);
316 MODULE_PARM_DESC(skd_isr_type
, "Interrupt type capability."
317 " (0==legacy, 1==MSI, 2==MSI-X, default==1)");
319 #define SKD_MAX_REQ_PER_MSG_DEFAULT 1
320 static int skd_max_req_per_msg
= SKD_MAX_REQ_PER_MSG_DEFAULT
;
322 module_param(skd_max_req_per_msg
, int, 0444);
323 MODULE_PARM_DESC(skd_max_req_per_msg
,
324 "Maximum SCSI requests packed in a single message."
325 " (1-" __stringify(SKD_MAX_REQ_PER_MSG
) ", default==1)");
327 #define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
328 #define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
329 static int skd_max_queue_depth
= SKD_MAX_QUEUE_DEPTH_DEFAULT
;
331 module_param(skd_max_queue_depth
, int, 0444);
332 MODULE_PARM_DESC(skd_max_queue_depth
,
333 "Maximum SCSI requests issued to s1120."
334 " (1-200, default==" SKD_MAX_QUEUE_DEPTH_DEFAULT_STR
")");
336 static int skd_sgs_per_request
= SKD_N_SG_PER_REQ_DEFAULT
;
337 module_param(skd_sgs_per_request
, int, 0444);
338 MODULE_PARM_DESC(skd_sgs_per_request
,
339 "Maximum SG elements per block request."
340 " (1-4096, default==256)");
342 static int skd_max_pass_thru
= 1;
343 module_param(skd_max_pass_thru
, int, 0444);
344 MODULE_PARM_DESC(skd_max_pass_thru
,
345 "Maximum SCSI pass-thru at a time. IGNORED");
347 module_param(skd_dbg_level
, int, 0444);
348 MODULE_PARM_DESC(skd_dbg_level
, "s1120 debug level (0,1,2)");
350 module_param(skd_isr_comp_limit
, int, 0444);
351 MODULE_PARM_DESC(skd_isr_comp_limit
, "s1120 isr comp limit (0=none) default=4");
353 /* Major device number dynamically assigned. */
354 static u32 skd_major
;
356 static void skd_destruct(struct skd_device
*skdev
);
357 static const struct block_device_operations skd_blockdev_ops
;
358 static void skd_send_fitmsg(struct skd_device
*skdev
,
359 struct skd_fitmsg_context
*skmsg
);
360 static void skd_send_special_fitmsg(struct skd_device
*skdev
,
361 struct skd_special_context
*skspcl
);
362 static bool skd_preop_sg_list(struct skd_device
*skdev
,
363 struct skd_request_context
*skreq
);
364 static void skd_postop_sg_list(struct skd_device
*skdev
,
365 struct skd_request_context
*skreq
);
367 static void skd_restart_device(struct skd_device
*skdev
);
368 static int skd_quiesce_dev(struct skd_device
*skdev
);
369 static int skd_unquiesce_dev(struct skd_device
*skdev
);
370 static void skd_disable_interrupts(struct skd_device
*skdev
);
371 static void skd_isr_fwstate(struct skd_device
*skdev
);
372 static void skd_recover_requests(struct skd_device
*skdev
);
373 static void skd_soft_reset(struct skd_device
*skdev
);
375 const char *skd_drive_state_to_str(int state
);
376 const char *skd_skdev_state_to_str(enum skd_drvr_state state
);
377 static void skd_log_skdev(struct skd_device
*skdev
, const char *event
);
378 static void skd_log_skreq(struct skd_device
*skdev
,
379 struct skd_request_context
*skreq
, const char *event
);
382 *****************************************************************************
383 * READ/WRITE REQUESTS
384 *****************************************************************************
386 static void skd_inc_in_flight(struct request
*rq
, void *data
, bool reserved
)
393 static int skd_in_flight(struct skd_device
*skdev
)
397 blk_mq_tagset_busy_iter(&skdev
->tag_set
, skd_inc_in_flight
, &count
);
403 skd_prep_rw_cdb(struct skd_scsi_request
*scsi_req
,
404 int data_dir
, unsigned lba
,
407 if (data_dir
== READ
)
408 scsi_req
->cdb
[0] = READ_10
;
410 scsi_req
->cdb
[0] = WRITE_10
;
412 scsi_req
->cdb
[1] = 0;
413 scsi_req
->cdb
[2] = (lba
& 0xff000000) >> 24;
414 scsi_req
->cdb
[3] = (lba
& 0xff0000) >> 16;
415 scsi_req
->cdb
[4] = (lba
& 0xff00) >> 8;
416 scsi_req
->cdb
[5] = (lba
& 0xff);
417 scsi_req
->cdb
[6] = 0;
418 scsi_req
->cdb
[7] = (count
& 0xff00) >> 8;
419 scsi_req
->cdb
[8] = count
& 0xff;
420 scsi_req
->cdb
[9] = 0;
424 skd_prep_zerosize_flush_cdb(struct skd_scsi_request
*scsi_req
,
425 struct skd_request_context
*skreq
)
427 skreq
->flush_cmd
= 1;
429 scsi_req
->cdb
[0] = SYNCHRONIZE_CACHE
;
430 scsi_req
->cdb
[1] = 0;
431 scsi_req
->cdb
[2] = 0;
432 scsi_req
->cdb
[3] = 0;
433 scsi_req
->cdb
[4] = 0;
434 scsi_req
->cdb
[5] = 0;
435 scsi_req
->cdb
[6] = 0;
436 scsi_req
->cdb
[7] = 0;
437 scsi_req
->cdb
[8] = 0;
438 scsi_req
->cdb
[9] = 0;
442 * Return true if and only if all pending requests should be failed.
444 static bool skd_fail_all(struct request_queue
*q
)
446 struct skd_device
*skdev
= q
->queuedata
;
448 SKD_ASSERT(skdev
->state
!= SKD_DRVR_STATE_ONLINE
);
450 skd_log_skdev(skdev
, "req_not_online");
451 switch (skdev
->state
) {
452 case SKD_DRVR_STATE_PAUSING
:
453 case SKD_DRVR_STATE_PAUSED
:
454 case SKD_DRVR_STATE_STARTING
:
455 case SKD_DRVR_STATE_RESTARTING
:
456 case SKD_DRVR_STATE_WAIT_BOOT
:
457 /* In case of starting, we haven't started the queue,
458 * so we can't get here... but requests are
459 * possibly hanging out waiting for us because we
460 * reported the dev/skd0 already. They'll wait
461 * forever if connect doesn't complete.
462 * What to do??? delay dev/skd0 ??
464 case SKD_DRVR_STATE_BUSY
:
465 case SKD_DRVR_STATE_BUSY_IMMINENT
:
466 case SKD_DRVR_STATE_BUSY_ERASE
:
469 case SKD_DRVR_STATE_BUSY_SANITIZE
:
470 case SKD_DRVR_STATE_STOPPING
:
471 case SKD_DRVR_STATE_SYNCING
:
472 case SKD_DRVR_STATE_FAULT
:
473 case SKD_DRVR_STATE_DISAPPEARED
:
479 static blk_status_t
skd_mq_queue_rq(struct blk_mq_hw_ctx
*hctx
,
480 const struct blk_mq_queue_data
*mqd
)
482 struct request
*const req
= mqd
->rq
;
483 struct request_queue
*const q
= req
->q
;
484 struct skd_device
*skdev
= q
->queuedata
;
485 struct skd_fitmsg_context
*skmsg
;
486 struct fit_msg_hdr
*fmh
;
487 const u32 tag
= blk_mq_unique_tag(req
);
488 struct skd_request_context
*const skreq
= blk_mq_rq_to_pdu(req
);
489 struct skd_scsi_request
*scsi_req
;
490 unsigned long flags
= 0;
491 const u32 lba
= blk_rq_pos(req
);
492 const u32 count
= blk_rq_sectors(req
);
493 const int data_dir
= rq_data_dir(req
);
495 if (unlikely(skdev
->state
!= SKD_DRVR_STATE_ONLINE
))
496 return skd_fail_all(q
) ? BLK_STS_IOERR
: BLK_STS_RESOURCE
;
498 blk_mq_start_request(req
);
500 WARN_ONCE(tag
>= skd_max_queue_depth
, "%#x > %#x (nr_requests = %lu)\n",
501 tag
, skd_max_queue_depth
, q
->nr_requests
);
503 SKD_ASSERT(skreq
->state
== SKD_REQ_STATE_IDLE
);
505 dev_dbg(&skdev
->pdev
->dev
,
506 "new req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n", req
, lba
,
507 lba
, count
, count
, data_dir
);
509 skreq
->id
= tag
+ SKD_ID_RW_REQUEST
;
510 skreq
->flush_cmd
= 0;
512 skreq
->sg_byte_count
= 0;
514 skreq
->fitmsg_id
= 0;
516 skreq
->data_dir
= data_dir
== READ
? DMA_FROM_DEVICE
: DMA_TO_DEVICE
;
518 if (req
->bio
&& !skd_preop_sg_list(skdev
, skreq
)) {
519 dev_dbg(&skdev
->pdev
->dev
, "error Out\n");
520 skreq
->status
= BLK_STS_RESOURCE
;
521 blk_mq_complete_request(req
);
525 dma_sync_single_for_device(&skdev
->pdev
->dev
, skreq
->sksg_dma_address
,
527 sizeof(struct fit_sg_descriptor
),
530 /* Either a FIT msg is in progress or we have to start one. */
531 if (skd_max_req_per_msg
== 1) {
534 spin_lock_irqsave(&skdev
->lock
, flags
);
535 skmsg
= skdev
->skmsg
;
538 skmsg
= &skdev
->skmsg_table
[tag
];
539 skdev
->skmsg
= skmsg
;
541 /* Initialize the FIT msg header */
542 fmh
= &skmsg
->msg_buf
->fmh
;
543 memset(fmh
, 0, sizeof(*fmh
));
544 fmh
->protocol_id
= FIT_PROTOCOL_ID_SOFIT
;
545 skmsg
->length
= sizeof(*fmh
);
547 fmh
= &skmsg
->msg_buf
->fmh
;
550 skreq
->fitmsg_id
= skmsg
->id
;
552 scsi_req
= &skmsg
->msg_buf
->scsi
[fmh
->num_protocol_cmds_coalesced
];
553 memset(scsi_req
, 0, sizeof(*scsi_req
));
555 scsi_req
->hdr
.tag
= skreq
->id
;
556 scsi_req
->hdr
.sg_list_dma_address
=
557 cpu_to_be64(skreq
->sksg_dma_address
);
559 if (req_op(req
) == REQ_OP_FLUSH
) {
560 skd_prep_zerosize_flush_cdb(scsi_req
, skreq
);
561 SKD_ASSERT(skreq
->flush_cmd
== 1);
563 skd_prep_rw_cdb(scsi_req
, data_dir
, lba
, count
);
566 if (req
->cmd_flags
& REQ_FUA
)
567 scsi_req
->cdb
[1] |= SKD_FUA_NV
;
569 scsi_req
->hdr
.sg_list_len_bytes
= cpu_to_be32(skreq
->sg_byte_count
);
571 /* Complete resource allocations. */
572 skreq
->state
= SKD_REQ_STATE_BUSY
;
574 skmsg
->length
+= sizeof(struct skd_scsi_request
);
575 fmh
->num_protocol_cmds_coalesced
++;
577 dev_dbg(&skdev
->pdev
->dev
, "req=0x%x busy=%d\n", skreq
->id
,
578 skd_in_flight(skdev
));
581 * If the FIT msg buffer is full send it.
583 if (skd_max_req_per_msg
== 1) {
584 skd_send_fitmsg(skdev
, skmsg
);
587 fmh
->num_protocol_cmds_coalesced
>= skd_max_req_per_msg
) {
588 skd_send_fitmsg(skdev
, skmsg
);
591 spin_unlock_irqrestore(&skdev
->lock
, flags
);
597 static enum blk_eh_timer_return
skd_timed_out(struct request
*req
,
600 struct skd_device
*skdev
= req
->q
->queuedata
;
602 dev_err(&skdev
->pdev
->dev
, "request with tag %#x timed out\n",
603 blk_mq_unique_tag(req
));
605 return BLK_EH_RESET_TIMER
;
608 static void skd_complete_rq(struct request
*req
)
610 struct skd_request_context
*skreq
= blk_mq_rq_to_pdu(req
);
612 blk_mq_end_request(req
, skreq
->status
);
615 static bool skd_preop_sg_list(struct skd_device
*skdev
,
616 struct skd_request_context
*skreq
)
618 struct request
*req
= blk_mq_rq_from_pdu(skreq
);
619 struct scatterlist
*sgl
= &skreq
->sg
[0], *sg
;
623 skreq
->sg_byte_count
= 0;
625 WARN_ON_ONCE(skreq
->data_dir
!= DMA_TO_DEVICE
&&
626 skreq
->data_dir
!= DMA_FROM_DEVICE
);
628 n_sg
= blk_rq_map_sg(skdev
->queue
, req
, sgl
);
633 * Map scatterlist to PCI bus addresses.
634 * Note PCI might change the number of entries.
636 n_sg
= pci_map_sg(skdev
->pdev
, sgl
, n_sg
, skreq
->data_dir
);
640 SKD_ASSERT(n_sg
<= skdev
->sgs_per_request
);
644 for_each_sg(sgl
, sg
, n_sg
, i
) {
645 struct fit_sg_descriptor
*sgd
= &skreq
->sksg_list
[i
];
646 u32 cnt
= sg_dma_len(sg
);
647 uint64_t dma_addr
= sg_dma_address(sg
);
649 sgd
->control
= FIT_SGD_CONTROL_NOT_LAST
;
650 sgd
->byte_count
= cnt
;
651 skreq
->sg_byte_count
+= cnt
;
652 sgd
->host_side_addr
= dma_addr
;
653 sgd
->dev_side_addr
= 0;
656 skreq
->sksg_list
[n_sg
- 1].next_desc_ptr
= 0LL;
657 skreq
->sksg_list
[n_sg
- 1].control
= FIT_SGD_CONTROL_LAST
;
659 if (unlikely(skdev
->dbg_level
> 1)) {
660 dev_dbg(&skdev
->pdev
->dev
,
661 "skreq=%x sksg_list=%p sksg_dma=%llx\n",
662 skreq
->id
, skreq
->sksg_list
, skreq
->sksg_dma_address
);
663 for (i
= 0; i
< n_sg
; i
++) {
664 struct fit_sg_descriptor
*sgd
= &skreq
->sksg_list
[i
];
666 dev_dbg(&skdev
->pdev
->dev
,
667 " sg[%d] count=%u ctrl=0x%x addr=0x%llx next=0x%llx\n",
668 i
, sgd
->byte_count
, sgd
->control
,
669 sgd
->host_side_addr
, sgd
->next_desc_ptr
);
676 static void skd_postop_sg_list(struct skd_device
*skdev
,
677 struct skd_request_context
*skreq
)
680 * restore the next ptr for next IO request so we
681 * don't have to set it every time.
683 skreq
->sksg_list
[skreq
->n_sg
- 1].next_desc_ptr
=
684 skreq
->sksg_dma_address
+
685 ((skreq
->n_sg
) * sizeof(struct fit_sg_descriptor
));
686 pci_unmap_sg(skdev
->pdev
, &skreq
->sg
[0], skreq
->n_sg
, skreq
->data_dir
);
690 *****************************************************************************
692 *****************************************************************************
695 static void skd_timer_tick_not_online(struct skd_device
*skdev
);
697 static void skd_start_queue(struct work_struct
*work
)
699 struct skd_device
*skdev
= container_of(work
, typeof(*skdev
),
703 * Although it is safe to call blk_start_queue() from interrupt
704 * context, blk_mq_start_hw_queues() must not be called from
707 blk_mq_start_hw_queues(skdev
->queue
);
710 static void skd_timer_tick(struct timer_list
*t
)
712 struct skd_device
*skdev
= from_timer(skdev
, t
, timer
);
713 unsigned long reqflags
;
716 if (skdev
->state
== SKD_DRVR_STATE_FAULT
)
717 /* The driver has declared fault, and we want it to
718 * stay that way until driver is reloaded.
722 spin_lock_irqsave(&skdev
->lock
, reqflags
);
724 state
= SKD_READL(skdev
, FIT_STATUS
);
725 state
&= FIT_SR_DRIVE_STATE_MASK
;
726 if (state
!= skdev
->drive_state
)
727 skd_isr_fwstate(skdev
);
729 if (skdev
->state
!= SKD_DRVR_STATE_ONLINE
)
730 skd_timer_tick_not_online(skdev
);
732 mod_timer(&skdev
->timer
, (jiffies
+ HZ
));
734 spin_unlock_irqrestore(&skdev
->lock
, reqflags
);
737 static void skd_timer_tick_not_online(struct skd_device
*skdev
)
739 switch (skdev
->state
) {
740 case SKD_DRVR_STATE_IDLE
:
741 case SKD_DRVR_STATE_LOAD
:
743 case SKD_DRVR_STATE_BUSY_SANITIZE
:
744 dev_dbg(&skdev
->pdev
->dev
,
745 "drive busy sanitize[%x], driver[%x]\n",
746 skdev
->drive_state
, skdev
->state
);
747 /* If we've been in sanitize for 3 seconds, we figure we're not
748 * going to get anymore completions, so recover requests now
750 if (skdev
->timer_countdown
> 0) {
751 skdev
->timer_countdown
--;
754 skd_recover_requests(skdev
);
757 case SKD_DRVR_STATE_BUSY
:
758 case SKD_DRVR_STATE_BUSY_IMMINENT
:
759 case SKD_DRVR_STATE_BUSY_ERASE
:
760 dev_dbg(&skdev
->pdev
->dev
, "busy[%x], countdown=%d\n",
761 skdev
->state
, skdev
->timer_countdown
);
762 if (skdev
->timer_countdown
> 0) {
763 skdev
->timer_countdown
--;
766 dev_dbg(&skdev
->pdev
->dev
,
767 "busy[%x], timedout=%d, restarting device.",
768 skdev
->state
, skdev
->timer_countdown
);
769 skd_restart_device(skdev
);
772 case SKD_DRVR_STATE_WAIT_BOOT
:
773 case SKD_DRVR_STATE_STARTING
:
774 if (skdev
->timer_countdown
> 0) {
775 skdev
->timer_countdown
--;
778 /* For now, we fault the drive. Could attempt resets to
779 * revcover at some point. */
780 skdev
->state
= SKD_DRVR_STATE_FAULT
;
782 dev_err(&skdev
->pdev
->dev
, "DriveFault Connect Timeout (%x)\n",
785 /*start the queue so we can respond with error to requests */
786 /* wakeup anyone waiting for startup complete */
787 schedule_work(&skdev
->start_queue
);
788 skdev
->gendisk_on
= -1;
789 wake_up_interruptible(&skdev
->waitq
);
792 case SKD_DRVR_STATE_ONLINE
:
793 /* shouldn't get here. */
796 case SKD_DRVR_STATE_PAUSING
:
797 case SKD_DRVR_STATE_PAUSED
:
800 case SKD_DRVR_STATE_RESTARTING
:
801 if (skdev
->timer_countdown
> 0) {
802 skdev
->timer_countdown
--;
805 /* For now, we fault the drive. Could attempt resets to
806 * revcover at some point. */
807 skdev
->state
= SKD_DRVR_STATE_FAULT
;
808 dev_err(&skdev
->pdev
->dev
,
809 "DriveFault Reconnect Timeout (%x)\n",
813 * Recovering does two things:
814 * 1. completes IO with error
815 * 2. reclaims dma resources
816 * When is it safe to recover requests?
817 * - if the drive state is faulted
818 * - if the state is still soft reset after out timeout
819 * - if the drive registers are dead (state = FF)
820 * If it is "unsafe", we still need to recover, so we will
821 * disable pci bus mastering and disable our interrupts.
824 if ((skdev
->drive_state
== FIT_SR_DRIVE_SOFT_RESET
) ||
825 (skdev
->drive_state
== FIT_SR_DRIVE_FAULT
) ||
826 (skdev
->drive_state
== FIT_SR_DRIVE_STATE_MASK
))
827 /* It never came out of soft reset. Try to
828 * recover the requests and then let them
829 * fail. This is to mitigate hung processes. */
830 skd_recover_requests(skdev
);
832 dev_err(&skdev
->pdev
->dev
, "Disable BusMaster (%x)\n",
834 pci_disable_device(skdev
->pdev
);
835 skd_disable_interrupts(skdev
);
836 skd_recover_requests(skdev
);
839 /*start the queue so we can respond with error to requests */
840 /* wakeup anyone waiting for startup complete */
841 schedule_work(&skdev
->start_queue
);
842 skdev
->gendisk_on
= -1;
843 wake_up_interruptible(&skdev
->waitq
);
846 case SKD_DRVR_STATE_RESUMING
:
847 case SKD_DRVR_STATE_STOPPING
:
848 case SKD_DRVR_STATE_SYNCING
:
849 case SKD_DRVR_STATE_FAULT
:
850 case SKD_DRVR_STATE_DISAPPEARED
:
856 static int skd_start_timer(struct skd_device
*skdev
)
860 timer_setup(&skdev
->timer
, skd_timer_tick
, 0);
862 rc
= mod_timer(&skdev
->timer
, (jiffies
+ HZ
));
864 dev_err(&skdev
->pdev
->dev
, "failed to start timer %d\n", rc
);
868 static void skd_kill_timer(struct skd_device
*skdev
)
870 del_timer_sync(&skdev
->timer
);
874 *****************************************************************************
875 * INTERNAL REQUESTS -- generated by driver itself
876 *****************************************************************************
879 static int skd_format_internal_skspcl(struct skd_device
*skdev
)
881 struct skd_special_context
*skspcl
= &skdev
->internal_skspcl
;
882 struct fit_sg_descriptor
*sgd
= &skspcl
->req
.sksg_list
[0];
883 struct fit_msg_hdr
*fmh
;
884 uint64_t dma_address
;
885 struct skd_scsi_request
*scsi
;
887 fmh
= &skspcl
->msg_buf
->fmh
;
888 fmh
->protocol_id
= FIT_PROTOCOL_ID_SOFIT
;
889 fmh
->num_protocol_cmds_coalesced
= 1;
891 scsi
= &skspcl
->msg_buf
->scsi
[0];
892 memset(scsi
, 0, sizeof(*scsi
));
893 dma_address
= skspcl
->req
.sksg_dma_address
;
894 scsi
->hdr
.sg_list_dma_address
= cpu_to_be64(dma_address
);
895 skspcl
->req
.n_sg
= 1;
896 sgd
->control
= FIT_SGD_CONTROL_LAST
;
898 sgd
->host_side_addr
= skspcl
->db_dma_address
;
899 sgd
->dev_side_addr
= 0;
900 sgd
->next_desc_ptr
= 0LL;
905 #define WR_BUF_SIZE SKD_N_INTERNAL_BYTES
907 static void skd_send_internal_skspcl(struct skd_device
*skdev
,
908 struct skd_special_context
*skspcl
,
911 struct fit_sg_descriptor
*sgd
= &skspcl
->req
.sksg_list
[0];
912 struct skd_scsi_request
*scsi
;
913 unsigned char *buf
= skspcl
->data_buf
;
916 if (skspcl
->req
.state
!= SKD_REQ_STATE_IDLE
)
918 * A refresh is already in progress.
919 * Just wait for it to finish.
923 skspcl
->req
.state
= SKD_REQ_STATE_BUSY
;
925 scsi
= &skspcl
->msg_buf
->scsi
[0];
926 scsi
->hdr
.tag
= skspcl
->req
.id
;
928 memset(scsi
->cdb
, 0, sizeof(scsi
->cdb
));
931 case TEST_UNIT_READY
:
932 scsi
->cdb
[0] = TEST_UNIT_READY
;
934 scsi
->hdr
.sg_list_len_bytes
= 0;
938 scsi
->cdb
[0] = READ_CAPACITY
;
939 sgd
->byte_count
= SKD_N_READ_CAP_BYTES
;
940 scsi
->hdr
.sg_list_len_bytes
= cpu_to_be32(sgd
->byte_count
);
944 scsi
->cdb
[0] = INQUIRY
;
945 scsi
->cdb
[1] = 0x01; /* evpd */
946 scsi
->cdb
[2] = 0x80; /* serial number page */
948 sgd
->byte_count
= 16;
949 scsi
->hdr
.sg_list_len_bytes
= cpu_to_be32(sgd
->byte_count
);
952 case SYNCHRONIZE_CACHE
:
953 scsi
->cdb
[0] = SYNCHRONIZE_CACHE
;
955 scsi
->hdr
.sg_list_len_bytes
= 0;
959 scsi
->cdb
[0] = WRITE_BUFFER
;
961 scsi
->cdb
[7] = (WR_BUF_SIZE
& 0xFF00) >> 8;
962 scsi
->cdb
[8] = WR_BUF_SIZE
& 0xFF;
963 sgd
->byte_count
= WR_BUF_SIZE
;
964 scsi
->hdr
.sg_list_len_bytes
= cpu_to_be32(sgd
->byte_count
);
965 /* fill incrementing byte pattern */
966 for (i
= 0; i
< sgd
->byte_count
; i
++)
971 scsi
->cdb
[0] = READ_BUFFER
;
973 scsi
->cdb
[7] = (WR_BUF_SIZE
& 0xFF00) >> 8;
974 scsi
->cdb
[8] = WR_BUF_SIZE
& 0xFF;
975 sgd
->byte_count
= WR_BUF_SIZE
;
976 scsi
->hdr
.sg_list_len_bytes
= cpu_to_be32(sgd
->byte_count
);
977 memset(skspcl
->data_buf
, 0, sgd
->byte_count
);
981 SKD_ASSERT("Don't know what to send");
985 skd_send_special_fitmsg(skdev
, skspcl
);
988 static void skd_refresh_device_data(struct skd_device
*skdev
)
990 struct skd_special_context
*skspcl
= &skdev
->internal_skspcl
;
992 skd_send_internal_skspcl(skdev
, skspcl
, TEST_UNIT_READY
);
995 static int skd_chk_read_buf(struct skd_device
*skdev
,
996 struct skd_special_context
*skspcl
)
998 unsigned char *buf
= skspcl
->data_buf
;
1001 /* check for incrementing byte pattern */
1002 for (i
= 0; i
< WR_BUF_SIZE
; i
++)
1003 if (buf
[i
] != (i
& 0xFF))
1009 static void skd_log_check_status(struct skd_device
*skdev
, u8 status
, u8 key
,
1010 u8 code
, u8 qual
, u8 fruc
)
1012 /* If the check condition is of special interest, log a message */
1013 if ((status
== SAM_STAT_CHECK_CONDITION
) && (key
== 0x02)
1014 && (code
== 0x04) && (qual
== 0x06)) {
1015 dev_err(&skdev
->pdev
->dev
,
1016 "*** LOST_WRITE_DATA ERROR *** key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
1017 key
, code
, qual
, fruc
);
1021 static void skd_complete_internal(struct skd_device
*skdev
,
1022 struct fit_completion_entry_v1
*skcomp
,
1023 struct fit_comp_error_info
*skerr
,
1024 struct skd_special_context
*skspcl
)
1026 u8
*buf
= skspcl
->data_buf
;
1029 struct skd_scsi_request
*scsi
= &skspcl
->msg_buf
->scsi
[0];
1031 lockdep_assert_held(&skdev
->lock
);
1033 SKD_ASSERT(skspcl
== &skdev
->internal_skspcl
);
1035 dev_dbg(&skdev
->pdev
->dev
, "complete internal %x\n", scsi
->cdb
[0]);
1037 dma_sync_single_for_cpu(&skdev
->pdev
->dev
,
1038 skspcl
->db_dma_address
,
1039 skspcl
->req
.sksg_list
[0].byte_count
,
1042 skspcl
->req
.completion
= *skcomp
;
1043 skspcl
->req
.state
= SKD_REQ_STATE_IDLE
;
1045 status
= skspcl
->req
.completion
.status
;
1047 skd_log_check_status(skdev
, status
, skerr
->key
, skerr
->code
,
1048 skerr
->qual
, skerr
->fruc
);
1050 switch (scsi
->cdb
[0]) {
1051 case TEST_UNIT_READY
:
1052 if (status
== SAM_STAT_GOOD
)
1053 skd_send_internal_skspcl(skdev
, skspcl
, WRITE_BUFFER
);
1054 else if ((status
== SAM_STAT_CHECK_CONDITION
) &&
1055 (skerr
->key
== MEDIUM_ERROR
))
1056 skd_send_internal_skspcl(skdev
, skspcl
, WRITE_BUFFER
);
1058 if (skdev
->state
== SKD_DRVR_STATE_STOPPING
) {
1059 dev_dbg(&skdev
->pdev
->dev
,
1060 "TUR failed, don't send anymore state 0x%x\n",
1064 dev_dbg(&skdev
->pdev
->dev
,
1065 "**** TUR failed, retry skerr\n");
1066 skd_send_internal_skspcl(skdev
, skspcl
,
1072 if (status
== SAM_STAT_GOOD
)
1073 skd_send_internal_skspcl(skdev
, skspcl
, READ_BUFFER
);
1075 if (skdev
->state
== SKD_DRVR_STATE_STOPPING
) {
1076 dev_dbg(&skdev
->pdev
->dev
,
1077 "write buffer failed, don't send anymore state 0x%x\n",
1081 dev_dbg(&skdev
->pdev
->dev
,
1082 "**** write buffer failed, retry skerr\n");
1083 skd_send_internal_skspcl(skdev
, skspcl
,
1089 if (status
== SAM_STAT_GOOD
) {
1090 if (skd_chk_read_buf(skdev
, skspcl
) == 0)
1091 skd_send_internal_skspcl(skdev
, skspcl
,
1094 dev_err(&skdev
->pdev
->dev
,
1095 "*** W/R Buffer mismatch %d ***\n",
1096 skdev
->connect_retries
);
1097 if (skdev
->connect_retries
<
1098 SKD_MAX_CONNECT_RETRIES
) {
1099 skdev
->connect_retries
++;
1100 skd_soft_reset(skdev
);
1102 dev_err(&skdev
->pdev
->dev
,
1103 "W/R Buffer Connect Error\n");
1109 if (skdev
->state
== SKD_DRVR_STATE_STOPPING
) {
1110 dev_dbg(&skdev
->pdev
->dev
,
1111 "read buffer failed, don't send anymore state 0x%x\n",
1115 dev_dbg(&skdev
->pdev
->dev
,
1116 "**** read buffer failed, retry skerr\n");
1117 skd_send_internal_skspcl(skdev
, skspcl
,
1123 skdev
->read_cap_is_valid
= 0;
1124 if (status
== SAM_STAT_GOOD
) {
1125 skdev
->read_cap_last_lba
=
1126 (buf
[0] << 24) | (buf
[1] << 16) |
1127 (buf
[2] << 8) | buf
[3];
1128 skdev
->read_cap_blocksize
=
1129 (buf
[4] << 24) | (buf
[5] << 16) |
1130 (buf
[6] << 8) | buf
[7];
1132 dev_dbg(&skdev
->pdev
->dev
, "last lba %d, bs %d\n",
1133 skdev
->read_cap_last_lba
,
1134 skdev
->read_cap_blocksize
);
1136 set_capacity(skdev
->disk
, skdev
->read_cap_last_lba
+ 1);
1138 skdev
->read_cap_is_valid
= 1;
1140 skd_send_internal_skspcl(skdev
, skspcl
, INQUIRY
);
1141 } else if ((status
== SAM_STAT_CHECK_CONDITION
) &&
1142 (skerr
->key
== MEDIUM_ERROR
)) {
1143 skdev
->read_cap_last_lba
= ~0;
1144 set_capacity(skdev
->disk
, skdev
->read_cap_last_lba
+ 1);
1145 dev_dbg(&skdev
->pdev
->dev
, "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n");
1146 skd_send_internal_skspcl(skdev
, skspcl
, INQUIRY
);
1148 dev_dbg(&skdev
->pdev
->dev
, "**** READCAP failed, retry TUR\n");
1149 skd_send_internal_skspcl(skdev
, skspcl
,
1155 skdev
->inquiry_is_valid
= 0;
1156 if (status
== SAM_STAT_GOOD
) {
1157 skdev
->inquiry_is_valid
= 1;
1159 for (i
= 0; i
< 12; i
++)
1160 skdev
->inq_serial_num
[i
] = buf
[i
+ 4];
1161 skdev
->inq_serial_num
[12] = 0;
1164 if (skd_unquiesce_dev(skdev
) < 0)
1165 dev_dbg(&skdev
->pdev
->dev
, "**** failed, to ONLINE device\n");
1166 /* connection is complete */
1167 skdev
->connect_retries
= 0;
1170 case SYNCHRONIZE_CACHE
:
1171 if (status
== SAM_STAT_GOOD
)
1172 skdev
->sync_done
= 1;
1174 skdev
->sync_done
= -1;
1175 wake_up_interruptible(&skdev
->waitq
);
1179 SKD_ASSERT("we didn't send this");
1184 *****************************************************************************
1186 *****************************************************************************
1189 static void skd_send_fitmsg(struct skd_device
*skdev
,
1190 struct skd_fitmsg_context
*skmsg
)
1194 dev_dbg(&skdev
->pdev
->dev
, "dma address 0x%llx, busy=%d\n",
1195 skmsg
->mb_dma_address
, skd_in_flight(skdev
));
1196 dev_dbg(&skdev
->pdev
->dev
, "msg_buf %p\n", skmsg
->msg_buf
);
1198 qcmd
= skmsg
->mb_dma_address
;
1199 qcmd
|= FIT_QCMD_QID_NORMAL
;
1201 if (unlikely(skdev
->dbg_level
> 1)) {
1202 u8
*bp
= (u8
*)skmsg
->msg_buf
;
1204 for (i
= 0; i
< skmsg
->length
; i
+= 8) {
1205 dev_dbg(&skdev
->pdev
->dev
, "msg[%2d] %8ph\n", i
,
1212 if (skmsg
->length
> 256)
1213 qcmd
|= FIT_QCMD_MSGSIZE_512
;
1214 else if (skmsg
->length
> 128)
1215 qcmd
|= FIT_QCMD_MSGSIZE_256
;
1216 else if (skmsg
->length
> 64)
1217 qcmd
|= FIT_QCMD_MSGSIZE_128
;
1220 * This makes no sense because the FIT msg header is
1221 * 64 bytes. If the msg is only 64 bytes long it has
1224 qcmd
|= FIT_QCMD_MSGSIZE_64
;
1226 dma_sync_single_for_device(&skdev
->pdev
->dev
, skmsg
->mb_dma_address
,
1227 skmsg
->length
, DMA_TO_DEVICE
);
1229 /* Make sure skd_msg_buf is written before the doorbell is triggered. */
1232 SKD_WRITEQ(skdev
, qcmd
, FIT_Q_COMMAND
);
1235 static void skd_send_special_fitmsg(struct skd_device
*skdev
,
1236 struct skd_special_context
*skspcl
)
1240 WARN_ON_ONCE(skspcl
->req
.n_sg
!= 1);
1242 if (unlikely(skdev
->dbg_level
> 1)) {
1243 u8
*bp
= (u8
*)skspcl
->msg_buf
;
1246 for (i
= 0; i
< SKD_N_SPECIAL_FITMSG_BYTES
; i
+= 8) {
1247 dev_dbg(&skdev
->pdev
->dev
, " spcl[%2d] %8ph\n", i
,
1253 dev_dbg(&skdev
->pdev
->dev
,
1254 "skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
1255 skspcl
, skspcl
->req
.id
, skspcl
->req
.sksg_list
,
1256 skspcl
->req
.sksg_dma_address
);
1257 for (i
= 0; i
< skspcl
->req
.n_sg
; i
++) {
1258 struct fit_sg_descriptor
*sgd
=
1259 &skspcl
->req
.sksg_list
[i
];
1261 dev_dbg(&skdev
->pdev
->dev
,
1262 " sg[%d] count=%u ctrl=0x%x addr=0x%llx next=0x%llx\n",
1263 i
, sgd
->byte_count
, sgd
->control
,
1264 sgd
->host_side_addr
, sgd
->next_desc_ptr
);
1269 * Special FIT msgs are always 128 bytes: a 64-byte FIT hdr
1270 * and one 64-byte SSDI command.
1272 qcmd
= skspcl
->mb_dma_address
;
1273 qcmd
|= FIT_QCMD_QID_NORMAL
+ FIT_QCMD_MSGSIZE_128
;
1275 dma_sync_single_for_device(&skdev
->pdev
->dev
, skspcl
->mb_dma_address
,
1276 SKD_N_SPECIAL_FITMSG_BYTES
, DMA_TO_DEVICE
);
1277 dma_sync_single_for_device(&skdev
->pdev
->dev
,
1278 skspcl
->req
.sksg_dma_address
,
1279 1 * sizeof(struct fit_sg_descriptor
),
1281 dma_sync_single_for_device(&skdev
->pdev
->dev
,
1282 skspcl
->db_dma_address
,
1283 skspcl
->req
.sksg_list
[0].byte_count
,
1286 /* Make sure skd_msg_buf is written before the doorbell is triggered. */
1289 SKD_WRITEQ(skdev
, qcmd
, FIT_Q_COMMAND
);
1293 *****************************************************************************
1295 *****************************************************************************
1298 static void skd_complete_other(struct skd_device
*skdev
,
1299 struct fit_completion_entry_v1
*skcomp
,
1300 struct fit_comp_error_info
*skerr
);
1309 enum skd_check_status_action action
;
1312 static struct sns_info skd_chkstat_table
[] = {
1314 { 0x70, 0x02, RECOVERED_ERROR
, 0, 0, 0x1c,
1315 SKD_CHECK_STATUS_REPORT_GOOD
},
1318 { 0x70, 0x02, NO_SENSE
, 0x0B, 0x00, 0x1E, /* warnings */
1319 SKD_CHECK_STATUS_REPORT_SMART_ALERT
},
1320 { 0x70, 0x02, NO_SENSE
, 0x5D, 0x00, 0x1E, /* thresholds */
1321 SKD_CHECK_STATUS_REPORT_SMART_ALERT
},
1322 { 0x70, 0x02, RECOVERED_ERROR
, 0x0B, 0x01, 0x1F, /* temperature over trigger */
1323 SKD_CHECK_STATUS_REPORT_SMART_ALERT
},
1325 /* Retry (with limits) */
1326 { 0x70, 0x02, 0x0B, 0, 0, 0x1C, /* This one is for DMA ERROR */
1327 SKD_CHECK_STATUS_REQUEUE_REQUEST
},
1328 { 0x70, 0x02, 0x06, 0x0B, 0x00, 0x1E, /* warnings */
1329 SKD_CHECK_STATUS_REQUEUE_REQUEST
},
1330 { 0x70, 0x02, 0x06, 0x5D, 0x00, 0x1E, /* thresholds */
1331 SKD_CHECK_STATUS_REQUEUE_REQUEST
},
1332 { 0x70, 0x02, 0x06, 0x80, 0x30, 0x1F, /* backup power */
1333 SKD_CHECK_STATUS_REQUEUE_REQUEST
},
1335 /* Busy (or about to be) */
1336 { 0x70, 0x02, 0x06, 0x3f, 0x01, 0x1F, /* fw changed */
1337 SKD_CHECK_STATUS_BUSY_IMMINENT
},
1341 * Look up status and sense data to decide how to handle the error
1343 * mask says which fields must match e.g., mask=0x18 means check
1344 * type and stat, ignore key, asc, ascq.
1347 static enum skd_check_status_action
1348 skd_check_status(struct skd_device
*skdev
,
1349 u8 cmp_status
, struct fit_comp_error_info
*skerr
)
1353 dev_err(&skdev
->pdev
->dev
, "key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
1354 skerr
->key
, skerr
->code
, skerr
->qual
, skerr
->fruc
);
1356 dev_dbg(&skdev
->pdev
->dev
,
1357 "stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
1358 skerr
->type
, cmp_status
, skerr
->key
, skerr
->code
, skerr
->qual
,
1361 /* Does the info match an entry in the good category? */
1362 for (i
= 0; i
< ARRAY_SIZE(skd_chkstat_table
); i
++) {
1363 struct sns_info
*sns
= &skd_chkstat_table
[i
];
1365 if (sns
->mask
& 0x10)
1366 if (skerr
->type
!= sns
->type
)
1369 if (sns
->mask
& 0x08)
1370 if (cmp_status
!= sns
->stat
)
1373 if (sns
->mask
& 0x04)
1374 if (skerr
->key
!= sns
->key
)
1377 if (sns
->mask
& 0x02)
1378 if (skerr
->code
!= sns
->asc
)
1381 if (sns
->mask
& 0x01)
1382 if (skerr
->qual
!= sns
->ascq
)
1385 if (sns
->action
== SKD_CHECK_STATUS_REPORT_SMART_ALERT
) {
1386 dev_err(&skdev
->pdev
->dev
,
1387 "SMART Alert: sense key/asc/ascq %02x/%02x/%02x\n",
1388 skerr
->key
, skerr
->code
, skerr
->qual
);
1393 /* No other match, so nonzero status means error,
1394 * zero status means good
1397 dev_dbg(&skdev
->pdev
->dev
, "status check: error\n");
1398 return SKD_CHECK_STATUS_REPORT_ERROR
;
1401 dev_dbg(&skdev
->pdev
->dev
, "status check good default\n");
1402 return SKD_CHECK_STATUS_REPORT_GOOD
;
1405 static void skd_resolve_req_exception(struct skd_device
*skdev
,
1406 struct skd_request_context
*skreq
,
1407 struct request
*req
)
1409 u8 cmp_status
= skreq
->completion
.status
;
1411 switch (skd_check_status(skdev
, cmp_status
, &skreq
->err_info
)) {
1412 case SKD_CHECK_STATUS_REPORT_GOOD
:
1413 case SKD_CHECK_STATUS_REPORT_SMART_ALERT
:
1414 skreq
->status
= BLK_STS_OK
;
1415 blk_mq_complete_request(req
);
1418 case SKD_CHECK_STATUS_BUSY_IMMINENT
:
1419 skd_log_skreq(skdev
, skreq
, "retry(busy)");
1420 blk_requeue_request(skdev
->queue
, req
);
1421 dev_info(&skdev
->pdev
->dev
, "drive BUSY imminent\n");
1422 skdev
->state
= SKD_DRVR_STATE_BUSY_IMMINENT
;
1423 skdev
->timer_countdown
= SKD_TIMER_MINUTES(20);
1424 skd_quiesce_dev(skdev
);
1427 case SKD_CHECK_STATUS_REQUEUE_REQUEST
:
1428 if ((unsigned long) ++req
->special
< SKD_MAX_RETRIES
) {
1429 skd_log_skreq(skdev
, skreq
, "retry");
1430 blk_requeue_request(skdev
->queue
, req
);
1435 case SKD_CHECK_STATUS_REPORT_ERROR
:
1437 skreq
->status
= BLK_STS_IOERR
;
1438 blk_mq_complete_request(req
);
1443 static void skd_release_skreq(struct skd_device
*skdev
,
1444 struct skd_request_context
*skreq
)
1447 * Reclaim the skd_request_context
1449 skreq
->state
= SKD_REQ_STATE_IDLE
;
1452 static int skd_isr_completion_posted(struct skd_device
*skdev
,
1453 int limit
, int *enqueued
)
1455 struct fit_completion_entry_v1
*skcmp
;
1456 struct fit_comp_error_info
*skerr
;
1461 struct skd_request_context
*skreq
;
1469 lockdep_assert_held(&skdev
->lock
);
1472 SKD_ASSERT(skdev
->skcomp_ix
< SKD_N_COMPLETION_ENTRY
);
1474 skcmp
= &skdev
->skcomp_table
[skdev
->skcomp_ix
];
1475 cmp_cycle
= skcmp
->cycle
;
1476 cmp_cntxt
= skcmp
->tag
;
1477 cmp_status
= skcmp
->status
;
1478 cmp_bytes
= be32_to_cpu(skcmp
->num_returned_bytes
);
1480 skerr
= &skdev
->skerr_table
[skdev
->skcomp_ix
];
1482 dev_dbg(&skdev
->pdev
->dev
,
1483 "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d busy=%d rbytes=0x%x proto=%d\n",
1484 skdev
->skcomp_cycle
, skdev
->skcomp_ix
, cmp_cycle
,
1485 cmp_cntxt
, cmp_status
, skd_in_flight(skdev
),
1486 cmp_bytes
, skdev
->proto_ver
);
1488 if (cmp_cycle
!= skdev
->skcomp_cycle
) {
1489 dev_dbg(&skdev
->pdev
->dev
, "end of completions\n");
1493 * Update the completion queue head index and possibly
1494 * the completion cycle count. 8-bit wrap-around.
1497 if (skdev
->skcomp_ix
>= SKD_N_COMPLETION_ENTRY
) {
1498 skdev
->skcomp_ix
= 0;
1499 skdev
->skcomp_cycle
++;
1503 * The command context is a unique 32-bit ID. The low order
1504 * bits help locate the request. The request is usually a
1505 * r/w request (see skd_start() above) or a special request.
1508 tag
= req_id
& SKD_ID_SLOT_AND_TABLE_MASK
;
1510 /* Is this other than a r/w request? */
1511 if (tag
>= skdev
->num_req_context
) {
1513 * This is not a completion for a r/w request.
1515 WARN_ON_ONCE(blk_mq_tag_to_rq(skdev
->tag_set
.tags
[hwq
],
1517 skd_complete_other(skdev
, skcmp
, skerr
);
1521 rq
= blk_mq_tag_to_rq(skdev
->tag_set
.tags
[hwq
], tag
);
1522 if (WARN(!rq
, "No request for tag %#x -> %#x\n", cmp_cntxt
,
1525 skreq
= blk_mq_rq_to_pdu(rq
);
1528 * Make sure the request ID for the slot matches.
1530 if (skreq
->id
!= req_id
) {
1531 dev_err(&skdev
->pdev
->dev
,
1532 "Completion mismatch comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
1533 req_id
, skreq
->id
, cmp_cntxt
);
1538 SKD_ASSERT(skreq
->state
== SKD_REQ_STATE_BUSY
);
1540 skreq
->completion
= *skcmp
;
1541 if (unlikely(cmp_status
== SAM_STAT_CHECK_CONDITION
)) {
1542 skreq
->err_info
= *skerr
;
1543 skd_log_check_status(skdev
, cmp_status
, skerr
->key
,
1544 skerr
->code
, skerr
->qual
,
1547 /* Release DMA resources for the request. */
1548 if (skreq
->n_sg
> 0)
1549 skd_postop_sg_list(skdev
, skreq
);
1551 skd_release_skreq(skdev
, skreq
);
1554 * Capture the outcome and post it back to the native request.
1556 if (likely(cmp_status
== SAM_STAT_GOOD
)) {
1557 skreq
->status
= BLK_STS_OK
;
1558 blk_mq_complete_request(rq
);
1560 skd_resolve_req_exception(skdev
, skreq
, rq
);
1563 /* skd_isr_comp_limit equal zero means no limit */
1565 if (++processed
>= limit
) {
1572 if (skdev
->state
== SKD_DRVR_STATE_PAUSING
&&
1573 skd_in_flight(skdev
) == 0) {
1574 skdev
->state
= SKD_DRVR_STATE_PAUSED
;
1575 wake_up_interruptible(&skdev
->waitq
);
1581 static void skd_complete_other(struct skd_device
*skdev
,
1582 struct fit_completion_entry_v1
*skcomp
,
1583 struct fit_comp_error_info
*skerr
)
1588 struct skd_special_context
*skspcl
;
1590 lockdep_assert_held(&skdev
->lock
);
1592 req_id
= skcomp
->tag
;
1593 req_table
= req_id
& SKD_ID_TABLE_MASK
;
1594 req_slot
= req_id
& SKD_ID_SLOT_MASK
;
1596 dev_dbg(&skdev
->pdev
->dev
, "table=0x%x id=0x%x slot=%d\n", req_table
,
1600 * Based on the request id, determine how to dispatch this completion.
1601 * This swich/case is finding the good cases and forwarding the
1602 * completion entry. Errors are reported below the switch.
1604 switch (req_table
) {
1605 case SKD_ID_RW_REQUEST
:
1607 * The caller, skd_isr_completion_posted() above,
1608 * handles r/w requests. The only way we get here
1609 * is if the req_slot is out of bounds.
1613 case SKD_ID_INTERNAL
:
1614 if (req_slot
== 0) {
1615 skspcl
= &skdev
->internal_skspcl
;
1616 if (skspcl
->req
.id
== req_id
&&
1617 skspcl
->req
.state
== SKD_REQ_STATE_BUSY
) {
1618 skd_complete_internal(skdev
,
1619 skcomp
, skerr
, skspcl
);
1625 case SKD_ID_FIT_MSG
:
1627 * These id's should never appear in a completion record.
1633 * These id's should never appear anywhere;
1639 * If we get here it is a bad or stale id.
1643 static void skd_reset_skcomp(struct skd_device
*skdev
)
1645 memset(skdev
->skcomp_table
, 0, SKD_SKCOMP_SIZE
);
1647 skdev
->skcomp_ix
= 0;
1648 skdev
->skcomp_cycle
= 1;
1652 *****************************************************************************
1654 *****************************************************************************
1656 static void skd_completion_worker(struct work_struct
*work
)
1658 struct skd_device
*skdev
=
1659 container_of(work
, struct skd_device
, completion_worker
);
1660 unsigned long flags
;
1661 int flush_enqueued
= 0;
1663 spin_lock_irqsave(&skdev
->lock
, flags
);
1666 * pass in limit=0, which means no limit..
1667 * process everything in compq
1669 skd_isr_completion_posted(skdev
, 0, &flush_enqueued
);
1670 schedule_work(&skdev
->start_queue
);
1672 spin_unlock_irqrestore(&skdev
->lock
, flags
);
1675 static void skd_isr_msg_from_dev(struct skd_device
*skdev
);
1678 skd_isr(int irq
, void *ptr
)
1680 struct skd_device
*skdev
= ptr
;
1685 int flush_enqueued
= 0;
1687 spin_lock(&skdev
->lock
);
1690 intstat
= SKD_READL(skdev
, FIT_INT_STATUS_HOST
);
1692 ack
= FIT_INT_DEF_MASK
;
1695 dev_dbg(&skdev
->pdev
->dev
, "intstat=0x%x ack=0x%x\n", intstat
,
1698 /* As long as there is an int pending on device, keep
1699 * running loop. When none, get out, but if we've never
1700 * done any processing, call completion handler?
1703 /* No interrupts on device, but run the completion
1707 if (likely (skdev
->state
1708 == SKD_DRVR_STATE_ONLINE
))
1715 SKD_WRITEL(skdev
, ack
, FIT_INT_STATUS_HOST
);
1717 if (likely((skdev
->state
!= SKD_DRVR_STATE_LOAD
) &&
1718 (skdev
->state
!= SKD_DRVR_STATE_STOPPING
))) {
1719 if (intstat
& FIT_ISH_COMPLETION_POSTED
) {
1721 * If we have already deferred completion
1722 * processing, don't bother running it again
1726 skd_isr_completion_posted(skdev
,
1727 skd_isr_comp_limit
, &flush_enqueued
);
1730 if (intstat
& FIT_ISH_FW_STATE_CHANGE
) {
1731 skd_isr_fwstate(skdev
);
1732 if (skdev
->state
== SKD_DRVR_STATE_FAULT
||
1734 SKD_DRVR_STATE_DISAPPEARED
) {
1735 spin_unlock(&skdev
->lock
);
1740 if (intstat
& FIT_ISH_MSG_FROM_DEV
)
1741 skd_isr_msg_from_dev(skdev
);
1745 if (unlikely(flush_enqueued
))
1746 schedule_work(&skdev
->start_queue
);
1749 schedule_work(&skdev
->completion_worker
);
1750 else if (!flush_enqueued
)
1751 schedule_work(&skdev
->start_queue
);
1753 spin_unlock(&skdev
->lock
);
1758 static void skd_drive_fault(struct skd_device
*skdev
)
1760 skdev
->state
= SKD_DRVR_STATE_FAULT
;
1761 dev_err(&skdev
->pdev
->dev
, "Drive FAULT\n");
1764 static void skd_drive_disappeared(struct skd_device
*skdev
)
1766 skdev
->state
= SKD_DRVR_STATE_DISAPPEARED
;
1767 dev_err(&skdev
->pdev
->dev
, "Drive DISAPPEARED\n");
1770 static void skd_isr_fwstate(struct skd_device
*skdev
)
1775 int prev_driver_state
= skdev
->state
;
1777 sense
= SKD_READL(skdev
, FIT_STATUS
);
1778 state
= sense
& FIT_SR_DRIVE_STATE_MASK
;
1780 dev_err(&skdev
->pdev
->dev
, "s1120 state %s(%d)=>%s(%d)\n",
1781 skd_drive_state_to_str(skdev
->drive_state
), skdev
->drive_state
,
1782 skd_drive_state_to_str(state
), state
);
1784 skdev
->drive_state
= state
;
1786 switch (skdev
->drive_state
) {
1787 case FIT_SR_DRIVE_INIT
:
1788 if (skdev
->state
== SKD_DRVR_STATE_PROTOCOL_MISMATCH
) {
1789 skd_disable_interrupts(skdev
);
1792 if (skdev
->state
== SKD_DRVR_STATE_RESTARTING
)
1793 skd_recover_requests(skdev
);
1794 if (skdev
->state
== SKD_DRVR_STATE_WAIT_BOOT
) {
1795 skdev
->timer_countdown
= SKD_STARTING_TIMO
;
1796 skdev
->state
= SKD_DRVR_STATE_STARTING
;
1797 skd_soft_reset(skdev
);
1800 mtd
= FIT_MXD_CONS(FIT_MTD_FITFW_INIT
, 0, 0);
1801 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1802 skdev
->last_mtd
= mtd
;
1805 case FIT_SR_DRIVE_ONLINE
:
1806 skdev
->cur_max_queue_depth
= skd_max_queue_depth
;
1807 if (skdev
->cur_max_queue_depth
> skdev
->dev_max_queue_depth
)
1808 skdev
->cur_max_queue_depth
= skdev
->dev_max_queue_depth
;
1810 skdev
->queue_low_water_mark
=
1811 skdev
->cur_max_queue_depth
* 2 / 3 + 1;
1812 if (skdev
->queue_low_water_mark
< 1)
1813 skdev
->queue_low_water_mark
= 1;
1814 dev_info(&skdev
->pdev
->dev
,
1815 "Queue depth limit=%d dev=%d lowat=%d\n",
1816 skdev
->cur_max_queue_depth
,
1817 skdev
->dev_max_queue_depth
,
1818 skdev
->queue_low_water_mark
);
1820 skd_refresh_device_data(skdev
);
1823 case FIT_SR_DRIVE_BUSY
:
1824 skdev
->state
= SKD_DRVR_STATE_BUSY
;
1825 skdev
->timer_countdown
= SKD_BUSY_TIMO
;
1826 skd_quiesce_dev(skdev
);
1828 case FIT_SR_DRIVE_BUSY_SANITIZE
:
1829 /* set timer for 3 seconds, we'll abort any unfinished
1830 * commands after that expires
1832 skdev
->state
= SKD_DRVR_STATE_BUSY_SANITIZE
;
1833 skdev
->timer_countdown
= SKD_TIMER_SECONDS(3);
1834 schedule_work(&skdev
->start_queue
);
1836 case FIT_SR_DRIVE_BUSY_ERASE
:
1837 skdev
->state
= SKD_DRVR_STATE_BUSY_ERASE
;
1838 skdev
->timer_countdown
= SKD_BUSY_TIMO
;
1840 case FIT_SR_DRIVE_OFFLINE
:
1841 skdev
->state
= SKD_DRVR_STATE_IDLE
;
1843 case FIT_SR_DRIVE_SOFT_RESET
:
1844 switch (skdev
->state
) {
1845 case SKD_DRVR_STATE_STARTING
:
1846 case SKD_DRVR_STATE_RESTARTING
:
1847 /* Expected by a caller of skd_soft_reset() */
1850 skdev
->state
= SKD_DRVR_STATE_RESTARTING
;
1854 case FIT_SR_DRIVE_FW_BOOTING
:
1855 dev_dbg(&skdev
->pdev
->dev
, "ISR FIT_SR_DRIVE_FW_BOOTING\n");
1856 skdev
->state
= SKD_DRVR_STATE_WAIT_BOOT
;
1857 skdev
->timer_countdown
= SKD_WAIT_BOOT_TIMO
;
1860 case FIT_SR_DRIVE_DEGRADED
:
1861 case FIT_SR_PCIE_LINK_DOWN
:
1862 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD
:
1865 case FIT_SR_DRIVE_FAULT
:
1866 skd_drive_fault(skdev
);
1867 skd_recover_requests(skdev
);
1868 schedule_work(&skdev
->start_queue
);
1871 /* PCIe bus returned all Fs? */
1873 dev_info(&skdev
->pdev
->dev
, "state=0x%x sense=0x%x\n", state
,
1875 skd_drive_disappeared(skdev
);
1876 skd_recover_requests(skdev
);
1877 schedule_work(&skdev
->start_queue
);
1881 * Uknown FW State. Wait for a state we recognize.
1885 dev_err(&skdev
->pdev
->dev
, "Driver state %s(%d)=>%s(%d)\n",
1886 skd_skdev_state_to_str(prev_driver_state
), prev_driver_state
,
1887 skd_skdev_state_to_str(skdev
->state
), skdev
->state
);
1890 static void skd_recover_request(struct request
*req
, void *data
, bool reserved
)
1892 struct skd_device
*const skdev
= data
;
1893 struct skd_request_context
*skreq
= blk_mq_rq_to_pdu(req
);
1895 if (skreq
->state
!= SKD_REQ_STATE_BUSY
)
1898 skd_log_skreq(skdev
, skreq
, "recover");
1900 /* Release DMA resources for the request. */
1901 if (skreq
->n_sg
> 0)
1902 skd_postop_sg_list(skdev
, skreq
);
1904 skreq
->state
= SKD_REQ_STATE_IDLE
;
1905 skreq
->status
= BLK_STS_IOERR
;
1906 blk_mq_complete_request(req
);
1909 static void skd_recover_requests(struct skd_device
*skdev
)
1911 blk_mq_tagset_busy_iter(&skdev
->tag_set
, skd_recover_request
, skdev
);
1914 static void skd_isr_msg_from_dev(struct skd_device
*skdev
)
1920 mfd
= SKD_READL(skdev
, FIT_MSG_FROM_DEVICE
);
1922 dev_dbg(&skdev
->pdev
->dev
, "mfd=0x%x last_mtd=0x%x\n", mfd
,
1925 /* ignore any mtd that is an ack for something we didn't send */
1926 if (FIT_MXD_TYPE(mfd
) != FIT_MXD_TYPE(skdev
->last_mtd
))
1929 switch (FIT_MXD_TYPE(mfd
)) {
1930 case FIT_MTD_FITFW_INIT
:
1931 skdev
->proto_ver
= FIT_PROTOCOL_MAJOR_VER(mfd
);
1933 if (skdev
->proto_ver
!= FIT_PROTOCOL_VERSION_1
) {
1934 dev_err(&skdev
->pdev
->dev
, "protocol mismatch\n");
1935 dev_err(&skdev
->pdev
->dev
, " got=%d support=%d\n",
1936 skdev
->proto_ver
, FIT_PROTOCOL_VERSION_1
);
1937 dev_err(&skdev
->pdev
->dev
, " please upgrade driver\n");
1938 skdev
->state
= SKD_DRVR_STATE_PROTOCOL_MISMATCH
;
1939 skd_soft_reset(skdev
);
1942 mtd
= FIT_MXD_CONS(FIT_MTD_GET_CMDQ_DEPTH
, 0, 0);
1943 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1944 skdev
->last_mtd
= mtd
;
1947 case FIT_MTD_GET_CMDQ_DEPTH
:
1948 skdev
->dev_max_queue_depth
= FIT_MXD_DATA(mfd
);
1949 mtd
= FIT_MXD_CONS(FIT_MTD_SET_COMPQ_DEPTH
, 0,
1950 SKD_N_COMPLETION_ENTRY
);
1951 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1952 skdev
->last_mtd
= mtd
;
1955 case FIT_MTD_SET_COMPQ_DEPTH
:
1956 SKD_WRITEQ(skdev
, skdev
->cq_dma_address
, FIT_MSG_TO_DEVICE_ARG
);
1957 mtd
= FIT_MXD_CONS(FIT_MTD_SET_COMPQ_ADDR
, 0, 0);
1958 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1959 skdev
->last_mtd
= mtd
;
1962 case FIT_MTD_SET_COMPQ_ADDR
:
1963 skd_reset_skcomp(skdev
);
1964 mtd
= FIT_MXD_CONS(FIT_MTD_CMD_LOG_HOST_ID
, 0, skdev
->devno
);
1965 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1966 skdev
->last_mtd
= mtd
;
1969 case FIT_MTD_CMD_LOG_HOST_ID
:
1970 /* hardware interface overflows in y2106 */
1971 skdev
->connect_time_stamp
= (u32
)ktime_get_real_seconds();
1972 data
= skdev
->connect_time_stamp
& 0xFFFF;
1973 mtd
= FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_LO
, 0, data
);
1974 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1975 skdev
->last_mtd
= mtd
;
1978 case FIT_MTD_CMD_LOG_TIME_STAMP_LO
:
1979 skdev
->drive_jiffies
= FIT_MXD_DATA(mfd
);
1980 data
= (skdev
->connect_time_stamp
>> 16) & 0xFFFF;
1981 mtd
= FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_HI
, 0, data
);
1982 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1983 skdev
->last_mtd
= mtd
;
1986 case FIT_MTD_CMD_LOG_TIME_STAMP_HI
:
1987 skdev
->drive_jiffies
|= (FIT_MXD_DATA(mfd
) << 16);
1988 mtd
= FIT_MXD_CONS(FIT_MTD_ARM_QUEUE
, 0, 0);
1989 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1990 skdev
->last_mtd
= mtd
;
1992 dev_err(&skdev
->pdev
->dev
, "Time sync driver=0x%x device=0x%x\n",
1993 skdev
->connect_time_stamp
, skdev
->drive_jiffies
);
1996 case FIT_MTD_ARM_QUEUE
:
1997 skdev
->last_mtd
= 0;
1999 * State should be, or soon will be, FIT_SR_DRIVE_ONLINE.
2008 static void skd_disable_interrupts(struct skd_device
*skdev
)
2012 sense
= SKD_READL(skdev
, FIT_CONTROL
);
2013 sense
&= ~FIT_CR_ENABLE_INTERRUPTS
;
2014 SKD_WRITEL(skdev
, sense
, FIT_CONTROL
);
2015 dev_dbg(&skdev
->pdev
->dev
, "sense 0x%x\n", sense
);
2017 /* Note that the 1s is written. A 1-bit means
2018 * disable, a 0 means enable.
2020 SKD_WRITEL(skdev
, ~0, FIT_INT_MASK_HOST
);
2023 static void skd_enable_interrupts(struct skd_device
*skdev
)
2027 /* unmask interrupts first */
2028 val
= FIT_ISH_FW_STATE_CHANGE
+
2029 FIT_ISH_COMPLETION_POSTED
+ FIT_ISH_MSG_FROM_DEV
;
2031 /* Note that the compliment of mask is written. A 1-bit means
2032 * disable, a 0 means enable. */
2033 SKD_WRITEL(skdev
, ~val
, FIT_INT_MASK_HOST
);
2034 dev_dbg(&skdev
->pdev
->dev
, "interrupt mask=0x%x\n", ~val
);
2036 val
= SKD_READL(skdev
, FIT_CONTROL
);
2037 val
|= FIT_CR_ENABLE_INTERRUPTS
;
2038 dev_dbg(&skdev
->pdev
->dev
, "control=0x%x\n", val
);
2039 SKD_WRITEL(skdev
, val
, FIT_CONTROL
);
2043 *****************************************************************************
2044 * START, STOP, RESTART, QUIESCE, UNQUIESCE
2045 *****************************************************************************
2048 static void skd_soft_reset(struct skd_device
*skdev
)
2052 val
= SKD_READL(skdev
, FIT_CONTROL
);
2053 val
|= (FIT_CR_SOFT_RESET
);
2054 dev_dbg(&skdev
->pdev
->dev
, "control=0x%x\n", val
);
2055 SKD_WRITEL(skdev
, val
, FIT_CONTROL
);
2058 static void skd_start_device(struct skd_device
*skdev
)
2060 unsigned long flags
;
2064 spin_lock_irqsave(&skdev
->lock
, flags
);
2066 /* ack all ghost interrupts */
2067 SKD_WRITEL(skdev
, FIT_INT_DEF_MASK
, FIT_INT_STATUS_HOST
);
2069 sense
= SKD_READL(skdev
, FIT_STATUS
);
2071 dev_dbg(&skdev
->pdev
->dev
, "initial status=0x%x\n", sense
);
2073 state
= sense
& FIT_SR_DRIVE_STATE_MASK
;
2074 skdev
->drive_state
= state
;
2075 skdev
->last_mtd
= 0;
2077 skdev
->state
= SKD_DRVR_STATE_STARTING
;
2078 skdev
->timer_countdown
= SKD_STARTING_TIMO
;
2080 skd_enable_interrupts(skdev
);
2082 switch (skdev
->drive_state
) {
2083 case FIT_SR_DRIVE_OFFLINE
:
2084 dev_err(&skdev
->pdev
->dev
, "Drive offline...\n");
2087 case FIT_SR_DRIVE_FW_BOOTING
:
2088 dev_dbg(&skdev
->pdev
->dev
, "FIT_SR_DRIVE_FW_BOOTING\n");
2089 skdev
->state
= SKD_DRVR_STATE_WAIT_BOOT
;
2090 skdev
->timer_countdown
= SKD_WAIT_BOOT_TIMO
;
2093 case FIT_SR_DRIVE_BUSY_SANITIZE
:
2094 dev_info(&skdev
->pdev
->dev
, "Start: BUSY_SANITIZE\n");
2095 skdev
->state
= SKD_DRVR_STATE_BUSY_SANITIZE
;
2096 skdev
->timer_countdown
= SKD_STARTED_BUSY_TIMO
;
2099 case FIT_SR_DRIVE_BUSY_ERASE
:
2100 dev_info(&skdev
->pdev
->dev
, "Start: BUSY_ERASE\n");
2101 skdev
->state
= SKD_DRVR_STATE_BUSY_ERASE
;
2102 skdev
->timer_countdown
= SKD_STARTED_BUSY_TIMO
;
2105 case FIT_SR_DRIVE_INIT
:
2106 case FIT_SR_DRIVE_ONLINE
:
2107 skd_soft_reset(skdev
);
2110 case FIT_SR_DRIVE_BUSY
:
2111 dev_err(&skdev
->pdev
->dev
, "Drive Busy...\n");
2112 skdev
->state
= SKD_DRVR_STATE_BUSY
;
2113 skdev
->timer_countdown
= SKD_STARTED_BUSY_TIMO
;
2116 case FIT_SR_DRIVE_SOFT_RESET
:
2117 dev_err(&skdev
->pdev
->dev
, "drive soft reset in prog\n");
2120 case FIT_SR_DRIVE_FAULT
:
2121 /* Fault state is bad...soft reset won't do it...
2122 * Hard reset, maybe, but does it work on device?
2123 * For now, just fault so the system doesn't hang.
2125 skd_drive_fault(skdev
);
2126 /*start the queue so we can respond with error to requests */
2127 dev_dbg(&skdev
->pdev
->dev
, "starting queue\n");
2128 schedule_work(&skdev
->start_queue
);
2129 skdev
->gendisk_on
= -1;
2130 wake_up_interruptible(&skdev
->waitq
);
2134 /* Most likely the device isn't there or isn't responding
2135 * to the BAR1 addresses. */
2136 skd_drive_disappeared(skdev
);
2137 /*start the queue so we can respond with error to requests */
2138 dev_dbg(&skdev
->pdev
->dev
,
2139 "starting queue to error-out reqs\n");
2140 schedule_work(&skdev
->start_queue
);
2141 skdev
->gendisk_on
= -1;
2142 wake_up_interruptible(&skdev
->waitq
);
2146 dev_err(&skdev
->pdev
->dev
, "Start: unknown state %x\n",
2147 skdev
->drive_state
);
2151 state
= SKD_READL(skdev
, FIT_CONTROL
);
2152 dev_dbg(&skdev
->pdev
->dev
, "FIT Control Status=0x%x\n", state
);
2154 state
= SKD_READL(skdev
, FIT_INT_STATUS_HOST
);
2155 dev_dbg(&skdev
->pdev
->dev
, "Intr Status=0x%x\n", state
);
2157 state
= SKD_READL(skdev
, FIT_INT_MASK_HOST
);
2158 dev_dbg(&skdev
->pdev
->dev
, "Intr Mask=0x%x\n", state
);
2160 state
= SKD_READL(skdev
, FIT_MSG_FROM_DEVICE
);
2161 dev_dbg(&skdev
->pdev
->dev
, "Msg from Dev=0x%x\n", state
);
2163 state
= SKD_READL(skdev
, FIT_HW_VERSION
);
2164 dev_dbg(&skdev
->pdev
->dev
, "HW version=0x%x\n", state
);
2166 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2169 static void skd_stop_device(struct skd_device
*skdev
)
2171 unsigned long flags
;
2172 struct skd_special_context
*skspcl
= &skdev
->internal_skspcl
;
2176 spin_lock_irqsave(&skdev
->lock
, flags
);
2178 if (skdev
->state
!= SKD_DRVR_STATE_ONLINE
) {
2179 dev_err(&skdev
->pdev
->dev
, "%s not online no sync\n", __func__
);
2183 if (skspcl
->req
.state
!= SKD_REQ_STATE_IDLE
) {
2184 dev_err(&skdev
->pdev
->dev
, "%s no special\n", __func__
);
2188 skdev
->state
= SKD_DRVR_STATE_SYNCING
;
2189 skdev
->sync_done
= 0;
2191 skd_send_internal_skspcl(skdev
, skspcl
, SYNCHRONIZE_CACHE
);
2193 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2195 wait_event_interruptible_timeout(skdev
->waitq
,
2196 (skdev
->sync_done
), (10 * HZ
));
2198 spin_lock_irqsave(&skdev
->lock
, flags
);
2200 switch (skdev
->sync_done
) {
2202 dev_err(&skdev
->pdev
->dev
, "%s no sync\n", __func__
);
2205 dev_err(&skdev
->pdev
->dev
, "%s sync done\n", __func__
);
2208 dev_err(&skdev
->pdev
->dev
, "%s sync error\n", __func__
);
2212 skdev
->state
= SKD_DRVR_STATE_STOPPING
;
2213 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2215 skd_kill_timer(skdev
);
2217 spin_lock_irqsave(&skdev
->lock
, flags
);
2218 skd_disable_interrupts(skdev
);
2220 /* ensure all ints on device are cleared */
2221 /* soft reset the device to unload with a clean slate */
2222 SKD_WRITEL(skdev
, FIT_INT_DEF_MASK
, FIT_INT_STATUS_HOST
);
2223 SKD_WRITEL(skdev
, FIT_CR_SOFT_RESET
, FIT_CONTROL
);
2225 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2227 /* poll every 100ms, 1 second timeout */
2228 for (i
= 0; i
< 10; i
++) {
2230 SKD_READL(skdev
, FIT_STATUS
) & FIT_SR_DRIVE_STATE_MASK
;
2231 if (dev_state
== FIT_SR_DRIVE_INIT
)
2233 set_current_state(TASK_INTERRUPTIBLE
);
2234 schedule_timeout(msecs_to_jiffies(100));
2237 if (dev_state
!= FIT_SR_DRIVE_INIT
)
2238 dev_err(&skdev
->pdev
->dev
, "%s state error 0x%02x\n", __func__
,
2242 /* assume spinlock is held */
2243 static void skd_restart_device(struct skd_device
*skdev
)
2247 /* ack all ghost interrupts */
2248 SKD_WRITEL(skdev
, FIT_INT_DEF_MASK
, FIT_INT_STATUS_HOST
);
2250 state
= SKD_READL(skdev
, FIT_STATUS
);
2252 dev_dbg(&skdev
->pdev
->dev
, "drive status=0x%x\n", state
);
2254 state
&= FIT_SR_DRIVE_STATE_MASK
;
2255 skdev
->drive_state
= state
;
2256 skdev
->last_mtd
= 0;
2258 skdev
->state
= SKD_DRVR_STATE_RESTARTING
;
2259 skdev
->timer_countdown
= SKD_RESTARTING_TIMO
;
2261 skd_soft_reset(skdev
);
2264 /* assume spinlock is held */
2265 static int skd_quiesce_dev(struct skd_device
*skdev
)
2269 switch (skdev
->state
) {
2270 case SKD_DRVR_STATE_BUSY
:
2271 case SKD_DRVR_STATE_BUSY_IMMINENT
:
2272 dev_dbg(&skdev
->pdev
->dev
, "stopping queue\n");
2273 blk_mq_stop_hw_queues(skdev
->queue
);
2275 case SKD_DRVR_STATE_ONLINE
:
2276 case SKD_DRVR_STATE_STOPPING
:
2277 case SKD_DRVR_STATE_SYNCING
:
2278 case SKD_DRVR_STATE_PAUSING
:
2279 case SKD_DRVR_STATE_PAUSED
:
2280 case SKD_DRVR_STATE_STARTING
:
2281 case SKD_DRVR_STATE_RESTARTING
:
2282 case SKD_DRVR_STATE_RESUMING
:
2285 dev_dbg(&skdev
->pdev
->dev
, "state [%d] not implemented\n",
2291 /* assume spinlock is held */
2292 static int skd_unquiesce_dev(struct skd_device
*skdev
)
2294 int prev_driver_state
= skdev
->state
;
2296 skd_log_skdev(skdev
, "unquiesce");
2297 if (skdev
->state
== SKD_DRVR_STATE_ONLINE
) {
2298 dev_dbg(&skdev
->pdev
->dev
, "**** device already ONLINE\n");
2301 if (skdev
->drive_state
!= FIT_SR_DRIVE_ONLINE
) {
2303 * If there has been an state change to other than
2304 * ONLINE, we will rely on controller state change
2305 * to come back online and restart the queue.
2306 * The BUSY state means that driver is ready to
2307 * continue normal processing but waiting for controller
2308 * to become available.
2310 skdev
->state
= SKD_DRVR_STATE_BUSY
;
2311 dev_dbg(&skdev
->pdev
->dev
, "drive BUSY state\n");
2316 * Drive has just come online, driver is either in startup,
2317 * paused performing a task, or bust waiting for hardware.
2319 switch (skdev
->state
) {
2320 case SKD_DRVR_STATE_PAUSED
:
2321 case SKD_DRVR_STATE_BUSY
:
2322 case SKD_DRVR_STATE_BUSY_IMMINENT
:
2323 case SKD_DRVR_STATE_BUSY_ERASE
:
2324 case SKD_DRVR_STATE_STARTING
:
2325 case SKD_DRVR_STATE_RESTARTING
:
2326 case SKD_DRVR_STATE_FAULT
:
2327 case SKD_DRVR_STATE_IDLE
:
2328 case SKD_DRVR_STATE_LOAD
:
2329 skdev
->state
= SKD_DRVR_STATE_ONLINE
;
2330 dev_err(&skdev
->pdev
->dev
, "Driver state %s(%d)=>%s(%d)\n",
2331 skd_skdev_state_to_str(prev_driver_state
),
2332 prev_driver_state
, skd_skdev_state_to_str(skdev
->state
),
2334 dev_dbg(&skdev
->pdev
->dev
,
2335 "**** device ONLINE...starting block queue\n");
2336 dev_dbg(&skdev
->pdev
->dev
, "starting queue\n");
2337 dev_info(&skdev
->pdev
->dev
, "STEC s1120 ONLINE\n");
2338 schedule_work(&skdev
->start_queue
);
2339 skdev
->gendisk_on
= 1;
2340 wake_up_interruptible(&skdev
->waitq
);
2343 case SKD_DRVR_STATE_DISAPPEARED
:
2345 dev_dbg(&skdev
->pdev
->dev
,
2346 "**** driver state %d, not implemented\n",
2354 *****************************************************************************
2355 * PCIe MSI/MSI-X INTERRUPT HANDLERS
2356 *****************************************************************************
2359 static irqreturn_t
skd_reserved_isr(int irq
, void *skd_host_data
)
2361 struct skd_device
*skdev
= skd_host_data
;
2362 unsigned long flags
;
2364 spin_lock_irqsave(&skdev
->lock
, flags
);
2365 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2366 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2367 dev_err(&skdev
->pdev
->dev
, "MSIX reserved irq %d = 0x%x\n", irq
,
2368 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2369 SKD_WRITEL(skdev
, FIT_INT_RESERVED_MASK
, FIT_INT_STATUS_HOST
);
2370 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2374 static irqreturn_t
skd_statec_isr(int irq
, void *skd_host_data
)
2376 struct skd_device
*skdev
= skd_host_data
;
2377 unsigned long flags
;
2379 spin_lock_irqsave(&skdev
->lock
, flags
);
2380 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2381 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2382 SKD_WRITEL(skdev
, FIT_ISH_FW_STATE_CHANGE
, FIT_INT_STATUS_HOST
);
2383 skd_isr_fwstate(skdev
);
2384 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2388 static irqreturn_t
skd_comp_q(int irq
, void *skd_host_data
)
2390 struct skd_device
*skdev
= skd_host_data
;
2391 unsigned long flags
;
2392 int flush_enqueued
= 0;
2395 spin_lock_irqsave(&skdev
->lock
, flags
);
2396 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2397 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2398 SKD_WRITEL(skdev
, FIT_ISH_COMPLETION_POSTED
, FIT_INT_STATUS_HOST
);
2399 deferred
= skd_isr_completion_posted(skdev
, skd_isr_comp_limit
,
2402 schedule_work(&skdev
->start_queue
);
2405 schedule_work(&skdev
->completion_worker
);
2406 else if (!flush_enqueued
)
2407 schedule_work(&skdev
->start_queue
);
2409 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2414 static irqreturn_t
skd_msg_isr(int irq
, void *skd_host_data
)
2416 struct skd_device
*skdev
= skd_host_data
;
2417 unsigned long flags
;
2419 spin_lock_irqsave(&skdev
->lock
, flags
);
2420 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2421 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2422 SKD_WRITEL(skdev
, FIT_ISH_MSG_FROM_DEV
, FIT_INT_STATUS_HOST
);
2423 skd_isr_msg_from_dev(skdev
);
2424 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2428 static irqreturn_t
skd_qfull_isr(int irq
, void *skd_host_data
)
2430 struct skd_device
*skdev
= skd_host_data
;
2431 unsigned long flags
;
2433 spin_lock_irqsave(&skdev
->lock
, flags
);
2434 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2435 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2436 SKD_WRITEL(skdev
, FIT_INT_QUEUE_FULL
, FIT_INT_STATUS_HOST
);
2437 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2442 *****************************************************************************
2443 * PCIe MSI/MSI-X SETUP
2444 *****************************************************************************
2447 struct skd_msix_entry
{
2451 struct skd_init_msix_entry
{
2453 irq_handler_t handler
;
2456 #define SKD_MAX_MSIX_COUNT 13
2457 #define SKD_MIN_MSIX_COUNT 7
2458 #define SKD_BASE_MSIX_IRQ 4
2460 static struct skd_init_msix_entry msix_entries
[SKD_MAX_MSIX_COUNT
] = {
2461 { "(DMA 0)", skd_reserved_isr
},
2462 { "(DMA 1)", skd_reserved_isr
},
2463 { "(DMA 2)", skd_reserved_isr
},
2464 { "(DMA 3)", skd_reserved_isr
},
2465 { "(State Change)", skd_statec_isr
},
2466 { "(COMPL_Q)", skd_comp_q
},
2467 { "(MSG)", skd_msg_isr
},
2468 { "(Reserved)", skd_reserved_isr
},
2469 { "(Reserved)", skd_reserved_isr
},
2470 { "(Queue Full 0)", skd_qfull_isr
},
2471 { "(Queue Full 1)", skd_qfull_isr
},
2472 { "(Queue Full 2)", skd_qfull_isr
},
2473 { "(Queue Full 3)", skd_qfull_isr
},
2476 static int skd_acquire_msix(struct skd_device
*skdev
)
2479 struct pci_dev
*pdev
= skdev
->pdev
;
2481 rc
= pci_alloc_irq_vectors(pdev
, SKD_MAX_MSIX_COUNT
, SKD_MAX_MSIX_COUNT
,
2484 dev_err(&skdev
->pdev
->dev
, "failed to enable MSI-X %d\n", rc
);
2488 skdev
->msix_entries
= kcalloc(SKD_MAX_MSIX_COUNT
,
2489 sizeof(struct skd_msix_entry
), GFP_KERNEL
);
2490 if (!skdev
->msix_entries
) {
2492 dev_err(&skdev
->pdev
->dev
, "msix table allocation error\n");
2496 /* Enable MSI-X vectors for the base queue */
2497 for (i
= 0; i
< SKD_MAX_MSIX_COUNT
; i
++) {
2498 struct skd_msix_entry
*qentry
= &skdev
->msix_entries
[i
];
2500 snprintf(qentry
->isr_name
, sizeof(qentry
->isr_name
),
2501 "%s%d-msix %s", DRV_NAME
, skdev
->devno
,
2502 msix_entries
[i
].name
);
2504 rc
= devm_request_irq(&skdev
->pdev
->dev
,
2505 pci_irq_vector(skdev
->pdev
, i
),
2506 msix_entries
[i
].handler
, 0,
2507 qentry
->isr_name
, skdev
);
2509 dev_err(&skdev
->pdev
->dev
,
2510 "Unable to register(%d) MSI-X handler %d: %s\n",
2511 rc
, i
, qentry
->isr_name
);
2516 dev_dbg(&skdev
->pdev
->dev
, "%d msix irq(s) enabled\n",
2517 SKD_MAX_MSIX_COUNT
);
2522 devm_free_irq(&pdev
->dev
, pci_irq_vector(pdev
, i
), skdev
);
2524 kfree(skdev
->msix_entries
);
2525 skdev
->msix_entries
= NULL
;
2529 static int skd_acquire_irq(struct skd_device
*skdev
)
2531 struct pci_dev
*pdev
= skdev
->pdev
;
2532 unsigned int irq_flag
= PCI_IRQ_LEGACY
;
2535 if (skd_isr_type
== SKD_IRQ_MSIX
) {
2536 rc
= skd_acquire_msix(skdev
);
2540 dev_err(&skdev
->pdev
->dev
,
2541 "failed to enable MSI-X, re-trying with MSI %d\n", rc
);
2544 snprintf(skdev
->isr_name
, sizeof(skdev
->isr_name
), "%s%d", DRV_NAME
,
2547 if (skd_isr_type
!= SKD_IRQ_LEGACY
)
2548 irq_flag
|= PCI_IRQ_MSI
;
2549 rc
= pci_alloc_irq_vectors(pdev
, 1, 1, irq_flag
);
2551 dev_err(&skdev
->pdev
->dev
,
2552 "failed to allocate the MSI interrupt %d\n", rc
);
2556 rc
= devm_request_irq(&pdev
->dev
, pdev
->irq
, skd_isr
,
2557 pdev
->msi_enabled
? 0 : IRQF_SHARED
,
2558 skdev
->isr_name
, skdev
);
2560 pci_free_irq_vectors(pdev
);
2561 dev_err(&skdev
->pdev
->dev
, "failed to allocate interrupt %d\n",
2569 static void skd_release_irq(struct skd_device
*skdev
)
2571 struct pci_dev
*pdev
= skdev
->pdev
;
2573 if (skdev
->msix_entries
) {
2576 for (i
= 0; i
< SKD_MAX_MSIX_COUNT
; i
++) {
2577 devm_free_irq(&pdev
->dev
, pci_irq_vector(pdev
, i
),
2581 kfree(skdev
->msix_entries
);
2582 skdev
->msix_entries
= NULL
;
2584 devm_free_irq(&pdev
->dev
, pdev
->irq
, skdev
);
2587 pci_free_irq_vectors(pdev
);
2591 *****************************************************************************
2593 *****************************************************************************
2596 static void *skd_alloc_dma(struct skd_device
*skdev
, struct kmem_cache
*s
,
2597 dma_addr_t
*dma_handle
, gfp_t gfp
,
2598 enum dma_data_direction dir
)
2600 struct device
*dev
= &skdev
->pdev
->dev
;
2603 buf
= kmem_cache_alloc(s
, gfp
);
2606 *dma_handle
= dma_map_single(dev
, buf
, s
->size
, dir
);
2607 if (dma_mapping_error(dev
, *dma_handle
)) {
2608 kmem_cache_free(s
, buf
);
2614 static void skd_free_dma(struct skd_device
*skdev
, struct kmem_cache
*s
,
2615 void *vaddr
, dma_addr_t dma_handle
,
2616 enum dma_data_direction dir
)
2621 dma_unmap_single(&skdev
->pdev
->dev
, dma_handle
, s
->size
, dir
);
2622 kmem_cache_free(s
, vaddr
);
2625 static int skd_cons_skcomp(struct skd_device
*skdev
)
2628 struct fit_completion_entry_v1
*skcomp
;
2630 dev_dbg(&skdev
->pdev
->dev
,
2631 "comp pci_alloc, total bytes %zd entries %d\n",
2632 SKD_SKCOMP_SIZE
, SKD_N_COMPLETION_ENTRY
);
2634 skcomp
= pci_zalloc_consistent(skdev
->pdev
, SKD_SKCOMP_SIZE
,
2635 &skdev
->cq_dma_address
);
2637 if (skcomp
== NULL
) {
2642 skdev
->skcomp_table
= skcomp
;
2643 skdev
->skerr_table
= (struct fit_comp_error_info
*)((char *)skcomp
+
2645 SKD_N_COMPLETION_ENTRY
);
2651 static int skd_cons_skmsg(struct skd_device
*skdev
)
2656 dev_dbg(&skdev
->pdev
->dev
,
2657 "skmsg_table kcalloc, struct %lu, count %u total %lu\n",
2658 sizeof(struct skd_fitmsg_context
), skdev
->num_fitmsg_context
,
2659 sizeof(struct skd_fitmsg_context
) * skdev
->num_fitmsg_context
);
2661 skdev
->skmsg_table
= kcalloc(skdev
->num_fitmsg_context
,
2662 sizeof(struct skd_fitmsg_context
),
2664 if (skdev
->skmsg_table
== NULL
) {
2669 for (i
= 0; i
< skdev
->num_fitmsg_context
; i
++) {
2670 struct skd_fitmsg_context
*skmsg
;
2672 skmsg
= &skdev
->skmsg_table
[i
];
2674 skmsg
->id
= i
+ SKD_ID_FIT_MSG
;
2676 skmsg
->msg_buf
= pci_alloc_consistent(skdev
->pdev
,
2678 &skmsg
->mb_dma_address
);
2680 if (skmsg
->msg_buf
== NULL
) {
2685 WARN(((uintptr_t)skmsg
->msg_buf
| skmsg
->mb_dma_address
) &
2686 (FIT_QCMD_ALIGN
- 1),
2687 "not aligned: msg_buf %p mb_dma_address %#llx\n",
2688 skmsg
->msg_buf
, skmsg
->mb_dma_address
);
2689 memset(skmsg
->msg_buf
, 0, SKD_N_FITMSG_BYTES
);
2696 static struct fit_sg_descriptor
*skd_cons_sg_list(struct skd_device
*skdev
,
2698 dma_addr_t
*ret_dma_addr
)
2700 struct fit_sg_descriptor
*sg_list
;
2702 sg_list
= skd_alloc_dma(skdev
, skdev
->sglist_cache
, ret_dma_addr
,
2703 GFP_DMA
| __GFP_ZERO
, DMA_TO_DEVICE
);
2705 if (sg_list
!= NULL
) {
2706 uint64_t dma_address
= *ret_dma_addr
;
2709 for (i
= 0; i
< n_sg
- 1; i
++) {
2711 ndp_off
= (i
+ 1) * sizeof(struct fit_sg_descriptor
);
2713 sg_list
[i
].next_desc_ptr
= dma_address
+ ndp_off
;
2715 sg_list
[i
].next_desc_ptr
= 0LL;
2721 static void skd_free_sg_list(struct skd_device
*skdev
,
2722 struct fit_sg_descriptor
*sg_list
,
2723 dma_addr_t dma_addr
)
2725 if (WARN_ON_ONCE(!sg_list
))
2728 skd_free_dma(skdev
, skdev
->sglist_cache
, sg_list
, dma_addr
,
2732 static int skd_init_request(struct blk_mq_tag_set
*set
, struct request
*rq
,
2733 unsigned int hctx_idx
, unsigned int numa_node
)
2735 struct skd_device
*skdev
= set
->driver_data
;
2736 struct skd_request_context
*skreq
= blk_mq_rq_to_pdu(rq
);
2738 skreq
->state
= SKD_REQ_STATE_IDLE
;
2739 skreq
->sg
= (void *)(skreq
+ 1);
2740 sg_init_table(skreq
->sg
, skd_sgs_per_request
);
2741 skreq
->sksg_list
= skd_cons_sg_list(skdev
, skd_sgs_per_request
,
2742 &skreq
->sksg_dma_address
);
2744 return skreq
->sksg_list
? 0 : -ENOMEM
;
2747 static void skd_exit_request(struct blk_mq_tag_set
*set
, struct request
*rq
,
2748 unsigned int hctx_idx
)
2750 struct skd_device
*skdev
= set
->driver_data
;
2751 struct skd_request_context
*skreq
= blk_mq_rq_to_pdu(rq
);
2753 skd_free_sg_list(skdev
, skreq
->sksg_list
, skreq
->sksg_dma_address
);
2756 static int skd_cons_sksb(struct skd_device
*skdev
)
2759 struct skd_special_context
*skspcl
;
2761 skspcl
= &skdev
->internal_skspcl
;
2763 skspcl
->req
.id
= 0 + SKD_ID_INTERNAL
;
2764 skspcl
->req
.state
= SKD_REQ_STATE_IDLE
;
2766 skspcl
->data_buf
= skd_alloc_dma(skdev
, skdev
->databuf_cache
,
2767 &skspcl
->db_dma_address
,
2768 GFP_DMA
| __GFP_ZERO
,
2770 if (skspcl
->data_buf
== NULL
) {
2775 skspcl
->msg_buf
= skd_alloc_dma(skdev
, skdev
->msgbuf_cache
,
2776 &skspcl
->mb_dma_address
,
2777 GFP_DMA
| __GFP_ZERO
, DMA_TO_DEVICE
);
2778 if (skspcl
->msg_buf
== NULL
) {
2783 skspcl
->req
.sksg_list
= skd_cons_sg_list(skdev
, 1,
2784 &skspcl
->req
.sksg_dma_address
);
2785 if (skspcl
->req
.sksg_list
== NULL
) {
2790 if (!skd_format_internal_skspcl(skdev
)) {
2799 static const struct blk_mq_ops skd_mq_ops
= {
2800 .queue_rq
= skd_mq_queue_rq
,
2801 .complete
= skd_complete_rq
,
2802 .timeout
= skd_timed_out
,
2803 .init_request
= skd_init_request
,
2804 .exit_request
= skd_exit_request
,
2807 static int skd_cons_disk(struct skd_device
*skdev
)
2810 struct gendisk
*disk
;
2811 struct request_queue
*q
;
2812 unsigned long flags
;
2814 disk
= alloc_disk(SKD_MINORS_PER_DEVICE
);
2821 sprintf(disk
->disk_name
, DRV_NAME
"%u", skdev
->devno
);
2823 disk
->major
= skdev
->major
;
2824 disk
->first_minor
= skdev
->devno
* SKD_MINORS_PER_DEVICE
;
2825 disk
->fops
= &skd_blockdev_ops
;
2826 disk
->private_data
= skdev
;
2828 memset(&skdev
->tag_set
, 0, sizeof(skdev
->tag_set
));
2829 skdev
->tag_set
.ops
= &skd_mq_ops
;
2830 skdev
->tag_set
.nr_hw_queues
= 1;
2831 skdev
->tag_set
.queue_depth
= skd_max_queue_depth
;
2832 skdev
->tag_set
.cmd_size
= sizeof(struct skd_request_context
) +
2833 skdev
->sgs_per_request
* sizeof(struct scatterlist
);
2834 skdev
->tag_set
.numa_node
= NUMA_NO_NODE
;
2835 skdev
->tag_set
.flags
= BLK_MQ_F_SHOULD_MERGE
|
2837 BLK_ALLOC_POLICY_TO_MQ_FLAG(BLK_TAG_ALLOC_FIFO
);
2838 skdev
->tag_set
.driver_data
= skdev
;
2839 rc
= blk_mq_alloc_tag_set(&skdev
->tag_set
);
2842 q
= blk_mq_init_queue(&skdev
->tag_set
);
2844 blk_mq_free_tag_set(&skdev
->tag_set
);
2848 q
->queuedata
= skdev
;
2853 blk_queue_write_cache(q
, true, true);
2854 blk_queue_max_segments(q
, skdev
->sgs_per_request
);
2855 blk_queue_max_hw_sectors(q
, SKD_N_MAX_SECTORS
);
2857 /* set optimal I/O size to 8KB */
2858 blk_queue_io_opt(q
, 8192);
2860 queue_flag_set_unlocked(QUEUE_FLAG_NONROT
, q
);
2861 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM
, q
);
2863 blk_queue_rq_timeout(q
, 8 * HZ
);
2865 spin_lock_irqsave(&skdev
->lock
, flags
);
2866 dev_dbg(&skdev
->pdev
->dev
, "stopping queue\n");
2867 blk_mq_stop_hw_queues(skdev
->queue
);
2868 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2874 #define SKD_N_DEV_TABLE 16u
2875 static u32 skd_next_devno
;
2877 static struct skd_device
*skd_construct(struct pci_dev
*pdev
)
2879 struct skd_device
*skdev
;
2880 int blk_major
= skd_major
;
2884 skdev
= kzalloc(sizeof(*skdev
), GFP_KERNEL
);
2887 dev_err(&pdev
->dev
, "memory alloc failure\n");
2891 skdev
->state
= SKD_DRVR_STATE_LOAD
;
2893 skdev
->devno
= skd_next_devno
++;
2894 skdev
->major
= blk_major
;
2895 skdev
->dev_max_queue_depth
= 0;
2897 skdev
->num_req_context
= skd_max_queue_depth
;
2898 skdev
->num_fitmsg_context
= skd_max_queue_depth
;
2899 skdev
->cur_max_queue_depth
= 1;
2900 skdev
->queue_low_water_mark
= 1;
2901 skdev
->proto_ver
= 99;
2902 skdev
->sgs_per_request
= skd_sgs_per_request
;
2903 skdev
->dbg_level
= skd_dbg_level
;
2905 spin_lock_init(&skdev
->lock
);
2907 INIT_WORK(&skdev
->start_queue
, skd_start_queue
);
2908 INIT_WORK(&skdev
->completion_worker
, skd_completion_worker
);
2910 size
= max(SKD_N_FITMSG_BYTES
, SKD_N_SPECIAL_FITMSG_BYTES
);
2911 skdev
->msgbuf_cache
= kmem_cache_create("skd-msgbuf", size
, 0,
2912 SLAB_HWCACHE_ALIGN
, NULL
);
2913 if (!skdev
->msgbuf_cache
)
2915 WARN_ONCE(kmem_cache_size(skdev
->msgbuf_cache
) < size
,
2916 "skd-msgbuf: %d < %zd\n",
2917 kmem_cache_size(skdev
->msgbuf_cache
), size
);
2918 size
= skd_sgs_per_request
* sizeof(struct fit_sg_descriptor
);
2919 skdev
->sglist_cache
= kmem_cache_create("skd-sglist", size
, 0,
2920 SLAB_HWCACHE_ALIGN
, NULL
);
2921 if (!skdev
->sglist_cache
)
2923 WARN_ONCE(kmem_cache_size(skdev
->sglist_cache
) < size
,
2924 "skd-sglist: %d < %zd\n",
2925 kmem_cache_size(skdev
->sglist_cache
), size
);
2926 size
= SKD_N_INTERNAL_BYTES
;
2927 skdev
->databuf_cache
= kmem_cache_create("skd-databuf", size
, 0,
2928 SLAB_HWCACHE_ALIGN
, NULL
);
2929 if (!skdev
->databuf_cache
)
2931 WARN_ONCE(kmem_cache_size(skdev
->databuf_cache
) < size
,
2932 "skd-databuf: %d < %zd\n",
2933 kmem_cache_size(skdev
->databuf_cache
), size
);
2935 dev_dbg(&skdev
->pdev
->dev
, "skcomp\n");
2936 rc
= skd_cons_skcomp(skdev
);
2940 dev_dbg(&skdev
->pdev
->dev
, "skmsg\n");
2941 rc
= skd_cons_skmsg(skdev
);
2945 dev_dbg(&skdev
->pdev
->dev
, "sksb\n");
2946 rc
= skd_cons_sksb(skdev
);
2950 dev_dbg(&skdev
->pdev
->dev
, "disk\n");
2951 rc
= skd_cons_disk(skdev
);
2955 dev_dbg(&skdev
->pdev
->dev
, "VICTORY\n");
2959 dev_dbg(&skdev
->pdev
->dev
, "construct failed\n");
2960 skd_destruct(skdev
);
2965 *****************************************************************************
2967 *****************************************************************************
2970 static void skd_free_skcomp(struct skd_device
*skdev
)
2972 if (skdev
->skcomp_table
)
2973 pci_free_consistent(skdev
->pdev
, SKD_SKCOMP_SIZE
,
2974 skdev
->skcomp_table
, skdev
->cq_dma_address
);
2976 skdev
->skcomp_table
= NULL
;
2977 skdev
->cq_dma_address
= 0;
2980 static void skd_free_skmsg(struct skd_device
*skdev
)
2984 if (skdev
->skmsg_table
== NULL
)
2987 for (i
= 0; i
< skdev
->num_fitmsg_context
; i
++) {
2988 struct skd_fitmsg_context
*skmsg
;
2990 skmsg
= &skdev
->skmsg_table
[i
];
2992 if (skmsg
->msg_buf
!= NULL
) {
2993 pci_free_consistent(skdev
->pdev
, SKD_N_FITMSG_BYTES
,
2995 skmsg
->mb_dma_address
);
2997 skmsg
->msg_buf
= NULL
;
2998 skmsg
->mb_dma_address
= 0;
3001 kfree(skdev
->skmsg_table
);
3002 skdev
->skmsg_table
= NULL
;
3005 static void skd_free_sksb(struct skd_device
*skdev
)
3007 struct skd_special_context
*skspcl
= &skdev
->internal_skspcl
;
3009 skd_free_dma(skdev
, skdev
->databuf_cache
, skspcl
->data_buf
,
3010 skspcl
->db_dma_address
, DMA_BIDIRECTIONAL
);
3012 skspcl
->data_buf
= NULL
;
3013 skspcl
->db_dma_address
= 0;
3015 skd_free_dma(skdev
, skdev
->msgbuf_cache
, skspcl
->msg_buf
,
3016 skspcl
->mb_dma_address
, DMA_TO_DEVICE
);
3018 skspcl
->msg_buf
= NULL
;
3019 skspcl
->mb_dma_address
= 0;
3021 skd_free_sg_list(skdev
, skspcl
->req
.sksg_list
,
3022 skspcl
->req
.sksg_dma_address
);
3024 skspcl
->req
.sksg_list
= NULL
;
3025 skspcl
->req
.sksg_dma_address
= 0;
3028 static void skd_free_disk(struct skd_device
*skdev
)
3030 struct gendisk
*disk
= skdev
->disk
;
3032 if (disk
&& (disk
->flags
& GENHD_FL_UP
))
3036 blk_cleanup_queue(skdev
->queue
);
3037 skdev
->queue
= NULL
;
3042 if (skdev
->tag_set
.tags
)
3043 blk_mq_free_tag_set(&skdev
->tag_set
);
3049 static void skd_destruct(struct skd_device
*skdev
)
3054 cancel_work_sync(&skdev
->start_queue
);
3056 dev_dbg(&skdev
->pdev
->dev
, "disk\n");
3057 skd_free_disk(skdev
);
3059 dev_dbg(&skdev
->pdev
->dev
, "sksb\n");
3060 skd_free_sksb(skdev
);
3062 dev_dbg(&skdev
->pdev
->dev
, "skmsg\n");
3063 skd_free_skmsg(skdev
);
3065 dev_dbg(&skdev
->pdev
->dev
, "skcomp\n");
3066 skd_free_skcomp(skdev
);
3068 kmem_cache_destroy(skdev
->databuf_cache
);
3069 kmem_cache_destroy(skdev
->sglist_cache
);
3070 kmem_cache_destroy(skdev
->msgbuf_cache
);
3072 dev_dbg(&skdev
->pdev
->dev
, "skdev\n");
3077 *****************************************************************************
3078 * BLOCK DEVICE (BDEV) GLUE
3079 *****************************************************************************
3082 static int skd_bdev_getgeo(struct block_device
*bdev
, struct hd_geometry
*geo
)
3084 struct skd_device
*skdev
;
3087 skdev
= bdev
->bd_disk
->private_data
;
3089 dev_dbg(&skdev
->pdev
->dev
, "%s: CMD[%s] getgeo device\n",
3090 bdev
->bd_disk
->disk_name
, current
->comm
);
3092 if (skdev
->read_cap_is_valid
) {
3093 capacity
= get_capacity(skdev
->disk
);
3096 geo
->cylinders
= (capacity
) / (255 * 64);
3103 static int skd_bdev_attach(struct device
*parent
, struct skd_device
*skdev
)
3105 dev_dbg(&skdev
->pdev
->dev
, "add_disk\n");
3106 device_add_disk(parent
, skdev
->disk
);
3110 static const struct block_device_operations skd_blockdev_ops
= {
3111 .owner
= THIS_MODULE
,
3112 .getgeo
= skd_bdev_getgeo
,
3116 *****************************************************************************
3118 *****************************************************************************
3121 static const struct pci_device_id skd_pci_tbl
[] = {
3122 { PCI_VENDOR_ID_STEC
, PCI_DEVICE_ID_S1120
,
3123 PCI_ANY_ID
, PCI_ANY_ID
, 0, 0, },
3124 { 0 } /* terminate list */
3127 MODULE_DEVICE_TABLE(pci
, skd_pci_tbl
);
3129 static char *skd_pci_info(struct skd_device
*skdev
, char *str
)
3133 strcpy(str
, "PCIe (");
3134 pcie_reg
= pci_find_capability(skdev
->pdev
, PCI_CAP_ID_EXP
);
3139 uint16_t pcie_lstat
, lspeed
, lwidth
;
3142 pci_read_config_word(skdev
->pdev
, pcie_reg
, &pcie_lstat
);
3143 lspeed
= pcie_lstat
& (0xF);
3144 lwidth
= (pcie_lstat
& 0x3F0) >> 4;
3147 strcat(str
, "2.5GT/s ");
3148 else if (lspeed
== 2)
3149 strcat(str
, "5.0GT/s ");
3151 strcat(str
, "<unknown> ");
3152 snprintf(lwstr
, sizeof(lwstr
), "%dX)", lwidth
);
3158 static int skd_pci_probe(struct pci_dev
*pdev
, const struct pci_device_id
*ent
)
3163 struct skd_device
*skdev
;
3165 dev_dbg(&pdev
->dev
, "vendor=%04X device=%04x\n", pdev
->vendor
,
3168 rc
= pci_enable_device(pdev
);
3171 rc
= pci_request_regions(pdev
, DRV_NAME
);
3174 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(64));
3176 if (pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64))) {
3177 dev_err(&pdev
->dev
, "consistent DMA mask error %d\n",
3181 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
3183 dev_err(&pdev
->dev
, "DMA mask error %d\n", rc
);
3184 goto err_out_regions
;
3189 rc
= register_blkdev(0, DRV_NAME
);
3191 goto err_out_regions
;
3196 skdev
= skd_construct(pdev
);
3197 if (skdev
== NULL
) {
3199 goto err_out_regions
;
3202 skd_pci_info(skdev
, pci_str
);
3203 dev_info(&pdev
->dev
, "%s 64bit\n", pci_str
);
3205 pci_set_master(pdev
);
3206 rc
= pci_enable_pcie_error_reporting(pdev
);
3209 "bad enable of PCIe error reporting rc=%d\n", rc
);
3210 skdev
->pcie_error_reporting_is_enabled
= 0;
3212 skdev
->pcie_error_reporting_is_enabled
= 1;
3214 pci_set_drvdata(pdev
, skdev
);
3216 for (i
= 0; i
< SKD_MAX_BARS
; i
++) {
3217 skdev
->mem_phys
[i
] = pci_resource_start(pdev
, i
);
3218 skdev
->mem_size
[i
] = (u32
)pci_resource_len(pdev
, i
);
3219 skdev
->mem_map
[i
] = ioremap(skdev
->mem_phys
[i
],
3220 skdev
->mem_size
[i
]);
3221 if (!skdev
->mem_map
[i
]) {
3223 "Unable to map adapter memory!\n");
3225 goto err_out_iounmap
;
3227 dev_dbg(&pdev
->dev
, "mem_map=%p, phyd=%016llx, size=%d\n",
3228 skdev
->mem_map
[i
], (uint64_t)skdev
->mem_phys
[i
],
3229 skdev
->mem_size
[i
]);
3232 rc
= skd_acquire_irq(skdev
);
3234 dev_err(&pdev
->dev
, "interrupt resource error %d\n", rc
);
3235 goto err_out_iounmap
;
3238 rc
= skd_start_timer(skdev
);
3242 init_waitqueue_head(&skdev
->waitq
);
3244 skd_start_device(skdev
);
3246 rc
= wait_event_interruptible_timeout(skdev
->waitq
,
3247 (skdev
->gendisk_on
),
3248 (SKD_START_WAIT_SECONDS
* HZ
));
3249 if (skdev
->gendisk_on
> 0) {
3250 /* device came on-line after reset */
3251 skd_bdev_attach(&pdev
->dev
, skdev
);
3254 /* we timed out, something is wrong with the device,
3255 don't add the disk structure */
3256 dev_err(&pdev
->dev
, "error: waiting for s1120 timed out %d!\n",
3258 /* in case of no error; we timeout with ENXIO */
3267 skd_stop_device(skdev
);
3268 skd_release_irq(skdev
);
3271 for (i
= 0; i
< SKD_MAX_BARS
; i
++)
3272 if (skdev
->mem_map
[i
])
3273 iounmap(skdev
->mem_map
[i
]);
3275 if (skdev
->pcie_error_reporting_is_enabled
)
3276 pci_disable_pcie_error_reporting(pdev
);
3278 skd_destruct(skdev
);
3281 pci_release_regions(pdev
);
3284 pci_disable_device(pdev
);
3285 pci_set_drvdata(pdev
, NULL
);
3289 static void skd_pci_remove(struct pci_dev
*pdev
)
3292 struct skd_device
*skdev
;
3294 skdev
= pci_get_drvdata(pdev
);
3296 dev_err(&pdev
->dev
, "no device data for PCI\n");
3299 skd_stop_device(skdev
);
3300 skd_release_irq(skdev
);
3302 for (i
= 0; i
< SKD_MAX_BARS
; i
++)
3303 if (skdev
->mem_map
[i
])
3304 iounmap(skdev
->mem_map
[i
]);
3306 if (skdev
->pcie_error_reporting_is_enabled
)
3307 pci_disable_pcie_error_reporting(pdev
);
3309 skd_destruct(skdev
);
3311 pci_release_regions(pdev
);
3312 pci_disable_device(pdev
);
3313 pci_set_drvdata(pdev
, NULL
);
3318 static int skd_pci_suspend(struct pci_dev
*pdev
, pm_message_t state
)
3321 struct skd_device
*skdev
;
3323 skdev
= pci_get_drvdata(pdev
);
3325 dev_err(&pdev
->dev
, "no device data for PCI\n");
3329 skd_stop_device(skdev
);
3331 skd_release_irq(skdev
);
3333 for (i
= 0; i
< SKD_MAX_BARS
; i
++)
3334 if (skdev
->mem_map
[i
])
3335 iounmap(skdev
->mem_map
[i
]);
3337 if (skdev
->pcie_error_reporting_is_enabled
)
3338 pci_disable_pcie_error_reporting(pdev
);
3340 pci_release_regions(pdev
);
3341 pci_save_state(pdev
);
3342 pci_disable_device(pdev
);
3343 pci_set_power_state(pdev
, pci_choose_state(pdev
, state
));
3347 static int skd_pci_resume(struct pci_dev
*pdev
)
3351 struct skd_device
*skdev
;
3353 skdev
= pci_get_drvdata(pdev
);
3355 dev_err(&pdev
->dev
, "no device data for PCI\n");
3359 pci_set_power_state(pdev
, PCI_D0
);
3360 pci_enable_wake(pdev
, PCI_D0
, 0);
3361 pci_restore_state(pdev
);
3363 rc
= pci_enable_device(pdev
);
3366 rc
= pci_request_regions(pdev
, DRV_NAME
);
3369 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(64));
3371 if (pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64))) {
3373 dev_err(&pdev
->dev
, "consistent DMA mask error %d\n",
3377 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
3380 dev_err(&pdev
->dev
, "DMA mask error %d\n", rc
);
3381 goto err_out_regions
;
3385 pci_set_master(pdev
);
3386 rc
= pci_enable_pcie_error_reporting(pdev
);
3389 "bad enable of PCIe error reporting rc=%d\n", rc
);
3390 skdev
->pcie_error_reporting_is_enabled
= 0;
3392 skdev
->pcie_error_reporting_is_enabled
= 1;
3394 for (i
= 0; i
< SKD_MAX_BARS
; i
++) {
3396 skdev
->mem_phys
[i
] = pci_resource_start(pdev
, i
);
3397 skdev
->mem_size
[i
] = (u32
)pci_resource_len(pdev
, i
);
3398 skdev
->mem_map
[i
] = ioremap(skdev
->mem_phys
[i
],
3399 skdev
->mem_size
[i
]);
3400 if (!skdev
->mem_map
[i
]) {
3401 dev_err(&pdev
->dev
, "Unable to map adapter memory!\n");
3403 goto err_out_iounmap
;
3405 dev_dbg(&pdev
->dev
, "mem_map=%p, phyd=%016llx, size=%d\n",
3406 skdev
->mem_map
[i
], (uint64_t)skdev
->mem_phys
[i
],
3407 skdev
->mem_size
[i
]);
3409 rc
= skd_acquire_irq(skdev
);
3411 dev_err(&pdev
->dev
, "interrupt resource error %d\n", rc
);
3412 goto err_out_iounmap
;
3415 rc
= skd_start_timer(skdev
);
3419 init_waitqueue_head(&skdev
->waitq
);
3421 skd_start_device(skdev
);
3426 skd_stop_device(skdev
);
3427 skd_release_irq(skdev
);
3430 for (i
= 0; i
< SKD_MAX_BARS
; i
++)
3431 if (skdev
->mem_map
[i
])
3432 iounmap(skdev
->mem_map
[i
]);
3434 if (skdev
->pcie_error_reporting_is_enabled
)
3435 pci_disable_pcie_error_reporting(pdev
);
3438 pci_release_regions(pdev
);
3441 pci_disable_device(pdev
);
3445 static void skd_pci_shutdown(struct pci_dev
*pdev
)
3447 struct skd_device
*skdev
;
3449 dev_err(&pdev
->dev
, "%s called\n", __func__
);
3451 skdev
= pci_get_drvdata(pdev
);
3453 dev_err(&pdev
->dev
, "no device data for PCI\n");
3457 dev_err(&pdev
->dev
, "calling stop\n");
3458 skd_stop_device(skdev
);
3461 static struct pci_driver skd_driver
= {
3463 .id_table
= skd_pci_tbl
,
3464 .probe
= skd_pci_probe
,
3465 .remove
= skd_pci_remove
,
3466 .suspend
= skd_pci_suspend
,
3467 .resume
= skd_pci_resume
,
3468 .shutdown
= skd_pci_shutdown
,
3472 *****************************************************************************
3474 *****************************************************************************
3477 const char *skd_drive_state_to_str(int state
)
3480 case FIT_SR_DRIVE_OFFLINE
:
3482 case FIT_SR_DRIVE_INIT
:
3484 case FIT_SR_DRIVE_ONLINE
:
3486 case FIT_SR_DRIVE_BUSY
:
3488 case FIT_SR_DRIVE_FAULT
:
3490 case FIT_SR_DRIVE_DEGRADED
:
3492 case FIT_SR_PCIE_LINK_DOWN
:
3494 case FIT_SR_DRIVE_SOFT_RESET
:
3495 return "SOFT_RESET";
3496 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD
:
3498 case FIT_SR_DRIVE_INIT_FAULT
:
3499 return "INIT_FAULT";
3500 case FIT_SR_DRIVE_BUSY_SANITIZE
:
3501 return "BUSY_SANITIZE";
3502 case FIT_SR_DRIVE_BUSY_ERASE
:
3503 return "BUSY_ERASE";
3504 case FIT_SR_DRIVE_FW_BOOTING
:
3505 return "FW_BOOTING";
3511 const char *skd_skdev_state_to_str(enum skd_drvr_state state
)
3514 case SKD_DRVR_STATE_LOAD
:
3516 case SKD_DRVR_STATE_IDLE
:
3518 case SKD_DRVR_STATE_BUSY
:
3520 case SKD_DRVR_STATE_STARTING
:
3522 case SKD_DRVR_STATE_ONLINE
:
3524 case SKD_DRVR_STATE_PAUSING
:
3526 case SKD_DRVR_STATE_PAUSED
:
3528 case SKD_DRVR_STATE_RESTARTING
:
3529 return "RESTARTING";
3530 case SKD_DRVR_STATE_RESUMING
:
3532 case SKD_DRVR_STATE_STOPPING
:
3534 case SKD_DRVR_STATE_SYNCING
:
3536 case SKD_DRVR_STATE_FAULT
:
3538 case SKD_DRVR_STATE_DISAPPEARED
:
3539 return "DISAPPEARED";
3540 case SKD_DRVR_STATE_BUSY_ERASE
:
3541 return "BUSY_ERASE";
3542 case SKD_DRVR_STATE_BUSY_SANITIZE
:
3543 return "BUSY_SANITIZE";
3544 case SKD_DRVR_STATE_BUSY_IMMINENT
:
3545 return "BUSY_IMMINENT";
3546 case SKD_DRVR_STATE_WAIT_BOOT
:
3554 static const char *skd_skreq_state_to_str(enum skd_req_state state
)
3557 case SKD_REQ_STATE_IDLE
:
3559 case SKD_REQ_STATE_SETUP
:
3561 case SKD_REQ_STATE_BUSY
:
3563 case SKD_REQ_STATE_COMPLETED
:
3565 case SKD_REQ_STATE_TIMEOUT
:
3572 static void skd_log_skdev(struct skd_device
*skdev
, const char *event
)
3574 dev_dbg(&skdev
->pdev
->dev
, "skdev=%p event='%s'\n", skdev
, event
);
3575 dev_dbg(&skdev
->pdev
->dev
, " drive_state=%s(%d) driver_state=%s(%d)\n",
3576 skd_drive_state_to_str(skdev
->drive_state
), skdev
->drive_state
,
3577 skd_skdev_state_to_str(skdev
->state
), skdev
->state
);
3578 dev_dbg(&skdev
->pdev
->dev
, " busy=%d limit=%d dev=%d lowat=%d\n",
3579 skd_in_flight(skdev
), skdev
->cur_max_queue_depth
,
3580 skdev
->dev_max_queue_depth
, skdev
->queue_low_water_mark
);
3581 dev_dbg(&skdev
->pdev
->dev
, " cycle=%d cycle_ix=%d\n",
3582 skdev
->skcomp_cycle
, skdev
->skcomp_ix
);
3585 static void skd_log_skreq(struct skd_device
*skdev
,
3586 struct skd_request_context
*skreq
, const char *event
)
3588 struct request
*req
= blk_mq_rq_from_pdu(skreq
);
3589 u32 lba
= blk_rq_pos(req
);
3590 u32 count
= blk_rq_sectors(req
);
3592 dev_dbg(&skdev
->pdev
->dev
, "skreq=%p event='%s'\n", skreq
, event
);
3593 dev_dbg(&skdev
->pdev
->dev
, " state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
3594 skd_skreq_state_to_str(skreq
->state
), skreq
->state
, skreq
->id
,
3596 dev_dbg(&skdev
->pdev
->dev
, " sg_dir=%d n_sg=%d\n",
3597 skreq
->data_dir
, skreq
->n_sg
);
3599 dev_dbg(&skdev
->pdev
->dev
,
3600 "req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n", req
, lba
, lba
,
3601 count
, count
, (int)rq_data_dir(req
));
3605 *****************************************************************************
3607 *****************************************************************************
3610 static int __init
skd_init(void)
3612 BUILD_BUG_ON(sizeof(struct fit_completion_entry_v1
) != 8);
3613 BUILD_BUG_ON(sizeof(struct fit_comp_error_info
) != 32);
3614 BUILD_BUG_ON(sizeof(struct skd_command_header
) != 16);
3615 BUILD_BUG_ON(sizeof(struct skd_scsi_request
) != 32);
3616 BUILD_BUG_ON(sizeof(struct driver_inquiry_data
) != 44);
3617 BUILD_BUG_ON(offsetof(struct skd_msg_buf
, fmh
) != 0);
3618 BUILD_BUG_ON(offsetof(struct skd_msg_buf
, scsi
) != 64);
3619 BUILD_BUG_ON(sizeof(struct skd_msg_buf
) != SKD_N_FITMSG_BYTES
);
3621 switch (skd_isr_type
) {
3622 case SKD_IRQ_LEGACY
:
3627 pr_err(PFX
"skd_isr_type %d invalid, re-set to %d\n",
3628 skd_isr_type
, SKD_IRQ_DEFAULT
);
3629 skd_isr_type
= SKD_IRQ_DEFAULT
;
3632 if (skd_max_queue_depth
< 1 ||
3633 skd_max_queue_depth
> SKD_MAX_QUEUE_DEPTH
) {
3634 pr_err(PFX
"skd_max_queue_depth %d invalid, re-set to %d\n",
3635 skd_max_queue_depth
, SKD_MAX_QUEUE_DEPTH_DEFAULT
);
3636 skd_max_queue_depth
= SKD_MAX_QUEUE_DEPTH_DEFAULT
;
3639 if (skd_max_req_per_msg
< 1 ||
3640 skd_max_req_per_msg
> SKD_MAX_REQ_PER_MSG
) {
3641 pr_err(PFX
"skd_max_req_per_msg %d invalid, re-set to %d\n",
3642 skd_max_req_per_msg
, SKD_MAX_REQ_PER_MSG_DEFAULT
);
3643 skd_max_req_per_msg
= SKD_MAX_REQ_PER_MSG_DEFAULT
;
3646 if (skd_sgs_per_request
< 1 || skd_sgs_per_request
> 4096) {
3647 pr_err(PFX
"skd_sg_per_request %d invalid, re-set to %d\n",
3648 skd_sgs_per_request
, SKD_N_SG_PER_REQ_DEFAULT
);
3649 skd_sgs_per_request
= SKD_N_SG_PER_REQ_DEFAULT
;
3652 if (skd_dbg_level
< 0 || skd_dbg_level
> 2) {
3653 pr_err(PFX
"skd_dbg_level %d invalid, re-set to %d\n",
3658 if (skd_isr_comp_limit
< 0) {
3659 pr_err(PFX
"skd_isr_comp_limit %d invalid, set to %d\n",
3660 skd_isr_comp_limit
, 0);
3661 skd_isr_comp_limit
= 0;
3664 return pci_register_driver(&skd_driver
);
3667 static void __exit
skd_exit(void)
3669 pci_unregister_driver(&skd_driver
);
3672 unregister_blkdev(skd_major
, DRV_NAME
);
3675 module_init(skd_init
);
3676 module_exit(skd_exit
);