1 // SPDX-License-Identifier: GPL-2.0+
3 * ufs.c - Universal Flash Storage (UFS) driver
5 * Taken from Linux Kernel v5.2 (drivers/scsi/ufs/ufshcd.c) and ported
8 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
11 #include <bouncebuf.h>
16 #include <dm/device_compat.h>
17 #include <dm/devres.h>
19 #include <dm/device-internal.h>
24 #include <asm/dma-mapping.h>
25 #include <linux/bitops.h>
26 #include <linux/delay.h>
27 #include <linux/dma-mapping.h>
31 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
34 /* maximum number of link-startup retries */
35 #define DME_LINKSTARTUP_RETRIES 3
37 /* maximum number of retries for a general UIC command */
38 #define UFS_UIC_COMMAND_RETRIES 3
40 /* Query request retries */
41 #define QUERY_REQ_RETRIES 3
42 /* Query request timeout */
43 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
45 /* maximum timeout in ms for a general UIC command */
46 #define UFS_UIC_CMD_TIMEOUT 1000
47 /* NOP OUT retries waiting for NOP IN response */
48 #define NOP_OUT_RETRIES 10
49 /* Timeout after 30 msecs if NOP OUT hangs without response */
50 #define NOP_OUT_TIMEOUT 30 /* msecs */
52 /* Only use one Task Tag for all requests */
55 /* Expose the flag value from utp_upiu_query.value */
56 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
58 #define MAX_PRDT_ENTRY 262144
60 /* maximum bytes per request */
61 #define UFS_MAX_BYTES (128 * 256 * 1024)
63 static inline bool ufshcd_is_hba_active(struct ufs_hba
*hba
);
64 static inline void ufshcd_hba_stop(struct ufs_hba
*hba
);
65 static int ufshcd_hba_enable(struct ufs_hba
*hba
);
68 * ufshcd_wait_for_register - wait for register value to change
70 static int ufshcd_wait_for_register(struct ufs_hba
*hba
, u32 reg
, u32 mask
,
71 u32 val
, unsigned long timeout_ms
)
74 unsigned long start
= get_timer(0);
76 /* ignore bits that we don't intend to wait on */
79 while ((ufshcd_readl(hba
, reg
) & mask
) != val
) {
80 if (get_timer(start
) > timeout_ms
) {
81 if ((ufshcd_readl(hba
, reg
) & mask
) != val
)
91 * ufshcd_init_pwr_info - setting the POR (power on reset)
92 * values in hba power info
94 static void ufshcd_init_pwr_info(struct ufs_hba
*hba
)
96 hba
->pwr_info
.gear_rx
= UFS_PWM_G1
;
97 hba
->pwr_info
.gear_tx
= UFS_PWM_G1
;
98 hba
->pwr_info
.lane_rx
= 1;
99 hba
->pwr_info
.lane_tx
= 1;
100 hba
->pwr_info
.pwr_rx
= SLOWAUTO_MODE
;
101 hba
->pwr_info
.pwr_tx
= SLOWAUTO_MODE
;
102 hba
->pwr_info
.hs_rate
= 0;
106 * ufshcd_print_pwr_info - print power params as saved in hba
109 static void ufshcd_print_pwr_info(struct ufs_hba
*hba
)
111 static const char * const names
[] = {
121 dev_err(hba
->dev
, "[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
122 hba
->pwr_info
.gear_rx
, hba
->pwr_info
.gear_tx
,
123 hba
->pwr_info
.lane_rx
, hba
->pwr_info
.lane_tx
,
124 names
[hba
->pwr_info
.pwr_rx
],
125 names
[hba
->pwr_info
.pwr_tx
],
126 hba
->pwr_info
.hs_rate
);
130 * ufshcd_ready_for_uic_cmd - Check if controller is ready
131 * to accept UIC commands
133 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba
*hba
)
135 if (ufshcd_readl(hba
, REG_CONTROLLER_STATUS
) & UIC_COMMAND_READY
)
142 * ufshcd_get_uic_cmd_result - Get the UIC command result
144 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba
*hba
)
146 return ufshcd_readl(hba
, REG_UIC_COMMAND_ARG_2
) &
147 MASK_UIC_COMMAND_RESULT
;
151 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
153 static inline u32
ufshcd_get_dme_attr_val(struct ufs_hba
*hba
)
155 return ufshcd_readl(hba
, REG_UIC_COMMAND_ARG_3
);
159 * ufshcd_is_device_present - Check if any device connected to
160 * the host controller
162 static inline bool ufshcd_is_device_present(struct ufs_hba
*hba
)
164 return (ufshcd_readl(hba
, REG_CONTROLLER_STATUS
) &
165 DEVICE_PRESENT
) ? true : false;
169 * ufshcd_send_uic_cmd - UFS Interconnect layer command API
172 static int ufshcd_send_uic_cmd(struct ufs_hba
*hba
, struct uic_command
*uic_cmd
)
174 unsigned long start
= 0;
176 u32 enabled_intr_status
;
178 if (!ufshcd_ready_for_uic_cmd(hba
)) {
180 "Controller not ready to accept UIC commands\n");
184 debug("sending uic command:%d\n", uic_cmd
->command
);
187 ufshcd_writel(hba
, uic_cmd
->argument1
, REG_UIC_COMMAND_ARG_1
);
188 ufshcd_writel(hba
, uic_cmd
->argument2
, REG_UIC_COMMAND_ARG_2
);
189 ufshcd_writel(hba
, uic_cmd
->argument3
, REG_UIC_COMMAND_ARG_3
);
192 ufshcd_writel(hba
, uic_cmd
->command
& COMMAND_OPCODE_MASK
,
195 start
= get_timer(0);
197 intr_status
= ufshcd_readl(hba
, REG_INTERRUPT_STATUS
);
198 enabled_intr_status
= intr_status
& hba
->intr_mask
;
199 ufshcd_writel(hba
, intr_status
, REG_INTERRUPT_STATUS
);
201 if (get_timer(start
) > UFS_UIC_CMD_TIMEOUT
) {
203 "Timedout waiting for UIC response\n");
208 if (enabled_intr_status
& UFSHCD_ERROR_MASK
) {
209 dev_err(hba
->dev
, "Error in status:%08x\n",
210 enabled_intr_status
);
214 } while (!(enabled_intr_status
& UFSHCD_UIC_MASK
));
216 uic_cmd
->argument2
= ufshcd_get_uic_cmd_result(hba
);
217 uic_cmd
->argument3
= ufshcd_get_dme_attr_val(hba
);
219 debug("Sent successfully\n");
225 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
228 int ufshcd_dme_set_attr(struct ufs_hba
*hba
, u32 attr_sel
, u8 attr_set
,
229 u32 mib_val
, u8 peer
)
231 struct uic_command uic_cmd
= {0};
232 static const char *const action
[] = {
236 const char *set
= action
[!!peer
];
238 int retries
= UFS_UIC_COMMAND_RETRIES
;
240 uic_cmd
.command
= peer
?
241 UIC_CMD_DME_PEER_SET
: UIC_CMD_DME_SET
;
242 uic_cmd
.argument1
= attr_sel
;
243 uic_cmd
.argument2
= UIC_ARG_ATTR_TYPE(attr_set
);
244 uic_cmd
.argument3
= mib_val
;
247 /* for peer attributes we retry upon failure */
248 ret
= ufshcd_send_uic_cmd(hba
, &uic_cmd
);
250 dev_dbg(hba
->dev
, "%s: attr-id 0x%x val 0x%x error code %d\n",
251 set
, UIC_GET_ATTR_ID(attr_sel
), mib_val
, ret
);
252 } while (ret
&& peer
&& --retries
);
255 dev_err(hba
->dev
, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
256 set
, UIC_GET_ATTR_ID(attr_sel
), mib_val
,
257 UFS_UIC_COMMAND_RETRIES
- retries
);
263 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
266 int ufshcd_dme_get_attr(struct ufs_hba
*hba
, u32 attr_sel
,
267 u32
*mib_val
, u8 peer
)
269 struct uic_command uic_cmd
= {0};
270 static const char *const action
[] = {
274 const char *get
= action
[!!peer
];
276 int retries
= UFS_UIC_COMMAND_RETRIES
;
278 uic_cmd
.command
= peer
?
279 UIC_CMD_DME_PEER_GET
: UIC_CMD_DME_GET
;
280 uic_cmd
.argument1
= attr_sel
;
283 /* for peer attributes we retry upon failure */
284 ret
= ufshcd_send_uic_cmd(hba
, &uic_cmd
);
286 dev_dbg(hba
->dev
, "%s: attr-id 0x%x error code %d\n",
287 get
, UIC_GET_ATTR_ID(attr_sel
), ret
);
288 } while (ret
&& peer
&& --retries
);
291 dev_err(hba
->dev
, "%s: attr-id 0x%x failed %d retries\n",
292 get
, UIC_GET_ATTR_ID(attr_sel
),
293 UFS_UIC_COMMAND_RETRIES
- retries
);
296 *mib_val
= uic_cmd
.argument3
;
301 static int ufshcd_disable_tx_lcc(struct ufs_hba
*hba
, bool peer
)
303 u32 tx_lanes
, i
, err
= 0;
306 ufshcd_dme_get(hba
, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES
),
309 ufshcd_dme_peer_get(hba
, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES
),
311 for (i
= 0; i
< tx_lanes
; i
++) {
313 err
= ufshcd_dme_set(hba
,
314 UIC_ARG_MIB_SEL(TX_LCC_ENABLE
,
315 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i
)),
318 err
= ufshcd_dme_peer_set(hba
,
319 UIC_ARG_MIB_SEL(TX_LCC_ENABLE
,
320 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i
)),
323 dev_err(hba
->dev
, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d\n",
324 __func__
, peer
, i
, err
);
332 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba
*hba
)
334 return ufshcd_disable_tx_lcc(hba
, true);
338 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
341 static int ufshcd_dme_link_startup(struct ufs_hba
*hba
)
343 struct uic_command uic_cmd
= {0};
346 uic_cmd
.command
= UIC_CMD_DME_LINK_STARTUP
;
348 ret
= ufshcd_send_uic_cmd(hba
, &uic_cmd
);
351 "dme-link-startup: error code %d\n", ret
);
356 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
359 static inline void ufshcd_disable_intr_aggr(struct ufs_hba
*hba
)
361 ufshcd_writel(hba
, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL
);
365 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
367 static inline int ufshcd_get_lists_status(u32 reg
)
369 return !((reg
& UFSHCD_STATUS_READY
) == UFSHCD_STATUS_READY
);
373 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
374 * When run-stop registers are set to 1, it indicates the
375 * host controller that it can process the requests
377 static void ufshcd_enable_run_stop_reg(struct ufs_hba
*hba
)
379 ufshcd_writel(hba
, UTP_TASK_REQ_LIST_RUN_STOP_BIT
,
380 REG_UTP_TASK_REQ_LIST_RUN_STOP
);
381 ufshcd_writel(hba
, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT
,
382 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP
);
386 * ufshcd_enable_intr - enable interrupts
388 static void ufshcd_enable_intr(struct ufs_hba
*hba
, u32 intrs
)
390 u32 set
= ufshcd_readl(hba
, REG_INTERRUPT_ENABLE
);
393 if (hba
->version
== UFSHCI_VERSION_10
) {
394 rw
= set
& INTERRUPT_MASK_RW_VER_10
;
395 set
= rw
| ((set
^ intrs
) & intrs
);
400 ufshcd_writel(hba
, set
, REG_INTERRUPT_ENABLE
);
402 hba
->intr_mask
= set
;
406 * ufshcd_make_hba_operational - Make UFS controller operational
408 * To bring UFS host controller to operational state,
409 * 1. Enable required interrupts
410 * 2. Configure interrupt aggregation
411 * 3. Program UTRL and UTMRL base address
412 * 4. Configure run-stop-registers
415 static int ufshcd_make_hba_operational(struct ufs_hba
*hba
)
420 /* Enable required interrupts */
421 ufshcd_enable_intr(hba
, UFSHCD_ENABLE_INTRS
);
423 /* Disable interrupt aggregation */
424 ufshcd_disable_intr_aggr(hba
);
426 /* Configure UTRL and UTMRL base address registers */
427 ufshcd_writel(hba
, lower_32_bits((dma_addr_t
)hba
->utrdl
),
428 REG_UTP_TRANSFER_REQ_LIST_BASE_L
);
429 ufshcd_writel(hba
, upper_32_bits((dma_addr_t
)hba
->utrdl
),
430 REG_UTP_TRANSFER_REQ_LIST_BASE_H
);
431 ufshcd_writel(hba
, lower_32_bits((dma_addr_t
)hba
->utmrdl
),
432 REG_UTP_TASK_REQ_LIST_BASE_L
);
433 ufshcd_writel(hba
, upper_32_bits((dma_addr_t
)hba
->utmrdl
),
434 REG_UTP_TASK_REQ_LIST_BASE_H
);
437 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
439 reg
= ufshcd_readl(hba
, REG_CONTROLLER_STATUS
);
440 if (!(ufshcd_get_lists_status(reg
))) {
441 ufshcd_enable_run_stop_reg(hba
);
444 "Host controller not ready to process requests\n");
454 * ufshcd_link_startup - Initialize unipro link startup
456 static int ufshcd_link_startup(struct ufs_hba
*hba
)
459 int retries
= DME_LINKSTARTUP_RETRIES
;
460 bool link_startup_again
= true;
464 ufshcd_ops_link_startup_notify(hba
, PRE_CHANGE
);
466 ret
= ufshcd_dme_link_startup(hba
);
468 /* check if device is detected by inter-connect layer */
469 if (!ret
&& !ufshcd_is_device_present(hba
)) {
470 dev_err(hba
->dev
, "%s: Device not present\n", __func__
);
476 * DME link lost indication is only received when link is up,
477 * but we can't be sure if the link is up until link startup
478 * succeeds. So reset the local Uni-Pro and try again.
480 if (ret
&& ufshcd_hba_enable(hba
))
482 } while (ret
&& retries
--);
485 /* failed to get the link up... retire */
488 if (link_startup_again
) {
489 link_startup_again
= false;
490 retries
= DME_LINKSTARTUP_RETRIES
;
494 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
495 ufshcd_init_pwr_info(hba
);
497 if (hba
->quirks
& UFSHCD_QUIRK_BROKEN_LCC
) {
498 ret
= ufshcd_disable_device_tx_lcc(hba
);
503 /* Include any host controller configuration via UIC commands */
504 ret
= ufshcd_ops_link_startup_notify(hba
, POST_CHANGE
);
508 ret
= ufshcd_make_hba_operational(hba
);
511 dev_err(hba
->dev
, "link startup failed %d\n", ret
);
517 * ufshcd_hba_stop - Send controller to reset state
519 static inline void ufshcd_hba_stop(struct ufs_hba
*hba
)
523 ufshcd_writel(hba
, CONTROLLER_DISABLE
, REG_CONTROLLER_ENABLE
);
524 err
= ufshcd_wait_for_register(hba
, REG_CONTROLLER_ENABLE
,
525 CONTROLLER_ENABLE
, CONTROLLER_DISABLE
,
528 dev_err(hba
->dev
, "%s: Controller disable failed\n", __func__
);
532 * ufshcd_is_hba_active - Get controller state
534 static inline bool ufshcd_is_hba_active(struct ufs_hba
*hba
)
536 return (ufshcd_readl(hba
, REG_CONTROLLER_ENABLE
) & CONTROLLER_ENABLE
)
541 * ufshcd_hba_start - Start controller initialization sequence
543 static inline void ufshcd_hba_start(struct ufs_hba
*hba
)
545 ufshcd_writel(hba
, CONTROLLER_ENABLE
, REG_CONTROLLER_ENABLE
);
549 * ufshcd_hba_enable - initialize the controller
551 static int ufshcd_hba_enable(struct ufs_hba
*hba
)
555 if (!ufshcd_is_hba_active(hba
))
556 /* change controller state to "reset state" */
557 ufshcd_hba_stop(hba
);
559 ufshcd_ops_hce_enable_notify(hba
, PRE_CHANGE
);
561 /* start controller initialization sequence */
562 ufshcd_hba_start(hba
);
565 * To initialize a UFS host controller HCE bit must be set to 1.
566 * During initialization the HCE bit value changes from 1->0->1.
567 * When the host controller completes initialization sequence
568 * it sets the value of HCE bit to 1. The same HCE bit is read back
569 * to check if the controller has completed initialization sequence.
570 * So without this delay the value HCE = 1, set in the previous
571 * instruction might be read back.
572 * This delay can be changed based on the controller.
576 /* wait for the host controller to complete initialization */
578 while (ufshcd_is_hba_active(hba
)) {
582 dev_err(hba
->dev
, "Controller enable failed\n");
588 /* enable UIC related interrupts */
589 ufshcd_enable_intr(hba
, UFSHCD_UIC_MASK
);
591 ufshcd_ops_hce_enable_notify(hba
, POST_CHANGE
);
597 * ufshcd_host_memory_configure - configure local reference block with
600 static void ufshcd_host_memory_configure(struct ufs_hba
*hba
)
602 struct utp_transfer_req_desc
*utrdlp
;
603 dma_addr_t cmd_desc_dma_addr
;
608 cmd_desc_dma_addr
= (dma_addr_t
)hba
->ucdl
;
610 utrdlp
->command_desc_base_addr_lo
=
611 cpu_to_le32(lower_32_bits(cmd_desc_dma_addr
));
612 utrdlp
->command_desc_base_addr_hi
=
613 cpu_to_le32(upper_32_bits(cmd_desc_dma_addr
));
615 response_offset
= offsetof(struct utp_transfer_cmd_desc
, response_upiu
);
616 prdt_offset
= offsetof(struct utp_transfer_cmd_desc
, prd_table
);
618 utrdlp
->response_upiu_offset
= cpu_to_le16(response_offset
>> 2);
619 utrdlp
->prd_table_offset
= cpu_to_le16(prdt_offset
>> 2);
620 utrdlp
->response_upiu_length
= cpu_to_le16(ALIGNED_UPIU_SIZE
>> 2);
622 hba
->ucd_req_ptr
= (struct utp_upiu_req
*)hba
->ucdl
;
624 (struct utp_upiu_rsp
*)&hba
->ucdl
->response_upiu
;
626 (struct ufshcd_sg_entry
*)&hba
->ucdl
->prd_table
;
630 * ufshcd_memory_alloc - allocate memory for host memory space data structures
632 static int ufshcd_memory_alloc(struct ufs_hba
*hba
)
634 /* Allocate one Transfer Request Descriptor
635 * Should be aligned to 1k boundary.
637 hba
->utrdl
= memalign(1024, sizeof(struct utp_transfer_req_desc
));
639 dev_err(hba
->dev
, "Transfer Descriptor memory allocation failed\n");
643 /* Allocate one Command Descriptor
644 * Should be aligned to 1k boundary.
646 hba
->ucdl
= memalign(1024, sizeof(struct utp_transfer_cmd_desc
));
648 dev_err(hba
->dev
, "Command descriptor memory allocation failed\n");
656 * ufshcd_get_intr_mask - Get the interrupt bit mask
658 static inline u32
ufshcd_get_intr_mask(struct ufs_hba
*hba
)
662 switch (hba
->version
) {
663 case UFSHCI_VERSION_10
:
664 intr_mask
= INTERRUPT_MASK_ALL_VER_10
;
666 case UFSHCI_VERSION_11
:
667 case UFSHCI_VERSION_20
:
668 intr_mask
= INTERRUPT_MASK_ALL_VER_11
;
670 case UFSHCI_VERSION_21
:
672 intr_mask
= INTERRUPT_MASK_ALL_VER_21
;
680 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
682 static inline u32
ufshcd_get_ufs_version(struct ufs_hba
*hba
)
684 return ufshcd_readl(hba
, REG_UFS_VERSION
);
688 * ufshcd_get_upmcrs - Get the power mode change request status
690 static inline u8
ufshcd_get_upmcrs(struct ufs_hba
*hba
)
692 return (ufshcd_readl(hba
, REG_CONTROLLER_STATUS
) >> 8) & 0x7;
696 * ufshcd_cache_flush_and_invalidate - Flush and invalidate cache
698 * Flush and invalidate cache in aligned address..address+size range.
699 * The invalidation is in place to avoid stale data in cache.
701 static void ufshcd_cache_flush_and_invalidate(void *addr
, unsigned long size
)
703 uintptr_t aaddr
= (uintptr_t)addr
& ~(ARCH_DMA_MINALIGN
- 1);
704 unsigned long asize
= ALIGN(size
, ARCH_DMA_MINALIGN
);
706 flush_dcache_range(aaddr
, aaddr
+ asize
);
707 invalidate_dcache_range(aaddr
, aaddr
+ asize
);
711 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
712 * descriptor according to request
714 static void ufshcd_prepare_req_desc_hdr(struct ufs_hba
*hba
,
716 enum dma_data_direction cmd_dir
)
718 struct utp_transfer_req_desc
*req_desc
= hba
->utrdl
;
722 if (cmd_dir
== DMA_FROM_DEVICE
) {
723 data_direction
= UTP_DEVICE_TO_HOST
;
724 *upiu_flags
= UPIU_CMD_FLAGS_READ
;
725 } else if (cmd_dir
== DMA_TO_DEVICE
) {
726 data_direction
= UTP_HOST_TO_DEVICE
;
727 *upiu_flags
= UPIU_CMD_FLAGS_WRITE
;
729 data_direction
= UTP_NO_DATA_TRANSFER
;
730 *upiu_flags
= UPIU_CMD_FLAGS_NONE
;
733 dword_0
= data_direction
| (0x1 << UPIU_COMMAND_TYPE_OFFSET
);
735 /* Enable Interrupt for command */
736 dword_0
|= UTP_REQ_DESC_INT_CMD
;
738 /* Transfer request descriptor header fields */
739 req_desc
->header
.dword_0
= cpu_to_le32(dword_0
);
740 /* dword_1 is reserved, hence it is set to 0 */
741 req_desc
->header
.dword_1
= 0;
743 * assigning invalid value for command status. Controller
744 * updates OCS on command completion, with the command
747 req_desc
->header
.dword_2
=
748 cpu_to_le32(OCS_INVALID_COMMAND_STATUS
);
749 /* dword_3 is reserved, hence it is set to 0 */
750 req_desc
->header
.dword_3
= 0;
752 req_desc
->prd_table_length
= 0;
754 ufshcd_cache_flush_and_invalidate(req_desc
, sizeof(*req_desc
));
757 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba
*hba
,
760 struct utp_upiu_req
*ucd_req_ptr
= hba
->ucd_req_ptr
;
761 struct ufs_query
*query
= &hba
->dev_cmd
.query
;
762 u16 len
= be16_to_cpu(query
->request
.upiu_req
.length
);
764 /* Query request header */
765 ucd_req_ptr
->header
.dword_0
=
766 UPIU_HEADER_DWORD(UPIU_TRANSACTION_QUERY_REQ
,
767 upiu_flags
, 0, TASK_TAG
);
768 ucd_req_ptr
->header
.dword_1
=
769 UPIU_HEADER_DWORD(0, query
->request
.query_func
,
772 /* Data segment length only need for WRITE_DESC */
773 if (query
->request
.upiu_req
.opcode
== UPIU_QUERY_OPCODE_WRITE_DESC
)
774 ucd_req_ptr
->header
.dword_2
=
775 UPIU_HEADER_DWORD(0, 0, (len
>> 8), (u8
)len
);
777 ucd_req_ptr
->header
.dword_2
= 0;
779 /* Copy the Query Request buffer as is */
780 memcpy(&ucd_req_ptr
->qr
, &query
->request
.upiu_req
, QUERY_OSF_SIZE
);
782 /* Copy the Descriptor */
783 if (query
->request
.upiu_req
.opcode
== UPIU_QUERY_OPCODE_WRITE_DESC
) {
784 memcpy(ucd_req_ptr
+ 1, query
->descriptor
, len
);
785 ufshcd_cache_flush_and_invalidate(ucd_req_ptr
, 2 * sizeof(*ucd_req_ptr
));
787 ufshcd_cache_flush_and_invalidate(ucd_req_ptr
, sizeof(*ucd_req_ptr
));
790 memset(hba
->ucd_rsp_ptr
, 0, sizeof(struct utp_upiu_rsp
));
791 ufshcd_cache_flush_and_invalidate(hba
->ucd_rsp_ptr
, sizeof(*hba
->ucd_rsp_ptr
));
794 static inline void ufshcd_prepare_utp_nop_upiu(struct ufs_hba
*hba
)
796 struct utp_upiu_req
*ucd_req_ptr
= hba
->ucd_req_ptr
;
798 memset(ucd_req_ptr
, 0, sizeof(struct utp_upiu_req
));
800 /* command descriptor fields */
801 ucd_req_ptr
->header
.dword_0
=
802 UPIU_HEADER_DWORD(UPIU_TRANSACTION_NOP_OUT
, 0, 0, TASK_TAG
);
803 /* clear rest of the fields of basic header */
804 ucd_req_ptr
->header
.dword_1
= 0;
805 ucd_req_ptr
->header
.dword_2
= 0;
807 memset(hba
->ucd_rsp_ptr
, 0, sizeof(struct utp_upiu_rsp
));
809 ufshcd_cache_flush_and_invalidate(ucd_req_ptr
, sizeof(*ucd_req_ptr
));
810 ufshcd_cache_flush_and_invalidate(hba
->ucd_rsp_ptr
, sizeof(*hba
->ucd_rsp_ptr
));
814 * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
815 * for Device Management Purposes
817 static int ufshcd_comp_devman_upiu(struct ufs_hba
*hba
,
818 enum dev_cmd_type cmd_type
)
823 hba
->dev_cmd
.type
= cmd_type
;
825 ufshcd_prepare_req_desc_hdr(hba
, &upiu_flags
, DMA_NONE
);
827 case DEV_CMD_TYPE_QUERY
:
828 ufshcd_prepare_utp_query_req_upiu(hba
, upiu_flags
);
830 case DEV_CMD_TYPE_NOP
:
831 ufshcd_prepare_utp_nop_upiu(hba
);
840 static int ufshcd_send_command(struct ufs_hba
*hba
, unsigned int task_tag
)
844 u32 enabled_intr_status
;
846 ufshcd_writel(hba
, 1 << task_tag
, REG_UTP_TRANSFER_REQ_DOOR_BELL
);
848 start
= get_timer(0);
850 intr_status
= ufshcd_readl(hba
, REG_INTERRUPT_STATUS
);
851 enabled_intr_status
= intr_status
& hba
->intr_mask
;
852 ufshcd_writel(hba
, intr_status
, REG_INTERRUPT_STATUS
);
854 if (get_timer(start
) > QUERY_REQ_TIMEOUT
) {
856 "Timedout waiting for UTP response\n");
861 if (enabled_intr_status
& UFSHCD_ERROR_MASK
) {
862 dev_err(hba
->dev
, "Error in status:%08x\n",
863 enabled_intr_status
);
867 } while (!(enabled_intr_status
& UTP_TRANSFER_REQ_COMPL
));
873 * ufshcd_get_req_rsp - returns the TR response transaction type
875 static inline int ufshcd_get_req_rsp(struct utp_upiu_rsp
*ucd_rsp_ptr
)
877 return be32_to_cpu(ucd_rsp_ptr
->header
.dword_0
) >> 24;
881 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
884 static inline int ufshcd_get_tr_ocs(struct ufs_hba
*hba
)
886 struct utp_transfer_req_desc
*req_desc
= hba
->utrdl
;
888 return le32_to_cpu(req_desc
->header
.dword_2
) & MASK_OCS
;
891 static inline int ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp
*ucd_rsp_ptr
)
893 return be32_to_cpu(ucd_rsp_ptr
->header
.dword_1
) & MASK_RSP_UPIU_RESULT
;
896 static int ufshcd_check_query_response(struct ufs_hba
*hba
)
898 struct ufs_query_res
*query_res
= &hba
->dev_cmd
.query
.response
;
900 /* Get the UPIU response */
901 query_res
->response
= ufshcd_get_rsp_upiu_result(hba
->ucd_rsp_ptr
) >>
902 UPIU_RSP_CODE_OFFSET
;
903 return query_res
->response
;
907 * ufshcd_copy_query_response() - Copy the Query Response and the data
910 static int ufshcd_copy_query_response(struct ufs_hba
*hba
)
912 struct ufs_query_res
*query_res
= &hba
->dev_cmd
.query
.response
;
914 memcpy(&query_res
->upiu_res
, &hba
->ucd_rsp_ptr
->qr
, QUERY_OSF_SIZE
);
916 /* Get the descriptor */
917 if (hba
->dev_cmd
.query
.descriptor
&&
918 hba
->ucd_rsp_ptr
->qr
.opcode
== UPIU_QUERY_OPCODE_READ_DESC
) {
919 u8
*descp
= (u8
*)hba
->ucd_rsp_ptr
+
920 GENERAL_UPIU_REQUEST_SIZE
;
924 /* data segment length */
925 resp_len
= be32_to_cpu(hba
->ucd_rsp_ptr
->header
.dword_2
) &
926 MASK_QUERY_DATA_SEG_LEN
;
928 be16_to_cpu(hba
->dev_cmd
.query
.request
.upiu_req
.length
);
929 if (likely(buf_len
>= resp_len
)) {
930 memcpy(hba
->dev_cmd
.query
.descriptor
, descp
, resp_len
);
933 "%s: Response size is bigger than buffer\n",
943 * ufshcd_exec_dev_cmd - API for sending device management requests
945 static int ufshcd_exec_dev_cmd(struct ufs_hba
*hba
, enum dev_cmd_type cmd_type
,
951 err
= ufshcd_comp_devman_upiu(hba
, cmd_type
);
955 err
= ufshcd_send_command(hba
, TASK_TAG
);
959 err
= ufshcd_get_tr_ocs(hba
);
961 dev_err(hba
->dev
, "Error in OCS:%d\n", err
);
965 resp
= ufshcd_get_req_rsp(hba
->ucd_rsp_ptr
);
967 case UPIU_TRANSACTION_NOP_IN
:
969 case UPIU_TRANSACTION_QUERY_RSP
:
970 err
= ufshcd_check_query_response(hba
);
972 err
= ufshcd_copy_query_response(hba
);
974 case UPIU_TRANSACTION_REJECT_UPIU
:
975 /* TODO: handle Reject UPIU Response */
977 dev_err(hba
->dev
, "%s: Reject UPIU not fully implemented\n",
982 dev_err(hba
->dev
, "%s: Invalid device management cmd response: %x\n",
990 * ufshcd_init_query() - init the query response and request parameters
992 static inline void ufshcd_init_query(struct ufs_hba
*hba
,
993 struct ufs_query_req
**request
,
994 struct ufs_query_res
**response
,
995 enum query_opcode opcode
,
996 u8 idn
, u8 index
, u8 selector
)
998 *request
= &hba
->dev_cmd
.query
.request
;
999 *response
= &hba
->dev_cmd
.query
.response
;
1000 memset(*request
, 0, sizeof(struct ufs_query_req
));
1001 memset(*response
, 0, sizeof(struct ufs_query_res
));
1002 (*request
)->upiu_req
.opcode
= opcode
;
1003 (*request
)->upiu_req
.idn
= idn
;
1004 (*request
)->upiu_req
.index
= index
;
1005 (*request
)->upiu_req
.selector
= selector
;
1009 * ufshcd_query_flag() - API function for sending flag query requests
1011 int ufshcd_query_flag(struct ufs_hba
*hba
, enum query_opcode opcode
,
1012 enum flag_idn idn
, bool *flag_res
)
1014 struct ufs_query_req
*request
= NULL
;
1015 struct ufs_query_res
*response
= NULL
;
1016 int err
, index
= 0, selector
= 0;
1017 int timeout
= QUERY_REQ_TIMEOUT
;
1019 ufshcd_init_query(hba
, &request
, &response
, opcode
, idn
, index
,
1023 case UPIU_QUERY_OPCODE_SET_FLAG
:
1024 case UPIU_QUERY_OPCODE_CLEAR_FLAG
:
1025 case UPIU_QUERY_OPCODE_TOGGLE_FLAG
:
1026 request
->query_func
= UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST
;
1028 case UPIU_QUERY_OPCODE_READ_FLAG
:
1029 request
->query_func
= UPIU_QUERY_FUNC_STANDARD_READ_REQUEST
;
1031 /* No dummy reads */
1032 dev_err(hba
->dev
, "%s: Invalid argument for read request\n",
1040 "%s: Expected query flag opcode but got = %d\n",
1046 err
= ufshcd_exec_dev_cmd(hba
, DEV_CMD_TYPE_QUERY
, timeout
);
1050 "%s: Sending flag query for idn %d failed, err = %d\n",
1051 __func__
, idn
, err
);
1056 *flag_res
= (be32_to_cpu(response
->upiu_res
.value
) &
1057 MASK_QUERY_UPIU_FLAG_LOC
) & 0x1;
1063 static int ufshcd_query_flag_retry(struct ufs_hba
*hba
,
1064 enum query_opcode opcode
,
1065 enum flag_idn idn
, bool *flag_res
)
1070 for (retries
= 0; retries
< QUERY_REQ_RETRIES
; retries
++) {
1071 ret
= ufshcd_query_flag(hba
, opcode
, idn
, flag_res
);
1074 "%s: failed with error %d, retries %d\n",
1075 __func__
, ret
, retries
);
1082 "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
1083 __func__
, opcode
, idn
, ret
, retries
);
1087 static int __ufshcd_query_descriptor(struct ufs_hba
*hba
,
1088 enum query_opcode opcode
,
1089 enum desc_idn idn
, u8 index
, u8 selector
,
1090 u8
*desc_buf
, int *buf_len
)
1092 struct ufs_query_req
*request
= NULL
;
1093 struct ufs_query_res
*response
= NULL
;
1097 dev_err(hba
->dev
, "%s: descriptor buffer required for opcode 0x%x\n",
1103 if (*buf_len
< QUERY_DESC_MIN_SIZE
|| *buf_len
> QUERY_DESC_MAX_SIZE
) {
1104 dev_err(hba
->dev
, "%s: descriptor buffer size (%d) is out of range\n",
1105 __func__
, *buf_len
);
1110 ufshcd_init_query(hba
, &request
, &response
, opcode
, idn
, index
,
1112 hba
->dev_cmd
.query
.descriptor
= desc_buf
;
1113 request
->upiu_req
.length
= cpu_to_be16(*buf_len
);
1116 case UPIU_QUERY_OPCODE_WRITE_DESC
:
1117 request
->query_func
= UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST
;
1119 case UPIU_QUERY_OPCODE_READ_DESC
:
1120 request
->query_func
= UPIU_QUERY_FUNC_STANDARD_READ_REQUEST
;
1123 dev_err(hba
->dev
, "%s: Expected query descriptor opcode but got = 0x%.2x\n",
1129 err
= ufshcd_exec_dev_cmd(hba
, DEV_CMD_TYPE_QUERY
, QUERY_REQ_TIMEOUT
);
1132 dev_err(hba
->dev
, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
1133 __func__
, opcode
, idn
, index
, err
);
1137 hba
->dev_cmd
.query
.descriptor
= NULL
;
1138 *buf_len
= be16_to_cpu(response
->upiu_res
.length
);
1145 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
1147 int ufshcd_query_descriptor_retry(struct ufs_hba
*hba
, enum query_opcode opcode
,
1148 enum desc_idn idn
, u8 index
, u8 selector
,
1149 u8
*desc_buf
, int *buf_len
)
1154 for (retries
= QUERY_REQ_RETRIES
; retries
> 0; retries
--) {
1155 err
= __ufshcd_query_descriptor(hba
, opcode
, idn
, index
,
1156 selector
, desc_buf
, buf_len
);
1157 if (!err
|| err
== -EINVAL
)
1165 * ufshcd_read_desc_length - read the specified descriptor length from header
1167 static int ufshcd_read_desc_length(struct ufs_hba
*hba
, enum desc_idn desc_id
,
1168 int desc_index
, int *desc_length
)
1171 u8 header
[QUERY_DESC_HDR_SIZE
];
1172 int header_len
= QUERY_DESC_HDR_SIZE
;
1174 if (desc_id
>= QUERY_DESC_IDN_MAX
)
1177 ret
= ufshcd_query_descriptor_retry(hba
, UPIU_QUERY_OPCODE_READ_DESC
,
1178 desc_id
, desc_index
, 0, header
,
1182 dev_err(hba
->dev
, "%s: Failed to get descriptor header id %d\n",
1185 } else if (desc_id
!= header
[QUERY_DESC_DESC_TYPE_OFFSET
]) {
1186 dev_warn(hba
->dev
, "%s: descriptor header id %d and desc_id %d mismatch\n",
1187 __func__
, header
[QUERY_DESC_DESC_TYPE_OFFSET
],
1192 *desc_length
= header
[QUERY_DESC_LENGTH_OFFSET
];
1197 static void ufshcd_init_desc_sizes(struct ufs_hba
*hba
)
1201 err
= ufshcd_read_desc_length(hba
, QUERY_DESC_IDN_DEVICE
, 0,
1202 &hba
->desc_size
.dev_desc
);
1204 hba
->desc_size
.dev_desc
= QUERY_DESC_DEVICE_DEF_SIZE
;
1206 err
= ufshcd_read_desc_length(hba
, QUERY_DESC_IDN_POWER
, 0,
1207 &hba
->desc_size
.pwr_desc
);
1209 hba
->desc_size
.pwr_desc
= QUERY_DESC_POWER_DEF_SIZE
;
1211 err
= ufshcd_read_desc_length(hba
, QUERY_DESC_IDN_INTERCONNECT
, 0,
1212 &hba
->desc_size
.interc_desc
);
1214 hba
->desc_size
.interc_desc
= QUERY_DESC_INTERCONNECT_DEF_SIZE
;
1216 err
= ufshcd_read_desc_length(hba
, QUERY_DESC_IDN_CONFIGURATION
, 0,
1217 &hba
->desc_size
.conf_desc
);
1219 hba
->desc_size
.conf_desc
= QUERY_DESC_CONFIGURATION_DEF_SIZE
;
1221 err
= ufshcd_read_desc_length(hba
, QUERY_DESC_IDN_UNIT
, 0,
1222 &hba
->desc_size
.unit_desc
);
1224 hba
->desc_size
.unit_desc
= QUERY_DESC_UNIT_DEF_SIZE
;
1226 err
= ufshcd_read_desc_length(hba
, QUERY_DESC_IDN_GEOMETRY
, 0,
1227 &hba
->desc_size
.geom_desc
);
1229 hba
->desc_size
.geom_desc
= QUERY_DESC_GEOMETRY_DEF_SIZE
;
1231 err
= ufshcd_read_desc_length(hba
, QUERY_DESC_IDN_HEALTH
, 0,
1232 &hba
->desc_size
.hlth_desc
);
1234 hba
->desc_size
.hlth_desc
= QUERY_DESC_HEALTH_DEF_SIZE
;
1238 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
1241 int ufshcd_map_desc_id_to_length(struct ufs_hba
*hba
, enum desc_idn desc_id
,
1245 case QUERY_DESC_IDN_DEVICE
:
1246 *desc_len
= hba
->desc_size
.dev_desc
;
1248 case QUERY_DESC_IDN_POWER
:
1249 *desc_len
= hba
->desc_size
.pwr_desc
;
1251 case QUERY_DESC_IDN_GEOMETRY
:
1252 *desc_len
= hba
->desc_size
.geom_desc
;
1254 case QUERY_DESC_IDN_CONFIGURATION
:
1255 *desc_len
= hba
->desc_size
.conf_desc
;
1257 case QUERY_DESC_IDN_UNIT
:
1258 *desc_len
= hba
->desc_size
.unit_desc
;
1260 case QUERY_DESC_IDN_INTERCONNECT
:
1261 *desc_len
= hba
->desc_size
.interc_desc
;
1263 case QUERY_DESC_IDN_STRING
:
1264 *desc_len
= QUERY_DESC_MAX_SIZE
;
1266 case QUERY_DESC_IDN_HEALTH
:
1267 *desc_len
= hba
->desc_size
.hlth_desc
;
1269 case QUERY_DESC_IDN_RFU_0
:
1270 case QUERY_DESC_IDN_RFU_1
:
1279 EXPORT_SYMBOL(ufshcd_map_desc_id_to_length
);
1282 * ufshcd_read_desc_param - read the specified descriptor parameter
1285 int ufshcd_read_desc_param(struct ufs_hba
*hba
, enum desc_idn desc_id
,
1286 int desc_index
, u8 param_offset
, u8
*param_read_buf
,
1292 bool is_kmalloc
= true;
1295 if (desc_id
>= QUERY_DESC_IDN_MAX
|| !param_size
)
1298 /* Get the max length of descriptor from structure filled up at probe
1301 ret
= ufshcd_map_desc_id_to_length(hba
, desc_id
, &buff_len
);
1304 if (ret
|| !buff_len
) {
1305 dev_err(hba
->dev
, "%s: Failed to get full descriptor length\n",
1310 /* Check whether we need temp memory */
1311 if (param_offset
!= 0 || param_size
< buff_len
) {
1312 desc_buf
= kmalloc(buff_len
, GFP_KERNEL
);
1316 desc_buf
= param_read_buf
;
1320 /* Request for full descriptor */
1321 ret
= ufshcd_query_descriptor_retry(hba
, UPIU_QUERY_OPCODE_READ_DESC
,
1322 desc_id
, desc_index
, 0, desc_buf
,
1326 dev_err(hba
->dev
, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
1327 __func__
, desc_id
, desc_index
, param_offset
, ret
);
1332 if (desc_buf
[QUERY_DESC_DESC_TYPE_OFFSET
] != desc_id
) {
1333 dev_err(hba
->dev
, "%s: invalid desc_id %d in descriptor header\n",
1334 __func__
, desc_buf
[QUERY_DESC_DESC_TYPE_OFFSET
]);
1339 /* Check wherher we will not copy more data, than available */
1340 if (is_kmalloc
&& param_size
> buff_len
)
1341 param_size
= buff_len
;
1344 memcpy(param_read_buf
, &desc_buf
[param_offset
], param_size
);
1351 /* replace non-printable or non-ASCII characters with spaces */
1352 static inline void ufshcd_remove_non_printable(uint8_t *val
)
1357 if (*val
< 0x20 || *val
> 0x7e)
1362 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
1363 * state) and waits for it to take effect.
1366 static int ufshcd_uic_pwr_ctrl(struct ufs_hba
*hba
, struct uic_command
*cmd
)
1368 unsigned long start
= 0;
1372 ret
= ufshcd_send_uic_cmd(hba
, cmd
);
1375 "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
1376 cmd
->command
, cmd
->argument3
, ret
);
1381 start
= get_timer(0);
1383 status
= ufshcd_get_upmcrs(hba
);
1384 if (get_timer(start
) > UFS_UIC_CMD_TIMEOUT
) {
1386 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
1387 cmd
->command
, status
);
1388 ret
= (status
!= PWR_OK
) ? status
: -1;
1391 } while (status
!= PWR_LOCAL
);
1397 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode change
1398 * using DME_SET primitives.
1400 static int ufshcd_uic_change_pwr_mode(struct ufs_hba
*hba
, u8 mode
)
1402 struct uic_command uic_cmd
= {0};
1405 uic_cmd
.command
= UIC_CMD_DME_SET
;
1406 uic_cmd
.argument1
= UIC_ARG_MIB(PA_PWRMODE
);
1407 uic_cmd
.argument3
= mode
;
1408 ret
= ufshcd_uic_pwr_ctrl(hba
, &uic_cmd
);
1414 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufs_hba
*hba
,
1415 struct scsi_cmd
*pccb
, u32 upiu_flags
)
1417 struct utp_upiu_req
*ucd_req_ptr
= hba
->ucd_req_ptr
;
1418 unsigned int cdb_len
;
1420 /* command descriptor fields */
1421 ucd_req_ptr
->header
.dword_0
=
1422 UPIU_HEADER_DWORD(UPIU_TRANSACTION_COMMAND
, upiu_flags
,
1423 pccb
->lun
, TASK_TAG
);
1424 ucd_req_ptr
->header
.dword_1
=
1425 UPIU_HEADER_DWORD(UPIU_COMMAND_SET_TYPE_SCSI
, 0, 0, 0);
1427 /* Total EHS length and Data segment length will be zero */
1428 ucd_req_ptr
->header
.dword_2
= 0;
1430 ucd_req_ptr
->sc
.exp_data_transfer_len
= cpu_to_be32(pccb
->datalen
);
1432 cdb_len
= min_t(unsigned short, pccb
->cmdlen
, UFS_CDB_SIZE
);
1433 memset(ucd_req_ptr
->sc
.cdb
, 0, UFS_CDB_SIZE
);
1434 memcpy(ucd_req_ptr
->sc
.cdb
, pccb
->cmd
, cdb_len
);
1436 memset(hba
->ucd_rsp_ptr
, 0, sizeof(struct utp_upiu_rsp
));
1437 ufshcd_cache_flush_and_invalidate(ucd_req_ptr
, sizeof(*ucd_req_ptr
));
1438 ufshcd_cache_flush_and_invalidate(hba
->ucd_rsp_ptr
, sizeof(*hba
->ucd_rsp_ptr
));
1441 static inline void prepare_prdt_desc(struct ufshcd_sg_entry
*entry
,
1442 unsigned char *buf
, ulong len
)
1444 entry
->size
= cpu_to_le32(len
) | GENMASK(1, 0);
1445 entry
->base_addr
= cpu_to_le32(lower_32_bits((unsigned long)buf
));
1446 entry
->upper_addr
= cpu_to_le32(upper_32_bits((unsigned long)buf
));
1449 static void prepare_prdt_table(struct ufs_hba
*hba
, struct scsi_cmd
*pccb
)
1451 struct utp_transfer_req_desc
*req_desc
= hba
->utrdl
;
1452 struct ufshcd_sg_entry
*prd_table
= hba
->ucd_prdt_ptr
;
1453 uintptr_t aaddr
= (uintptr_t)(pccb
->pdata
) & ~(ARCH_DMA_MINALIGN
- 1);
1454 ulong datalen
= pccb
->datalen
;
1460 req_desc
->prd_table_length
= 0;
1461 ufshcd_cache_flush_and_invalidate(req_desc
, sizeof(*req_desc
));
1465 if (pccb
->dma_dir
== DMA_TO_DEVICE
) { /* Write to device */
1466 flush_dcache_range(aaddr
, aaddr
+
1467 ALIGN(datalen
, ARCH_DMA_MINALIGN
));
1470 /* In any case, invalidate cache to avoid stale data in it. */
1471 invalidate_dcache_range(aaddr
, aaddr
+
1472 ALIGN(datalen
, ARCH_DMA_MINALIGN
));
1474 table_length
= DIV_ROUND_UP(pccb
->datalen
, MAX_PRDT_ENTRY
);
1478 prepare_prdt_desc(&prd_table
[table_length
- i
- 1], buf
,
1479 MAX_PRDT_ENTRY
- 1);
1480 buf
+= MAX_PRDT_ENTRY
;
1481 datalen
-= MAX_PRDT_ENTRY
;
1484 prepare_prdt_desc(&prd_table
[table_length
- i
- 1], buf
, datalen
- 1);
1486 req_desc
->prd_table_length
= table_length
;
1487 ufshcd_cache_flush_and_invalidate(prd_table
, sizeof(*prd_table
) * table_length
);
1488 ufshcd_cache_flush_and_invalidate(req_desc
, sizeof(*req_desc
));
1491 static int ufs_scsi_exec(struct udevice
*scsi_dev
, struct scsi_cmd
*pccb
)
1493 struct ufs_hba
*hba
= dev_get_uclass_priv(scsi_dev
->parent
);
1495 int ocs
, result
= 0;
1498 ufshcd_prepare_req_desc_hdr(hba
, &upiu_flags
, pccb
->dma_dir
);
1499 ufshcd_prepare_utp_scsi_cmd_upiu(hba
, pccb
, upiu_flags
);
1500 prepare_prdt_table(hba
, pccb
);
1502 ufshcd_send_command(hba
, TASK_TAG
);
1504 ocs
= ufshcd_get_tr_ocs(hba
);
1507 result
= ufshcd_get_req_rsp(hba
->ucd_rsp_ptr
);
1509 case UPIU_TRANSACTION_RESPONSE
:
1510 result
= ufshcd_get_rsp_upiu_result(hba
->ucd_rsp_ptr
);
1512 scsi_status
= result
& MASK_SCSI_STATUS
;
1517 case UPIU_TRANSACTION_REJECT_UPIU
:
1518 /* TODO: handle Reject UPIU Response */
1520 "Reject UPIU not fully implemented\n");
1524 "Unexpected request response code = %x\n",
1530 dev_err(hba
->dev
, "OCS error from controller = %x\n", ocs
);
1537 static inline int ufshcd_read_desc(struct ufs_hba
*hba
, enum desc_idn desc_id
,
1538 int desc_index
, u8
*buf
, u32 size
)
1540 return ufshcd_read_desc_param(hba
, desc_id
, desc_index
, 0, buf
, size
);
1543 static int ufshcd_read_device_desc(struct ufs_hba
*hba
, u8
*buf
, u32 size
)
1545 return ufshcd_read_desc(hba
, QUERY_DESC_IDN_DEVICE
, 0, buf
, size
);
1549 * ufshcd_read_string_desc - read string descriptor
1552 int ufshcd_read_string_desc(struct ufs_hba
*hba
, int desc_index
,
1553 u8
*buf
, u32 size
, bool ascii
)
1557 err
= ufshcd_read_desc(hba
, QUERY_DESC_IDN_STRING
, desc_index
, buf
,
1561 dev_err(hba
->dev
, "%s: reading String Desc failed after %d retries. err = %d\n",
1562 __func__
, QUERY_REQ_RETRIES
, err
);
1573 /* remove header and divide by 2 to move from UTF16 to UTF8 */
1574 ascii_len
= (desc_len
- QUERY_DESC_HDR_SIZE
) / 2 + 1;
1575 if (size
< ascii_len
+ QUERY_DESC_HDR_SIZE
) {
1576 dev_err(hba
->dev
, "%s: buffer allocated size is too small\n",
1582 buff_ascii
= kmalloc(ascii_len
, GFP_KERNEL
);
1589 * the descriptor contains string in UTF16 format
1590 * we need to convert to utf-8 so it can be displayed
1592 utf16_to_utf8(buff_ascii
,
1593 (uint16_t *)&buf
[QUERY_DESC_HDR_SIZE
], ascii_len
);
1595 /* replace non-printable or non-ASCII characters with spaces */
1596 for (i
= 0; i
< ascii_len
; i
++)
1597 ufshcd_remove_non_printable(&buff_ascii
[i
]);
1599 memset(buf
+ QUERY_DESC_HDR_SIZE
, 0,
1600 size
- QUERY_DESC_HDR_SIZE
);
1601 memcpy(buf
+ QUERY_DESC_HDR_SIZE
, buff_ascii
, ascii_len
);
1602 buf
[QUERY_DESC_LENGTH_OFFSET
] = ascii_len
+ QUERY_DESC_HDR_SIZE
;
1609 static int ufs_get_device_desc(struct ufs_hba
*hba
,
1610 struct ufs_dev_desc
*dev_desc
)
1617 buff_len
= max_t(size_t, hba
->desc_size
.dev_desc
,
1618 QUERY_DESC_MAX_SIZE
+ 1);
1619 desc_buf
= kmalloc(buff_len
, GFP_KERNEL
);
1625 err
= ufshcd_read_device_desc(hba
, desc_buf
, hba
->desc_size
.dev_desc
);
1627 dev_err(hba
->dev
, "%s: Failed reading Device Desc. err = %d\n",
1633 * getting vendor (manufacturerID) and Bank Index in big endian
1636 dev_desc
->wmanufacturerid
= desc_buf
[DEVICE_DESC_PARAM_MANF_ID
] << 8 |
1637 desc_buf
[DEVICE_DESC_PARAM_MANF_ID
+ 1];
1639 model_index
= desc_buf
[DEVICE_DESC_PARAM_PRDCT_NAME
];
1641 /* Zero-pad entire buffer for string termination. */
1642 memset(desc_buf
, 0, buff_len
);
1644 err
= ufshcd_read_string_desc(hba
, model_index
, desc_buf
,
1645 QUERY_DESC_MAX_SIZE
, true/*ASCII*/);
1647 dev_err(hba
->dev
, "%s: Failed reading Product Name. err = %d\n",
1652 desc_buf
[QUERY_DESC_MAX_SIZE
] = '\0';
1653 strlcpy(dev_desc
->model
, (char *)(desc_buf
+ QUERY_DESC_HDR_SIZE
),
1654 min_t(u8
, desc_buf
[QUERY_DESC_LENGTH_OFFSET
],
1657 /* Null terminate the model string */
1658 dev_desc
->model
[MAX_MODEL_LEN
] = '\0';
1666 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
1668 static int ufshcd_get_max_pwr_mode(struct ufs_hba
*hba
)
1670 struct ufs_pa_layer_attr
*pwr_info
= &hba
->max_pwr_info
.info
;
1672 if (hba
->max_pwr_info
.is_valid
)
1675 if (hba
->quirks
& UFSHCD_QUIRK_HIBERN_FASTAUTO
) {
1676 pwr_info
->pwr_tx
= FASTAUTO_MODE
;
1677 pwr_info
->pwr_rx
= FASTAUTO_MODE
;
1679 pwr_info
->pwr_tx
= FAST_MODE
;
1680 pwr_info
->pwr_rx
= FAST_MODE
;
1682 pwr_info
->hs_rate
= PA_HS_MODE_B
;
1684 /* Get the connected lane count */
1685 ufshcd_dme_get(hba
, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES
),
1686 &pwr_info
->lane_rx
);
1687 ufshcd_dme_get(hba
, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES
),
1688 &pwr_info
->lane_tx
);
1690 if (!pwr_info
->lane_rx
|| !pwr_info
->lane_tx
) {
1691 dev_err(hba
->dev
, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
1692 __func__
, pwr_info
->lane_rx
, pwr_info
->lane_tx
);
1697 * First, get the maximum gears of HS speed.
1698 * If a zero value, it means there is no HSGEAR capability.
1699 * Then, get the maximum gears of PWM speed.
1701 ufshcd_dme_get(hba
, UIC_ARG_MIB(PA_MAXRXHSGEAR
), &pwr_info
->gear_rx
);
1702 if (!pwr_info
->gear_rx
) {
1703 ufshcd_dme_get(hba
, UIC_ARG_MIB(PA_MAXRXPWMGEAR
),
1704 &pwr_info
->gear_rx
);
1705 if (!pwr_info
->gear_rx
) {
1706 dev_err(hba
->dev
, "%s: invalid max pwm rx gear read = %d\n",
1707 __func__
, pwr_info
->gear_rx
);
1710 pwr_info
->pwr_rx
= SLOW_MODE
;
1713 ufshcd_dme_peer_get(hba
, UIC_ARG_MIB(PA_MAXRXHSGEAR
),
1714 &pwr_info
->gear_tx
);
1715 if (!pwr_info
->gear_tx
) {
1716 ufshcd_dme_peer_get(hba
, UIC_ARG_MIB(PA_MAXRXPWMGEAR
),
1717 &pwr_info
->gear_tx
);
1718 if (!pwr_info
->gear_tx
) {
1719 dev_err(hba
->dev
, "%s: invalid max pwm tx gear read = %d\n",
1720 __func__
, pwr_info
->gear_tx
);
1723 pwr_info
->pwr_tx
= SLOW_MODE
;
1726 hba
->max_pwr_info
.is_valid
= true;
1730 static int ufshcd_change_power_mode(struct ufs_hba
*hba
,
1731 struct ufs_pa_layer_attr
*pwr_mode
)
1735 /* if already configured to the requested pwr_mode */
1736 if (pwr_mode
->gear_rx
== hba
->pwr_info
.gear_rx
&&
1737 pwr_mode
->gear_tx
== hba
->pwr_info
.gear_tx
&&
1738 pwr_mode
->lane_rx
== hba
->pwr_info
.lane_rx
&&
1739 pwr_mode
->lane_tx
== hba
->pwr_info
.lane_tx
&&
1740 pwr_mode
->pwr_rx
== hba
->pwr_info
.pwr_rx
&&
1741 pwr_mode
->pwr_tx
== hba
->pwr_info
.pwr_tx
&&
1742 pwr_mode
->hs_rate
== hba
->pwr_info
.hs_rate
) {
1743 dev_dbg(hba
->dev
, "%s: power already configured\n", __func__
);
1748 * Configure attributes for power mode change with below.
1749 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
1750 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
1753 ufshcd_dme_set(hba
, UIC_ARG_MIB(PA_RXGEAR
), pwr_mode
->gear_rx
);
1754 ufshcd_dme_set(hba
, UIC_ARG_MIB(PA_ACTIVERXDATALANES
),
1756 if (pwr_mode
->pwr_rx
== FASTAUTO_MODE
|| pwr_mode
->pwr_rx
== FAST_MODE
)
1757 ufshcd_dme_set(hba
, UIC_ARG_MIB(PA_RXTERMINATION
), TRUE
);
1759 ufshcd_dme_set(hba
, UIC_ARG_MIB(PA_RXTERMINATION
), FALSE
);
1761 ufshcd_dme_set(hba
, UIC_ARG_MIB(PA_TXGEAR
), pwr_mode
->gear_tx
);
1762 ufshcd_dme_set(hba
, UIC_ARG_MIB(PA_ACTIVETXDATALANES
),
1764 if (pwr_mode
->pwr_tx
== FASTAUTO_MODE
|| pwr_mode
->pwr_tx
== FAST_MODE
)
1765 ufshcd_dme_set(hba
, UIC_ARG_MIB(PA_TXTERMINATION
), TRUE
);
1767 ufshcd_dme_set(hba
, UIC_ARG_MIB(PA_TXTERMINATION
), FALSE
);
1769 if (pwr_mode
->pwr_rx
== FASTAUTO_MODE
||
1770 pwr_mode
->pwr_tx
== FASTAUTO_MODE
||
1771 pwr_mode
->pwr_rx
== FAST_MODE
||
1772 pwr_mode
->pwr_tx
== FAST_MODE
)
1773 ufshcd_dme_set(hba
, UIC_ARG_MIB(PA_HSSERIES
),
1776 ret
= ufshcd_uic_change_pwr_mode(hba
, pwr_mode
->pwr_rx
<< 4 |
1781 "%s: power mode change failed %d\n", __func__
, ret
);
1786 /* Copy new Power Mode to power info */
1787 memcpy(&hba
->pwr_info
, pwr_mode
, sizeof(struct ufs_pa_layer_attr
));
1793 * ufshcd_verify_dev_init() - Verify device initialization
1796 static int ufshcd_verify_dev_init(struct ufs_hba
*hba
)
1801 for (retries
= NOP_OUT_RETRIES
; retries
> 0; retries
--) {
1802 err
= ufshcd_exec_dev_cmd(hba
, DEV_CMD_TYPE_NOP
,
1804 if (!err
|| err
== -ETIMEDOUT
)
1807 dev_dbg(hba
->dev
, "%s: error %d retrying\n", __func__
, err
);
1811 dev_err(hba
->dev
, "%s: NOP OUT failed %d\n", __func__
, err
);
1817 * ufshcd_complete_dev_init() - checks device readiness
1819 static int ufshcd_complete_dev_init(struct ufs_hba
*hba
)
1825 err
= ufshcd_query_flag_retry(hba
, UPIU_QUERY_OPCODE_SET_FLAG
,
1826 QUERY_FLAG_IDN_FDEVICEINIT
, NULL
);
1829 "%s setting fDeviceInit flag failed with error %d\n",
1834 /* poll for max. 1000 iterations for fDeviceInit flag to clear */
1835 for (i
= 0; i
< 1000 && !err
&& flag_res
; i
++)
1836 err
= ufshcd_query_flag_retry(hba
, UPIU_QUERY_OPCODE_READ_FLAG
,
1837 QUERY_FLAG_IDN_FDEVICEINIT
,
1842 "%s reading fDeviceInit flag failed with error %d\n",
1846 "%s fDeviceInit was not cleared by the device\n",
1853 static void ufshcd_def_desc_sizes(struct ufs_hba
*hba
)
1855 hba
->desc_size
.dev_desc
= QUERY_DESC_DEVICE_DEF_SIZE
;
1856 hba
->desc_size
.pwr_desc
= QUERY_DESC_POWER_DEF_SIZE
;
1857 hba
->desc_size
.interc_desc
= QUERY_DESC_INTERCONNECT_DEF_SIZE
;
1858 hba
->desc_size
.conf_desc
= QUERY_DESC_CONFIGURATION_DEF_SIZE
;
1859 hba
->desc_size
.unit_desc
= QUERY_DESC_UNIT_DEF_SIZE
;
1860 hba
->desc_size
.geom_desc
= QUERY_DESC_GEOMETRY_DEF_SIZE
;
1861 hba
->desc_size
.hlth_desc
= QUERY_DESC_HEALTH_DEF_SIZE
;
1864 int ufs_start(struct ufs_hba
*hba
)
1866 struct ufs_dev_desc card
= {0};
1869 ret
= ufshcd_link_startup(hba
);
1873 ret
= ufshcd_verify_dev_init(hba
);
1877 ret
= ufshcd_complete_dev_init(hba
);
1881 /* Init check for device descriptor sizes */
1882 ufshcd_init_desc_sizes(hba
);
1884 ret
= ufs_get_device_desc(hba
, &card
);
1886 dev_err(hba
->dev
, "%s: Failed getting device info. err = %d\n",
1892 if (ufshcd_get_max_pwr_mode(hba
)) {
1894 "%s: Failed getting max supported power mode\n",
1897 ret
= ufshcd_change_power_mode(hba
, &hba
->max_pwr_info
.info
);
1899 dev_err(hba
->dev
, "%s: Failed setting power mode, err = %d\n",
1905 printf("Device at %s up at:", hba
->dev
->name
);
1906 ufshcd_print_pwr_info(hba
);
1912 int ufshcd_probe(struct udevice
*ufs_dev
, struct ufs_hba_ops
*hba_ops
)
1914 struct ufs_hba
*hba
= dev_get_uclass_priv(ufs_dev
);
1915 struct scsi_plat
*scsi_plat
;
1916 struct udevice
*scsi_dev
;
1917 void __iomem
*mmio_base
;
1920 device_find_first_child(ufs_dev
, &scsi_dev
);
1924 scsi_plat
= dev_get_uclass_plat(scsi_dev
);
1925 scsi_plat
->max_id
= UFSHCD_MAX_ID
;
1926 scsi_plat
->max_lun
= UFS_MAX_LUNS
;
1927 scsi_plat
->max_bytes_per_req
= UFS_MAX_BYTES
;
1932 if (device_is_on_pci_bus(ufs_dev
)) {
1933 mmio_base
= dm_pci_map_bar(ufs_dev
, PCI_BASE_ADDRESS_0
, 0, 0,
1934 PCI_REGION_TYPE
, PCI_REGION_MEM
);
1936 mmio_base
= dev_read_addr_ptr(ufs_dev
);
1938 hba
->mmio_base
= mmio_base
;
1940 /* Set descriptor lengths to specification defaults */
1941 ufshcd_def_desc_sizes(hba
);
1943 ufshcd_ops_init(hba
);
1945 /* Read capabilties registers */
1946 hba
->capabilities
= ufshcd_readl(hba
, REG_CONTROLLER_CAPABILITIES
);
1947 if (hba
->quirks
& UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS
)
1948 hba
->capabilities
&= ~MASK_64_ADDRESSING_SUPPORT
;
1950 /* Get UFS version supported by the controller */
1951 hba
->version
= ufshcd_get_ufs_version(hba
);
1952 if (hba
->version
!= UFSHCI_VERSION_10
&&
1953 hba
->version
!= UFSHCI_VERSION_11
&&
1954 hba
->version
!= UFSHCI_VERSION_20
&&
1955 hba
->version
!= UFSHCI_VERSION_21
&&
1956 hba
->version
!= UFSHCI_VERSION_30
)
1957 dev_err(hba
->dev
, "invalid UFS version 0x%x\n",
1960 /* Get Interrupt bit mask per version */
1961 hba
->intr_mask
= ufshcd_get_intr_mask(hba
);
1963 /* Allocate memory for host memory space */
1964 err
= ufshcd_memory_alloc(hba
);
1966 dev_err(hba
->dev
, "Memory allocation failed\n");
1970 /* Configure Local data structures */
1971 ufshcd_host_memory_configure(hba
);
1974 * In order to avoid any spurious interrupt immediately after
1975 * registering UFS controller interrupt handler, clear any pending UFS
1976 * interrupt status and disable all the UFS interrupts.
1978 ufshcd_writel(hba
, ufshcd_readl(hba
, REG_INTERRUPT_STATUS
),
1979 REG_INTERRUPT_STATUS
);
1980 ufshcd_writel(hba
, 0, REG_INTERRUPT_ENABLE
);
1982 err
= ufshcd_hba_enable(hba
);
1984 dev_err(hba
->dev
, "Host controller enable failed\n");
1988 err
= ufs_start(hba
);
1995 int ufs_scsi_bind(struct udevice
*ufs_dev
, struct udevice
**scsi_devp
)
1997 int ret
= device_bind_driver(ufs_dev
, "ufs_scsi", "ufs_scsi",
2003 #if IS_ENABLED(CONFIG_BOUNCE_BUFFER)
2004 static int ufs_scsi_buffer_aligned(struct udevice
*scsi_dev
, struct bounce_buffer
*state
)
2006 #ifdef CONFIG_PHYS_64BIT
2007 struct ufs_hba
*hba
= dev_get_uclass_priv(scsi_dev
->parent
);
2008 uintptr_t ubuf
= (uintptr_t)state
->user_buffer
;
2009 size_t len
= state
->len_aligned
;
2011 /* Check if below 32bit boundary */
2012 if ((hba
->quirks
& UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS
) &&
2013 ((ubuf
>> 32) || (ubuf
+ len
) >> 32)) {
2014 dev_dbg(scsi_dev
, "Buffer above 32bit boundary %lx-%lx\n",
2021 #endif /* CONFIG_BOUNCE_BUFFER */
2023 static struct scsi_ops ufs_ops
= {
2024 .exec
= ufs_scsi_exec
,
2025 #if IS_ENABLED(CONFIG_BOUNCE_BUFFER)
2026 .buffer_aligned
= ufs_scsi_buffer_aligned
,
2027 #endif /* CONFIG_BOUNCE_BUFFER */
2030 int ufs_probe_dev(int index
)
2032 struct udevice
*dev
;
2034 return uclass_get_device(UCLASS_UFS
, index
, &dev
);
2039 struct udevice
*dev
;
2043 ret
= uclass_get_device(UCLASS_UFS
, i
, &dev
);
2051 U_BOOT_DRIVER(ufs_scsi
) = {