[thirdparty/u-boot.git] / drivers / nvme / nvme_apple.c

// SPDX-License-Identifier: GPL-2.0+
/*
 * (C) Copyright 2021 Mark Kettenis <kettenis@openbsd.org>
 */

#include <common.h>
#include <dm.h>
#include <mailbox.h>
#include <mapmem.h>
#include "nvme.h"
#include <reset.h>

#include <asm/io.h>
#include <asm/arch/rtkit.h>
#include <asm/arch/sart.h>
#include <linux/iopoll.h>

/* ASC registers */
#define REG_CPU_CTRL		0x0044
#define  REG_CPU_CTRL_RUN	BIT(4)

/* Apple NVMe registers */
#define ANS_MAX_PEND_CMDS_CTRL	0x01210
#define  ANS_MAX_QUEUE_DEPTH	64
#define ANS_BOOT_STATUS		0x01300
#define  ANS_BOOT_STATUS_OK	0xde71ce55
#define ANS_MODESEL		0x01304
#define ANS_UNKNOWN_CTRL	0x24008
#define  ANS_PRP_NULL_CHECK	(1 << 11)
#define ANS_LINEAR_SQ_CTRL	0x24908
#define  ANS_LINEAR_SQ_CTRL_EN	(1 << 0)
#define ANS_ASQ_DB		0x2490c
#define ANS_IOSQ_DB		0x24910
#define ANS_NVMMU_NUM		0x28100
#define ANS_NVMMU_BASE_ASQ	0x28108
#define ANS_NVMMU_BASE_IOSQ	0x28110
#define ANS_NVMMU_TCB_INVAL	0x28118
#define ANS_NVMMU_TCB_STAT	0x28120

#define ANS_NVMMU_TCB_SIZE	0x4000
#define ANS_NVMMU_TCB_PITCH	0x80

/*
 * The Apple NVMe controller includes an IOMMU known as NVMMU.  The
 * NVMMU is programmed through an array of TCBs. These TCBs are paired
 * with the corresponding slot in the submission queues and need to be
 * configured with the command details before a command is allowed to
 * execute. This is necessary even for commands that don't do DMA.
 */
struct ans_nvmmu_tcb {
	u8 opcode;
	u8 flags;
	u8 slot;
	u8 pad0;
	u32 prpl_len;
	u8 pad1[16];
	u64 prp1;
	u64 prp2;
};

#define ANS_NVMMU_TCB_WRITE	BIT(0)
#define ANS_NVMMU_TCB_READ	BIT(1)

struct apple_nvme_priv {
	struct nvme_dev ndev;
	void *base;		/* NVMe registers */
	void *asc;		/* ASC registers */
	struct reset_ctl_bulk resets; /* ASC reset */
	struct mbox_chan chan;
	struct apple_sart *sart;
	struct apple_rtkit *rtk;
	struct ans_nvmmu_tcb *tcbs[NVME_Q_NUM]; /* Submission queue TCBs */
	u32 __iomem *q_db[NVME_Q_NUM]; /* Submission queue doorbell */
};

static int apple_nvme_setup_queue(struct nvme_queue *nvmeq)
{
	struct apple_nvme_priv *priv =
		container_of(nvmeq->dev, struct apple_nvme_priv, ndev);
	struct nvme_dev *dev = nvmeq->dev;

	switch (nvmeq->qid) {
	case NVME_ADMIN_Q:
	case NVME_IO_Q:
		break;
	default:
		return -EINVAL;
	}

	priv->tcbs[nvmeq->qid] = (void *)memalign(4096, ANS_NVMMU_TCB_SIZE);
	memset((void *)priv->tcbs[nvmeq->qid], 0, ANS_NVMMU_TCB_SIZE);

	switch (nvmeq->qid) {
	case NVME_ADMIN_Q:
		priv->q_db[nvmeq->qid] =
			((void __iomem *)dev->bar) + ANS_ASQ_DB;
		nvme_writeq((ulong)priv->tcbs[nvmeq->qid],
			    ((void __iomem *)dev->bar) + ANS_NVMMU_BASE_ASQ);
		break;
	case NVME_IO_Q:
		priv->q_db[nvmeq->qid] =
			((void __iomem *)dev->bar) + ANS_IOSQ_DB;
		nvme_writeq((ulong)priv->tcbs[nvmeq->qid],
			    ((void __iomem *)dev->bar) + ANS_NVMMU_BASE_IOSQ);
		break;
	}

	return 0;
}

static void apple_nvme_submit_cmd(struct nvme_queue *nvmeq,
				  struct nvme_command *cmd)
{
	struct apple_nvme_priv *priv =
		container_of(nvmeq->dev, struct apple_nvme_priv, ndev);
	struct ans_nvmmu_tcb *tcb;
	u16 tail = nvmeq->sq_tail;

	tcb = ((void *)priv->tcbs[nvmeq->qid]) + tail * ANS_NVMMU_TCB_PITCH;
	memset(tcb, 0, sizeof(*tcb));
	tcb->opcode = cmd->common.opcode;
	tcb->flags = ANS_NVMMU_TCB_WRITE | ANS_NVMMU_TCB_READ;
	tcb->slot = tail;
	tcb->prpl_len = cmd->rw.length;
	tcb->prp1 = cmd->common.prp1;
	tcb->prp2 = cmd->common.prp2;

	writel(tail, priv->q_db[nvmeq->qid]);
}

static void apple_nvme_complete_cmd(struct nvme_queue *nvmeq,
				    struct nvme_command *cmd)
{
	struct apple_nvme_priv *priv =
		container_of(nvmeq->dev, struct apple_nvme_priv, ndev);
	struct ans_nvmmu_tcb *tcb;
	u16 tail = nvmeq->sq_tail;

	tcb = ((void *)priv->tcbs[nvmeq->qid]) + tail * ANS_NVMMU_TCB_PITCH;
	memset(tcb, 0, sizeof(*tcb));
	writel(tail, ((void __iomem *)nvmeq->dev->bar) + ANS_NVMMU_TCB_INVAL);
	readl(((void __iomem *)nvmeq->dev->bar) + ANS_NVMMU_TCB_STAT);

	if (++tail == nvmeq->q_depth)
		tail = 0;
	nvmeq->sq_tail = tail;
}

static int nvme_shmem_setup(void *cookie, struct apple_rtkit_buffer *buf)
{
	struct apple_nvme_priv *priv = (struct apple_nvme_priv *)cookie;

	if (!buf || buf->dva || !buf->size)
		return -1;

	buf->buffer = memalign(SZ_16K, ALIGN(buf->size, SZ_16K));
	if (!buf->buffer)
		return -ENOMEM;

	if (!sart_add_allowed_region(priv->sart, buf->buffer, buf->size)) {
		free(buf->buffer);
		buf->buffer = NULL;
		buf->size = 0;
		return -1;
	}

	buf->dva = (u64)buf->buffer;

	return 0;
}

static void nvme_shmem_destroy(void *cookie, struct apple_rtkit_buffer *buf)
{
	struct apple_nvme_priv *priv = (struct apple_nvme_priv *)cookie;

	if (!buf)
		return;

	if (buf->buffer) {
		sart_remove_allowed_region(priv->sart, buf->buffer, buf->size);
		free(buf->buffer);
		buf->buffer = NULL;
		buf->size = 0;
		buf->dva = 0;
	}
}

static int apple_nvme_probe(struct udevice *dev)
{
	struct apple_nvme_priv *priv = dev_get_priv(dev);
	fdt_addr_t addr;
	ofnode of_sart;
	u32 ctrl, stat, phandle;
	int ret;

	priv->base = dev_read_addr_ptr(dev);
	if (!priv->base)
		return -EINVAL;

	addr = dev_read_addr_index(dev, 1);
	if (addr == FDT_ADDR_T_NONE)
		return -EINVAL;
	priv->asc = map_sysmem(addr, 0);

	ret = reset_get_bulk(dev, &priv->resets);
	if (ret < 0)
		return ret;

	ret = mbox_get_by_index(dev, 0, &priv->chan);
	if (ret < 0)
		return ret;

	ret = dev_read_u32(dev, "apple,sart", &phandle);
	if (ret < 0)
		return ret;

	of_sart = ofnode_get_by_phandle(phandle);
	priv->sart = sart_init(of_sart);
	if (!priv->sart)
		return -EINVAL;

	ctrl = readl(priv->asc + REG_CPU_CTRL);
	writel(ctrl | REG_CPU_CTRL_RUN, priv->asc + REG_CPU_CTRL);

	priv->rtk = apple_rtkit_init(&priv->chan, priv, nvme_shmem_setup, nvme_shmem_destroy);
	if (!priv->rtk)
		return -ENOMEM;

	ret = apple_rtkit_boot(priv->rtk);
	if (ret < 0) {
		printf("%s: NVMe apple_rtkit_boot returned: %d\n", __func__, ret);
		return ret;
	}

	ret = readl_poll_sleep_timeout(priv->base + ANS_BOOT_STATUS, stat,
				       (stat == ANS_BOOT_STATUS_OK), 100,
				       500000);
	if (ret < 0) {
		printf("%s: NVMe firmware didn't boot\n", __func__);
		return -ETIMEDOUT;
	}

	writel(ANS_LINEAR_SQ_CTRL_EN, priv->base + ANS_LINEAR_SQ_CTRL);
	writel(((ANS_MAX_QUEUE_DEPTH << 16) | ANS_MAX_QUEUE_DEPTH),
	       priv->base + ANS_MAX_PEND_CMDS_CTRL);

	writel(readl(priv->base + ANS_UNKNOWN_CTRL) & ~ANS_PRP_NULL_CHECK,
	       priv->base + ANS_UNKNOWN_CTRL);

	strcpy(priv->ndev.vendor, "Apple");

	writel((ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH) - 1,
	       priv->base + ANS_NVMMU_NUM);
	writel(0, priv->base + ANS_MODESEL);

	priv->ndev.bar = priv->base;
	return nvme_init(dev);
}

static int apple_nvme_remove(struct udevice *dev)
{
	struct apple_nvme_priv *priv = dev_get_priv(dev);
	u32 ctrl;

	nvme_shutdown(dev);

	apple_rtkit_shutdown(priv->rtk, APPLE_RTKIT_PWR_STATE_SLEEP);

	ctrl = readl(priv->asc + REG_CPU_CTRL);
	writel(ctrl & ~REG_CPU_CTRL_RUN, priv->asc + REG_CPU_CTRL);

	apple_rtkit_free(priv->rtk);
	priv->rtk = NULL;

	sart_free(priv->sart);
	priv->sart = NULL;

	reset_assert_bulk(&priv->resets);
	reset_deassert_bulk(&priv->resets);

	return 0;
}

static const struct nvme_ops apple_nvme_ops = {
	.setup_queue = apple_nvme_setup_queue,
	.submit_cmd = apple_nvme_submit_cmd,
	.complete_cmd = apple_nvme_complete_cmd,
};

static const struct udevice_id apple_nvme_ids[] = {
	{ .compatible = "apple,nvme-ans2" },
	{ /* sentinel */ }
};

U_BOOT_DRIVER(apple_nvme) = {
	.name = "apple_nvme",
	.id = UCLASS_NVME,
	.of_match = apple_nvme_ids,
	.priv_auto = sizeof(struct apple_nvme_priv),
	.probe = apple_nvme_probe,
	.remove = apple_nvme_remove,
	.ops = &apple_nvme_ops,
	.flags = DM_FLAG_OS_PREPARE,
};
Commit	Line	Data
50333c94 MK	1	// SPDX-License-Identifier: GPL-2.0+
	2	/*
	3	* (C) Copyright 2021 Mark Kettenis <kettenis@openbsd.org>
	4	*/
	5
d678a59d	6	#include <common.h>
50333c94 MK	7	#include <dm.h>
	8	#include <mailbox.h>
	9	#include <mapmem.h>
	10	#include "nvme.h"
	11	#include <reset.h>
	12
	13	#include <asm/io.h>
	14	#include <asm/arch/rtkit.h>
e44d59c6	15	#include <asm/arch/sart.h>
50333c94 MK	16	#include <linux/iopoll.h>
	17
	18	/* ASC registers */
	19	#define REG_CPU_CTRL 0x0044
	20	#define REG_CPU_CTRL_RUN BIT(4)
	21
	22	/* Apple NVMe registers */
	23	#define ANS_MAX_PEND_CMDS_CTRL 0x01210
	24	#define ANS_MAX_QUEUE_DEPTH 64
	25	#define ANS_BOOT_STATUS 0x01300
	26	#define ANS_BOOT_STATUS_OK 0xde71ce55
	27	#define ANS_MODESEL 0x01304
	28	#define ANS_UNKNOWN_CTRL 0x24008
	29	#define ANS_PRP_NULL_CHECK (1 << 11)
	30	#define ANS_LINEAR_SQ_CTRL 0x24908
	31	#define ANS_LINEAR_SQ_CTRL_EN (1 << 0)
	32	#define ANS_ASQ_DB 0x2490c
	33	#define ANS_IOSQ_DB 0x24910
	34	#define ANS_NVMMU_NUM 0x28100
	35	#define ANS_NVMMU_BASE_ASQ 0x28108
	36	#define ANS_NVMMU_BASE_IOSQ 0x28110
	37	#define ANS_NVMMU_TCB_INVAL 0x28118
	38	#define ANS_NVMMU_TCB_STAT 0x28120
	39
	40	#define ANS_NVMMU_TCB_SIZE 0x4000
	41	#define ANS_NVMMU_TCB_PITCH 0x80
	42
	43	/*
	44	* The Apple NVMe controller includes an IOMMU known as NVMMU. The
	45	* NVMMU is programmed through an array of TCBs. These TCBs are paired
	46	* with the corresponding slot in the submission queues and need to be
	47	* configured with the command details before a command is allowed to
	48	* execute. This is necessary even for commands that don't do DMA.
	49	*/
	50	struct ans_nvmmu_tcb {
	51	u8 opcode;
	52	u8 flags;
	53	u8 slot;
	54	u8 pad0;
	55	u32 prpl_len;
	56	u8 pad1[16];
	57	u64 prp1;
	58	u64 prp2;
	59	};
	60
	61	#define ANS_NVMMU_TCB_WRITE BIT(0)
	62	#define ANS_NVMMU_TCB_READ BIT(1)
	63
	64	struct apple_nvme_priv {
	65	struct nvme_dev ndev;
	66	void base; / NVMe registers */
	67	void asc; / ASC registers */
	68	struct reset_ctl_bulk resets; /* ASC reset */
	69	struct mbox_chan chan;
e44d59c6 JG	70	struct apple_sart *sart;
e44d59c6 JG	71	struct apple_rtkit *rtk;
50333c94 MK	72	struct ans_nvmmu_tcb tcbs[NVME_Q_NUM]; / Submission queue TCBs */
	73	u32 __iomem q_db[NVME_Q_NUM]; / Submission queue doorbell */
	74	};
	75
	76	static int apple_nvme_setup_queue(struct nvme_queue *nvmeq)
	77	{
	78	struct apple_nvme_priv *priv =
	79	container_of(nvmeq->dev, struct apple_nvme_priv, ndev);
	80	struct nvme_dev *dev = nvmeq->dev;
	81
	82	switch (nvmeq->qid) {
	83	case NVME_ADMIN_Q:
	84	case NVME_IO_Q:
	85	break;
	86	default:
	87	return -EINVAL;
	88	}
	89
	90	priv->tcbs[nvmeq->qid] = (void *)memalign(4096, ANS_NVMMU_TCB_SIZE);
	91	memset((void *)priv->tcbs[nvmeq->qid], 0, ANS_NVMMU_TCB_SIZE);
	92
	93	switch (nvmeq->qid) {
	94	case NVME_ADMIN_Q:
	95	priv->q_db[nvmeq->qid] =
	96	((void __iomem *)dev->bar) + ANS_ASQ_DB;
	97	nvme_writeq((ulong)priv->tcbs[nvmeq->qid],
	98	((void __iomem *)dev->bar) + ANS_NVMMU_BASE_ASQ);
	99	break;
	100	case NVME_IO_Q:
	101	priv->q_db[nvmeq->qid] =
	102	((void __iomem *)dev->bar) + ANS_IOSQ_DB;
	103	nvme_writeq((ulong)priv->tcbs[nvmeq->qid],
	104	((void __iomem *)dev->bar) + ANS_NVMMU_BASE_IOSQ);
	105	break;
	106	}
	107
	108	return 0;
	109	}
	110
	111	static void apple_nvme_submit_cmd(struct nvme_queue *nvmeq,
	112	struct nvme_command *cmd)
	113	{
	114	struct apple_nvme_priv *priv =
	115	container_of(nvmeq->dev, struct apple_nvme_priv, ndev);
	116	struct ans_nvmmu_tcb *tcb;
	117	u16 tail = nvmeq->sq_tail;
	118
	119	tcb = ((void )priv->tcbs[nvmeq->qid]) + tail ANS_NVMMU_TCB_PITCH;
	120	memset(tcb, 0, sizeof(*tcb));
	121	tcb->opcode = cmd->common.opcode;
	122	tcb->flags = ANS_NVMMU_TCB_WRITE \| ANS_NVMMU_TCB_READ;
	123	tcb->slot = tail;
	124	tcb->prpl_len = cmd->rw.length;
	125	tcb->prp1 = cmd->common.prp1;
	126	tcb->prp2 = cmd->common.prp2;
	127
	128	writel(tail, priv->q_db[nvmeq->qid]);
	129	}
	130
	131	static void apple_nvme_complete_cmd(struct nvme_queue *nvmeq,
	132	struct nvme_command *cmd)
	133	{
	134	struct apple_nvme_priv *priv =
	135	container_of(nvmeq->dev, struct apple_nvme_priv, ndev);
136	struct ans_nvmmu_tcb *tcb;
137	u16 tail = nvmeq->sq_tail;
138
139	tcb = ((void )priv->tcbs[nvmeq->qid]) + tail ANS_NVMMU_TCB_PITCH;
140	memset(tcb, 0, sizeof(*tcb));
141	writel(tail, ((void __iomem *)nvmeq->dev->bar) + ANS_NVMMU_TCB_INVAL);
142	readl(((void __iomem *)nvmeq->dev->bar) + ANS_NVMMU_TCB_STAT);
143
144	if (++tail == nvmeq->q_depth)
145	tail = 0;
146	nvmeq->sq_tail = tail;
147	}
148
e44d59c6 JG	149	static int nvme_shmem_setup(void cookie, struct apple_rtkit_buffer buf)
	150	{
	151	struct apple_nvme_priv priv = (struct apple_nvme_priv )cookie;
	152
	153	if (!buf \|\| buf->dva \|\| !buf->size)
	154	return -1;
	155
	156	buf->buffer = memalign(SZ_16K, ALIGN(buf->size, SZ_16K));
	157	if (!buf->buffer)
	158	return -ENOMEM;
	159
	160	if (!sart_add_allowed_region(priv->sart, buf->buffer, buf->size)) {
	161	free(buf->buffer);
	162	buf->buffer = NULL;
	163	buf->size = 0;
	164	return -1;
	165	}
	166
	167	buf->dva = (u64)buf->buffer;
	168
	169	return 0;
	170	}
	171
	172	static void nvme_shmem_destroy(void cookie, struct apple_rtkit_buffer buf)
	173	{
	174	struct apple_nvme_priv priv = (struct apple_nvme_priv )cookie;
	175
	176	if (!buf)
	177	return;
	178
	179	if (buf->buffer) {
	180	sart_remove_allowed_region(priv->sart, buf->buffer, buf->size);
	181	free(buf->buffer);
	182	buf->buffer = NULL;
	183	buf->size = 0;
	184	buf->dva = 0;
	185	}
	186	}
	187
50333c94 MK	188	static int apple_nvme_probe(struct udevice *dev)
	189	{
	190	struct apple_nvme_priv *priv = dev_get_priv(dev);
	191	fdt_addr_t addr;
e44d59c6 JG	192	ofnode of_sart;
e44d59c6 JG	193	u32 ctrl, stat, phandle;
50333c94 MK	194	int ret;
	195
	196	priv->base = dev_read_addr_ptr(dev);
	197	if (!priv->base)
	198	return -EINVAL;
	199
	200	addr = dev_read_addr_index(dev, 1);
	201	if (addr == FDT_ADDR_T_NONE)
	202	return -EINVAL;
	203	priv->asc = map_sysmem(addr, 0);
	204
	205	ret = reset_get_bulk(dev, &priv->resets);
	206	if (ret < 0)
	207	return ret;
	208
	209	ret = mbox_get_by_index(dev, 0, &priv->chan);
	210	if (ret < 0)
	211	return ret;
	212
e44d59c6 JG	213	ret = dev_read_u32(dev, "apple,sart", &phandle);
	214	if (ret < 0)
	215	return ret;
	216
	217	of_sart = ofnode_get_by_phandle(phandle);
	218	priv->sart = sart_init(of_sart);
	219	if (!priv->sart)
	220	return -EINVAL;
	221
50333c94 MK	222	ctrl = readl(priv->asc + REG_CPU_CTRL);
	223	writel(ctrl \| REG_CPU_CTRL_RUN, priv->asc + REG_CPU_CTRL);
	224
e44d59c6 JG	225	priv->rtk = apple_rtkit_init(&priv->chan, priv, nvme_shmem_setup, nvme_shmem_destroy);
	226	if (!priv->rtk)
	227	return -ENOMEM;
	228
	229	ret = apple_rtkit_boot(priv->rtk);
	230	if (ret < 0) {
	231	printf("%s: NVMe apple_rtkit_boot returned: %d\n", __func__, ret);
50333c94	232	return ret;
e44d59c6	233	}
50333c94 MK	234
	235	ret = readl_poll_sleep_timeout(priv->base + ANS_BOOT_STATUS, stat,
	236	(stat == ANS_BOOT_STATUS_OK), 100,
	237	500000);
	238	if (ret < 0) {
	239	printf("%s: NVMe firmware didn't boot\n", __func__);
	240	return -ETIMEDOUT;
	241	}
	242
	243	writel(ANS_LINEAR_SQ_CTRL_EN, priv->base + ANS_LINEAR_SQ_CTRL);
	244	writel(((ANS_MAX_QUEUE_DEPTH << 16) \| ANS_MAX_QUEUE_DEPTH),
	245	priv->base + ANS_MAX_PEND_CMDS_CTRL);
	246
	247	writel(readl(priv->base + ANS_UNKNOWN_CTRL) & ~ANS_PRP_NULL_CHECK,
	248	priv->base + ANS_UNKNOWN_CTRL);
	249
	250	strcpy(priv->ndev.vendor, "Apple");
	251
	252	writel((ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH) - 1,
	253	priv->base + ANS_NVMMU_NUM);
	254	writel(0, priv->base + ANS_MODESEL);
	255
	256	priv->ndev.bar = priv->base;
	257	return nvme_init(dev);
	258	}
	259
	260	static int apple_nvme_remove(struct udevice *dev)
	261	{
	262	struct apple_nvme_priv *priv = dev_get_priv(dev);
	263	u32 ctrl;
	264
	265	nvme_shutdown(dev);
	266
e44d59c6	267	apple_rtkit_shutdown(priv->rtk, APPLE_RTKIT_PWR_STATE_SLEEP);
50333c94 MK	268
	269	ctrl = readl(priv->asc + REG_CPU_CTRL);
	270	writel(ctrl & ~REG_CPU_CTRL_RUN, priv->asc + REG_CPU_CTRL);
	271
e44d59c6 JG	272	apple_rtkit_free(priv->rtk);
	273	priv->rtk = NULL;
	274
	275	sart_free(priv->sart);
	276	priv->sart = NULL;
	277
50333c94 MK	278	reset_assert_bulk(&priv->resets);
	279	reset_deassert_bulk(&priv->resets);
	280
	281	return 0;
	282	}
	283
	284	static const struct nvme_ops apple_nvme_ops = {
	285	.setup_queue = apple_nvme_setup_queue,
	286	.submit_cmd = apple_nvme_submit_cmd,
	287	.complete_cmd = apple_nvme_complete_cmd,
	288	};
	289
	290	static const struct udevice_id apple_nvme_ids[] = {
	291	{ .compatible = "apple,nvme-ans2" },
	292	{ /* sentinel */ }
	293	};
	294
	295	U_BOOT_DRIVER(apple_nvme) = {
	296	.name = "apple_nvme",
	297	.id = UCLASS_NVME,
	298	.of_match = apple_nvme_ids,
	299	.priv_auto = sizeof(struct apple_nvme_priv),
	300	.probe = apple_nvme_probe,
	301	.remove = apple_nvme_remove,
	302	.ops = &apple_nvme_ops,
	303	.flags = DM_FLAG_OS_PREPARE,
	304	};