[thirdparty/kernel/stable.git] / drivers / nvme / host / multipath.c

/*
 * Copyright (c) 2017-2018 Christoph Hellwig.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <linux/moduleparam.h>
#include <trace/events/block.h>
#include "nvme.h"

static bool multipath = true;
module_param(multipath, bool, 0444);
MODULE_PARM_DESC(multipath,
	"turn on native support for multiple controllers per subsystem");

inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
{
	return multipath && ctrl->subsys && (ctrl->subsys->cmic & (1 << 3));
}

/*
 * If multipathing is enabled we need to always use the subsystem instance
 * number for numbering our devices to avoid conflicts between subsystems that
 * have multiple controllers and thus use the multipath-aware subsystem node
 * and those that have a single controller and use the controller node
 * directly.
 */
void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
			struct nvme_ctrl *ctrl, int *flags)
{
	if (!multipath) {
		sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
	} else if (ns->head->disk) {
		sprintf(disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
				ctrl->cntlid, ns->head->instance);
		*flags = GENHD_FL_HIDDEN;
	} else {
		sprintf(disk_name, "nvme%dn%d", ctrl->subsys->instance,
				ns->head->instance);
	}
}

void nvme_failover_req(struct request *req)
{
	struct nvme_ns *ns = req->q->queuedata;
	u16 status = nvme_req(req)->status;
	unsigned long flags;

	spin_lock_irqsave(&ns->head->requeue_lock, flags);
	blk_steal_bios(&ns->head->requeue_list, req);
	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
	blk_mq_end_request(req, 0);

	switch (status & 0x7ff) {
	case NVME_SC_ANA_TRANSITION:
	case NVME_SC_ANA_INACCESSIBLE:
	case NVME_SC_ANA_PERSISTENT_LOSS:
		/*
		 * If we got back an ANA error we know the controller is alive,
		 * but not ready to serve this namespaces.  The spec suggests
		 * we should update our general state here, but due to the fact
		 * that the admin and I/O queues are not serialized that is
		 * fundamentally racy.  So instead just clear the current path,
		 * mark the the path as pending and kick of a re-read of the ANA
		 * log page ASAP.
		 */
		nvme_mpath_clear_current_path(ns);
		if (ns->ctrl->ana_log_buf) {
			set_bit(NVME_NS_ANA_PENDING, &ns->flags);
			queue_work(nvme_wq, &ns->ctrl->ana_work);
		}
		break;
	case NVME_SC_HOST_PATH_ERROR:
		/*
		 * Temporary transport disruption in talking to the controller.
		 * Try to send on a new path.
		 */
		nvme_mpath_clear_current_path(ns);
		break;
	default:
		/*
		 * Reset the controller for any non-ANA error as we don't know
		 * what caused the error.
		 */
		nvme_reset_ctrl(ns->ctrl);
		break;
	}

	kblockd_schedule_work(&ns->head->requeue_work);
}

void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns;

	down_read(&ctrl->namespaces_rwsem);
	list_for_each_entry(ns, &ctrl->namespaces, list) {
		if (ns->head->disk)
			kblockd_schedule_work(&ns->head->requeue_work);
	}
	up_read(&ctrl->namespaces_rwsem);
}

static const char *nvme_ana_state_names[] = {
	[0]				= "invalid state",
	[NVME_ANA_OPTIMIZED]		= "optimized",
	[NVME_ANA_NONOPTIMIZED]		= "non-optimized",
	[NVME_ANA_INACCESSIBLE]		= "inaccessible",
	[NVME_ANA_PERSISTENT_LOSS]	= "persistent-loss",
	[NVME_ANA_CHANGE]		= "change",
};

void nvme_mpath_clear_current_path(struct nvme_ns *ns)
{
	struct nvme_ns_head *head = ns->head;
	int node;

	if (!head)
		return;

	for_each_node(node) {
		if (ns == rcu_access_pointer(head->current_path[node]))
			rcu_assign_pointer(head->current_path[node], NULL);
	}
}

static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
{
	int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
	struct nvme_ns *found = NULL, *fallback = NULL, *ns;

	list_for_each_entry_rcu(ns, &head->list, siblings) {
		if (ns->ctrl->state != NVME_CTRL_LIVE ||
		    test_bit(NVME_NS_ANA_PENDING, &ns->flags))
			continue;

		distance = node_distance(node, dev_to_node(ns->ctrl->dev));

		switch (ns->ana_state) {
		case NVME_ANA_OPTIMIZED:
			if (distance < found_distance) {
				found_distance = distance;
				found = ns;
			}
			break;
		case NVME_ANA_NONOPTIMIZED:
			if (distance < fallback_distance) {
				fallback_distance = distance;
				fallback = ns;
			}
			break;
		default:
			break;
		}
	}

	if (!found)
		found = fallback;
	if (found)
		rcu_assign_pointer(head->current_path[node], found);
	return found;
}

static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
{
	return ns->ctrl->state == NVME_CTRL_LIVE &&
		ns->ana_state == NVME_ANA_OPTIMIZED;
}

inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
{
	int node = numa_node_id();
	struct nvme_ns *ns;

	ns = srcu_dereference(head->current_path[node], &head->srcu);
	if (unlikely(!ns || !nvme_path_is_optimized(ns)))
		ns = __nvme_find_path(head, node);
	return ns;
}

static blk_qc_t nvme_ns_head_make_request(struct request_queue *q,
		struct bio *bio)
{
	struct nvme_ns_head *head = q->queuedata;
	struct device *dev = disk_to_dev(head->disk);
	struct nvme_ns *ns;
	blk_qc_t ret = BLK_QC_T_NONE;
	int srcu_idx;

	srcu_idx = srcu_read_lock(&head->srcu);
	ns = nvme_find_path(head);
	if (likely(ns)) {
		bio->bi_disk = ns->disk;
		bio->bi_opf |= REQ_NVME_MPATH;
		trace_block_bio_remap(bio->bi_disk->queue, bio,
				      disk_devt(ns->head->disk),
				      bio->bi_iter.bi_sector);
		ret = direct_make_request(bio);
	} else if (!list_empty_careful(&head->list)) {
		dev_warn_ratelimited(dev, "no path available - requeuing I/O\n");

		spin_lock_irq(&head->requeue_lock);
		bio_list_add(&head->requeue_list, bio);
		spin_unlock_irq(&head->requeue_lock);
	} else {
		dev_warn_ratelimited(dev, "no path - failing I/O\n");

		bio->bi_status = BLK_STS_IOERR;
		bio_endio(bio);
	}

	srcu_read_unlock(&head->srcu, srcu_idx);
	return ret;
}

static int nvme_ns_head_poll(struct request_queue *q, blk_qc_t qc, bool spin)
{
	struct nvme_ns_head *head = q->queuedata;
	struct nvme_ns *ns;
	int found = 0;
	int srcu_idx;

	srcu_idx = srcu_read_lock(&head->srcu);
	ns = srcu_dereference(head->current_path[numa_node_id()], &head->srcu);
	if (likely(ns && nvme_path_is_optimized(ns)))
		found = ns->queue->poll_fn(q, qc, spin);
	srcu_read_unlock(&head->srcu, srcu_idx);
	return found;
}

static void nvme_requeue_work(struct work_struct *work)
{
	struct nvme_ns_head *head =
		container_of(work, struct nvme_ns_head, requeue_work);
	struct bio *bio, *next;

	spin_lock_irq(&head->requeue_lock);
	next = bio_list_get(&head->requeue_list);
	spin_unlock_irq(&head->requeue_lock);

	while ((bio = next) != NULL) {
		next = bio->bi_next;
		bio->bi_next = NULL;

		/*
		 * Reset disk to the mpath node and resubmit to select a new
		 * path.
		 */
		bio->bi_disk = head->disk;
		generic_make_request(bio);
	}
}

int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
{
	struct request_queue *q;
	bool vwc = false;

	mutex_init(&head->lock);
	bio_list_init(&head->requeue_list);
	spin_lock_init(&head->requeue_lock);
	INIT_WORK(&head->requeue_work, nvme_requeue_work);

	/*
	 * Add a multipath node if the subsystems supports multiple controllers.
	 * We also do this for private namespaces as the namespace sharing data could
	 * change after a rescan.
	 */
	if (!(ctrl->subsys->cmic & (1 << 1)) || !multipath)
		return 0;

	q = blk_alloc_queue_node(GFP_KERNEL, NUMA_NO_NODE);
	if (!q)
		goto out;
	q->queuedata = head;
	blk_queue_make_request(q, nvme_ns_head_make_request);
	q->poll_fn = nvme_ns_head_poll;
	blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
	/* set to a default value for 512 until disk is validated */
	blk_queue_logical_block_size(q, 512);
	blk_set_stacking_limits(&q->limits);

	/* we need to propagate up the VMC settings */
	if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
		vwc = true;
	blk_queue_write_cache(q, vwc, vwc);

	head->disk = alloc_disk(0);
	if (!head->disk)
		goto out_cleanup_queue;
	head->disk->fops = &nvme_ns_head_ops;
	head->disk->private_data = head;
	head->disk->queue = q;
	head->disk->flags = GENHD_FL_EXT_DEVT;
	sprintf(head->disk->disk_name, "nvme%dn%d",
			ctrl->subsys->instance, head->instance);
	return 0;

out_cleanup_queue:
	blk_cleanup_queue(q);
out:
	return -ENOMEM;
}

static void nvme_mpath_set_live(struct nvme_ns *ns)
{
	struct nvme_ns_head *head = ns->head;

	lockdep_assert_held(&ns->head->lock);

	if (!head->disk)
		return;

	if (!(head->disk->flags & GENHD_FL_UP))
		device_add_disk(&head->subsys->dev, head->disk,
				nvme_ns_id_attr_groups);

	if (nvme_path_is_optimized(ns)) {
		int node, srcu_idx;

		srcu_idx = srcu_read_lock(&head->srcu);
		for_each_node(node)
			__nvme_find_path(head, node);
		srcu_read_unlock(&head->srcu, srcu_idx);
	}

	kblockd_schedule_work(&ns->head->requeue_work);
}

static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
		int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *,
			void *))
{
	void *base = ctrl->ana_log_buf;
	size_t offset = sizeof(struct nvme_ana_rsp_hdr);
	int error, i;

	lockdep_assert_held(&ctrl->ana_lock);

	for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
		struct nvme_ana_group_desc *desc = base + offset;
		u32 nr_nsids = le32_to_cpu(desc->nnsids);
		size_t nsid_buf_size = nr_nsids * sizeof(__le32);

		if (WARN_ON_ONCE(desc->grpid == 0))
			return -EINVAL;
		if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))
			return -EINVAL;
		if (WARN_ON_ONCE(desc->state == 0))
			return -EINVAL;
		if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
			return -EINVAL;

		offset += sizeof(*desc);
		if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
			return -EINVAL;

		error = cb(ctrl, desc, data);
		if (error)
			return error;

		offset += nsid_buf_size;
		if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
			return -EINVAL;
	}

	return 0;
}

static inline bool nvme_state_is_live(enum nvme_ana_state state)
{
	return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
}

static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
		struct nvme_ns *ns)
{
	enum nvme_ana_state old;

	mutex_lock(&ns->head->lock);
	old = ns->ana_state;
	ns->ana_grpid = le32_to_cpu(desc->grpid);
	ns->ana_state = desc->state;
	clear_bit(NVME_NS_ANA_PENDING, &ns->flags);

	if (nvme_state_is_live(ns->ana_state) && !nvme_state_is_live(old))
		nvme_mpath_set_live(ns);
	mutex_unlock(&ns->head->lock);
}

static int nvme_update_ana_state(struct nvme_ctrl *ctrl,
		struct nvme_ana_group_desc *desc, void *data)
{
	u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
	unsigned *nr_change_groups = data;
	struct nvme_ns *ns;

	dev_info(ctrl->device, "ANA group %d: %s.\n",
			le32_to_cpu(desc->grpid),
			nvme_ana_state_names[desc->state]);

	if (desc->state == NVME_ANA_CHANGE)
		(*nr_change_groups)++;

	if (!nr_nsids)
		return 0;

	down_write(&ctrl->namespaces_rwsem);
	list_for_each_entry(ns, &ctrl->namespaces, list) {
		if (ns->head->ns_id != le32_to_cpu(desc->nsids[n]))
			continue;
		nvme_update_ns_ana_state(desc, ns);
		if (++n == nr_nsids)
			break;
	}
	up_write(&ctrl->namespaces_rwsem);
	WARN_ON_ONCE(n < nr_nsids);
	return 0;
}

static int nvme_read_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
{
	u32 nr_change_groups = 0;
	int error;

	mutex_lock(&ctrl->ana_lock);
	error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA,
			groups_only ? NVME_ANA_LOG_RGO : 0,
			ctrl->ana_log_buf, ctrl->ana_log_size, 0);
	if (error) {
		dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
		goto out_unlock;
	}

	error = nvme_parse_ana_log(ctrl, &nr_change_groups,
			nvme_update_ana_state);
	if (error)
		goto out_unlock;

	/*
	 * In theory we should have an ANATT timer per group as they might enter
	 * the change state at different times.  But that is a lot of overhead
	 * just to protect against a target that keeps entering new changes
	 * states while never finishing previous ones.  But we'll still
	 * eventually time out once all groups are in change state, so this
	 * isn't a big deal.
	 *
	 * We also double the ANATT value to provide some slack for transports
	 * or AEN processing overhead.
	 */
	if (nr_change_groups)
		mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);
	else
		del_timer_sync(&ctrl->anatt_timer);
out_unlock:
	mutex_unlock(&ctrl->ana_lock);
	return error;
}

static void nvme_ana_work(struct work_struct *work)
{
	struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);

	nvme_read_ana_log(ctrl, false);
}

static void nvme_anatt_timeout(struct timer_list *t)
{
	struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);

	dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");
	nvme_reset_ctrl(ctrl);
}

void nvme_mpath_stop(struct nvme_ctrl *ctrl)
{
	if (!nvme_ctrl_use_ana(ctrl))
		return;
	del_timer_sync(&ctrl->anatt_timer);
	cancel_work_sync(&ctrl->ana_work);
}

static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
		char *buf)
{
	return sprintf(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid);
}
DEVICE_ATTR_RO(ana_grpid);

static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
		char *buf)
{
	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);

	return sprintf(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);
}
DEVICE_ATTR_RO(ana_state);

static int nvme_set_ns_ana_state(struct nvme_ctrl *ctrl,
		struct nvme_ana_group_desc *desc, void *data)
{
	struct nvme_ns *ns = data;

	if (ns->ana_grpid == le32_to_cpu(desc->grpid)) {
		nvme_update_ns_ana_state(desc, ns);
		return -ENXIO; /* just break out of the loop */
	}

	return 0;
}

void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id)
{
	if (nvme_ctrl_use_ana(ns->ctrl)) {
		mutex_lock(&ns->ctrl->ana_lock);
		ns->ana_grpid = le32_to_cpu(id->anagrpid);
		nvme_parse_ana_log(ns->ctrl, ns, nvme_set_ns_ana_state);
		mutex_unlock(&ns->ctrl->ana_lock);
	} else {
		mutex_lock(&ns->head->lock);
		ns->ana_state = NVME_ANA_OPTIMIZED; 
		nvme_mpath_set_live(ns);
		mutex_unlock(&ns->head->lock);
	}
}

void nvme_mpath_remove_disk(struct nvme_ns_head *head)
{
	if (!head->disk)
		return;
	if (head->disk->flags & GENHD_FL_UP)
		del_gendisk(head->disk);
	blk_set_queue_dying(head->disk->queue);
	/* make sure all pending bios are cleaned up */
	kblockd_schedule_work(&head->requeue_work);
	flush_work(&head->requeue_work);
	blk_cleanup_queue(head->disk->queue);
	put_disk(head->disk);
}

int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
{
	int error;

	if (!nvme_ctrl_use_ana(ctrl))
		return 0;

	ctrl->anacap = id->anacap;
	ctrl->anatt = id->anatt;
	ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
	ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);

	mutex_init(&ctrl->ana_lock);
	timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
	ctrl->ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
		ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc);
	if (!(ctrl->anacap & (1 << 6)))
		ctrl->ana_log_size += ctrl->max_namespaces * sizeof(__le32);

	if (ctrl->ana_log_size > ctrl->max_hw_sectors << SECTOR_SHIFT) {
		dev_err(ctrl->device,
			"ANA log page size (%zd) larger than MDTS (%d).\n",
			ctrl->ana_log_size,
			ctrl->max_hw_sectors << SECTOR_SHIFT);
		dev_err(ctrl->device, "disabling ANA support.\n");
		return 0;
	}

	INIT_WORK(&ctrl->ana_work, nvme_ana_work);
	ctrl->ana_log_buf = kmalloc(ctrl->ana_log_size, GFP_KERNEL);
	if (!ctrl->ana_log_buf) {
		error = -ENOMEM;
		goto out;
	}

	error = nvme_read_ana_log(ctrl, true);
	if (error)
		goto out_free_ana_log_buf;
	return 0;
out_free_ana_log_buf:
	kfree(ctrl->ana_log_buf);
out:
	return error;
}

void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
{
	kfree(ctrl->ana_log_buf);
}
Commit	Line	Data
32acab31	1	/*
0d0b660f	2	* Copyright (c) 2017-2018 Christoph Hellwig.
32acab31 CH	3	*
	4	* This program is free software; you can redistribute it and/or modify it
	5	* under the terms and conditions of the GNU General Public License,
	6	* version 2, as published by the Free Software Foundation.
	7	*
	8	* This program is distributed in the hope it will be useful, but WITHOUT
	9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	10	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	11	* more details.
	12	*/
	13
	14	#include <linux/moduleparam.h>
2796b569	15	#include <trace/events/block.h>
32acab31 CH	16	#include "nvme.h"
	17
	18	static bool multipath = true;
5cadde80	19	module_param(multipath, bool, 0444);
32acab31 CH	20	MODULE_PARM_DESC(multipath,
	21	"turn on native support for multiple controllers per subsystem");
	22
0d0b660f CH	23	inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
0d0b660f CH	24	{
8f220c41	25	return multipath && ctrl->subsys && (ctrl->subsys->cmic & (1 << 3));
0d0b660f CH	26	}
0d0b660f CH	27
a785dbcc KB	28	/*
	29	* If multipathing is enabled we need to always use the subsystem instance
	30	* number for numbering our devices to avoid conflicts between subsystems that
	31	* have multiple controllers and thus use the multipath-aware subsystem node
	32	* and those that have a single controller and use the controller node
	33	* directly.
	34	*/
	35	void nvme_set_disk_name(char disk_name, struct nvme_ns ns,
	36	struct nvme_ctrl ctrl, int flags)
	37	{
	38	if (!multipath) {
	39	sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
	40	} else if (ns->head->disk) {
	41	sprintf(disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
	42	ctrl->cntlid, ns->head->instance);
	43	*flags = GENHD_FL_HIDDEN;
	44	} else {
	45	sprintf(disk_name, "nvme%dn%d", ctrl->subsys->instance,
	46	ns->head->instance);
	47	}
	48	}
	49
32acab31 CH	50	void nvme_failover_req(struct request *req)
	51	{
	52	struct nvme_ns *ns = req->q->queuedata;
0d0b660f	53	u16 status = nvme_req(req)->status;
32acab31 CH	54	unsigned long flags;
	55
	56	spin_lock_irqsave(&ns->head->requeue_lock, flags);
	57	blk_steal_bios(&ns->head->requeue_list, req);
	58	spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
	59	blk_mq_end_request(req, 0);
	60
0d0b660f CH	61	switch (status & 0x7ff) {
	62	case NVME_SC_ANA_TRANSITION:
	63	case NVME_SC_ANA_INACCESSIBLE:
	64	case NVME_SC_ANA_PERSISTENT_LOSS:
	65	/*
	66	* If we got back an ANA error we know the controller is alive,
	67	* but not ready to serve this namespaces. The spec suggests
	68	* we should update our general state here, but due to the fact
	69	* that the admin and I/O queues are not serialized that is
	70	* fundamentally racy. So instead just clear the current path,
	71	* mark the the path as pending and kick of a re-read of the ANA
	72	* log page ASAP.
	73	*/
	74	nvme_mpath_clear_current_path(ns);
	75	if (ns->ctrl->ana_log_buf) {
	76	set_bit(NVME_NS_ANA_PENDING, &ns->flags);
	77	queue_work(nvme_wq, &ns->ctrl->ana_work);
	78	}
	79	break;
783f4a44 JS	80	case NVME_SC_HOST_PATH_ERROR:
	81	/*
	82	* Temporary transport disruption in talking to the controller.
	83	* Try to send on a new path.
	84	*/
	85	nvme_mpath_clear_current_path(ns);
	86	break;
0d0b660f CH	87	default:
	88	/*
	89	* Reset the controller for any non-ANA error as we don't know
	90	* what caused the error.
	91	*/
	92	nvme_reset_ctrl(ns->ctrl);
	93	break;
	94	}
	95
32acab31 CH	96	kblockd_schedule_work(&ns->head->requeue_work);
	97	}
	98
32acab31 CH	99	void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
	100	{
	101	struct nvme_ns *ns;
	102
765cc031	103	down_read(&ctrl->namespaces_rwsem);
32acab31 CH	104	list_for_each_entry(ns, &ctrl->namespaces, list) {
	105	if (ns->head->disk)
	106	kblockd_schedule_work(&ns->head->requeue_work);
	107	}
765cc031	108	up_read(&ctrl->namespaces_rwsem);
32acab31 CH	109	}
32acab31 CH	110
0d0b660f CH	111	static const char *nvme_ana_state_names[] = {
	112	[0] = "invalid state",
	113	[NVME_ANA_OPTIMIZED] = "optimized",
	114	[NVME_ANA_NONOPTIMIZED] = "non-optimized",
	115	[NVME_ANA_INACCESSIBLE] = "inaccessible",
	116	[NVME_ANA_PERSISTENT_LOSS] = "persistent-loss",
	117	[NVME_ANA_CHANGE] = "change",
	118	};
	119
f3334447	120	void nvme_mpath_clear_current_path(struct nvme_ns *ns)
32acab31	121	{
f3334447 CH	122	struct nvme_ns_head *head = ns->head;
	123	int node;
	124
	125	if (!head)
	126	return;
	127
	128	for_each_node(node) {
	129	if (ns == rcu_access_pointer(head->current_path[node]))
	130	rcu_assign_pointer(head->current_path[node], NULL);
	131	}
	132	}
	133
	134	static struct nvme_ns __nvme_find_path(struct nvme_ns_head head, int node)
	135	{
	136	int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
	137	struct nvme_ns found = NULL, fallback = NULL, *ns;
32acab31 CH	138
32acab31 CH	139	list_for_each_entry_rcu(ns, &head->list, siblings) {
0d0b660f CH	140	if (ns->ctrl->state != NVME_CTRL_LIVE \|\|
	141	test_bit(NVME_NS_ANA_PENDING, &ns->flags))
	142	continue;
f3334447 CH	143
	144	distance = node_distance(node, dev_to_node(ns->ctrl->dev));
	145
0d0b660f CH	146	switch (ns->ana_state) {
0d0b660f CH	147	case NVME_ANA_OPTIMIZED:
f3334447 CH	148	if (distance < found_distance) {
	149	found_distance = distance;
	150	found = ns;
	151	}
	152	break;
0d0b660f	153	case NVME_ANA_NONOPTIMIZED:
f3334447 CH	154	if (distance < fallback_distance) {
	155	fallback_distance = distance;
	156	fallback = ns;
	157	}
0d0b660f CH	158	break;
	159	default:
	160	break;
32acab31 CH	161	}
	162	}
	163
f3334447 CH	164	if (!found)
	165	found = fallback;
	166	if (found)
	167	rcu_assign_pointer(head->current_path[node], found);
	168	return found;
0d0b660f CH	169	}
	170
	171	static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
	172	{
	173	return ns->ctrl->state == NVME_CTRL_LIVE &&
	174	ns->ana_state == NVME_ANA_OPTIMIZED;
32acab31 CH	175	}
	176
	177	inline struct nvme_ns nvme_find_path(struct nvme_ns_head head)
	178	{
f3334447 CH	179	int node = numa_node_id();
f3334447 CH	180	struct nvme_ns *ns;
32acab31	181
f3334447	182	ns = srcu_dereference(head->current_path[node], &head->srcu);
0d0b660f	183	if (unlikely(!ns \|\| !nvme_path_is_optimized(ns)))
f3334447	184	ns = __nvme_find_path(head, node);
32acab31 CH	185	return ns;
	186	}
	187
	188	static blk_qc_t nvme_ns_head_make_request(struct request_queue *q,
	189	struct bio *bio)
	190	{
	191	struct nvme_ns_head *head = q->queuedata;
	192	struct device *dev = disk_to_dev(head->disk);
	193	struct nvme_ns *ns;
	194	blk_qc_t ret = BLK_QC_T_NONE;
	195	int srcu_idx;
	196
	197	srcu_idx = srcu_read_lock(&head->srcu);
	198	ns = nvme_find_path(head);
	199	if (likely(ns)) {
	200	bio->bi_disk = ns->disk;
	201	bio->bi_opf \|= REQ_NVME_MPATH;
2796b569 HR	202	trace_block_bio_remap(bio->bi_disk->queue, bio,
	203	disk_devt(ns->head->disk),
	204	bio->bi_iter.bi_sector);
32acab31 CH	205	ret = direct_make_request(bio);
32acab31 CH	206	} else if (!list_empty_careful(&head->list)) {
89c4aff6	207	dev_warn_ratelimited(dev, "no path available - requeuing I/O\n");
32acab31 CH	208
	209	spin_lock_irq(&head->requeue_lock);
	210	bio_list_add(&head->requeue_list, bio);
	211	spin_unlock_irq(&head->requeue_lock);
	212	} else {
	213	dev_warn_ratelimited(dev, "no path - failing I/O\n");
	214
	215	bio->bi_status = BLK_STS_IOERR;
	216	bio_endio(bio);
	217	}
	218
	219	srcu_read_unlock(&head->srcu, srcu_idx);
	220	return ret;
	221	}
	222
0a1b8b87	223	static int nvme_ns_head_poll(struct request_queue *q, blk_qc_t qc, bool spin)
32acab31 CH	224	{
	225	struct nvme_ns_head *head = q->queuedata;
	226	struct nvme_ns *ns;
85f4d4b6	227	int found = 0;
32acab31 CH	228	int srcu_idx;
	229
	230	srcu_idx = srcu_read_lock(&head->srcu);
f3334447	231	ns = srcu_dereference(head->current_path[numa_node_id()], &head->srcu);
0d0b660f	232	if (likely(ns && nvme_path_is_optimized(ns)))
0a1b8b87	233	found = ns->queue->poll_fn(q, qc, spin);
32acab31 CH	234	srcu_read_unlock(&head->srcu, srcu_idx);
	235	return found;
	236	}
	237
	238	static void nvme_requeue_work(struct work_struct *work)
	239	{
	240	struct nvme_ns_head *head =
	241	container_of(work, struct nvme_ns_head, requeue_work);
	242	struct bio bio, next;
	243
	244	spin_lock_irq(&head->requeue_lock);
	245	next = bio_list_get(&head->requeue_list);
	246	spin_unlock_irq(&head->requeue_lock);
	247
	248	while ((bio = next) != NULL) {
	249	next = bio->bi_next;
	250	bio->bi_next = NULL;
	251
	252	/*
	253	* Reset disk to the mpath node and resubmit to select a new
	254	* path.
	255	*/
	256	bio->bi_disk = head->disk;
	257	generic_make_request(bio);
	258	}
	259	}
	260
	261	int nvme_mpath_alloc_disk(struct nvme_ctrl ctrl, struct nvme_ns_head head)
	262	{
	263	struct request_queue *q;
	264	bool vwc = false;
	265
0d0b660f	266	mutex_init(&head->lock);
32acab31 CH	267	bio_list_init(&head->requeue_list);
	268	spin_lock_init(&head->requeue_lock);
	269	INIT_WORK(&head->requeue_work, nvme_requeue_work);
	270
	271	/*
	272	* Add a multipath node if the subsystems supports multiple controllers.
	273	* We also do this for private namespaces as the namespace sharing data could
	274	* change after a rescan.
	275	*/
	276	if (!(ctrl->subsys->cmic & (1 << 1)) \|\| !multipath)
	277	return 0;
	278
6d469642	279	q = blk_alloc_queue_node(GFP_KERNEL, NUMA_NO_NODE);
32acab31 CH	280	if (!q)
	281	goto out;
	282	q->queuedata = head;
	283	blk_queue_make_request(q, nvme_ns_head_make_request);
	284	q->poll_fn = nvme_ns_head_poll;
8b904b5b	285	blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
32acab31 CH	286	/* set to a default value for 512 until disk is validated */
32acab31 CH	287	blk_queue_logical_block_size(q, 512);
8f676b85	288	blk_set_stacking_limits(&q->limits);
32acab31 CH	289
	290	/* we need to propagate up the VMC settings */
	291	if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
	292	vwc = true;
	293	blk_queue_write_cache(q, vwc, vwc);
	294
	295	head->disk = alloc_disk(0);
	296	if (!head->disk)
	297	goto out_cleanup_queue;
	298	head->disk->fops = &nvme_ns_head_ops;
	299	head->disk->private_data = head;
	300	head->disk->queue = q;
	301	head->disk->flags = GENHD_FL_EXT_DEVT;
	302	sprintf(head->disk->disk_name, "nvme%dn%d",
	303	ctrl->subsys->instance, head->instance);
	304	return 0;
	305
	306	out_cleanup_queue:
	307	blk_cleanup_queue(q);
	308	out:
	309	return -ENOMEM;
	310	}
	311
0d0b660f	312	static void nvme_mpath_set_live(struct nvme_ns *ns)
32acab31	313	{
0d0b660f CH	314	struct nvme_ns_head *head = ns->head;
	315
	316	lockdep_assert_held(&ns->head->lock);
	317
32acab31 CH	318	if (!head->disk)
32acab31 CH	319	return;
9bd82b1a	320
33b14f67 HR	321	if (!(head->disk->flags & GENHD_FL_UP))
	322	device_add_disk(&head->subsys->dev, head->disk,
	323	nvme_ns_id_attr_groups);
0d0b660f	324
886fabf6 KB	325	if (nvme_path_is_optimized(ns)) {
	326	int node, srcu_idx;
	327
	328	srcu_idx = srcu_read_lock(&head->srcu);
	329	for_each_node(node)
	330	__nvme_find_path(head, node);
	331	srcu_read_unlock(&head->srcu, srcu_idx);
	332	}
	333
0d0b660f CH	334	kblockd_schedule_work(&ns->head->requeue_work);
	335	}
	336
	337	static int nvme_parse_ana_log(struct nvme_ctrl ctrl, void data,
	338	int (cb)(struct nvme_ctrl ctrl, struct nvme_ana_group_desc *,
	339	void *))
	340	{
	341	void *base = ctrl->ana_log_buf;
	342	size_t offset = sizeof(struct nvme_ana_rsp_hdr);
	343	int error, i;
	344
	345	lockdep_assert_held(&ctrl->ana_lock);
	346
	347	for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
	348	struct nvme_ana_group_desc *desc = base + offset;
	349	u32 nr_nsids = le32_to_cpu(desc->nnsids);
	350	size_t nsid_buf_size = nr_nsids * sizeof(__le32);
	351
	352	if (WARN_ON_ONCE(desc->grpid == 0))
	353	return -EINVAL;
	354	if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))
	355	return -EINVAL;
	356	if (WARN_ON_ONCE(desc->state == 0))
	357	return -EINVAL;
	358	if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
	359	return -EINVAL;
	360
	361	offset += sizeof(*desc);
	362	if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
	363	return -EINVAL;
	364
	365	error = cb(ctrl, desc, data);
	366	if (error)
	367	return error;
	368
	369	offset += nsid_buf_size;
	370	if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
	371	return -EINVAL;
	372	}
	373
	374	return 0;
	375	}
	376
	377	static inline bool nvme_state_is_live(enum nvme_ana_state state)
	378	{
	379	return state == NVME_ANA_OPTIMIZED \|\| state == NVME_ANA_NONOPTIMIZED;
	380	}
	381
	382	static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
	383	struct nvme_ns *ns)
	384	{
	385	enum nvme_ana_state old;
	386
	387	mutex_lock(&ns->head->lock);
	388	old = ns->ana_state;
	389	ns->ana_grpid = le32_to_cpu(desc->grpid);
	390	ns->ana_state = desc->state;
	391	clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
	392
	393	if (nvme_state_is_live(ns->ana_state) && !nvme_state_is_live(old))
	394	nvme_mpath_set_live(ns);
	395	mutex_unlock(&ns->head->lock);
	396	}
	397
398	static int nvme_update_ana_state(struct nvme_ctrl *ctrl,
399	struct nvme_ana_group_desc desc, void data)
400	{
401	u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
402	unsigned *nr_change_groups = data;
403	struct nvme_ns *ns;
404
405	dev_info(ctrl->device, "ANA group %d: %s.\n",
406	le32_to_cpu(desc->grpid),
407	nvme_ana_state_names[desc->state]);
408
409	if (desc->state == NVME_ANA_CHANGE)
410	(*nr_change_groups)++;
411
412	if (!nr_nsids)
413	return 0;
414
415	down_write(&ctrl->namespaces_rwsem);
416	list_for_each_entry(ns, &ctrl->namespaces, list) {
417	if (ns->head->ns_id != le32_to_cpu(desc->nsids[n]))
418	continue;
419	nvme_update_ns_ana_state(desc, ns);
420	if (++n == nr_nsids)
421	break;
422	}
423	up_write(&ctrl->namespaces_rwsem);
424	WARN_ON_ONCE(n < nr_nsids);
425	return 0;
426	}
427
428	static int nvme_read_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
429	{
430	u32 nr_change_groups = 0;
431	int error;
432
433	mutex_lock(&ctrl->ana_lock);
434	error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA,
435	groups_only ? NVME_ANA_LOG_RGO : 0,
436	ctrl->ana_log_buf, ctrl->ana_log_size, 0);
437	if (error) {
438	dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
439	goto out_unlock;
440	}
441
442	error = nvme_parse_ana_log(ctrl, &nr_change_groups,
443	nvme_update_ana_state);
444	if (error)
445	goto out_unlock;
446
447	/*
448	* In theory we should have an ANATT timer per group as they might enter
449	* the change state at different times. But that is a lot of overhead
450	* just to protect against a target that keeps entering new changes
451	* states while never finishing previous ones. But we'll still
452	* eventually time out once all groups are in change state, so this
453	* isn't a big deal.
454	*
455	* We also double the ANATT value to provide some slack for transports
456	* or AEN processing overhead.
457	*/
458	if (nr_change_groups)
459	mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);
460	else
461	del_timer_sync(&ctrl->anatt_timer);
462	out_unlock:
463	mutex_unlock(&ctrl->ana_lock);
464	return error;
465	}
466
467	static void nvme_ana_work(struct work_struct *work)
468	{
469	struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
470
471	nvme_read_ana_log(ctrl, false);
472	}
473
474	static void nvme_anatt_timeout(struct timer_list *t)
475	{
476	struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
477
478	dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");
479	nvme_reset_ctrl(ctrl);
480	}
481
482	void nvme_mpath_stop(struct nvme_ctrl *ctrl)
483	{
484	if (!nvme_ctrl_use_ana(ctrl))
485	return;
486	del_timer_sync(&ctrl->anatt_timer);
487	cancel_work_sync(&ctrl->ana_work);
488	}
489
490	static ssize_t ana_grpid_show(struct device dev, struct device_attribute attr,
491	char *buf)
492	{
493	return sprintf(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid);
494	}
495	DEVICE_ATTR_RO(ana_grpid);
496
497	static ssize_t ana_state_show(struct device dev, struct device_attribute attr,
498	char *buf)
499	{
500	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
501
502	return sprintf(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);
503	}
504	DEVICE_ATTR_RO(ana_state);
505
506	static int nvme_set_ns_ana_state(struct nvme_ctrl *ctrl,
507	struct nvme_ana_group_desc desc, void data)
508	{
509	struct nvme_ns *ns = data;
510
511	if (ns->ana_grpid == le32_to_cpu(desc->grpid)) {
512	nvme_update_ns_ana_state(desc, ns);
513	return -ENXIO; /* just break out of the loop */
514	}
515
516	return 0;
517	}
518
519	void nvme_mpath_add_disk(struct nvme_ns ns, struct nvme_id_ns id)
520	{
521	if (nvme_ctrl_use_ana(ns->ctrl)) {
522	mutex_lock(&ns->ctrl->ana_lock);
523	ns->ana_grpid = le32_to_cpu(id->anagrpid);
524	nvme_parse_ana_log(ns->ctrl, ns, nvme_set_ns_ana_state);
525	mutex_unlock(&ns->ctrl->ana_lock);
526	} else {
527	mutex_lock(&ns->head->lock);
528	ns->ana_state = NVME_ANA_OPTIMIZED;
529	nvme_mpath_set_live(ns);
530	mutex_unlock(&ns->head->lock);
9bd82b1a	531	}
32acab31 CH	532	}
	533
	534	void nvme_mpath_remove_disk(struct nvme_ns_head *head)
	535	{
	536	if (!head->disk)
	537	return;
33b14f67	538	if (head->disk->flags & GENHD_FL_UP)
0d0b660f	539	del_gendisk(head->disk);
32acab31 CH	540	blk_set_queue_dying(head->disk->queue);
	541	/* make sure all pending bios are cleaned up */
	542	kblockd_schedule_work(&head->requeue_work);
	543	flush_work(&head->requeue_work);
	544	blk_cleanup_queue(head->disk->queue);
	545	put_disk(head->disk);
	546	}
0d0b660f CH	547
	548	int nvme_mpath_init(struct nvme_ctrl ctrl, struct nvme_id_ctrl id)
	549	{
	550	int error;
	551
	552	if (!nvme_ctrl_use_ana(ctrl))
	553	return 0;
	554
	555	ctrl->anacap = id->anacap;
	556	ctrl->anatt = id->anatt;
	557	ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
	558	ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
	559
	560	mutex_init(&ctrl->ana_lock);
	561	timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
	562	ctrl->ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
	563	ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc);
	564	if (!(ctrl->anacap & (1 << 6)))
	565	ctrl->ana_log_size += ctrl->max_namespaces * sizeof(__le32);
	566
	567	if (ctrl->ana_log_size > ctrl->max_hw_sectors << SECTOR_SHIFT) {
	568	dev_err(ctrl->device,
	569	"ANA log page size (%zd) larger than MDTS (%d).\n",
	570	ctrl->ana_log_size,
	571	ctrl->max_hw_sectors << SECTOR_SHIFT);
	572	dev_err(ctrl->device, "disabling ANA support.\n");
	573	return 0;
	574	}
	575
	576	INIT_WORK(&ctrl->ana_work, nvme_ana_work);
	577	ctrl->ana_log_buf = kmalloc(ctrl->ana_log_size, GFP_KERNEL);
bb830add SD	578	if (!ctrl->ana_log_buf) {
bb830add SD	579	error = -ENOMEM;
0d0b660f	580	goto out;
bb830add	581	}
0d0b660f CH	582
	583	error = nvme_read_ana_log(ctrl, true);
	584	if (error)
	585	goto out_free_ana_log_buf;
	586	return 0;
	587	out_free_ana_log_buf:
	588	kfree(ctrl->ana_log_buf);
	589	out:
bb830add	590	return error;
0d0b660f CH	591	}
	592
	593	void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
	594	{
	595	kfree(ctrl->ana_log_buf);
	596	}
	597