nvme: export I/O failure count when no path is available via sysfs

When I/O is submitted to the NVMe namespace head and no available path
can handle the request, the driver fails the I/O immediately. Currently,
such failures are only reported via kernel log messages, which may be
lost over time since dmesg is a circular buffer.

Add a new ns-head sysfs counter io_fail_no_available_path_count, under
diag attribute group to expose the number of I/Os that failed due to the
absence of an available path. This provides persistent visibility into
path-related I/O failures and can help users diagnose the cause of I/O
errors. This counter is also writable and so user may reset its value,
if needed.

This counter can also be consumed by monitoring tools such as nvme-top.

Tested-by: Venkat Rao Bagalkote <venkat88@linux.ibm.com>
Signed-off-by: Nilay Shroff <nilay@linux.ibm.com>
Signed-off-by: Keith Busch <kbusch@kernel.org>

diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index 9021fd44f193e8407c4198412a1cdfd68597947a..96337ae2b5524854c4c67356a1f472a21796d018 100644 (file)
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -543,6 +543,7 @@ static void nvme_ns_head_submit_bio(struct bio *bio)
                dev_warn_ratelimited(dev, "no available path - failing I/O\n");
 
                bio_io_error(bio);
+               atomic_long_inc(&head->io_fail_no_available_path_count);
        }
 
        srcu_read_unlock(&head->srcu, srcu_idx);
@@ -1222,6 +1223,35 @@ static ssize_t io_requeue_no_usable_path_count_store(struct device *dev,
 
 DEVICE_ATTR_RW(io_requeue_no_usable_path_count);
 
+static ssize_t io_fail_no_available_path_count_show(struct device *dev,
+               struct device_attribute *attr, char *buf)
+{
+       struct gendisk *disk = dev_to_disk(dev);
+       struct nvme_ns_head *head = disk->private_data;
+
+       return sysfs_emit(buf, "%lu\n",
+                   atomic_long_read(&head->io_fail_no_available_path_count));
+}
+
+static ssize_t io_fail_no_available_path_count_store(struct device *dev,
+               struct device_attribute *attr, const char *buf, size_t count)
+{
+       int err;
+       unsigned long fail_cnt;
+       struct gendisk *disk = dev_to_disk(dev);
+       struct nvme_ns_head *head = disk->private_data;
+
+       err = kstrtoul(buf, 0, &fail_cnt);
+       if (err)
+               return -EINVAL;
+
+       atomic_long_set(&head->io_fail_no_available_path_count, fail_cnt);
+
+       return count;
+}
+
+DEVICE_ATTR_RW(io_fail_no_available_path_count);
+
 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl,
                struct nvme_ana_group_desc *desc, void *data)
 {

diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index bfd427184d69e5754c6a2e36b91dad241e19316d..249f1f8dde4043e5404145c15ce31c4efe2ccc33 100644 (file)
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -567,6 +567,7 @@ struct nvme_ns_head {
        struct delayed_work     remove_work;
        unsigned int            delayed_removal_secs;
        atomic_long_t           io_requeue_no_usable_path_count;
+       atomic_long_t           io_fail_no_available_path_count;
 #define NVME_NSHEAD_DISK_LIVE          0
 #define NVME_NSHEAD_QUEUE_IF_NO_PATH   1
        struct nvme_ns __rcu    *current_path[];
@@ -1071,6 +1072,7 @@ extern struct device_attribute dev_attr_numa_nodes;
 extern struct device_attribute dev_attr_delayed_removal_secs;
 extern struct device_attribute dev_attr_multipath_failover_count;
 extern struct device_attribute dev_attr_io_requeue_no_usable_path_count;
+extern struct device_attribute dev_attr_io_fail_no_available_path_count;
 extern struct device_attribute subsys_attr_iopolicy;
 
 static inline bool nvme_disk_is_ns_head(struct gendisk *disk)

diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c
index 7f0575b7cdd0ef4187c2fff339b3a4b833862c9d..d2c7d943b23fccf749b332e5a7eed93797dda557 100644 (file)
--- a/drivers/nvme/host/sysfs.c
+++ b/drivers/nvme/host/sysfs.c
@@ -404,6 +404,7 @@ static struct attribute *nvme_ns_diag_attrs[] = {
 #ifdef CONFIG_NVME_MULTIPATH
        &dev_attr_multipath_failover_count.attr,
        &dev_attr_io_requeue_no_usable_path_count.attr,
+       &dev_attr_io_fail_no_available_path_count.attr,
 #endif
        NULL,
 };
@@ -432,6 +433,10 @@ static umode_t nvme_ns_diag_attrs_are_visible(struct kobject *kobj,
                if (!nvme_disk_is_ns_head(dev_to_disk(dev)))
                        return 0;
        }
+       if (a == &dev_attr_io_fail_no_available_path_count.attr) {
+               if (!nvme_disk_is_ns_head(dev_to_disk(dev)))
+                       return 0;
+       }
 #endif
        return a->mode;
 }