rdma: Add resource FRMR pools show command

Allow users to see the FRMR pools that were created on the devices,
their properties and their usage statistics.
The set of properties of each pool are encoded as a colon-separated
list of hexadecimal fields (vendor_key:num_dma_blocks:access_flags:ats)
to simplify referencing a specific pool in 'set' commands.

Sample output:

$rdma resource show frmr_pools
dev rocep8s0f0 key 0:1000:0:0 queue 0 in_use 0 max_in_use 200
dev rocep8s0f0 key 0:800:0:0 queue 0 in_use 0 max_in_use 200
dev rocep8s0f0 key 0:400:0:0 queue 0 in_use 0 max_in_use 200

$rdma resource show frmr_pools -d
dev rocep8s0f0 key 0:1000:0:0 ats 0 access_flags 0 vendor_key 0 num_dma_blocks 4096 queue 0 in_use 0 max_in_use 200
dev rocep8s0f0 key 0:800:0:0 ats 0 access_flags 0 vendor_key 0 num_dma_blocks 2048 queue 0 in_use 0 max_in_use 200
dev rocep8s0f0 key 0:400:0:0 ats 0 access_flags 0 vendor_key 0 num_dma_blocks 1024 queue 0 in_use 0 max_in_use 200

$rdma resource show frmr_pools num_dma_blocks 2048
dev rocep8s0f0 key 0:800:0:0 queue 0 in_use 0 max_in_use 200

Signed-off-by: Michael Guralnik <michaelgur@nvidia.com>
Reviewed-by: Patrisious Haddad <phaddad@nvidia.com>
Reviewed-by: Chiara Meiohas <cmeiohas@nvidia.com>
Signed-off-by: David Ahern <dsahern@kernel.org>

diff --git a/man/man8/rdma-resource.8 b/man/man8/rdma-resource.8
index 61bec471a0968394adb380e195bd4f96d57cd3aa..4dfb8218b9a2035862854d5fd26b19a627dcf75e 100644 (file)
--- a/man/man8/rdma-resource.8
+++ b/man/man8/rdma-resource.8
@@ -13,7 +13,8 @@ rdma-resource \- rdma resource configuration
 
 .ti -8
 .IR RESOURCE " := { "
-.BR cm_id " | " cq " | " mr " | " pd " | " qp " | " ctx " | " srq " }"
+.BR cm_id " | " cq " | " mr " | " pd " | " qp " | " ctx " | " srq " | "
+.BR frmr_pools " }"
 .sp
 
 .ti -8
@@ -113,6 +114,11 @@ rdma resource show srq lqpn 5-7
 Show SRQs that the QPs with lqpn 5-7 are associated with.
 .RE
 .PP
+rdma resource show frmr_pools ats 1
+.RS 4
+Show FRMR pools that have ats attribute set.
+.RE
+.PP
 
 .SH SEE ALSO
 .BR rdma (8),

diff --git a/rdma/Makefile b/rdma/Makefile
index ed3c1c1c1b97b443d87e5d3dde1d98c81e6a0382..66fe53f9b778442ee11eeb2fe6c9bc7ddbaa1020 100644 (file)
--- a/rdma/Makefile
+++ b/rdma/Makefile
@@ -5,7 +5,7 @@ CFLAGS += -I./include/uapi/
 
 RDMA_OBJ = rdma.o utils.o dev.o link.o res.o res-pd.o res-mr.o res-cq.o \
           res-cmid.o res-qp.o sys.o stat.o stat-mr.o res-ctx.o res-srq.o \
-          monitor.o
+          monitor.o res-frmr-pools.o
 
 TARGETS += rdma
 

diff --git a/rdma/res-frmr-pools.c b/rdma/res-frmr-pools.c
new file mode 100644 (file)
index 0000000..7d99a72
--- /dev/null
+++ b/rdma/res-frmr-pools.c
@@ -0,0 +1,174 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * res-frmr-pools.c    RDMA tool
+ * Authors:    Michael Guralnik <michaelgur@nvidia.com>
+ */
+
+#include "res.h"
+#include <inttypes.h>
+
+struct frmr_pool_key {
+       uint64_t vendor_key;
+       uint64_t num_dma_blocks;
+       uint32_t access_flags;
+       uint8_t ats;
+};
+
+/* vendor_key(16) + ':' + num_dma_blocks(16) + ':' + access_flags(8) + ':' + ats(1) + '\0' */
+#define FRMR_POOL_KEY_MAX_LEN 45
+
+static int res_frmr_pools_line(struct rd *rd, const char *name, int idx,
+                              struct nlattr **nla_line)
+{
+       uint64_t in_use = 0, max_in_use = 0, kernel_vendor_key = 0;
+       struct nlattr *key_tb[RDMA_NLDEV_ATTR_MAX] = {};
+       char key_str[FRMR_POOL_KEY_MAX_LEN];
+       struct frmr_pool_key key = { 0 };
+       uint32_t queue_handles = 0;
+
+       if (nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY]) {
+               if (mnl_attr_parse_nested(
+                           nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY],
+                           rd_attr_cb, key_tb) != MNL_CB_OK)
+                       return MNL_CB_ERROR;
+
+               if (key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_ATS])
+                       key.ats = mnl_attr_get_u8(
+                               key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_ATS]);
+               if (key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_ACCESS_FLAGS])
+                       key.access_flags = mnl_attr_get_u32(
+                               key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_ACCESS_FLAGS]);
+               if (key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_VENDOR_KEY])
+                       key.vendor_key = mnl_attr_get_u64(
+                               key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_VENDOR_KEY]);
+               if (key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_NUM_DMA_BLOCKS])
+                       key.num_dma_blocks = mnl_attr_get_u64(
+                               key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_NUM_DMA_BLOCKS]);
+               if (key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_KERNEL_VENDOR_KEY])
+                       kernel_vendor_key = mnl_attr_get_u64(
+                               key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_KERNEL_VENDOR_KEY]);
+
+               if (rd_is_filtered_attr(
+                           rd, "ats", key.ats,
+                           key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_ATS]))
+                       goto out;
+
+               if (rd_is_filtered_attr(
+                           rd, "access_flags", key.access_flags,
+                           key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_ACCESS_FLAGS]))
+                       goto out;
+
+               if (rd_is_filtered_attr(
+                           rd, "vendor_key", key.vendor_key,
+                           key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_VENDOR_KEY]))
+                       goto out;
+
+               if (rd_is_filtered_attr(
+                           rd, "num_dma_blocks", key.num_dma_blocks,
+                           key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_NUM_DMA_BLOCKS]))
+                       goto out;
+       }
+
+       if (nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_QUEUE_HANDLES])
+               queue_handles = mnl_attr_get_u32(
+                       nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_QUEUE_HANDLES]);
+       if (rd_is_filtered_attr(
+                   rd, "queue", queue_handles,
+                   nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_QUEUE_HANDLES]))
+               goto out;
+
+       if (nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_IN_USE])
+               in_use = mnl_attr_get_u64(
+                       nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_IN_USE]);
+       if (rd_is_filtered_attr(rd, "in_use", in_use,
+                               nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_IN_USE]))
+               goto out;
+
+       if (nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_MAX_IN_USE])
+               max_in_use = mnl_attr_get_u64(
+                       nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_MAX_IN_USE]);
+       if (rd_is_filtered_attr(
+                   rd, "max_in_use", max_in_use,
+                   nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_MAX_IN_USE]))
+               goto out;
+
+       open_json_object(NULL);
+       print_dev(idx, name);
+
+       if (nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY]) {
+               snprintf(key_str, sizeof(key_str),
+                        "%" PRIx64 ":%" PRIx64 ":%x:%s",
+                        key.vendor_key, key.num_dma_blocks,
+                        key.access_flags, key.ats ? "1" : "0");
+               print_string(PRINT_ANY, "key", "key %s ", key_str);
+
+               if (rd->show_details) {
+                       res_print_u32(
+                               "ats", key.ats,
+                               key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_ATS]);
+                       res_print_u32(
+                               "access_flags", key.access_flags,
+                               key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_ACCESS_FLAGS]);
+                       res_print_u64(
+                               "vendor_key", key.vendor_key,
+                               key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_VENDOR_KEY]);
+                       res_print_u64(
+                               "num_dma_blocks", key.num_dma_blocks,
+                               key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_NUM_DMA_BLOCKS]);
+                       res_print_u64(
+                               "kernel_vendor_key", kernel_vendor_key,
+                               key_tb[RDMA_NLDEV_ATTR_RES_FRMR_POOL_KEY_KERNEL_VENDOR_KEY]);
+               }
+       }
+
+       res_print_u32("queue", queue_handles,
+                     nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_QUEUE_HANDLES]);
+       res_print_u64("in_use", in_use,
+                     nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_IN_USE]);
+       res_print_u64("max_in_use", max_in_use,
+                     nla_line[RDMA_NLDEV_ATTR_RES_FRMR_POOL_MAX_IN_USE]);
+
+       print_driver_table(rd, nla_line[RDMA_NLDEV_ATTR_DRIVER]);
+       close_json_object();
+       newline();
+
+out:
+       return MNL_CB_OK;
+}
+
+int res_frmr_pools_idx_parse_cb(const struct nlmsghdr *nlh, void *data)
+{
+       return MNL_CB_OK;
+}
+
+int res_frmr_pools_parse_cb(const struct nlmsghdr *nlh, void *data)
+{
+       struct nlattr *tb[RDMA_NLDEV_ATTR_MAX] = {};
+       struct nlattr *nla_table, *nla_entry;
+       struct rd *rd = data;
+       int ret = MNL_CB_OK;
+       const char *name;
+       uint32_t idx;
+
+       mnl_attr_parse(nlh, 0, rd_attr_cb, tb);
+       if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_DEV_NAME] ||
+           !tb[RDMA_NLDEV_ATTR_RES_FRMR_POOLS])
+               return MNL_CB_ERROR;
+
+       name = mnl_attr_get_str(tb[RDMA_NLDEV_ATTR_DEV_NAME]);
+       idx = mnl_attr_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+       nla_table = tb[RDMA_NLDEV_ATTR_RES_FRMR_POOLS];
+
+       mnl_attr_for_each_nested(nla_entry, nla_table) {
+               struct nlattr *nla_line[RDMA_NLDEV_ATTR_MAX] = {};
+
+               ret = mnl_attr_parse_nested(nla_entry, rd_attr_cb, nla_line);
+               if (ret != MNL_CB_OK)
+                       break;
+
+               ret = res_frmr_pools_line(rd, name, idx, nla_line);
+               if (ret != MNL_CB_OK)
+                       break;
+       }
+       return ret;
+}

diff --git a/rdma/res.c b/rdma/res.c
index 7e7de04267c32c7a85d94489c4d2d0335b0b28a2..f1f13d74bde48f4aa4fc002fb974a99bdb005b22 100644 (file)
--- a/rdma/res.c
+++ b/rdma/res.c
@@ -11,7 +11,7 @@ static int res_help(struct rd *rd)
 {
        pr_out("Usage: %s resource\n", rd->filename);
        pr_out("          resource show [DEV]\n");
-       pr_out("          resource show [qp|cm_id|pd|mr|cq|ctx|srq]\n");
+       pr_out("          resource show [qp|cm_id|pd|mr|cq|ctx|srq|frmr_pools]\n");
        pr_out("          resource show qp link [DEV/PORT]\n");
        pr_out("          resource show qp link [DEV/PORT] [FILTER-NAME FILTER-VALUE]\n");
        pr_out("          resource show cm_id link [DEV/PORT]\n");
@@ -26,6 +26,8 @@ static int res_help(struct rd *rd)
        pr_out("          resource show ctx dev [DEV] [FILTER-NAME FILTER-VALUE]\n");
        pr_out("          resource show srq dev [DEV]\n");
        pr_out("          resource show srq dev [DEV] [FILTER-NAME FILTER-VALUE]\n");
+       pr_out("          resource show frmr_pools dev [DEV]\n");
+       pr_out("          resource show frmr_pools dev [DEV] [FILTER-NAME FILTER-VALUE]\n");
        return 0;
 }
 
@@ -237,6 +239,7 @@ static int res_show(struct rd *rd)
                { "pd",         res_pd          },
                { "ctx",        res_ctx         },
                { "srq",        res_srq         },
+               { "frmr_pools", res_frmr_pools  },
                { 0 }
        };
 

diff --git a/rdma/res.h b/rdma/res.h
index fd09ce7dc0847a6cb55e945aa7153b00e434abd8..7c3e88f217a933e5d36eb6ccc22cc09c76327031 100644 (file)
--- a/rdma/res.h
+++ b/rdma/res.h
@@ -26,6 +26,8 @@ int res_ctx_parse_cb(const struct nlmsghdr *nlh, void *data);
 int res_ctx_idx_parse_cb(const struct nlmsghdr *nlh, void *data);
 int res_srq_parse_cb(const struct nlmsghdr *nlh, void *data);
 int res_srq_idx_parse_cb(const struct nlmsghdr *nlh, void *data);
+int res_frmr_pools_parse_cb(const struct nlmsghdr *nlh, void *data);
+int res_frmr_pools_idx_parse_cb(const struct nlmsghdr *nlh, void *data);
 
 static inline uint32_t res_get_command(uint32_t command, struct rd *rd)
 {
@@ -185,6 +187,21 @@ struct filters srq_valid_filters[MAX_NUMBER_OF_FILTERS] = {
 RES_FUNC(res_srq, RDMA_NLDEV_CMD_RES_SRQ_GET, srq_valid_filters, true,
         RDMA_NLDEV_ATTR_RES_SRQN);
 
+static const
+struct filters frmr_pools_valid_filters[MAX_NUMBER_OF_FILTERS] = {
+       { .name = "dev", .is_number = false },
+       { .name = "ats", .is_number = true },
+       { .name = "access_flags", .is_number = true },
+       { .name = "vendor_key", .is_number = true },
+       { .name = "num_dma_blocks", .is_number = true },
+       { .name = "queue", .is_number = true },
+       { .name = "in_use", .is_number = true },
+       { .name = "max_in_use", .is_number = true },
+};
+
+RES_FUNC(res_frmr_pools, RDMA_NLDEV_CMD_RES_FRMR_POOLS_GET,
+        frmr_pools_valid_filters, true, 0);
+
 void print_dev(uint32_t idx, const char *name);
 void print_link(uint32_t idx, const char *name, uint32_t port, struct nlattr **nla_line);
 void print_key(const char *name, uint64_t val, struct nlattr *nlattr);