RDMA/bnxt_re: Add a debugfs entry for CQE coalescing tuning

This patch adds debugfs interfaces that allows the user to
enable/disable the RoCE CQ coalescing and fine tune certain
CQ coalescing parameters which would be helpful during debug.

Signed-off-by: Damodharam Ammepalli <damodharam.ammepalli@broadcom.com>
Reviewed-by: Hongguang Gao <hongguang.gao@broadcom.com>
Signed-off-by: Kalesh AP <kalesh-anakkur.purayil@broadcom.com>
Link: https://patch.msgid.link/20251103043425.234846-1-kalesh-anakkur.purayil@broadcom.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index 3485e495ac6aa24348fa882d66173792884c35da..3a7ce4729fcfc8ca8ad6dd29222a4cdf554a723a 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -224,6 +224,8 @@ struct bnxt_re_dev {
        struct workqueue_struct         *dcb_wq;
        struct dentry                   *cc_config;
        struct bnxt_re_dbg_cc_config_params *cc_config_params;
+       struct dentry                   *cq_coal_cfg;
+       struct bnxt_re_dbg_cq_coal_params *cq_coal_cfg_params;
 #define BNXT_VPD_FLD_LEN               32
        char board_partno[BNXT_VPD_FLD_LEN];
        /* RoCE mirror */

diff --git a/drivers/infiniband/hw/bnxt_re/debugfs.c b/drivers/infiniband/hw/bnxt_re/debugfs.c
index be5e9b5ca2f0656762f2f05af46d95943fcf56d1..d03f5bb0d890fdc337402ab502418c6568f6672d 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/debugfs.c
+++ b/drivers/infiniband/hw/bnxt_re/debugfs.c
@@ -23,6 +23,14 @@
 
 static struct dentry *bnxt_re_debugfs_root;
 
+static const char * const bnxt_re_cq_coal_str[] = {
+       "buf_maxtime",
+       "normal_maxbuf",
+       "during_maxbuf",
+       "en_ring_idle_mode",
+       "enable",
+};
+
 static const char * const bnxt_re_cc_gen0_name[] = {
        "enable_cc",
        "run_avg_weight_g",
@@ -349,6 +357,123 @@ static void bnxt_re_debugfs_add_info(struct bnxt_re_dev *rdev)
        debugfs_create_file("info", 0400, rdev->dbg_root, rdev, &info_fops);
 }
 
+static ssize_t cq_coal_cfg_write(struct file *file,
+                                const char __user *buf,
+                                size_t count, loff_t *pos)
+{
+       struct seq_file *s = file->private_data;
+       struct bnxt_re_cq_coal_param *param = s->private;
+       struct bnxt_re_dev *rdev = param->rdev;
+       int offset = param->offset;
+       char lbuf[16] = { };
+       u32 val;
+
+       if (count > sizeof(lbuf))
+               return -EINVAL;
+
+       if (copy_from_user(lbuf, buf, count))
+               return -EFAULT;
+
+       lbuf[sizeof(lbuf) - 1] = '\0';
+
+       if (kstrtou32(lbuf, 0, &val))
+               return -EINVAL;
+
+       switch (offset) {
+       case BNXT_RE_COAL_CQ_BUF_MAXTIME:
+               if (val < 1 || val > BNXT_QPLIB_CQ_COAL_MAX_BUF_MAXTIME)
+                       return -EINVAL;
+               rdev->cq_coalescing.buf_maxtime = val;
+               break;
+       case BNXT_RE_COAL_CQ_NORMAL_MAXBUF:
+               if (val < 1 || val > BNXT_QPLIB_CQ_COAL_MAX_NORMAL_MAXBUF)
+                       return -EINVAL;
+               rdev->cq_coalescing.normal_maxbuf = val;
+               break;
+       case BNXT_RE_COAL_CQ_DURING_MAXBUF:
+               if (val < 1 || val > BNXT_QPLIB_CQ_COAL_MAX_DURING_MAXBUF)
+                       return -EINVAL;
+               rdev->cq_coalescing.during_maxbuf = val;
+               break;
+       case BNXT_RE_COAL_CQ_EN_RING_IDLE_MODE:
+               if (val > BNXT_QPLIB_CQ_COAL_MAX_EN_RING_IDLE_MODE)
+                       return -EINVAL;
+               rdev->cq_coalescing.en_ring_idle_mode = val;
+               break;
+       case BNXT_RE_COAL_CQ_ENABLE:
+               if (val > 1)
+                       return -EINVAL;
+               rdev->cq_coalescing.enable = val;
+               break;
+       default:
+               return -EINVAL;
+       }
+       return  count;
+}
+
+static int cq_coal_cfg_show(struct seq_file *s, void *unused)
+{
+       struct bnxt_re_cq_coal_param *param = s->private;
+       struct bnxt_re_dev *rdev = param->rdev;
+       int offset = param->offset;
+       u32 val = 0;
+
+       switch (offset) {
+       case BNXT_RE_COAL_CQ_BUF_MAXTIME:
+               val = rdev->cq_coalescing.buf_maxtime;
+               break;
+       case BNXT_RE_COAL_CQ_NORMAL_MAXBUF:
+               val = rdev->cq_coalescing.normal_maxbuf;
+               break;
+       case BNXT_RE_COAL_CQ_DURING_MAXBUF:
+               val = rdev->cq_coalescing.during_maxbuf;
+               break;
+       case BNXT_RE_COAL_CQ_EN_RING_IDLE_MODE:
+               val = rdev->cq_coalescing.en_ring_idle_mode;
+               break;
+       case BNXT_RE_COAL_CQ_ENABLE:
+               val = rdev->cq_coalescing.enable;
+               break;
+       default:
+               return -EINVAL;
+       }
+
+       seq_printf(s, "%u\n", val);
+       return 0;
+}
+DEFINE_SHOW_STORE_ATTRIBUTE(cq_coal_cfg);
+
+static void bnxt_re_cleanup_cq_coal_debugfs(struct bnxt_re_dev *rdev)
+{
+       debugfs_remove_recursive(rdev->cq_coal_cfg);
+       kfree(rdev->cq_coal_cfg_params);
+}
+
+static void bnxt_re_init_cq_coal_debugfs(struct bnxt_re_dev *rdev)
+{
+       struct bnxt_re_dbg_cq_coal_params *dbg_cq_coal_params;
+       int i;
+
+       if (_is_cq_coalescing_supported(rdev->dev_attr->dev_cap_flags2))
+               return;
+
+       dbg_cq_coal_params = kzalloc(sizeof(*dbg_cq_coal_params), GFP_KERNEL);
+       if (!dbg_cq_coal_params)
+               return;
+
+       rdev->cq_coal_cfg = debugfs_create_dir("cq_coal_cfg", rdev->dbg_root);
+       rdev->cq_coal_cfg_params = dbg_cq_coal_params;
+
+       for (i = 0; i < BNXT_RE_COAL_CQ_MAX; i++) {
+               dbg_cq_coal_params->params[i].offset = i;
+               dbg_cq_coal_params->params[i].rdev = rdev;
+               debugfs_create_file(bnxt_re_cq_coal_str[i],
+                                   0600, rdev->cq_coal_cfg,
+                                   &dbg_cq_coal_params->params[i],
+                                   &cq_coal_cfg_fops);
+       }
+}
+
 void bnxt_re_debugfs_add_pdev(struct bnxt_re_dev *rdev)
 {
        struct pci_dev *pdev = rdev->en_dev->pdev;
@@ -374,10 +499,13 @@ void bnxt_re_debugfs_add_pdev(struct bnxt_re_dev *rdev)
                                                         rdev->cc_config, tmp_params,
                                                         &bnxt_re_cc_config_ops);
        }
+
+       bnxt_re_init_cq_coal_debugfs(rdev);
 }
 
 void bnxt_re_debugfs_rem_pdev(struct bnxt_re_dev *rdev)
 {
+       bnxt_re_cleanup_cq_coal_debugfs(rdev);
        debugfs_remove_recursive(rdev->qp_debugfs);
        debugfs_remove_recursive(rdev->cc_config);
        kfree(rdev->cc_config_params);

diff --git a/drivers/infiniband/hw/bnxt_re/debugfs.h b/drivers/infiniband/hw/bnxt_re/debugfs.h
index 8f101df4e83893382e6e8dc4c49a7f46f321d4b6..98f4620ef245617ee78f944672889f06997e4acb 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/debugfs.h
+++ b/drivers/infiniband/hw/bnxt_re/debugfs.h
@@ -33,4 +33,23 @@ struct bnxt_re_cc_param {
 struct bnxt_re_dbg_cc_config_params {
        struct bnxt_re_cc_param gen0_parms[BNXT_RE_CC_PARAM_GEN0];
 };
+
+struct bnxt_re_cq_coal_param {
+       struct bnxt_re_dev      *rdev;
+       u32                     offset;
+};
+
+enum bnxt_re_cq_coal_types {
+       BNXT_RE_COAL_CQ_BUF_MAXTIME,
+       BNXT_RE_COAL_CQ_NORMAL_MAXBUF,
+       BNXT_RE_COAL_CQ_DURING_MAXBUF,
+       BNXT_RE_COAL_CQ_EN_RING_IDLE_MODE,
+       BNXT_RE_COAL_CQ_ENABLE,
+       BNXT_RE_COAL_CQ_MAX
+
+};
+
+struct bnxt_re_dbg_cq_coal_params {
+       struct bnxt_re_cq_coal_param    params[BNXT_RE_COAL_CQ_MAX];
+};
 #endif

diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index b13810572c2e898ca1b973d8df0c68e2f558c6be..73003ad25ee836707f8b0309aebac79b82e16cdc 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -1453,6 +1453,7 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct auxiliary_device *adev,
        atomic_set(&rdev->stats.res.pd_count, 0);
        rdev->cosq[0] = 0xFFFF;
        rdev->cosq[1] = 0xFFFF;
+       rdev->cq_coalescing.enable = 1;
        rdev->cq_coalescing.buf_maxtime = BNXT_QPLIB_CQ_COAL_DEF_BUF_MAXTIME;
        if (bnxt_re_chip_gen_p7(en_dev->chip_num)) {
                rdev->cq_coalescing.normal_maxbuf = BNXT_QPLIB_CQ_COAL_DEF_NORMAL_MAXBUF_P7;

diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index ce90d3d834d4a50e89b0b012018cf9e692e98057..c88f049136fcb0de2fda90a916f618c049944446 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -2226,7 +2226,8 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
        req.cq_handle = cpu_to_le64(cq->cq_handle);
        req.cq_size = cpu_to_le32(cq->max_wqe);
 
-       if (_is_cq_coalescing_supported(res->dattr->dev_cap_flags2)) {
+       if (_is_cq_coalescing_supported(res->dattr->dev_cap_flags2) &&
+           cq->coalescing->enable) {
                req.flags |= cpu_to_le16(CMDQ_CREATE_CQ_FLAGS_COALESCING_VALID);
                coalescing |= ((cq->coalescing->buf_maxtime <<
                                CMDQ_CREATE_CQ_BUF_MAXTIME_SFT) &

diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index b990d0c0ce1a2399e9848e47893344c0bb32ee24..1b414a73b46d6812544b2829dc88fedfa2f8d186 100644 (file)
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -395,6 +395,7 @@ struct bnxt_qplib_cq_coal_param {
        u8 normal_maxbuf;
        u8 during_maxbuf;
        u8 en_ring_idle_mode;
+       u8 enable;
 };
 
 #define BNXT_QPLIB_CQ_COAL_DEF_BUF_MAXTIME             0x1