RDMA/mlx5: Add support for rate limit in UD and UC QPs

Rate limiting is currently supported only for raw packet QPs, where
the packet pacing index is programmed into the SQC during SQ modify.

Extend rate limit support to UD and UC QPs by setting the pacing
index in the QPC during RTR2RTS and RTS2RTS transitions.

Signed-off-by: Maher Sanalla <msanalla@nvidia.com>
Reviewed-by: Michael Guralnik <michaelgur@nvidia.com>
Signed-off-by: Edward Srouji <edwards@nvidia.com>
Link: https://patch.msgid.link/20260524-packet-pacing-v1-3-3d79439f8d08@nvidia.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 6924fceb99c5b07ed284af056fcffb7a1c64206e..1f8d6dd6057c6ffa12a251d57cad3e44de33913b 100644 (file)
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -2754,6 +2754,10 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
                if (err)
                        mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
                                     base->mqp.qpn);
+               if (qp->rl.rate) {
+                       mlx5_rl_remove_rate(dev->mdev, &qp->rl);
+                       memset(&qp->rl, 0, sizeof(qp->rl));
+               }
        }
 
        destroy_qp(dev, qp, base, udata);
@@ -3703,8 +3707,10 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q
                                          MLX5_QP_OPTPAR_RNR_TIMEOUT,
                        [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
                                          MLX5_QP_OPTPAR_RWE            |
-                                         MLX5_QP_OPTPAR_PM_STATE,
-                       [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
+                                         MLX5_QP_OPTPAR_PM_STATE       |
+                                         MLX5_QP_OPTPAR_PP_INDEX,
+                       [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY          |
+                                         MLX5_QP_OPTPAR_PP_INDEX,
                        [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
                                          MLX5_QP_OPTPAR_RRE            |
                                          MLX5_QP_OPTPAR_RAE            |
@@ -3723,10 +3729,12 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q
                                          MLX5_QP_OPTPAR_ALT_ADDR_PATH,
                        [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE            |
                                          MLX5_QP_OPTPAR_PM_STATE       |
-                                         MLX5_QP_OPTPAR_ALT_ADDR_PATH,
+                                         MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
+                                         MLX5_QP_OPTPAR_PP_INDEX,
                        [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY          |
                                          MLX5_QP_OPTPAR_SRQN           |
-                                         MLX5_QP_OPTPAR_CQN_RCV,
+                                         MLX5_QP_OPTPAR_CQN_RCV        |
+                                         MLX5_QP_OPTPAR_PP_INDEX,
                        [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE           |
                                          MLX5_QP_OPTPAR_RAE            |
                                          MLX5_QP_OPTPAR_RWE            |
@@ -3870,11 +3878,31 @@ out:
        return err;
 }
 
+static bool qp_rate_limit_supported(struct mlx5_ib_dev *dev,
+                                   struct mlx5_ib_qp *qp)
+{
+       if (qp->type == IB_QPT_RAW_PACKET ||
+           qp->flags & IB_QP_CREATE_SOURCE_QPN)
+               return true;
+
+       if (qp->type == IB_QPT_UD)
+               return MLX5_CAP_QOS(dev->mdev, packet_pacing_req_ud);
+
+       if (qp->type == IB_QPT_UC)
+               return MLX5_CAP_QOS(dev->mdev, packet_pacing_req_uc);
+
+       return false;
+}
+
 static int qp_rl_parse(struct mlx5_ib_dev *dev,
+                      struct mlx5_ib_qp *qp,
                       const struct ib_qp_attr *attr,
                       const struct mlx5_ib_modify_qp *ucmd,
                       struct mlx5_rate_limit *rl_desired)
 {
+       if (!qp_rate_limit_supported(dev, qp))
+               return -EOPNOTSUPP;
+
        rl_desired->rate = attr->rate_limit;
 
        if (ucmd->burst_info.max_burst_sz) {
@@ -3935,15 +3963,20 @@ static void qp_rl_rollback(struct mlx5_core_dev *dev,
 
 static void qp_rl_commit(struct mlx5_core_dev *dev,
                         struct mlx5_ib_qp *qp,
-                        struct mlx5_rate_limit_ctx *ctx)
+                        struct mlx5_rate_limit_ctx *ctx,
+                        enum ib_qp_state new_state)
 {
-       if (!ctx->rl_changed)
-               return;
-
-       if (ctx->rl_old.rate)
-               mlx5_rl_remove_rate(dev, &ctx->rl_old);
+       if (ctx->rl_changed) {
+               if (ctx->rl_old.rate)
+                       mlx5_rl_remove_rate(dev, &ctx->rl_old);
+               qp->rl = ctx->rl_desired;
+       }
 
-       qp->rl = ctx->rl_desired;
+       if (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR) {
+               if (qp->rl.rate)
+                       mlx5_rl_remove_rate(dev, &qp->rl);
+               memset(&qp->rl, 0, sizeof(qp->rl));
+       }
 }
 
 static int modify_raw_packet_qp_sq(
@@ -4250,6 +4283,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
        struct mlx5_ib_qp *qp = to_mqp(ibqp);
        struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
        struct mlx5_ib_cq *send_cq, *recv_cq;
+       struct mlx5_rate_limit_ctx rl_ctx = {};
        struct mlx5_ib_pd *pd;
        enum mlx5_qp_state mlx5_cur, mlx5_new;
        void *qpc, *pri_path, *alt_path;
@@ -4440,20 +4474,31 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
                goto out;
        }
 
+       if (attr_mask & IB_QP_RATE_LIMIT) {
+               err = qp_rl_parse(dev, qp, attr, ucmd, &rl_ctx.rl_desired);
+               if (err)
+                       goto out;
+       } else {
+               rl_ctx.rl_desired = qp->rl;
+       }
+
        op = optab[mlx5_cur][mlx5_new];
+       if (!mlx5_rl_are_equal(&rl_ctx.rl_desired, &qp->rl)) {
+               err = qp_rl_prepare(dev, qp, op, &rl_ctx);
+               if (err)
+                       goto out;
+       }
        optpar |= ib_mask_to_mlx5_opt(attr_mask);
+       if (rl_ctx.rl_changed)
+               optpar |= MLX5_QP_OPTPAR_PP_INDEX;
        optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
 
-       if (attr_mask & IB_QP_RATE_LIMIT && qp->type != IB_QPT_RAW_PACKET) {
-               err = -EOPNOTSUPP;
-               goto out;
-       }
-
        if (qp->type == IB_QPT_RAW_PACKET ||
            qp->flags & IB_QP_CREATE_SOURCE_QPN) {
                struct mlx5_modify_raw_qp_param raw_qp_param = {};
 
                raw_qp_param.operation = op;
+               raw_qp_param.rl_ctx = rl_ctx;
                if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
                        raw_qp_param.rq_q_ctr_id = set_id;
                        raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
@@ -4462,22 +4507,8 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
                if (attr_mask & IB_QP_PORT)
                        raw_qp_param.port = attr->port_num;
 
-               if (attr_mask & IB_QP_RATE_LIMIT) {
-                       err = qp_rl_parse(dev, qp, attr, ucmd,
-                                         &raw_qp_param.rl_ctx.rl_desired);
-                       if (err)
-                               goto out;
-
-                       if (!mlx5_rl_are_equal(&raw_qp_param.rl_ctx.rl_desired,
-                                              &qp->rl)) {
-                               err = qp_rl_prepare(dev, qp, op,
-                                                   &raw_qp_param.rl_ctx);
-                               if (err)
-                                       goto out;
-                       }
-
+               if (rl_ctx.rl_changed)
                        raw_qp_param.set_mask |= MLX5_RAW_QP_RATE_LIMIT;
-               }
 
                err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity);
                if (err) {
@@ -4485,8 +4516,13 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
                        goto out;
                }
 
-               qp_rl_commit(dev->mdev, qp, &raw_qp_param.rl_ctx);
+               qp_rl_commit(dev->mdev, qp, &raw_qp_param.rl_ctx, new_state);
        } else {
+               if (rl_ctx.rl_changed) {
+                       MLX5_SET(qpc, qpc, packet_pacing_rate_limit_index,
+                                rl_ctx.rl_desired_index);
+               }
+
                if (udata) {
                        /* For the kernel flows, the resp will stay zero */
                        resp->ece_options =
@@ -4496,6 +4532,13 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
                }
                err = mlx5_core_qp_modify(dev, op, optpar, qpc, &base->mqp,
                                          &resp->ece_options);
+
+               if (err) {
+                       qp_rl_rollback(dev->mdev, &rl_ctx);
+                       goto out;
+               }
+
+               qp_rl_commit(dev->mdev, qp, &rl_ctx, new_state);
        }
 
        if (err)

diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h
index d67aedc6ea68ae9f514a1de11626792bfe847c04..40f889403b075ebc6b860705206843aafa5d9a32 100644 (file)
--- a/include/linux/mlx5/qp.h
+++ b/include/linux/mlx5/qp.h
@@ -72,6 +72,7 @@ enum mlx5_qp_optpar {
        MLX5_QP_OPTPAR_CQN_RCV                  = 1 << 19,
        MLX5_QP_OPTPAR_DC_HS                    = 1 << 20,
        MLX5_QP_OPTPAR_DC_KEY                   = 1 << 21,
+       MLX5_QP_OPTPAR_PP_INDEX                 = 1 << 22,
        MLX5_QP_OPTPAR_COUNTER_SET_ID           = 1 << 25,
 };