drivers/infiniband/hw/hfi1/opfn.c

   1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
   2 /*
   3  * Copyright(c) 2018 Intel Corporation.
   4  *
   5  */
   6 #include "hfi.h"
   7 #include "trace.h"
   8 #include "qp.h"
   9 #include "opfn.h"
  10
  11 #define IB_BTHE_E                 BIT(IB_BTHE_E_SHIFT)
  12
  13 #define OPFN_CODE(code) BIT((code) - 1)
  14 #define OPFN_MASK(code) OPFN_CODE(STL_VERBS_EXTD_##code)
  15
  16 struct hfi1_opfn_type {
  17         bool (*request)(struct rvt_qp *qp, u64 *data);
  18         bool (*response)(struct rvt_qp *qp, u64 *data);
  19         bool (*reply)(struct rvt_qp *qp, u64 data);
  20         void (*error)(struct rvt_qp *qp);
  21 };
  22
  23 static struct hfi1_opfn_type hfi1_opfn_handlers[STL_VERBS_EXTD_MAX] = {
  24         [STL_VERBS_EXTD_TID_RDMA] = {
  25                 .request = tid_rdma_conn_req,
  26                 .response = tid_rdma_conn_resp,
  27                 .reply = tid_rdma_conn_reply,
  28                 .error = tid_rdma_conn_error,
  29         },
  30 };
  31
  32 static struct workqueue_struct *opfn_wq;
  33
  34 static void opfn_schedule_conn_request(struct rvt_qp *qp);
  35
  36 static bool hfi1_opfn_extended(u32 bth1)
  37 {
  38         return !!(bth1 & IB_BTHE_E);
  39 }
  40
  41 static void opfn_conn_request(struct rvt_qp *qp)
  42 {
  43         struct hfi1_qp_priv *priv = qp->priv;
  44         struct ib_atomic_wr wr;
  45         u16 mask, capcode;
  46         struct hfi1_opfn_type *extd;
  47         u64 data;
  48         unsigned long flags;
  49         int ret = 0;
  50
  51         spin_lock_irqsave(&priv->opfn.lock, flags);
  52         /*
  53          * Exit if the extended bit is not set, or if nothing is requested, or
  54          * if we have completed all requests, or if a previous request is in
  55          * progress
  56          */
  57         if (!priv->opfn.extended || !priv->opfn.requested ||
  58             priv->opfn.requested == priv->opfn.completed || priv->opfn.curr)
  59                 goto done;
  60
  61         mask = priv->opfn.requested & ~priv->opfn.completed;
  62         capcode = ilog2(mask & ~(mask - 1)) + 1;
  63         if (capcode >= STL_VERBS_EXTD_MAX) {
  64                 priv->opfn.completed |= OPFN_CODE(capcode);
  65                 goto done;
  66         }
  67
  68         extd = &hfi1_opfn_handlers[capcode];
  69         if (!extd || !extd->request || !extd->request(qp, &data)) {
  70                 /*
  71                  * Either there is no handler for this capability or the request
  72                  * packet could not be generated. Either way, mark it as done so
  73                  * we don't keep attempting to complete it.
  74                  */
  75                 priv->opfn.completed |= OPFN_CODE(capcode);
  76                 goto done;
  77         }
  78
  79         data = (data & ~0xf) | capcode;
  80
  81         memset(&wr, 0, sizeof(wr));
  82         wr.wr.opcode = IB_WR_OPFN;
  83         wr.remote_addr = HFI1_VERBS_E_ATOMIC_VADDR;
  84         wr.compare_add = data;
  85
  86         priv->opfn.curr = capcode;      /* A new request is now in progress */
  87         /* Drop opfn.lock before calling ib_post_send() */
  88         spin_unlock_irqrestore(&priv->opfn.lock, flags);
  89
  90         ret = ib_post_send(&qp->ibqp, &wr.wr, NULL);
  91         if (ret)
  92                 goto err;
  93         return;
  94 err:
  95         spin_lock_irqsave(&priv->opfn.lock, flags);
  96         /*
  97          * In case of an unexpected error return from ib_post_send
  98          * clear opfn.curr and reschedule to try again
  99          */
 100         priv->opfn.curr = STL_VERBS_EXTD_NONE;
 101         opfn_schedule_conn_request(qp);
 102 done:
 103         spin_unlock_irqrestore(&priv->opfn.lock, flags);
 104 }
 105
 106 void opfn_send_conn_request(struct work_struct *work)
 107 {
 108         struct hfi1_opfn_data *od;
 109         struct hfi1_qp_priv *qpriv;
 110
 111         od = container_of(work, struct hfi1_opfn_data, opfn_work);
 112         qpriv = container_of(od, struct hfi1_qp_priv, opfn);
 113
 114         opfn_conn_request(qpriv->owner);
 115 }
 116
 117 /*
 118  * When QP s_lock is held in the caller, the OPFN request must be scheduled
 119  * to a different workqueue to avoid double locking QP s_lock in call to
 120  * ib_post_send in opfn_conn_request
 121  */
 122 static void opfn_schedule_conn_request(struct rvt_qp *qp)
 123 {
 124         struct hfi1_qp_priv *priv = qp->priv;
 125
 126         queue_work(opfn_wq, &priv->opfn.opfn_work);
 127 }
 128
 129 void opfn_conn_response(struct rvt_qp *qp, struct rvt_ack_entry *e,
 130                         struct ib_atomic_eth *ateth)
 131 {
 132         struct hfi1_qp_priv *priv = qp->priv;
 133         u64 data = be64_to_cpu(ateth->compare_data);
 134         struct hfi1_opfn_type *extd;
 135         u8 capcode;
 136         unsigned long flags;
 137
 138         capcode = data & 0xf;
 139         if (!capcode || capcode >= STL_VERBS_EXTD_MAX)
 140                 return;
 141
 142         extd = &hfi1_opfn_handlers[capcode];
 143
 144         if (!extd || !extd->response) {
 145                 e->atomic_data = capcode;
 146                 return;
 147         }
 148
 149         spin_lock_irqsave(&priv->opfn.lock, flags);
 150         if (priv->opfn.completed & OPFN_CODE(capcode)) {
 151                 /*
 152                  * We are receiving a request for a feature that has already
 153                  * been negotiated. This may mean that the other side has reset
 154                  */
 155                 priv->opfn.completed &= ~OPFN_CODE(capcode);
 156                 if (extd->error)
 157                         extd->error(qp);
 158         }
 159
 160         if (extd->response(qp, &data))
 161                 priv->opfn.completed |= OPFN_CODE(capcode);
 162         e->atomic_data = (data & ~0xf) | capcode;
 163         spin_unlock_irqrestore(&priv->opfn.lock, flags);
 164 }
 165
 166 void opfn_conn_reply(struct rvt_qp *qp, u64 data)
 167 {
 168         struct hfi1_qp_priv *priv = qp->priv;
 169         struct hfi1_opfn_type *extd;
 170         u8 capcode;
 171         unsigned long flags;
 172
 173         capcode = data & 0xf;
 174         if (!capcode || capcode >= STL_VERBS_EXTD_MAX)
 175                 return;
 176
 177         spin_lock_irqsave(&priv->opfn.lock, flags);
 178         /*
 179          * Either there is no previous request or the reply is not for the
 180          * current request
 181          */
 182         if (!priv->opfn.curr || capcode != priv->opfn.curr)
 183                 goto done;
 184
 185         extd = &hfi1_opfn_handlers[capcode];
 186
 187         if (!extd || !extd->reply)
 188                 goto clear;
 189
 190         if (extd->reply(qp, data))
 191                 priv->opfn.completed |= OPFN_CODE(capcode);
 192 clear:
 193         /*
 194          * Clear opfn.curr to indicate that the previous request is no longer in
 195          * progress
 196          */
 197         priv->opfn.curr = STL_VERBS_EXTD_NONE;
 198 done:
 199         spin_unlock_irqrestore(&priv->opfn.lock, flags);
 200 }
 201
 202 void opfn_conn_error(struct rvt_qp *qp)
 203 {
 204         struct hfi1_qp_priv *priv = qp->priv;
 205         struct hfi1_opfn_type *extd = NULL;
 206         unsigned long flags;
 207         u16 capcode;
 208
 209         /*
 210          * The QP has gone into the Error state. We have to invalidate all
 211          * negotiated feature, including the one in progress (if any). The RC
 212          * QP handling will clean the WQE for the connection request.
 213          */
 214         spin_lock_irqsave(&priv->opfn.lock, flags);
 215         while (priv->opfn.completed) {
 216                 capcode = priv->opfn.completed & ~(priv->opfn.completed - 1);
 217                 extd = &hfi1_opfn_handlers[ilog2(capcode) + 1];
 218                 if (extd->error)
 219                         extd->error(qp);
 220                 priv->opfn.completed &= ~OPFN_CODE(capcode);
 221         }
 222         priv->opfn.extended = 0;
 223         priv->opfn.requested = 0;
 224         priv->opfn.curr = STL_VERBS_EXTD_NONE;
 225         spin_unlock_irqrestore(&priv->opfn.lock, flags);
 226 }
 227
 228 void opfn_qp_init(struct rvt_qp *qp, struct ib_qp_attr *attr, int attr_mask)
 229 {
 230         struct ib_qp *ibqp = &qp->ibqp;
 231         struct hfi1_qp_priv *priv = qp->priv;
 232         unsigned long flags;
 233
 234         spin_lock_irqsave(&priv->opfn.lock, flags);
 235         if (ibqp->qp_type == IB_QPT_RC && HFI1_CAP_IS_KSET(TID_RDMA)) {
 236                 struct tid_rdma_params *local = &priv->tid_rdma.local;
 237
 238                 if (qp->pmtu == enum_to_mtu(OPA_MTU_4096) ||
 239                     qp->pmtu == enum_to_mtu(OPA_MTU_8192)) {
 240                         tid_rdma_opfn_init(qp, local);
 241                         /*
 242                          * We only want to set the OPFN requested bit when the
 243                          * QP transitions to RTS.
 244                          */
 245                         if (attr_mask & IB_QP_STATE &&
 246                             attr->qp_state == IB_QPS_RTS) {
 247                                 priv->opfn.requested |= OPFN_MASK(TID_RDMA);
 248                                 /*
 249                                  * If the QP is transitioning to RTS and the
 250                                  * opfn.completed for TID RDMA has already been
 251                                  * set, the QP is being moved *back* into RTS.
 252                                  * We can now renegotiate the TID RDMA
 253                                  * parameters.
 254                                  */
 255                                 if (priv->opfn.completed &
 256                                     OPFN_MASK(TID_RDMA)) {
 257                                         priv->opfn.completed &=
 258                                                 ~OPFN_MASK(TID_RDMA);
 259                                         /*
 260                                          * Since the opfn.completed bit was
 261                                          * already set, it is safe to assume
 262                                          * that the opfn.extended is also set.
 263                                          */
 264                                         opfn_schedule_conn_request(qp);
 265                                 }
 266                         }
 267                 } else {
 268                         memset(local, 0, sizeof(*local));
 269                 }
 270         }
 271         spin_unlock_irqrestore(&priv->opfn.lock, flags);
 272 }
 273
 274 void opfn_trigger_conn_request(struct rvt_qp *qp, u32 bth1)
 275 {
 276         struct hfi1_qp_priv *priv = qp->priv;
 277
 278         if (!priv->opfn.extended && hfi1_opfn_extended(bth1) &&
 279             HFI1_CAP_IS_KSET(OPFN)) {
 280                 priv->opfn.extended = 1;
 281                 if (qp->state == IB_QPS_RTS)
 282                         opfn_conn_request(qp);
 283         }
 284 }
 285
 286 int opfn_init(void)
 287 {
 288         opfn_wq = alloc_workqueue("hfi_opfn",
 289                                   WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE |
 290                                   WQ_MEM_RECLAIM,
 291                                   HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES);
 292         if (!opfn_wq)
 293                 return -ENOMEM;
 294
 295         return 0;
 296 }
 297
 298 void opfn_exit(void)
 299 {
 300         if (opfn_wq) {
 301                 destroy_workqueue(opfn_wq);
 302                 opfn_wq = NULL;
 303         }
 304 }