drivers/infiniband/hw/erdma/erdma_qp.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2
   3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
   4 /*          Kai Shen <kaishen@linux.alibaba.com> */
   5 /* Copyright (c) 2020-2021, Alibaba Group */
   6 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
   7 /* Copyright (c) 2008-2019, IBM Corporation */
   8
   9 #include <linux/errno.h>
  10 #include <linux/pci.h>
  11 #include <linux/scatterlist.h>
  12 #include <linux/types.h>
  13
  14 #include <rdma/ib_user_verbs.h>
  15 #include <rdma/ib_verbs.h>
  16
  17 #include "erdma.h"
  18 #include "erdma_cm.h"
  19 #include "erdma_verbs.h"
  20
  21 void erdma_qp_llp_close(struct erdma_qp *qp)
  22 {
  23         struct erdma_qp_attrs qp_attrs;
  24
  25         down_write(&qp->state_lock);
  26
  27         switch (qp->attrs.state) {
  28         case ERDMA_QP_STATE_RTS:
  29         case ERDMA_QP_STATE_RTR:
  30         case ERDMA_QP_STATE_IDLE:
  31         case ERDMA_QP_STATE_TERMINATE:
  32                 qp_attrs.state = ERDMA_QP_STATE_CLOSING;
  33                 erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE);
  34                 break;
  35         case ERDMA_QP_STATE_CLOSING:
  36                 qp->attrs.state = ERDMA_QP_STATE_IDLE;
  37                 break;
  38         default:
  39                 break;
  40         }
  41
  42         if (qp->cep) {
  43                 erdma_cep_put(qp->cep);
  44                 qp->cep = NULL;
  45         }
  46
  47         up_write(&qp->state_lock);
  48 }
  49
  50 struct ib_qp *erdma_get_ibqp(struct ib_device *ibdev, int id)
  51 {
  52         struct erdma_qp *qp = find_qp_by_qpn(to_edev(ibdev), id);
  53
  54         if (qp)
  55                 return &qp->ibqp;
  56
  57         return NULL;
  58 }
  59
  60 static int erdma_modify_qp_state_to_rts(struct erdma_qp *qp,
  61                                         struct erdma_qp_attrs *attrs,
  62                                         enum erdma_qp_attr_mask mask)
  63 {
  64         int ret;
  65         struct erdma_dev *dev = qp->dev;
  66         struct erdma_cmdq_modify_qp_req req;
  67         struct tcp_sock *tp;
  68         struct erdma_cep *cep = qp->cep;
  69         struct sockaddr_storage local_addr, remote_addr;
  70
  71         if (!(mask & ERDMA_QP_ATTR_LLP_HANDLE))
  72                 return -EINVAL;
  73
  74         if (!(mask & ERDMA_QP_ATTR_MPA))
  75                 return -EINVAL;
  76
  77         ret = getname_local(cep->sock, &local_addr);
  78         if (ret < 0)
  79                 return ret;
  80
  81         ret = getname_peer(cep->sock, &remote_addr);
  82         if (ret < 0)
  83                 return ret;
  84
  85         qp->attrs.state = ERDMA_QP_STATE_RTS;
  86
  87         tp = tcp_sk(qp->cep->sock->sk);
  88
  89         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
  90                                 CMDQ_OPCODE_MODIFY_QP);
  91
  92         req.cfg = FIELD_PREP(ERDMA_CMD_MODIFY_QP_STATE_MASK, qp->attrs.state) |
  93                   FIELD_PREP(ERDMA_CMD_MODIFY_QP_CC_MASK, qp->attrs.cc) |
  94                   FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp));
  95
  96         req.cookie = be32_to_cpu(qp->cep->mpa.ext_data.cookie);
  97         req.dip = to_sockaddr_in(remote_addr).sin_addr.s_addr;
  98         req.sip = to_sockaddr_in(local_addr).sin_addr.s_addr;
  99         req.dport = to_sockaddr_in(remote_addr).sin_port;
 100         req.sport = to_sockaddr_in(local_addr).sin_port;
 101
 102         req.send_nxt = tp->snd_nxt;
 103         /* rsvd tcp seq for mpa-rsp in server. */
 104         if (qp->attrs.qp_type == ERDMA_QP_PASSIVE)
 105                 req.send_nxt += MPA_DEFAULT_HDR_LEN + qp->attrs.pd_len;
 106         req.recv_nxt = tp->rcv_nxt;
 107
 108         return erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL,
 109                                    NULL);
 110 }
 111
 112 static int erdma_modify_qp_state_to_stop(struct erdma_qp *qp,
 113                                          struct erdma_qp_attrs *attrs,
 114                                          enum erdma_qp_attr_mask mask)
 115 {
 116         struct erdma_dev *dev = qp->dev;
 117         struct erdma_cmdq_modify_qp_req req;
 118
 119         qp->attrs.state = attrs->state;
 120
 121         erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
 122                                 CMDQ_OPCODE_MODIFY_QP);
 123
 124         req.cfg = FIELD_PREP(ERDMA_CMD_MODIFY_QP_STATE_MASK, attrs->state) |
 125                   FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp));
 126
 127         return erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL,
 128                                    NULL);
 129 }
 130
 131 int erdma_modify_qp_internal(struct erdma_qp *qp, struct erdma_qp_attrs *attrs,
 132                              enum erdma_qp_attr_mask mask)
 133 {
 134         int drop_conn, ret = 0;
 135
 136         if (!mask)
 137                 return 0;
 138
 139         if (!(mask & ERDMA_QP_ATTR_STATE))
 140                 return 0;
 141
 142         switch (qp->attrs.state) {
 143         case ERDMA_QP_STATE_IDLE:
 144         case ERDMA_QP_STATE_RTR:
 145                 if (attrs->state == ERDMA_QP_STATE_RTS) {
 146                         ret = erdma_modify_qp_state_to_rts(qp, attrs, mask);
 147                 } else if (attrs->state == ERDMA_QP_STATE_ERROR) {
 148                         qp->attrs.state = ERDMA_QP_STATE_ERROR;
 149                         if (qp->cep) {
 150                                 erdma_cep_put(qp->cep);
 151                                 qp->cep = NULL;
 152                         }
 153                         ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
 154                 }
 155                 break;
 156         case ERDMA_QP_STATE_RTS:
 157                 drop_conn = 0;
 158
 159                 if (attrs->state == ERDMA_QP_STATE_CLOSING) {
 160                         ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
 161                         drop_conn = 1;
 162                 } else if (attrs->state == ERDMA_QP_STATE_TERMINATE) {
 163                         qp->attrs.state = ERDMA_QP_STATE_TERMINATE;
 164                         ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
 165                         drop_conn = 1;
 166                 } else if (attrs->state == ERDMA_QP_STATE_ERROR) {
 167                         ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
 168                         qp->attrs.state = ERDMA_QP_STATE_ERROR;
 169                         drop_conn = 1;
 170                 }
 171
 172                 if (drop_conn)
 173                         erdma_qp_cm_drop(qp);
 174
 175                 break;
 176         case ERDMA_QP_STATE_TERMINATE:
 177                 if (attrs->state == ERDMA_QP_STATE_ERROR)
 178                         qp->attrs.state = ERDMA_QP_STATE_ERROR;
 179                 break;
 180         case ERDMA_QP_STATE_CLOSING:
 181                 if (attrs->state == ERDMA_QP_STATE_IDLE) {
 182                         qp->attrs.state = ERDMA_QP_STATE_IDLE;
 183                 } else if (attrs->state == ERDMA_QP_STATE_ERROR) {
 184                         ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
 185                         qp->attrs.state = ERDMA_QP_STATE_ERROR;
 186                 } else if (attrs->state != ERDMA_QP_STATE_CLOSING) {
 187                         return -ECONNABORTED;
 188                 }
 189                 break;
 190         default:
 191                 break;
 192         }
 193
 194         return ret;
 195 }
 196
 197 static void erdma_qp_safe_free(struct kref *ref)
 198 {
 199         struct erdma_qp *qp = container_of(ref, struct erdma_qp, ref);
 200
 201         complete(&qp->safe_free);
 202 }
 203
 204 void erdma_qp_put(struct erdma_qp *qp)
 205 {
 206         WARN_ON(kref_read(&qp->ref) < 1);
 207         kref_put(&qp->ref, erdma_qp_safe_free);
 208 }
 209
 210 void erdma_qp_get(struct erdma_qp *qp)
 211 {
 212         kref_get(&qp->ref);
 213 }
 214
 215 static int fill_inline_data(struct erdma_qp *qp,
 216                             const struct ib_send_wr *send_wr, u16 wqe_idx,
 217                             u32 sgl_offset, __le32 *length_field)
 218 {
 219         u32 remain_size, copy_size, data_off, bytes = 0;
 220         char *data;
 221         int i = 0;
 222
 223         wqe_idx += (sgl_offset >> SQEBB_SHIFT);
 224         sgl_offset &= (SQEBB_SIZE - 1);
 225         data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, qp->attrs.sq_size,
 226                                SQEBB_SHIFT);
 227
 228         while (i < send_wr->num_sge) {
 229                 bytes += send_wr->sg_list[i].length;
 230                 if (bytes > (int)ERDMA_MAX_INLINE)
 231                         return -EINVAL;
 232
 233                 remain_size = send_wr->sg_list[i].length;
 234                 data_off = 0;
 235
 236                 while (1) {
 237                         copy_size = min(remain_size, SQEBB_SIZE - sgl_offset);
 238
 239                         memcpy(data + sgl_offset,
 240                                (void *)(uintptr_t)send_wr->sg_list[i].addr +
 241                                        data_off,
 242                                copy_size);
 243                         remain_size -= copy_size;
 244                         data_off += copy_size;
 245                         sgl_offset += copy_size;
 246                         wqe_idx += (sgl_offset >> SQEBB_SHIFT);
 247                         sgl_offset &= (SQEBB_SIZE - 1);
 248
 249                         data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
 250                                                qp->attrs.sq_size, SQEBB_SHIFT);
 251                         if (!remain_size)
 252                                 break;
 253                 }
 254
 255                 i++;
 256         }
 257         *length_field = cpu_to_le32(bytes);
 258
 259         return bytes;
 260 }
 261
 262 static int fill_sgl(struct erdma_qp *qp, const struct ib_send_wr *send_wr,
 263                     u16 wqe_idx, u32 sgl_offset, __le32 *length_field)
 264 {
 265         int i = 0;
 266         u32 bytes = 0;
 267         char *sgl;
 268
 269         if (send_wr->num_sge > qp->dev->attrs.max_send_sge)
 270                 return -EINVAL;
 271
 272         if (sgl_offset & 0xF)
 273                 return -EINVAL;
 274
 275         while (i < send_wr->num_sge) {
 276                 wqe_idx += (sgl_offset >> SQEBB_SHIFT);
 277                 sgl_offset &= (SQEBB_SIZE - 1);
 278                 sgl = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
 279                                       qp->attrs.sq_size, SQEBB_SHIFT);
 280
 281                 bytes += send_wr->sg_list[i].length;
 282                 memcpy(sgl + sgl_offset, &send_wr->sg_list[i],
 283                        sizeof(struct ib_sge));
 284
 285                 sgl_offset += sizeof(struct ib_sge);
 286                 i++;
 287         }
 288
 289         *length_field = cpu_to_le32(bytes);
 290         return 0;
 291 }
 292
 293 static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
 294                               const struct ib_send_wr *send_wr)
 295 {
 296         u32 wqe_size, wqebb_cnt, hw_op, flags, sgl_offset;
 297         u32 idx = *pi & (qp->attrs.sq_size - 1);
 298         enum ib_wr_opcode op = send_wr->opcode;
 299         struct erdma_readreq_sqe *read_sqe;
 300         struct erdma_reg_mr_sqe *regmr_sge;
 301         struct erdma_write_sqe *write_sqe;
 302         struct erdma_send_sqe *send_sqe;
 303         struct ib_rdma_wr *rdma_wr;
 304         struct erdma_mr *mr;
 305         __le32 *length_field;
 306         u64 wqe_hdr, *entry;
 307         struct ib_sge *sge;
 308         u32 attrs;
 309         int ret;
 310
 311         entry = get_queue_entry(qp->kern_qp.sq_buf, idx, qp->attrs.sq_size,
 312                                 SQEBB_SHIFT);
 313
 314         /* Clear the SQE header section. */
 315         *entry = 0;
 316
 317         qp->kern_qp.swr_tbl[idx] = send_wr->wr_id;
 318         flags = send_wr->send_flags;
 319         wqe_hdr = FIELD_PREP(
 320                 ERDMA_SQE_HDR_CE_MASK,
 321                 ((flags & IB_SEND_SIGNALED) || qp->kern_qp.sig_all) ? 1 : 0);
 322         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SE_MASK,
 323                               flags & IB_SEND_SOLICITED ? 1 : 0);
 324         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_FENCE_MASK,
 325                               flags & IB_SEND_FENCE ? 1 : 0);
 326         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_INLINE_MASK,
 327                               flags & IB_SEND_INLINE ? 1 : 0);
 328         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp));
 329
 330         switch (op) {
 331         case IB_WR_RDMA_WRITE:
 332         case IB_WR_RDMA_WRITE_WITH_IMM:
 333                 hw_op = ERDMA_OP_WRITE;
 334                 if (op == IB_WR_RDMA_WRITE_WITH_IMM)
 335                         hw_op = ERDMA_OP_WRITE_WITH_IMM;
 336                 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op);
 337                 rdma_wr = container_of(send_wr, struct ib_rdma_wr, wr);
 338                 write_sqe = (struct erdma_write_sqe *)entry;
 339
 340                 write_sqe->imm_data = send_wr->ex.imm_data;
 341                 write_sqe->sink_stag = cpu_to_le32(rdma_wr->rkey);
 342                 write_sqe->sink_to_h =
 343                         cpu_to_le32(upper_32_bits(rdma_wr->remote_addr));
 344                 write_sqe->sink_to_l =
 345                         cpu_to_le32(lower_32_bits(rdma_wr->remote_addr));
 346
 347                 length_field = &write_sqe->length;
 348                 wqe_size = sizeof(struct erdma_write_sqe);
 349                 sgl_offset = wqe_size;
 350                 break;
 351         case IB_WR_RDMA_READ:
 352         case IB_WR_RDMA_READ_WITH_INV:
 353                 read_sqe = (struct erdma_readreq_sqe *)entry;
 354                 if (unlikely(send_wr->num_sge != 1))
 355                         return -EINVAL;
 356                 hw_op = ERDMA_OP_READ;
 357                 if (op == IB_WR_RDMA_READ_WITH_INV) {
 358                         hw_op = ERDMA_OP_READ_WITH_INV;
 359                         read_sqe->invalid_stag =
 360                                 cpu_to_le32(send_wr->ex.invalidate_rkey);
 361                 }
 362
 363                 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op);
 364                 rdma_wr = container_of(send_wr, struct ib_rdma_wr, wr);
 365                 read_sqe->length = cpu_to_le32(send_wr->sg_list[0].length);
 366                 read_sqe->sink_stag = cpu_to_le32(send_wr->sg_list[0].lkey);
 367                 read_sqe->sink_to_l =
 368                         cpu_to_le32(lower_32_bits(send_wr->sg_list[0].addr));
 369                 read_sqe->sink_to_h =
 370                         cpu_to_le32(upper_32_bits(send_wr->sg_list[0].addr));
 371
 372                 sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
 373                                       qp->attrs.sq_size, SQEBB_SHIFT);
 374                 sge->addr = rdma_wr->remote_addr;
 375                 sge->lkey = rdma_wr->rkey;
 376                 sge->length = send_wr->sg_list[0].length;
 377                 wqe_size = sizeof(struct erdma_readreq_sqe) +
 378                            send_wr->num_sge * sizeof(struct ib_sge);
 379
 380                 goto out;
 381         case IB_WR_SEND:
 382         case IB_WR_SEND_WITH_IMM:
 383         case IB_WR_SEND_WITH_INV:
 384                 send_sqe = (struct erdma_send_sqe *)entry;
 385                 hw_op = ERDMA_OP_SEND;
 386                 if (op == IB_WR_SEND_WITH_IMM) {
 387                         hw_op = ERDMA_OP_SEND_WITH_IMM;
 388                         send_sqe->imm_data = send_wr->ex.imm_data;
 389                 } else if (op == IB_WR_SEND_WITH_INV) {
 390                         hw_op = ERDMA_OP_SEND_WITH_INV;
 391                         send_sqe->invalid_stag =
 392                                 cpu_to_le32(send_wr->ex.invalidate_rkey);
 393                 }
 394                 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op);
 395                 length_field = &send_sqe->length;
 396                 wqe_size = sizeof(struct erdma_send_sqe);
 397                 sgl_offset = wqe_size;
 398
 399                 break;
 400         case IB_WR_REG_MR:
 401                 wqe_hdr |=
 402                         FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, ERDMA_OP_REG_MR);
 403                 regmr_sge = (struct erdma_reg_mr_sqe *)entry;
 404                 mr = to_emr(reg_wr(send_wr)->mr);
 405
 406                 mr->access = ERDMA_MR_ACC_LR |
 407                              to_erdma_access_flags(reg_wr(send_wr)->access);
 408                 regmr_sge->addr = cpu_to_le64(mr->ibmr.iova);
 409                 regmr_sge->length = cpu_to_le32(mr->ibmr.length);
 410                 regmr_sge->stag = cpu_to_le32(mr->ibmr.lkey);
 411                 attrs = FIELD_PREP(ERDMA_SQE_MR_MODE_MASK, 0) |
 412                         FIELD_PREP(ERDMA_SQE_MR_ACCESS_MASK, mr->access) |
 413                         FIELD_PREP(ERDMA_SQE_MR_MTT_CNT_MASK,
 414                                    mr->mem.mtt_nents);
 415
 416                 if (mr->mem.mtt_nents < ERDMA_MAX_INLINE_MTT_ENTRIES) {
 417                         attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 0);
 418                         /* Copy SGLs to SQE content to accelerate */
 419                         memcpy(get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
 420                                                qp->attrs.sq_size, SQEBB_SHIFT),
 421                                mr->mem.mtt_buf, MTT_SIZE(mr->mem.mtt_nents));
 422                         wqe_size = sizeof(struct erdma_reg_mr_sqe) +
 423                                    MTT_SIZE(mr->mem.mtt_nents);
 424                 } else {
 425                         attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 1);
 426                         wqe_size = sizeof(struct erdma_reg_mr_sqe);
 427                 }
 428
 429                 regmr_sge->attrs = cpu_to_le32(attrs);
 430                 goto out;
 431         case IB_WR_LOCAL_INV:
 432                 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,
 433                                       ERDMA_OP_LOCAL_INV);
 434                 regmr_sge = (struct erdma_reg_mr_sqe *)entry;
 435                 regmr_sge->stag = cpu_to_le32(send_wr->ex.invalidate_rkey);
 436                 wqe_size = sizeof(struct erdma_reg_mr_sqe);
 437                 goto out;
 438         default:
 439                 return -EOPNOTSUPP;
 440         }
 441
 442         if (flags & IB_SEND_INLINE) {
 443                 ret = fill_inline_data(qp, send_wr, idx, sgl_offset,
 444                                        length_field);
 445                 if (ret < 0)
 446                         return -EINVAL;
 447                 wqe_size += ret;
 448                 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SGL_LEN_MASK, ret);
 449         } else {
 450                 ret = fill_sgl(qp, send_wr, idx, sgl_offset, length_field);
 451                 if (ret)
 452                         return -EINVAL;
 453                 wqe_size += send_wr->num_sge * sizeof(struct ib_sge);
 454                 wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SGL_LEN_MASK,
 455                                       send_wr->num_sge);
 456         }
 457
 458 out:
 459         wqebb_cnt = SQEBB_COUNT(wqe_size);
 460         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_WQEBB_CNT_MASK, wqebb_cnt - 1);
 461         *pi += wqebb_cnt;
 462         wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, *pi);
 463
 464         *entry = wqe_hdr;
 465
 466         return 0;
 467 }
 468
 469 static void kick_sq_db(struct erdma_qp *qp, u16 pi)
 470 {
 471         u64 db_data = FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp)) |
 472                       FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, pi);
 473
 474         *(u64 *)qp->kern_qp.sq_db_info = db_data;
 475         writeq(db_data, qp->kern_qp.hw_sq_db);
 476 }
 477
 478 int erdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *send_wr,
 479                     const struct ib_send_wr **bad_send_wr)
 480 {
 481         struct erdma_qp *qp = to_eqp(ibqp);
 482         int ret = 0;
 483         const struct ib_send_wr *wr = send_wr;
 484         unsigned long flags;
 485         u16 sq_pi;
 486
 487         if (!send_wr)
 488                 return -EINVAL;
 489
 490         spin_lock_irqsave(&qp->lock, flags);
 491         sq_pi = qp->kern_qp.sq_pi;
 492
 493         while (wr) {
 494                 if ((u16)(sq_pi - qp->kern_qp.sq_ci) >= qp->attrs.sq_size) {
 495                         ret = -ENOMEM;
 496                         *bad_send_wr = send_wr;
 497                         break;
 498                 }
 499
 500                 ret = erdma_push_one_sqe(qp, &sq_pi, wr);
 501                 if (ret) {
 502                         *bad_send_wr = wr;
 503                         break;
 504                 }
 505                 qp->kern_qp.sq_pi = sq_pi;
 506                 kick_sq_db(qp, sq_pi);
 507
 508                 wr = wr->next;
 509         }
 510         spin_unlock_irqrestore(&qp->lock, flags);
 511
 512         return ret;
 513 }
 514
 515 static int erdma_post_recv_one(struct erdma_qp *qp,
 516                                const struct ib_recv_wr *recv_wr)
 517 {
 518         struct erdma_rqe *rqe =
 519                 get_queue_entry(qp->kern_qp.rq_buf, qp->kern_qp.rq_pi,
 520                                 qp->attrs.rq_size, RQE_SHIFT);
 521
 522         rqe->qe_idx = cpu_to_le16(qp->kern_qp.rq_pi + 1);
 523         rqe->qpn = cpu_to_le32(QP_ID(qp));
 524
 525         if (recv_wr->num_sge == 0) {
 526                 rqe->length = 0;
 527         } else if (recv_wr->num_sge == 1) {
 528                 rqe->stag = cpu_to_le32(recv_wr->sg_list[0].lkey);
 529                 rqe->to = cpu_to_le64(recv_wr->sg_list[0].addr);
 530                 rqe->length = cpu_to_le32(recv_wr->sg_list[0].length);
 531         } else {
 532                 return -EINVAL;
 533         }
 534
 535         *(u64 *)qp->kern_qp.rq_db_info = *(u64 *)rqe;
 536         writeq(*(u64 *)rqe, qp->kern_qp.hw_rq_db);
 537
 538         qp->kern_qp.rwr_tbl[qp->kern_qp.rq_pi & (qp->attrs.rq_size - 1)] =
 539                 recv_wr->wr_id;
 540         qp->kern_qp.rq_pi++;
 541
 542         return 0;
 543 }
 544
 545 int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr,
 546                     const struct ib_recv_wr **bad_recv_wr)
 547 {
 548         const struct ib_recv_wr *wr = recv_wr;
 549         struct erdma_qp *qp = to_eqp(ibqp);
 550         unsigned long flags;
 551         int ret;
 552
 553         spin_lock_irqsave(&qp->lock, flags);
 554
 555         while (wr) {
 556                 ret = erdma_post_recv_one(qp, wr);
 557                 if (ret) {
 558                         *bad_recv_wr = wr;
 559                         break;
 560                 }
 561                 wr = wr->next;
 562         }
 563
 564         spin_unlock_irqrestore(&qp->lock, flags);
 565         return ret;
 566 }