drivers/infiniband/hw/hfi1/tid_rdma.h

   1 /* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
   2 /*
   3  * Copyright(c) 2018 Intel Corporation.
   4  *
   5  */
   6 #ifndef HFI1_TID_RDMA_H
   7 #define HFI1_TID_RDMA_H
   8
   9 #include <linux/circ_buf.h>
  10 #include "common.h"
  11
  12 /* Add a convenience helper */
  13 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1))
  14 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size)
  15 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size)
  16
  17 #define TID_RDMA_MIN_SEGMENT_SIZE       BIT(18)   /* 256 KiB (for now) */
  18 #define TID_RDMA_MAX_SEGMENT_SIZE       BIT(18)   /* 256 KiB (for now) */
  19 #define TID_RDMA_MAX_PAGES              (BIT(18) >> PAGE_SHIFT)
  20
  21 /*
  22  * Bit definitions for priv->s_flags.
  23  * These bit flags overload the bit flags defined for the QP's s_flags.
  24  * Due to the fact that these bit fields are used only for the QP priv
  25  * s_flags, there are no collisions.
  26  *
  27  * HFI1_S_TID_WAIT_INTERLCK - QP is waiting for requester interlock
  28  */
  29 #define HFI1_R_TID_RSC_TIMER      BIT(2)
  30 /* BIT(4) reserved for RVT_S_ACK_PENDING. */
  31 #define HFI1_S_TID_WAIT_INTERLCK  BIT(5)
  32 #define HFI1_R_TID_SW_PSN         BIT(19)
  33
  34 /*
  35  * Unlike regular IB RDMA VERBS, which do not require an entry
  36  * in the s_ack_queue, TID RDMA WRITE requests do because they
  37  * generate responses.
  38  * Therefore, the s_ack_queue needs to be extended by a certain
  39  * amount. The key point is that the queue needs to be extended
  40  * without letting the "user" know so they user doesn't end up
  41  * using these extra entries.
  42  */
  43 #define HFI1_TID_RDMA_WRITE_CNT 8
  44
  45 struct tid_rdma_params {
  46         struct rcu_head rcu_head;
  47         u32 qp;
  48         u32 max_len;
  49         u16 jkey;
  50         u8 max_read;
  51         u8 max_write;
  52         u8 timeout;
  53         u8 urg;
  54         u8 version;
  55 };
  56
  57 struct tid_rdma_qp_params {
  58         struct work_struct trigger_work;
  59         struct tid_rdma_params local;
  60         struct tid_rdma_params __rcu *remote;
  61 };
  62
  63 /* Track state for each hardware flow */
  64 struct tid_flow_state {
  65         u32 generation;
  66         u32 psn;
  67         u32 r_next_psn;      /* next PSN to be received (in TID space) */
  68         u8 index;
  69         u8 last_index;
  70         u8 flags;
  71 };
  72
  73 enum tid_rdma_req_state {
  74         TID_REQUEST_INACTIVE = 0,
  75         TID_REQUEST_INIT,
  76         TID_REQUEST_INIT_RESEND,
  77         TID_REQUEST_ACTIVE,
  78         TID_REQUEST_RESEND,
  79         TID_REQUEST_RESEND_ACTIVE,
  80         TID_REQUEST_QUEUED,
  81         TID_REQUEST_SYNC,
  82         TID_REQUEST_RNR_NAK,
  83         TID_REQUEST_COMPLETE,
  84 };
  85
  86 struct tid_rdma_request {
  87         struct rvt_qp *qp;
  88         struct hfi1_ctxtdata *rcd;
  89         union {
  90                 struct rvt_swqe *swqe;
  91                 struct rvt_ack_entry *ack;
  92         } e;
  93
  94         struct tid_rdma_flow *flows;    /* array of tid flows */
  95         struct rvt_sge_state ss; /* SGE state for TID RDMA requests */
  96         u16 n_flows;            /* size of the flow buffer window */
  97         u16 setup_head;         /* flow index we are setting up */
  98         u16 clear_tail;         /* flow index we are clearing */
  99         u16 flow_idx;           /* flow index most recently set up */
 100         u16 acked_tail;
 101
 102         u32 seg_len;
 103         u32 total_len;
 104         u32 r_ack_psn;          /* next expected ack PSN */
 105         u32 r_flow_psn;         /* IB PSN of next segment start */
 106         u32 r_last_acked;       /* IB PSN of last ACK'ed packet */
 107         u32 s_next_psn;         /* IB PSN of next segment start for read */
 108
 109         u32 total_segs;         /* segments required to complete a request */
 110         u32 cur_seg;            /* index of current segment */
 111         u32 comp_seg;           /* index of last completed segment */
 112         u32 ack_seg;            /* index of last ack'ed segment */
 113         u32 alloc_seg;          /* index of next segment to be allocated */
 114         u32 isge;               /* index of "current" sge */
 115         u32 ack_pending;        /* num acks pending for this request */
 116
 117         enum tid_rdma_req_state state;
 118 };
 119
 120 /*
 121  * When header suppression is used, PSNs associated with a "flow" are
 122  * relevant (and not the PSNs maintained by verbs). Track per-flow
 123  * PSNs here for a TID RDMA segment.
 124  *
 125  */
 126 struct flow_state {
 127         u32 flags;
 128         u32 resp_ib_psn;     /* The IB PSN of the response for this flow */
 129         u32 generation;      /* generation of flow */
 130         u32 spsn;            /* starting PSN in TID space */
 131         u32 lpsn;            /* last PSN in TID space */
 132         u32 r_next_psn;      /* next PSN to be received (in TID space) */
 133
 134         /* For tid rdma read */
 135         u32 ib_spsn;         /* starting PSN in Verbs space */
 136         u32 ib_lpsn;         /* last PSn in Verbs space */
 137 };
 138
 139 struct tid_rdma_pageset {
 140         dma_addr_t addr : 48; /* Only needed for the first page */
 141         u8 idx: 8;
 142         u8 count : 7;
 143         u8 mapped: 1;
 144 };
 145
 146 /**
 147  * kern_tid_node - used for managing TID's in TID groups
 148  *
 149  * @grp_idx: rcd relative index to tid_group
 150  * @map: grp->map captured prior to programming this TID group in HW
 151  * @cnt: Only @cnt of available group entries are actually programmed
 152  */
 153 struct kern_tid_node {
 154         struct tid_group *grp;
 155         u8 map;
 156         u8 cnt;
 157 };
 158
 159 /* Overall info for a TID RDMA segment */
 160 struct tid_rdma_flow {
 161         /*
 162          * While a TID RDMA segment is being transferred, it uses a QP number
 163          * from the "KDETH section of QP numbers" (which is different from the
 164          * QP number that originated the request). Bits 11-15 of these QP
 165          * numbers identify the "TID flow" for the segment.
 166          */
 167         struct flow_state flow_state;
 168         struct tid_rdma_request *req;
 169         u32 tid_qpn;
 170         u32 tid_offset;
 171         u32 length;
 172         u32 sent;
 173         u8 tnode_cnt;
 174         u8 tidcnt;
 175         u8 tid_idx;
 176         u8 idx;
 177         u8 npagesets;
 178         u8 npkts;
 179         u8 pkt;
 180         u8 resync_npkts;
 181         struct kern_tid_node tnode[TID_RDMA_MAX_PAGES];
 182         struct tid_rdma_pageset pagesets[TID_RDMA_MAX_PAGES];
 183         u32 tid_entry[TID_RDMA_MAX_PAGES];
 184 };
 185
 186 enum tid_rnr_nak_state {
 187         TID_RNR_NAK_INIT = 0,
 188         TID_RNR_NAK_SEND,
 189         TID_RNR_NAK_SENT,
 190 };
 191
 192 bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data);
 193 bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data);
 194 bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data);
 195 void tid_rdma_conn_error(struct rvt_qp *qp);
 196 void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p);
 197
 198 int hfi1_kern_exp_rcv_init(struct hfi1_ctxtdata *rcd, int reinit);
 199 int hfi1_kern_exp_rcv_setup(struct tid_rdma_request *req,
 200                             struct rvt_sge_state *ss, bool *last);
 201 int hfi1_kern_exp_rcv_clear(struct tid_rdma_request *req);
 202 void hfi1_kern_exp_rcv_clear_all(struct tid_rdma_request *req);
 203 void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe);
 204
 205 /**
 206  * trdma_clean_swqe - clean flows for swqe if large send queue
 207  * @qp: the qp
 208  * @wqe: the send wqe
 209  */
 210 static inline void trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe)
 211 {
 212         if (!wqe->priv)
 213                 return;
 214         __trdma_clean_swqe(qp, wqe);
 215 }
 216
 217 void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp);
 218
 219 int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp,
 220                       struct ib_qp_init_attr *init_attr);
 221 void hfi1_qp_priv_tid_free(struct rvt_dev_info *rdi, struct rvt_qp *qp);
 222
 223 void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp);
 224
 225 int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp);
 226 void hfi1_kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp);
 227 void hfi1_kern_init_ctxt_generations(struct hfi1_ctxtdata *rcd);
 228
 229 struct cntr_entry;
 230 u64 hfi1_access_sw_tid_wait(const struct cntr_entry *entry,
 231                             void *context, int vl, int mode, u64 data);
 232
 233 u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe,
 234                                     struct ib_other_headers *ohdr,
 235                                     u32 *bth1, u32 *bth2, u32 *len);
 236 u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe,
 237                                  struct ib_other_headers *ohdr, u32 *bth1,
 238                                  u32 *bth2, u32 *len);
 239 void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet);
 240 u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e,
 241                                   struct ib_other_headers *ohdr, u32 *bth0,
 242                                   u32 *bth1, u32 *bth2, u32 *len, bool *last);
 243 void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet);
 244 bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd,
 245                               struct hfi1_pportdata *ppd,
 246                               struct hfi1_packet *packet);
 247 void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe,
 248                                u32 *bth2);
 249 void hfi1_qp_kern_exp_rcv_clear_all(struct rvt_qp *qp);
 250 bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe);
 251
 252 void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe);
 253 static inline void hfi1_setup_tid_rdma_wqe(struct rvt_qp *qp,
 254                                            struct rvt_swqe *wqe)
 255 {
 256         if (wqe->priv &&
 257             wqe->wr.opcode == IB_WR_RDMA_READ &&
 258             wqe->length >= TID_RDMA_MIN_SEGMENT_SIZE)
 259                 setup_tid_rdma_wqe(qp, wqe);
 260 }
 261
 262 u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe,
 263                                   struct ib_other_headers *ohdr,
 264                                   u32 *bth1, u32 *bth2, u32 *len);
 265
 266 void hfi1_compute_tid_rdma_flow_wt(void);
 267
 268 void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet);
 269
 270 u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e,
 271                                    struct ib_other_headers *ohdr, u32 *bth1,
 272                                    u32 bth2, u32 *len,
 273                                    struct rvt_sge_state **ss);
 274
 275 void hfi1_del_tid_reap_timer(struct rvt_qp *qp);
 276
 277 void hfi1_rc_rcv_tid_rdma_write_resp(struct hfi1_packet *packet);
 278
 279 bool hfi1_build_tid_rdma_packet(struct rvt_swqe *wqe,
 280                                 struct ib_other_headers *ohdr,
 281                                 u32 *bth1, u32 *bth2, u32 *len);
 282
 283 void hfi1_rc_rcv_tid_rdma_write_data(struct hfi1_packet *packet);
 284
 285 u32 hfi1_build_tid_rdma_write_ack(struct rvt_qp *qp, struct rvt_ack_entry *e,
 286                                   struct ib_other_headers *ohdr, u16 iflow,
 287                                   u32 *bth1, u32 *bth2);
 288
 289 void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet);
 290
 291 #endif /* HFI1_TID_RDMA_H */