xprtrdma: Remove temp allocation of rpcrdma_rep objects

The original code was designed so that most calls to
rpcrdma_rep_create() would occur on the NUMA node that the device
preferred. There are a few cases where that's not possible, so
those reps are marked as temporary.

However, we have the device (and its preferred node) already in
rpcrdma_rep_create(), so let's use that to guarantee the memory
is allocated from the correct node.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>

diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
index 190a4de239c85a83f0646d812bd6597195028933..1478c41c7e9da7775b22f0bb4c45d44ed8263820 100644 (file)
--- a/net/sunrpc/xprtrdma/rpc_rdma.c
+++ b/net/sunrpc/xprtrdma/rpc_rdma.c
@@ -1471,8 +1471,7 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep)
                credits = 1;    /* don't deadlock */
        else if (credits > r_xprt->rx_ep->re_max_requests)
                credits = r_xprt->rx_ep->re_max_requests;
-       rpcrdma_post_recvs(r_xprt, credits + (buf->rb_bc_srv_max_requests << 1),
-                          false);
+       rpcrdma_post_recvs(r_xprt, credits + (buf->rb_bc_srv_max_requests << 1));
        if (buf->rb_credits != credits)
                rpcrdma_update_cwnd(r_xprt, credits);
 

diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 2904454adb06548ee79afbde071199b63a90eb7e..63262ef0c2e3a9f75d2a7192a3faf24755086f28 100644 (file)
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -69,13 +69,15 @@ static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,
                                       struct rpcrdma_sendctx *sc);
 static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt);
 static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt);
-static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep);
 static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt);
 static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
 static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt);
 static void rpcrdma_ep_get(struct rpcrdma_ep *ep);
 static int rpcrdma_ep_put(struct rpcrdma_ep *ep);
 static struct rpcrdma_regbuf *
+rpcrdma_regbuf_alloc_node(size_t size, enum dma_data_direction direction,
+                         int node);
+static struct rpcrdma_regbuf *
 rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction);
 static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb);
 static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb);
@@ -510,7 +512,7 @@ int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt)
         * outstanding Receives.
         */
        rpcrdma_ep_get(ep);
-       rpcrdma_post_recvs(r_xprt, 1, true);
+       rpcrdma_post_recvs(r_xprt, 1);
 
        rc = rdma_connect(ep->re_id, &ep->re_remote_cma);
        if (rc)
@@ -943,18 +945,20 @@ static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt)
 }
 
 static noinline
-struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt,
-                                      bool temp)
+struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt)
 {
        struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
+       struct rpcrdma_ep *ep = r_xprt->rx_ep;
+       struct ib_device *device = ep->re_id->device;
        struct rpcrdma_rep *rep;
 
        rep = kzalloc(sizeof(*rep), XPRTRDMA_GFP_FLAGS);
        if (rep == NULL)
                goto out;
 
-       rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep->re_inline_recv,
-                                              DMA_FROM_DEVICE);
+       rep->rr_rdmabuf = rpcrdma_regbuf_alloc_node(ep->re_inline_recv,
+                                                   DMA_FROM_DEVICE,
+                                                   ibdev_to_node(device));
        if (!rep->rr_rdmabuf)
                goto out_free;
 
@@ -969,7 +973,6 @@ struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt,
        rep->rr_recv_wr.wr_cqe = &rep->rr_cqe;
        rep->rr_recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov;
        rep->rr_recv_wr.num_sge = 1;
-       rep->rr_temp = temp;
 
        spin_lock(&buf->rb_lock);
        list_add(&rep->rr_all, &buf->rb_all_reps);
@@ -988,17 +991,6 @@ static void rpcrdma_rep_free(struct rpcrdma_rep *rep)
        kfree(rep);
 }
 
-static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep)
-{
-       struct rpcrdma_buffer *buf = &rep->rr_rxprt->rx_buf;
-
-       spin_lock(&buf->rb_lock);
-       list_del(&rep->rr_all);
-       spin_unlock(&buf->rb_lock);
-
-       rpcrdma_rep_free(rep);
-}
-
 static struct rpcrdma_rep *rpcrdma_rep_get_locked(struct rpcrdma_buffer *buf)
 {
        struct llist_node *node;
@@ -1030,10 +1022,8 @@ static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt)
        struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
        struct rpcrdma_rep *rep;
 
-       list_for_each_entry(rep, &buf->rb_all_reps, rr_all) {
+       list_for_each_entry(rep, &buf->rb_all_reps, rr_all)
                rpcrdma_regbuf_dma_unmap(rep->rr_rdmabuf);
-               rep->rr_temp = true;    /* Mark this rep for destruction */
-       }
 }
 
 static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf)
@@ -1250,14 +1240,15 @@ void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req)
  * or Replies they may be registered externally via frwr_map.
  */
 static struct rpcrdma_regbuf *
-rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction)
+rpcrdma_regbuf_alloc_node(size_t size, enum dma_data_direction direction,
+                         int node)
 {
        struct rpcrdma_regbuf *rb;
 
-       rb = kmalloc(sizeof(*rb), XPRTRDMA_GFP_FLAGS);
+       rb = kmalloc_node(sizeof(*rb), XPRTRDMA_GFP_FLAGS, node);
        if (!rb)
                return NULL;
-       rb->rg_data = kmalloc(size, XPRTRDMA_GFP_FLAGS);
+       rb->rg_data = kmalloc_node(size, XPRTRDMA_GFP_FLAGS, node);
        if (!rb->rg_data) {
                kfree(rb);
                return NULL;
@@ -1269,6 +1260,12 @@ rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction)
        return rb;
 }
 
+static struct rpcrdma_regbuf *
+rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction)
+{
+       return rpcrdma_regbuf_alloc_node(size, direction, NUMA_NO_NODE);
+}
+
 /**
  * rpcrdma_regbuf_realloc - re-allocate a SEND/RECV buffer
  * @rb: regbuf to reallocate
@@ -1346,10 +1343,9 @@ static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb)
  * rpcrdma_post_recvs - Refill the Receive Queue
  * @r_xprt: controlling transport instance
  * @needed: current credit grant
- * @temp: mark Receive buffers to be deleted after one use
  *
  */
-void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp)
+void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed)
 {
        struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
        struct rpcrdma_ep *ep = r_xprt->rx_ep;
@@ -1363,8 +1359,7 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp)
        if (likely(ep->re_receive_count > needed))
                goto out;
        needed -= ep->re_receive_count;
-       if (!temp)
-               needed += RPCRDMA_MAX_RECV_BATCH;
+       needed += RPCRDMA_MAX_RECV_BATCH;
 
        if (atomic_inc_return(&ep->re_receiving) > 1)
                goto out;
@@ -1373,12 +1368,8 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp)
        wr = NULL;
        while (needed) {
                rep = rpcrdma_rep_get_locked(buf);
-               if (rep && rep->rr_temp) {
-                       rpcrdma_rep_destroy(rep);
-                       continue;
-               }
                if (!rep)
-                       rep = rpcrdma_rep_create(r_xprt, temp);
+                       rep = rpcrdma_rep_create(r_xprt);
                if (!rep)
                        break;
                if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf)) {

diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index 341725c66ec81da09e40ed9b975be80b70d40b10..8147d2b414940e55b66cf40b36c7fe258b31a013 100644 (file)
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -200,7 +200,6 @@ struct rpcrdma_rep {
        __be32                  rr_proc;
        int                     rr_wc_flags;
        u32                     rr_inv_rkey;
-       bool                    rr_temp;
        struct rpcrdma_regbuf   *rr_rdmabuf;
        struct rpcrdma_xprt     *rr_rxprt;
        struct rpc_rqst         *rr_rqst;
@@ -468,7 +467,7 @@ void rpcrdma_flush_disconnect(struct rpcrdma_xprt *r_xprt, struct ib_wc *wc);
 int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt);
 void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt);
 
-void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp);
+void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed);
 
 /*
  * Buffer calls - xprtrdma/verbs.c