SUNRPC: Track consumed rq_pages entries

The rq_pages array holds pages allocated for incoming RPC requests.
Two transport receive paths NULL entries in rq_pages to prevent
svc_rqst_release_pages() from freeing pages that the transport has
taken ownership of:

- svc_tcp_save_pages() moves partial request data pages to
  svsk->sk_pages during multi-fragment TCP reassembly.

- svc_rdma_clear_rqst_pages() moves request data pages to
  head->rc_pages because they are targets of active RDMA Read WRs.

A new rq_pages_nfree field in struct svc_rqst records how many
entries were NULLed. svc_alloc_arg() uses it to refill only those
entries rather than scanning the full rq_pages array. In steady
state, the transport NULLs a handful of entries per RPC, so the
allocator visits only those entries instead of the full ~259 slots
(for 1MB messages).

Reviewed-by: Jeff Layton <jlayton@kernel.org>
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>

diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index 3b1a98ab5cbab8d0f5e46046da1015977262be00..c3399cf645241f64249044db9a82516e6104d27d 100644 (file)
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -143,6 +143,15 @@ extern u32 svc_max_payload(const struct svc_rqst *rqstp);
  * server thread needs to allocate more to replace those used in
  * sending.
  *
+ * rq_pages request page contract:
+ *
+ * Transport receive paths that move request data pages out of
+ * rq_pages -- TCP multi-fragment reassembly (svc_tcp_save_pages)
+ * and RDMA Read I/O (svc_rdma_clear_rqst_pages) -- NULL those
+ * entries to prevent svc_rqst_release_pages() from freeing pages
+ * still in transport use, and set rq_pages_nfree to the count.
+ * svc_alloc_arg() refills only that many rq_pages entries.
+ *
  * xdr_buf holds responses; the structure fits NFS read responses
  * (header, data pages, optional tail) and enables sharing of
  * client-side routines.
@@ -204,6 +213,7 @@ struct svc_rqst {
        struct folio            *rq_scratch_folio;
        struct xdr_buf          rq_res;
        unsigned long           rq_maxpages;    /* entries per page array */
+       unsigned long           rq_pages_nfree; /* rq_pages entries NULLed by transport */
        struct page *           *rq_pages;      /* Call buffer pages */
        struct page *           *rq_respages;   /* Reply buffer pages */
        struct page *           *rq_next_page; /* next reply page to use */

diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 0ce16e9abdf63399baece6fdd101f3e7eae88c4b..6e57e35fa6d6f93ff002076c376d52f91642899e 100644 (file)
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -655,6 +655,7 @@ svc_init_buffer(struct svc_rqst *rqstp, const struct svc_serv *serv, int node)
                return false;
        }
 
+       rqstp->rq_pages_nfree = rqstp->rq_maxpages;
        return true;
 }
 

diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c
index e027765f43071f64ad6e50da7adf491eaed97cc3..795b5729525fe56eb6f14f35fee3e265bdde7fcd 100644 (file)
--- a/net/sunrpc/svc_xprt.c
+++ b/net/sunrpc/svc_xprt.c
@@ -675,12 +675,17 @@ static bool svc_fill_pages(struct svc_rqst *rqstp, struct page **pages,
 static bool svc_alloc_arg(struct svc_rqst *rqstp)
 {
        struct xdr_buf *arg = &rqstp->rq_arg;
-       unsigned long pages;
+       unsigned long pages, nfree;
 
        pages = rqstp->rq_maxpages;
 
-       if (!svc_fill_pages(rqstp, rqstp->rq_pages, pages))
-               return false;
+       nfree = rqstp->rq_pages_nfree;
+       if (nfree) {
+               if (!svc_fill_pages(rqstp, rqstp->rq_pages, nfree))
+                       return false;
+               rqstp->rq_pages_nfree = 0;
+       }
+
        if (!svc_fill_pages(rqstp, rqstp->rq_respages, pages))
                return false;
        rqstp->rq_next_page = rqstp->rq_respages;

diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c
index c86f28f720f7fea4db38f0b91432e6b953fd1e79..2ce43f9995f16b3c06fa4b202a764178d77ed6c4 100644 (file)
--- a/net/sunrpc/svcsock.c
+++ b/net/sunrpc/svcsock.c
@@ -1009,6 +1009,7 @@ static void svc_tcp_save_pages(struct svc_sock *svsk, struct svc_rqst *rqstp)
                svsk->sk_pages[i] = rqstp->rq_pages[i];
                rqstp->rq_pages[i] = NULL;
        }
+       rqstp->rq_pages_nfree = npages;
 }
 
 static void svc_tcp_clear_pages(struct svc_sock *svsk)

diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index 4ec2f9ae06aa58005ec5c04f0a1eb4dedf1bdbae..cf4a1762b629e3b38d47366dff9c609cf0ae9dcc 100644 (file)
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -1107,6 +1107,7 @@ static void svc_rdma_clear_rqst_pages(struct svc_rqst *rqstp,
                head->rc_pages[i] = rqstp->rq_pages[i];
                rqstp->rq_pages[i] = NULL;
        }
+       rqstp->rq_pages_nfree = head->rc_page_count;
 }
 
 /**