queue-5.15/sunrpc-use-rmw-bitops-in-single-threaded-hot-paths.patch

   1 From 989cd25c8fc4466d88a2d04f625e4a652fc7b89a Mon Sep 17 00:00:00 2001
   2 From: Sasha Levin <sashal@kernel.org>
   3 Date: Fri, 29 Apr 2022 10:06:21 -0400
   4 Subject: SUNRPC: Use RMW bitops in single-threaded hot paths
   5
   6 From: Chuck Lever <chuck.lever@oracle.com>
   7
   8 [ Upstream commit 28df0988815f63e2af5e6718193c9f68681ad7ff ]
   9
  10 I noticed CPU pipeline stalls while using perf.
  11
  12 Once an svc thread is scheduled and executing an RPC, no other
  13 processes will touch svc_rqst::rq_flags. Thus bus-locked atomics are
  14 not needed outside the svc thread scheduler.
  15
  16 Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
  17 ---
  18  fs/nfsd/nfs4proc.c                       | 7 ++++---
  19  fs/nfsd/nfs4xdr.c                        | 2 +-
  20  net/sunrpc/auth_gss/svcauth_gss.c        | 4 ++--
  21  net/sunrpc/svc.c                         | 6 +++---
  22  net/sunrpc/svc_xprt.c                    | 2 +-
  23  net/sunrpc/svcsock.c                     | 8 ++++----
  24  net/sunrpc/xprtrdma/svc_rdma_transport.c | 2 +-
  25  7 files changed, 16 insertions(+), 15 deletions(-)
  26
  27 diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
  28 index 3ac2978c596ae..5b56877c7fb57 100644
  29 --- a/fs/nfsd/nfs4proc.c
  30 +++ b/fs/nfsd/nfs4proc.c
  31 @@ -970,7 +970,7 @@ nfsd4_read(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
  32          * the client wants us to do more in this compound:
  33          */
  34         if (!nfsd4_last_compound_op(rqstp))
  35 -               clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
  36 +               __clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
  37
  38         /* check stateid */
  39         status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
  40 @@ -2644,11 +2644,12 @@ nfsd4_proc_compound(struct svc_rqst *rqstp)
  41         cstate->minorversion = args->minorversion;
  42         fh_init(current_fh, NFS4_FHSIZE);
  43         fh_init(save_fh, NFS4_FHSIZE);
  44 +
  45         /*
  46          * Don't use the deferral mechanism for NFSv4; compounds make it
  47          * too hard to avoid non-idempotency problems.
  48          */
  49 -       clear_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
  50 +       __clear_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
  51
  52         /*
  53          * According to RFC3010, this takes precedence over all other errors.
  54 @@ -2770,7 +2771,7 @@ nfsd4_proc_compound(struct svc_rqst *rqstp)
  55  out:
  56         cstate->status = status;
  57         /* Reset deferral mechanism for RPC deferrals */
  58 -       set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
  59 +       __set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
  60         return rpc_success;
  61  }
  62
  63 diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
  64 index 771d3057577ef..96d41b1cc2d17 100644
  65 --- a/fs/nfsd/nfs4xdr.c
  66 +++ b/fs/nfsd/nfs4xdr.c
  67 @@ -2408,7 +2408,7 @@ nfsd4_decode_compound(struct nfsd4_compoundargs *argp)
  68         argp->rqstp->rq_cachetype = cachethis ? RC_REPLBUFF : RC_NOCACHE;
  69
  70         if (readcount > 1 || max_reply > PAGE_SIZE - auth_slack)
  71 -               clear_bit(RQ_SPLICE_OK, &argp->rqstp->rq_flags);
  72 +               __clear_bit(RQ_SPLICE_OK, &argp->rqstp->rq_flags);
  73
  74         return true;
  75  }
  76 diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c
  77 index 48b608cb5f5ec..2381c5d1b0710 100644
  78 --- a/net/sunrpc/auth_gss/svcauth_gss.c
  79 +++ b/net/sunrpc/auth_gss/svcauth_gss.c
  80 @@ -900,7 +900,7 @@ unwrap_integ_data(struct svc_rqst *rqstp, struct xdr_buf *buf, u32 seq, struct g
  81          * rejecting the server-computed MIC in this somewhat rare case,
  82          * do not use splice with the GSS integrity service.
  83          */
  84 -       clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
  85 +       __clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
  86
  87         /* Did we already verify the signature on the original pass through? */
  88         if (rqstp->rq_deferred)
  89 @@ -972,7 +972,7 @@ unwrap_priv_data(struct svc_rqst *rqstp, struct xdr_buf *buf, u32 seq, struct gs
  90         int pad, remaining_len, offset;
  91         u32 rseqno;
  92
  93 -       clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
  94 +       __clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
  95
  96         priv_len = svc_getnl(&buf->head[0]);
  97         if (rqstp->rq_deferred) {
  98 diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
  99 index 87da3ff46ce9a..f2a8c1ee8530e 100644
 100 --- a/net/sunrpc/svc.c
 101 +++ b/net/sunrpc/svc.c
 102 @@ -1281,10 +1281,10 @@ svc_process_common(struct svc_rqst *rqstp, struct kvec *argv, struct kvec *resv)
 103                 goto err_short_len;
 104
 105         /* Will be turned off by GSS integrity and privacy services */
 106 -       set_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
 107 +       __set_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
 108         /* Will be turned off only when NFSv4 Sessions are used */
 109 -       set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
 110 -       clear_bit(RQ_DROPME, &rqstp->rq_flags);
 111 +       __set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
 112 +       __clear_bit(RQ_DROPME, &rqstp->rq_flags);
 113
 114         svc_putu32(resv, rqstp->rq_xid);
 115
 116 diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c
 117 index 67ccf1a6459ae..39acef5134f5c 100644
 118 --- a/net/sunrpc/svc_xprt.c
 119 +++ b/net/sunrpc/svc_xprt.c
 120 @@ -1251,7 +1251,7 @@ static struct cache_deferred_req *svc_defer(struct cache_req *req)
 121         trace_svc_defer(rqstp);
 122         svc_xprt_get(rqstp->rq_xprt);
 123         dr->xprt = rqstp->rq_xprt;
 124 -       set_bit(RQ_DROPME, &rqstp->rq_flags);
 125 +       __set_bit(RQ_DROPME, &rqstp->rq_flags);
 126
 127         dr->handle.revisit = svc_revisit;
 128         return &dr->handle;
 129 diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c
 130 index be7081284a098..46cea0e413aeb 100644
 131 --- a/net/sunrpc/svcsock.c
 132 +++ b/net/sunrpc/svcsock.c
 133 @@ -298,9 +298,9 @@ static void svc_sock_setbufsize(struct svc_sock *svsk, unsigned int nreqs)
 134  static void svc_sock_secure_port(struct svc_rqst *rqstp)
 135  {
 136         if (svc_port_is_privileged(svc_addr(rqstp)))
 137 -               set_bit(RQ_SECURE, &rqstp->rq_flags);
 138 +               __set_bit(RQ_SECURE, &rqstp->rq_flags);
 139         else
 140 -               clear_bit(RQ_SECURE, &rqstp->rq_flags);
 141 +               __clear_bit(RQ_SECURE, &rqstp->rq_flags);
 142  }
 143
 144  /*
 145 @@ -1005,9 +1005,9 @@ static int svc_tcp_recvfrom(struct svc_rqst *rqstp)
 146         rqstp->rq_xprt_ctxt   = NULL;
 147         rqstp->rq_prot        = IPPROTO_TCP;
 148         if (test_bit(XPT_LOCAL, &svsk->sk_xprt.xpt_flags))
 149 -               set_bit(RQ_LOCAL, &rqstp->rq_flags);
 150 +               __set_bit(RQ_LOCAL, &rqstp->rq_flags);
 151         else
 152 -               clear_bit(RQ_LOCAL, &rqstp->rq_flags);
 153 +               __clear_bit(RQ_LOCAL, &rqstp->rq_flags);
 154
 155         p = (__be32 *)rqstp->rq_arg.head[0].iov_base;
 156         calldir = p[1];
 157 diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
 158 index f776f0cb471f0..ac147304fb0e9 100644
 159 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
 160 +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
 161 @@ -602,7 +602,7 @@ static int svc_rdma_has_wspace(struct svc_xprt *xprt)
 162
 163  static void svc_rdma_secure_port(struct svc_rqst *rqstp)
 164  {
 165 -       set_bit(RQ_SECURE, &rqstp->rq_flags);
 166 +       __set_bit(RQ_SECURE, &rqstp->rq_flags);
 167  }
 168
 169  static void svc_rdma_kill_temp_xprt(struct svc_xprt *xprt)
 170 --
 171 2.43.0
 172