nfs: probe for LOCALIO when v3 client reconnects to server

Re-enabling NFSv3 LOCALIO is made more complex (than NFSv4) because v3
is stateless.  As such, the hueristic used to identify a LOCALIO probe
point is more adhoc by nature: if/when NFSv3 client IO begins to
complete again in terms of normal RPC-based NFSv3 server IO, attempt
nfs_local_probe_async().

Care is taken to throttle the frequency of nfs_local_probe_async(),
otherwise there could be a flood of repeat calls to
nfs_local_probe_async().

The throttle is admin controlled using a new module parameter for
nfsv3, e.g.:
  echo 512 > /sys/module/nfsv3/parameters/nfs3_localio_probe_throttle

Probe for NFSv3 LOCALIO every N IO requests (512 in this case). Must
be power-of-2, defaults to 0 (probing disabled).

On systems that expect to use LOCALIO with NFSv3 the admin should
configure the 'nfs3_localio_probe_throttle' module parameter.

This commit backfills module parameter documentation in localio.rst

Signed-off-by: Mike Snitzer <snitzer@kernel.org>
Reviewed-by: Jeff Layton <jlayton@kernel.org>
Signed-off-by: Anna Schumaker <anna.schumaker@oracle.com>

diff --git a/Documentation/filesystems/nfs/localio.rst b/Documentation/filesystems/nfs/localio.rst
index 7d2dbf75e96db215609c5c95c1bd2bb10f232663..79808b37d74578f4a95f9941b0a0358e3f83a74f 100644 (file)
--- a/Documentation/filesystems/nfs/localio.rst
+++ b/Documentation/filesystems/nfs/localio.rst
@@ -291,7 +291,7 @@ align their IO).
 Security
 ========
 
-Localio is only supported when UNIX-style authentication (AUTH_UNIX, aka
+LOCALIO is only supported when UNIX-style authentication (AUTH_UNIX, aka
 AUTH_SYS) is used.
 
 Care is taken to ensure the same NFS security mechanisms are used
@@ -306,6 +306,24 @@ client is afforded this same level of access (albeit in terms of the NFS
 protocol via SUNRPC). No other namespaces (user, mount, etc) have been
 altered or purposely extended from the server to the client.
 
+Module Parameters
+=================
+
+/sys/module/nfs/parameters/localio_enabled (bool)
+controls if LOCALIO is enabled, defaults to Y. If client and server are
+local but 'localio_enabled' is set to N then LOCALIO will not be used.
+
+/sys/module/nfs/parameters/localio_O_DIRECT_semantics (bool)
+controls if O_DIRECT extends down to the underlying filesystem, defaults
+to N. Application IO must be logical blocksize aligned, otherwise
+O_DIRECT will fail.
+
+/sys/module/nfsv3/parameters/nfs3_localio_probe_throttle (uint)
+controls if NFSv3 read and write IOs will trigger (re)enabling of
+LOCALIO every N (nfs3_localio_probe_throttle) IOs, defaults to 0
+(disabled). Must be power-of-2, admin keeps all the pieces if they
+misconfigure (too low a value or non-power-of-2).
+
 Testing
 =======
 

diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c
index 1566163c6d85b998f0391ec48b3429d245a54651..7359e1a3bd84c12231e652e3e70dff7b13342363 100644 (file)
--- a/fs/nfs/nfs3proc.c
+++ b/fs/nfs/nfs3proc.c
@@ -844,6 +844,41 @@ nfs3_proc_pathconf(struct nfs_server *server, struct nfs_fh *fhandle,
        return status;
 }
 
+#if IS_ENABLED(CONFIG_NFS_LOCALIO)
+
+static unsigned nfs3_localio_probe_throttle __read_mostly = 0;
+module_param(nfs3_localio_probe_throttle, uint, 0644);
+MODULE_PARM_DESC(nfs3_localio_probe_throttle,
+                "Probe for NFSv3 LOCALIO every N IO requests. Must be power-of-2, defaults to 0 (probing disabled).");
+
+static void nfs3_localio_probe(struct nfs_server *server)
+{
+       struct nfs_client *clp = server->nfs_client;
+
+       /* Throttled to reduce nfs_local_probe_async() frequency */
+       if (!nfs3_localio_probe_throttle || nfs_server_is_local(clp))
+               return;
+
+       /*
+        * Try (re)enabling LOCALIO if isn't enabled -- admin deems
+        * it worthwhile to periodically check if LOCALIO possible by
+        * setting the 'nfs3_localio_probe_throttle' module parameter.
+        *
+        * This is useful if LOCALIO was previously enabled, but was
+        * disabled due to server restart, and IO has successfully
+        * completed in terms of normal RPC.
+        */
+       if ((clp->cl_uuid.nfs3_localio_probe_count++ &
+            (nfs3_localio_probe_throttle - 1)) == 0) {
+               if (!nfs_server_is_local(clp))
+                       nfs_local_probe_async(clp);
+       }
+}
+
+#else
+static void nfs3_localio_probe(struct nfs_server *server) {}
+#endif
+
 static int nfs3_read_done(struct rpc_task *task, struct nfs_pgio_header *hdr)
 {
        struct inode *inode = hdr->inode;
@@ -855,8 +890,11 @@ static int nfs3_read_done(struct rpc_task *task, struct nfs_pgio_header *hdr)
        if (nfs3_async_handle_jukebox(task, inode))
                return -EAGAIN;
 
-       if (task->tk_status >= 0 && !server->read_hdrsize)
-               cmpxchg(&server->read_hdrsize, 0, hdr->res.replen);
+       if (task->tk_status >= 0) {
+               if (!server->read_hdrsize)
+                       cmpxchg(&server->read_hdrsize, 0, hdr->res.replen);
+               nfs3_localio_probe(server);
+       }
 
        nfs_invalidate_atime(inode);
        nfs_refresh_inode(inode, &hdr->fattr);
@@ -886,8 +924,10 @@ static int nfs3_write_done(struct rpc_task *task, struct nfs_pgio_header *hdr)
 
        if (nfs3_async_handle_jukebox(task, inode))
                return -EAGAIN;
-       if (task->tk_status >= 0)
+       if (task->tk_status >= 0) {
                nfs_writeback_update_inode(hdr);
+               nfs3_localio_probe(NFS_SERVER(inode));
+       }
        return 0;
 }
 

diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c
index bad7691e32b94d1a3104ebff2644c7e49b0ba85d..6a0bdea6d6449f84a512f277b246b8ba76755188 100644 (file)
--- a/fs/nfs_common/nfslocalio.c
+++ b/fs/nfs_common/nfslocalio.c
@@ -43,6 +43,7 @@ void nfs_uuid_init(nfs_uuid_t *nfs_uuid)
        INIT_LIST_HEAD(&nfs_uuid->list);
        INIT_LIST_HEAD(&nfs_uuid->files);
        spin_lock_init(&nfs_uuid->lock);
+       nfs_uuid->nfs3_localio_probe_count = 0;
 }
 EXPORT_SYMBOL_GPL(nfs_uuid_init);
 

diff --git a/include/linux/nfslocalio.h b/include/linux/nfslocalio.h
index 05817d6ef3d1e34efe91ec8bd28186bd4c18b829..9aa8a43843d7179c9f71857599d7fb27a36727aa 100644 (file)
--- a/include/linux/nfslocalio.h
+++ b/include/linux/nfslocalio.h
@@ -27,7 +27,8 @@ struct nfs_file_localio;
  */
 typedef struct {
        uuid_t uuid;
-       /* sadly this struct is just over a cacheline, avoid bouncing */
+       unsigned nfs3_localio_probe_count;
+       /* this struct is over a cacheline, avoid bouncing */
        spinlock_t ____cacheline_aligned lock;
        struct list_head list;
        spinlock_t *list_lock; /* nn->local_clients_lock */