sunrpc: introduce the concept of a minimum number of threads per pool

Add a new pool->sp_nrthrmin field to track the minimum number of threads
in a pool. Add min_threads parameters to both svc_set_num_threads() and
svc_set_pool_threads(). If min_threads is non-zero and less than the
max, svc_set_num_threads() will ensure that the number of running
threads is between the min and the max.

If the min is 0 or greater than the max, then it is ignored, and the
maximum number of threads will be started, and never spun down.

For now, the min_threads is always 0, but a later patch will pass the
proper value through from nfsd.

Signed-off-by: Jeff Layton <jlayton@kernel.org>
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>

diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c
index fbf132b4e08d11a91784c21ee0209fd7c149fd9d..e2a1b12272f564392bf8d5379e6a25852ca1431b 100644 (file)
--- a/fs/lockd/svc.c
+++ b/fs/lockd/svc.c
@@ -340,7 +340,7 @@ static int lockd_get(void)
                return -ENOMEM;
        }
 
-       error = svc_set_num_threads(serv, 1);
+       error = svc_set_num_threads(serv, 0, 1);
        if (error < 0) {
                svc_destroy(&serv);
                return error;
@@ -368,7 +368,7 @@ static void lockd_put(void)
        unregister_inet6addr_notifier(&lockd_inet6addr_notifier);
 #endif
 
-       svc_set_num_threads(nlmsvc_serv, 0);
+       svc_set_num_threads(nlmsvc_serv, 0, 0);
        timer_delete_sync(&nlmsvc_retry);
        svc_destroy(&nlmsvc_serv);
        dprintk("lockd_down: service destroyed\n");

diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c
index d01de143927bfeab2b44e60928512a03183e7244..6889818138e3a553ab55ce22293a8c87541d042d 100644 (file)
--- a/fs/nfs/callback.c
+++ b/fs/nfs/callback.c
@@ -119,9 +119,9 @@ static int nfs_callback_start_svc(int minorversion, struct rpc_xprt *xprt,
        if (serv->sv_nrthreads == nrservs)
                return 0;
 
-       ret = svc_set_num_threads(serv, nrservs);
+       ret = svc_set_num_threads(serv, 0, nrservs);
        if (ret) {
-               svc_set_num_threads(serv, 0);
+               svc_set_num_threads(serv, 0, 0);
                return ret;
        }
        dprintk("nfs_callback_up: service started\n");
@@ -242,7 +242,7 @@ int nfs_callback_up(u32 minorversion, struct rpc_xprt *xprt)
        cb_info->users++;
 err_net:
        if (!cb_info->users) {
-               svc_set_num_threads(cb_info->serv, 0);
+               svc_set_num_threads(cb_info->serv, 0, 0);
                svc_destroy(&cb_info->serv);
        }
 err_create:
@@ -268,7 +268,7 @@ void nfs_callback_down(int minorversion, struct net *net, struct rpc_xprt *xprt)
        nfs_callback_down_net(minorversion, serv, net);
        cb_info->users--;
        if (cb_info->users == 0) {
-               svc_set_num_threads(serv, 0);
+               svc_set_num_threads(serv, 0, 0);
                dprintk("nfs_callback_down: service destroyed\n");
                xprt_svc_destroy_nullify_bc(xprt, &cb_info->serv);
        }

diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
index 049165ee26afca9f3d6d4ba471823597c93262a5..1b3a143e0b29603e594f8dbb1f88a20b99b67e8c 100644 (file)
--- a/fs/nfsd/nfssvc.c
+++ b/fs/nfsd/nfssvc.c
@@ -580,7 +580,7 @@ void nfsd_shutdown_threads(struct net *net)
        }
 
        /* Kill outstanding nfsd threads */
-       svc_set_num_threads(serv, 0);
+       svc_set_num_threads(serv, 0, 0);
        nfsd_destroy_serv(net);
        mutex_unlock(&nfsd_mutex);
 }
@@ -690,7 +690,7 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net)
 
        /* Special case: When n == 1, distribute threads equally among pools. */
        if (n == 1)
-               return svc_set_num_threads(nn->nfsd_serv, nthreads[0]);
+               return svc_set_num_threads(nn->nfsd_serv, 0, nthreads[0]);
 
        if (n > nn->nfsd_serv->sv_nrpools)
                n = nn->nfsd_serv->sv_nrpools;
@@ -718,7 +718,7 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net)
        for (i = 0; i < n; i++) {
                err = svc_set_pool_threads(nn->nfsd_serv,
                                           &nn->nfsd_serv->sv_pools[i],
-                                          nthreads[i]);
+                                          0, nthreads[i]);
                if (err)
                        goto out;
        }
@@ -727,7 +727,7 @@ int nfsd_set_nrthreads(int n, int *nthreads, struct net *net)
        for (i = n; i < nn->nfsd_serv->sv_nrpools; ++i) {
                err = svc_set_pool_threads(nn->nfsd_serv,
                                           &nn->nfsd_serv->sv_pools[i],
-                                          0);
+                                          0, 0);
                if (err)
                        goto out;
        }

diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index ec2b6ef5482352e61a9861a19f0ae4a610985ae9..8fd511d02f3b36a614db5595c3b88afe9fce92a2 100644 (file)
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -36,6 +36,7 @@
 struct svc_pool {
        unsigned int            sp_id;          /* pool id; also node id on NUMA */
        unsigned int            sp_nrthreads;   /* # of threads currently running in pool */
+       unsigned int            sp_nrthrmin;    /* Min number of threads to run per pool */
        unsigned int            sp_nrthrmax;    /* Max requested number of threads in pool */
        struct lwq              sp_xprts;       /* pending transports */
        struct list_head        sp_all_threads; /* all server threads */
@@ -72,7 +73,7 @@ struct svc_serv {
        struct svc_stat *       sv_stats;       /* RPC statistics */
        spinlock_t              sv_lock;
        unsigned int            sv_nprogs;      /* Number of sv_programs */
-       unsigned int            sv_nrthreads;   /* # of server threads */
+       unsigned int            sv_nrthreads;   /* # of running server threads */
        unsigned int            sv_max_payload; /* datagram payload size */
        unsigned int            sv_max_mesg;    /* max_payload + 1 page for overheads */
        unsigned int            sv_xdrsize;     /* XDR buffer size */
@@ -448,8 +449,9 @@ struct svc_serv *  svc_create_pooled(struct svc_program *prog,
                                     unsigned int bufsize,
                                     int (*threadfn)(void *data));
 int               svc_set_pool_threads(struct svc_serv *serv, struct svc_pool *pool,
-                                       unsigned int nrservs);
-int               svc_set_num_threads(struct svc_serv *serv, unsigned int nrservs);
+                                       unsigned int min_threads, unsigned int max_threads);
+int               svc_set_num_threads(struct svc_serv *serv, unsigned int min_threads,
+                                      unsigned int nrservs);
 int               svc_pool_stats_open(struct svc_info *si, struct file *file);
 void              svc_process(struct svc_rqst *rqstp);
 void              svc_process_bc(struct rpc_rqst *req, struct svc_rqst *rqstp);

diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index dd22906705f1a3105b74db25a4f5750cb17f7a67..92bad06755efdfb83fbac42d35ecb95a917b054e 100644 (file)
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -820,9 +820,14 @@ svc_stop_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  * svc_set_pool_threads - adjust number of threads per pool
  * @serv: RPC service to adjust
  * @pool: Specific pool from which to choose threads
- * @nrservs: New number of threads for @serv (0 means kill all threads)
+ * @min_threads: min number of threads to run in @pool
+ * @max_threads: max number of threads in @pool (0 means kill all threads)
+ *
+ * Create or destroy threads in @pool to bring it into an acceptable range
+ * between @min_threads and @max_threads.
  *
- * Create or destroy threads in @pool to bring it to @nrservs.
+ * If @min_threads is 0 or larger than @max_threads, then it is ignored and
+ * the pool will be set to run a static @max_threads number of threads.
  *
  * Caller must ensure mutual exclusion between this and server startup or
  * shutdown.
@@ -832,16 +837,36 @@ svc_stop_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
  */
 int
 svc_set_pool_threads(struct svc_serv *serv, struct svc_pool *pool,
-                    unsigned int nrservs)
+                    unsigned int min_threads, unsigned int max_threads)
 {
-       int delta = nrservs;
+       int delta;
 
        if (!pool)
                return -EINVAL;
 
-       pool->sp_nrthrmax = nrservs;
-       delta -= pool->sp_nrthreads;
+       /* clamp min threads to the max */
+       if (min_threads > max_threads)
+               min_threads = max_threads;
+
+       pool->sp_nrthrmin = min_threads;
+       pool->sp_nrthrmax = max_threads;
+
+       /*
+        * When min_threads is set, then only change the number of
+        * threads to bring it within an acceptable range.
+        */
+       if (min_threads) {
+               if (pool->sp_nrthreads > max_threads)
+                       delta = max_threads;
+               else if (pool->sp_nrthreads < min_threads)
+                       delta = min_threads;
+               else
+                       return 0;
+       } else {
+               delta = max_threads;
+       }
 
+       delta -= pool->sp_nrthreads;
        if (delta > 0)
                return svc_start_kthreads(serv, pool, delta);
        if (delta < 0)
@@ -853,6 +878,7 @@ EXPORT_SYMBOL_GPL(svc_set_pool_threads);
 /**
  * svc_set_num_threads - adjust number of threads in serv
  * @serv: RPC service to adjust
+ * @min_threads: min number of threads to run per pool
  * @nrservs: New number of threads for @serv (0 means kill all threads)
  *
  * Create or destroy threads in @serv to bring it to @nrservs. If there
@@ -867,20 +893,23 @@ EXPORT_SYMBOL_GPL(svc_set_pool_threads);
  * adjusted; the caller is responsible for recovery.
  */
 int
-svc_set_num_threads(struct svc_serv *serv, unsigned int nrservs)
+svc_set_num_threads(struct svc_serv *serv, unsigned int min_threads,
+                   unsigned int nrservs)
 {
        unsigned int base = nrservs / serv->sv_nrpools;
        unsigned int remain = nrservs % serv->sv_nrpools;
        int i, err = 0;
 
        for (i = 0; i < serv->sv_nrpools; ++i) {
+               struct svc_pool *pool = &serv->sv_pools[i];
                int threads = base;
 
                if (remain) {
                        ++threads;
                        --remain;
                }
-               err = svc_set_pool_threads(serv, &serv->sv_pools[i], threads);
+
+               err = svc_set_pool_threads(serv, pool, min_threads, threads);
                if (err)
                        break;
        }