]> git.ipfire.org Git - thirdparty/apache/httpd.git/commitdiff
Merge r1705922, r1706523, r1738464, r1738466, r1738486 from trunk:
authorJim Jagielski <jim@apache.org>
Fri, 2 Dec 2016 11:44:57 +0000 (11:44 +0000)
committerJim Jagielski <jim@apache.org>
Fri, 2 Dec 2016 11:44:57 +0000 (11:44 +0000)
When shutting down a process, free resources early

Due to lingering connections, shutting down a process may take a very
long time. Free all recycled pools early in the hope that we can already
give some memory back to the OS.

rename some variables to be more descriptive

pid -> pslot
tid -> tslot
remove unused 'sd'

Terminate keep-alive connections when dying

When shutting down a process gracefully, terminate keep-alive connections so
that we don't get any new requests which may keep the dying process alive
longer.

Exit threads early during shutdown

During graceful shutdown, if there are more running worker threads than open
connections, terminate some threads. This frees resources faster, which may be
needed for new processes.

Exit threads early during shutdown, part 2

Follow up to r1738466: During graceful shutdown, when the listener thread is
closing a connection, it needs to wake up a worker thread so that it may
terminate.

Submitted by: sf
Reviewed/backported by: jim

git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/branches/2.4.x@1772334 13f79535-47bb-0310-9956-ffa450edef68

CHANGES
STATUS
server/mpm/event/event.c
server/mpm/event/fdqueue.c
server/mpm/event/fdqueue.h

diff --git a/CHANGES b/CHANGES
index 3f06e2edd7d8f8fd5698ff04baad403c74683075..5d2462669e16261c454db3bea76a04c38bcb1ff2 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -15,6 +15,9 @@ Changes with Apache 2.4.24
   *) mod_ssl: Fix quick renegotiation (OptRenegotiaton) with no intermediate
      in the client certificate chain.  PR 55786.  [Yann Ylavic]
 
+  *) mpm_event: Free memory earlier when shutting down processes.
+     [Stefan Fritsch]
+
   *) mod_status: Display the process slot number in the async connection
      overview. [Stefan Fritsch]
 
diff --git a/STATUS b/STATUS
index 2c55ccec64775d3760ea6593b076448c77869e75..71988742f01dbdf9119e1c0767412ff36bc68f94 100644 (file)
--- a/STATUS
+++ b/STATUS
@@ -117,25 +117,6 @@ RELEASE SHOWSTOPPERS:
 PATCHES ACCEPTED TO BACKPORT FROM TRUNK:
   [ start all new proposals below, under PATCHES PROPOSED. ]
 
-  *) Improve mod_status view of async connections
-     trunk patches:
-       https://svn.apache.org/r1738628
-       https://svn.apache.org/r1757009
-       https://svn.apache.org/r1756848
-       https://svn.apache.org/r1757029
-     2.4.x patch: https://people.apache.org/~sf/PR53555_2_mod_status.diff
-     +1: sf, jim, wrowe
-
-  *) mpm_event: Free resources earlier during shutdown
-     trunk patches:
-       https://svn.apache.org/r1705922
-       https://svn.apache.org/r1706523
-       https://svn.apache.org/r1738464
-       https://svn.apache.org/r1738466
-       https://svn.apache.org/r1738486
-     2.4.x patch: https://people.apache.org/~sf/PR53555_3_free_early.diff
-     +1: sf, jim, wrowe
-
   *) mpm_event: Use all free scoreboard slots up to ServerLimit, but don't
      re-use scoreboard slots of still running, gracefully finishing processes.
      PR: 53555
index 9871a5f0f07203f7adf81f9f4073c0687fb3c983..bd74f24e7c552afbcc58caa65eb1bf5fa8900597 100644 (file)
@@ -181,6 +181,8 @@ static apr_uint32_t connection_count = 0;   /* Number of open connections */
 static apr_uint32_t lingering_count = 0;    /* Number of connections in lingering close */
 static apr_uint32_t suspended_count = 0;    /* Number of suspended connections */
 static apr_uint32_t clogged_count = 0;      /* Number of threads processing ssl conns */
+static apr_uint32_t threads_shutdown = 0;   /* Number of threads that have shutdown
+                                               early during graceful termination */
 static int resource_shortage = 0;
 static fd_queue_t *worker_queue;
 static fd_queue_info_t *worker_queue_info;
@@ -288,9 +290,8 @@ static apr_pollset_t *event_pollset;
 /* The structure used to pass unique initialization info to each thread */
 typedef struct
 {
-    int pid;
-    int tid;
-    int sd;
+    int pslot;  /* process slot */
+    int tslot;  /* worker slot of the thread */
 } proc_info;
 
 /* Structure used to pass information to the thread responsible for
@@ -911,6 +912,8 @@ static int start_lingering_close_nonblocking(event_conn_state_t *cs)
         || apr_socket_shutdown(csd, APR_SHUTDOWN_WRITE) != APR_SUCCESS) {
         apr_socket_close(csd);
         ap_push_pool(worker_queue_info, cs->p);
+        if (dying)
+            ap_queue_interrupt_one(worker_queue);
         return 0;
     }
     return start_lingering_close_common(cs, 0);
@@ -934,6 +937,8 @@ static int stop_lingering_close(event_conn_state_t *cs)
         AP_DEBUG_ASSERT(0);
     }
     ap_push_pool(worker_queue_info, cs->p);
+    if (dying)
+        ap_queue_interrupt_one(worker_queue);
     return 0;
 }
 
@@ -1219,6 +1224,9 @@ static void close_listeners(int process_slot, int *closed)
         }
         /* wake up the main thread */
         kill(ap_my_pid, SIGTERM);
+
+        ap_free_idle_pools(worker_queue_info);
+        ap_queue_interrupt_all(worker_queue);
     }
 }
 
@@ -1439,6 +1447,8 @@ static void process_lingering_close(event_conn_state_t *cs, const apr_pollfd_t *
     TO_QUEUE_ELEM_INIT(cs);
 
     ap_push_pool(worker_queue_info, cs->p);
+    if (dying)
+        ap_queue_interrupt_one(worker_queue);
 }
 
 /* call 'func' for all elements of 'q' with timeout less than 'timeout_time'.
@@ -1518,7 +1528,7 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy)
     timer_event_t *te;
     apr_status_t rc;
     proc_info *ti = dummy;
-    int process_slot = ti->pid;
+    int process_slot = ti->pslot;
     apr_pool_t *tpool = apr_thread_pool_get(thd);
     void *csd = NULL;
     apr_pool_t *ptrans;         /* Pool for per-transaction stuff */
@@ -1584,6 +1594,12 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy)
                              *keepalive_q->total,
                              apr_atomic_read32(&lingering_count),
                              apr_atomic_read32(&suspended_count));
+                if (dying) {
+                    ap_log_error(APLOG_MARK, APLOG_TRACE6, 0, ap_server_conf,
+                                 "%u/%u workers shutdown",
+                                 apr_atomic_read32(&threads_shutdown),
+                                 threads_per_child);
+                }
                 apr_thread_mutex_unlock(timeout_mutex);
             }
         }
@@ -1818,11 +1834,12 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy)
             /* If all workers are busy, we kill older keep-alive connections so that they
              * may connect to another process.
              */
-            if (workers_were_busy && *keepalive_q->total) {
-                ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, ap_server_conf,
-                             "All workers are busy, will close %d keep-alive "
-                             "connections",
-                             *keepalive_q->total);
+            if ((workers_were_busy || dying) && *keepalive_q->total) {
+                if (!dying)
+                    ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, ap_server_conf,
+                                 "All workers are busy, will close %d keep-alive "
+                                 "connections",
+                                 *keepalive_q->total);
                 process_timeout_queue(keepalive_q, 0,
                                       start_lingering_close_nonblocking);
             }
@@ -1869,6 +1886,34 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy)
     return NULL;
 }
 
+/*
+ * During graceful shutdown, if there are more running worker threads than
+ * open connections, exit one worker thread.
+ *
+ * return 1 if thread should exit, 0 if it should continue running.
+ */
+static int worker_thread_should_exit_early(void)
+{
+    for (;;) {
+        apr_uint32_t conns = apr_atomic_read32(&connection_count);
+        apr_uint32_t dead = apr_atomic_read32(&threads_shutdown);
+        apr_uint32_t newdead;
+
+        AP_DEBUG_ASSERT(dead <= threads_per_child);
+        if (conns >= threads_per_child - dead)
+            return 0;
+
+        newdead = dead + 1;
+        if (apr_atomic_cas32(&threads_shutdown, newdead, dead) == dead) {
+            /*
+             * No other thread has exited in the mean time, safe to exit
+             * this one.
+             */
+            return 1;
+        }
+    }
+}
+
 /* XXX For ungraceful termination/restart, we definitely don't want to
  *     wait for active connections to finish but we may want to wait
  *     for idle workers to get out of the queue code and release mutexes,
@@ -1879,8 +1924,8 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy)
 static void *APR_THREAD_FUNC worker_thread(apr_thread_t * thd, void *dummy)
 {
     proc_info *ti = dummy;
-    int process_slot = ti->pid;
-    int thread_slot = ti->tid;
+    int process_slot = ti->pslot;
+    int thread_slot = ti->tslot;
     apr_socket_t *csd = NULL;
     event_conn_state_t *cs;
     apr_pool_t *ptrans;         /* Pool for per-transaction stuff */
@@ -1916,6 +1961,9 @@ static void *APR_THREAD_FUNC worker_thread(apr_thread_t * thd, void *dummy)
         if (workers_may_exit) {
             break;
         }
+        if (dying && worker_thread_should_exit_early()) {
+            break;
+        }
 
         te = NULL;
         rv = ap_queue_pop_something(worker_queue, &csd, &cs, &ptrans, &te);
@@ -1993,9 +2041,8 @@ static void create_listener_thread(thread_starter * ts)
     apr_status_t rv;
 
     my_info = (proc_info *) ap_malloc(sizeof(proc_info));
-    my_info->pid = my_child_num;
-    my_info->tid = -1;          /* listener thread doesn't have a thread slot */
-    my_info->sd = 0;
+    my_info->pslot = my_child_num;
+    my_info->tslot = -1;      /* listener thread doesn't have a thread slot */
     rv = apr_thread_create(&ts->listener, thread_attr, listener_thread,
                            my_info, pchild);
     if (rv != APR_SUCCESS) {
@@ -2108,9 +2155,8 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy)
             }
 
             my_info = (proc_info *) ap_malloc(sizeof(proc_info));
-            my_info->pid = my_child_num;
-            my_info->tid = i;
-            my_info->sd = 0;
+            my_info->pslot = my_child_num;
+            my_info->tslot = i;
 
             /* We are creating threads right now */
             ap_update_child_status_from_indexes(my_child_num, i,
index 343146f4c65a3f3b4d224f2b85932260d467be4c..64b318d0e06a35fb6bd3e826e73d41803a47aae5 100644 (file)
@@ -280,6 +280,19 @@ void ap_pop_pool(apr_pool_t ** recycled_pool, fd_queue_info_t * queue_info)
     }
 }
 
+void ap_free_idle_pools(fd_queue_info_t *queue_info)
+{
+    apr_pool_t *p;
+
+    queue_info->max_recycled_pools = 0;
+    do {
+        ap_pop_pool(&p, queue_info);
+        if (p != NULL)
+            apr_pool_destroy(p);
+    } while (p != NULL);
+}
+
+
 apr_status_t ap_queue_info_term(fd_queue_info_t * queue_info)
 {
     apr_status_t rv;
@@ -477,17 +490,30 @@ apr_status_t ap_queue_pop_something(fd_queue_t * queue, apr_socket_t ** sd,
     return rv;
 }
 
-apr_status_t ap_queue_interrupt_all(fd_queue_t * queue)
+static apr_status_t queue_interrupt(fd_queue_t * queue, int all)
 {
     apr_status_t rv;
 
     if ((rv = apr_thread_mutex_lock(queue->one_big_mutex)) != APR_SUCCESS) {
         return rv;
     }
-    apr_thread_cond_broadcast(queue->not_empty);
+    if (all)
+        apr_thread_cond_broadcast(queue->not_empty);
+    else
+        apr_thread_cond_signal(queue->not_empty);
     return apr_thread_mutex_unlock(queue->one_big_mutex);
 }
 
+apr_status_t ap_queue_interrupt_all(fd_queue_t * queue)
+{
+    return queue_interrupt(queue, 1);
+}
+
+apr_status_t ap_queue_interrupt_one(fd_queue_t * queue)
+{
+    return queue_interrupt(queue, 0);
+}
+
 apr_status_t ap_queue_term(fd_queue_t * queue)
 {
     apr_status_t rv;
index 955816b7d2aa1e12a69327c069fedda8b8c6217a..37be684d150b72a99db82b3308873bde693f4847 100644 (file)
@@ -52,6 +52,7 @@ apr_status_t ap_queue_info_wait_for_idler(fd_queue_info_t * queue_info,
                                           int *had_to_block);
 apr_status_t ap_queue_info_term(fd_queue_info_t * queue_info);
 apr_uint32_t ap_queue_info_get_idlers(fd_queue_info_t * queue_info);
+void ap_free_idle_pools(fd_queue_info_t *queue_info);
 
 struct fd_queue_elem_t
 {
@@ -98,6 +99,7 @@ apr_status_t ap_queue_pop_something(fd_queue_t * queue, apr_socket_t ** sd,
                                     event_conn_state_t ** ecs, apr_pool_t ** p,
                                     timer_event_t ** te);
 apr_status_t ap_queue_interrupt_all(fd_queue_t * queue);
+apr_status_t ap_queue_interrupt_one(fd_queue_t * queue);
 apr_status_t ap_queue_term(fd_queue_t * queue);
 
 #endif /* FDQUEUE_H */