From: Jim Jagielski Date: Fri, 2 Dec 2016 11:44:57 +0000 (+0000) Subject: Merge r1705922, r1706523, r1738464, r1738466, r1738486 from trunk: X-Git-Tag: 2.4.24~86 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=15dc4248820cc80356d14030d9ba02f71c0ca722;p=thirdparty%2Fapache%2Fhttpd.git Merge r1705922, r1706523, r1738464, r1738466, r1738486 from trunk: When shutting down a process, free resources early Due to lingering connections, shutting down a process may take a very long time. Free all recycled pools early in the hope that we can already give some memory back to the OS. rename some variables to be more descriptive pid -> pslot tid -> tslot remove unused 'sd' Terminate keep-alive connections when dying When shutting down a process gracefully, terminate keep-alive connections so that we don't get any new requests which may keep the dying process alive longer. Exit threads early during shutdown During graceful shutdown, if there are more running worker threads than open connections, terminate some threads. This frees resources faster, which may be needed for new processes. Exit threads early during shutdown, part 2 Follow up to r1738466: During graceful shutdown, when the listener thread is closing a connection, it needs to wake up a worker thread so that it may terminate. Submitted by: sf Reviewed/backported by: jim git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/branches/2.4.x@1772334 13f79535-47bb-0310-9956-ffa450edef68 --- diff --git a/CHANGES b/CHANGES index 3f06e2edd7d..5d2462669e1 100644 --- a/CHANGES +++ b/CHANGES @@ -15,6 +15,9 @@ Changes with Apache 2.4.24 *) mod_ssl: Fix quick renegotiation (OptRenegotiaton) with no intermediate in the client certificate chain. PR 55786. [Yann Ylavic] + *) mpm_event: Free memory earlier when shutting down processes. + [Stefan Fritsch] + *) mod_status: Display the process slot number in the async connection overview. [Stefan Fritsch] diff --git a/STATUS b/STATUS index 2c55ccec647..71988742f01 100644 --- a/STATUS +++ b/STATUS @@ -117,25 +117,6 @@ RELEASE SHOWSTOPPERS: PATCHES ACCEPTED TO BACKPORT FROM TRUNK: [ start all new proposals below, under PATCHES PROPOSED. ] - *) Improve mod_status view of async connections - trunk patches: - https://svn.apache.org/r1738628 - https://svn.apache.org/r1757009 - https://svn.apache.org/r1756848 - https://svn.apache.org/r1757029 - 2.4.x patch: https://people.apache.org/~sf/PR53555_2_mod_status.diff - +1: sf, jim, wrowe - - *) mpm_event: Free resources earlier during shutdown - trunk patches: - https://svn.apache.org/r1705922 - https://svn.apache.org/r1706523 - https://svn.apache.org/r1738464 - https://svn.apache.org/r1738466 - https://svn.apache.org/r1738486 - 2.4.x patch: https://people.apache.org/~sf/PR53555_3_free_early.diff - +1: sf, jim, wrowe - *) mpm_event: Use all free scoreboard slots up to ServerLimit, but don't re-use scoreboard slots of still running, gracefully finishing processes. PR: 53555 diff --git a/server/mpm/event/event.c b/server/mpm/event/event.c index 9871a5f0f07..bd74f24e7c5 100644 --- a/server/mpm/event/event.c +++ b/server/mpm/event/event.c @@ -181,6 +181,8 @@ static apr_uint32_t connection_count = 0; /* Number of open connections */ static apr_uint32_t lingering_count = 0; /* Number of connections in lingering close */ static apr_uint32_t suspended_count = 0; /* Number of suspended connections */ static apr_uint32_t clogged_count = 0; /* Number of threads processing ssl conns */ +static apr_uint32_t threads_shutdown = 0; /* Number of threads that have shutdown + early during graceful termination */ static int resource_shortage = 0; static fd_queue_t *worker_queue; static fd_queue_info_t *worker_queue_info; @@ -288,9 +290,8 @@ static apr_pollset_t *event_pollset; /* The structure used to pass unique initialization info to each thread */ typedef struct { - int pid; - int tid; - int sd; + int pslot; /* process slot */ + int tslot; /* worker slot of the thread */ } proc_info; /* Structure used to pass information to the thread responsible for @@ -911,6 +912,8 @@ static int start_lingering_close_nonblocking(event_conn_state_t *cs) || apr_socket_shutdown(csd, APR_SHUTDOWN_WRITE) != APR_SUCCESS) { apr_socket_close(csd); ap_push_pool(worker_queue_info, cs->p); + if (dying) + ap_queue_interrupt_one(worker_queue); return 0; } return start_lingering_close_common(cs, 0); @@ -934,6 +937,8 @@ static int stop_lingering_close(event_conn_state_t *cs) AP_DEBUG_ASSERT(0); } ap_push_pool(worker_queue_info, cs->p); + if (dying) + ap_queue_interrupt_one(worker_queue); return 0; } @@ -1219,6 +1224,9 @@ static void close_listeners(int process_slot, int *closed) } /* wake up the main thread */ kill(ap_my_pid, SIGTERM); + + ap_free_idle_pools(worker_queue_info); + ap_queue_interrupt_all(worker_queue); } } @@ -1439,6 +1447,8 @@ static void process_lingering_close(event_conn_state_t *cs, const apr_pollfd_t * TO_QUEUE_ELEM_INIT(cs); ap_push_pool(worker_queue_info, cs->p); + if (dying) + ap_queue_interrupt_one(worker_queue); } /* call 'func' for all elements of 'q' with timeout less than 'timeout_time'. @@ -1518,7 +1528,7 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) timer_event_t *te; apr_status_t rc; proc_info *ti = dummy; - int process_slot = ti->pid; + int process_slot = ti->pslot; apr_pool_t *tpool = apr_thread_pool_get(thd); void *csd = NULL; apr_pool_t *ptrans; /* Pool for per-transaction stuff */ @@ -1584,6 +1594,12 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) *keepalive_q->total, apr_atomic_read32(&lingering_count), apr_atomic_read32(&suspended_count)); + if (dying) { + ap_log_error(APLOG_MARK, APLOG_TRACE6, 0, ap_server_conf, + "%u/%u workers shutdown", + apr_atomic_read32(&threads_shutdown), + threads_per_child); + } apr_thread_mutex_unlock(timeout_mutex); } } @@ -1818,11 +1834,12 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) /* If all workers are busy, we kill older keep-alive connections so that they * may connect to another process. */ - if (workers_were_busy && *keepalive_q->total) { - ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, ap_server_conf, - "All workers are busy, will close %d keep-alive " - "connections", - *keepalive_q->total); + if ((workers_were_busy || dying) && *keepalive_q->total) { + if (!dying) + ap_log_error(APLOG_MARK, APLOG_TRACE1, 0, ap_server_conf, + "All workers are busy, will close %d keep-alive " + "connections", + *keepalive_q->total); process_timeout_queue(keepalive_q, 0, start_lingering_close_nonblocking); } @@ -1869,6 +1886,34 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) return NULL; } +/* + * During graceful shutdown, if there are more running worker threads than + * open connections, exit one worker thread. + * + * return 1 if thread should exit, 0 if it should continue running. + */ +static int worker_thread_should_exit_early(void) +{ + for (;;) { + apr_uint32_t conns = apr_atomic_read32(&connection_count); + apr_uint32_t dead = apr_atomic_read32(&threads_shutdown); + apr_uint32_t newdead; + + AP_DEBUG_ASSERT(dead <= threads_per_child); + if (conns >= threads_per_child - dead) + return 0; + + newdead = dead + 1; + if (apr_atomic_cas32(&threads_shutdown, newdead, dead) == dead) { + /* + * No other thread has exited in the mean time, safe to exit + * this one. + */ + return 1; + } + } +} + /* XXX For ungraceful termination/restart, we definitely don't want to * wait for active connections to finish but we may want to wait * for idle workers to get out of the queue code and release mutexes, @@ -1879,8 +1924,8 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) static void *APR_THREAD_FUNC worker_thread(apr_thread_t * thd, void *dummy) { proc_info *ti = dummy; - int process_slot = ti->pid; - int thread_slot = ti->tid; + int process_slot = ti->pslot; + int thread_slot = ti->tslot; apr_socket_t *csd = NULL; event_conn_state_t *cs; apr_pool_t *ptrans; /* Pool for per-transaction stuff */ @@ -1916,6 +1961,9 @@ static void *APR_THREAD_FUNC worker_thread(apr_thread_t * thd, void *dummy) if (workers_may_exit) { break; } + if (dying && worker_thread_should_exit_early()) { + break; + } te = NULL; rv = ap_queue_pop_something(worker_queue, &csd, &cs, &ptrans, &te); @@ -1993,9 +2041,8 @@ static void create_listener_thread(thread_starter * ts) apr_status_t rv; my_info = (proc_info *) ap_malloc(sizeof(proc_info)); - my_info->pid = my_child_num; - my_info->tid = -1; /* listener thread doesn't have a thread slot */ - my_info->sd = 0; + my_info->pslot = my_child_num; + my_info->tslot = -1; /* listener thread doesn't have a thread slot */ rv = apr_thread_create(&ts->listener, thread_attr, listener_thread, my_info, pchild); if (rv != APR_SUCCESS) { @@ -2108,9 +2155,8 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) } my_info = (proc_info *) ap_malloc(sizeof(proc_info)); - my_info->pid = my_child_num; - my_info->tid = i; - my_info->sd = 0; + my_info->pslot = my_child_num; + my_info->tslot = i; /* We are creating threads right now */ ap_update_child_status_from_indexes(my_child_num, i, diff --git a/server/mpm/event/fdqueue.c b/server/mpm/event/fdqueue.c index 343146f4c65..64b318d0e06 100644 --- a/server/mpm/event/fdqueue.c +++ b/server/mpm/event/fdqueue.c @@ -280,6 +280,19 @@ void ap_pop_pool(apr_pool_t ** recycled_pool, fd_queue_info_t * queue_info) } } +void ap_free_idle_pools(fd_queue_info_t *queue_info) +{ + apr_pool_t *p; + + queue_info->max_recycled_pools = 0; + do { + ap_pop_pool(&p, queue_info); + if (p != NULL) + apr_pool_destroy(p); + } while (p != NULL); +} + + apr_status_t ap_queue_info_term(fd_queue_info_t * queue_info) { apr_status_t rv; @@ -477,17 +490,30 @@ apr_status_t ap_queue_pop_something(fd_queue_t * queue, apr_socket_t ** sd, return rv; } -apr_status_t ap_queue_interrupt_all(fd_queue_t * queue) +static apr_status_t queue_interrupt(fd_queue_t * queue, int all) { apr_status_t rv; if ((rv = apr_thread_mutex_lock(queue->one_big_mutex)) != APR_SUCCESS) { return rv; } - apr_thread_cond_broadcast(queue->not_empty); + if (all) + apr_thread_cond_broadcast(queue->not_empty); + else + apr_thread_cond_signal(queue->not_empty); return apr_thread_mutex_unlock(queue->one_big_mutex); } +apr_status_t ap_queue_interrupt_all(fd_queue_t * queue) +{ + return queue_interrupt(queue, 1); +} + +apr_status_t ap_queue_interrupt_one(fd_queue_t * queue) +{ + return queue_interrupt(queue, 0); +} + apr_status_t ap_queue_term(fd_queue_t * queue) { apr_status_t rv; diff --git a/server/mpm/event/fdqueue.h b/server/mpm/event/fdqueue.h index 955816b7d2a..37be684d150 100644 --- a/server/mpm/event/fdqueue.h +++ b/server/mpm/event/fdqueue.h @@ -52,6 +52,7 @@ apr_status_t ap_queue_info_wait_for_idler(fd_queue_info_t * queue_info, int *had_to_block); apr_status_t ap_queue_info_term(fd_queue_info_t * queue_info); apr_uint32_t ap_queue_info_get_idlers(fd_queue_info_t * queue_info); +void ap_free_idle_pools(fd_queue_info_t *queue_info); struct fd_queue_elem_t { @@ -98,6 +99,7 @@ apr_status_t ap_queue_pop_something(fd_queue_t * queue, apr_socket_t ** sd, event_conn_state_t ** ecs, apr_pool_t ** p, timer_event_t ** te); apr_status_t ap_queue_interrupt_all(fd_queue_t * queue); +apr_status_t ap_queue_interrupt_one(fd_queue_t * queue); apr_status_t ap_queue_term(fd_queue_t * queue); #endif /* FDQUEUE_H */