From: Yann Ylavic Date: Fri, 3 Aug 2018 09:53:42 +0000 (+0000) Subject: event, worker: initialize the objects used by signal_threads() first. X-Git-Tag: 2.5.0-alpha2-ci-test-only~2414 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8cea5079860a4b1485fa75220d80f5738cd2b54e;p=thirdparty%2Fapache%2Fhttpd.git event, worker: initialize the objects used by signal_threads() first. Follow up to r1835845. If a signal is received early when the MPM children start, signal_threads() may be called concurrently with start_streads() thus before the latter (or its underlying threads like the listener_thread) had a chance to create and init the queues, mutexes, pollset and sockets array used by the former. So move those initializations to a new setup_threads_runtime() function called before start_threads(), where the pruntime pool is also created. git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1837354 13f79535-47bb-0310-9956-ffa450edef68 --- diff --git a/server/mpm/event/event.c b/server/mpm/event/event.c index 27071b1e374..03c3e7d3389 100644 --- a/server/mpm/event/event.c +++ b/server/mpm/event/event.c @@ -465,6 +465,7 @@ int raise_sigstop_flags; static apr_pool_t *pconf; /* Pool for config stuff */ static apr_pool_t *pchild; /* Pool for httpd child stuff */ +static apr_pool_t *pruntime; /* Pool for MPM threads stuff */ static pid_t ap_my_pid; /* Linux getpid() doesn't work except in main thread. Use this instead */ @@ -1367,37 +1368,11 @@ static apr_status_t s_socket_remove(void *user_baton, } #endif -static apr_status_t init_pollset(apr_pool_t *p) -{ #if HAVE_SERF +static void init_serf(apr_pool_t *p) +{ s_baton_t *baton = NULL; -#endif - ap_listen_rec *lr; - listener_poll_type *pt; - int i = 0; - - listener_pollfd = apr_palloc(p, sizeof(apr_pollfd_t) * num_listensocks); - for (lr = my_bucket->listeners; lr != NULL; lr = lr->next, i++) { - apr_pollfd_t *pfd; - AP_DEBUG_ASSERT(i < num_listensocks); - pfd = &listener_pollfd[i]; - pt = apr_pcalloc(p, sizeof(*pt)); - pfd->desc_type = APR_POLL_SOCKET; - pfd->desc.s = lr->sd; - pfd->reqevents = APR_POLLIN; - - pt->type = PT_ACCEPT; - pt->baton = lr; - - pfd->client_data = pt; - - apr_socket_opt_set(pfd->desc.s, APR_SO_NONBLOCK, 1); - apr_pollset_add(event_pollset, pfd); - lr->accept_func = ap_unixd_accept; - } - -#if HAVE_SERF baton = apr_pcalloc(p, sizeof(*baton)); baton->pollset = event_pollset; /* TODO: subpools, threads, reuse, etc. -- currently use malloc() inside :( */ @@ -1409,11 +1384,8 @@ static apr_status_t init_pollset(apr_pool_t *p) ap_register_provider(p, "mpm_serf", "instance", "0", g_serf); - -#endif - - return APR_SUCCESS; } +#endif static apr_status_t push_timer2worker(timer_event_t* te) { @@ -1822,7 +1794,6 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) proc_info *ti = dummy; int process_slot = ti->pslot; struct process_score *ps = ap_get_scoreboard_process(process_slot); - apr_pool_t *tpool = apr_thread_pool_get(thd); int closed = 0; int have_idle_worker = 0; apr_time_t last_log; @@ -1830,16 +1801,9 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy) last_log = apr_time_now(); free(ti); - rc = init_pollset(tpool); - if (rc != APR_SUCCESS) { - ap_log_error(APLOG_MARK, APLOG_ERR, rc, ap_server_conf, - APLOGNO(03266) - "failed to initialize pollset, " - "shutdown process now"); - resource_shortage = 1; - signal_threads(ST_UNGRACEFUL); - return NULL; - } +#if HAVE_SERF + init_serf(apr_thread_pool_get(thd)); +#endif /* Unblock the signal used to wake this thread up, and set a handler for * it. @@ -2444,8 +2408,6 @@ static int check_signal(int signum) return 0; } - - static void create_listener_thread(thread_starter * ts, apr_pool_t *pool) { int my_child_num = ts->child_num_arg; @@ -2467,26 +2429,12 @@ static void create_listener_thread(thread_starter * ts, apr_pool_t *pool) apr_os_thread_get(&listener_os_thread, ts->listener); } -/* XXX under some circumstances not understood, children can get stuck - * in start_threads forever trying to take over slots which will - * never be cleaned up; for now there is an APLOG_DEBUG message issued - * every so often when this condition occurs - */ -static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) +static void setup_threads_runtime(void) { - thread_starter *ts = dummy; - apr_thread_t **threads = ts->threads; - apr_threadattr_t *thread_attr = ts->threadattr; - int my_child_num = ts->child_num_arg; - apr_pool_t *pruntime = NULL; - proc_info *my_info; apr_status_t rv; - int i; - int threads_created = 0; - int listener_started = 0; - int loops; - int prev_threads_created; - int max_recycled_pools = -1; + ap_listen_rec *lr; + apr_pool_t *pskip = NULL; + int max_recycled_pools = -1, i; const int good_methods[] = { APR_POLLSET_KQUEUE, APR_POLLSET_PORT, APR_POLLSET_EPOLL }; @@ -2497,6 +2445,21 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) (async_factor > 2 ? async_factor : 2); int pollset_flags; + /* Event's skiplist operations will happen concurrently with other modules' + * runtime so they need their own pool for allocations, and its lifetime + * should be at least the one of the connections (ptrans). Thus pskip is + * created as a subpool of pconf like/before ptrans (before so that it's + * destroyed after). In forked mode pconf is never destroyed so we are good + * anyway, but in ONE_PROCESS mode this ensures that the skiplist works + * from connection/ptrans cleanups (even after pchild is destroyed). + */ + apr_pool_create(&pskip, pconf); + apr_pool_tag(pskip, "mpm_skiplist"); + apr_thread_mutex_create(&g_timer_skiplist_mtx, APR_THREAD_MUTEX_DEFAULT, pskip); + APR_RING_INIT(&timer_free_ring, timer_event_t, link); + apr_skiplist_init(&timer_skiplist, pskip); + apr_skiplist_set_compare(timer_skiplist, timer_comp, timer_comp); + /* All threads (listener, workers) and synchronization objects (queues, * pollset, mutexes...) created here should have at least the lifetime of * the connections they handle (i.e. ptrans). We can't use this thread's @@ -2577,8 +2540,52 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) clean_child_exit(APEXIT_CHILDFATAL); } + /* Add listeners to the main pollset */ + listener_pollfd = apr_pcalloc(pruntime, num_listensocks * + sizeof(apr_pollfd_t)); + for (i = 0, lr = my_bucket->listeners; lr; lr = lr->next, i++) { + apr_pollfd_t *pfd; + listener_poll_type *pt; + + AP_DEBUG_ASSERT(i < num_listensocks); + pfd = &listener_pollfd[i]; + + pfd->reqevents = APR_POLLIN; + pfd->desc_type = APR_POLL_SOCKET; + pfd->desc.s = lr->sd; + + pt = apr_pcalloc(pruntime, sizeof(*pt)); + pfd->client_data = pt; + pt->type = PT_ACCEPT; + pt->baton = lr; + + apr_socket_opt_set(pfd->desc.s, APR_SO_NONBLOCK, 1); + apr_pollset_add(event_pollset, pfd); + + lr->accept_func = ap_unixd_accept; + } + worker_sockets = apr_pcalloc(pruntime, threads_per_child * sizeof(apr_socket_t *)); +} + +/* XXX under some circumstances not understood, children can get stuck + * in start_threads forever trying to take over slots which will + * never be cleaned up; for now there is an APLOG_DEBUG message issued + * every so often when this condition occurs + */ +static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy) +{ + thread_starter *ts = dummy; + apr_thread_t **threads = ts->threads; + apr_threadattr_t *thread_attr = ts->threadattr; + int my_child_num = ts->child_num_arg; + proc_info *my_info; + apr_status_t rv; + int threads_created = 0; + int listener_started = 0; + int prev_threads_created; + int loops, i; ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(02471) "start_threads: Using %s (%swakeable)", @@ -2728,7 +2735,6 @@ static void child_main(int child_num_arg, int child_bucket) thread_starter *ts; apr_threadattr_t *thread_attr; apr_thread_t *start_thread_id; - apr_pool_t *pskip; int i; /* for benefit of any hooks that run as this child initializes */ @@ -2760,21 +2766,6 @@ static void child_main(int child_num_arg, int child_bucket) clean_child_exit(APEXIT_CHILDFATAL); } - /* Event's skiplist operations will happen concurrently with other modules' - * runtime so they need their own pool for allocations, and its lifetime - * should be at least the one of the connections (ptrans). Thus pskip is - * created as a subpool of pconf like/before ptrans (before so that it's - * destroyed after). In forked mode pconf is never destroyed so we are good - * anyway, but in ONE_PROCESS mode this ensures that the skiplist works - * from connection/ptrans cleanups (even after pchild is destroyed). - */ - apr_pool_create(&pskip, pconf); - apr_pool_tag(pskip, "mpm_skiplist"); - apr_thread_mutex_create(&g_timer_skiplist_mtx, APR_THREAD_MUTEX_DEFAULT, pskip); - APR_RING_INIT(&timer_free_ring, timer_event_t, link); - apr_skiplist_init(&timer_skiplist, pskip); - apr_skiplist_set_compare(timer_skiplist, timer_comp, timer_comp); - /* Just use the standard apr_setup_signal_thread to block all signals * from being received. The child processes no longer use signals for * any communication with the parent process. Let's also do this before @@ -2798,7 +2789,10 @@ static void child_main(int child_num_arg, int child_bucket) conns_this_child = APR_INT32_MAX; } - /* Setup worker threads */ + /* Setup threads */ + + /* Globals used by signal_threads() so to be initialized before */ + setup_threads_runtime(); /* clear the storage; we may not create all our threads immediately, * and we want a 0 entry to indicate a thread which was not created diff --git a/server/mpm/worker/worker.c b/server/mpm/worker/worker.c index 052b852d2a8..da6c6030a63 100644 --- a/server/mpm/worker/worker.c +++ b/server/mpm/worker/worker.c @@ -135,6 +135,8 @@ static int num_listensocks = 0; static int resource_shortage = 0; static fd_queue_t *worker_queue; static fd_queue_info_t *worker_queue_info; +static apr_pollset_t *worker_pollset; + /* data retained by worker across load/unload of the module * allocated on first call to pre-config hook; located on @@ -219,6 +221,7 @@ int raise_sigstop_flags; static apr_pool_t *pconf; /* Pool for config stuff */ static apr_pool_t *pchild; /* Pool for httpd child stuff */ +static apr_pool_t *pruntime; /* Pool for MPM threads stuff */ static pid_t ap_my_pid; /* Linux getpid() doesn't work except in main thread. Use this instead */ @@ -539,47 +542,15 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t *thd, void * dummy) { proc_info * ti = dummy; int process_slot = ti->pid; - apr_pool_t *tpool = apr_thread_pool_get(thd); void *csd = NULL; apr_pool_t *ptrans = NULL; /* Pool for per-transaction stuff */ - apr_pollset_t *pollset; apr_status_t rv; - ap_listen_rec *lr; + ap_listen_rec *lr = NULL; int have_idle_worker = 0; int last_poll_idx = 0; free(ti); - rv = apr_pollset_create(&pollset, num_listensocks, tpool, - APR_POLLSET_NOCOPY); - if (rv != APR_SUCCESS) { - ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03285) - "Couldn't create pollset in thread;" - " check system or user limits"); - /* let the parent decide how bad this really is */ - clean_child_exit(APEXIT_CHILDSICK); - } - - for (lr = my_bucket->listeners; lr != NULL; lr = lr->next) { - apr_pollfd_t *pfd = apr_pcalloc(tpool, sizeof *pfd); - - pfd->desc_type = APR_POLL_SOCKET; - pfd->desc.s = lr->sd; - pfd->reqevents = APR_POLLIN; - pfd->client_data = lr; - - rv = apr_pollset_add(pollset, pfd); - if (rv != APR_SUCCESS) { - ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03286) - "Couldn't create add listener to pollset;" - " check system or user limits"); - /* let the parent decide how bad this really is */ - clean_child_exit(APEXIT_CHILDSICK); - } - - lr->accept_func = ap_unixd_accept; - } - /* Unblock the signal used to wake this thread up, and set a handler for * it. */ @@ -631,7 +602,7 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t *thd, void * dummy) apr_int32_t numdesc; const apr_pollfd_t *pdesc; - rv = apr_pollset_poll(pollset, -1, &numdesc, &pdesc); + rv = apr_pollset_poll(worker_pollset, -1, &numdesc, &pdesc); if (rv != APR_SUCCESS) { if (APR_STATUS_IS_EINTR(rv)) { continue; @@ -886,25 +857,10 @@ static void create_listener_thread(thread_starter *ts, apr_pool_t *pool) apr_os_thread_get(&listener_os_thread, ts->listener); } -/* XXX under some circumstances not understood, children can get stuck - * in start_threads forever trying to take over slots which will - * never be cleaned up; for now there is an APLOG_DEBUG message issued - * every so often when this condition occurs - */ -static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) +static void setup_threads_runtime(void) { - thread_starter *ts = dummy; - apr_thread_t **threads = ts->threads; - apr_threadattr_t *thread_attr = ts->threadattr; - int my_child_num = ts->child_num_arg; - apr_pool_t *pruntime = NULL; - proc_info *my_info; + ap_listen_rec *lr; apr_status_t rv; - int i; - int threads_created = 0; - int listener_started = 0; - int loops; - int prev_threads_created; /* All threads (listener, workers) and synchronization objects (queues, * pollset, mutexes...) created here should have at least the lifetime of @@ -936,8 +892,58 @@ static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) clean_child_exit(APEXIT_CHILDFATAL); } + /* Create the main pollset */ + rv = apr_pollset_create(&worker_pollset, num_listensocks, pruntime, + APR_POLLSET_NOCOPY); + if (rv != APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03285) + "Couldn't create pollset in thread;" + " check system or user limits"); + /* let the parent decide how bad this really is */ + clean_child_exit(APEXIT_CHILDSICK); + } + + for (lr = my_bucket->listeners; lr != NULL; lr = lr->next) { + apr_pollfd_t *pfd = apr_pcalloc(pruntime, sizeof *pfd); + + pfd->desc_type = APR_POLL_SOCKET; + pfd->desc.s = lr->sd; + pfd->reqevents = APR_POLLIN; + pfd->client_data = lr; + + rv = apr_pollset_add(worker_pollset, pfd); + if (rv != APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03286) + "Couldn't create add listener to pollset;" + " check system or user limits"); + /* let the parent decide how bad this really is */ + clean_child_exit(APEXIT_CHILDSICK); + } + + lr->accept_func = ap_unixd_accept; + } + worker_sockets = apr_pcalloc(pruntime, threads_per_child * sizeof(apr_socket_t *)); +} + +/* XXX under some circumstances not understood, children can get stuck + * in start_threads forever trying to take over slots which will + * never be cleaned up; for now there is an APLOG_DEBUG message issued + * every so often when this condition occurs + */ +static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) +{ + thread_starter *ts = dummy; + apr_thread_t **threads = ts->threads; + apr_threadattr_t *thread_attr = ts->threadattr; + int my_child_num = ts->child_num_arg; + proc_info *my_info; + apr_status_t rv; + int threads_created = 0; + int listener_started = 0; + int prev_threads_created; + int loops, i; loops = prev_threads_created = 0; while (1) { @@ -1156,7 +1162,10 @@ static void child_main(int child_num_arg, int child_bucket) requests_this_child = INT_MAX; } - /* Setup worker threads */ + /* Setup threads */ + + /* Globals used by signal_threads() so to be initialized before */ + setup_threads_runtime(); /* clear the storage; we may not create all our threads immediately, * and we want a 0 entry to indicate a thread which was not created