From: Jim Jagielski Date: Tue, 3 Jun 2014 15:46:48 +0000 (+0000) Subject: Update eventopt w/ bucket listeners X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=59094b605c6818d3c3f5feb501de58e54c4fc1bd;p=thirdparty%2Fapache%2Fhttpd.git Update eventopt w/ bucket listeners git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1599624 13f79535-47bb-0310-9956-ffa450edef68 --- diff --git a/server/mpm/eventopt/eventopt.c b/server/mpm/eventopt/eventopt.c index 93f34655f76..1a4a146210c 100644 --- a/server/mpm/eventopt/eventopt.c +++ b/server/mpm/eventopt/eventopt.c @@ -338,7 +338,7 @@ typedef struct event_retained_data { * doubled up to MAX_SPAWN_RATE, and reset only when a cycle goes by * without the need to spawn. */ - int idle_spawn_rate; + int *idle_spawn_rate; #ifndef MAX_SPAWN_RATE #define MAX_SPAWN_RATE (32) #endif @@ -348,7 +348,10 @@ static event_retained_data *retained; #define ID_FROM_CHILD_THREAD(c, t) ((c * thread_limit) + t) -static ap_pod_t *pod; +static ap_pod_t **pod; +static ap_pod_t *child_pod; +static ap_listen_rec *child_listen; +static int *bucket; /* bucket array for the httpd child processes */ /* The eventopt MPM respects a couple of runtime flags that can aid * in debugging. Setting the -DNO_DETACH flag will prevent the root process @@ -541,6 +544,9 @@ static int event_query(int query_code, int *result, apr_status_t *rv) case AP_MPMQ_GENERATION: *result = retained->my_generation; break; + case AP_MPMQ_CAN_SUSPEND: + *result = 1; + break; default: *rv = APR_ENOTIMPL; break; @@ -1244,7 +1250,7 @@ static apr_status_t init_pollset(apr_pool_t *p) TO_QUEUE_INIT(short_linger_q); listener_pollfd = apr_palloc(p, sizeof(apr_pollfd_t) * num_listensocks); - for (lr = ap_listeners; lr != NULL; lr = lr->next, i++) { + for (lr = child_listen; lr != NULL; lr = lr->next, i++) { apr_pollfd_t *pfd; AP_DEBUG_ASSERT(i < num_listensocks); pfd = &listener_pollfd[i]; @@ -2252,6 +2258,8 @@ static void child_main(int child_num_arg) apr_threadattr_t *thread_attr; apr_thread_t *start_thread_id; apr_pool_t *pskip; + int i; + ap_listen_rec *lr; mpm_state = AP_MPMQ_STARTING; /* for benefit of any hooks that run as this * child initializes @@ -2260,6 +2268,19 @@ static void child_main(int child_num_arg) ap_fatal_signal_child_setup(ap_server_conf); apr_pool_create(&pchild, pconf); + /* close unused listeners and pods */ + for (i = 0; i < num_buckets; i++) { + if (i != bucket[child_num_arg]) { + lr = mpm_listen[i]; + while(lr) { + apr_socket_close(lr->sd); + lr = lr->next; + } + mpm_listen[i]->active = 0; + ap_mpm_podx_close(pod[i]); + } + } + /*stuff to do before we switch id's, so we have permissions. */ ap_reopen_scoreboard(pchild, NULL, 0); @@ -2370,7 +2391,7 @@ static void child_main(int child_num_arg) apr_signal(SIGTERM, dummy_signal_handler); /* Watch for any messages from the parent over the POD */ while (1) { - rv = ap_mpm_podx_check(pod); + rv = ap_mpm_podx_check(child_pod); if (rv == AP_MPM_PODX_NORESTART) { /* see if termination was triggered while we slept */ switch (terminate_mode) { @@ -2423,6 +2444,9 @@ static int make_child(server_rec * s, int slot) /* NOTREACHED */ } + child_listen = mpm_listen[bucket[slot]]; + child_pod = pod[bucket[slot]]; + if ((pid = fork()) == -1) { ap_log_error(APLOG_MARK, APLOG_ERR, errno, s, APLOGNO(00481) "fork: Unable to fork new process"); @@ -2483,6 +2507,7 @@ static void startup_children(int number_to_start) if (ap_scoreboard_image->parent[i].pid != 0) { continue; } + bucket[i] = i % num_buckets; if (make_child(ap_server_conf, i) < 0) { break; } @@ -2490,7 +2515,7 @@ static void startup_children(int number_to_start) } } -static void perform_idle_server_maintenance(void) +static void perform_idle_server_maintenance(int child_bucket) { int i, j; int idle_thread_count; @@ -2520,7 +2545,7 @@ static void perform_idle_server_maintenance(void) int child_threads_active = 0; if (i >= retained->max_daemons_limit - && totally_free_length == retained->idle_spawn_rate) + && totally_free_length == retained->idle_spawn_rate[child_bucket]) /* short cut if all active processes have been examined and * enough empty scoreboard slots have been found */ @@ -2547,7 +2572,8 @@ static void perform_idle_server_maintenance(void) if (ps->pid != 0) { /* XXX just set all_dead_threads in outer for loop if no pid? not much else matters */ if (status <= SERVER_READY && !ps->quiescing && !ps->not_accepting - && ps->generation == retained->my_generation) + && ps->generation == retained->my_generation && + bucket[i] == child_bucket) { ++idle_thread_count; } @@ -2558,8 +2584,8 @@ static void perform_idle_server_maintenance(void) } active_thread_count += child_threads_active; if (any_dead_threads - && totally_free_length < retained->idle_spawn_rate - && free_length < MAX_SPAWN_RATE + && totally_free_length < retained->idle_spawn_rate[child_bucket] + && free_length < MAX_SPAWN_RATE/num_buckets && (!ps->pid /* no process in the slot */ || ps->quiescing)) { /* or at least one is going away */ if (all_dead_threads) { @@ -2614,12 +2640,12 @@ static void perform_idle_server_maintenance(void) retained->max_daemons_limit = last_non_dead + 1; - if (idle_thread_count > max_spare_threads) { + if (idle_thread_count > max_spare_threads/num_buckets) { /* Kill off one child */ - ap_mpm_podx_signal(pod, AP_MPM_PODX_GRACEFUL); - retained->idle_spawn_rate = 1; + ap_mpm_podx_signal(pod[child_bucket], AP_MPM_PODX_GRACEFUL); + retained->idle_spawn_rate[child_bucket] = 1; } - else if (idle_thread_count < min_spare_threads) { + else if (idle_thread_count < min_spare_threads/num_buckets) { /* terminate the free list */ if (free_length == 0) { /* scoreboard is full, can't fork */ @@ -2637,13 +2663,13 @@ static void perform_idle_server_maintenance(void) ap_log_error(APLOG_MARK, APLOG_ERR, 0, ap_server_conf, APLOGNO(00485) "scoreboard is full, not at MaxRequestWorkers"); } - retained->idle_spawn_rate = 1; + retained->idle_spawn_rate[child_bucket] = 1; } else { - if (free_length > retained->idle_spawn_rate) { - free_length = retained->idle_spawn_rate; + if (free_length > retained->idle_spawn_rate[child_bucket]) { + free_length = retained->idle_spawn_rate[child_bucket]; } - if (retained->idle_spawn_rate >= 8) { + if (retained->idle_spawn_rate[child_bucket] >= 8) { ap_log_error(APLOG_MARK, APLOG_INFO, 0, ap_server_conf, APLOGNO(00486) "server seems busy, (you may need " "to increase StartServers, ThreadsPerChild " @@ -2653,6 +2679,7 @@ static void perform_idle_server_maintenance(void) idle_thread_count, total_non_dead); } for (i = 0; i < free_length; ++i) { + bucket[free_slots[i]] = child_bucket; make_child(ap_server_conf, free_slots[i]); } /* the next time around we want to spawn twice as many if this @@ -2661,13 +2688,13 @@ static void perform_idle_server_maintenance(void) if (retained->hold_off_on_exponential_spawning) { --retained->hold_off_on_exponential_spawning; } - else if (retained->idle_spawn_rate < MAX_SPAWN_RATE) { - retained->idle_spawn_rate *= 2; + else if (retained->idle_spawn_rate[child_bucket] < MAX_SPAWN_RATE/num_buckets) { + retained->idle_spawn_rate[child_bucket] *= 2; } } } else { - retained->idle_spawn_rate = 1; + retained->idle_spawn_rate[child_bucket] = 1; } } @@ -2724,7 +2751,7 @@ static void server_main_loop(int remaining_children_to_start) ap_scoreboard_image->parent[child_slot].quiescing = 0; if (processed_status == APEXIT_CHILDSICK) { /* resource shortage, minimize the fork rate */ - retained->idle_spawn_rate = 1; + retained->idle_spawn_rate[bucket[child_slot]] = 1; } else if (remaining_children_to_start && child_slot < ap_daemons_limit) { @@ -2742,7 +2769,9 @@ static void server_main_loop(int remaining_children_to_start) if (processed_status == APEXIT_CHILDSICK && old_gen == retained->my_generation) { /* resource shortage, minimize the fork rate */ - retained->idle_spawn_rate = 1; + for (i = 0; i < num_buckets; i++) { + retained->idle_spawn_rate[i] = 1; + } } #if APR_HAS_OTHER_CHILD } @@ -2781,7 +2810,9 @@ static void server_main_loop(int remaining_children_to_start) continue; } - perform_idle_server_maintenance(); + for (i = 0; i < num_buckets; i++) { + perform_idle_server_maintenance(i); + } } } @@ -2789,6 +2820,8 @@ static int event_run(apr_pool_t * _pconf, apr_pool_t * plog, server_rec * s) { int remaining_children_to_start; + int i; + ap_log_pid(pconf, ap_pid_fname); if (!retained->is_graceful) { @@ -2802,11 +2835,13 @@ static int event_run(apr_pool_t * _pconf, apr_pool_t * plog, server_rec * s) ap_scoreboard_image->global->running_generation = retained->my_generation; } + bucket = apr_palloc(_pconf, sizeof(int) * ap_daemons_limit); + restart_pending = shutdown_pending = 0; set_signals(); /* Don't thrash... */ - if (max_spare_threads < min_spare_threads + threads_per_child) - max_spare_threads = min_spare_threads + threads_per_child; + if (max_spare_threads < min_spare_threads + threads_per_child * num_buckets) + max_spare_threads = min_spare_threads + threads_per_child * num_buckets; /* If we're doing a graceful_restart then we're going to see a lot * of children exiting immediately when we get into the main loop @@ -2848,7 +2883,9 @@ static int event_run(apr_pool_t * _pconf, apr_pool_t * plog, server_rec * s) /* Time to shut down: * Kill child processes, tell them to call child_exit, etc... */ - ap_mpm_podx_killpg(pod, ap_daemons_limit, AP_MPM_PODX_RESTART); + for (i = 0; i < num_buckets; i++) { + ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_RESTART); + } ap_reclaim_child_processes(1, /* Start with SIGTERM */ event_note_child_killed); @@ -2869,7 +2906,9 @@ static int event_run(apr_pool_t * _pconf, apr_pool_t * plog, server_rec * s) /* Close our listeners, and then ask our children to do same */ ap_close_listeners(); - ap_mpm_podx_killpg(pod, ap_daemons_limit, AP_MPM_PODX_GRACEFUL); + for (i = 0; i < num_buckets; i++) { + ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_GRACEFUL); + } ap_relieve_child_processes(event_note_child_killed); if (!child_fatal) { @@ -2909,7 +2948,9 @@ static int event_run(apr_pool_t * _pconf, apr_pool_t * plog, server_rec * s) * way, try and make sure that all of our processes are * really dead. */ - ap_mpm_podx_killpg(pod, ap_daemons_limit, AP_MPM_PODX_RESTART); + for (i = 0; i < num_buckets; i++) { + ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_RESTART); + } ap_reclaim_child_processes(1, event_note_child_killed); return DONE; @@ -2935,8 +2976,9 @@ static int event_run(apr_pool_t * _pconf, apr_pool_t * plog, server_rec * s) AP_SIG_GRACEFUL_STRING " received. Doing graceful restart"); /* wake up the children...time to die. But we'll have more soon */ - ap_mpm_podx_killpg(pod, ap_daemons_limit, AP_MPM_PODX_GRACEFUL); - + for (i = 0; i < num_buckets; i++) { + ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_GRACEFUL); + } /* This is mostly for debugging... so that we know what is still * gracefully dealing with existing request. @@ -2948,7 +2990,9 @@ static int event_run(apr_pool_t * _pconf, apr_pool_t * plog, server_rec * s) * and a SIGHUP, we may as well use the same signal, because some user * pthreads are stealing signals from us left and right. */ - ap_mpm_podx_killpg(pod, ap_daemons_limit, AP_MPM_PODX_RESTART); + for (i = 0; i < num_buckets; i++) { + ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_RESTART); + } ap_reclaim_child_processes(1, /* Start with SIGTERM */ event_note_child_killed); @@ -2968,6 +3012,8 @@ static int event_open_logs(apr_pool_t * p, apr_pool_t * plog, int startup = 0; int level_flags = 0; apr_status_t rv; + int i; + int num_of_cores = 0; pconf = p; @@ -2977,6 +3023,8 @@ static int event_open_logs(apr_pool_t * p, apr_pool_t * plog, level_flags |= APLOG_STARTUP; } + enable_default_listener = 0; + if ((num_listensocks = ap_setup_listeners(ap_server_conf)) < 1) { ap_log_error(APLOG_MARK, APLOG_ALERT | level_flags, 0, (startup ? NULL : s), @@ -2984,12 +3032,36 @@ static int event_open_logs(apr_pool_t * p, apr_pool_t * plog, return DONE; } + enable_default_listener = 1; + if (have_so_reuseport) { +#ifdef _SC_NPROCESSORS_ONLN + num_of_cores = sysconf(_SC_NPROCESSORS_ONLN); +#else + num_of_cores = 1; +#endif + if (num_of_cores > 8) { + num_buckets = num_of_cores/8; + } + else { + num_buckets = 1; + } + } + else { + num_buckets = 1; + } + + ap_duplicate_listeners(ap_server_conf, pconf, num_buckets); + + pod = apr_palloc(pconf, sizeof(ap_pod_t *) * num_buckets); + if (!one_process) { - if ((rv = ap_mpm_podx_open(pconf, &pod))) { - ap_log_error(APLOG_MARK, APLOG_CRIT | level_flags, rv, - (startup ? NULL : s), - "could not open pipe-of-death"); - return DONE; + for (i = 0; i < num_buckets; i++) { + if ((rv = ap_mpm_podx_open(pconf, &pod[i]))) { + ap_log_error(APLOG_MARK, APLOG_CRIT | level_flags, rv, + (startup ? NULL : s), + "could not open pipe-of-death"); + return DONE; + } } } /* for skiplist */ @@ -3003,6 +3075,7 @@ static int event_pre_config(apr_pool_t * pconf, apr_pool_t * plog, int no_detach, debug, foreground; apr_status_t rv; const char *userdata_key = "mpm_eventopt_module"; + int i; mpm_state = AP_MPMQ_STARTING; @@ -3023,7 +3096,6 @@ static int event_pre_config(apr_pool_t * pconf, apr_pool_t * plog, if (!retained) { retained = ap_retained_data_create(userdata_key, sizeof(*retained)); retained->max_daemons_limit = -1; - retained->idle_spawn_rate = 1; } ++retained->module_loads; if (retained->module_loads == 2) { @@ -3037,6 +3109,10 @@ static int event_pre_config(apr_pool_t * pconf, apr_pool_t * plog, "atomics not working as expected - add32 of negative number"); return HTTP_INTERNAL_SERVER_ERROR; } + retained->idle_spawn_rate = apr_palloc(pconf, sizeof(int) * num_buckets); + for (i = 0; i< num_buckets; i++) { + retained->idle_spawn_rate[i] = 1; + } rv = apr_pollset_create(&event_pollset, 1, plog, APR_POLLSET_WAKEABLE|APR_POLLSET_NOCOPY); if (rv != APR_SUCCESS) {