From: Yann Ylavic Date: Tue, 7 Oct 2014 15:16:02 +0000 (+0000) Subject: core: Use process scoreboard to store each child's listener bucket, X-Git-Tag: 2.5.0-alpha~3796 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=a527db4331ac8ffa95858ce79e7d735e14fbbbb6;p=apache core: Use process scoreboard to store each child's listener bucket, and silently adjust the configured number of processes/threads to be above the computed number of listener buckets (depending on the CPU cores). git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1629909 13f79535-47bb-0310-9956-ffa450edef68 --- diff --git a/include/ap_mmn.h b/include/ap_mmn.h index 3744cb8bee..eb2efc8cbc 100644 --- a/include/ap_mmn.h +++ b/include/ap_mmn.h @@ -469,6 +469,7 @@ * 20140627.5 (2.5.0-dev) Add r->trailers_{in,out} * 20140627.6 (2.5.0-dev) Added ap_pcre_version_string(), AP_REG_PCRE_COMPILED * and AP_REG_PCRE_LOADED to ap_regex.h. + * 20140627.7 (2.5.0-dev) Add listener bucket in scoreboard.h's process_score. */ #define MODULE_MAGIC_COOKIE 0x41503235UL /* "AP25" */ @@ -476,7 +477,7 @@ #ifndef MODULE_MAGIC_NUMBER_MAJOR #define MODULE_MAGIC_NUMBER_MAJOR 20140627 #endif -#define MODULE_MAGIC_NUMBER_MINOR 6 /* 0...n */ +#define MODULE_MAGIC_NUMBER_MINOR 7 /* 0...n */ /** * Determine if the server's current MODULE_MAGIC_NUMBER is at least a diff --git a/include/scoreboard.h b/include/scoreboard.h index c41d7d3f0b..c07b28cb89 100644 --- a/include/scoreboard.h +++ b/include/scoreboard.h @@ -142,6 +142,7 @@ struct process_score { apr_uint32_t lingering_close; /* async connections in lingering close */ apr_uint32_t keep_alive; /* async connections in keep alive */ apr_uint32_t suspended; /* connections suspended by some module */ + int bucket; /* Listener bucket used by this child */ }; /* Scoreboard is now in 'local' memory, since it isn't updated once created, diff --git a/server/mpm/event/event.c b/server/mpm/event/event.c index ccd9eb3c66..d5e40c3916 100644 --- a/server/mpm/event/event.c +++ b/server/mpm/event/event.c @@ -366,7 +366,6 @@ static event_retained_data *retained; static ap_pod_t **pod; static ap_pod_t *child_pod; static ap_listen_rec *child_listen; -static int *bucket; /* bucket array for the httpd child processes */ /* The event MPM respects a couple of runtime flags that can aid * in debugging. Setting the -DNO_DETACH flag will prevent the root process @@ -2410,7 +2409,7 @@ static void join_start_thread(apr_thread_t * start_thread_id) } } -static void child_main(int child_num_arg) +static void child_main(int child_num_arg, int child_bucket) { apr_thread_t **threads; apr_status_t rv; @@ -2428,9 +2427,12 @@ static void child_main(int child_num_arg) ap_fatal_signal_child_setup(ap_server_conf); apr_pool_create(&pchild, pconf); + child_listen = mpm_listen[child_bucket]; + child_pod = pod[child_bucket]; + /* close unused listeners and pods */ for (i = 0; i < num_buckets; i++) { - if (i != bucket[child_num_arg]) { + if (i != child_bucket) { lr = mpm_listen[i]; while(lr) { apr_socket_close(lr->sd); @@ -2589,7 +2591,7 @@ static void child_main(int child_num_arg) clean_child_exit(resource_shortage ? APEXIT_CHILDSICK : 0); } -static int make_child(server_rec * s, int slot) +static int make_child(server_rec * s, int slot, int bucket) { int pid; @@ -2597,13 +2599,10 @@ static int make_child(server_rec * s, int slot) retained->max_daemons_limit = slot + 1; } - child_listen = mpm_listen[bucket[slot]]; - child_pod = pod[bucket[slot]]; - if (one_process) { set_signals(); event_note_child_started(slot, getpid()); - child_main(slot); + child_main(0, 0); /* NOTREACHED */ } @@ -2641,7 +2640,7 @@ static int make_child(server_rec * s, int slot) RAISE_SIGSTOP(MAKE_CHILD); apr_signal(SIGTERM, just_die); - child_main(slot); + child_main(slot, bucket); /* NOTREACHED */ } /* else */ @@ -2654,6 +2653,7 @@ static int make_child(server_rec * s, int slot) } ap_scoreboard_image->parent[slot].quiescing = 0; ap_scoreboard_image->parent[slot].not_accepting = 0; + ap_scoreboard_image->parent[slot].bucket = bucket; event_note_child_started(slot, pid); return 0; } @@ -2667,8 +2667,7 @@ static void startup_children(int number_to_start) if (ap_scoreboard_image->parent[i].pid != 0) { continue; } - bucket[i] = i % num_buckets; - if (make_child(ap_server_conf, i) < 0) { + if (make_child(ap_server_conf, i, i % num_buckets) < 0) { break; } --number_to_start; @@ -2732,8 +2731,8 @@ static void perform_idle_server_maintenance(int child_bucket) if (ps->pid != 0) { /* XXX just set all_dead_threads in outer for loop if no pid? not much else matters */ if (status <= SERVER_READY && !ps->quiescing && !ps->not_accepting - && ps->generation == retained->my_generation && - bucket[i] == child_bucket) + && ps->generation == retained->my_generation + && ps->bucket == child_bucket) { ++idle_thread_count; } @@ -2745,7 +2744,7 @@ static void perform_idle_server_maintenance(int child_bucket) active_thread_count += child_threads_active; if (any_dead_threads && totally_free_length < retained->idle_spawn_rate[child_bucket] - && free_length < MAX_SPAWN_RATE/num_buckets + && free_length < MAX_SPAWN_RATE / num_buckets && (!ps->pid /* no process in the slot */ || ps->quiescing)) { /* or at least one is going away */ if (all_dead_threads) { @@ -2801,12 +2800,12 @@ static void perform_idle_server_maintenance(int child_bucket) retained->max_daemons_limit = last_non_dead + 1; - if (idle_thread_count > max_spare_threads/num_buckets) { + if (idle_thread_count > max_spare_threads / num_buckets) { /* Kill off one child */ ap_mpm_podx_signal(pod[child_bucket], AP_MPM_PODX_GRACEFUL); retained->idle_spawn_rate[child_bucket] = 1; } - else if (idle_thread_count < min_spare_threads/num_buckets) { + else if (idle_thread_count < min_spare_threads) { /* terminate the free list */ if (free_length == 0) { /* scoreboard is full, can't fork */ @@ -2840,8 +2839,7 @@ static void perform_idle_server_maintenance(int child_bucket) idle_thread_count, total_non_dead); } for (i = 0; i < free_length; ++i) { - bucket[free_slots[i]] = child_bucket; - make_child(ap_server_conf, free_slots[i]); + make_child(ap_server_conf, free_slots[i], child_bucket); } /* the next time around we want to spawn twice as many if this * wasn't good enough, but not if we've just done a graceful @@ -2849,7 +2847,8 @@ static void perform_idle_server_maintenance(int child_bucket) if (retained->hold_off_on_exponential_spawning) { --retained->hold_off_on_exponential_spawning; } - else if (retained->idle_spawn_rate[child_bucket] < MAX_SPAWN_RATE/num_buckets) { + else if (retained->idle_spawn_rate[child_bucket] < + MAX_SPAWN_RATE / num_buckets) { retained->idle_spawn_rate[child_bucket] *= 2; } } @@ -2903,23 +2902,26 @@ static void server_main_loop(int remaining_children_to_start) } /* non-fatal death... note that it's gone in the scoreboard. */ if (child_slot >= 0) { + process_score *ps; + for (i = 0; i < threads_per_child; i++) ap_update_child_status_from_indexes(child_slot, i, SERVER_DEAD, (request_rec *) NULL); event_note_child_killed(child_slot, 0, 0); - ap_scoreboard_image->parent[child_slot].quiescing = 0; + ps = &ap_scoreboard_image->parent[child_slot]; + ps->quiescing = 0; if (processed_status == APEXIT_CHILDSICK) { /* resource shortage, minimize the fork rate */ - retained->idle_spawn_rate[bucket[child_slot]] = 1; + retained->idle_spawn_rate[ps->bucket] = 1; } else if (remaining_children_to_start && child_slot < ap_daemons_limit) { /* we're still doing a 1-for-1 replacement of dead * children with new children */ - make_child(ap_server_conf, child_slot); + make_child(ap_server_conf, child_slot, ps->bucket); --remaining_children_to_start; } } @@ -2996,8 +2998,6 @@ static int event_run(apr_pool_t * _pconf, apr_pool_t * plog, server_rec * s) ap_scoreboard_image->global->running_generation = retained->my_generation; } - bucket = apr_palloc(_pconf, sizeof(int) * ap_daemons_limit); - restart_pending = shutdown_pending = 0; set_signals(); /* Don't thrash... */ @@ -3174,7 +3174,6 @@ static int event_open_logs(apr_pool_t * p, apr_pool_t * plog, int level_flags = 0; apr_status_t rv; int i; - int num_of_cores = 0; pconf = p; @@ -3185,7 +3184,6 @@ static int event_open_logs(apr_pool_t * p, apr_pool_t * plog, } enable_default_listener = 0; - if ((num_listensocks = ap_setup_listeners(ap_server_conf)) < 1) { ap_log_error(APLOG_MARK, APLOG_ALERT | level_flags, 0, (startup ? NULL : s), @@ -3194,22 +3192,6 @@ static int event_open_logs(apr_pool_t * p, apr_pool_t * plog, } enable_default_listener = 1; - if (have_so_reuseport) { -#ifdef _SC_NPROCESSORS_ONLN - num_of_cores = sysconf(_SC_NPROCESSORS_ONLN); -#else - num_of_cores = 1; -#endif - if (num_of_cores > 8) { - num_buckets = num_of_cores/8; - } - else { - num_buckets = 1; - } - } - else { - num_buckets = 1; - } ap_duplicate_listeners(ap_server_conf, pconf, num_buckets); @@ -3258,6 +3240,17 @@ static int event_pre_config(apr_pool_t * pconf, apr_pool_t * plog, retained = ap_retained_data_create(userdata_key, sizeof(*retained)); retained->max_daemons_limit = -1; } + if (!retained->is_graceful) { + num_buckets = 1; +#ifdef _SC_NPROCESSORS_ONLN + if (have_so_reuseport) { + int num_online_cores = sysconf(_SC_NPROCESSORS_ONLN); + if (num_online_cores > 8) { + num_buckets = num_online_cores / 8; + } + } +#endif + } ++retained->module_loads; if (retained->module_loads == 2) { /* test for correct operation of fdqueue */ @@ -3511,9 +3504,15 @@ static int event_check_config(apr_pool_t *p, apr_pool_t *plog, } ap_daemons_limit = server_limit; } + else if (ap_daemons_limit < num_buckets) { + /* Don't thrash since num_buckets depends on + * the system and the number of CPU cores. + */ + ap_daemons_limit = num_buckets; + } /* ap_daemons_to_start > ap_daemons_limit checked in ap_mpm_run() */ - if (ap_daemons_to_start < 0) { + if (ap_daemons_to_start < 1) { if (startup) { ap_log_error(APLOG_MARK, APLOG_WARNING | APLOG_STARTUP, 0, NULL, APLOGNO(00517) "WARNING: StartServers of %d not allowed, " @@ -3525,6 +3524,12 @@ static int event_check_config(apr_pool_t *p, apr_pool_t *plog, } ap_daemons_to_start = 1; } + if (ap_daemons_to_start < num_buckets) { + /* Don't thrash since num_buckets depends on + * the system and the number of CPU cores. + */ + ap_daemons_to_start = num_buckets; + } if (min_spare_threads < 1) { if (startup) { @@ -3542,6 +3547,12 @@ static int event_check_config(apr_pool_t *p, apr_pool_t *plog, } min_spare_threads = 1; } + if (min_spare_threads < num_buckets) { + /* Don't thrash since num_buckets depends on + * the system and the number of CPU cores. + */ + min_spare_threads = num_buckets; + } /* max_spare_threads < min_spare_threads + threads_per_child * checked in ap_mpm_run() diff --git a/server/mpm/eventopt/eventopt.c b/server/mpm/eventopt/eventopt.c index dbdc315039..f1089257bc 100644 --- a/server/mpm/eventopt/eventopt.c +++ b/server/mpm/eventopt/eventopt.c @@ -351,7 +351,6 @@ static event_retained_data *retained; static ap_pod_t **pod; static ap_pod_t *child_pod; static ap_listen_rec *child_listen; -static int *bucket; /* bucket array for the httpd child processes */ /* The eventopt MPM respects a couple of runtime flags that can aid * in debugging. Setting the -DNO_DETACH flag will prevent the root process @@ -2232,7 +2231,7 @@ static void join_start_thread(apr_thread_t * start_thread_id) } } -static void child_main(int child_num_arg) +static void child_main(int child_num_arg, int child_bucket) { apr_thread_t **threads; apr_status_t rv; @@ -2250,9 +2249,12 @@ static void child_main(int child_num_arg) ap_fatal_signal_child_setup(ap_server_conf); apr_pool_create(&pchild, pconf); + child_listen = mpm_listen[child_bucket]; + child_pod = pod[child_bucket]; + /* close unused listeners and pods */ for (i = 0; i < num_buckets; i++) { - if (i != bucket[child_num_arg]) { + if (i != child_bucket) { lr = mpm_listen[i]; while(lr) { apr_socket_close(lr->sd); @@ -2411,7 +2413,7 @@ static void child_main(int child_num_arg) clean_child_exit(resource_shortage ? APEXIT_CHILDSICK : 0); } -static int make_child(server_rec * s, int slot) +static int make_child(server_rec * s, int slot, int bucket) { int pid; @@ -2419,13 +2421,10 @@ static int make_child(server_rec * s, int slot) retained->max_daemons_limit = slot + 1; } - child_listen = mpm_listen[bucket[slot]]; - child_pod = pod[bucket[slot]]; - if (one_process) { set_signals(); event_note_child_started(slot, getpid()); - child_main(slot); + child_main(0, 0); /* NOTREACHED */ } @@ -2463,7 +2462,7 @@ static int make_child(server_rec * s, int slot) RAISE_SIGSTOP(MAKE_CHILD); apr_signal(SIGTERM, just_die); - child_main(slot); + child_main(slot, bucket); /* NOTREACHED */ } /* else */ @@ -2476,6 +2475,7 @@ static int make_child(server_rec * s, int slot) } ap_scoreboard_image->parent[slot].quiescing = 0; ap_scoreboard_image->parent[slot].not_accepting = 0; + ap_scoreboard_image->parent[slot].bucket = bucket; event_note_child_started(slot, pid); return 0; } @@ -2489,8 +2489,7 @@ static void startup_children(int number_to_start) if (ap_scoreboard_image->parent[i].pid != 0) { continue; } - bucket[i] = i % num_buckets; - if (make_child(ap_server_conf, i) < 0) { + if (make_child(ap_server_conf, i, i % num_buckets) < 0) { break; } --number_to_start; @@ -2554,8 +2553,8 @@ static void perform_idle_server_maintenance(int child_bucket) if (ps->pid != 0) { /* XXX just set all_dead_threads in outer for loop if no pid? not much else matters */ if (status <= SERVER_READY && !ps->quiescing && !ps->not_accepting - && ps->generation == retained->my_generation && - bucket[i] == child_bucket) + && ps->generation == retained->my_generation + && ps->bucket == child_bucket) { ++idle_thread_count; } @@ -2567,7 +2566,7 @@ static void perform_idle_server_maintenance(int child_bucket) active_thread_count += child_threads_active; if (any_dead_threads && totally_free_length < retained->idle_spawn_rate[child_bucket] - && free_length < MAX_SPAWN_RATE/num_buckets + && free_length < MAX_SPAWN_RATE / num_buckets && (!ps->pid /* no process in the slot */ || ps->quiescing)) { /* or at least one is going away */ if (all_dead_threads) { @@ -2622,12 +2621,12 @@ static void perform_idle_server_maintenance(int child_bucket) retained->max_daemons_limit = last_non_dead + 1; - if (idle_thread_count > max_spare_threads/num_buckets) { + if (idle_thread_count > max_spare_threads / num_buckets) { /* Kill off one child */ ap_mpm_podx_signal(pod[child_bucket], AP_MPM_PODX_GRACEFUL); retained->idle_spawn_rate[child_bucket] = 1; } - else if (idle_thread_count < min_spare_threads/num_buckets) { + else if (idle_thread_count < min_spare_threads) { /* terminate the free list */ if (free_length == 0) { /* scoreboard is full, can't fork */ @@ -2661,8 +2660,7 @@ static void perform_idle_server_maintenance(int child_bucket) idle_thread_count, total_non_dead); } for (i = 0; i < free_length; ++i) { - bucket[free_slots[i]] = child_bucket; - make_child(ap_server_conf, free_slots[i]); + make_child(ap_server_conf, free_slots[i], child_bucket); } /* the next time around we want to spawn twice as many if this * wasn't good enough, but not if we've just done a graceful @@ -2670,7 +2668,8 @@ static void perform_idle_server_maintenance(int child_bucket) if (retained->hold_off_on_exponential_spawning) { --retained->hold_off_on_exponential_spawning; } - else if (retained->idle_spawn_rate[child_bucket] < MAX_SPAWN_RATE/num_buckets) { + else if (retained->idle_spawn_rate[child_bucket] + < MAX_SPAWN_RATE / num_buckets) { retained->idle_spawn_rate[child_bucket] *= 2; } } @@ -2724,23 +2723,26 @@ static void server_main_loop(int remaining_children_to_start) } /* non-fatal death... note that it's gone in the scoreboard. */ if (child_slot >= 0) { + process_score *ps; + for (i = 0; i < threads_per_child; i++) ap_update_child_status_from_indexes(child_slot, i, SERVER_DEAD, (request_rec *) NULL); event_note_child_killed(child_slot, 0, 0); - ap_scoreboard_image->parent[child_slot].quiescing = 0; + ps = &ap_scoreboard_image->parent[child_slot]; + ps->quiescing = 0; if (processed_status == APEXIT_CHILDSICK) { /* resource shortage, minimize the fork rate */ - retained->idle_spawn_rate[bucket[child_slot]] = 1; + retained->idle_spawn_rate[ps->bucket] = 1; } else if (remaining_children_to_start && child_slot < ap_daemons_limit) { /* we're still doing a 1-for-1 replacement of dead * children with new children */ - make_child(ap_server_conf, child_slot); + make_child(ap_server_conf, child_slot, ps->bucket); --remaining_children_to_start; } } @@ -2817,8 +2819,6 @@ static int event_run(apr_pool_t * _pconf, apr_pool_t * plog, server_rec * s) ap_scoreboard_image->global->running_generation = retained->my_generation; } - bucket = apr_palloc(_pconf, sizeof(int) * ap_daemons_limit); - restart_pending = shutdown_pending = 0; set_signals(); /* Don't thrash... */ @@ -2995,7 +2995,6 @@ static int event_open_logs(apr_pool_t * p, apr_pool_t * plog, int level_flags = 0; apr_status_t rv; int i; - int num_of_cores = 0; pconf = p; @@ -3015,23 +3014,6 @@ static int event_open_logs(apr_pool_t * p, apr_pool_t * plog, } enable_default_listener = 1; - if (have_so_reuseport) { -#ifdef _SC_NPROCESSORS_ONLN - num_of_cores = sysconf(_SC_NPROCESSORS_ONLN); -#else - num_of_cores = 1; -#endif - if (num_of_cores > 8) { - num_buckets = num_of_cores/8; - } - else { - num_buckets = 1; - } - } - else { - num_buckets = 1; - } - ap_duplicate_listeners(ap_server_conf, pconf, num_buckets); pod = apr_palloc(pconf, sizeof(ap_pod_t *) * num_buckets); @@ -3079,6 +3061,17 @@ static int event_pre_config(apr_pool_t * pconf, apr_pool_t * plog, retained = ap_retained_data_create(userdata_key, sizeof(*retained)); retained->max_daemons_limit = -1; } + if (!retained->is_graceful) { + num_buckets = 1; +#ifdef _SC_NPROCESSORS_ONLN + if (have_so_reuseport) { + int num_online_cores = sysconf(_SC_NPROCESSORS_ONLN); + if (num_online_cores > 8) { + num_buckets = num_online_cores / 8; + } + } +#endif + } ++retained->module_loads; if (retained->module_loads == 2) { /* test for correct operation of fdqueue */ @@ -3330,9 +3323,15 @@ static int event_check_config(apr_pool_t *p, apr_pool_t *plog, } ap_daemons_limit = server_limit; } + else if (ap_daemons_limit < num_buckets) { + /* Don't thrash since num_buckets depends on + * the system and the number of CPU cores. + */ + ap_daemons_limit = num_buckets; + } /* ap_daemons_to_start > ap_daemons_limit checked in ap_mpm_run() */ - if (ap_daemons_to_start < 0) { + if (ap_daemons_to_start < 1) { if (startup) { ap_log_error(APLOG_MARK, APLOG_WARNING | APLOG_STARTUP, 0, NULL, APLOGNO(00517) "WARNING: StartServers of %d not allowed, " @@ -3344,6 +3343,12 @@ static int event_check_config(apr_pool_t *p, apr_pool_t *plog, } ap_daemons_to_start = 1; } + if (ap_daemons_to_start < num_buckets) { + /* Don't thrash since num_buckets depends on + * the system and the number of CPU cores. + */ + ap_daemons_to_start = num_buckets; + } if (min_spare_threads < 1) { if (startup) { @@ -3361,6 +3366,12 @@ static int event_check_config(apr_pool_t *p, apr_pool_t *plog, } min_spare_threads = 1; } + if (min_spare_threads < num_buckets) { + /* Don't thrash since num_buckets depends on + * the system and the number of CPU cores. + */ + min_spare_threads = num_buckets; + } /* max_spare_threads < min_spare_threads + threads_per_child * checked in ap_mpm_run() diff --git a/server/mpm/prefork/prefork.c b/server/mpm/prefork/prefork.c index d6c038bcee..318d5de0f3 100644 --- a/server/mpm/prefork/prefork.c +++ b/server/mpm/prefork/prefork.c @@ -100,7 +100,6 @@ static ap_pod_t **pod; static ap_pod_t *child_pod; static apr_proc_mutex_t *child_mutex; static ap_listen_rec *child_listen; -static int *bucket; /* bucket array for the httpd child processes */ /* data retained by prefork across load/unload of the module @@ -488,7 +487,7 @@ static void set_signals(void) static int requests_this_child; static int num_listensocks = 0; -static void child_main(int child_num_arg) +static void child_main(int child_num_arg, int child_bucket) { #if APR_HAS_THREADS apr_thread_t *thd = NULL; @@ -513,6 +512,10 @@ static void child_main(int child_num_arg) ap_my_pid = getpid(); requests_this_child = 0; + child_listen = mpm_listen[child_bucket]; + child_mutex = accept_mutex[child_bucket]; + child_pod = pod[child_bucket]; + ap_fatal_signal_child_setup(ap_server_conf); /* Get a sub context for global allocations in this child, so that @@ -534,7 +537,7 @@ static void child_main(int child_num_arg) /* close unused listeners and pods */ for (i = 0; i < num_buckets; i++) { - if (i != bucket[my_child_num]) { + if (i != child_bucket) { lr = mpm_listen[i]; while(lr) { apr_socket_close(lr->sd); @@ -750,7 +753,7 @@ static void child_main(int child_num_arg) } -static int make_child(server_rec *s, int slot) +static int make_child(server_rec *s, int slot, int bucket) { int pid; @@ -758,10 +761,6 @@ static int make_child(server_rec *s, int slot) retained->max_daemons_limit = slot + 1; } - child_listen = mpm_listen[bucket[slot]]; - child_mutex = accept_mutex[bucket[slot]]; - child_pod = pod[bucket[slot]]; - if (one_process) { apr_signal(SIGHUP, sig_term); /* Don't catch AP_SIG_GRACEFUL in ONE_PROCESS mode :) */ @@ -771,7 +770,7 @@ static int make_child(server_rec *s, int slot) #endif apr_signal(SIGTERM, sig_term); prefork_note_child_started(slot, getpid()); - child_main(slot); + child_main(slot, bucket); /* NOTREACHED */ } @@ -831,9 +830,10 @@ static int make_child(server_rec *s, int slot) * The pod is used for signalling the graceful restart. */ apr_signal(AP_SIG_GRACEFUL, stop_listening); - child_main(slot); + child_main(slot, bucket); } + ap_scoreboard_image->parent[slot].bucket = bucket; prefork_note_child_started(slot, pid); return 0; @@ -849,8 +849,7 @@ static void startup_children(int number_to_start) if (ap_scoreboard_image->servers[i][0].status != SERVER_DEAD) { continue; } - bucket[i] = i % num_buckets; - if (make_child(ap_server_conf, i) < 0) { + if (make_child(ap_server_conf, i, i % num_buckets) < 0) { break; } --number_to_start; @@ -937,8 +936,9 @@ static void perform_idle_server_maintenance(apr_pool_t *p) idle_count, total_non_dead); } for (i = 0; i < free_length; ++i) { - bucket[free_slots[i]]= (++bucket_make_child_record) % num_buckets; - make_child(ap_server_conf, free_slots[i]); + bucket_make_child_record++; + make_child(ap_server_conf, free_slots[i], + bucket_make_child_record % num_buckets); } /* the next time around we want to spawn twice as many if this * wasn't good enough, but not if we've just done a graceful @@ -969,7 +969,6 @@ static int prefork_run(apr_pool_t *_pconf, apr_pool_t *plog, server_rec *s) ap_log_pid(pconf, ap_pid_fname); - bucket = apr_palloc(_pconf, sizeof(int) * ap_daemons_limit); /* Initialize cross-process accept lock for each bucket*/ accept_mutex = apr_palloc(_pconf, sizeof(apr_proc_mutex_t *) * num_buckets); for (i = 0; i < num_buckets; i++) { @@ -997,12 +996,14 @@ static int prefork_run(apr_pool_t *_pconf, apr_pool_t *plog, server_rec *s) if (one_process) { AP_MONCONTROL(1); - bucket[0] = 0; - make_child(ap_server_conf, 0); + make_child(ap_server_conf, 0, 0); /* NOTREACHED */ + ap_assert(0); + return DONE; } - else { - if (ap_daemons_max_free < ap_daemons_min_free + num_buckets) /* Don't thrash... */ + + /* Don't thrash... */ + if (ap_daemons_max_free < ap_daemons_min_free + num_buckets) ap_daemons_max_free = ap_daemons_min_free + num_buckets; /* If we're doing a graceful_restart then we're going to see a lot @@ -1094,7 +1095,8 @@ static int prefork_run(apr_pool_t *_pconf, apr_pool_t *plog, server_rec *s) /* we're still doing a 1-for-1 replacement of dead * children with new children */ - make_child(ap_server_conf, child_slot); + make_child(ap_server_conf, child_slot, + ap_get_scoreboard_process(child_slot)->bucket); --remaining_children_to_start; } #if APR_HAS_OTHER_CHILD @@ -1135,7 +1137,6 @@ static int prefork_run(apr_pool_t *_pconf, apr_pool_t *plog, server_rec *s) perform_idle_server_maintenance(pconf); } - } /* one_process */ mpm_state = AP_MPMQ_STOPPING; @@ -1290,7 +1291,6 @@ static int prefork_open_logs(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp, int level_flags = 0; apr_status_t rv; int i; - int num_of_cores = 0; pconf = p; @@ -1309,23 +1309,6 @@ static int prefork_open_logs(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp, } enable_default_listener = 1; - if (have_so_reuseport) { -#ifdef _SC_NPROCESSORS_ONLN - num_of_cores = sysconf(_SC_NPROCESSORS_ONLN); -#else - num_of_cores = 1; -#endif - if (num_of_cores > 8) { - num_buckets = num_of_cores/8; - } - else { - num_buckets = 1; - } - } - else { - num_buckets = 1; - } - ap_duplicate_listeners(ap_server_conf, pconf, num_buckets); pod = apr_palloc(pconf, sizeof(ap_pod_t *) * num_buckets); @@ -1371,6 +1354,17 @@ static int prefork_pre_config(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp retained->max_daemons_limit = -1; retained->idle_spawn_rate = 1; } + if (!retained->is_graceful) { + num_buckets = 1; +#ifdef _SC_NPROCESSORS_ONLN + if (have_so_reuseport) { + int num_online_cores = sysconf(_SC_NPROCESSORS_ONLN); + if (num_online_cores > 8) { + num_buckets = num_online_cores / 8; + } + } +#endif + } ++retained->module_loads; if (retained->module_loads == 2) { if (!one_process && !foreground) { @@ -1484,6 +1478,12 @@ static int prefork_check_config(apr_pool_t *p, apr_pool_t *plog, } ap_daemons_limit = 1; } + if (ap_daemons_limit < num_buckets) { + /* Don't thrash since num_buckets depends on + * the system and the number of CPU cores. + */ + ap_daemons_limit = num_buckets; + } /* ap_daemons_to_start > ap_daemons_limit checked in prefork_run() */ if (ap_daemons_to_start < 0) { @@ -1498,6 +1498,12 @@ static int prefork_check_config(apr_pool_t *p, apr_pool_t *plog, } ap_daemons_to_start = 1; } + if (ap_daemons_to_start < num_buckets) { + /* Don't thrash since num_buckets depends on + * the system and the number of CPU cores. + */ + ap_daemons_to_start = num_buckets; + } if (ap_daemons_min_free < 1) { if (startup) { @@ -1515,6 +1521,12 @@ static int prefork_check_config(apr_pool_t *p, apr_pool_t *plog, } ap_daemons_min_free = 1; } + if (ap_daemons_min_free < num_buckets) { + /* Don't thrash since num_buckets depends on + * the system and the number of CPU cores. + */ + ap_daemons_min_free = num_buckets; + } /* ap_daemons_max_free < ap_daemons_min_free + 1 checked in prefork_run() */ diff --git a/server/mpm/worker/worker.c b/server/mpm/worker/worker.c index 35d5e04def..f2c235efc6 100644 --- a/server/mpm/worker/worker.c +++ b/server/mpm/worker/worker.c @@ -226,7 +226,6 @@ static apr_os_thread_t *listener_os_thread; static apr_proc_mutex_t **accept_mutex; static apr_proc_mutex_t *child_mutex; static ap_listen_rec *child_listen; -static int *bucket; /* bucket array for the httpd child processes */ #ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT #define SAFE_ACCEPT(stmt) (child_listen->next ? (stmt) : APR_SUCCESS) @@ -1218,7 +1217,7 @@ static void join_start_thread(apr_thread_t *start_thread_id) } } -static void child_main(int child_num_arg) +static void child_main(int child_num_arg, int child_bucket) { apr_thread_t **threads; apr_status_t rv; @@ -1235,9 +1234,13 @@ static void child_main(int child_num_arg) ap_fatal_signal_child_setup(ap_server_conf); apr_pool_create(&pchild, pconf); + child_listen = mpm_listen[child_bucket]; + child_mutex = accept_mutex[child_bucket]; + child_pod = pod[child_bucket]; + /* close unused listeners and pods */ for (i = 0; i < num_buckets; i++) { - if (i != bucket[child_num_arg]) { + if (i != child_bucket) { lr = mpm_listen[i]; while(lr) { apr_socket_close(lr->sd); @@ -1399,7 +1402,7 @@ static void child_main(int child_num_arg) clean_child_exit(resource_shortage ? APEXIT_CHILDSICK : 0); } -static int make_child(server_rec *s, int slot) +static int make_child(server_rec *s, int slot, int bucket) { int pid; @@ -1407,14 +1410,10 @@ static int make_child(server_rec *s, int slot) retained->max_daemons_limit = slot + 1; } - child_listen = mpm_listen[bucket[slot]]; - child_mutex = accept_mutex[bucket[slot]]; - child_pod = pod[bucket[slot]]; - if (one_process) { set_signals(); worker_note_child_started(slot, getpid()); - child_main(slot); + child_main(0, 0); /* NOTREACHED */ } @@ -1451,7 +1450,7 @@ static int make_child(server_rec *s, int slot) RAISE_SIGSTOP(MAKE_CHILD); apr_signal(SIGTERM, just_die); - child_main(slot); + child_main(slot, bucket); /* NOTREACHED */ } /* else */ @@ -1463,6 +1462,7 @@ static int make_child(server_rec *s, int slot) worker_note_child_lost_slot(slot, pid); } ap_scoreboard_image->parent[slot].quiescing = 0; + ap_scoreboard_image->parent[slot].bucket = bucket; worker_note_child_started(slot, pid); return 0; } @@ -1476,8 +1476,7 @@ static void startup_children(int number_to_start) if (ap_scoreboard_image->parent[i].pid != 0) { continue; } - bucket[i] = i % num_buckets; - if (make_child(ap_server_conf, i) < 0) { + if (make_child(ap_server_conf, i, i % num_buckets) < 0) { break; } --number_to_start; @@ -1542,7 +1541,7 @@ static void perform_idle_server_maintenance(int child_bucket) if (status <= SERVER_READY && !ps->quiescing && ps->generation == retained->my_generation && - bucket[i] == child_bucket) { + ps->bucket == child_bucket) { ++idle_thread_count; } if (status >= SERVER_READY && status < SERVER_GRACEFUL) { @@ -1552,7 +1551,7 @@ static void perform_idle_server_maintenance(int child_bucket) } active_thread_count += child_threads_active; if (any_dead_threads && totally_free_length < retained->idle_spawn_rate[child_bucket] - && free_length < MAX_SPAWN_RATE/num_buckets + && free_length < MAX_SPAWN_RATE / num_buckets && (!ps->pid /* no process in the slot */ || ps->quiescing)) { /* or at least one is going away */ if (all_dead_threads) { @@ -1608,12 +1607,12 @@ static void perform_idle_server_maintenance(int child_bucket) retained->max_daemons_limit = last_non_dead + 1; - if (idle_thread_count > max_spare_threads/num_buckets) { + if (idle_thread_count > max_spare_threads / num_buckets) { /* Kill off one child */ ap_mpm_podx_signal(pod[child_bucket], AP_MPM_PODX_GRACEFUL); retained->idle_spawn_rate[child_bucket] = 1; } - else if (idle_thread_count < min_spare_threads/num_buckets) { + else if (idle_thread_count < min_spare_threads) { /* terminate the free list */ if (free_length == 0) { /* scoreboard is full, can't fork */ @@ -1661,8 +1660,7 @@ static void perform_idle_server_maintenance(int child_bucket) idle_thread_count, total_non_dead); } for (i = 0; i < free_length; ++i) { - bucket[free_slots[i]] = child_bucket; - make_child(ap_server_conf, free_slots[i]); + make_child(ap_server_conf, free_slots[i], child_bucket); } /* the next time around we want to spawn twice as many if this * wasn't good enough, but not if we've just done a graceful @@ -1670,13 +1668,14 @@ static void perform_idle_server_maintenance(int child_bucket) if (retained->hold_off_on_exponential_spawning) { --retained->hold_off_on_exponential_spawning; } - else if (retained->idle_spawn_rate[child_bucket] < MAX_SPAWN_RATE/num_buckets) { + else if (retained->idle_spawn_rate[child_bucket] + < MAX_SPAWN_RATE / num_buckets) { retained->idle_spawn_rate[child_bucket] *= 2; } } } else { - retained->idle_spawn_rate[child_bucket] = 1; + retained->idle_spawn_rate[child_bucket] = 1; } } @@ -1724,22 +1723,25 @@ static void server_main_loop(int remaining_children_to_start) } /* non-fatal death... note that it's gone in the scoreboard. */ if (child_slot >= 0) { + process_score *ps; + for (i = 0; i < threads_per_child; i++) ap_update_child_status_from_indexes(child_slot, i, SERVER_DEAD, (request_rec *) NULL); worker_note_child_killed(child_slot, 0, 0); - ap_scoreboard_image->parent[child_slot].quiescing = 0; + ps = &ap_scoreboard_image->parent[child_slot]; + ps->quiescing = 0; if (processed_status == APEXIT_CHILDSICK) { /* resource shortage, minimize the fork rate */ - retained->idle_spawn_rate[bucket[child_slot]] = 1; + retained->idle_spawn_rate[ps->bucket] = 1; } else if (remaining_children_to_start && child_slot < ap_daemons_limit) { /* we're still doing a 1-for-1 replacement of dead * children with new children */ - make_child(ap_server_conf, child_slot); + make_child(ap_server_conf, child_slot, ps->bucket); --remaining_children_to_start; } } @@ -1804,7 +1806,6 @@ static int worker_run(apr_pool_t *_pconf, apr_pool_t *plog, server_rec *s) ap_log_pid(pconf, ap_pid_fname); - bucket = apr_palloc(_pconf, sizeof(int) * ap_daemons_limit); /* Initialize cross-process accept lock */ accept_mutex = apr_palloc(_pconf, sizeof(apr_proc_mutex_t *) * num_buckets); for (i = 0; i < num_buckets; i++) { @@ -2004,7 +2005,6 @@ static int worker_open_logs(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp, int level_flags = 0; apr_status_t rv; int i; - int num_of_cores = 0; pconf = p; @@ -2021,24 +2021,8 @@ static int worker_open_logs(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp, "no listening sockets available, shutting down"); return DONE; } - enable_default_listener = 1; - if (have_so_reuseport) { -#ifdef _SC_NPROCESSORS_ONLN - num_of_cores = sysconf(_SC_NPROCESSORS_ONLN); -#else - num_of_cores = 1; -#endif - if (num_of_cores > 8) { - num_buckets = num_of_cores/8; - } - else { - num_buckets = 1; - } - } - else { - num_buckets = 1; - } + enable_default_listener = 1; ap_duplicate_listeners(ap_server_conf, pconf, num_buckets); pod = apr_palloc(pconf, sizeof(ap_pod_t *) * num_buckets); @@ -2085,6 +2069,17 @@ static int worker_pre_config(apr_pool_t *pconf, apr_pool_t *plog, retained = ap_retained_data_create(userdata_key, sizeof(*retained)); retained->max_daemons_limit = -1; } + if (!retained->is_graceful) { + num_buckets = 1; +#ifdef _SC_NPROCESSORS_ONLN + if (have_so_reuseport) { + int num_online_cores = sysconf(_SC_NPROCESSORS_ONLN); + if (num_online_cores > 8) { + num_buckets = num_online_cores / 8; + } + } +#endif + } ++retained->module_loads; if (retained->module_loads == 2) { if (!one_process && !foreground) { @@ -2317,9 +2312,15 @@ static int worker_check_config(apr_pool_t *p, apr_pool_t *plog, } ap_daemons_limit = server_limit; } + else if (ap_daemons_limit < num_buckets) { + /* Don't thrash since num_buckets depends on + * the system and the number of CPU cores. + */ + ap_daemons_limit = num_buckets; + } /* ap_daemons_to_start > ap_daemons_limit checked in worker_run() */ - if (ap_daemons_to_start < 0) { + if (ap_daemons_to_start < 1) { if (startup) { ap_log_error(APLOG_MARK, APLOG_WARNING | APLOG_STARTUP, 0, NULL, APLOGNO(00320) "WARNING: StartServers of %d not allowed, " @@ -2331,6 +2332,12 @@ static int worker_check_config(apr_pool_t *p, apr_pool_t *plog, } ap_daemons_to_start = 1; } + if (ap_daemons_to_start < num_buckets) { + /* Don't thrash since num_buckets depends on + * the system and the number of CPU cores. + */ + ap_daemons_to_start = num_buckets; + } if (min_spare_threads < 1) { if (startup) { @@ -2348,6 +2355,12 @@ static int worker_check_config(apr_pool_t *p, apr_pool_t *plog, } min_spare_threads = 1; } + if (min_spare_threads < num_buckets) { + /* Don't thrash since num_buckets depends on + * the system and the number of CPU cores. + */ + min_spare_threads = num_buckets; + } /* max_spare_threads < min_spare_threads + threads_per_child * checked in worker_run()