From 1703da8e3787f59093eaaa3c78b43230cf0ff7d3 Mon Sep 17 00:00:00 2001 From: Ryan Bloom Date: Mon, 11 Feb 2002 04:56:10 +0000 Subject: [PATCH] Remove all signal handling from the worker MPM's child processes. Instead, we use the pipe of death for all communication between parent and child. git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@93358 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 4 + server/mpm/worker/Makefile.in | 2 +- server/mpm/worker/mpm.h | 1 - server/mpm/worker/pod.c | 234 ++++++++++++++++++++++++++++++++++ server/mpm/worker/pod.h | 94 ++++++++++++++ server/mpm/worker/worker.c | 88 +++++-------- 6 files changed, 366 insertions(+), 57 deletions(-) create mode 100644 server/mpm/worker/pod.c create mode 100644 server/mpm/worker/pod.h diff --git a/CHANGES b/CHANGES index 72a5ccc776..189a80a8ee 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,9 @@ Changes with Apache 2.0.32-dev + *) Remove all signals from the worker MPM's child process. Instead, + the parent uses the Pipe of Death for all communication with the + child processes. [Ryan Bloom] + *) Fix prefork to not kill the parent if a child hits a resource shortage on accept(). [Greg Ames] diff --git a/server/mpm/worker/Makefile.in b/server/mpm/worker/Makefile.in index 64a02436f3..b45b848341 100644 --- a/server/mpm/worker/Makefile.in +++ b/server/mpm/worker/Makefile.in @@ -1,5 +1,5 @@ LTLIBRARY_NAME = libworker.la -LTLIBRARY_SOURCES = worker.c fdqueue.c +LTLIBRARY_SOURCES = worker.c fdqueue.c pod.c include $(top_srcdir)/build/ltlib.mk diff --git a/server/mpm/worker/mpm.h b/server/mpm/worker/mpm.h index ac34100972..5db57028f6 100644 --- a/server/mpm/worker/mpm.h +++ b/server/mpm/worker/mpm.h @@ -76,7 +76,6 @@ #define AP_MPM_WANT_SET_ACCEPT_LOCK_MECH #define AP_MPM_DISABLE_NAGLE_ACCEPTED_SOCK -#define AP_MPM_USES_POD 1 #define MPM_SYNC_CHILD_TABLE() (ap_sync_scoreboard_image()) #define MPM_CHILD_PID(i) (ap_scoreboard_image->parent[i].pid) #define MPM_NOTE_CHILD_KILLED(i) (MPM_CHILD_PID(i) = 0) diff --git a/server/mpm/worker/pod.c b/server/mpm/worker/pod.c new file mode 100644 index 0000000000..c869e43832 --- /dev/null +++ b/server/mpm/worker/pod.c @@ -0,0 +1,234 @@ +/* ==================================================================== + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2000-2001 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, + * if any, must include the following acknowledgment: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowledgment may appear in the software itself, + * if and wherever such third-party acknowledgments normally appear. + * + * 4. The names "Apache" and "Apache Software Foundation" must + * not be used to endorse or promote products derived from this + * software without prior written permission. For written + * permission, please contact apache@apache.org. + * + * 5. Products derived from this software may not be called "Apache", + * nor may "Apache" appear in their name, without prior written + * permission of the Apache Software Foundation. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + * Portions of this software are based upon public domain software + * originally written at the National Center for Supercomputing Applications, + * University of Illinois, Urbana-Champaign. + */ + +#include "apr.h" +#include "apr_strings.h" +#include "apr_lock.h" +#define APR_WANT_STRFUNC +#include "apr_want.h" + +#include "httpd.h" +#include "http_config.h" +#include "http_log.h" +#include "http_main.h" +#include "mpm.h" +#include "pod.h" +#include "mpm_common.h" +#include "ap_mpm.h" +#include "ap_listen.h" +#include "mpm_default.h" + +AP_DECLARE(apr_status_t) ap_mpm_pod_open(apr_pool_t *p, ap_pod_t **pod) +{ + apr_status_t rv; + + *pod = apr_palloc(p, sizeof(**pod)); + rv = apr_file_pipe_create(&((*pod)->pod_in), &((*pod)->pod_out), p); + if (rv != APR_SUCCESS) { + return rv; + } +/* + apr_file_pipe_timeout_set((*pod)->pod_in, 0); +*/ + (*pod)->p = p; + + apr_sockaddr_info_get(&(*pod)->sa, ap_listeners->bind_addr->hostname, + APR_UNSPEC, ap_listeners->bind_addr->port, 0, p); + + return APR_SUCCESS; +} + +AP_DECLARE(int) ap_mpm_pod_check(ap_pod_t *pod) +{ + char c; + apr_size_t len = 1; + apr_status_t rv; + + rv = apr_file_read(pod->pod_in, &c, &len); + + if ((rv == APR_SUCCESS) && (len ==1)) { + if (c == RESTART_CHAR) { + return AP_RESTART; + } + if (c == GRACEFUL_CHAR) { + return AP_GRACEFUL; + } + } + else if (rv != APR_SUCCESS) { + return rv; + } + return AP_NORESTART; +} + +AP_DECLARE(apr_status_t) ap_mpm_pod_close(ap_pod_t *pod) +{ + apr_status_t rv; + + rv = apr_file_close(pod->pod_out); + if (rv != APR_SUCCESS) { + return rv; + } + + rv = apr_file_close(pod->pod_in); + if (rv != APR_SUCCESS) { + return rv; + } + return rv; +} + +static apr_status_t pod_signal_internal(ap_pod_t *pod, int graceful) +{ + apr_status_t rv; + char char_of_death = graceful ? GRACEFUL_CHAR : RESTART_CHAR; + apr_size_t one = 1; + + do { + rv = apr_file_write(pod->pod_out, &char_of_death, &one); + } while (APR_STATUS_IS_EINTR(rv)); + if (rv != APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_WARNING, rv, ap_server_conf, + "write pipe_of_death"); + } + return rv; +} + +/* This function connects to the server, then immediately closes the connection. + * This permits the MPM to skip the poll when there is only one listening + * socket, because it provides a alternate way to unblock an accept() when + * the pod is used. + */ + +static apr_status_t dummy_connection(ap_pod_t *pod) +{ + apr_status_t rv; + apr_socket_t *sock; + apr_pool_t *p; + + /* create a temporary pool for the socket. pconf stays around too long */ + rv = apr_pool_create(&p, pod->p); + if (rv != APR_SUCCESS) { + return rv; + } + + rv = apr_socket_create(&sock, pod->sa->family, SOCK_STREAM, p); + if (rv != APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_WARNING, rv, ap_server_conf, + "get socket to connect to listener"); + return rv; + } + /* on some platforms (e.g., FreeBSD), the kernel won't accept many + * queued connections before it starts blocking local connects... + * we need to keep from blocking too long and instead return an error, + * because the MPM won't want to hold up a graceful restart for a + * long time + */ + rv = apr_setsocketopt(sock, APR_SO_TIMEOUT, 3 * APR_USEC_PER_SEC); + if (rv != APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_WARNING, rv, ap_server_conf, + "set timeout on socket to connect to listener"); + return rv; + } + + rv = apr_connect(sock, pod->sa); + if (rv != APR_SUCCESS) { + int log_level = APLOG_WARNING; + + if (APR_STATUS_IS_TIMEUP(rv)) { + /* probably some server processes bailed out already and there + * is nobody around to call accept and clear out the kernel + * connection queue; usually this is not worth logging + */ + log_level = APLOG_DEBUG; + } + + ap_log_error(APLOG_MARK, log_level, rv, ap_server_conf, + "connect to listener"); + } + + apr_socket_close(sock); + apr_pool_destroy(p); + + return rv; +} + +AP_DECLARE(apr_status_t) ap_mpm_pod_signal(ap_pod_t *pod, int graceful) +{ + apr_status_t rv; + + rv = pod_signal_internal(pod, graceful); + if (rv != APR_SUCCESS) { + return rv; + } + return dummy_connection(pod); +} + +AP_DECLARE(void) ap_mpm_pod_killpg(ap_pod_t *pod, int num, int graceful) +{ + int i; + apr_status_t rv = APR_SUCCESS; + + for (i = 0; i < num && rv == APR_SUCCESS; i++) { + rv = pod_signal_internal(pod, graceful); + } + if (rv == APR_SUCCESS) { + for (i = 0; i < num && rv == APR_SUCCESS; i++) { + rv = dummy_connection(pod); + } + } +} + diff --git a/server/mpm/worker/pod.h b/server/mpm/worker/pod.h new file mode 100644 index 0000000000..cc68b4b6e3 --- /dev/null +++ b/server/mpm/worker/pod.h @@ -0,0 +1,94 @@ +/* ==================================================================== + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2000-2001 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, + * if any, must include the following acknowledgment: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowledgment may appear in the software itself, + * if and wherever such third-party acknowledgments normally appear. + * + * 4. The names "Apache" and "Apache Software Foundation" must + * not be used to endorse or promote products derived from this + * software without prior written permission. For written + * permission, please contact apache@apache.org. + * + * 5. Products derived from this software may not be called "Apache", + * nor may "Apache" appear in their name, without prior written + * permission of the Apache Software Foundation. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + * Portions of this software are based upon public domain software + * originally written at the National Center for Supercomputing Applications, + * University of Illinois, Urbana-Champaign. + */ + +#include "apr.h" +#include "apr_strings.h" +#include "apr_lock.h" +#define APR_WANT_STRFUNC +#include "apr_want.h" + +#include "httpd.h" +#include "http_config.h" +#include "http_log.h" +#include "http_main.h" +#include "mpm.h" +#include "mpm_common.h" +#include "ap_mpm.h" +#include "ap_listen.h" +#include "mpm_default.h" + +#define RESTART_CHAR '$' +#define GRACEFUL_CHAR '!' + +#define AP_RESTART 0 +#define AP_GRACEFUL 1 + +typedef struct ap_pod_t ap_pod_t; + +struct ap_pod_t { + apr_file_t *pod_in; + apr_file_t *pod_out; + apr_pool_t *p; + apr_sockaddr_t *sa; +}; + +AP_DECLARE(apr_status_t) ap_mpm_pod_open(apr_pool_t *p, ap_pod_t **pod); +AP_DECLARE(int) ap_mpm_pod_check(ap_pod_t *pod); +AP_DECLARE(apr_status_t) ap_mpm_pod_close(ap_pod_t *pod); +AP_DECLARE(apr_status_t) ap_mpm_pod_signal(ap_pod_t *pod, int graceful); +AP_DECLARE(void) ap_mpm_pod_killpg(ap_pod_t *pod, int num, int graceful); diff --git a/server/mpm/worker/worker.c b/server/mpm/worker/worker.c index 9664e2ffe6..23733f6f93 100644 --- a/server/mpm/worker/worker.c +++ b/server/mpm/worker/worker.c @@ -101,6 +101,7 @@ #include "http_core.h" /* for get_remote_host */ #include "http_connection.h" #include "ap_mpm.h" +#include "pod.h" #include "mpm_common.h" #include "ap_listen.h" #include "scoreboard.h" @@ -695,10 +696,6 @@ static void *listener_thread(apr_thread_t *thd, void * dummy) rv); } } - if (ap_mpm_pod_check(pod) == APR_SUCCESS) { - signal_workers(); - break; - } } else { if ((rv = SAFE_ACCEPT(apr_proc_mutex_unlock(accept_mutex))) @@ -760,16 +757,6 @@ static void * APR_THREAD_FUNC worker_thread(apr_thread_t *thd, void * dummy) return NULL; } -static int check_signal(int signum) -{ - switch (signum) { - case SIGTERM: - case SIGINT: - return 1; - } - return 0; -} - static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy) { thread_starter *ts = dummy; @@ -893,6 +880,10 @@ static void child_main(int child_num_arg) /* done with init critical section */ + /* Just use the standard apr_setup_signal_thread to block all signals + * from being received. The child processes no longer use signals for + * any communication with the parent process. + */ rv = apr_setup_signal_thread(); if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, @@ -947,24 +938,28 @@ static void child_main(int child_num_arg) clean_child_exit(APEXIT_CHILDFATAL); } - apr_signal_thread(check_signal); + /* Watch for any messages from the parent over the POD */ + while (1) { + rv = ap_mpm_pod_check(pod); + if (rv == AP_GRACEFUL || rv == AP_RESTART) { + signal_workers(); + break; + } + } - signal_workers(); /* helps us terminate a little more quickly when - * the dispatch of the signal thread - * beats the Pipe of Death and the browsers - */ - - /* A terminating signal was received. Now join each of the workers to - * clean them up. - * If the worker already exited, then the join frees their resources - * and returns. - * If the worker hasn't exited, then this blocks until they have (then - * cleans up). - */ - apr_thread_join(&rv, start_thread_id); - for (i = 0; i < ap_threads_per_child; i++) { - if (threads[i]) { /* if we ever created this thread */ - apr_thread_join(&rv, threads[i]); + if (rv == AP_GRACEFUL) { + /* A terminating signal was received. Now join each of the workers to + * clean them up. + * If the worker already exited, then the join frees their resources + * and returns. + * If the worker hasn't exited, then this blocks until they have (then + * cleans up). + */ + apr_thread_join(&rv, start_thread_id); + for (i = 0; i < ap_threads_per_child; i++) { + if (threads[i]) { /* if we ever created this thread */ + apr_thread_join(&rv, threads[i]); + } } } @@ -1029,16 +1024,6 @@ static int make_child(server_rec *s, int slot) return 0; } -/* If there aren't many connections coming in from the network, the child - * processes may need to be awakened from their network i/o waits. - * The pipe of death is an effective prod. - */ - -static void wake_up_and_die(void) -{ - ap_mpm_pod_killpg(pod, ap_daemons_limit); -} - /* start up a bunch of children */ static void startup_children(int number_to_start) { @@ -1158,7 +1143,7 @@ static void perform_idle_server_maintenance(void) if (idle_thread_count > max_spare_threads) { /* Kill off one child */ - ap_mpm_pod_signal(pod); + ap_mpm_pod_signal(pod, TRUE); idle_spawn_rate = 1; } else if (idle_thread_count < min_spare_threads) { @@ -1373,12 +1358,7 @@ int ap_mpm_run(apr_pool_t *_pconf, apr_pool_t *plog, server_rec *s) /* Time to gracefully shut down: * Kill child processes, tell them to call child_exit, etc... */ - wake_up_and_die(); - - if (ap_os_killpg(getpgrp(), SIGTERM) < 0) { - ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf, - "killpg SIGTERM"); - } + ap_mpm_pod_killpg(pod, ap_daemons_limit, TRUE); ap_reclaim_child_processes(1); /* Start with SIGTERM */ if (!child_fatal) { @@ -1413,12 +1393,12 @@ int ap_mpm_run(apr_pool_t *_pconf, apr_pool_t *plog, server_rec *s) ap_scoreboard_image->global->running_generation = ap_my_generation; update_scoreboard_global(); - /* wake up the children...time to die. But we'll have more soon */ - wake_up_and_die(); - if (is_graceful) { ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_NOTICE, 0, ap_server_conf, AP_SIG_GRACEFUL_STRING " received. Doing graceful restart"); + /* wake up the children...time to die. But we'll have more soon */ + ap_mpm_pod_killpg(pod, ap_daemons_limit, TRUE); + /* This is mostly for debugging... so that we know what is still * gracefully dealing with existing request. @@ -1430,10 +1410,8 @@ int ap_mpm_run(apr_pool_t *_pconf, apr_pool_t *plog, server_rec *s) * and a SIGHUP, we may as well use the same signal, because some user * pthreads are stealing signals from us left and right. */ - if (ap_os_killpg(getpgrp(), SIGTERM) < 0) { - ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf, - "killpg SIGTERM"); - } + ap_mpm_pod_killpg(pod, ap_daemons_limit, FALSE); + ap_reclaim_child_processes(1); /* Start with SIGTERM */ ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_NOTICE, 0, ap_server_conf, "SIGHUP received. Attempting to restart"); -- 2.40.0