Implement the worker retry functionality.

It uses either worker->retry option or default 60 second retry
that is on each revolution extended by another 60 seconds.

Submitted by: mturk

git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@104619 13f79535-47bb-0310-9956-ffa450edef68

diff --git a/modules/proxy/mod_proxy.h b/modules/proxy/mod_proxy.h
index 9e808b4ada6ac33aa46d7568cf51d3f931c975ad..3161e3ac59fab8de061f4e5f7e5c66eb155a8e89 100644 (file)
--- a/modules/proxy/mod_proxy.h
+++ b/modules/proxy/mod_proxy.h
@@ -384,6 +384,7 @@ PROXY_DECLARE(int) ap_proxy_pre_request(proxy_worker **worker, proxy_balancer **
 PROXY_DECLARE(int) ap_proxy_determine_connection(apr_pool_t *p, request_rec *r, proxy_server_conf *conf, proxy_worker *worker, proxy_conn_rec *conn,
                                                  apr_pool_t *ppool, apr_uri_t *uri, char **url, const char *proxyname, apr_port_t proxyport,
                                                  char *server_portstr, int server_portstr_size);
+PROXY_DECLARE(int) ap_proxy_retry_worker(const char *proxy_function, proxy_worker *worker, server_rec *s);
 PROXY_DECLARE(int) ap_proxy_acquire_connection(const char *proxy_function, proxy_conn_rec **conn, proxy_worker *worker, server_rec *s);
 PROXY_DECLARE(int) ap_proxy_release_connection(const char *proxy_function, proxy_conn_rec *conn, server_rec *s);
 PROXY_DECLARE(apr_status_t) ap_proxy_close_connection(proxy_conn_rec *conn);

diff --git a/modules/proxy/proxy_balancer.c b/modules/proxy/proxy_balancer.c
index ab832399d151ccb7e234d52f574387652367042b..61a674b7deb306a288e48ac30ee4892f22f391da 100644 (file)
--- a/modules/proxy/proxy_balancer.c
+++ b/modules/proxy/proxy_balancer.c
@@ -146,11 +146,14 @@ static proxy_runtime_worker *find_best_worker(proxy_balancer *balancer,
 
     /* First try to see if we have available candidate */
     for (i = 0; i < balancer->workers->nelts; i++) {
-        /* If the worker is not error state
-         * or not in disabled mode
+        /* See if the retry timeout is ellapsed
+         * for the workers flagged as IN_ERROR
+         */
+        if (!PROXY_WORKER_IS_USABLE(worker->w))
+            ap_proxy_retry_worker("BALANCER", worker->w, r->server)
+        /* If the worker is not in error state
+         * or not disabled.
          */
-
-        /* TODO: read the scoreboard status */
         if (PROXY_WORKER_IS_USABLE(worker->w)) {
             if (!candidate)
                 candidate = worker;

diff --git a/modules/proxy/proxy_util.c b/modules/proxy/proxy_util.c
index f31ede2cf16c272f96102e8d09ad4c3652cca3f0..e1ed3b5ad4b11d6ca7a8a3922b12e32303fd2866 100644 (file)
--- a/modules/proxy/proxy_util.c
+++ b/modules/proxy/proxy_util.c
@@ -1471,6 +1471,31 @@ static apr_status_t init_conn_worker(proxy_worker *worker, server_rec *s)
     return rv;
 }
 
+PROXY_DECLARE(int) ap_proxy_retry_worker(const char *proxy_function,
+                                         proxy_worker *worker,
+                                         server_rec *s)
+{
+    if (worker->status & PROXY_WORKER_IN_ERROR) {
+        apr_interval_time_t diff;
+        apr_time_t now = apr_time_now();
+        if (worker->retry)
+            diff = worker->retry;
+        else
+            diff = apr_time_from_sec(60 + 60 * worker->retries++);
+        if (now > worker->error_time + diff) {
+            worker->status &= ~PROXY_WORKER_IN_ERROR;
+            ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s,
+                         "proxy: %s: retrying the worker for (%s)",
+                         proxy_function, worker->hostname);
+            return OK;
+        }
+        else
+            return DECLINED;
+    }
+    else
+        return OK;
+}
+
 PROXY_DECLARE(int) ap_proxy_acquire_connection(const char *proxy_function,
                                                proxy_conn_rec **conn,
                                                proxy_worker *worker,
@@ -1483,10 +1508,22 @@ PROXY_DECLARE(int) ap_proxy_acquire_connection(const char *proxy_function,
             ap_log_error(APLOG_MARK, APLOG_ERR, rv, s,
                          "proxy: %s: failed to initialize worker for (%s)",
                          proxy_function, worker->hostname);
-            return DECLINED;
+            return HTTP_INTERNAL_SERVER_ERROR;
         }
         worker->status = PROXY_WORKER_INITIALIZED;
     }
+
+    if (!PROXY_WORKER_IS_USABLE(worker)) {
+        /* Retry the worker */
+        ap_proxy_retry_worker(proxy_function, worker, s);
+    
+        if (!PROXY_WORKER_IS_USABLE(worker)) {
+            ap_log_error(APLOG_MARK, APLOG_ERR, rv, s,
+                         "proxy: %s: disabled connection for (%s)",
+                         proxy_function, worker->hostname);
+            return HTTP_SERVICE_UNAVAILABLE;
+        }
+    }
 #if APR_HAS_THREADS
     if (worker->hmax) {
         rv = apr_reslist_acquire(worker->cp->res, (void **)conn);
@@ -1508,7 +1545,7 @@ PROXY_DECLARE(int) ap_proxy_acquire_connection(const char *proxy_function,
         ap_log_error(APLOG_MARK, APLOG_ERR, rv, s,
                      "proxy: %s: failed to acquire connection for (%s)",
                      proxy_function, worker->hostname);
-        return DECLINED;
+        return HTTP_SERVICE_UNAVAILABLE;
     }
     return OK;
 }
@@ -1723,6 +1760,14 @@ PROXY_DECLARE(int) ap_proxy_connect_backend(const char *proxy_function,
         conn->worker = worker;
         connected    = 1;
     }
+    /* Put the entire worker to error state
+     * Altrough some connections may be alive
+     * no further connections to the worker could be made
+     */
+    if (!connected && PROXY_WORKER_IS_USABLE(worker)) {
+        worker->status |= PROXY_WORKER_IN_ERROR;
+        worker->error_time = apr_time_now();
+    }
     return connected ? OK : DECLINED;
 }