granicus.if.org Git - apache/blob - modules/proxy/mod_proxy_balancer.c

   1 /* Licensed to the Apache Software Foundation (ASF) under one or more
   2  * contributor license agreements.  See the NOTICE file distributed with
   3  * this work for additional information regarding copyright ownership.
   4  * The ASF licenses this file to You under the Apache License, Version 2.0
   5  * (the "License"); you may not use this file except in compliance with
   6  * the License.  You may obtain a copy of the License at
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /* Load balancer module for Apache proxy */
  18
  19 #define CORE_PRIVATE
  20
  21 #include "mod_proxy.h"
  22 #include "ap_mpm.h"
  23 #include "apr_version.h"
  24 #include "apr_hooks.h"
  25
  26 #if APR_HAVE_UNISTD_H
  27 #include <unistd.h> /* for getpid() */
  28 #endif
  29
  30 module AP_MODULE_DECLARE_DATA proxy_balancer_module;
  31
  32 static int proxy_balancer_canon(request_rec *r, char *url)
  33 {
  34     char *host, *path, *search;
  35     const char *err;
  36     apr_port_t port = 0;
  37
  38     if (strncasecmp(url, "balancer:", 9) == 0) {
  39         url += 9;
  40     }
  41     else {
  42         return DECLINED;
  43     }
  44
  45     ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
  46              "proxy: BALANCER: canonicalising URL %s", url);
  47
  48     /* do syntatic check.
  49      * We break the URL into host, port, path, search
  50      */
  51     err = ap_proxy_canon_netloc(r->pool, &url, NULL, NULL, &host, &port);
  52     if (err) {
  53         ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r,
  54                       "error parsing URL %s: %s",
  55                       url, err);
  56         return HTTP_BAD_REQUEST;
  57     }
  58     /* now parse path/search args, according to rfc1738 */
  59     /* N.B. if this isn't a true proxy request, then the URL _path_
  60      * has already been decoded.  True proxy requests have r->uri
  61      * == r->unparsed_uri, and no others have that property.
  62      */
  63     if (r->uri == r->unparsed_uri) {
  64         search = strchr(url, '?');
  65         if (search != NULL)
  66             *(search++) = '\0';
  67     }
  68     else
  69         search = r->args;
  70
  71     /* process path */
  72     path = ap_proxy_canonenc(r->pool, url, strlen(url), enc_path, 0, r->proxyreq);
  73     if (path == NULL)
  74         return HTTP_BAD_REQUEST;
  75
  76     r->filename = apr_pstrcat(r->pool, "proxy:balancer://", host,
  77             "/", path, (search) ? "?" : "", (search) ? search : "", NULL);
  78
  79     r->path_info = apr_pstrcat(r->pool, "/", path, NULL);
  80
  81     return OK;
  82 }
  83
  84 static int init_balancer_members(proxy_server_conf *conf, server_rec *s,
  85                                  proxy_balancer *balancer)
  86 {
  87     int i;
  88     proxy_worker *workers;
  89
  90     workers = (proxy_worker *)balancer->workers->elts;
  91
  92     for (i = 0; i < balancer->workers->nelts; i++) {
  93         ap_proxy_initialize_worker_share(conf, workers, s);
  94         ap_proxy_initialize_worker(workers, s);
  95         ++workers;
  96     }
  97
  98     workers = (proxy_worker *)balancer->workers->elts;
  99     for (i = 0; i < balancer->workers->nelts; i++) {
 100         /* Set to the original configuration */
 101         workers[i].s->lbstatus = workers[i].s->lbfactor =
 102           (workers[i].lbfactor ? workers[i].lbfactor : 1);
 103         workers[i].s->lbset = workers[i].lbset;
 104     }
 105     /* Set default number of attempts to the number of
 106      * workers.
 107      */
 108     if (!balancer->max_attempts_set && balancer->workers->nelts > 1) {
 109         balancer->max_attempts = balancer->workers->nelts - 1;
 110         balancer->max_attempts_set = 1;
 111     }
 112     return 0;
 113 }
 114
 115 /* Retrieve the parameter with the given name
 116  * Something like 'JSESSIONID=12345...N'
 117  */
 118 static char *get_path_param(apr_pool_t *pool, char *url,
 119                             const char *name)
 120 {
 121     char *path = NULL;
 122
 123     for (path = strstr(url, name); path; path = strstr(path + 1, name)) {
 124         path += strlen(name);
 125         if (*path == '=') {
 126             /*
 127              * Session path was found, get it's value
 128              */
 129             ++path;
 130             if (strlen(path)) {
 131                 char *q;
 132                 path = apr_strtok(apr_pstrdup(pool, path), "?&", &q);
 133                 return path;
 134             }
 135         }
 136     }
 137     return NULL;
 138 }
 139
 140 static char *get_cookie_param(request_rec *r, const char *name)
 141 {
 142     const char *cookies;
 143     const char *start_cookie;
 144
 145     if ((cookies = apr_table_get(r->headers_in, "Cookie"))) {
 146         for (start_cookie = ap_strstr_c(cookies, name); start_cookie;
 147              start_cookie = ap_strstr_c(start_cookie + 1, name)) {
 148             if (start_cookie == cookies ||
 149                 start_cookie[-1] == ';' ||
 150                 start_cookie[-1] == ',' ||
 151                 isspace(start_cookie[-1])) {
 152
 153                 start_cookie += strlen(name);
 154                 while(*start_cookie && isspace(*start_cookie))
 155                     ++start_cookie;
 156                 if (*start_cookie == '=' && start_cookie[1]) {
 157                     /*
 158                      * Session cookie was found, get it's value
 159                      */
 160                     char *end_cookie, *cookie;
 161                     ++start_cookie;
 162                     cookie = apr_pstrdup(r->pool, start_cookie);
 163                     if ((end_cookie = strchr(cookie, ';')) != NULL)
 164                         *end_cookie = '\0';
 165                     if((end_cookie = strchr(cookie, ',')) != NULL)
 166                         *end_cookie = '\0';
 167                     return cookie;
 168                 }
 169             }
 170         }
 171     }
 172     return NULL;
 173 }
 174
 175 /* Find the worker that has the 'route' defined
 176  */
 177 static proxy_worker *find_route_worker(proxy_balancer *balancer,
 178                                        const char *route, request_rec *r)
 179 {
 180     int i;
 181     int checking_standby;
 182     int checked_standby;
 183
 184     proxy_worker *worker;
 185
 186     checking_standby = checked_standby = 0;
 187     while (!checked_standby) {
 188         worker = (proxy_worker *)balancer->workers->elts;
 189         for (i = 0; i < balancer->workers->nelts; i++, worker++) {
 190             if ( (checking_standby ? !PROXY_WORKER_IS_STANDBY(worker) : PROXY_WORKER_IS_STANDBY(worker)) )
 191                 continue;
 192             if (*(worker->s->route) && strcmp(worker->s->route, route) == 0) {
 193                 if (worker && PROXY_WORKER_IS_USABLE(worker)) {
 194                     return worker;
 195                 } else {
 196                     /*
 197                      * If the worker is in error state run
 198                      * retry on that worker. It will be marked as
 199                      * operational if the retry timeout is elapsed.
 200                      * The worker might still be unusable, but we try
 201                      * anyway.
 202                      */
 203                     ap_proxy_retry_worker("BALANCER", worker, r->server);
 204                     if (PROXY_WORKER_IS_USABLE(worker)) {
 205                             return worker;
 206                     } else {
 207                         /*
 208                          * We have a worker that is unusable.
 209                          * It can be in error or disabled, but in case
 210                          * it has a redirection set use that redirection worker.
 211                          * This enables to safely remove the member from the
 212                          * balancer. Of course you will need some kind of
 213                          * session replication between those two remote.
 214                          */
 215                         if (*worker->s->redirect) {
 216                             proxy_worker *rworker = NULL;
 217                             rworker = find_route_worker(balancer, worker->s->redirect, r);
 218                             /* Check if the redirect worker is usable */
 219                             if (rworker && !PROXY_WORKER_IS_USABLE(rworker)) {
 220                                 /*
 221                                  * If the worker is in error state run
 222                                  * retry on that worker. It will be marked as
 223                                  * operational if the retry timeout is elapsed.
 224                                  * The worker might still be unusable, but we try
 225                                  * anyway.
 226                                  */
 227                                 ap_proxy_retry_worker("BALANCER", rworker, r->server);
 228                             }
 229                             if (rworker && PROXY_WORKER_IS_USABLE(rworker))
 230                                 return rworker;
 231                         }
 232                     }
 233                 }
 234             }
 235         }
 236         checked_standby = checking_standby++;
 237     }
 238     return NULL;
 239 }
 240
 241 static proxy_worker *find_session_route(proxy_balancer *balancer,
 242                                         request_rec *r,
 243                                         char **route,
 244                                         char **url)
 245 {
 246     proxy_worker *worker = NULL;
 247
 248     if (!balancer->sticky)
 249         return NULL;
 250     /* Try to find the sticky route inside url */
 251     *route = get_path_param(r->pool, *url, balancer->sticky);
 252     if (!*route)
 253         *route = get_cookie_param(r, balancer->sticky);
 254     ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
 255                             "proxy: BALANCER: Found value %s for "
 256                             "stickysession %s", *route, balancer->sticky);
 257     /*
 258      * If we found a value for sticksession, find the first '.' within.
 259      * Everything after '.' (if present) is our route.
 260      */
 261     if ((*route) && ((*route = strchr(*route, '.')) != NULL ))
 262         (*route)++;
 263     if ((*route) && (**route)) {
 264         ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
 265                                   "proxy: BALANCER: Found route %s", *route);
 266         /* We have a route in path or in cookie
 267          * Find the worker that has this route defined.
 268          */
 269         worker = find_route_worker(balancer, *route, r);
 270         if (worker && strcmp(*route, worker->s->route)) {
 271             /*
 272              * Notice that the route of the worker chosen is different from
 273              * the route supplied by the client.
 274              */
 275             apr_table_setn(r->subprocess_env, "BALANCER_ROUTE_CHANGED", "1");
 276             ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
 277                          "proxy: BALANCER: Route changed from %s to %s",
 278                          *route, worker->s->route);
 279         }
 280         return worker;
 281     }
 282     else
 283         return NULL;
 284 }
 285
 286 static proxy_worker *find_best_worker(proxy_balancer *balancer,
 287                                       request_rec *r)
 288 {
 289     proxy_worker *candidate = NULL;
 290     apr_status_t rv;
 291
 292     if ((rv = PROXY_THREAD_LOCK(balancer)) != APR_SUCCESS) {
 293         ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
 294         "proxy: BALANCER: (%s). Lock failed for find_best_worker()", balancer->name);
 295         return NULL;
 296     }
 297
 298     candidate = (*balancer->lbmethod->finder)(balancer, r);
 299
 300     if (candidate)
 301         candidate->s->elected++;
 302
 303 /*
 304         PROXY_THREAD_UNLOCK(balancer);
 305         return NULL;
 306 */
 307
 308     if ((rv = PROXY_THREAD_UNLOCK(balancer)) != APR_SUCCESS) {
 309         ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
 310         "proxy: BALANCER: (%s). Unlock failed for find_best_worker()", balancer->name);
 311     }
 312
 313     if (candidate == NULL) {
 314         /* All the workers are in error state or disabled.
 315          * If the balancer has a timeout sleep for a while
 316          * and try again to find the worker. The chances are
 317          * that some other thread will release a connection.
 318          * By default the timeout is not set, and the server
 319          * returns SERVER_BUSY.
 320          */
 321 #if APR_HAS_THREADS
 322         if (balancer->timeout) {
 323             /* XXX: This can perhaps be build using some
 324              * smarter mechanism, like tread_cond.
 325              * But since the statuses can came from
 326              * different childs, use the provided algo.
 327              */
 328             apr_interval_time_t timeout = balancer->timeout;
 329             apr_interval_time_t step, tval = 0;
 330             /* Set the timeout to 0 so that we don't
 331              * end in infinite loop
 332              */
 333             balancer->timeout = 0;
 334             step = timeout / 100;
 335             while (tval < timeout) {
 336                 apr_sleep(step);
 337                 /* Try again */
 338                 if ((candidate = find_best_worker(balancer, r)))
 339                     break;
 340                 tval += step;
 341             }
 342             /* restore the timeout */
 343             balancer->timeout = timeout;
 344         }
 345 #endif
 346     }
 347     return candidate;
 348 }
 349
 350 static int rewrite_url(request_rec *r, proxy_worker *worker,
 351                         char **url)
 352 {
 353     const char *scheme = strstr(*url, "://");
 354     const char *path = NULL;
 355
 356     if (scheme)
 357         path = ap_strchr_c(scheme + 3, '/');
 358
 359     /* we break the URL into host, port, uri */
 360     if (!worker) {
 361         return ap_proxyerror(r, HTTP_BAD_REQUEST, apr_pstrcat(r->pool,
 362                              "missing worker. URI cannot be parsed: ", *url,
 363                              NULL));
 364     }
 365
 366     *url = apr_pstrcat(r->pool, worker->name, path, NULL);
 367
 368     return OK;
 369 }
 370
 371 static void force_recovery(proxy_balancer *balancer, server_rec *s)
 372 {
 373     int i;
 374     int ok = 0;
 375     proxy_worker *worker;
 376
 377     worker = (proxy_worker *)balancer->workers->elts;
 378     for (i = 0; i < balancer->workers->nelts; i++, worker++) {
 379         if (!(worker->s->status & PROXY_WORKER_IN_ERROR)) {
 380             ok = 1;
 381             break;
 382         }
 383     }
 384     if (!ok) {
 385         /* If all workers are in error state force the recovery.
 386          */
 387         worker = (proxy_worker *)balancer->workers->elts;
 388         for (i = 0; i < balancer->workers->nelts; i++, worker++) {
 389             ++worker->s->retries;
 390             worker->s->status &= ~PROXY_WORKER_IN_ERROR;
 391             ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s,
 392                          "proxy: BALANCER: (%s). Forcing recovery for worker (%s)",
 393                          balancer->name, worker->hostname);
 394         }
 395     }
 396 }
 397
 398 static int proxy_balancer_pre_request(proxy_worker **worker,
 399                                       proxy_balancer **balancer,
 400                                       request_rec *r,
 401                                       proxy_server_conf *conf, char **url)
 402 {
 403     int access_status;
 404     proxy_worker *runtime;
 405     char *route = NULL;
 406     apr_status_t rv;
 407
 408     *worker = NULL;
 409     /* Step 1: check if the url is for us
 410      * The url we can handle starts with 'balancer://'
 411      * If balancer is already provided skip the search
 412      * for balancer, because this is failover attempt.
 413      */
 414     if (!*balancer &&
 415         !(*balancer = ap_proxy_get_balancer(r->pool, conf, *url)))
 416         return DECLINED;
 417
 418     /* Step 2: Lock the LoadBalancer
 419      * XXX: perhaps we need the process lock here
 420      */
 421     if ((rv = PROXY_THREAD_LOCK(*balancer)) != APR_SUCCESS) {
 422         ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
 423                      "proxy: BALANCER: (%s). Lock failed for pre_request",
 424                      (*balancer)->name);
 425         return DECLINED;
 426     }
 427
 428     /* Step 3: force recovery */
 429     force_recovery(*balancer, r->server);
 430
 431     /* Step 4: find the session route */
 432     runtime = find_session_route(*balancer, r, &route, url);
 433     if (runtime) {
 434         int i, total_factor = 0;
 435         proxy_worker *workers;
 436         /* We have a sticky load balancer
 437          * Update the workers status
 438          * so that even session routes get
 439          * into account.
 440          */
 441         workers = (proxy_worker *)(*balancer)->workers->elts;
 442         for (i = 0; i < (*balancer)->workers->nelts; i++) {
 443             /* Take into calculation only the workers that are
 444              * not in error state or not disabled.
 445              *
 446              * TODO: Abstract the below, since this is dependent
 447              *       on the LB implementation
 448              */
 449             if (PROXY_WORKER_IS_USABLE(workers)) {
 450                 workers->s->lbstatus += workers->s->lbfactor;
 451                 total_factor += workers->s->lbfactor;
 452             }
 453             workers++;
 454         }
 455         runtime->s->lbstatus -= total_factor;
 456         runtime->s->elected++;
 457
 458         *worker = runtime;
 459     }
 460     else if (route && (*balancer)->sticky_force) {
 461         ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,
 462                      "proxy: BALANCER: (%s). All workers are in error state for route (%s)",
 463                      (*balancer)->name, route);
 464         if ((rv = PROXY_THREAD_UNLOCK(*balancer)) != APR_SUCCESS) {
 465             ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
 466                          "proxy: BALANCER: (%s). Unlock failed for pre_request",
 467                          (*balancer)->name);
 468         }
 469         return HTTP_SERVICE_UNAVAILABLE;
 470     }
 471
 472     if ((rv = PROXY_THREAD_UNLOCK(*balancer)) != APR_SUCCESS) {
 473         ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
 474                      "proxy: BALANCER: (%s). Unlock failed for pre_request",
 475                      (*balancer)->name);
 476     }
 477     if (!*worker) {
 478         runtime = find_best_worker(*balancer, r);
 479         if (!runtime) {
 480             ap_log_error(APLOG_MARK, APLOG_ERR, 0, r->server,
 481                          "proxy: BALANCER: (%s). All workers are in error state",
 482                          (*balancer)->name);
 483
 484             return HTTP_SERVICE_UNAVAILABLE;
 485         }
 486         if ((*balancer)->sticky && runtime) {
 487             /*
 488              * This balancer has sticky sessions and the client either has not
 489              * supplied any routing information or all workers for this route
 490              * including possible redirect and hotstandby workers are in error
 491              * state, but we have found another working worker for this
 492              * balancer where we can send the request. Thus notice that we have
 493              * changed the route to the backend.
 494              */
 495             apr_table_setn(r->subprocess_env, "BALANCER_ROUTE_CHANGED", "1");
 496         }
 497         *worker = runtime;
 498     }
 499
 500     /* Add balancer/worker info to env. */
 501     apr_table_setn(r->subprocess_env,
 502                    "BALANCER_NAME", (*balancer)->name);
 503     apr_table_setn(r->subprocess_env,
 504                    "BALANCER_WORKER_NAME", (*worker)->name);
 505     apr_table_setn(r->subprocess_env,
 506                    "BALANCER_WORKER_ROUTE", (*worker)->s->route);
 507
 508     /* Rewrite the url from 'balancer://url'
 509      * to the 'worker_scheme://worker_hostname[:worker_port]/url'
 510      * This replaces the balancers fictional name with the
 511      * real hostname of the elected worker.
 512      */
 513     access_status = rewrite_url(r, *worker, url);
 514     /* Add the session route to request notes if present */
 515     if (route) {
 516         apr_table_setn(r->notes, "session-sticky", (*balancer)->sticky);
 517         apr_table_setn(r->notes, "session-route", route);
 518
 519         /* Add session info to env. */
 520         apr_table_setn(r->subprocess_env,
 521                        "BALANCER_SESSION_STICKY", (*balancer)->sticky);
 522         apr_table_setn(r->subprocess_env,
 523                        "BALANCER_SESSION_ROUTE", route);
 524     }
 525     ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
 526                  "proxy: BALANCER (%s) worker (%s) rewritten to %s",
 527                  (*balancer)->name, (*worker)->name, *url);
 528
 529     return access_status;
 530 }
 531
 532 static int proxy_balancer_post_request(proxy_worker *worker,
 533                                        proxy_balancer *balancer,
 534                                        request_rec *r,
 535                                        proxy_server_conf *conf)
 536 {
 537
 538 #if 0
 539     apr_status_t rv;
 540
 541     if ((rv = PROXY_THREAD_LOCK(balancer)) != APR_SUCCESS) {
 542         ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
 543             "proxy: BALANCER: (%s). Lock failed for post_request",
 544             balancer->name);
 545         return HTTP_INTERNAL_SERVER_ERROR;
 546     }
 547     /* TODO: placeholder for post_request actions
 548      */
 549
 550     if ((rv = PROXY_THREAD_UNLOCK(balancer)) != APR_SUCCESS) {
 551         ap_log_error(APLOG_MARK, APLOG_ERR, rv, r->server,
 552             "proxy: BALANCER: (%s). Unlock failed for post_request",
 553             balancer->name);
 554     }
 555     ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
 556                  "proxy_balancer_post_request for (%s)", balancer->name);
 557
 558 #endif
 559
 560     return OK;
 561 }
 562
 563 static void recalc_factors(proxy_balancer *balancer)
 564 {
 565     int i;
 566     proxy_worker *workers;
 567
 568
 569     /* Recalculate lbfactors */
 570     workers = (proxy_worker *)balancer->workers->elts;
 571     /* Special case if there is only one worker it's
 572      * load factor will always be 1
 573      */
 574     if (balancer->workers->nelts == 1) {
 575         workers->s->lbstatus = workers->s->lbfactor = 1;
 576         return;
 577     }
 578     for (i = 0; i < balancer->workers->nelts; i++) {
 579         /* Update the status entries */
 580         workers[i].s->lbstatus = workers[i].s->lbfactor;
 581     }
 582 }
 583
 584 /* Manages the loadfactors and member status
 585  */
 586 static int balancer_handler(request_rec *r)
 587 {
 588     void *sconf = r->server->module_config;
 589     proxy_server_conf *conf = (proxy_server_conf *)
 590         ap_get_module_config(sconf, &proxy_module);
 591     proxy_balancer *balancer, *bsel = NULL;
 592     proxy_worker *worker, *wsel = NULL;
 593     apr_table_t *params = apr_table_make(r->pool, 10);
 594     int access_status;
 595     int i, n;
 596     const char *name;
 597
 598     /* is this for us? */
 599     if (strcmp(r->handler, "balancer-manager"))
 600         return DECLINED;
 601     r->allowed = (AP_METHOD_BIT << M_GET);
 602     if (r->method_number != M_GET)
 603         return DECLINED;
 604
 605     if (r->args) {
 606         char *args = apr_pstrdup(r->pool, r->args);
 607         char *tok, *val;
 608         while (args && *args) {
 609             if ((val = ap_strchr(args, '='))) {
 610                 *val++ = '\0';
 611                 if ((tok = ap_strchr(val, '&')))
 612                     *tok++ = '\0';
 613                 /*
 614                  * Special case: workers are allowed path information
 615                  */
 616                 if ((access_status = ap_unescape_url(val)) != OK)
 617                     if (strcmp(args, "w") || (access_status !=  HTTP_NOT_FOUND))
 618                         return access_status;
 619                 apr_table_setn(params, args, val);
 620                 args = tok;
 621             }
 622             else
 623                 return HTTP_BAD_REQUEST;
 624         }
 625     }
 626     if ((name = apr_table_get(params, "b")))
 627         bsel = ap_proxy_get_balancer(r->pool, conf,
 628             apr_pstrcat(r->pool, "balancer://", name, NULL));
 629     if ((name = apr_table_get(params, "w"))) {
 630         proxy_worker *ws;
 631
 632         ws = ap_proxy_get_worker(r->pool, conf, name);
 633         if (ws) {
 634             worker = (proxy_worker *)bsel->workers->elts;
 635             for (n = 0; n < bsel->workers->nelts; n++) {
 636                 if (strcasecmp(worker->name, ws->name) == 0) {
 637                     wsel = worker;
 638                     break;
 639                 }
 640                 ++worker;
 641             }
 642         }
 643     }
 644     /* First set the params */
 645     if (bsel) {
 646         const char *val;
 647         if ((val = apr_table_get(params, "ss"))) {
 648             if (strlen(val))
 649                 bsel->sticky = apr_pstrdup(conf->pool, val);
 650             else
 651                 bsel->sticky = NULL;
 652         }
 653         if ((val = apr_table_get(params, "tm"))) {
 654             int ival = atoi(val);
 655             if (ival >= 0)
 656                 bsel->timeout = apr_time_from_sec(ival);
 657         }
 658         if ((val = apr_table_get(params, "fa"))) {
 659             int ival = atoi(val);
 660             if (ival >= 0)
 661                 bsel->max_attempts = ival;
 662             bsel->max_attempts_set = 1;
 663         }
 664         if ((val = apr_table_get(params, "lm"))) {
 665             proxy_balancer_method *provider;
 666             provider = ap_lookup_provider(PROXY_LBMETHOD, val, "0");
 667             if (provider) {
 668                 bsel->lbmethod = provider;
 669             }
 670         }
 671     }
 672     if (wsel) {
 673         const char *val;
 674         if ((val = apr_table_get(params, "lf"))) {
 675             int ival = atoi(val);
 676             if (ival >= 1 && ival <= 100) {
 677                 wsel->s->lbfactor = ival;
 678                 if (bsel)
 679                     recalc_factors(bsel);
 680             }
 681         }
 682         if ((val = apr_table_get(params, "wr"))) {
 683             if (strlen(val) && strlen(val) < PROXY_WORKER_MAX_ROUTE_SIZ)
 684                 strcpy(wsel->s->route, val);
 685             else
 686                 *wsel->s->route = '\0';
 687         }
 688         if ((val = apr_table_get(params, "rr"))) {
 689             if (strlen(val) && strlen(val) < PROXY_WORKER_MAX_ROUTE_SIZ)
 690                 strcpy(wsel->s->redirect, val);
 691             else
 692                 *wsel->s->redirect = '\0';
 693         }
 694         if ((val = apr_table_get(params, "dw"))) {
 695             if (!strcasecmp(val, "Disable"))
 696                 wsel->s->status |= PROXY_WORKER_DISABLED;
 697             else if (!strcasecmp(val, "Enable"))
 698                 wsel->s->status &= ~PROXY_WORKER_DISABLED;
 699         }
 700         if ((val = apr_table_get(params, "ls"))) {
 701             int ival = atoi(val);
 702             if (ival >= 0 && ival <= 99) {
 703                 wsel->s->lbset = ival;
 704              }
 705         }
 706
 707     }
 708     if (apr_table_get(params, "xml")) {
 709         ap_set_content_type(r, "text/xml");
 710         ap_rputs("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n", r);
 711         ap_rputs("<httpd:manager xmlns:httpd=\"http://httpd.apache.org\">\n", r);
 712         ap_rputs("  <httpd:balancers>\n", r);
 713         balancer = (proxy_balancer *)conf->balancers->elts;
 714         for (i = 0; i < conf->balancers->nelts; i++) {
 715             ap_rputs("    <httpd:balancer>\n", r);
 716             ap_rvputs(r, "      <httpd:name>", balancer->name, "</httpd:name>\n", NULL);
 717             ap_rputs("      <httpd:workers>\n", r);
 718             worker = (proxy_worker *)balancer->workers->elts;
 719             for (n = 0; n < balancer->workers->nelts; n++) {
 720                 ap_rputs("        <httpd:worker>\n", r);
 721                 ap_rvputs(r, "          <httpd:scheme>", worker->scheme,
 722                           "</httpd:scheme>\n", NULL);
 723                 ap_rvputs(r, "          <httpd:hostname>", worker->hostname,
 724                           "</httpd:hostname>\n", NULL);
 725                ap_rprintf(r, "          <httpd:loadfactor>%d</httpd:loadfactor>\n",
 726                           worker->s->lbfactor);
 727                 ap_rputs("        </httpd:worker>\n", r);
 728                 ++worker;
 729             }
 730             ap_rputs("      </httpd:workers>\n", r);
 731             ap_rputs("    </httpd:balancer>\n", r);
 732             ++balancer;
 733         }
 734         ap_rputs("  </httpd:balancers>\n", r);
 735         ap_rputs("</httpd:manager>", r);
 736     }
 737     else {
 738         ap_set_content_type(r, "text/html");
 739         ap_rputs(DOCTYPE_HTML_3_2
 740                  "<html><head><title>Balancer Manager</title></head>\n", r);
 741         ap_rputs("<body><h1>Load Balancer Manager for ", r);
 742         ap_rvputs(r, ap_get_server_name(r), "</h1>\n\n", NULL);
 743         ap_rvputs(r, "<dl><dt>Server Version: ",
 744                   ap_get_server_description(), "</dt>\n", NULL);
 745         ap_rvputs(r, "<dt>Server Built: ",
 746                   ap_get_server_built(), "\n</dt></dl>\n", NULL);
 747         balancer = (proxy_balancer *)conf->balancers->elts;
 748         for (i = 0; i < conf->balancers->nelts; i++) {
 749
 750             ap_rputs("<hr />\n<h3>LoadBalancer Status for ", r);
 751             ap_rvputs(r, "<a href=\"", r->uri, "?b=",
 752                       balancer->name + sizeof("balancer://") - 1,
 753                       "\">", NULL);
 754             ap_rvputs(r, balancer->name, "</a></h3>\n\n", NULL);
 755             ap_rputs("\n\n<table border=\"0\" style=\"text-align: left;\"><tr>"
 756                 "<th>StickySession</th><th>Timeout</th><th>FailoverAttempts</th><th>Method</th>"
 757                 "</tr>\n<tr>", r);
 758             ap_rvputs(r, "<td>", balancer->sticky, NULL);
 759             ap_rprintf(r, "</td><td>%" APR_TIME_T_FMT "</td>",
 760                 apr_time_sec(balancer->timeout));
 761             ap_rprintf(r, "<td>%d</td>\n", balancer->max_attempts);
 762             ap_rprintf(r, "<td>%s</td>\n",
 763                        balancer->lbmethod->name);
 764             ap_rputs("</table>\n<br />", r);
 765             ap_rputs("\n\n<table border=\"0\" style=\"text-align: left;\"><tr>"
 766                 "<th>Worker URL</th>"
 767                 "<th>Route</th><th>RouteRedir</th>"
 768                 "<th>Factor</th><th>Set</th><th>Status</th>"
 769                 "<th>Elected</th><th>To</th><th>From</th>"
 770                 "</tr>\n", r);
 771
 772             worker = (proxy_worker *)balancer->workers->elts;
 773             for (n = 0; n < balancer->workers->nelts; n++) {
 774                 char fbuf[50];
 775                 ap_rvputs(r, "<tr>\n<td><a href=\"", r->uri, "?b=",
 776                           balancer->name + sizeof("balancer://") - 1, "&w=",
 777                           ap_escape_uri(r->pool, worker->name),
 778                           "\">", NULL);
 779                 ap_rvputs(r, worker->name, "</a></td>", NULL);
 780                 ap_rvputs(r, "<td>", worker->s->route, NULL);
 781                 ap_rvputs(r, "</td><td>", worker->s->redirect, NULL);
 782                 ap_rprintf(r, "</td><td>%d</td>", worker->s->lbfactor);
 783                 ap_rprintf(r, "<td>%d</td><td>", worker->s->lbset);
 784                 if (worker->s->status & PROXY_WORKER_DISABLED)
 785                    ap_rputs("Dis ", r);
 786                 if (worker->s->status & PROXY_WORKER_IN_ERROR)
 787                    ap_rputs("Err ", r);
 788                 if (worker->s->status & PROXY_WORKER_STOPPED)
 789                    ap_rputs("Stop ", r);
 790                 if (worker->s->status & PROXY_WORKER_HOT_STANDBY)
 791                    ap_rputs("Stby ", r);
 792                 if (PROXY_WORKER_IS_USABLE(worker))
 793                     ap_rputs("Ok", r);
 794                 if (!PROXY_WORKER_IS_INITIALIZED(worker))
 795                     ap_rputs("-", r);
 796                 ap_rputs("</td>", r);
 797                 ap_rprintf(r, "<td>%" APR_SIZE_T_FMT "</td><td>", worker->s->elected);
 798                 ap_rputs(apr_strfsize(worker->s->transferred, fbuf), r);
 799                 ap_rputs("</td><td>", r);
 800                 ap_rputs(apr_strfsize(worker->s->read, fbuf), r);
 801                 ap_rputs("</td></tr>\n", r);
 802
 803                 ++worker;
 804             }
 805             ap_rputs("</table>\n", r);
 806             ++balancer;
 807         }
 808         ap_rputs("<hr />\n", r);
 809         if (wsel && bsel) {
 810             ap_rputs("<h3>Edit worker settings for ", r);
 811             ap_rvputs(r, wsel->name, "</h3>\n", NULL);
 812             ap_rvputs(r, "<form method=\"GET\" action=\"", NULL);
 813             ap_rvputs(r, r->uri, "\">\n<dl>", NULL);
 814             ap_rputs("<table><tr><td>Load factor:</td><td><input name=\"lf\" type=text ", r);
 815             ap_rprintf(r, "value=\"%d\"></td></tr>\n", wsel->s->lbfactor);
 816             ap_rputs("<tr><td>LB Set:</td><td><input name=\"ls\" type=text ", r);
 817             ap_rprintf(r, "value=\"%d\"></td></tr>\n", wsel->s->lbset);
 818             ap_rputs("<tr><td>Route:</td><td><input name=\"wr\" type=text ", r);
 819             ap_rvputs(r, "value=\"", wsel->s->route, NULL);
 820             ap_rputs("\"></td></tr>\n", r);
 821             ap_rputs("<tr><td>Route Redirect:</td><td><input name=\"rr\" type=text ", r);
 822             ap_rvputs(r, "value=\"", wsel->s->redirect, NULL);
 823             ap_rputs("\"></td></tr>\n", r);
 824             ap_rputs("<tr><td>Status:</td><td>Disabled: <input name=\"dw\" value=\"Disable\" type=radio", r);
 825             if (wsel->s->status & PROXY_WORKER_DISABLED)
 826                 ap_rputs(" checked", r);
 827             ap_rputs("> | Enabled: <input name=\"dw\" value=\"Enable\" type=radio", r);
 828             if (!(wsel->s->status & PROXY_WORKER_DISABLED))
 829                 ap_rputs(" checked", r);
 830             ap_rputs("></td></tr>\n", r);
 831             ap_rputs("<tr><td colspan=2><input type=submit value=\"Submit\"></td></tr>\n", r);
 832             ap_rvputs(r, "</table>\n<input type=hidden name=\"w\" ",  NULL);
 833             ap_rvputs(r, "value=\"", ap_escape_uri(r->pool, wsel->name), "\">\n", NULL);
 834             ap_rvputs(r, "<input type=hidden name=\"b\" ", NULL);
 835             ap_rvputs(r, "value=\"", bsel->name + sizeof("balancer://") - 1,
 836                       "\">\n</form>\n", NULL);
 837             ap_rputs("<hr />\n", r);
 838         }
 839         else if (bsel) {
 840             ap_rputs("<h3>Edit balancer settings for ", r);
 841             ap_rvputs(r, bsel->name, "</h3>\n", NULL);
 842             ap_rvputs(r, "<form method=\"GET\" action=\"", NULL);
 843             ap_rvputs(r, r->uri, "\">\n<dl>", NULL);
 844             ap_rputs("<table><tr><td>StickySession Identifier:</td><td><input name=\"ss\" type=text ", r);
 845             if (bsel->sticky)
 846                 ap_rvputs(r, "value=\"", bsel->sticky, "\"", NULL);
 847             ap_rputs("></td><tr>\n<tr><td>Timeout:</td><td><input name=\"tm\" type=text ", r);
 848             ap_rprintf(r, "value=\"%" APR_TIME_T_FMT "\"></td></tr>\n",
 849                        apr_time_sec(bsel->timeout));
 850             ap_rputs("<tr><td>Failover Attempts:</td><td><input name=\"fa\" type=text ", r);
 851             ap_rprintf(r, "value=\"%d\"></td></tr>\n",
 852                        bsel->max_attempts);
 853             ap_rputs("<tr><td>LB Method:</td><td><select name=\"lm\">", r);
 854             {
 855                 apr_array_header_t *methods;
 856                 ap_list_provider_names_t *method;
 857                 int i;
 858                 methods = ap_list_provider_names(r->pool, PROXY_LBMETHOD, "0");
 859                 method = (ap_list_provider_names_t *)methods->elts;
 860                 for (i = 0; i < methods->nelts; i++) {
 861                     ap_rprintf(r, "<option value=\"%s\" %s>%s</option>", method->provider_name,
 862                        (!strcasecmp(bsel->lbmethod->name, method->provider_name)) ? "selected" : "",
 863                        method->provider_name);
 864                     method++;
 865                 }
 866             }
 867             ap_rputs("</select></td></tr>\n", r);
 868             ap_rputs("<tr><td colspan=2><input type=submit value=\"Submit\"></td></tr>\n", r);
 869             ap_rvputs(r, "</table>\n<input type=hidden name=\"b\" ", NULL);
 870             ap_rvputs(r, "value=\"", bsel->name + sizeof("balancer://") - 1,
 871                       "\">\n</form>\n", NULL);
 872             ap_rputs("<hr />\n", r);
 873         }
 874         ap_rputs(ap_psignature("",r), r);
 875         ap_rputs("</body></html>\n", r);
 876     }
 877     return OK;
 878 }
 879
 880 static void child_init(apr_pool_t *p, server_rec *s)
 881 {
 882     while (s) {
 883         void *sconf = s->module_config;
 884         proxy_server_conf *conf;
 885         proxy_balancer *balancer;
 886         int i;
 887         conf = (proxy_server_conf *)ap_get_module_config(sconf, &proxy_module);
 888
 889         /* Initialize shared scoreboard data */
 890         balancer = (proxy_balancer *)conf->balancers->elts;
 891         for (i = 0; i < conf->balancers->nelts; i++) {
 892             init_balancer_members(conf, s, balancer);
 893             balancer++;
 894         }
 895         s = s->next;
 896     }
 897
 898 }
 899
 900 /*
 901  * The idea behind the find_best_byrequests scheduler is the following:
 902  *
 903  * lbfactor is "how much we expect this worker to work", or "the worker's
 904  * normalized work quota".
 905  *
 906  * lbstatus is "how urgent this worker has to work to fulfill its quota
 907  * of work".
 908  *
 909  * We distribute each worker's work quota to the worker, and then look
 910  * which of them needs to work most urgently (biggest lbstatus).  This
 911  * worker is then selected for work, and its lbstatus reduced by the
 912  * total work quota we distributed to all workers.  Thus the sum of all
 913  * lbstatus does not change.(*)
 914  *
 915  * If some workers are disabled, the others will
 916  * still be scheduled correctly.
 917  *
 918  * If a balancer is configured as follows:
 919  *
 920  * worker     a    b    c    d
 921  * lbfactor  25   25   25   25
 922  *
 923  * And b gets disabled, the following schedule is produced:
 924  *
 925  *    a c d a c d a c d ...
 926  *
 927  * Note that the above lbfactor setting is the *exact* same as:
 928  *
 929  * worker     a    b    c    d
 930  * lbfactor   1    1    1    1
 931  *
 932  * Asymmetric configurations work as one would expect. For
 933  * example:
 934  *
 935  * worker     a    b    c    d
 936  * lbfactor   1    1    1    2
 937  *
 938  * would have a, b and c all handling about the same
 939  * amount of load with d handling twice what a or b
 940  * or c handles individually. So we could see:
 941  *
 942  *   b a d c d a c d b d ...
 943  *
 944  */
 945
 946 static proxy_worker *find_best_byrequests(proxy_balancer *balancer,
 947                                 request_rec *r)
 948 {
 949     int i;
 950     int total_factor = 0;
 951     proxy_worker *worker;
 952     proxy_worker *mycandidate = NULL;
 953     int cur_lbset = 0;
 954     int max_lbset = 0;
 955     int checking_standby;
 956     int checked_standby;
 957
 958     ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
 959                  "proxy: Entering byrequests for BALANCER (%s)",
 960                  balancer->name);
 961
 962     /* First try to see if we have available candidate */
 963     do {
 964         checking_standby = checked_standby = 0;
 965         while (!mycandidate && !checked_standby) {
 966             worker = (proxy_worker *)balancer->workers->elts;
 967             for (i = 0; i < balancer->workers->nelts; i++, worker++) {
 968                 if (!checking_standby) {    /* first time through */
 969                     if (worker->s->lbset > max_lbset)
 970                         max_lbset = worker->s->lbset;
 971                 }
 972                 if (worker->s->lbset > cur_lbset)
 973                     continue;
 974                 if ( (checking_standby ? !PROXY_WORKER_IS_STANDBY(worker) : PROXY_WORKER_IS_STANDBY(worker)) )
 975                     continue;
 976                 /* If the worker is in error state run
 977                  * retry on that worker. It will be marked as
 978                  * operational if the retry timeout is elapsed.
 979                  * The worker might still be unusable, but we try
 980                  * anyway.
 981                  */
 982                 if (!PROXY_WORKER_IS_USABLE(worker))
 983                     ap_proxy_retry_worker("BALANCER", worker, r->server);
 984                 /* Take into calculation only the workers that are
 985                  * not in error state or not disabled.
 986                  */
 987                 if (PROXY_WORKER_IS_USABLE(worker)) {
 988                     worker->s->lbstatus += worker->s->lbfactor;
 989                     total_factor += worker->s->lbfactor;
 990                     if (!mycandidate || worker->s->lbstatus > mycandidate->s->lbstatus)
 991                         mycandidate = worker;
 992                 }
 993             }
 994             checked_standby = checking_standby++;
 995         }
 996         cur_lbset++;
 997     } while (cur_lbset <= max_lbset && !mycandidate);
 998
 999     if (mycandidate) {
1000         mycandidate->s->lbstatus -= total_factor;
1001     }
1002
1003     return mycandidate;
1004 }
1005
1006 /*
1007  * The idea behind the find_best_bytraffic scheduler is the following:
1008  *
1009  * We know the amount of traffic (bytes in and out) handled by each
1010  * worker. We normalize that traffic by each workers' weight. So assuming
1011  * a setup as below:
1012  *
1013  * worker     a    b    c
1014  * lbfactor   1    1    3
1015  *
1016  * the scheduler will allow worker c to handle 3 times the
1017  * traffic of a and b. If each request/response results in the
1018  * same amount of traffic, then c would be accessed 3 times as
1019  * often as a or b. If, for example, a handled a request that
1020  * resulted in a large i/o bytecount, then b and c would be
1021  * chosen more often, to even things out.
1022  */
1023 static proxy_worker *find_best_bytraffic(proxy_balancer *balancer,
1024                                          request_rec *r)
1025 {
1026     int i;
1027     apr_off_t mytraffic = 0;
1028     apr_off_t curmin = 0;
1029     proxy_worker *worker;
1030     proxy_worker *mycandidate = NULL;
1031     int cur_lbset = 0;
1032     int max_lbset = 0;
1033     int checking_standby;
1034     int checked_standby;
1035
1036     ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, r->server,
1037                  "proxy: Entering bytraffic for BALANCER (%s)",
1038                  balancer->name);
1039
1040     /* First try to see if we have available candidate */
1041     do {
1042         checking_standby = checked_standby = 0;
1043         while (!mycandidate && !checked_standby) {
1044             worker = (proxy_worker *)balancer->workers->elts;
1045             for (i = 0; i < balancer->workers->nelts; i++, worker++) {
1046                 if (!checking_standby) {    /* first time through */
1047                     if (worker->s->lbset > max_lbset)
1048                         max_lbset = worker->s->lbset;
1049                 }
1050                 if (worker->s->lbset > cur_lbset)
1051                     continue;
1052                 if ( (checking_standby ? !PROXY_WORKER_IS_STANDBY(worker) : PROXY_WORKER_IS_STANDBY(worker)) )
1053                     continue;
1054                 /* If the worker is in error state run
1055                  * retry on that worker. It will be marked as
1056                  * operational if the retry timeout is elapsed.
1057                  * The worker might still be unusable, but we try
1058                  * anyway.
1059                  */
1060                 if (!PROXY_WORKER_IS_USABLE(worker))
1061                     ap_proxy_retry_worker("BALANCER", worker, r->server);
1062                 /* Take into calculation only the workers that are
1063                  * not in error state or not disabled.
1064                  */
1065                 if (PROXY_WORKER_IS_USABLE(worker)) {
1066                     mytraffic = (worker->s->transferred/worker->s->lbfactor) +
1067                                 (worker->s->read/worker->s->lbfactor);
1068                     if (!mycandidate || mytraffic < curmin) {
1069                         mycandidate = worker;
1070                         curmin = mytraffic;
1071                     }
1072                 }
1073             }
1074             checked_standby = checking_standby++;
1075         }
1076         cur_lbset++;
1077     } while (cur_lbset <= max_lbset && !mycandidate);
1078
1079     return mycandidate;
1080 }
1081
1082 /*
1083  * How to add additional lbmethods:
1084  *   1. Create func which determines "best" candidate worker
1085  *      (eg: find_best_bytraffic, above)
1086  *   2. Register it as a provider.
1087  */
1088 static const proxy_balancer_method byrequests =
1089 {
1090     "byrequests",
1091     &find_best_byrequests,
1092     NULL
1093 };
1094
1095 static const proxy_balancer_method bytraffic =
1096 {
1097     "bytraffic",
1098     &find_best_bytraffic,
1099     NULL
1100 };
1101
1102 static void ap_proxy_balancer_register_hook(apr_pool_t *p)
1103 {
1104     /* Only the mpm_winnt has child init hook handler.
1105      * make sure that we are called after the mpm
1106      * initializes and after the mod_proxy
1107      */
1108     static const char *const aszPred[] = { "mpm_winnt.c", "mod_proxy.c", NULL};
1109      /* manager handler */
1110     ap_hook_handler(balancer_handler, NULL, NULL, APR_HOOK_FIRST);
1111     ap_hook_child_init(child_init, aszPred, NULL, APR_HOOK_MIDDLE);
1112     proxy_hook_pre_request(proxy_balancer_pre_request, NULL, NULL, APR_HOOK_FIRST);
1113     proxy_hook_post_request(proxy_balancer_post_request, NULL, NULL, APR_HOOK_FIRST);
1114     proxy_hook_canon_handler(proxy_balancer_canon, NULL, NULL, APR_HOOK_FIRST);
1115     ap_register_provider(p, PROXY_LBMETHOD, "bytraffic", "0", &bytraffic);
1116     ap_register_provider(p, PROXY_LBMETHOD, "byrequests", "0", &byrequests);
1117 }
1118
1119 module AP_MODULE_DECLARE_DATA proxy_balancer_module = {
1120     STANDARD20_MODULE_STUFF,
1121     NULL,       /* create per-directory config structure */
1122     NULL,       /* merge per-directory config structures */
1123     NULL,       /* create per-server config structure */
1124     NULL,       /* merge per-server config structures */
1125     NULL,       /* command apr_table_t */
1126     ap_proxy_balancer_register_hook /* register hooks */
1127 };