From 93d3db6999becf39536df05d70388ffed92e40e7 Mon Sep 17 00:00:00 2001 From: Graham Leggett Date: Sat, 15 Nov 2008 15:49:28 +0000 Subject: [PATCH] mod_buffer: Optional support for buffering of the input and output filter stacks. Can collapse many small buckets into fewer larger buckets, and prevents excessively small chunks being sent over the wire. git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@717867 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 5 + docs/manual/mod/mod_buffer.xml | 83 ++++++++ docs/manual/mod/mod_buffer.xml.meta | 12 ++ modules/filters/config.m4 | 1 + modules/filters/mod_buffer.c | 288 ++++++++++++++++++++++++++++ 5 files changed, 389 insertions(+) create mode 100644 docs/manual/mod/mod_buffer.xml create mode 100644 docs/manual/mod/mod_buffer.xml.meta create mode 100644 modules/filters/mod_buffer.c diff --git a/CHANGES b/CHANGES index 5bb889f165..a38639bebd 100644 --- a/CHANGES +++ b/CHANGES @@ -2,6 +2,11 @@ Changes with Apache 2.3.0 [ When backported to 2.2.x, remove entry from this file ] + *) mod_buffer: Optional support for buffering of the input and output + filter stacks. Can collapse many small buckets into fewer larger + buckets, and prevents excessively small chunks being sent over + the wire. [Graham Leggett] + *) mod_privileges: new module to make httpd on Solaris privileges-aware and to enable different virtualhosts to run with different privileges and Unix user/group IDs [Nick Kew] diff --git a/docs/manual/mod/mod_buffer.xml b/docs/manual/mod/mod_buffer.xml new file mode 100644 index 0000000000..4c85668950 --- /dev/null +++ b/docs/manual/mod/mod_buffer.xml @@ -0,0 +1,83 @@ + + + + + + + + + +mod_buffer +Support for request buffering +Extension +mod_buffer.c +buffer_module +Available in Apache 2.3 and later + + +

This module provides the ability to buffer the input and output + filter stacks.

+ +

Under certain circumstances, content generators might create + content in small chunks. In order to promote memory reuse, in + memory chunks are always 8k in size, regardless of the size of the + chunk itself. When many small chunks are generated by a request, + this can create a large memory footprint while the request is + being processed, and an unnecessarily large amount of data on the + wire. The addition of a buffer collapses the response into the + fewest chunks possible.

+ +

When httpd is used in front of an expensive content generator, + buffering the response may allow the backend to complete + processing and release resources sooner, depending on how the + backend is designed.

+ +

The buffer filter may be added to either the input or the + output filter stacks, as appropriate, using the + SetInputFilter, + SetOutputFilter, + AddOutputFilter or + AddOutputFilterByType directives.

+ + Using buffer with mod_include + AddOutputFilterByType INCLUDES;BUFFER text/html
+ + + +Filters + + +BufferSize +Maximum size in bytes to buffer by the buffer filter +BufferSize integer +BufferSize 131072 +server config +virtual host +directory +.htaccess + + + +

The BufferSize + directive specifies the amount of data in bytes that will be + buffered before being read from or written to each request. + The default is 128 kilobytes.

+ + + + diff --git a/docs/manual/mod/mod_buffer.xml.meta b/docs/manual/mod/mod_buffer.xml.meta new file mode 100644 index 0000000000..8b2eb8efd8 --- /dev/null +++ b/docs/manual/mod/mod_buffer.xml.meta @@ -0,0 +1,12 @@ + + + + + mod_buffer + /mod/ + .. + + + en + + diff --git a/modules/filters/config.m4 b/modules/filters/config.m4 index af334dd1c6..1db984de84 100644 --- a/modules/filters/config.m4 +++ b/modules/filters/config.m4 @@ -4,6 +4,7 @@ dnl APACHE_MODULE(name, helptext[, objects[, structname[, default[, config]]]]) APACHE_MODPATH_INIT(filters) +APACHE_MODULE(buffer, Filter Buffering, , , yes) APACHE_MODULE(ext_filter, external filter module, , , most) APACHE_MODULE(request, Request Body Filtering, , , yes) APACHE_MODULE(include, Server Side Includes, , , yes) diff --git a/modules/filters/mod_buffer.c b/modules/filters/mod_buffer.c new file mode 100644 index 0000000000..f7231dad4d --- /dev/null +++ b/modules/filters/mod_buffer.c @@ -0,0 +1,288 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * mod_buffer.c --- Buffer the input and output filter stacks, collapse + * many small buckets into fewer large buckets. + */ + +#include "apr.h" +#include "apr_strings.h" +#include "apr_buckets.h" +#include "apr_lib.h" + +#include "ap_config.h" +#include "util_filter.h" +#include "httpd.h" +#include "http_config.h" +#include "http_log.h" +#include "http_request.h" + +static const char bufferFilterName[] = "BUFFER"; +module AP_MODULE_DECLARE_DATA buffer_module; + +#define DEFAULT_BUFFER_SIZE 128*1024 + +typedef struct buffer_conf { + apr_off_t size; /* size of the buffer */ + int size_set; /* has the size been set */ +} buffer_conf; + +typedef struct buffer_ctx { + apr_bucket_brigade *bb; + buffer_conf *conf; +} buffer_ctx; + +/** + * Buffer buckets being written to the output filter stack. + */ +static apr_status_t buffer_out_filter(ap_filter_t *f, apr_bucket_brigade *bb) { + apr_bucket *e; + request_rec *r = f->r; + buffer_ctx *ctx = f->ctx; + apr_status_t rv = APR_SUCCESS; + + /* first time in? create a context */ + if (!ctx) { + + /* buffering won't work on subrequests, it would be nice if + * it did. Within subrequests, we have no EOS to check for, + * so we don't know when to flush the buffer to the network + */ + if (!ap_is_initial_req(f->r)) { + ap_remove_output_filter(f); + return ap_pass_brigade(f->next, bb); + } + + ctx = f->ctx = apr_pcalloc(r->pool, sizeof(*ctx)); + ctx->bb = apr_brigade_create(r->pool, f->c->bucket_alloc); + ctx->conf = ap_get_module_config(f->r->per_dir_config, &buffer_module); + + } + + /* Do nothing if asked to filter nothing. */ + if (APR_BRIGADE_EMPTY(bb)) { + return ap_pass_brigade(f->next, bb); + } + + while (APR_SUCCESS == rv && !APR_BRIGADE_EMPTY(bb)) { + const char *data; + apr_off_t len; + apr_size_t size; + + e = APR_BRIGADE_FIRST(bb); + + /* EOS means we are done. */ + if (APR_BUCKET_IS_EOS(e)) { + + /* should we add an etag? */ + + /* pass the EOS across */ + APR_BUCKET_REMOVE(e); + APR_BRIGADE_INSERT_TAIL(ctx->bb, e); + + /* pass what we have down the chain */ + rv = ap_pass_brigade(f->next, ctx->bb); + continue; + } + + /* A flush takes precedence over buffering */ + if (APR_BUCKET_IS_FLUSH(e)) { + + /* pass the flush bucket across */ + APR_BUCKET_REMOVE(e); + APR_BRIGADE_INSERT_TAIL(ctx->bb, e); + + /* pass what we have down the chain */ + rv = ap_pass_brigade(f->next, ctx->bb); + continue; + } + + /* metadata buckets are preserved as is */ + if (APR_BUCKET_IS_METADATA(e)) { + /* + * Remove meta data bucket from old brigade and insert into the + * new. + */ + APR_BUCKET_REMOVE(e); + APR_BRIGADE_INSERT_TAIL(ctx->bb, e); + continue; + } + + /* is our buffer full? + * If so, send what we have down the filter chain. If the buffer + * gets full, we can no longer compute a content length. + */ + apr_brigade_length(ctx->bb, 1, &len); + if (len > ctx->conf->size) { + + /* pass what we have down the chain */ + rv = ap_pass_brigade(f->next, ctx->bb); + } + + /* at this point we are ready to buffer. + * Buffering takes advantage of an optimisation in the handling of + * bucket brigades. Heap buckets are always created at a fixed + * size, regardless of the size of the data placed into them. + * The apr_brigade_write() call will first try and pack the data + * into any free space in the most recent heap bucket, before + * allocating a new bucket if necessary. + */ + if (APR_SUCCESS == (rv = apr_bucket_read(e, &data, &size, + APR_BLOCK_READ))) { + apr_brigade_write(ctx->bb, NULL, NULL, data, size); + } + + apr_bucket_delete(e); + } + + return rv; + +} + +/** + * Buffer buckets being read from the input filter stack. + */ +static apr_status_t buffer_in_filter(ap_filter_t *f, apr_bucket_brigade *bb, + ap_input_mode_t mode, apr_read_type_e block, apr_off_t readbytes) { + apr_bucket *e; + apr_bucket_brigade *tmp; + apr_status_t rv; + buffer_conf *c; + apr_off_t remaining; + + /* buffer on main requests only */ + if (!ap_is_initial_req(f->r)) { + ap_remove_input_filter(f); + return ap_get_brigade(f->next, bb, mode, block, readbytes); + } + + /* just get out of the way of things we don't want. */ + if (mode != AP_MODE_READBYTES) { + return ap_get_brigade(f->next, bb, mode, block, readbytes); + } + + c = ap_get_module_config(f->r->per_dir_config, &buffer_module); + + tmp = apr_brigade_create(f->r->pool, f->c->bucket_alloc); + + remaining = readbytes; + while (remaining > 0) { + const char *data; + apr_off_t len; + apr_size_t size; + + rv = ap_get_brigade(f->next, tmp, mode, block, remaining); + + /* if an error was received, bail out now */ + if (rv != APR_SUCCESS) { + APR_BRIGADE_CONCAT(bb, tmp); + return rv; + } + + apr_brigade_length(tmp, 1, &len); + remaining -= len; + + for (e = APR_BRIGADE_FIRST(tmp); e != APR_BRIGADE_SENTINEL(tmp); e + = APR_BUCKET_NEXT(e)) { + + /* if we see an EOS, we are done */ + if (APR_BUCKET_IS_EOS(e)) { + APR_BUCKET_REMOVE(e); + APR_BRIGADE_INSERT_TAIL(bb, e); + remaining = 0; + break; + } + + /* pass flush buckets through */ + if (APR_BUCKET_IS_FLUSH(e)) { + APR_BUCKET_REMOVE(e); + APR_BRIGADE_INSERT_TAIL(bb, e); + continue; + } + + /* pass metadata buckets through */ + if (APR_BUCKET_IS_METADATA(e)) { + APR_BUCKET_REMOVE(e); + APR_BRIGADE_INSERT_TAIL(bb, e); + continue; + } + + /* read */ + if (APR_SUCCESS == (rv = apr_bucket_read(e, &data, &size, + APR_BLOCK_READ))) { + apr_brigade_write(bb, NULL, NULL, data, size); + } + + } + apr_brigade_cleanup(tmp); + } + + return APR_SUCCESS; +} + +static void *create_buffer_config(apr_pool_t *p, char *dummy) { + buffer_conf *new = (buffer_conf *) apr_pcalloc(p, sizeof(buffer_conf)); + + new->size_set = 0; /* unset */ + new->size = DEFAULT_BUFFER_SIZE; /* default size */ + + return (void *) new; +} + +static void *merge_buffer_config(apr_pool_t *p, void *basev, void *addv) { + buffer_conf *new = (buffer_conf *) apr_pcalloc(p, sizeof(buffer_conf)); + buffer_conf *add = (buffer_conf *) addv; + buffer_conf *base = (buffer_conf *) basev; + + new->size = (add->size_set == 0) ? base->size : add->size; + new->size_set = add->size_set || base->size_set; + + return new; +} + +static const char *set_buffer_size(cmd_parms *cmd, void *dconf, const char *arg) { + buffer_conf *conf = dconf; + + if (APR_SUCCESS != apr_strtoff(&(conf->size), arg, NULL, 0) || conf->size + <= 0) { + return "BufferSize must be a size in bytes, and greater than zero"; + } + conf->size_set = 1; + + return NULL; +} + +static const command_rec buffer_cmds[] = { AP_INIT_TAKE1("BufferSize", + set_buffer_size, NULL, ACCESS_CONF, + "Maximum size of the buffer used by the buffer filter"), { NULL } }; + +static void register_hooks(apr_pool_t *p) { + ap_register_output_filter(bufferFilterName, buffer_out_filter, NULL, + AP_FTYPE_CONTENT_SET); + ap_register_input_filter(bufferFilterName, buffer_in_filter, NULL, + AP_FTYPE_CONTENT_SET); +} + +module AP_MODULE_DECLARE_DATA buffer_module = { + STANDARD20_MODULE_STUFF, + create_buffer_config, /* create per-directory config structure */ + merge_buffer_config, /* merge per-directory config structures */ + NULL, /* create per-server config structure */ + NULL, /* merge per-server config structures */ + buffer_cmds, /* command apr_table_t */ + register_hooks /* register hooks */ +}; -- 2.40.0