From: Stefan Fritsch Date: Sat, 13 Aug 2011 09:06:35 +0000 (+0000) Subject: Add ap_varbuf API for resizable buffers. X-Git-Tag: 2.3.15~388 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=84a104bb220cfa71edd29a705958dc64f37f316e;p=apache Add ap_varbuf API for resizable buffers. Increase length limit of lines in the configuration file to 16MB. Increase length limit of lines in the group file to 16MB. PR: 45888, 50824, 43084 Windows and Netware build changes are untested. git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1157354 13f79535-47bb-0310-9956-ffa450edef68 --- diff --git a/CHANGES b/CHANGES index 3c7c6126ed..1297f0e50a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,14 @@ -*- coding: utf-8 -*- Changes with Apache 2.3.15 + *) mod_authz_groupfile: Increase length limit of lines in the group file to + 16MB. PR 43084. [Stefan Fritsch] + + *) core: Increase length limit of lines in the configuration file to 16MB. + PR 45888. PR 50824. [Stefan Fritsch] + + *) core: Add API for resizable buffers. [Stefan Fritsch] + *) mod_ldap: Enable LDAPConnectionTimeout for LDAP toolkits that have LDAP_OPT_CONNECT_TIMEOUT instead of LDAP_OPT_NETWORK_TIMEOUT, such as Tivoli Directory Server 6.3 and later. [Eric Covener] diff --git a/build/nw_export.inc b/build/nw_export.inc index f1526150f8..0d448ec77a 100644 --- a/build/nw_export.inc +++ b/build/nw_export.inc @@ -56,6 +56,7 @@ #include "util_mutex.h" #include "util_script.h" #include "util_time.h" +#include "util_varbuf.h" #include "util_xml.h" #include "mod_core.h" diff --git a/docs/manual/configuring.xml b/docs/manual/configuring.xml index 4b6c9ded2b..aa5630362e 100644 --- a/docs/manual/configuring.xml +++ b/docs/manual/configuring.xml @@ -96,10 +96,6 @@ Server.

module="mod_env">SetEnv, take effect too late to be used for expansions in the configuration file.

-

The maximum length of a line in the configuration file, after - variable substitution, joining any continued lines and removing leading - and trailing white space, is 8192 characters.

-

You can check your configuration files for syntax errors without starting the server by using apachectl configtest or the -t command line diff --git a/include/ap_mmn.h b/include/ap_mmn.h index 565e4ea47c..b892e0e48b 100644 --- a/include/ap_mmn.h +++ b/include/ap_mmn.h @@ -347,6 +347,7 @@ * core_dir_config and htaccess_result * 20110724.1 (2.3.15-dev) add NOT_IN_HTACCESS * 20110724.2 (2.3.15-dev) retries and retry_delay in util_ldap_state_t + * 20110724.3 (2.3.15-dev) add util_varbuf.h / ap_varbuf API */ #define MODULE_MAGIC_COOKIE 0x41503234UL /* "AP24" */ @@ -354,7 +355,7 @@ #ifndef MODULE_MAGIC_NUMBER_MAJOR #define MODULE_MAGIC_NUMBER_MAJOR 20110724 #endif -#define MODULE_MAGIC_NUMBER_MINOR 2 /* 0...n */ +#define MODULE_MAGIC_NUMBER_MINOR 3 /* 0...n */ /** * Determine if the server's current MODULE_MAGIC_NUMBER is at least a diff --git a/libhttpd.dsp b/libhttpd.dsp index 6e656d084d..d7ff1a2687 100644 --- a/libhttpd.dsp +++ b/libhttpd.dsp @@ -607,6 +607,10 @@ SOURCE=.\server\util_time.c # End Source File # Begin Source File +SOURCE=.\include\util_varbuf.h +# End Source File +# Begin Source File + SOURCE=.\os\win32\util_win32.c # End Source File # Begin Source File diff --git a/modules/aaa/mod_authz_groupfile.c b/modules/aaa/mod_authz_groupfile.c index 7da27a455c..94128f19bc 100644 --- a/modules/aaa/mod_authz_groupfile.c +++ b/modules/aaa/mod_authz_groupfile.c @@ -52,6 +52,7 @@ #include "http_log.h" #include "http_protocol.h" #include "http_request.h" +#include "util_varbuf.h" #include "mod_auth.h" @@ -90,13 +91,15 @@ static const command_rec authz_groupfile_cmds[] = module AP_MODULE_DECLARE_DATA authz_groupfile_module; +#define VARBUF_INIT_LEN 512 +#define VARBUF_MAX_LEN (16*1024*1024) static apr_status_t groups_for_user(apr_pool_t *p, char *user, char *grpfile, apr_table_t ** out) { ap_configfile_t *f; apr_table_t *grps = apr_table_make(p, 15); apr_pool_t *sp; - char l[MAX_STRING_LEN]; + struct ap_varbuf vb; const char *group_name, *ll, *w; apr_status_t status; apr_size_t group_len; @@ -106,12 +109,13 @@ static apr_status_t groups_for_user(apr_pool_t *p, char *user, char *grpfile, } apr_pool_create(&sp, p); + ap_varbuf_init(p, &vb, VARBUF_INIT_LEN); - while (!(ap_cfg_getline(l, MAX_STRING_LEN, f))) { - if ((l[0] == '#') || (!l[0])) { + while (!(ap_varbuf_cfg_getline(&vb, f, VARBUF_MAX_LEN))) { + if ((vb.buf[0] == '#') || (!vb.buf[0])) { continue; } - ll = l; + ll = vb.buf; apr_pool_clear(sp); group_name = ap_getword(sp, &ll, ':'); @@ -132,6 +136,7 @@ static apr_status_t groups_for_user(apr_pool_t *p, char *user, char *grpfile, } ap_cfg_closefile(f); apr_pool_destroy(sp); + ap_varbuf_free(&vb); *out = grps; return APR_SUCCESS; diff --git a/server/config.c b/server/config.c index 2239352576..1068b1f9a2 100644 --- a/server/config.c +++ b/server/config.c @@ -49,6 +49,7 @@ #include "http_main.h" #include "http_vhost.h" #include "util_cfgtree.h" +#include "util_varbuf.h" #include "mpm_common.h" #define APLOG_UNSET (APLOG_NO_MODULE - 1) @@ -1186,6 +1187,9 @@ static const char *ap_build_config_sub(apr_pool_t *p, apr_pool_t *temp_pool, return retval; } +#define VARBUF_INIT_LEN 200 +#define VARBUF_MAX_LEN (16*1024*1024) + AP_DECLARE(const char *) ap_build_cont_config(apr_pool_t *p, apr_pool_t *temp_pool, cmd_parms *parms, @@ -1193,27 +1197,23 @@ AP_DECLARE(const char *) ap_build_cont_config(apr_pool_t *p, ap_directive_t **curr_parent, char *orig_directive) { - char *l; char *bracket; const char *retval; ap_directive_t *sub_tree = NULL; apr_status_t rc; - - /* Since this function can be called recursively, allocate - * the temporary 8k string buffer from the temp_pool rather - * than the stack to avoid over-running a fixed length stack. - */ - l = apr_palloc(temp_pool, MAX_STRING_LEN); + struct ap_varbuf vb; bracket = apr_pstrcat(temp_pool, orig_directive + 1, ">", NULL); - while ((rc = ap_cfg_getline(l, MAX_STRING_LEN, parms->config_file)) + ap_varbuf_init(temp_pool, &vb, VARBUF_INIT_LEN); + + while ((rc = ap_varbuf_cfg_getline(&vb, parms->config_file, VARBUF_MAX_LEN)) == APR_SUCCESS) { - if (!memcmp(l, "config_file, rc); @@ -1319,10 +1320,12 @@ AP_DECLARE(const char *) ap_build_config(cmd_parms *parms, { ap_directive_t *current = *conftree; ap_directive_t *curr_parent = NULL; - char *l = apr_palloc (temp_pool, MAX_STRING_LEN); const char *errmsg; ap_directive_t **last_ptr = NULL; apr_status_t rc; + struct ap_varbuf vb; + + ap_varbuf_init(temp_pool, &vb, VARBUF_INIT_LEN); if (current != NULL) { /* If we have to traverse the whole tree again for every included @@ -1346,9 +1349,9 @@ AP_DECLARE(const char *) ap_build_config(cmd_parms *parms, } } - while ((rc = ap_cfg_getline(l, MAX_STRING_LEN, parms->config_file)) + while ((rc = ap_varbuf_cfg_getline(&vb, parms->config_file, VARBUF_MAX_LEN)) == APR_SUCCESS) { - errmsg = ap_build_config_sub(p, temp_pool, l, parms, + errmsg = ap_build_config_sub(p, temp_pool, vb.buf, parms, ¤t, &curr_parent, conftree); if (errmsg != NULL) return errmsg; @@ -1361,6 +1364,7 @@ AP_DECLARE(const char *) ap_build_config(cmd_parms *parms, *conftree = current; } } + ap_varbuf_free(&vb); if (rc != APR_EOF && rc != APR_SUCCESS) return ap_pcfg_strerror(temp_pool, parms->config_file, rc); @@ -1532,17 +1536,19 @@ AP_DECLARE(char *) ap_server_root_relative(apr_pool_t *p, const char *file) AP_DECLARE(const char *) ap_soak_end_container(cmd_parms *cmd, char *directive) { - char l[MAX_STRING_LEN]; + struct ap_varbuf vb; const char *args; char *cmd_name; apr_status_t rc; - while((rc = ap_cfg_getline(l, MAX_STRING_LEN, cmd->config_file)) + ap_varbuf_init(cmd->temp_pool, &vb, VARBUF_INIT_LEN); + + while((rc = ap_varbuf_cfg_getline(&vb, cmd->config_file, VARBUF_MAX_LEN)) == APR_SUCCESS) { #if RESOLVE_ENV_PER_TOKEN - args = l; + args = vb.buf; #else - args = ap_resolve_env(cmd->temp_pool, l); + args = ap_resolve_env(cmd->temp_pool, vb.buf); #endif cmd_name = ap_getword_conf(cmd->temp_pool, &args); @@ -1556,6 +1562,7 @@ AP_DECLARE(const char *) ap_soak_end_container(cmd_parms *cmd, char *directive) cmd_name, ">", NULL); } + ap_varbuf_free(&vb); return NULL; /* found end of container */ } else { diff --git a/server/util.c b/server/util.c index 8e1a6209ae..9e20d37bba 100644 --- a/server/util.c +++ b/server/util.c @@ -54,6 +54,7 @@ #include "http_config.h" #include "http_core.h" #include "util_ebcdic.h" +#include "util_varbuf.h" #ifdef HAVE_PWD_H #include @@ -907,10 +908,10 @@ AP_DECLARE(const char *) ap_pcfg_strerror(apr_pool_t *p, ap_configfile_t *cfp, /* Read one line from open ap_configfile_t, strip LF, increase line number */ /* If custom handler does not define a getstr() function, read char by char */ -AP_DECLARE(apr_status_t) ap_cfg_getline(char *buf, size_t bufsize, ap_configfile_t *cfp) +static apr_status_t ap_cfg_getline_core(char *buf, size_t bufsize, + ap_configfile_t *cfp) { apr_status_t rc; - char *src, *dst; /* If a "get string" function is defined, use it */ if (cfp->getstr != NULL) { char *cp; @@ -922,12 +923,12 @@ AP_DECLARE(apr_status_t) ap_cfg_getline(char *buf, size_t bufsize, ap_configfile rc = cfp->getstr(cbuf, cbufsize, cfp->param); if (rc == APR_EOF) { if (cbuf != buf) { - *cbuf = '\0'; + *cbuf = '\0'; break; - } + } else { return APR_EOF; - } + } } if (rc != APR_SUCCESS) { return rc; @@ -997,27 +998,69 @@ AP_DECLARE(apr_status_t) ap_cfg_getline(char *buf, size_t bufsize, ap_configfile buf[i] = c; ++i; } - buf[i] = '\0'; + buf[i] = '\0'; } + return APR_SUCCESS; +} +static int cfg_trim_line(char *buf) +{ + char *start, *end; /* * Leading and trailing white space is eliminated completely */ - src = buf; - while (apr_isspace(*src)) - ++src; + start = buf; + while (apr_isspace(*start)) + ++start; /* blast trailing whitespace */ - dst = &src[strlen(src)]; - while (--dst >= src && apr_isspace(*dst)) - *dst = '\0'; + end = &start[strlen(start)]; + while (--end >= start && apr_isspace(*end)) + *end = '\0'; /* Zap leading whitespace by shifting */ - if (src != buf) - memmove(buf, src, dst - src + 2); - + if (start != buf) + memmove(buf, start, end - start + 2); #ifdef DEBUG_CFG_LINES ap_log_error(APLOG_MARK, APLOG_NOTICE, 0, NULL, "Read config: '%s'", buf); #endif - return APR_SUCCESS; + return end - start + 1; +} + +/* Read one line from open ap_configfile_t, strip LF, increase line number */ +/* If custom handler does not define a getstr() function, read char by char */ +AP_DECLARE(apr_status_t) ap_cfg_getline(char *buf, size_t bufsize, ap_configfile_t *cfp) +{ + apr_status_t rc = ap_cfg_getline_core(buf, bufsize, cfp); + if (rc == APR_SUCCESS) + cfg_trim_line(buf); + return rc; +} + +AP_DECLARE(apr_status_t) ap_varbuf_cfg_getline(struct ap_varbuf *vb, + ap_configfile_t *cfp, + apr_size_t max_len) +{ + apr_status_t rc; + vb->strlen = 0; + *vb->buf = '\0'; + + for (;;) { + apr_size_t new_len; + rc = ap_cfg_getline_core(vb->buf + vb->strlen, vb->avail - vb->strlen, cfp); + if (rc == APR_ENOSPC || rc == APR_SUCCESS) + vb->strlen += strlen(vb->buf + vb->strlen); + if (rc != APR_ENOSPC) + break; + if (vb->avail >= max_len) + return APR_ENOSPC; + new_len = vb->avail * 2; + if (new_len > max_len) + new_len = max_len; + ap_varbuf_grow(vb, new_len); + --cfp->line_number; + } + if (rc == APR_SUCCESS) + vb->strlen = cfg_trim_line(vb->buf); + return rc; } /* Size an HTTP header field list item, as separated by a comma. @@ -2400,3 +2443,142 @@ AP_DECLARE(int) ap_parse_form_data(request_rec *r, ap_filter_t *f, return OK; } + +#define VARBUF_SMALL_SIZE 2048 +#define VARBUF_MAX_SIZE (APR_SIZE_MAX - 1 - \ + APR_ALIGN_DEFAULT(sizeof(struct ap_varbuf_info))) + +struct ap_varbuf_info { + struct apr_memnode_t *node; + apr_allocator_t *allocator; +}; + +static apr_status_t varbuf_cleanup(void *info_) +{ + struct ap_varbuf_info *info = info_; + info->node->next = NULL; + apr_allocator_free(info->allocator, info->node); + return APR_SUCCESS; +} + +AP_DECLARE(void) ap_varbuf_init(apr_pool_t *p, struct ap_varbuf *vb, + apr_size_t init_size) +{ + vb->buf = NULL; + vb->avail = 0; + vb->strlen = AP_VARBUF_UNKNOWN; + vb->pool = p; + vb->info = NULL; + + ap_varbuf_grow(vb, init_size); +} + +AP_DECLARE(void) ap_varbuf_grow(struct ap_varbuf *vb, apr_size_t new_len) +{ + apr_memnode_t *new_node = NULL; + apr_allocator_t *allocator; + struct ap_varbuf_info *new_info; + char *new; + + if (new_len <= vb->avail) + return; + + if (new_len < 2 * vb->avail && vb->avail < VARBUF_MAX_SIZE/2) { + /* at least double the size, to avoid repeated reallocations */ + new_len = 2 * vb->avail; + } + else if (new_len > VARBUF_MAX_SIZE) { + apr_abortfunc_t abort_fn = apr_pool_abort_get(vb->pool); + ap_assert(abort_fn != NULL); + abort_fn(APR_ENOMEM); + return; + } + + new_len++; /* add space for trailing \0 */ + if (new_len <= VARBUF_SMALL_SIZE) { + new_len = APR_ALIGN_DEFAULT(new_len); + new = apr_palloc(vb->pool, new_len); + if (vb->buf && vb->strlen > 0) { + AP_DEBUG_ASSERT(vb->avail > 0); + if (new == vb->buf + vb->avail + 1) { + /* We are lucky: the new memory lies directly after our old + * buffer, we can now use both. + */ + vb->avail += new_len; + return; + } + else { + /* copy up to vb->strlen + 1 bytes */ + memcpy(new, vb->buf, vb->strlen > vb->avail ? + vb->avail + 1 : vb->strlen + 1); + } + } + else { + *new = '\0'; + } + vb->avail = new_len - 1; + vb->buf = new; + return; + } + + /* The required block is rather larger. Use allocator directly so that + * the memory can be freed independently from the pool. */ + allocator = apr_pool_allocator_get(vb->pool); + if (new_len <= VARBUF_MAX_SIZE) + new_node = apr_allocator_alloc(allocator, + new_len + APR_ALIGN_DEFAULT(sizeof(*new_info))); + if (!new_node) { + apr_abortfunc_t abort_fn = apr_pool_abort_get(vb->pool); + ap_assert(abort_fn != NULL); + abort_fn(APR_ENOMEM); + return; + } + new_info = (struct ap_varbuf_info *)new_node->first_avail; + new_node->first_avail += APR_ALIGN_DEFAULT(sizeof(*new_info)); + new_info->node = new_node; + new_info->allocator = allocator; + new = new_node->first_avail; + AP_DEBUG_ASSERT(new_node->endp - new_node->first_avail >= new_len); + new_len = new_node->endp - new_node->first_avail; + + if (vb->buf && vb->strlen > 0) + memcpy(new, vb->buf, vb->strlen > vb->avail ? + vb->avail + 1: vb->strlen + 1); + else + *new = '\0'; + if (vb->info) + apr_pool_cleanup_run(vb->pool, vb->info, varbuf_cleanup); + apr_pool_cleanup_register(vb->pool, new_info, varbuf_cleanup, + apr_pool_cleanup_null); + vb->info = new_info; + vb->buf = new; + vb->avail = new_len - 1; +} + +AP_DECLARE(void) ap_varbuf_strmemcat(struct ap_varbuf *vb, const char *str, + int len) +{ + AP_DEBUG_ASSERT(len == strlen(str)); + if (!vb->avail) { + ap_varbuf_grow(vb, len); + memcpy(vb->buf, str, len + 1); + return; + } + if (vb->strlen > vb->avail) { + AP_DEBUG_ASSERT(vb->strlen == AP_VARBUF_UNKNOWN); + vb->strlen = strlen(vb->buf); + } + ap_varbuf_grow(vb, vb->strlen + len); + memcpy(vb->buf + vb->strlen, str, len); + vb->strlen += len; + vb->buf[vb->strlen] = '\0'; +} + +AP_DECLARE(void) ap_varbuf_free(struct ap_varbuf *vb) +{ + if (vb->info) { + apr_pool_cleanup_run(vb->pool, vb->info, varbuf_cleanup); + vb->info = NULL; + } + vb->buf = NULL; +}