From 2d63cb2faa99e8b286962bd7f35cdbdba0bd0a08 Mon Sep 17 00:00:00 2001 From: Ruediger Pluem Date: Sat, 29 Jul 2006 13:24:37 +0000 Subject: [PATCH] * Rework inflate out filter and give it a similar workflow as the deflate out filter. This fixes the following bugs in the inflate out filter: - Incorrect handling of flush buckets. - Excessive memory usage for large compressed content (because we now already sent parts down the chain and do not process the whole brigade first before sending something down the chain). - Handle the case correctly where the validation bytes at the end of the compressed data stream are distributed across different buckets / brigades. - Fix a memory leak due to not cleaning up the internal structures of libz in some error cases. PR: 39854 git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@426799 13f79535-47bb-0310-9956-ffa450edef68 --- modules/filters/mod_deflate.c | 207 +++++++++++++++++++++++----------- 1 file changed, 139 insertions(+), 68 deletions(-) diff --git a/modules/filters/mod_deflate.c b/modules/filters/mod_deflate.c index 6050b3e733..9cdb2c2777 100644 --- a/modules/filters/mod_deflate.c +++ b/modules/filters/mod_deflate.c @@ -213,6 +213,8 @@ typedef struct deflate_ctx_t unsigned long crc; apr_bucket_brigade *bb, *proc_bb; int (*libz_end_func)(z_streamp); + unsigned char *validation_buffer; + apr_size_t validation_buffer_length; } deflate_ctx; /* Number of validation bytes (CRC and length) after the compressed data */ @@ -891,8 +893,8 @@ static apr_status_t inflate_out_filter(ap_filter_t *f, { int zlib_method; int zlib_flags; - int deflate_init = 1; - apr_bucket *bkt; + int inflate_init = 1; + apr_bucket *e; request_rec *r = f->r; deflate_ctx *ctx = f->ctx; int zRC; @@ -950,9 +952,11 @@ static apr_status_t inflate_out_filter(ap_filter_t *f, f->ctx = ctx = apr_pcalloc(f->r->pool, sizeof(*ctx)); - ctx->proc_bb = apr_brigade_create(r->pool, f->c->bucket_alloc); + ctx->bb = apr_brigade_create(r->pool, f->c->bucket_alloc); ctx->buffer = apr_palloc(r->pool, c->bufferSize); - + ctx->libz_end_func = inflateEnd; + ctx->validation_buffer = NULL; + ctx->validation_buffer_length = 0; zRC = inflateInit2(&ctx->stream, c->windowSize); @@ -963,40 +967,117 @@ static apr_status_t inflate_out_filter(ap_filter_t *f, "unable to init Zlib: " "inflateInit2 returned %d: URL %s", zRC, r->uri); + /* + * Remove ourselves as it does not make sense to return: + * We are not able to init libz and pass data down the chain + * compressed. + */ ap_remove_output_filter(f); return ap_pass_brigade(f->next, bb); } - /* initialize deflate output buffer */ + /* + * Register a cleanup function to ensure that we cleanup the internal + * libz resources. + */ + apr_pool_cleanup_register(r->pool, ctx, deflate_ctx_cleanup, + apr_pool_cleanup_null); + + apr_table_unset(r->headers_out, "Content-Length"); + + /* initialize inflate output buffer */ ctx->stream.next_out = ctx->buffer; ctx->stream.avail_out = c->bufferSize; - deflate_init = 0; + inflate_init = 0; } - for (bkt = APR_BRIGADE_FIRST(bb); - bkt != APR_BRIGADE_SENTINEL(bb); - bkt = APR_BUCKET_NEXT(bkt)) + while (!APR_BRIGADE_EMPTY(bb)) { const char *data; + apr_bucket *b; apr_size_t len; - /* If we actually see the EOS, that means we screwed up! */ - /* no it doesn't - not in a HEAD or 204/304 */ - if (APR_BUCKET_IS_EOS(bkt)) { + e = APR_BRIGADE_FIRST(bb); + + if (APR_BUCKET_IS_EOS(e)) { + ctx->stream.avail_in = 0; /* should be zero already anyway */ + /* + * Flush the remaining data from the zlib buffers. It is correct + * to use Z_SYNC_FLUSH in this case and not Z_FINISH as in the + * deflate case. In the inflate case Z_FINISH requires to have a + * large enough output buffer to put ALL data in otherwise it + * fails, whereas in the deflate case you can empty a filled output + * buffer and call it again until no more output can be created. + */ + flush_libz_buffer(ctx, c, f->c->bucket_alloc, inflate, Z_SYNC_FLUSH, + UPDATE_CRC); + ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, + "Zlib: Inflated %ld to %ld : URL %s", + ctx->stream.total_in, ctx->stream.total_out, r->uri); + + if (ctx->validation_buffer_length == VALIDATION_SIZE) { + unsigned long compCRC, compLen; + compCRC = getLong(ctx->validation_buffer); + if (ctx->crc != compCRC) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Zlib: Checksum of inflated stream invalid"); + return APR_EGENERAL; + } + ctx->validation_buffer += VALIDATION_SIZE / 2; + compLen = getLong(ctx->validation_buffer); + if (ctx->stream.total_out != compLen) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Zlib: Length of inflated stream invalid"); + return APR_EGENERAL; + } + } + else { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, + "Zlib: Validation bytes not present"); + return APR_EGENERAL; + } + inflateEnd(&ctx->stream); - return ap_pass_brigade(f->next, bb); + /* No need for cleanup any longer */ + apr_pool_cleanup_kill(r->pool, ctx, deflate_ctx_cleanup); + + /* Remove EOS from the old list, and insert into the new. */ + APR_BUCKET_REMOVE(e); + APR_BRIGADE_INSERT_TAIL(ctx->bb, e); + + /* + * Okay, we've seen the EOS. + * Time to pass it along down the chain. + */ + return ap_pass_brigade(f->next, ctx->bb); } - if (APR_BUCKET_IS_FLUSH(bkt)) { + if (APR_BUCKET_IS_FLUSH(e)) { + apr_status_t rv; + + /* flush the remaining data from the zlib buffers */ + zRC = flush_libz_buffer(ctx, c, f->c->bucket_alloc, inflate, + Z_SYNC_FLUSH, UPDATE_CRC); + if (zRC != Z_OK) { + return APR_EGENERAL; + } + + /* Remove flush bucket from old brigade anf insert into the new. */ + APR_BUCKET_REMOVE(e); + APR_BRIGADE_INSERT_TAIL(ctx->bb, e); + rv = ap_pass_brigade(f->next, ctx->bb); + if (rv != APR_SUCCESS) { + return rv; + } continue; } /* read */ - apr_bucket_read(bkt, &data, &len, APR_BLOCK_READ); + apr_bucket_read(e, &data, &len, APR_BLOCK_READ); /* first bucket contains zlib header */ - if (!deflate_init++) { + if (!inflate_init++) { if (len < 10) { ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "Insufficient data for inflate"); @@ -1050,82 +1131,72 @@ static apr_status_t inflate_out_filter(ap_filter_t *f, ctx->stream.next_in = (unsigned char *)data; ctx->stream.avail_in = len; + if (ctx->validation_buffer) { + if (ctx->validation_buffer_length < VALIDATION_SIZE) { + apr_size_t copy_size; + + copy_size = VALIDATION_SIZE - ctx->validation_buffer_length; + if (copy_size > ctx->stream.avail_in) + copy_size = ctx->stream.avail_in; + memcpy(ctx->validation_buffer, ctx->stream.next_in, copy_size); + } else { + ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, + "Zlib: %d bytes of garbage at the end of " + "compressed stream.", ctx->stream.avail_in); + } + } + zRC = Z_OK; while (ctx->stream.avail_in != 0) { if (ctx->stream.avail_out == 0) { - apr_bucket *tmp_heap; + ctx->stream.next_out = ctx->buffer; len = c->bufferSize - ctx->stream.avail_out; ctx->crc = crc32(ctx->crc, (const Bytef *)ctx->buffer, len); - tmp_heap = apr_bucket_heap_create((char *)ctx->buffer, len, - NULL, f->c->bucket_alloc); - APR_BRIGADE_INSERT_TAIL(ctx->proc_bb, tmp_heap); + b = apr_bucket_heap_create((char *)ctx->buffer, len, + NULL, f->c->bucket_alloc); + APR_BRIGADE_INSERT_TAIL(ctx->bb, b); ctx->stream.avail_out = c->bufferSize; + /* Send what we have right now to the next filter. */ + rv = ap_pass_brigade(f->next, ctx->bb); + if (rv != APR_SUCCESS) { + return rv; + } } zRC = inflate(&ctx->stream, Z_NO_FLUSH); if (zRC == Z_STREAM_END) { + /* + * We have inflated all data. Now try to capture the + * validation bytes. We may not have them all available + * right now, but capture what is there. + */ + ctx->validation_buffer = apr_pcalloc(f->r->pool, + VALIDATION_SIZE); + if (ctx->stream.avail_in > VALIDATION_SIZE) { + ctx->validation_buffer_length = VALIDATION_SIZE; + } else if (ctx->stream.avail_in > 0) { + ctx->validation_buffer_length = ctx->stream.avail_in; + } + if (ctx->validation_buffer_length) + memcpy(ctx->validation_buffer, ctx->stream.next_in, + ctx->validation_buffer_length); break; } if (zRC != Z_OK) { - inflateEnd(&ctx->stream); - return APR_EGENERAL; - } - } - if (zRC == Z_STREAM_END) { - apr_bucket *tmp_heap, *eos; - - ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, - "Zlib: Inflated %ld to %ld : URL %s", - ctx->stream.total_in, ctx->stream.total_out, - r->uri); - - len = c->bufferSize - ctx->stream.avail_out; - - ctx->crc = crc32(ctx->crc, (const Bytef *)ctx->buffer, len); - tmp_heap = apr_bucket_heap_create((char *)ctx->buffer, len, - NULL, f->c->bucket_alloc); - APR_BRIGADE_INSERT_TAIL(ctx->proc_bb, tmp_heap); - ctx->stream.avail_out = c->bufferSize; - - /* Is the remaining 8 bytes already in the avail stream? */ - if (ctx->stream.avail_in >= 8) { - unsigned long compCRC, compLen; - compCRC = getLong(ctx->stream.next_in); - if (ctx->crc != compCRC) { - inflateEnd(&ctx->stream); - return APR_EGENERAL; - } - ctx->stream.next_in += 4; - compLen = getLong(ctx->stream.next_in); - if (ctx->stream.total_out != compLen) { - inflateEnd(&ctx->stream); - return APR_EGENERAL; - } - } - else { - /* FIXME: We need to grab the 8 verification bytes - * from the wire! */ - inflateEnd(&ctx->stream); return APR_EGENERAL; } - - inflateEnd(&ctx->stream); - - eos = apr_bucket_eos_create(f->c->bucket_alloc); - APR_BRIGADE_INSERT_TAIL(ctx->proc_bb, eos); - break; } + apr_bucket_delete(e); } - rv = ap_pass_brigade(f->next, ctx->proc_bb); - apr_brigade_cleanup(ctx->proc_bb); - return rv ; + apr_brigade_cleanup(bb); + return APR_SUCCESS; } #define PROTO_FLAGS AP_FILTER_PROTO_CHANGE|AP_FILTER_PROTO_CHANGE_LENGTH -- 2.40.0