From 1de680cb5f906ec4d416f2bdc203c66e792b88fa Mon Sep 17 00:00:00 2001 From: Stefan Eissing Date: Tue, 27 Oct 2015 14:15:56 +0000 Subject: [PATCH] http2 TLS record size handling configuration, improved output write frequency git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1710825 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 3 ++ docs/manual/mod/mod_http2.xml | 96 +++++++++++++++++++++++++++++++++++ modules/http2/h2_config.c | 38 ++++++++++++++ modules/http2/h2_config.h | 5 ++ modules/http2/h2_conn_io.c | 50 ++++++++++++------ modules/http2/h2_conn_io.h | 6 ++- modules/http2/h2_session.c | 60 +++++++++++++++------- modules/http2/h2_session.h | 4 +- 8 files changed, 226 insertions(+), 36 deletions(-) diff --git a/CHANGES b/CHANGES index b0840e967a..02155ccef0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,9 @@ -*- coding: utf-8 -*- Changes with Apache 2.5.0 + *) mod_http2: new directives 'H2TLSWarmUpSize' and 'H2TLSCoolDownSecs' + to control TLS record sizes during connection lifetime. + *) mod_cache: Accept HT (Horizontal Tab) when parsing cache related header fields as described in RFC2616. [Christophe Jaillet] diff --git a/docs/manual/mod/mod_http2.xml b/docs/manual/mod/mod_http2.xml index b355865ff4..e58af67a8b 100644 --- a/docs/manual/mod/mod_http2.xml +++ b/docs/manual/mod/mod_http2.xml @@ -383,4 +383,100 @@ + + H2TLSWarmUpSize + + H2TLSWarmUpSize amount + H2TLSWarmUpSize 1048576 + + server config + virtual host + + + +

+ This directive sets the number of bytes to be sent in small + TLS records (~1300 bytes) until doing maximum sized writes (16k) + on https: HTTP/2 connections. + This can be used server wide or for specific + VirtualHosts. +

+

+ Measurements by google performance + labs show that best performance on TLS connections is reached, + if initial record sizes stay below the MTU level, to allow a + complete record to fit into an IP packet. +

+

+ While TCP adjust its flow-control and window sizes, longer TLS + records can get stuck in queues or get lost and need retransmission. + This is of course true for all packets. TLS however needs the + whole record in order to decrypt it. Any missing bytes at the end + will stall usage of the received ones. +

+

+ After a sufficient number of bytes have been send successfully, + the TCP state of the connection is stable and maximum TLS record + sizes (16 KB) can be used for optimal performance. +

+

+ In deployments where servers are reached locally or over reliable + connections only, the value might be decreased with 0 disabling + any warmup phase alltogether. +

+

+ The following example sets the size to zero, effectively disabling + any warmup phase. +

+ Example + + H2TLSWarmUpSize 0 + + + + + + + H2TLSCoolDownSecs + + H2TLSCoolDownSecs seconds + H2TLSCoolDownSecs 1 + + server config + virtual host + + + +

+ This directive sets the number of seconds of idle time on a TLS + connection before the TLS write size falls back to small (~1300 bytes) + length. + This can be used server wide or for specific + VirtualHosts. +

+

+ See H2TLSWarmUpSize for a + description of TLS warmup. H2TLSCoolDownSecs reflects the fact + that connections may detoriate over time (and TCP flow adjusts) + for idle connections as well. It is beneficial to overall performance + to fall back to the pre-warmup phase after a number of seconds that + no data has been sent. +

+

+ In deployments where connections can be considered reliable, this + timer can be disabled by setting it to 0. +

+

+ The following example sets the seconds to zero, effectively disabling + any cooldown. Warmed up TLS connections stay on maximum record + size. +

+ Example + + H2TLSCoolDownSecs 0 + + + + + diff --git a/modules/http2/h2_config.c b/modules/http2/h2_config.c index a8744f0a5f..2b1cdd9800 100644 --- a/modules/http2/h2_config.c +++ b/modules/http2/h2_config.c @@ -51,6 +51,8 @@ static h2_config defconf = { -1, /* # session extra files */ 1, /* modern TLS only */ -1, /* HTTP/1 Upgrade support */ + 1024*1024, /* TLS warmup size */ + 1, /* TLS cooldown secs */ }; static int files_per_session = 0; @@ -104,6 +106,9 @@ static void *h2_config_create(apr_pool_t *pool, conf->session_extra_files = DEF_VAL; conf->modern_tls_only = DEF_VAL; conf->h2_upgrade = DEF_VAL; + conf->tls_warmup_size = DEF_VAL; + conf->tls_cooldown_secs = DEF_VAL; + return conf; } @@ -144,11 +149,18 @@ void *h2_config_merge(apr_pool_t *pool, void *basev, void *addv) n->session_extra_files = H2_CONFIG_GET(add, base, session_extra_files); n->modern_tls_only = H2_CONFIG_GET(add, base, modern_tls_only); n->h2_upgrade = H2_CONFIG_GET(add, base, h2_upgrade); + n->tls_warmup_size = H2_CONFIG_GET(add, base, tls_warmup_size); + n->tls_cooldown_secs = H2_CONFIG_GET(add, base, tls_cooldown_secs); return n; } int h2_config_geti(h2_config *conf, h2_config_var_t var) +{ + return (int)h2_config_geti64(conf, var); +} + +apr_int64_t h2_config_geti64(h2_config *conf, h2_config_var_t var) { int n; switch(var) { @@ -180,6 +192,10 @@ int h2_config_geti(h2_config *conf, h2_config_var_t var) n = files_per_session; } return n; + case H2_CONF_TLS_WARMUP_SIZE: + return H2_CONFIG_GET(conf, &defconf, tls_warmup_size); + case H2_CONF_TLS_COOLDOWN_SECS: + return H2_CONFIG_GET(conf, &defconf, tls_cooldown_secs); default: return DEF_VAL; } @@ -376,6 +392,24 @@ static const char *h2_conf_set_upgrade(cmd_parms *parms, return "value must be On or Off"; } +static const char *h2_conf_set_tls_warmup_size(cmd_parms *parms, + void *arg, const char *value) +{ + h2_config *cfg = h2_config_sget(parms->server); + cfg->tls_warmup_size = apr_atoi64(value); + (void)arg; + return NULL; +} + +static const char *h2_conf_set_tls_cooldown_secs(cmd_parms *parms, + void *arg, const char *value) +{ + h2_config *cfg = h2_config_sget(parms->server); + cfg->tls_cooldown_secs = (int)apr_atoi64(value); + (void)arg; + return NULL; +} + #define AP_END_CMD AP_INIT_TAKE1(NULL, NULL, NULL, RSRC_CONF, NULL) @@ -406,6 +440,10 @@ const command_rec h2_cmds[] = { RSRC_CONF, "on to enable direct HTTP/2 mode"), AP_INIT_TAKE1("H2SessionExtraFiles", h2_conf_set_session_extra_files, NULL, RSRC_CONF, "number of extra file a session might keep open"), + AP_INIT_TAKE1("H2TLSWarmUpSize", h2_conf_set_tls_warmup_size, NULL, + RSRC_CONF, "number of bytes on TLS connection before doing max writes"), + AP_INIT_TAKE1("H2TLSCoolDownSecs", h2_conf_set_tls_cooldown_secs, NULL, + RSRC_CONF, "seconds of idle time on TLS before shrinking writes"), AP_END_CMD }; diff --git a/modules/http2/h2_config.h b/modules/http2/h2_config.h index 781ca902d3..9c59817a99 100644 --- a/modules/http2/h2_config.h +++ b/modules/http2/h2_config.h @@ -36,6 +36,8 @@ typedef enum { H2_CONF_SESSION_FILES, H2_CONF_MODERN_TLS_ONLY, H2_CONF_UPGRADE, + H2_CONF_TLS_WARMUP_SIZE, + H2_CONF_TLS_COOLDOWN_SECS, } h2_config_var_t; /* Apache httpd module configuration for h2. */ @@ -55,6 +57,8 @@ typedef struct h2_config { int session_extra_files; /* # of extra files a session may keep open */ int modern_tls_only; /* Accept only modern TLS in HTTP/2 connections */ int h2_upgrade; /* Allow HTTP/1 upgrade to h2/h2c */ + apr_int64_t tls_warmup_size; /* Amount of TLS data to send before going full write size */ + int tls_cooldown_secs; /* Seconds of idle time before going back to small TLS records */ } h2_config; @@ -71,6 +75,7 @@ h2_config *h2_config_sget(server_rec *s); h2_config *h2_config_rget(request_rec *r); int h2_config_geti(h2_config *conf, h2_config_var_t var); +apr_int64_t h2_config_geti64(h2_config *conf, h2_config_var_t var); void h2_config_init(apr_pool_t *pool); diff --git a/modules/http2/h2_conn_io.c b/modules/http2/h2_conn_io.c index 3a094218f4..cf03651434 100644 --- a/modules/http2/h2_conn_io.c +++ b/modules/http2/h2_conn_io.c @@ -28,24 +28,40 @@ #include "h2_h2.h" #include "h2_util.h" -#define WRITE_BUFFER_SIZE (64*1024) +#define WRITE_BUFFER_SIZE (128*1024) +/* Calculated like this: assuming MTU 1500 bytes + * 1500 - 40 (IP) - 20 (TCP) - 40 (TCP options) + * - TLS overhead (60-100) + * ~= 1300 bytes */ #define WRITE_SIZE_INITIAL 1300 -#define WRITE_SIZE_MAX (16*1024) -#define WRITE_SIZE_IDLE_USEC (1*APR_USEC_PER_SEC) -#define WRITE_SIZE_THRESHOLD (1*1024*1024) +/* Calculated like this: max TLS record size + * 16*1024 + * - TLS overhead (60-100) */ +#define WRITE_SIZE_MAX (16*1024 - 100) apr_status_t h2_conn_io_init(h2_conn_io *io, conn_rec *c) { - io->connection = c; - io->input = apr_brigade_create(c->pool, c->bucket_alloc); - io->output = apr_brigade_create(c->pool, c->bucket_alloc); - io->buflen = 0; + h2_config *cfg = h2_config_get(c); + + io->connection = c; + io->input = apr_brigade_create(c->pool, c->bucket_alloc); + io->output = apr_brigade_create(c->pool, c->bucket_alloc); + io->buflen = 0; /* That is where we start with, * see https://issues.apache.org/jira/browse/TS-2503 */ - io->write_size = WRITE_SIZE_INITIAL; - io->last_write = 0; - io->buffer_output = h2_h2_is_tls(c); + io->tls_warmup_size = h2_config_geti64(cfg, H2_CONF_TLS_WARMUP_SIZE); + io->tls_cooldown_usecs = (h2_config_geti(cfg, H2_CONF_TLS_COOLDOWN_SECS) + * APR_USEC_PER_SEC); + io->write_size = WRITE_SIZE_INITIAL; + io->last_write = 0; + io->buffer_output = h2_h2_is_tls(c); + if (APLOGctrace1(c)) { + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, io->connection, + "h2_conn_io(%ld): init, buffering=%d, warmup_size=%ld, cd_secs=%f", + io->connection->id, io->buffer_output, (long)io->tls_warmup_size, + ((float)io->tls_cooldown_usecs/APR_USEC_PER_SEC)); + } /* Currently we buffer only for TLS output. The reason this gives * improved performance is that buckets send to the mod_ssl network * filter will be encrypted in chunks. There is a special filter @@ -159,7 +175,7 @@ apr_status_t h2_conn_io_read(h2_conn_io *io, status = ap_get_brigade(io->connection->input_filters, io->input, AP_MODE_READBYTES, - block, 16 * 4096); + block, 64 * 4096); switch (status) { case APR_SUCCESS: return h2_conn_io_bucket_read(io, block, on_read_cb, puser, &done); @@ -187,6 +203,9 @@ static apr_status_t flush_out(apr_bucket_brigade *bb, void *ctx) ap_update_child_status(io->connection->sbh, SERVER_BUSY_WRITE, NULL); status = apr_brigade_length(bb, 1, &bblen); if (status == APR_SUCCESS) { + ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, io->connection, + "h2_conn_io(%ld): flush, brigade %ld bytes", + io->connection->id, (long)bblen); status = ap_pass_brigade(io->connection->output_filters, bb); if (status == APR_SUCCESS) { io->bytes_written += (apr_size_t)bblen; @@ -206,8 +225,9 @@ static apr_status_t bucketeer_buffer(h2_conn_io *io) { apr_bucket *b; int bcount, i; - if (io->write_size > WRITE_SIZE_INITIAL - && (apr_time_now() - io->last_write) >= WRITE_SIZE_IDLE_USEC) { + if (io->write_size > WRITE_SIZE_INITIAL + && (io->tls_cooldown_usecs > 0) + && (apr_time_now() - io->last_write) >= io->tls_cooldown_usecs) { /* long time not written, reset write size */ io->write_size = WRITE_SIZE_INITIAL; io->bytes_written = 0; @@ -216,7 +236,7 @@ static apr_status_t bucketeer_buffer(h2_conn_io *io) { (long)io->connection->id, (long)io->write_size); } else if (io->write_size < WRITE_SIZE_MAX - && io->bytes_written >= WRITE_SIZE_THRESHOLD) { + && io->bytes_written >= io->tls_warmup_size) { /* connection is hot, use max size */ io->write_size = WRITE_SIZE_MAX; ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, io->connection, diff --git a/modules/http2/h2_conn_io.h b/modules/http2/h2_conn_io.h index 084445ef2b..b76ed39643 100644 --- a/modules/http2/h2_conn_io.h +++ b/modules/http2/h2_conn_io.h @@ -29,7 +29,11 @@ typedef struct { int buffer_output; int write_size; apr_time_t last_write; - apr_size_t bytes_written; + apr_int64_t bytes_written; + + apr_time_t tls_cooldown_usecs; + apr_int64_t tls_warmup_size; + char *buffer; apr_size_t buflen; diff --git a/modules/http2/h2_session.c b/modules/http2/h2_session.c index 718fb21867..d0e0dfea99 100644 --- a/modules/http2/h2_session.c +++ b/modules/http2/h2_session.c @@ -186,6 +186,20 @@ static int before_frame_send_cb(nghttp2_session *ngh2, if (session->aborted) { return NGHTTP2_ERR_CALLBACK_FAILURE; } + /* Set the need to flush output when we have added one of the + * following frame types */ + switch (frame->hd.type) { + case NGHTTP2_RST_STREAM: + case NGHTTP2_WINDOW_UPDATE: + case NGHTTP2_PUSH_PROMISE: + case NGHTTP2_PING: + case NGHTTP2_GOAWAY: + session->flush = 1; + break; + default: + break; + + } if (APLOGctrace2(session->c)) { char buffer[256]; frame_print(frame, buffer, sizeof(buffer)/sizeof(buffer[0])); @@ -201,9 +215,14 @@ static int on_frame_send_cb(nghttp2_session *ngh2, void *userp) { h2_session *session = (h2_session *)userp; - (void)ngh2; (void)frame; - ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, session->c, - "h2_session(%ld): on_frame_send", session->id); + (void)ngh2; + if (APLOGctrace2(session->c)) { + char buffer[256]; + frame_print(frame, buffer, sizeof(buffer)/sizeof(buffer[0])); + ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c, + "h2_session(%ld): on_frame_send %s", + session->id, buffer); + } return 0; } @@ -270,6 +289,9 @@ static int on_stream_close_cb(nghttp2_session *ngh2, int32_t stream_id, session->id, (int)stream_id, error_code); } + /* always flush on eos */ + session->flush = 1; + return 0; } @@ -518,7 +540,6 @@ static int on_send_data_cb(nghttp2_session *ngh2, return h2_session_status_from_apr_status(status); } - #define NGH2_SET_CALLBACK(callbacks, name, fn)\ nghttp2_session_callbacks_set_##name##_callback(callbacks, fn) @@ -893,6 +914,12 @@ static void update_window(void *ctx, int stream_id, apr_size_t bytes_read) nghttp2_session_consume(session->ngh2, stream_id, bytes_read); } +static apr_status_t h2_session_flush(h2_session *session) +{ + session->flush = 0; + return h2_conn_io_flush(&session->io); +} + static apr_status_t h2_session_update_windows(h2_session *session) { return h2_mplx_in_update_windows(session->mplx, update_window, session); @@ -902,16 +929,12 @@ apr_status_t h2_session_write(h2_session *session, apr_interval_time_t timeout) { apr_status_t status = APR_EAGAIN; h2_stream *stream = NULL; - int flush_output = 0; AP_DEBUG_ASSERT(session); /* Check that any pending window updates are sent. */ status = h2_session_update_windows(session); - if (status == APR_SUCCESS) { - flush_output = 1; - } - else if (!APR_STATUS_IS_EAGAIN(status)) { + if (status != APR_SUCCESS && !APR_STATUS_IS_EAGAIN(status)) { return status; } @@ -927,27 +950,24 @@ apr_status_t h2_session_write(h2_session *session, apr_interval_time_t timeout) status = APR_ECONNABORTED; } } - flush_output = 1; } /* If we have responses ready, submit them now. */ while (!session->aborted && (stream = h2_mplx_next_submit(session->mplx, session->streams)) != NULL) { status = h2_session_handle_response(session, stream); - flush_output = 1; } if (!session->aborted && h2_session_resume_streams_with_data(session) > 0) { - flush_output = 1; } - if (!session->aborted && !flush_output - && timeout > 0 && !h2_session_want_write(session)) { + if (!session->aborted && !session->flush && timeout > 0 + && !h2_session_want_write(session)) { + h2_session_flush(session); status = h2_mplx_out_trywait(session->mplx, timeout, session->iowait); if (status != APR_TIMEUP && h2_session_resume_streams_with_data(session) > 0) { - flush_output = 1; } else { /* nothing happened to ongoing streams, do some house-keeping */ @@ -966,11 +986,10 @@ apr_status_t h2_session_write(h2_session *session, apr_interval_time_t timeout) status = APR_ECONNABORTED; } } - flush_output = 1; } - if (flush_output) { - h2_conn_io_flush(&session->io); + if (session->flush) { + h2_session_flush(session); } return status; @@ -1015,6 +1034,11 @@ static apr_status_t session_receive(const char *data, apr_size_t len, apr_status_t h2_session_read(h2_session *session, apr_read_type_e block) { AP_DEBUG_ASSERT(session); + if (block == APR_BLOCK_READ) { + /* before we do a blocking read, make sure that all our output + * is send out. Otherwise we might deadlock. */ + h2_session_flush(session); + } return h2_conn_io_read(&session->io, block, session_receive, session); } diff --git a/modules/http2/h2_session.h b/modules/http2/h2_session.h index ddf29a06df..a176181164 100644 --- a/modules/http2/h2_session.h +++ b/modules/http2/h2_session.h @@ -58,6 +58,7 @@ struct h2_session { request_rec *r; /* the request that started this in case * of 'h2c', NULL otherwise */ int aborted; /* this session is being aborted */ + int flush; /* if != 0, flush output on next occasion */ apr_size_t frames_received; /* number of http/2 frames received */ apr_size_t max_stream_count; /* max number of open streams */ apr_size_t max_stream_mem; /* max buffer memory for a single stream */ @@ -147,8 +148,7 @@ apr_status_t h2_session_read(h2_session *session, apr_read_type_e block); * a maximum of timeout micro-seconds and return to the caller. If timeout * occurred, APR_TIMEUP will be returned. */ -apr_status_t h2_session_write(h2_session *session, - apr_interval_time_t timeout); +apr_status_t h2_session_write(h2_session *session, apr_interval_time_t timeout); /* Start submitting the response to a stream request. This is possible * once we have all the response headers. */ -- 2.40.0