url: add option CURLOPT_SUPPRESS_CONNECT_HEADERS

- Add new option CURLOPT_SUPPRESS_CONNECT_HEADERS to allow suppressing
  proxy CONNECT response headers from the user callback functions
  CURLOPT_HEADERFUNCTION and CURLOPT_WRITEFUNCTION.

- Add new tool option --suppress-connect-headers to expose
  CURLOPT_SUPPRESS_CONNECT_HEADERS and allow suppressing proxy CONNECT
  response headers from --dump-header and --include.

Assisted-by: Jay Satiro
Assisted-by: CarloCannas@users.noreply.github.com
Closes https://github.com/curl/curl/pull/783

diff --git a/docs/cmdline-opts/Makefile.am b/docs/cmdline-opts/Makefile.am
index f425ddd08d7dfad95c2d282fb7cc7f1f773b1795..5cfd67605c15c636722bd6e527d52ca6b1bfeaf0 100644 (file)
--- a/docs/cmdline-opts/Makefile.am
+++ b/docs/cmdline-opts/Makefile.am
@@ -59,7 +59,8 @@ DPAGES = abstract-unix-socket.d anyauth.d append.d basic.d cacert.d capath.d cer
   service-name.d show-error.d silent.d socks4a.d socks4.d socks5.d     \
   socks5-gssapi-nec.d socks5-gssapi-service.d socks5-hostname.d                \
   speed-limit.d speed-time.d ssl-allow-beast.d ssl.d ssl-no-revoke.d   \
-  ssl-reqd.d sslv2.d sslv3.d stderr.d tcp-fastopen.d tcp-nodelay.d     \
+  ssl-reqd.d sslv2.d sslv3.d stderr.d suppress-connect-headers.d       \
+  tcp-fastopen.d tcp-nodelay.d                                         \
   telnet-option.d tftp-blksize.d tftp-no-options.d time-cond.d         \
   tls-max.d \
   tlsauthtype.d tlspassword.d tlsuser.d tlsv1.0.d tlsv1.1.d tlsv1.2.d  \

diff --git a/docs/cmdline-opts/proxytunnel.d b/docs/cmdline-opts/proxytunnel.d
index 09855ed34729050201e94b2faa10b96a69f50ac6..f2e8b802d24bea727cfae3528aa920585c9ad1a3 100644 (file)
--- a/docs/cmdline-opts/proxytunnel.d
+++ b/docs/cmdline-opts/proxytunnel.d
@@ -8,3 +8,6 @@ to attempt to tunnel through the proxy instead of merely using it to do
 HTTP-like operations. The tunnel approach is made with the HTTP proxy CONNECT
 request and requires that the proxy allows direct connect to the remote port
 number curl wants to tunnel through to.
+
+To suppress proxy CONNECT response headers when curl is set to output headers
+use --suppress-connect-headers.

diff --git a/docs/cmdline-opts/suppress-connect-headers.d b/docs/cmdline-opts/suppress-connect-headers.d
new file mode 100644 (file)
index 0000000..d208b89
--- /dev/null
+++ b/docs/cmdline-opts/suppress-connect-headers.d
@@ -0,0 +1,8 @@
+Long: suppress-connect-headers
+Help: Suppress proxy CONNECT response headers
+See-also: dump-header include proxytunnel
+---
+When --proxytunnel is used and a CONNECT request is made don't output proxy
+CONNECT response headers. This option is meant to be used with --dump-header or
+--include which are used to show protocol headers in the output. It has no
+effect on debug options such as --verbose or --trace, or any statistics.

diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3
index 422cb85690f5b100035b60275f8f89f1d9ee458b..cb6884766be712791910e8d6255704375220063d 100644 (file)
--- a/docs/libcurl/curl_easy_setopt.3
+++ b/docs/libcurl/curl_easy_setopt.3
@@ -137,6 +137,8 @@ Data pointer to pass to the chunk callbacks. See \fICURLOPT_CHUNK_DATA(3)\fP
 Callback for wildcard matching. See \fICURLOPT_FNMATCH_FUNCTION(3)\fP
 .IP CURLOPT_FNMATCH_DATA
 Data pointer to pass to the wildcard matching callback. See \fICURLOPT_FNMATCH_DATA(3)\fP
+.IP CURLOPT_SUPPRESS_CONNECT_HEADERS
+Suppress proxy CONNECT response headers from user callbacks. See \fICURLOPT_SUPPRESS_CONNECT_HEADERS(3)\fP
 .SH ERROR OPTIONS
 .IP CURLOPT_ERRORBUFFER
 Error message buffer. See \fICURLOPT_ERRORBUFFER(3)\fP

diff --git a/docs/libcurl/opts/CURLINFO_HEADER_SIZE.3 b/docs/libcurl/opts/CURLINFO_HEADER_SIZE.3
index b4b9b578f25513d5f6f1a0d9e43da84d220fd02c..b687c80b96dbefba8a7ce565abc3f7dce2f2b390 100644 (file)
--- a/docs/libcurl/opts/CURLINFO_HEADER_SIZE.3
+++ b/docs/libcurl/opts/CURLINFO_HEADER_SIZE.3
@@ -30,6 +30,9 @@ CURLcode curl_easy_getinfo(CURL *handle, CURLINFO_HEADER_SIZE, long *sizep);
 .SH DESCRIPTION
 Pass a pointer to a long to receive the total size of all the headers
 received. Measured in number of bytes.
+
+The total includes the size of any received headers suppressed by
+\fICURLOPT_SUPPRESS_CONNECT_HEADERS(3)\fP.
 .SH PROTOCOLS
 All
 .SH EXAMPLE

diff --git a/docs/libcurl/opts/CURLOPT_HTTPPROXYTUNNEL.3 b/docs/libcurl/opts/CURLOPT_HTTPPROXYTUNNEL.3
index 6b48a0527f86ef428a97d993d79585c7dc067d30..bdd04dfff90cf5f6746b58cf3138c10f69a17cb0 100644 (file)
--- a/docs/libcurl/opts/CURLOPT_HTTPPROXYTUNNEL.3
+++ b/docs/libcurl/opts/CURLOPT_HTTPPROXYTUNNEL.3
@@ -40,6 +40,9 @@ it allows CONNECT requests to and often only port 80 and 443 are allowed.
 
 When using this, it only makes sense to use \fICURLOPT_PROXYTYPE(3)\fP set to
 a HTTP proxy.
+
+To suppress proxy CONNECT response headers from user callbacks use
+\fICURLOPT_SUPPRESS_CONNECT_HEADERS(3)\fP.
 .SH DEFAULT
 0
 .SH PROTOCOLS

diff --git a/docs/libcurl/opts/CURLOPT_SUPPRESS_CONNECT_HEADERS.3 b/docs/libcurl/opts/CURLOPT_SUPPRESS_CONNECT_HEADERS.3
new file mode 100644 (file)
index 0000000..8cfec7a
--- /dev/null
+++ b/docs/libcurl/opts/CURLOPT_SUPPRESS_CONNECT_HEADERS.3
@@ -0,0 +1,95 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_SUPPRESS_CONNECT_HEADERS 3 "13 February 2017" "libcurl 7.54.0" "curl_easy_setopt options"
+.SH NAME
+CURLOPT_SUPPRESS_CONNECT_HEADERS \- Suppress proxy CONNECT response headers from user callbacks
+.SH SYNOPSIS
+.nf
+#include <curl/curl.h>
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_SUPPRESS_CONNECT_HEADERS, long onoff);
+.fi
+.SH DESCRIPTION
+When \fICURLOPT_HTTPPROXYTUNNEL(3)\fP is used and a CONNECT request is made,
+suppress proxy CONNECT response headers from the user callback functions
+\fICURLOPT_HEADERFUNCTION(3)\fP and \fICURLOPT_WRITEFUNCTION(3)\fP.
+
+Proxy CONNECT response headers can complicate header processing since it's
+essentially a separate set of headers. You can enable this option to suppress
+those headers.
+
+For example let's assume an HTTPS URL is to be retrieved via CONNECT. On
+success there would normally be two sets of headers, and each header line sent
+to the header function and/or the write function. The data given to the
+callbacks would look like this:
+
+.nf
+HTTP/1.1 200 Connection established
+{headers}...
+
+HTTP/1.1 200 OK
+Content-Type: application/json
+{headers}...
+
+{body}...
+.fi
+
+However by enabling this option the CONNECT response headers are suppressed, so
+the data given to the callbacks would look like this:
+
+.nf
+HTTP/1.1 200 OK
+Content-Type: application/json
+{headers}...
+
+{body}...
+.fi
+
+.SH DEFAULT
+0
+.SH PROTOCOLS
+All
+.SH EXAMPLE
+.nf
+CURL *curl = curl_easy_init();
+if(curl) {
+  curl_easy_setopt(curl, CURLOPT_URL, "https://example.com");
+
+  curl_easy_setopt(curl, CURLOPT_HEADER, 1L);
+  curl_easy_setopt(curl, CURLOPT_PROXY, "http://foo:3128");
+  curl_easy_setopt(curl, CURLOPT_HTTPPROXYTUNNEL, 1L);
+  curl_easy_setopt(curl, CURLOPT_SUPPRESS_CONNECT_HEADERS, 1L);
+
+  curl_easy_perform(curl);
+
+  /* always cleanup */
+  curl_easy_cleanup(curl);
+}
+.fi
+.SH AVAILABILITY
+Added in 7.54.0
+.SH RETURN VALUE
+CURLE_OK or an error such as CURLE_UNKNOWN_OPTION.
+.SH "SEE ALSO"
+.BR CURLOPT_HEADER "(3), " CURLOPT_PROXY "(3), "
+.BR CURLOPT_HTTPPROXYTUNNEL "(3), "

diff --git a/docs/libcurl/opts/Makefile.am b/docs/libcurl/opts/Makefile.am
index 5829a144febf03d1ef8120079c21c23821053c78..26bb2eeb9c8afc37bfc44cef8d70aab32948608a 100644 (file)
--- a/docs/libcurl/opts/Makefile.am
+++ b/docs/libcurl/opts/Makefile.am
@@ -300,6 +300,7 @@ man_MANS =                                      \
  CURLOPT_STREAM_DEPENDS.3                       \
  CURLOPT_STREAM_DEPENDS_E.3                     \
  CURLOPT_STREAM_WEIGHT.3                        \
+ CURLOPT_SUPPRESS_CONNECT_HEADERS.3             \
  CURLOPT_TCP_FASTOPEN.3                         \
  CURLOPT_TCP_KEEPALIVE.3                        \
  CURLOPT_TCP_KEEPIDLE.3                         \

diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions
index f4cd9805f2a47a8c40fe06919382de19b5ff14ad..6ac8869cadf0277f55ee698d4d681505f7474227 100644 (file)
--- a/docs/libcurl/symbols-in-versions
+++ b/docs/libcurl/symbols-in-versions
@@ -559,6 +559,7 @@ CURLOPT_STDERR                  7.1
 CURLOPT_STREAM_DEPENDS          7.46.0
 CURLOPT_STREAM_DEPENDS_E        7.46.0
 CURLOPT_STREAM_WEIGHT           7.46.0
+CURLOPT_SUPPRESS_CONNECT_HEADERS 7.54.0
 CURLOPT_TCP_KEEPALIVE           7.25.0
 CURLOPT_TCP_KEEPIDLE            7.25.0
 CURLOPT_TCP_KEEPINTVL           7.25.0

diff --git a/include/curl/curl.h b/include/curl/curl.h
index 8b221ef8ff2cce60de3807fb247fb6097199ca96..7c7d28d6a812928abfe8e14073dce212be7cd6f6 100644 (file)
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -1778,6 +1778,9 @@ typedef enum {
   /* Path to an abstract Unix domain socket */
   CINIT(ABSTRACT_UNIX_SOCKET, STRINGPOINT, 264),
 
+  /* Suppress proxy CONNECT response headers from user callbacks */
+  CINIT(SUPPRESS_CONNECT_HEADERS, LONG, 265),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

diff --git a/lib/http_proxy.c b/lib/http_proxy.c
index a673286478a4c37f7773e6444216b4405d16eb1c..f09304a26f8dbe93e77429aed32ac245fbefd91f 100644 (file)
--- a/lib/http_proxy.c
+++ b/lib/http_proxy.c
@@ -316,8 +316,6 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
       perline = 0;
 
       while(nread < BUFSIZE && keepon && !error) {
-        int writetype;
-
         if(Curl_pgrsUpdate(conn))
           return CURLE_ABORTED_BY_CALLBACK;
 
@@ -419,19 +417,20 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
           Curl_debug(data, CURLINFO_HEADER_IN,
                      line_start, (size_t)perline, conn);
 
-        /* send the header to the callback */
-        writetype = CLIENTWRITE_HEADER;
-        if(data->set.include_header)
-          writetype |= CLIENTWRITE_BODY;
+        if(!data->set.suppress_connect_headers) {
+          /* send the header to the callback */
+          int writetype = CLIENTWRITE_HEADER;
+          if(data->set.include_header)
+            writetype |= CLIENTWRITE_BODY;
 
-        result = Curl_client_write(conn, writetype, line_start, perline);
+          result = Curl_client_write(conn, writetype, line_start, perline);
+          if(result)
+            return result;
+        }
 
         data->info.header_size += (long)perline;
         data->req.headerbytecount += (long)perline;
 
-        if(result)
-          return result;
-
         /* Newlines are CRLF, so the CR is ignored as the line isn't
            really terminated until the LF comes. Treat a following CR
            as end-of-headers as well.*/

diff --git a/lib/url.c b/lib/url.c
index f9c32426913db66cd0325e13e4ab46ea593abd31..c306b577682893ff2407f180f820dc2de413b2f7 100644 (file)
--- a/lib/url.c
+++ b/lib/url.c
@@ -2894,6 +2894,9 @@ CURLcode Curl_setopt(struct Curl_easy *data, CURLoption option,
   case CURLOPT_CONNECT_TO:
     data->set.connect_to = va_arg(param, struct curl_slist *);
     break;
+  case CURLOPT_SUPPRESS_CONNECT_HEADERS:
+    data->set.suppress_connect_headers = (0 != va_arg(param, long))?TRUE:FALSE;
+    break;
   default:
     /* unknown tag and its companion, just ignore: */
     result = CURLE_UNKNOWN_OPTION;

diff --git a/lib/urldata.h b/lib/urldata.h
index 7402332d1957a6983cee77ccb1635841d216762e..2dd7938e0ee927fcada4566df6acddfbbc57a82d 100644 (file)
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1756,6 +1756,8 @@ struct UserDefined {
   bool pipewait;        /* wait for pipe/multiplex status before starting a
                            new connection */
   long expect_100_timeout; /* in milliseconds */
+  bool suppress_connect_headers;  /* suppress proxy CONNECT response headers
+                                     from user callbacks */
 
   struct Curl_easy *stream_depends_on;
   bool stream_depends_e; /* set or don't set the Exclusive bit */

diff --git a/packages/OS400/curl.inc.in b/packages/OS400/curl.inc.in
index b9b0808c7d01d5e9c59672257709e64db3d715cf..e4d2d30cafd8595985a517cd92e5a7597bdde884 100644 (file)
--- a/packages/OS400/curl.inc.in
+++ b/packages/OS400/curl.inc.in
@@ -1310,6 +1310,8 @@
      d                 c                   10263
      d  CURLOPT_ABSTRACT_UNIX_SOCKET...
      d                 c                   10264
+     d  CURLOPT_SUPPRESS_CONNECT_HEADERS...
+     d                 c                   00265
       *
       /if not defined(CURL_NO_OLDIES)
      d  CURLOPT_FILE   c                   10001

diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h
index b05c440e5388b68a89e21209ab949362b146a674..b7fe1d340a8f1791de420dd9d4c43bc9e8202f73 100644 (file)
--- a/src/tool_cfgable.h
+++ b/src/tool_cfgable.h
@@ -235,6 +235,8 @@ struct OperationConfig {
   bool falsestart;
   bool path_as_is;
   double expect100timeout;
+  bool suppress_connect_headers;  /* suppress proxy CONNECT response headers
+                                     from user callbacks */
   struct GlobalConfig *global;
   struct OperationConfig *prev;
   struct OperationConfig *next;   /* Always last in the struct */

diff --git a/src/tool_getparam.c b/src/tool_getparam.c
index 85d75ae6e36609c1f7df76533a2e4bdce9d76ce2..e7ac35476a0db4a9623eb0f56acc40b35278a26b 100644 (file)
--- a/src/tool_getparam.c
+++ b/src/tool_getparam.c
@@ -185,6 +185,7 @@ static const struct LongShort aliases[]= {
   {"$U", "connect-to",               TRUE},
   {"$W", "abstract-unix-socket",     TRUE},
   {"$X", "tls-max",                  TRUE},
+  {"$Y", "suppress-connect-headers", FALSE},
   {"0",   "http1.0",                 FALSE},
   {"01",  "http1.1",                 FALSE},
   {"02",  "http2",                   FALSE},
@@ -1066,6 +1067,9 @@ ParameterError getparameter(char *flag,    /* f or -long-flag */
         if(err)
           return err;
         break;
+      case 'Y': /* --suppress-connect-headers */
+        config->suppress_connect_headers = toggle;
+        break;
       }
       break;
     case '#': /* --progress-bar */

diff --git a/src/tool_help.c b/src/tool_help.c
index f6fe3527b8174c173d95a8fb571b4509ed86ad40..86d35899ad4880beb1efd5c0ce0d193414985cd3 100644 (file)
--- a/src/tool_help.c
+++ b/src/tool_help.c
@@ -249,6 +249,7 @@ static const char *const helptext[] = {
   "     --ssl-allow-beast  Allow security flaw to improve interop (SSL)",
   "     --ssl-no-revoke    Disable cert revocation checks (WinSSL)",
   "     --stderr FILE   Where to redirect stderr (use \"-\" for stdout)",
+  "     --suppress-connect-headers  Suppress proxy CONNECT response headers",
   "     --tcp-nodelay   Use the TCP_NODELAY option",
   "     --tcp-fastopen  Use TCP Fast Open",
   " -t, --telnet-option OPT=VAL  Set telnet option",

diff --git a/src/tool_operate.c b/src/tool_operate.c
index a489b8dbdb9d40645c9f1b97c5c3450965a91b9b..8f767152a2f6d16c2af1af7c22b28415be32d5ed 100644 (file)
--- a/src/tool_operate.c
+++ b/src/tool_operate.c
@@ -886,8 +886,11 @@ static CURLcode operate_do(struct GlobalConfig *global,
 
           /* new in libcurl 7.19.4 */
           my_setopt_str(curl, CURLOPT_NOPROXY, config->noproxy);
+
+          my_setopt(curl, CURLOPT_SUPPRESS_CONNECT_HEADERS,
+                    config->suppress_connect_headers?1L:0L);
         }
-#endif
+#endif /* !CURL_DISABLE_PROXY */
 
         my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror?1L:0L);
         my_setopt(curl, CURLOPT_UPLOAD, uploadfile?1L:0L);

diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index a115ab69d2565edaeaa0d01baee120eb176f8055..7742bcfecf9d92fead61ac895271fbcbf10b62b2 100644 (file)
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -132,6 +132,7 @@ test1252 test1253 test1254 test1255 test1256 test1257 test1258 test1259 \
 test1260 \
 \
 test1280 test1281 test1282 test1283 test1284 test1285 test1286 test1287 \
+test1288 \
 \
 test1300 test1301 test1302 test1303 test1304 test1305 test1306 test1307 \
 test1308 test1309 test1310 test1311 test1312 test1313 test1314 test1315 \

diff --git a/tests/data/test1288 b/tests/data/test1288
new file mode 100644 (file)
index 0000000..543aa3d
--- /dev/null
+++ b/tests/data/test1288
@@ -0,0 +1,96 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+HTTP CONNECT
+HTTP proxy
+proxytunnel
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<connect>
+HTTP/1.1 200 Mighty fine indeed\r
+Server: test tunnel 2000\r
+\r
+</connect>
+
+<data nocheck="yes">
+HTTP/1.1 200 OK\r
+Date: Thu, 09 Nov 2010 14:49:00 GMT\r
+Server: test-server/fake\r
+Content-Type: text/html\r
+Funny-head: yesyes\r
+Content-Length: 9\r
+Connection: keep-alive\r
+\r
+contents
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+http-proxy
+</server>
+<name>
+Suppress proxy CONNECT response headers
+</name>
+<command>
+--proxytunnel --suppress-connect-headers --dump-header - --include --write-out "\nCONNECT CODE: %{http_connect}\nRECEIVED HEADER BYTE TOTAL: %{size_header}\n" --proxy %HOSTIP:%PROXYPORT http://%HOSTIP.1288:%HTTPPORT/we/want/that/page/1288
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<proxy>
+CONNECT %HOSTIP.1288:%HTTPPORT HTTP/1.1\r
+Host: %HOSTIP.1288:%HTTPPORT\r
+Proxy-Connection: Keep-Alive\r
+\r
+</proxy>
+<protocol>
+GET /we/want/that/page/1288 HTTP/1.1\r
+Host: %HOSTIP.1288:%HTTPPORT\r
+Accept: */*\r
+\r
+</protocol>
+
+# This test is structured to test all the expectations of
+# --suppress-connect-headers, which are:
+# Must suppress in --include and --dump-header
+# Must not suppress in --verbose and --trace
+# Must not suppress in statistics (eg received header byte total)
+<stdout>
+HTTP/1.1 200 OK\r
+HTTP/1.1 200 OK\r
+Date: Thu, 09 Nov 2010 14:49:00 GMT\r
+Date: Thu, 09 Nov 2010 14:49:00 GMT\r
+Server: test-server/fake\r
+Server: test-server/fake\r
+Content-Type: text/html\r
+Content-Type: text/html\r
+Funny-head: yesyes\r
+Funny-head: yesyes\r
+Content-Length: 9\r
+Content-Length: 9\r
+Connection: keep-alive\r
+Connection: keep-alive\r
+\r
+\r
+contents
+
+CONNECT CODE: 200
+RECEIVED HEADER BYTE TOTAL: 231
+</stdout>
+</verify>
+</testcase>