ssh: add the ability to enable compression (for SCP/SFTP)

The required low-level logic was already available as part of
`libssh2` (via `LIBSSH2_FLAG_COMPRESS` `libssh2_session_flag()`[1]
option.)

This patch adds the new `libcurl` option `CURLOPT_SSH_COMPRESSION`
(boolean) and the new `curl` command-line option `--compressed-ssh`
to request this `libssh2` feature. To have compression enabled, it
is required that the SSH server supports a (zlib) compatible
compression method and that `libssh2` was built with `zlib` support
enabled.

[1] https://www.libssh2.org/libssh2_session_flag.html

Ref: https://github.com/curl/curl/issues/1732
Closes https://github.com/curl/curl/pull/1735

diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc
index 7eea5c6c5b455b9a9e38dd70319735084c6707e1..e8f46410bc538e4d0fce0cbda0cf23a91f9bb2a1 100644 (file)
--- a/docs/cmdline-opts/Makefile.inc
+++ b/docs/cmdline-opts/Makefile.inc
@@ -1,7 +1,8 @@
 # Shared between Makefile.am and CMakeLists.txt
 
 DPAGES = abstract-unix-socket.d anyauth.d append.d basic.d cacert.d capath.d cert.d \
-  cert-status.d cert-type.d ciphers.d compressed.d config.d             \
+  cert-status.d cert-type.d ciphers.d compressed.d compressed-ssh.d     \
+  config.d                                                              \
   connect-timeout.d connect-to.d continue-at.d cookie.d cookie-jar.d    \
   create-dirs.d crlf.d crlfile.d data-ascii.d data-binary.d data.d      \
   data-raw.d data-urlencode.d delegation.d digest.d disable.d           \

diff --git a/docs/cmdline-opts/compressed-ssh.d b/docs/cmdline-opts/compressed-ssh.d
new file mode 100644 (file)
index 0000000..beb5cb7
--- /dev/null
+++ b/docs/cmdline-opts/compressed-ssh.d
@@ -0,0 +1,6 @@
+Long: compressed-ssh
+Help: Enable SSH compression
+Protocols: SCP SFTP
+---
+Enables built-in SSH compression.
+This is a request, not an order; the server may or may not do it.

diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3
index 26f73ee1402067f242346d8254c29d98467e907e..01dfa85b64f960d6e4170d0a7792f6a25ecba603 100644 (file)
--- a/docs/libcurl/curl_easy_setopt.3
+++ b/docs/libcurl/curl_easy_setopt.3
@@ -547,6 +547,8 @@ Disable GSS-API delegation. See \fICURLOPT_GSSAPI_DELEGATION(3)\fP
 .SH SSH OPTIONS
 .IP CURLOPT_SSH_AUTH_TYPES
 SSH authentication types. See \fICURLOPT_SSH_AUTH_TYPES(3)\fP
+.IP CURLOPT_SSH_COMPRESSION
+Enable SSH compression. See \fICURLOPT_SSH_COMPRESSION(3)\fP
 .IP CURLOPT_SSH_HOST_PUBLIC_KEY_MD5
 MD5 of host's public key. See \fICURLOPT_SSH_HOST_PUBLIC_KEY_MD5(3)\fP
 .IP CURLOPT_SSH_PUBLIC_KEYFILE

diff --git a/docs/libcurl/opts/CURLOPT_SSH_COMPRESSION.3 b/docs/libcurl/opts/CURLOPT_SSH_COMPRESSION.3
new file mode 100644 (file)
index 0000000..1933205
--- /dev/null
+++ b/docs/libcurl/opts/CURLOPT_SSH_COMPRESSION.3
@@ -0,0 +1,58 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_SSH_COMPRESSION 3 "05 Aug 2017" "libcurl 7.56.0" "curl_easy_setopt options"
+.SH NAME
+CURLOPT_SSH_COMPRESSION \- enables automatic decompression of HTTP downloads
+.SH SYNOPSIS
+#include <curl/curl.h>
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_SSH_COMPRESSION, long enable);
+.SH DESCRIPTION
+Pass a long as parameter to enable or disable.
+
+Enables built-in SSH compression.
+This is a request, not an order; the server may or may not do it.
+.SH DEFAULT
+
+.SH PROTOCOLS
+All SSH based protocols: SCP, SFTP
+.SH EXAMPLE
+.nf
+CURL *curl = curl_easy_init();
+if(curl) {
+  curl_easy_setopt(curl, CURLOPT_URL, "sftp://example.com");
+
+  /* enable built-in compression */
+  curl_easy_setopt(curl, CURLOPT_SSH_COMPRESSION, 1L);
+
+  /* Perform the request */
+  curl_easy_perform(curl);
+}
+.fi
+.SH AVAILABILITY
+Added in 7.56.0
+.SH RETURN VALUE
+Returns CURLE_OK if the option is supported, CURLE_UNKNOWN_OPTION if not, or
+CURLE_OUT_OF_MEMORY if there was insufficient heap space.
+.SH "SEE ALSO"
+.BR CURLOPT_ACCEPT_ENCODING "(3), " CURLOPT_TRANSFER_ENCODING "(3), "

diff --git a/docs/libcurl/opts/Makefile.inc b/docs/libcurl/opts/Makefile.inc
index c3db5d225181005da79287b70a2ae1d6c1ad7013..ad09dce2d32c2517b0f401ccf1e894ecf53ee780 100644 (file)
--- a/docs/libcurl/opts/Makefile.inc
+++ b/docs/libcurl/opts/Makefile.inc
@@ -258,6 +258,7 @@ man_MANS =                                      \
   CURLOPT_SOCKS5_GSSAPI_NEC.3                   \
   CURLOPT_SOCKS5_GSSAPI_SERVICE.3               \
   CURLOPT_SSH_AUTH_TYPES.3                      \
+  CURLOPT_SSH_COMPRESSION.3                     \
   CURLOPT_SSH_HOST_PUBLIC_KEY_MD5.3             \
   CURLOPT_SSH_KEYDATA.3                         \
   CURLOPT_SSH_KEYFUNCTION.3                     \

diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions
index e141b6a5726222f03a2dbe040fd295e9466b631e..d3a2ff8866bbef95eaf42a93ab10d3eee6eb671b 100644 (file)
--- a/docs/libcurl/symbols-in-versions
+++ b/docs/libcurl/symbols-in-versions
@@ -540,6 +540,7 @@ CURLOPT_SOURCE_QUOTE            7.13.0        -           7.15.5
 CURLOPT_SOURCE_URL              7.13.0        -           7.15.5
 CURLOPT_SOURCE_USERPWD          7.12.1        -           7.15.5
 CURLOPT_SSH_AUTH_TYPES          7.16.1
+CURLOPT_SSH_COMPRESSION         7.56.0
 CURLOPT_SSH_HOST_PUBLIC_KEY_MD5 7.17.1
 CURLOPT_SSH_KEYDATA             7.19.6
 CURLOPT_SSH_KEYFUNCTION         7.19.6

diff --git a/include/curl/curl.h b/include/curl/curl.h
index 82fb8aeb83dd3f71df7f7667ff7a8b629b919e30..3936d1e43c848dcad1b01b114a09e46761b1c03f 100644 (file)
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -1788,6 +1788,9 @@ typedef enum {
   /* bitmask of allowed auth methods for connections to SOCKS5 proxies */
   CINIT(SOCKS5_AUTH, LONG, 267),
 
+  /* Enable/disable SSH compression */
+  CINIT(SSH_COMPRESSION, LONG, 268),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

diff --git a/lib/ssh.c b/lib/ssh.c
index cc5178fe74ec2f6c8353955c3ec601c87673529b..28f709ada70043d04e25eafe1db5d6294945aba2 100644 (file)
--- a/lib/ssh.c
+++ b/lib/ssh.c
@@ -2942,6 +2942,13 @@ static CURLcode ssh_connect(struct connectdata *conn, bool *done)
     return CURLE_FAILED_INIT;
   }
 
+  if(data->set.ssh_compression) {
+#if LIBSSH2_VERSION_NUM >= 0x010208
+    if(libssh2_session_flag(ssh->ssh_session, LIBSSH2_FLAG_COMPRESS, 1) < 0)
+#endif
+      infof(data, "Failed to enable compression for ssh session\n");
+  }
+
 #ifdef HAVE_LIBSSH2_KNOWNHOST_API
   if(data->set.str[STRING_SSH_KNOWNHOSTS]) {
     int rc;

diff --git a/lib/url.c b/lib/url.c
index 71d4d8bb7fa949a1f2c87f1c69569a2681ee6e0d..32623abe59721a7866db1f6596fe4013bf07f238 100644 (file)
--- a/lib/url.c
+++ b/lib/url.c
@@ -2936,6 +2936,9 @@ CURLcode Curl_setopt(struct Curl_easy *data, CURLoption option,
   case CURLOPT_SUPPRESS_CONNECT_HEADERS:
     data->set.suppress_connect_headers = (0 != va_arg(param, long))?TRUE:FALSE;
     break;
+  case CURLOPT_SSH_COMPRESSION:
+    data->set.ssh_compression = (0 != va_arg(param, long))?TRUE:FALSE;
+    break;
   default:
     /* unknown tag and its companion, just ignore: */
     result = CURLE_UNKNOWN_OPTION;

diff --git a/lib/urldata.h b/lib/urldata.h
index e45baff17c9eb85474eaa2ff227bf06b88c3a79a..b0faa9a355acd4011c79d53d7e2edc94ff88e42c 100644 (file)
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1742,6 +1742,7 @@ struct UserDefined {
 
   curl_sshkeycallback ssh_keyfunc; /* key matching callback */
   void *ssh_keyfunc_userp;         /* custom pointer to callback */
+  bool ssh_compression;            /* enable SSH compression */
 
 /* Here follows boolean settings that define how to behave during
    this session. They are STATIC, set by libcurl users or at least initially

diff --git a/packages/OS400/curl.inc.in b/packages/OS400/curl.inc.in
index e4d2d30cafd8595985a517cd92e5a7597bdde884..41b6be190b2531067422cddf986fee550e9f43dc 100644 (file)
--- a/packages/OS400/curl.inc.in
+++ b/packages/OS400/curl.inc.in
@@ -1312,6 +1312,8 @@
      d                 c                   10264
      d  CURLOPT_SUPPRESS_CONNECT_HEADERS...
      d                 c                   00265
+     d  CURLOPT_SSH_COMPRESSION...
+     d                 c                   00268
       *
       /if not defined(CURL_NO_OLDIES)
      d  CURLOPT_FILE   c                   10001

diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h
index 254805c8c876dcb6abc11f8576cc6ee0a6b402b3..2b436657f24e334bae4da9ebd5eb29156291b5ab 100644 (file)
--- a/src/tool_cfgable.h
+++ b/src/tool_cfgable.h
@@ -247,6 +247,7 @@ struct OperationConfig {
                                      from user callbacks */
   curl_error synthetic_error;     /* if non-zero, it overrides any libcurl
                                      error */
+  bool ssh_compression;           /* enable/disable SSH compression */
   struct GlobalConfig *global;
   struct OperationConfig *prev;
   struct OperationConfig *next;   /* Always last in the struct */

diff --git a/src/tool_getparam.c b/src/tool_getparam.c
index 40b39a8aa25d025afaa37372c5064f6d73a0653a..3dd1dec3fe9308a9a92d6128beb7a3ab4736cf5e 100644 (file)
--- a/src/tool_getparam.c
+++ b/src/tool_getparam.c
@@ -188,6 +188,7 @@ static const struct LongShort aliases[]= {
   {"$W", "abstract-unix-socket",     ARG_STRING},
   {"$X", "tls-max",                  ARG_STRING},
   {"$Y", "suppress-connect-headers", ARG_BOOL},
+  {"$Z", "compressed-ssh",           ARG_BOOL},
   {"0",   "http1.0",                 ARG_NONE},
   {"01",  "http1.1",                 ARG_NONE},
   {"02",  "http2",                   ARG_NONE},
@@ -1076,6 +1077,9 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
       case 'Y': /* --suppress-connect-headers */
         config->suppress_connect_headers = toggle;
         break;
+      case 'Z': /* --compressed-ssh */
+        config->ssh_compression = toggle;
+        break;
       }
       break;
     case '#': /* --progress-bar */

diff --git a/src/tool_help.c b/src/tool_help.c
index a5bfaba24e1f4e8b10f7bd2c055841881f5d1c4b..04d84dd7a8184775d4c20d9540b28a2b879f3612 100644 (file)
--- a/src/tool_help.c
+++ b/src/tool_help.c
@@ -68,6 +68,8 @@ static const struct helptxt helptext[] = {
    "SSL ciphers to use"},
   {"    --compressed",
    "Request compressed response"},
+  {"    --compressed-ssh",
+   "Enable SSH compression"},
   {"-K, --config <file>",
    "Read config from a file"},
   {"    --connect-timeout <seconds>",

diff --git a/src/tool_operate.c b/src/tool_operate.c
index 1e8d0073c23215d93dda0c5553ca1bd7263fc79a..3c8c6ed2fe3f3acc97f0de446e38eb49a955efe8 100644 (file)
--- a/src/tool_operate.c
+++ b/src/tool_operate.c
@@ -1091,6 +1091,10 @@ static CURLcode operate_do(struct GlobalConfig *global,
              to fail if we are not talking to who we think we should */
           my_setopt_str(curl, CURLOPT_SSH_HOST_PUBLIC_KEY_MD5,
                         config->hostpubmd5);
+
+          /* new in libcurl 7.56.0 */
+          if(config->ssh_compression)
+            my_setopt(curl, CURLOPT_SSH_COMPRESSION, 1L);
         }
 
         if(config->cacert)

diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index b92f7f92ff114e10690fcca09978479fac17f5f9..88e5e2a9a6e2fb5f810472a599cc812c7a5f24c5 100644 (file)
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -77,7 +77,7 @@ test599 test600 test601 test602 test603 test604 test605 test606 test607 \
 test608 test609 test610 test611 test612 test613 test614 test615 test616 \
 test617 test618 test619 test620 test621 test622 test623 test624 test625 \
 test626 test627 test628 test629 test630 test631 test632 test633 test634 \
-test635 test636 test637 test638 test639 test640 test641 \
+test635 test636 test637 test638 test639 test640 test641 test642 \
 \
 test700 test701 test702 test703 test704 test705 test706 test707 test708 \
 test709 test710 test711 test712 test713 test714 test715 \

diff --git a/tests/data/test642 b/tests/data/test642
new file mode 100644 (file)
index 0000000..41fd444
--- /dev/null
+++ b/tests/data/test642
@@ -0,0 +1,42 @@
+<testcase>
+<info>
+<keywords>
+SFTP
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data>
+Test data
+for ssh test
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+sftp
+</server>
+ <name>
+SFTP retrieval
+ </name>
+ <command>
+--key curl_client_key --pubkey curl_client_key.pub -u %USER: --compressed-ssh sftp://%HOSTIP:%SSHPORT%PWD/log/file642.txt --insecure
+</command>
+<file name="log/file642.txt">
+Test data
+for ssh test
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<valgrind>
+disable
+</valgrind>
+</verify>
+</testcase>