ntlm: avoid integer overflow for malloc size

Reported-by: Alex Nichols
Assisted-by: Kamil Dudka and Max Dymond
CVE-2017-8816

Bug: https://curl.haxx.se/docs/adv_2017-11e7.html

diff --git a/lib/curl_ntlm_core.c b/lib/curl_ntlm_core.c
index 1309bf0d9989cbfcbe157e67c1088b4af1199a4a..43a290574c786b3a2287b423bc33c9378e9504a1 100644 (file)
--- a/lib/curl_ntlm_core.c
+++ b/lib/curl_ntlm_core.c
@@ -646,6 +646,12 @@ CURLcode Curl_hmac_md5(const unsigned char *key, unsigned int keylen,
   return CURLE_OK;
 }
 
+#if defined(SIZEOF_SIZE_T) && (SIZEOF_SIZE_T > 4)
+#define SIZE_T_MAX 18446744073709551615U
+#else
+#define SIZE_T_MAX 4294967295U
+#endif
+
 /* This creates the NTLMv2 hash by using NTLM hash as the key and Unicode
  * (uppercase UserName + Domain) as the data
  */
@@ -655,10 +661,20 @@ CURLcode Curl_ntlm_core_mk_ntlmv2_hash(const char *user, size_t userlen,
                                        unsigned char *ntlmv2hash)
 {
   /* Unicode representation */
-  size_t identity_len = (userlen + domlen) * 2;
-  unsigned char *identity = malloc(identity_len);
+  size_t identity_len;
+  unsigned char *identity;
   CURLcode result = CURLE_OK;
 
+  /* we do the length checks below separately to avoid integer overflow risk
+     on extreme data lengths */
+  if((userlen > SIZE_T_MAX/2) ||
+     (domlen > SIZE_T_MAX/2) ||
+     ((userlen + domlen) > SIZE_T_MAX/2))
+    return CURLE_OUT_OF_MEMORY;
+
+  identity_len = (userlen + domlen) * 2;
+  identity = malloc(identity_len);
+
   if(!identity)
     return CURLE_OUT_OF_MEMORY;