vauth/cleartext: fix integer overflow check

Make the integer overflow check not rely on the undefined behavior that
a size_t wraps around on overflow.

Detected by lgtm.com
Closes #2408

diff --git a/lib/curl_ntlm_core.c b/lib/curl_ntlm_core.c
index e8962769ca4712da4cc9cf6e5bc426ba7ada7cb4..72eda34ad19c620a6fe27fddfc68987f3cb108da 100644 (file)
--- a/lib/curl_ntlm_core.c
+++ b/lib/curl_ntlm_core.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -646,15 +646,6 @@ CURLcode Curl_hmac_md5(const unsigned char *key, unsigned int keylen,
   return CURLE_OK;
 }
 
-#ifndef SIZE_T_MAX
-/* some limits.h headers have this defined, some don't */
-#if defined(SIZEOF_SIZE_T) && (SIZEOF_SIZE_T > 4)
-#define SIZE_T_MAX 18446744073709551615U
-#else
-#define SIZE_T_MAX 4294967295U
-#endif
-#endif
-
 /* This creates the NTLMv2 hash by using NTLM hash as the key and Unicode
  * (uppercase UserName + Domain) as the data
  */

diff --git a/lib/curl_setup.h b/lib/curl_setup.h
index f128696e9d075b7875992ef19e6466d544136333..e4503c64caccc63a602371d2824097e7b2291415 100644 (file)
--- a/lib/curl_setup.h
+++ b/lib/curl_setup.h
@@ -447,6 +447,15 @@
 #  endif
 #endif
 
+#ifndef SIZE_T_MAX
+/* some limits.h headers have this defined, some don't */
+#if defined(SIZEOF_SIZE_T) && (SIZEOF_SIZE_T > 4)
+#define SIZE_T_MAX 18446744073709551615U
+#else
+#define SIZE_T_MAX 4294967295U
+#endif
+#endif
+
 /*
  * Arg 2 type for gethostname in case it hasn't been defined in config file.
  */

diff --git a/lib/vauth/cleartext.c b/lib/vauth/cleartext.c
index a761ae7846616104b46e2514852ecdc1a3848a47..5d61ce6dc2e5e924a3d95aeed0fcdd9128f8e973 100644 (file)
--- a/lib/vauth/cleartext.c
+++ b/lib/vauth/cleartext.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -73,16 +73,10 @@ CURLcode Curl_auth_create_plain_message(struct Curl_easy *data,
   ulen = strlen(userp);
   plen = strlen(passwdp);
 
-  /* Compute binary message length, checking for overflows. */
-  plainlen = 2 * ulen;
-  if(plainlen < ulen)
-    return CURLE_OUT_OF_MEMORY;
-  plainlen += plen;
-  if(plainlen < plen)
-    return CURLE_OUT_OF_MEMORY;
-  plainlen += 2;
-  if(plainlen < 2)
+  /* Compute binary message length. Check for overflows. */
+  if((ulen > SIZE_T_MAX/2) || (plen > (SIZE_T_MAX/2 - 2)))
     return CURLE_OUT_OF_MEMORY;
+  plainlen = 2 * ulen + plen + 2;
 
   plainauth = malloc(plainlen);
   if(!plainauth)