From: Matt Caswell <matt@openssl.org>
Date: Tue, 31 May 2016 10:28:14 +0000 (+0100)
Subject: Fix BN_mod_word bug
X-Git-Tag: OpenSSL_1_1_0-pre6~521
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=37258dadaa9e36db4b96a3aa54aa6c67136160cc;p=openssl

Fix BN_mod_word bug

On systems where we do not have BN_ULLONG (e.g. typically 64 bit systems)
then BN_mod_word() can return incorrect results if the supplied modulus is
too big.

RT#4501

Reviewed-by: Andy Polyakov <appro@openssl.org>
---

diff --git a/crypto/bn/bn_word.c b/crypto/bn/bn_word.c
index fd282980d0..a34244c4ad 100644
--- a/crypto/bn/bn_word.c
+++ b/crypto/bn/bn_word.c
@@ -22,10 +22,32 @@ BN_ULONG BN_mod_word(const BIGNUM *a, BN_ULONG w)
     if (w == 0)
         return (BN_ULONG)-1;
 
+#ifndef BN_LLONG
+    /*
+     * If |w| is too long and we don't have BN_ULLONG then we need to fall
+     * back to using BN_div_word
+     */
+    if (w > ((BN_ULONG)1 << BN_BITS4)) {
+        BIGNUM *tmp = BN_dup(a);
+        if (tmp == NULL)
+            return (BN_ULONG)-1;
+
+        ret = BN_div_word(tmp, w);
+        BN_free(tmp);
+
+        return ret;
+    }
+#endif
+
     bn_check_top(a);
     w &= BN_MASK2;
     for (i = a->top - 1; i >= 0; i--) {
 #ifndef BN_LLONG
+        /*
+         * We can assume here that | w <= ((BN_ULONG)1 << BN_BITS4) | and so
+         * | ret < ((BN_ULONG)1 << BN_BITS4) | and therefore the shifts here are
+         * safe and will not overflow
+         */
         ret = ((ret << BN_BITS4) | ((a->d[i] >> BN_BITS4) & BN_MASK2l)) % w;
         ret = ((ret << BN_BITS4) | (a->d[i] & BN_MASK2l)) % w;
 #else