Numbers greater than 1 are usually non-negative.

BN_is_prime_fasttest_ex begins by rejecting if a <= 1. Then it goes to
set A := abs(a), but a cannot be negative at this point.

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3275)
(cherry picked from commit 8b24f94209676bbe9933affd2879a686b1ed044d)

diff --git a/crypto/bn/bn_prime.c b/crypto/bn/bn_prime.c
index 8177fd294772f29f64a2a28bd56db27cca6d6a6f..e911e157859aae24ef7d9d5c54ddbc49e45f98e0 100644 (file)
--- a/crypto/bn/bn_prime.c
+++ b/crypto/bn/bn_prime.c
@@ -252,7 +252,6 @@ int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
     BN_CTX *ctx = NULL;
     BIGNUM *A1, *A1_odd, *check; /* taken from ctx */
     BN_MONT_CTX *mont = NULL;
-    const BIGNUM *A = NULL;
 
     if (BN_cmp(a, BN_value_one()) <= 0)
         return 0;
@@ -278,25 +277,14 @@ int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
         goto err;
     BN_CTX_start(ctx);
 
-    /* A := abs(a) */
-    if (a->neg) {
-        BIGNUM *t;
-        if ((t = BN_CTX_get(ctx)) == NULL)
-            goto err;
-        if (BN_copy(t, a) == NULL)
-            goto err;
-        t->neg = 0;
-        A = t;
-    } else
-        A = a;
     A1 = BN_CTX_get(ctx);
     A1_odd = BN_CTX_get(ctx);
     check = BN_CTX_get(ctx);
     if (check == NULL)
         goto err;
 
-    /* compute A1 := A - 1 */
-    if (!BN_copy(A1, A))
+    /* compute A1 := a - 1 */
+    if (!BN_copy(A1, a))
         goto err;
     if (!BN_sub_word(A1, 1))
         goto err;
@@ -312,11 +300,11 @@ int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
     if (!BN_rshift(A1_odd, A1, k))
         goto err;
 
-    /* Montgomery setup for computations mod A */
+    /* Montgomery setup for computations mod a */
     mont = BN_MONT_CTX_new();
     if (mont == NULL)
         goto err;
-    if (!BN_MONT_CTX_set(mont, A, ctx))
+    if (!BN_MONT_CTX_set(mont, a, ctx))
         goto err;
 
     for (i = 0; i < checks; i++) {
@@ -324,9 +312,9 @@ int BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
             goto err;
         if (!BN_add_word(check, 1))
             goto err;
-        /* now 1 <= check < A */
+        /* now 1 <= check < a */
 
-        j = witness(check, A, A1, A1_odd, k, ctx, mont);
+        j = witness(check, a, A1, A1_odd, k, ctx, mont);
         if (j == -1)
             goto err;
         if (j) {