Change the return type of EVP_EncodeUpdate

Previously EVP_EncodeUpdate returned a void. However there are a couple
of error conditions that can occur. Therefore the return type has been
changed to an int, with 0 indicating error and 1 indicating success.

Reviewed-by: Rich Salz <rsalz@openssl.org>

diff --git a/CHANGES b/CHANGES
index 792f602d767eab30643c83db54f4e2a6ddb468b9..ef01b27e4800b3067a08f3c4b54562766b8b589a 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -4,6 +4,11 @@
 
  Changes between 1.0.2h and 1.1.0  [xx XXX 2016]
 
+  *) The EVP_EncryptUpdate() function has had its return type changed from void
+     to int. A return of 0 indicates and error while a return of 1 indicates
+     success.
+     [Matt Caswell]
+
   *) The flags RSA_FLAG_NO_CONSTTIME, DSA_FLAG_NO_EXP_CONSTTIME and
      DH_FLAG_NO_EXP_CONSTTIME which previously provided the ability to switch
      off the constant time implementation for RSA, DSA and DH have been made

diff --git a/crypto/evp/bio_b64.c b/crypto/evp/bio_b64.c
index 9067848578d369438050a2c4db380438a9f14829..32a884a711049d7f09e6611bdf83c941a3611e86 100644 (file)
--- a/crypto/evp/bio_b64.c
+++ b/crypto/evp/bio_b64.c
@@ -403,9 +403,10 @@ static int b64_write(BIO *b, const char *in, int inl)
                 ret += n;
             }
         } else {
-            EVP_EncodeUpdate(ctx->base64,
-                             (unsigned char *)ctx->buf, &ctx->buf_len,
-                             (unsigned char *)in, n);
+            if (!EVP_EncodeUpdate(ctx->base64,
+                                 (unsigned char *)ctx->buf, &ctx->buf_len,
+                                 (unsigned char *)in, n))
+                return ((ret == 0) ? -1 : ret);
             OPENSSL_assert(ctx->buf_len <= (int)sizeof(ctx->buf));
             OPENSSL_assert(ctx->buf_len >= ctx->buf_off);
             ret += n;

diff --git a/crypto/evp/encode.c b/crypto/evp/encode.c
index bd2bbc0b099a9882f9748b84b2874208f5a76877..e026a8f0caf89b5a8f73ebf4b2bcb089f7b6d380 100644 (file)
--- a/crypto/evp/encode.c
+++ b/crypto/evp/encode.c
@@ -114,7 +114,7 @@ void EVP_EncodeInit(EVP_ENCODE_CTX *ctx)
     ctx->line_num = 0;
 }
 
-void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
+int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
                       const unsigned char *in, int inl)
 {
     int i, j;
@@ -122,12 +122,12 @@ void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
 
     *outl = 0;
     if (inl <= 0)
-        return;
+        return 0;
     OPENSSL_assert(ctx->length <= (int)sizeof(ctx->enc_data));
     if (ctx->length - ctx->num > inl) {
         memcpy(&(ctx->enc_data[ctx->num]), in, inl);
         ctx->num += inl;
-        return;
+        return 1;
     }
     if (ctx->num != 0) {
         i = ctx->length - ctx->num;
@@ -153,12 +153,14 @@ void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
     if (total > INT_MAX) {
         /* Too much output data! */
         *outl = 0;
-        return;
+        return 0;
     }
     if (inl != 0)
         memcpy(&(ctx->enc_data[0]), in, inl);
     ctx->num = inl;
     *outl = total;
+
+    return 1;
 }
 
 void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl)

diff --git a/crypto/pem/pem_lib.c b/crypto/pem/pem_lib.c
index 90893f1954552f80907b7270fc99fde636e36cb0..8965fda8d0207e6f1abbfcd7e16951bdfe136985 100644 (file)
--- a/crypto/pem/pem_lib.c
+++ b/crypto/pem/pem_lib.c
@@ -618,7 +618,8 @@ int PEM_write_bio(BIO *bp, const char *name, const char *header,
     i = j = 0;
     while (len > 0) {
         n = (int)((len > (PEM_BUFSIZE * 5)) ? (PEM_BUFSIZE * 5) : len);
-        EVP_EncodeUpdate(ctx, buf, &outl, &(data[j]), n);
+        if (!EVP_EncodeUpdate(ctx, buf, &outl, &(data[j]), n))
+            goto err;
         if ((outl) && (BIO_write(bp, (char *)buf, outl) != outl))
             goto err;
         i += outl;

diff --git a/doc/crypto/EVP_EncodeInit.pod b/doc/crypto/EVP_EncodeInit.pod
index f65322617d806c4a9846e78394331075ec78028b..4f62e71ec61c9fb480f0699494ada95c29512b1b 100644 (file)
--- a/doc/crypto/EVP_EncodeInit.pod
+++ b/doc/crypto/EVP_EncodeInit.pod
@@ -15,8 +15,8 @@ routines
  void EVP_ENCODE_CTX_free(EVP_ENCODE_CTX *ctx);
  int EVP_ENCODE_CTX_num(EVP_ENCODE_CTX *ctx);
  void EVP_EncodeInit(EVP_ENCODE_CTX *ctx);
- void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
-                       const unsigned char *in, int inl);
+ int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
+                      const unsigned char *in, int inl);
  void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl);
  int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int n);
 
@@ -66,7 +66,8 @@ any remainder). This gives the number of blocks of data that will be processed.
 Ensure the output buffer contains 65 bytes of storage for each block, plus an
 additional byte for a NUL terminator. EVP_EncodeUpdate() may be called
 repeatedly to process large amounts of input data. In the event of an error
-EVP_EncodeUpdate() will set B<*outl> to 0.
+EVP_EncodeUpdate() will set B<*outl> to 0 and return 0. On success 1 wil be
+returned.
 
 EVP_EncodeFinal() must be called at the end of an encoding operation. It will
 process any partial block of data remaining in the B<ctx> object. The output
@@ -129,6 +130,8 @@ object or NULL on error.
 EVP_ENCODE_CTX_num() returns the number of bytes pending encoding or decoding in
 B<ctx>.
 
+EVP_EncodeUpdate() returns 0 on error or 1 on success.
+
 EVP_EncodeBlock() returns the number of bytes encoded excluding the NUL
 terminator.
 

diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index 796f4ccb3cdc7d9f5fbcd1ed3ac3ec6df6101099..343cd9fd17cdb28d5789a79380e2bd8b3cd203fb 100644 (file)
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -610,8 +610,8 @@ EVP_ENCODE_CTX *EVP_ENCODE_CTX_new(void);
 void EVP_ENCODE_CTX_free(EVP_ENCODE_CTX *ctx);
 int EVP_ENCODE_CTX_num(EVP_ENCODE_CTX *ctx);
 void EVP_EncodeInit(EVP_ENCODE_CTX *ctx);
-void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
-                      const unsigned char *in, int inl);
+int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
+                     const unsigned char *in, int inl);
 void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl);
 int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int n);