Add notes about ignoring initialization failures on contexts

Fixes #20130

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/20136)

diff --git a/doc/man3/EVP_DigestInit.pod b/doc/man3/EVP_DigestInit.pod
index b8a0cb91f8d70d13e72a15be25220fc471cbd2dd..167ebbbad53f3f04beedc3ed36b0aba0952eca51 100644 (file)
--- a/doc/man3/EVP_DigestInit.pod
+++ b/doc/man3/EVP_DigestInit.pod
@@ -573,6 +573,7 @@ Returns 1 for success or 0 for failure.
 EVP_Digest(),
 EVP_DigestInit_ex2(),
 EVP_DigestInit_ex(),
+EVP_DigestInit(),
 EVP_DigestUpdate(),
 EVP_DigestFinal_ex(),
 EVP_DigestFinalXOF(), and
@@ -651,6 +652,13 @@ are still in common use.
 For most applications the I<impl> parameter to EVP_DigestInit_ex() will be
 set to NULL to use the default digest implementation.
 
+Ignoring failure returns of EVP_DigestInit_ex(), EVP_DigestInit_ex2(), or
+EVP_DigestInit() can lead to undefined behavior on subsequent calls
+updating or finalizing the B<EVP_MD_CTX> such as the EVP_DigestUpdate() or
+EVP_DigestFinal() functions. The only valid calls on the B<EVP_MD_CTX>
+when initialization fails are calls that attempt another initialization of
+the context or release the context.
+
 The functions EVP_DigestInit(), EVP_DigestFinal() and EVP_MD_CTX_copy() are
 obsolete but are retained to maintain compatibility with existing code. New
 applications should use EVP_DigestInit_ex(), EVP_DigestFinal_ex() and
@@ -698,10 +706,26 @@ digest name passed on the command line.
      }
 
      mdctx = EVP_MD_CTX_new();
-     EVP_DigestInit_ex2(mdctx, md, NULL);
-     EVP_DigestUpdate(mdctx, mess1, strlen(mess1));
-     EVP_DigestUpdate(mdctx, mess2, strlen(mess2));
-     EVP_DigestFinal_ex(mdctx, md_value, &md_len);
+     if (!EVP_DigestInit_ex2(mdctx, md, NULL)) {
+         printf("Message digest initialization failed.\n");
+         EVP_MD_CTX_free(mdctx);
+         exit(1);
+     }
+     if (!EVP_DigestUpdate(mdctx, mess1, strlen(mess1))) {
+         printf("Message digest update failed.\n");
+         EVP_MD_CTX_free(mdctx);
+         exit(1);
+     }
+     if (!EVP_DigestUpdate(mdctx, mess2, strlen(mess2))) {
+         printf("Message digest update failed.\n");
+         EVP_MD_CTX_free(mdctx);
+         exit(1);
+     }
+     if (!EVP_DigestFinal_ex(mdctx, md_value, &md_len)) {
+         printf("Message digest finalization failed.\n");
+         EVP_MD_CTX_free(mdctx);
+         exit(1);
+     }
      EVP_MD_CTX_free(mdctx);
 
      printf("Digest is: ");

diff --git a/doc/man3/EVP_DigestSignInit.pod b/doc/man3/EVP_DigestSignInit.pod
index f8bd14145a79c9c22ede64131ada00733f552468..fe4a68618405b9578a5edcde9c0f62ed3e43c62c 100644 (file)
--- a/doc/man3/EVP_DigestSignInit.pod
+++ b/doc/man3/EVP_DigestSignInit.pod
@@ -172,6 +172,10 @@ multiple times on a context and the parameters set by previous calls should be
 preserved if the I<pkey> parameter is NULL. The call then just resets the state
 of the I<ctx>.
 
+Ignoring failure returns of EVP_DigestSignInit() and EVP_DigestSignInit_ex()
+functions can lead to subsequent undefined behavior when calling
+EVP_DigestSignUpdate(), EVP_DigestSignFinal(), or EVP_DigestSign().
+
 The use of EVP_PKEY_get_size() with these functions is discouraged because some
 signature operations may have a signature length which depends on the
 parameters set. As a result EVP_PKEY_get_size() would have to return a value

diff --git a/doc/man3/EVP_DigestVerifyInit.pod b/doc/man3/EVP_DigestVerifyInit.pod
index 42612d48d057fe879908a62bb7e565353b527b8d..d4bb8163d1f8def45872ff7c507bcff808ffe08c 100644 (file)
--- a/doc/man3/EVP_DigestVerifyInit.pod
+++ b/doc/man3/EVP_DigestVerifyInit.pod
@@ -161,6 +161,10 @@ multiple times on a context and the parameters set by previous calls should be
 preserved if the I<pkey> parameter is NULL. The call then just resets the state
 of the I<ctx>.
 
+Ignoring failure returns of EVP_DigestVerifyInit() and EVP_DigestVerifyInit_ex()
+functions can lead to subsequent undefined behavior when calling
+EVP_DigestVerifyUpdate(), EVP_DigestVerifyFinal(), or EVP_DigestVerify().
+
 =head1 SEE ALSO
 
 L<EVP_DigestSignInit(3)>,

diff --git a/doc/man3/EVP_EncryptInit.pod b/doc/man3/EVP_EncryptInit.pod
index 3a6755e27a615735ab15bc359a870d35c2ea62fe..514d72c54b25c3d89190933be41a5f6672a12d0c 100644 (file)
--- a/doc/man3/EVP_EncryptInit.pod
+++ b/doc/man3/EVP_EncryptInit.pod
@@ -1536,6 +1536,12 @@ removed, and it is especially important for the
 B<EVP_CIPHER_CTX_FLAG_WRAP_ALLOW> flag treated specially in
 EVP_CipherInit_ex().
 
+Ignoring failure returns of the B<EVP_CIPHER_CTX> initialization functions can
+lead to subsequent undefined behavior when calling the functions that update or
+finalize the context. The only valid calls on the B<EVP_CIPHER_CTX> when
+initialization fails are calls that attempt another initialization of the
+context or release the context.
+
 EVP_get_cipherbynid(), and EVP_get_cipherbyobj() are implemented as macros.
 
 =head1 BUGS
@@ -1568,7 +1574,11 @@ Encrypt a string using IDEA:
      FILE *out;
 
      ctx = EVP_CIPHER_CTX_new();
-     EVP_EncryptInit_ex2(ctx, EVP_idea_cbc(), key, iv, NULL);
+     if (!EVP_EncryptInit_ex2(ctx, EVP_idea_cbc(), key, iv, NULL)) {
+         /* Error */
+         EVP_CIPHER_CTX_free(ctx);
+         return 0;
+     }
 
      if (!EVP_EncryptUpdate(ctx, outbuf, &outlen, intext, strlen(intext))) {
          /* Error */
@@ -1626,13 +1636,21 @@ with a 128-bit key:
 
      /* Don't set key or IV right away; we want to check lengths */
      ctx = EVP_CIPHER_CTX_new();
-     EVP_CipherInit_ex2(ctx, EVP_aes_128_cbc(), NULL, NULL,
-                        do_encrypt, NULL);
+     if (!EVP_CipherInit_ex2(ctx, EVP_aes_128_cbc(), NULL, NULL,
+                             do_encrypt, NULL)) {
+         /* Error */
+         EVP_CIPHER_CTX_free(ctx);
+         return 0;
+     }
      OPENSSL_assert(EVP_CIPHER_CTX_get_key_length(ctx) == 16);
      OPENSSL_assert(EVP_CIPHER_CTX_get_iv_length(ctx) == 16);
 
      /* Now we can set key and IV */
-     EVP_CipherInit_ex2(ctx, NULL, key, iv, do_encrypt, NULL);
+     if (!EVP_CipherInit_ex2(ctx, NULL, key, iv, do_encrypt, NULL)) {
+         /* Error */
+         EVP_CIPHER_CTX_free(ctx);
+         return 0;
+     }
 
      for (;;) {
          inlen = fread(inbuf, 1, 1024, in);