sm2: check buffer size before writing ciphertext

The SM2 encryption may write past the caller-provided output buffer
when the required ciphertext size exceeds the supplied buffer length.

Reject outputs that do not fit in the caller-provided buffer.

Reviewed-by: Viktor Dukhovni <viktor@openssl.org>
Reviewed-by: Eugene Syromiatnikov <esyr@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.foundation>
MergeDate: Wed May  6 16:51:00 2026
(Merged from https://github.com/openssl/openssl/pull/30614)

diff --git a/crypto/sm2/sm2_crypt.c b/crypto/sm2/sm2_crypt.c
index 37993bc8c3f8a5da7fd7ecd54aca22ec18b78de1..e7ae6a8bd0bd89ba069b62801a767938121bae8e 100644 (file)
--- a/crypto/sm2/sm2_crypt.c
+++ b/crypto/sm2/sm2_crypt.c
@@ -253,12 +253,23 @@ again:
         goto done;
     }
 
-    ciphertext_leni = i2d_SM2_Ciphertext(&ctext_struct, &ciphertext_buf);
+    ciphertext_leni = i2d_SM2_Ciphertext(&ctext_struct, NULL);
     /* Ensure cast to size_t is safe */
     if (ciphertext_leni < 0) {
         ERR_raise(ERR_LIB_SM2, ERR_R_INTERNAL_ERROR);
         goto done;
     }
+
+    if (*ciphertext_len < (size_t)ciphertext_leni) {
+        ERR_raise(ERR_LIB_SM2, SM2_R_BUFFER_TOO_SMALL);
+        goto done;
+    }
+
+    ciphertext_leni = i2d_SM2_Ciphertext(&ctext_struct, &ciphertext_buf);
+    if (ciphertext_leni < 0) {
+        ERR_raise(ERR_LIB_SM2, ERR_R_INTERNAL_ERROR);
+        goto done;
+    }
     *ciphertext_len = (size_t)ciphertext_leni;
 
     rc = 1;