Update rsasve_recover to properly store outlen on success

Outlen was never validated in this function prior to use, nor is it set
to the decrypted value on sucess.  Add both of those operations

Fixes #25509

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
Reviewed-by: Viktor Dukhovni <viktor@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/25522)

(cherry picked from commit 0f9516855e3139ef999b58f2fa551afb3b6c2b15)

diff --git a/providers/implementations/kem/rsa_kem.c b/providers/implementations/kem/rsa_kem.c
index 365ae3d7d69b9f0677d6bc2def08eb0283aefb46..6b48c8e92b8c0b063719862b8756845eec5b2c83 100644 (file)
--- a/providers/implementations/kem/rsa_kem.c
+++ b/providers/implementations/kem/rsa_kem.c
@@ -265,6 +265,12 @@ static int rsasve_generate(PROV_RSA_CTX *prsactx,
             *secretlen = nlen;
         return 1;
     }
+
+    if (outlen != NULL && *outlen < nlen) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_OUTPUT_LENGTH);
+        return 0;
+    }
+
     /*
      * Step (2): Generate a random byte string z of nlen bytes where
      *            1 < z < n - 1
@@ -286,15 +292,33 @@ static int rsasve_generate(PROV_RSA_CTX *prsactx,
     return ret;
 }
 
-/*
- * NIST.SP.800-56Br2
+/**
+ * rsasve_recover - Recovers a secret value from ciphertext using an RSA
+ * private key.  Once, recovered, the secret value is considered to be a
+ * shared secret.  Algorithm is preformed as per
+ * NIST SP 800-56B Rev 2
  * 7.2.1.3 RSASVE Recovery Operation (RSASVE.RECOVER).
+ *
+ * This function performs RSA decryption using the private key from the
+ * provided RSA context (`prsactx`). It takes the input ciphertext, decrypts
+ * it, and writes the decrypted message to the output buffer.
+ *
+ * @prsactx:      The RSA context containing the private key.
+ * @out:          The output buffer to store the decrypted message.
+ * @outlen:       On input, the size of the output buffer. On successful
+ *                completion, the actual length of the decrypted message.
+ * @in:           The input buffer containing the ciphertext to be decrypted.
+ * @inlen:        The length of the input ciphertext in bytes.
+ *
+ * Returns 1 on success, or 0 on error. In case of error, appropriate
+ * error messages are raised using the ERR_raise function.
  */
 static int rsasve_recover(PROV_RSA_CTX *prsactx,
                           unsigned char *out, size_t *outlen,
                           const unsigned char *in, size_t inlen)
 {
     size_t nlen;
+    int ret;
 
     /* Step (1): get the byte length of n */
     nlen = RSA_size(prsactx->rsa);
@@ -308,13 +332,24 @@ static int rsasve_recover(PROV_RSA_CTX *prsactx,
         return 1;
     }
 
-    /* Step (2): check the input ciphertext 'inlen' matches the nlen */
+    /*
+     * Step (2): check the input ciphertext 'inlen' matches the nlen
+     * and that outlen is at least nlen bytes
+     */
     if (inlen != nlen) {
         ERR_raise(ERR_LIB_PROV, PROV_R_BAD_LENGTH);
         return 0;
     }
+    if (outlen != NULL && *outlen < nlen) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_OUTPUT_LENGTH);
+        return 0;
+    }
+
     /* Step (3): out = RSADP((n,d), in) */
-    return (RSA_private_decrypt(inlen, in, out, prsactx->rsa, RSA_NO_PADDING) > 0);
+    ret = RSA_private_decrypt(inlen, in, out, prsactx->rsa, RSA_NO_PADDING);
+    if (ret > 0 && outlen != NULL)
+        *outlen = ret;
+    return ret > 0;
 }
 
 static int rsakem_generate(void *vprsactx, unsigned char *out, size_t *outlen,