OpenSSL: Fix HPKE in some corner cases

EVP_PKEY_derive() might report a larger maximum size of the output than
HPKE_MAX_SHARED_SECRET_LEN under some conditions. That should be allowed
as long as the real final length is within the maximum limit.
Furthermore, since we are using a fixed length buffer for this, there is
no need to call EVP_PKEY_derive() twice to first learn the maximum
length. Use a bit longer buffer and allow OpenSSL to take care of the
update to the final length internally with than single call.

This showed up using the following test case sequence:
dbus_pkcs11 dpp_private_peer_introduction

Signed-off-by: Jouni Malinen <quic_jouni@quicinc.com>

diff --git a/src/crypto/crypto_openssl.c b/src/crypto/crypto_openssl.c
index b9fc1106357f4ca12c3d4cf41560898df915e200..59458762f4cc86506789b6d91db64bf6c43e9c18 100644 (file)
--- a/src/crypto/crypto_openssl.c
+++ b/src/crypto/crypto_openssl.c
@@ -5020,7 +5020,7 @@ static int hpke_encap(struct hpke_context *ctx, struct crypto_ec_key *pk_r,
        EVP_PKEY_CTX *pctx = NULL;
        struct crypto_ec_key *sk_e;
        int res = -1;
-       u8 dhss[HPKE_MAX_SHARED_SECRET_LEN];
+       u8 dhss[HPKE_MAX_SHARED_SECRET_LEN + 16];
        size_t dhss_len;
        struct wpabuf *enc_buf = NULL, *pk_rm = NULL;
 
@@ -5033,13 +5033,13 @@ static int hpke_encap(struct hpke_context *ctx, struct crypto_ec_key *pk_r,
        }
 
        /* dh = DH(skE, pkR) */
+       dhss_len = sizeof(dhss);
        pctx = EVP_PKEY_CTX_new((EVP_PKEY *) sk_e, NULL);
        if (!pctx ||
            EVP_PKEY_derive_init(pctx) != 1 ||
            EVP_PKEY_derive_set_peer(pctx, (EVP_PKEY *) pk_r) != 1 ||
-           EVP_PKEY_derive(pctx, NULL, &dhss_len) != 1 ||
-           dhss_len > HPKE_MAX_SHARED_SECRET_LEN ||
-           EVP_PKEY_derive(pctx, dhss, &dhss_len) != 1) {
+           EVP_PKEY_derive(pctx, dhss, &dhss_len) != 1 ||
+           dhss_len > HPKE_MAX_SHARED_SECRET_LEN) {
                wpa_printf(MSG_INFO, "OpenSSL: EVP_PKEY_derive failed: %s",
                           ERR_error_string(ERR_get_error(), NULL));
                goto fail;
@@ -5184,7 +5184,7 @@ static int hpke_decap(struct hpke_context *ctx, const u8 *enc,
        size_t len;
        int res = -1;
        struct crypto_ec_key *pk_e = NULL;
-       u8 dhss[HPKE_MAX_SHARED_SECRET_LEN];
+       u8 dhss[HPKE_MAX_SHARED_SECRET_LEN + 16];
        size_t dhss_len;
 
        /* pkE = DeserializePublicKey(enc) */
@@ -5198,13 +5198,13 @@ static int hpke_decap(struct hpke_context *ctx, const u8 *enc,
        if (!pk_e)
                return -1; /* invalid public key point */
        /* dh = DH(skR, pkE) */
+       dhss_len = sizeof(dhss);
        pctx = EVP_PKEY_CTX_new((EVP_PKEY *) sk_r, NULL);
        if (!pctx ||
            EVP_PKEY_derive_init(pctx) != 1 ||
            EVP_PKEY_derive_set_peer(pctx, (EVP_PKEY *) pk_e) != 1 ||
-           EVP_PKEY_derive(pctx, NULL, &dhss_len) != 1 ||
-           dhss_len > HPKE_MAX_SHARED_SECRET_LEN ||
-           EVP_PKEY_derive(pctx, dhss, &dhss_len) != 1) {
+           EVP_PKEY_derive(pctx, dhss, &dhss_len) != 1 ||
+           dhss_len > HPKE_MAX_SHARED_SECRET_LEN) {
                wpa_printf(MSG_INFO, "OpenSSL: EVP_PKEY_derive failed: %s",
                           ERR_error_string(ERR_get_error(), NULL));
                goto fail;