[thirdparty/openssl.git] / crypto / rsa / rsa_sign.c

/*
 * Copyright 1995-2019 The OpenSSL Project Authors. All Rights Reserved.
 *
 * Licensed under the Apache License 2.0 (the "License").  You may not use
 * this file except in compliance with the License.  You can obtain a copy
 * in the file LICENSE in the source distribution or at
 * https://www.openssl.org/source/license.html
 */

/*
 * RSA low level APIs are deprecated for public use, but still ok for
 * internal use.
 */
#include "internal/deprecated.h"

#include <stdio.h>
#include "internal/cryptlib.h"
#include <openssl/bn.h>
#include <openssl/rsa.h>
#include <openssl/objects.h>
#include <openssl/x509.h>
#include "crypto/x509.h"
#ifndef OPENSSL_NO_MD2
# include <openssl/md2.h> /* uses MD2_DIGEST_LENGTH */
#endif
#ifndef OPENSSL_NO_MD5
# include <openssl/md5.h> /* uses MD5_DIGEST_LENGTH */
#endif
#ifndef OPENSSL_NO_MDC2
# include <openssl/mdc2.h> /* uses MDC2_DIGEST_LENGTH */
#endif
#include <openssl/sha.h> /* uses SHA???_DIGEST_LENGTH */
#include "rsa_local.h"

/*
 * The general purpose ASN1 code is not available inside the FIPS provider.
 * To remove the dependency RSASSA-PKCS1-v1_5 DigestInfo encodings can be
 * treated as a special case by pregenerating the required ASN1 encoding.
 * This encoding will also be shared by the default provider.
 *
 * The EMSA-PKCS1-v1_5 encoding method includes an ASN.1 value of type
 * DigestInfo, where the type DigestInfo has the syntax
 *
 *     DigestInfo ::= SEQUENCE {
 *         digestAlgorithm DigestAlgorithm,
 *         digest OCTET STRING
 *     }
 *
 *     DigestAlgorithm ::= AlgorithmIdentifier {
 *         {PKCS1-v1-5DigestAlgorithms}
 *     }
 *
 * The AlgorithmIdentifier is a sequence containing the digest OID and
 * parameters (a value of type NULL).
 *
 * The ENCODE_DIGESTINFO_SHA() and ENCODE_DIGESTINFO_MD() macros define an
 * initialized array containing the DER encoded DigestInfo for the specified
 * SHA or MD digest. The content of the OCTET STRING is not included.
 * |name| is the digest name.
 * |n| is last byte in the encoded OID for the digest.
 * |sz| is the digest length in bytes. It must not be greater than 110.
 */

#define ASN1_SEQUENCE 0x30
#define ASN1_OCTET_STRING 0x04
#define ASN1_NULL 0x05
#define ASN1_OID 0x06

/* SHA OIDs are of the form: (2 16 840 1 101 3 4 2 |n|) */
#define ENCODE_DIGESTINFO_SHA(name, n, sz)                                     \
static const unsigned char digestinfo_##name##_der[] = {                       \
    ASN1_SEQUENCE, 0x11 + sz,                                                  \
      ASN1_SEQUENCE, 0x0d,                                                     \
        ASN1_OID, 0x09, 2 * 40 + 16, 0x86, 0x48, 1, 101, 3, 4, 2, n,           \
        ASN1_NULL, 0x00,                                                       \
      ASN1_OCTET_STRING, sz                                                    \
};

/* MD2 and MD5 OIDs are of the form: (1 2 840 113549 2 |n|) */
#define ENCODE_DIGESTINFO_MD(name, n, sz)                                      \
static const unsigned char digestinfo_##name##_der[] = {                       \
    ASN1_SEQUENCE, 0x10 + sz,                                                  \
      ASN1_SEQUENCE, 0x0c,                                                     \
        ASN1_OID, 0x08, 1 * 40 + 2, 0x86, 0x48, 0x86, 0xf7, 0x0d, 2, n,        \
        ASN1_NULL, 0x00,                                                       \
      ASN1_OCTET_STRING, sz                                                    \
};

#ifndef FIPS_MODE
# ifndef OPENSSL_NO_MD2
ENCODE_DIGESTINFO_MD(md2, 0x02, MD2_DIGEST_LENGTH)
# endif
# ifndef OPENSSL_NO_MD5
ENCODE_DIGESTINFO_MD(md5, 0x05, MD5_DIGEST_LENGTH)
# endif
# ifndef OPENSSL_NO_MDC2
/* MDC-2 (2 5 8 3 101) */
static const unsigned char digestinfo_mdc2_der[] = {
    ASN1_SEQUENCE, 0x0c + MDC2_DIGEST_LENGTH,
      ASN1_SEQUENCE, 0x08,
        ASN1_OID, 0x04, 2 * 40 + 5, 8, 3, 101,
        ASN1_NULL, 0x00,
      ASN1_OCTET_STRING, MDC2_DIGEST_LENGTH
};
# endif
/* SHA-1 (1 3 14 3 2 26) */
static const unsigned char digestinfo_sha1_der[] = {
    ASN1_SEQUENCE, 0x0d + SHA_DIGEST_LENGTH,
      ASN1_SEQUENCE, 0x09,
        ASN1_OID, 0x05, 1 * 40 + 3, 14, 3, 2, 26,
        ASN1_NULL, 0x00,
      ASN1_OCTET_STRING, SHA_DIGEST_LENGTH
};

#endif /* FIPS_MODE */

ENCODE_DIGESTINFO_SHA(sha256, 0x01, SHA256_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha384, 0x02, SHA384_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha512, 0x03, SHA512_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha224, 0x04, SHA224_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha512_224, 0x05, SHA224_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha512_256, 0x06, SHA256_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha3_224, 0x07, SHA224_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha3_256, 0x08, SHA256_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha3_384, 0x09, SHA384_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha3_512, 0x0a, SHA512_DIGEST_LENGTH)

#define MD_CASE(name)                                                          \
    case NID_##name:                                                           \
        *len = sizeof(digestinfo_##name##_der);                                \
        return digestinfo_##name##_der;

static const unsigned char *digestinfo_encoding(int nid, size_t *len)
{
    switch (nid) {
#ifndef FIPS_MODE
# ifndef OPENSSL_NO_MDC2
    MD_CASE(mdc2)
# endif
# ifndef OPENSSL_NO_MD2
    MD_CASE(md2)
# endif
# ifndef OPENSSL_NO_MD5
    MD_CASE(md5)
# endif
    MD_CASE(sha1)
#endif /* FIPS_MODE */
    MD_CASE(sha224)
    MD_CASE(sha256)
    MD_CASE(sha384)
    MD_CASE(sha512)
    MD_CASE(sha512_224)
    MD_CASE(sha512_256)
    MD_CASE(sha3_224)
    MD_CASE(sha3_256)
    MD_CASE(sha3_384)
    MD_CASE(sha3_512)
    default:
        return NULL;
    }
}

/* Size of an SSL signature: MD5+SHA1 */
#define SSL_SIG_LENGTH  36

/*
 * Encodes a DigestInfo prefix of hash |type| and digest |m|, as
 * described in EMSA-PKCS1-v1_5-ENCODE, RFC 3447 section 9.2 step 2. This
 * encodes the DigestInfo (T and tLen) but does not add the padding.
 *
 * On success, it returns one and sets |*out| to a newly allocated buffer
 * containing the result and |*out_len| to its length. The caller must free
 * |*out| with OPENSSL_free(). Otherwise, it returns zero.
 */
static int encode_pkcs1(unsigned char **out, size_t *out_len, int type,
                        const unsigned char *m, size_t m_len)
{
    size_t di_prefix_len, dig_info_len;
    const unsigned char *di_prefix;
    unsigned char *dig_info;

    if (type == NID_undef) {
        RSAerr(RSA_F_ENCODE_PKCS1, RSA_R_UNKNOWN_ALGORITHM_TYPE);
        return 0;
    }
    di_prefix = digestinfo_encoding(type, &di_prefix_len);
    if (di_prefix == NULL) {
        RSAerr(RSA_F_ENCODE_PKCS1,
               RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD);
        return 0;
    }
    dig_info_len = di_prefix_len + m_len;
    dig_info = OPENSSL_malloc(dig_info_len);
    if (dig_info == NULL) {
        RSAerr(RSA_F_ENCODE_PKCS1, ERR_R_MALLOC_FAILURE);
        return 0;
    }
    memcpy(dig_info, di_prefix, di_prefix_len);
    memcpy(dig_info + di_prefix_len, m, m_len);

    *out = dig_info;
    *out_len = dig_info_len;
    return 1;
}

int RSA_sign(int type, const unsigned char *m, unsigned int m_len,
             unsigned char *sigret, unsigned int *siglen, RSA *rsa)
{
    int encrypt_len, ret = 0;
    size_t encoded_len = 0;
    unsigned char *tmps = NULL;
    const unsigned char *encoded = NULL;

    if (rsa->meth->rsa_sign != NULL)
        return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa);

    /* Compute the encoded digest. */
    if (type == NID_md5_sha1) {
        /*
         * NID_md5_sha1 corresponds to the MD5/SHA1 combination in TLS 1.1 and
         * earlier. It has no DigestInfo wrapper but otherwise is
         * RSASSA-PKCS1-v1_5.
         */
        if (m_len != SSL_SIG_LENGTH) {
            RSAerr(RSA_F_RSA_SIGN, RSA_R_INVALID_MESSAGE_LENGTH);
            return 0;
        }
        encoded_len = SSL_SIG_LENGTH;
        encoded = m;
    } else {
        if (!encode_pkcs1(&tmps, &encoded_len, type, m, m_len))
            goto err;
        encoded = tmps;
    }

    if (encoded_len + RSA_PKCS1_PADDING_SIZE > (size_t)RSA_size(rsa)) {
        RSAerr(RSA_F_RSA_SIGN, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
        goto err;
    }
    encrypt_len = RSA_private_encrypt((int)encoded_len, encoded, sigret, rsa,
                                      RSA_PKCS1_PADDING);
    if (encrypt_len <= 0)
        goto err;

    *siglen = encrypt_len;
    ret = 1;

err:
    OPENSSL_clear_free(tmps, encoded_len);
    return ret;
}

/*
 * Verify an RSA signature in |sigbuf| using |rsa|.
 * |type| is the NID of the digest algorithm to use.
 * If |rm| is NULL, it verifies the signature for digest |m|, otherwise
 * it recovers the digest from the signature, writing the digest to |rm| and
 * the length to |*prm_len|.
 *
 * It returns one on successful verification or zero otherwise.
 */
int int_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
                   unsigned char *rm, size_t *prm_len,
                   const unsigned char *sigbuf, size_t siglen, RSA *rsa)
{
    int len, ret = 0;
    size_t decrypt_len, encoded_len = 0;
    unsigned char *decrypt_buf = NULL, *encoded = NULL;

    if (siglen != (size_t)RSA_size(rsa)) {
        RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_WRONG_SIGNATURE_LENGTH);
        return 0;
    }

    /* Recover the encoded digest. */
    decrypt_buf = OPENSSL_malloc(siglen);
    if (decrypt_buf == NULL) {
        RSAerr(RSA_F_INT_RSA_VERIFY, ERR_R_MALLOC_FAILURE);
        goto err;
    }

    len = RSA_public_decrypt((int)siglen, sigbuf, decrypt_buf, rsa,
                             RSA_PKCS1_PADDING);
    if (len <= 0)
        goto err;
    decrypt_len = len;

    if (type == NID_md5_sha1) {
        /*
         * NID_md5_sha1 corresponds to the MD5/SHA1 combination in TLS 1.1 and
         * earlier. It has no DigestInfo wrapper but otherwise is
         * RSASSA-PKCS1-v1_5.
         */
        if (decrypt_len != SSL_SIG_LENGTH) {
            RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
            goto err;
        }

        if (rm != NULL) {
            memcpy(rm, decrypt_buf, SSL_SIG_LENGTH);
            *prm_len = SSL_SIG_LENGTH;
        } else {
            if (m_len != SSL_SIG_LENGTH) {
                RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_MESSAGE_LENGTH);
                goto err;
            }

            if (memcmp(decrypt_buf, m, SSL_SIG_LENGTH) != 0) {
                RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
                goto err;
            }
        }
    } else if (type == NID_mdc2 && decrypt_len == 2 + 16
               && decrypt_buf[0] == 0x04 && decrypt_buf[1] == 0x10) {
        /*
         * Oddball MDC2 case: signature can be OCTET STRING. check for correct
         * tag and length octets.
         */
        if (rm != NULL) {
            memcpy(rm, decrypt_buf + 2, 16);
            *prm_len = 16;
        } else {
            if (m_len != 16) {
                RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_MESSAGE_LENGTH);
                goto err;
            }

            if (memcmp(m, decrypt_buf + 2, 16) != 0) {
                RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
                goto err;
            }
        }
    } else {
        /*
         * If recovering the digest, extract a digest-sized output from the end
         * of |decrypt_buf| for |encode_pkcs1|, then compare the decryption
         * output as in a standard verification.
         */
        if (rm != NULL) {
            const EVP_MD *md = EVP_get_digestbynid(type);
            if (md == NULL) {
                RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
                goto err;
            }

            len = EVP_MD_size(md);
            if (len <= 0)
                goto err;
            m_len = (unsigned int)len;
            if (m_len > decrypt_len) {
                RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_DIGEST_LENGTH);
                goto err;
            }
            m = decrypt_buf + decrypt_len - m_len;
        }

        /* Construct the encoded digest and ensure it matches. */
        if (!encode_pkcs1(&encoded, &encoded_len, type, m, m_len))
            goto err;

        if (encoded_len != decrypt_len
                || memcmp(encoded, decrypt_buf, encoded_len) != 0) {
            RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
            goto err;
        }

        /* Output the recovered digest. */
        if (rm != NULL) {
            memcpy(rm, m, m_len);
            *prm_len = m_len;
        }
    }

    ret = 1;

err:
    OPENSSL_clear_free(encoded, encoded_len);
    OPENSSL_clear_free(decrypt_buf, siglen);
    return ret;
}

int RSA_verify(int type, const unsigned char *m, unsigned int m_len,
               const unsigned char *sigbuf, unsigned int siglen, RSA *rsa)
{

    if (rsa->meth->rsa_verify != NULL)
        return rsa->meth->rsa_verify(type, m, m_len, sigbuf, siglen, rsa);

    return int_rsa_verify(type, m, m_len, NULL, NULL, sigbuf, siglen, rsa);
}
Commit	Line	Data
2039c421	1	/*
169e422e	2	* Copyright 1995-2019 The OpenSSL Project Authors. All Rights Reserved.
d02b48c6	3	*
2a7b6f39	4	* Licensed under the Apache License 2.0 (the "License"). You may not use
2039c421 RS	5	* this file except in compliance with the License. You can obtain a copy
	6	* in the file LICENSE in the source distribution or at
	7	* https://www.openssl.org/source/license.html
d02b48c6 RE	8	*/
d02b48c6 RE	9
c5f87134 P	10	/*
	11	* RSA low level APIs are deprecated for public use, but still ok for
	12	* internal use.
	13	*/
	14	#include "internal/deprecated.h"
	15
d02b48c6	16	#include <stdio.h>
b39fc560	17	#include "internal/cryptlib.h"
ec577822 BM	18	#include <openssl/bn.h>
	19	#include <openssl/rsa.h>
	20	#include <openssl/objects.h>
	21	#include <openssl/x509.h>
25f2138b	22	#include "crypto/x509.h"
169e422e SL	23	#ifndef OPENSSL_NO_MD2
	24	# include <openssl/md2.h> /* uses MD2_DIGEST_LENGTH */
	25	#endif
	26	#ifndef OPENSSL_NO_MD5
	27	# include <openssl/md5.h> /* uses MD5_DIGEST_LENGTH */
	28	#endif
	29	#ifndef OPENSSL_NO_MDC2
	30	# include <openssl/mdc2.h> /* uses MDC2_DIGEST_LENGTH */
	31	#endif
	32	#include <openssl/sha.h> /* uses SHA???_DIGEST_LENGTH */
706457b7	33	#include "rsa_local.h"
d02b48c6	34
169e422e SL	35	/*
	36	* The general purpose ASN1 code is not available inside the FIPS provider.
	37	* To remove the dependency RSASSA-PKCS1-v1_5 DigestInfo encodings can be
	38	* treated as a special case by pregenerating the required ASN1 encoding.
	39	* This encoding will also be shared by the default provider.
	40	*
	41	* The EMSA-PKCS1-v1_5 encoding method includes an ASN.1 value of type
	42	* DigestInfo, where the type DigestInfo has the syntax
	43	*
	44	* DigestInfo ::= SEQUENCE {
	45	* digestAlgorithm DigestAlgorithm,
	46	* digest OCTET STRING
	47	* }
	48	*
	49	* DigestAlgorithm ::= AlgorithmIdentifier {
	50	* {PKCS1-v1-5DigestAlgorithms}
	51	* }
	52	*
	53	* The AlgorithmIdentifier is a sequence containing the digest OID and
	54	* parameters (a value of type NULL).
	55	*
	56	* The ENCODE_DIGESTINFO_SHA() and ENCODE_DIGESTINFO_MD() macros define an
	57	* initialized array containing the DER encoded DigestInfo for the specified
	58	* SHA or MD digest. The content of the OCTET STRING is not included.
	59	* \|name\| is the digest name.
	60	* \|n\| is last byte in the encoded OID for the digest.
	61	* \|sz\| is the digest length in bytes. It must not be greater than 110.
	62	*/
	63
	64	#define ASN1_SEQUENCE 0x30
	65	#define ASN1_OCTET_STRING 0x04
	66	#define ASN1_NULL 0x05
	67	#define ASN1_OID 0x06
	68
	69	/* SHA OIDs are of the form: (2 16 840 1 101 3 4 2 \|n\|) */
	70	#define ENCODE_DIGESTINFO_SHA(name, n, sz) \
	71	static const unsigned char digestinfo_##name##_der[] = { \
	72	ASN1_SEQUENCE, 0x11 + sz, \
	73	ASN1_SEQUENCE, 0x0d, \
	74	ASN1_OID, 0x09, 2 * 40 + 16, 0x86, 0x48, 1, 101, 3, 4, 2, n, \
	75	ASN1_NULL, 0x00, \
	76	ASN1_OCTET_STRING, sz \
	77	};
	78
	79	/* MD2 and MD5 OIDs are of the form: (1 2 840 113549 2 \|n\|) */
	80	#define ENCODE_DIGESTINFO_MD(name, n, sz) \
	81	static const unsigned char digestinfo_##name##_der[] = { \
	82	ASN1_SEQUENCE, 0x10 + sz, \
	83	ASN1_SEQUENCE, 0x0c, \
	84	ASN1_OID, 0x08, 1 * 40 + 2, 0x86, 0x48, 0x86, 0xf7, 0x0d, 2, n, \
	85	ASN1_NULL, 0x00, \
	86	ASN1_OCTET_STRING, sz \
	87	};
	88
	89	#ifndef FIPS_MODE
	90	# ifndef OPENSSL_NO_MD2
	91	ENCODE_DIGESTINFO_MD(md2, 0x02, MD2_DIGEST_LENGTH)
	92	# endif
	93	# ifndef OPENSSL_NO_MD5
	94	ENCODE_DIGESTINFO_MD(md5, 0x05, MD5_DIGEST_LENGTH)
	95	# endif
	96	# ifndef OPENSSL_NO_MDC2
	97	/* MDC-2 (2 5 8 3 101) */
	98	static const unsigned char digestinfo_mdc2_der[] = {
99	ASN1_SEQUENCE, 0x0c + MDC2_DIGEST_LENGTH,
100	ASN1_SEQUENCE, 0x08,
101	ASN1_OID, 0x04, 2 * 40 + 5, 8, 3, 101,
102	ASN1_NULL, 0x00,
103	ASN1_OCTET_STRING, MDC2_DIGEST_LENGTH
104	};
105	# endif
106	/* SHA-1 (1 3 14 3 2 26) */
107	static const unsigned char digestinfo_sha1_der[] = {
108	ASN1_SEQUENCE, 0x0d + SHA_DIGEST_LENGTH,
109	ASN1_SEQUENCE, 0x09,
110	ASN1_OID, 0x05, 1 * 40 + 3, 14, 3, 2, 26,
111	ASN1_NULL, 0x00,
112	ASN1_OCTET_STRING, SHA_DIGEST_LENGTH
113	};
114
115	#endif /* FIPS_MODE */
116
117	ENCODE_DIGESTINFO_SHA(sha256, 0x01, SHA256_DIGEST_LENGTH)
118	ENCODE_DIGESTINFO_SHA(sha384, 0x02, SHA384_DIGEST_LENGTH)
119	ENCODE_DIGESTINFO_SHA(sha512, 0x03, SHA512_DIGEST_LENGTH)
120	ENCODE_DIGESTINFO_SHA(sha224, 0x04, SHA224_DIGEST_LENGTH)
121	ENCODE_DIGESTINFO_SHA(sha512_224, 0x05, SHA224_DIGEST_LENGTH)
122	ENCODE_DIGESTINFO_SHA(sha512_256, 0x06, SHA256_DIGEST_LENGTH)
123	ENCODE_DIGESTINFO_SHA(sha3_224, 0x07, SHA224_DIGEST_LENGTH)
124	ENCODE_DIGESTINFO_SHA(sha3_256, 0x08, SHA256_DIGEST_LENGTH)
125	ENCODE_DIGESTINFO_SHA(sha3_384, 0x09, SHA384_DIGEST_LENGTH)
126	ENCODE_DIGESTINFO_SHA(sha3_512, 0x0a, SHA512_DIGEST_LENGTH)
127
128	#define MD_CASE(name) \
129	case NID_##name: \
130	*len = sizeof(digestinfo_##name##_der); \
131	return digestinfo_##name##_der;
132
133	static const unsigned char digestinfo_encoding(int nid, size_t len)
134	{
135	switch (nid) {
136	#ifndef FIPS_MODE
137	# ifndef OPENSSL_NO_MDC2
138	MD_CASE(mdc2)
139	# endif
140	# ifndef OPENSSL_NO_MD2
141	MD_CASE(md2)
142	# endif
143	# ifndef OPENSSL_NO_MD5
144	MD_CASE(md5)
145	# endif
146	MD_CASE(sha1)
147	#endif /* FIPS_MODE */
148	MD_CASE(sha224)
149	MD_CASE(sha256)
150	MD_CASE(sha384)
151	MD_CASE(sha512)
152	MD_CASE(sha512_224)
153	MD_CASE(sha512_256)
154	MD_CASE(sha3_224)
155	MD_CASE(sha3_256)
156	MD_CASE(sha3_384)
157	MD_CASE(sha3_512)
158	default:
159	return NULL;
160	}
161	}
162
1c80019a	163	/* Size of an SSL signature: MD5+SHA1 */
0f113f3e	164	#define SSL_SIG_LENGTH 36
1c80019a	165
608a0264	166	/*
169e422e	167	* Encodes a DigestInfo prefix of hash \|type\| and digest \|m\|, as
608a0264 DB	168	* described in EMSA-PKCS1-v1_5-ENCODE, RFC 3447 section 9.2 step 2. This
	169	* encodes the DigestInfo (T and tLen) but does not add the padding.
	170	*
	171	* On success, it returns one and sets \|*out\| to a newly allocated buffer
	172	* containing the result and \|*out_len\| to its length. The caller must free
169e422e	173	* \|*out\| with OPENSSL_free(). Otherwise, it returns zero.
608a0264	174	*/
169e422e SL	175	static int encode_pkcs1(unsigned char *out, size_t out_len, int type,
169e422e SL	176	const unsigned char *m, size_t m_len)
0f113f3e	177	{
169e422e SL	178	size_t di_prefix_len, dig_info_len;
	179	const unsigned char *di_prefix;
	180	unsigned char *dig_info;
	181
	182	if (type == NID_undef) {
608a0264 DB	183	RSAerr(RSA_F_ENCODE_PKCS1, RSA_R_UNKNOWN_ALGORITHM_TYPE);
	184	return 0;
	185	}
169e422e SL	186	di_prefix = digestinfo_encoding(type, &di_prefix_len);
169e422e SL	187	if (di_prefix == NULL) {
608a0264 DB	188	RSAerr(RSA_F_ENCODE_PKCS1,
	189	RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD);
	190	return 0;
	191	}
169e422e SL	192	dig_info_len = di_prefix_len + m_len;
	193	dig_info = OPENSSL_malloc(dig_info_len);
	194	if (dig_info == NULL) {
	195	RSAerr(RSA_F_ENCODE_PKCS1, ERR_R_MALLOC_FAILURE);
608a0264	196	return 0;
169e422e SL	197	}
	198	memcpy(dig_info, di_prefix, di_prefix_len);
	199	memcpy(dig_info + di_prefix_len, m, m_len);
608a0264	200
169e422e SL	201	*out = dig_info;
169e422e SL	202	*out_len = dig_info_len;
608a0264 DB	203	return 1;
	204	}
	205
	206	int RSA_sign(int type, const unsigned char *m, unsigned int m_len,
	207	unsigned char sigret, unsigned int siglen, RSA *rsa)
	208	{
169e422e SL	209	int encrypt_len, ret = 0;
169e422e SL	210	size_t encoded_len = 0;
608a0264 DB	211	unsigned char *tmps = NULL;
	212	const unsigned char *encoded = NULL;
	213
169e422e	214	if (rsa->meth->rsa_sign != NULL)
0f113f3e	215	return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa);
608a0264 DB	216
608a0264 DB	217	/* Compute the encoded digest. */
0f113f3e	218	if (type == NID_md5_sha1) {
608a0264 DB	219	/*
	220	* NID_md5_sha1 corresponds to the MD5/SHA1 combination in TLS 1.1 and
	221	* earlier. It has no DigestInfo wrapper but otherwise is
	222	* RSASSA-PKCS1-v1_5.
	223	*/
0f113f3e MC	224	if (m_len != SSL_SIG_LENGTH) {
0f113f3e MC	225	RSAerr(RSA_F_RSA_SIGN, RSA_R_INVALID_MESSAGE_LENGTH);
608a0264	226	return 0;
0f113f3e	227	}
608a0264 DB	228	encoded_len = SSL_SIG_LENGTH;
608a0264 DB	229	encoded = m;
0f113f3e	230	} else {
608a0264 DB	231	if (!encode_pkcs1(&tmps, &encoded_len, type, m, m_len))
	232	goto err;
	233	encoded = tmps;
0f113f3e	234	}
608a0264	235
169e422e	236	if (encoded_len + RSA_PKCS1_PADDING_SIZE > (size_t)RSA_size(rsa)) {
0f113f3e	237	RSAerr(RSA_F_RSA_SIGN, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
608a0264	238	goto err;
0f113f3e	239	}
169e422e	240	encrypt_len = RSA_private_encrypt((int)encoded_len, encoded, sigret, rsa,
608a0264 DB	241	RSA_PKCS1_PADDING);
	242	if (encrypt_len <= 0)
	243	goto err;
d02b48c6	244
608a0264 DB	245	*siglen = encrypt_len;
	246	ret = 1;
	247
	248	err:
169e422e	249	OPENSSL_clear_free(tmps, encoded_len);
0f113f3e MC	250	return ret;
0f113f3e MC	251	}
1cfd255c	252
608a0264	253	/*
169e422e SL	254	* Verify an RSA signature in \|sigbuf\| using \|rsa\|.
	255	* \|type\| is the NID of the digest algorithm to use.
	256	* If \|rm\| is NULL, it verifies the signature for digest \|m\|, otherwise
	257	* it recovers the digest from the signature, writing the digest to \|rm\| and
	258	* the length to \|*prm_len\|.
	259	*
	260	* It returns one on successful verification or zero otherwise.
608a0264 DB	261	*/
608a0264 DB	262	int int_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
0f113f3e MC	263	unsigned char rm, size_t prm_len,
	264	const unsigned char sigbuf, size_t siglen, RSA rsa)
	265	{
169e422e SL	266	int len, ret = 0;
169e422e SL	267	size_t decrypt_len, encoded_len = 0;
608a0264	268	unsigned char decrypt_buf = NULL, encoded = NULL;
d02b48c6	269
608a0264	270	if (siglen != (size_t)RSA_size(rsa)) {
0f113f3e	271	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_WRONG_SIGNATURE_LENGTH);
608a0264	272	return 0;
0f113f3e	273	}
d02b48c6	274
608a0264 DB	275	/* Recover the encoded digest. */
	276	decrypt_buf = OPENSSL_malloc(siglen);
	277	if (decrypt_buf == NULL) {
0f113f3e MC	278	RSAerr(RSA_F_INT_RSA_VERIFY, ERR_R_MALLOC_FAILURE);
	279	goto err;
	280	}
d02b48c6	281
169e422e SL	282	len = RSA_public_decrypt((int)siglen, sigbuf, decrypt_buf, rsa,
	283	RSA_PKCS1_PADDING);
	284	if (len <= 0)
0f113f3e	285	goto err;
169e422e	286	decrypt_len = len;
608a0264 DB	287
	288	if (type == NID_md5_sha1) {
	289	/*
	290	* NID_md5_sha1 corresponds to the MD5/SHA1 combination in TLS 1.1 and
	291	* earlier. It has no DigestInfo wrapper but otherwise is
	292	* RSASSA-PKCS1-v1_5.
	293	*/
	294	if (decrypt_len != SSL_SIG_LENGTH) {
0f113f3e	295	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
608a0264	296	goto err;
dffe5109	297	}
b79aa05e	298
608a0264 DB	299	if (rm != NULL) {
	300	memcpy(rm, decrypt_buf, SSL_SIG_LENGTH);
	301	*prm_len = SSL_SIG_LENGTH;
	302	} else {
	303	if (m_len != SSL_SIG_LENGTH) {
	304	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_MESSAGE_LENGTH);
	305	goto err;
	306	}
b79aa05e	307
608a0264 DB	308	if (memcmp(decrypt_buf, m, SSL_SIG_LENGTH) != 0) {
	309	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
	310	goto err;
	311	}
0f113f3e	312	}
608a0264 DB	313	} else if (type == NID_mdc2 && decrypt_len == 2 + 16
	314	&& decrypt_buf[0] == 0x04 && decrypt_buf[1] == 0x10) {
	315	/*
	316	* Oddball MDC2 case: signature can be OCTET STRING. check for correct
	317	* tag and length octets.
	318	*/
	319	if (rm != NULL) {
	320	memcpy(rm, decrypt_buf + 2, 16);
	321	*prm_len = 16;
	322	} else {
	323	if (m_len != 16) {
	324	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_MESSAGE_LENGTH);
	325	goto err;
	326	}
b79aa05e	327
608a0264 DB	328	if (memcmp(m, decrypt_buf + 2, 16) != 0) {
	329	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
	330	goto err;
	331	}
	332	}
	333	} else {
0f113f3e	334	/*
608a0264 DB	335	* If recovering the digest, extract a digest-sized output from the end
	336	* of \|decrypt_buf\| for \|encode_pkcs1\|, then compare the decryption
	337	* output as in a standard verification.
0f113f3e	338	*/
608a0264 DB	339	if (rm != NULL) {
	340	const EVP_MD *md = EVP_get_digestbynid(type);
	341	if (md == NULL) {
	342	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
	343	goto err;
	344	}
	345
169e422e SL	346	len = EVP_MD_size(md);
	347	if (len <= 0)
	348	goto err;
	349	m_len = (unsigned int)len;
	350	if (m_len > decrypt_len) {
608a0264 DB	351	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_DIGEST_LENGTH);
	352	goto err;
	353	}
	354	m = decrypt_buf + decrypt_len - m_len;
0f113f3e	355	}
d02b48c6	356
608a0264 DB	357	/* Construct the encoded digest and ensure it matches. */
	358	if (!encode_pkcs1(&encoded, &encoded_len, type, m, m_len))
	359	goto err;
dfeab068	360
608a0264	361	if (encoded_len != decrypt_len
169e422e	362	\|\| memcmp(encoded, decrypt_buf, encoded_len) != 0) {
608a0264	363	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
3d0cf918	364	goto err;
0f113f3e	365	}
608a0264 DB	366
	367	/* Output the recovered digest. */
	368	if (rm != NULL) {
	369	memcpy(rm, m, m_len);
	370	*prm_len = m_len;
	371	}
0f113f3e	372	}
608a0264 DB	373
	374	ret = 1;
	375
	376	err:
169e422e	377	OPENSSL_clear_free(encoded, encoded_len);
608a0264 DB	378	OPENSSL_clear_free(decrypt_buf, siglen);
608a0264 DB	379	return ret;
0f113f3e	380	}
d02b48c6	381
608a0264	382	int RSA_verify(int type, const unsigned char *m, unsigned int m_len,
0f113f3e MC	383	const unsigned char sigbuf, unsigned int siglen, RSA rsa)
0f113f3e MC	384	{
b2a97be7	385
169e422e	386	if (rsa->meth->rsa_verify != NULL)
608a0264	387	return rsa->meth->rsa_verify(type, m, m_len, sigbuf, siglen, rsa);
b2a97be7	388
608a0264	389	return int_rsa_verify(type, m, m_len, NULL, NULL, sigbuf, siglen, rsa);
0f113f3e	390	}