[thirdparty/openssl.git] / crypto / rsa / rsa_sign.c

/*
 * Copyright 1995-2019 The OpenSSL Project Authors. All Rights Reserved.
 *
 * Licensed under the Apache License 2.0 (the "License").  You may not use
 * this file except in compliance with the License.  You can obtain a copy
 * in the file LICENSE in the source distribution or at
 * https://www.openssl.org/source/license.html
 */

#include <stdio.h>
#include "internal/cryptlib.h"
#include <openssl/bn.h>
#include <openssl/rsa.h>
#include <openssl/objects.h>
#include <openssl/x509.h>
#include "crypto/x509.h"
#ifndef OPENSSL_NO_MD2
# include <openssl/md2.h> /* uses MD2_DIGEST_LENGTH */
#endif
#ifndef OPENSSL_NO_MD5
# include <openssl/md5.h> /* uses MD5_DIGEST_LENGTH */
#endif
#ifndef OPENSSL_NO_MDC2
# include <openssl/mdc2.h> /* uses MDC2_DIGEST_LENGTH */
#endif
#include <openssl/sha.h> /* uses SHA???_DIGEST_LENGTH */
#include "rsa_local.h"

/*
 * The general purpose ASN1 code is not available inside the FIPS provider.
 * To remove the dependency RSASSA-PKCS1-v1_5 DigestInfo encodings can be
 * treated as a special case by pregenerating the required ASN1 encoding.
 * This encoding will also be shared by the default provider.
 *
 * The EMSA-PKCS1-v1_5 encoding method includes an ASN.1 value of type
 * DigestInfo, where the type DigestInfo has the syntax
 *
 *     DigestInfo ::= SEQUENCE {
 *         digestAlgorithm DigestAlgorithm,
 *         digest OCTET STRING
 *     }
 *
 *     DigestAlgorithm ::= AlgorithmIdentifier {
 *         {PKCS1-v1-5DigestAlgorithms}
 *     }
 *
 * The AlgorithmIdentifier is a sequence containing the digest OID and
 * parameters (a value of type NULL).
 *
 * The ENCODE_DIGESTINFO_SHA() and ENCODE_DIGESTINFO_MD() macros define an
 * initialized array containing the DER encoded DigestInfo for the specified
 * SHA or MD digest. The content of the OCTET STRING is not included.
 * |name| is the digest name.
 * |n| is last byte in the encoded OID for the digest.
 * |sz| is the digest length in bytes. It must not be greater than 110.
 */

#define ASN1_SEQUENCE 0x30
#define ASN1_OCTET_STRING 0x04
#define ASN1_NULL 0x05
#define ASN1_OID 0x06

/* SHA OIDs are of the form: (2 16 840 1 101 3 4 2 |n|) */
#define ENCODE_DIGESTINFO_SHA(name, n, sz)                                     \
static const unsigned char digestinfo_##name##_der[] = {                       \
    ASN1_SEQUENCE, 0x11 + sz,                                                  \
      ASN1_SEQUENCE, 0x0d,                                                     \
        ASN1_OID, 0x09, 2 * 40 + 16, 0x86, 0x48, 1, 101, 3, 4, 2, n,           \
        ASN1_NULL, 0x00,                                                       \
      ASN1_OCTET_STRING, sz                                                    \
};

/* MD2 and MD5 OIDs are of the form: (1 2 840 113549 2 |n|) */
#define ENCODE_DIGESTINFO_MD(name, n, sz)                                      \
static const unsigned char digestinfo_##name##_der[] = {                       \
    ASN1_SEQUENCE, 0x10 + sz,                                                  \
      ASN1_SEQUENCE, 0x0c,                                                     \
        ASN1_OID, 0x08, 1 * 40 + 2, 0x86, 0x48, 0x86, 0xf7, 0x0d, 2, n,        \
        ASN1_NULL, 0x00,                                                       \
      ASN1_OCTET_STRING, sz                                                    \
};

#ifndef FIPS_MODE
# ifndef OPENSSL_NO_MD2
ENCODE_DIGESTINFO_MD(md2, 0x02, MD2_DIGEST_LENGTH)
# endif
# ifndef OPENSSL_NO_MD5
ENCODE_DIGESTINFO_MD(md5, 0x05, MD5_DIGEST_LENGTH)
# endif
# ifndef OPENSSL_NO_MDC2
/* MDC-2 (2 5 8 3 101) */
static const unsigned char digestinfo_mdc2_der[] = {
    ASN1_SEQUENCE, 0x0c + MDC2_DIGEST_LENGTH,
      ASN1_SEQUENCE, 0x08,
        ASN1_OID, 0x04, 2 * 40 + 5, 8, 3, 101,
        ASN1_NULL, 0x00,
      ASN1_OCTET_STRING, MDC2_DIGEST_LENGTH
};
# endif
/* SHA-1 (1 3 14 3 2 26) */
static const unsigned char digestinfo_sha1_der[] = {
    ASN1_SEQUENCE, 0x0d + SHA_DIGEST_LENGTH,
      ASN1_SEQUENCE, 0x09,
        ASN1_OID, 0x05, 1 * 40 + 3, 14, 3, 2, 26,
        ASN1_NULL, 0x00,
      ASN1_OCTET_STRING, SHA_DIGEST_LENGTH
};

#endif /* FIPS_MODE */

ENCODE_DIGESTINFO_SHA(sha256, 0x01, SHA256_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha384, 0x02, SHA384_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha512, 0x03, SHA512_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha224, 0x04, SHA224_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha512_224, 0x05, SHA224_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha512_256, 0x06, SHA256_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha3_224, 0x07, SHA224_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha3_256, 0x08, SHA256_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha3_384, 0x09, SHA384_DIGEST_LENGTH)
ENCODE_DIGESTINFO_SHA(sha3_512, 0x0a, SHA512_DIGEST_LENGTH)

#define MD_CASE(name)                                                          \
    case NID_##name:                                                           \
        *len = sizeof(digestinfo_##name##_der);                                \
        return digestinfo_##name##_der;

static const unsigned char *digestinfo_encoding(int nid, size_t *len)
{
    switch (nid) {
#ifndef FIPS_MODE
# ifndef OPENSSL_NO_MDC2
    MD_CASE(mdc2)
# endif
# ifndef OPENSSL_NO_MD2
    MD_CASE(md2)
# endif
# ifndef OPENSSL_NO_MD5
    MD_CASE(md5)
# endif
    MD_CASE(sha1)
#endif /* FIPS_MODE */
    MD_CASE(sha224)
    MD_CASE(sha256)
    MD_CASE(sha384)
    MD_CASE(sha512)
    MD_CASE(sha512_224)
    MD_CASE(sha512_256)
    MD_CASE(sha3_224)
    MD_CASE(sha3_256)
    MD_CASE(sha3_384)
    MD_CASE(sha3_512)
    default:
        return NULL;
    }
}

/* Size of an SSL signature: MD5+SHA1 */
#define SSL_SIG_LENGTH  36

/*
 * Encodes a DigestInfo prefix of hash |type| and digest |m|, as
 * described in EMSA-PKCS1-v1_5-ENCODE, RFC 3447 section 9.2 step 2. This
 * encodes the DigestInfo (T and tLen) but does not add the padding.
 *
 * On success, it returns one and sets |*out| to a newly allocated buffer
 * containing the result and |*out_len| to its length. The caller must free
 * |*out| with OPENSSL_free(). Otherwise, it returns zero.
 */
static int encode_pkcs1(unsigned char **out, size_t *out_len, int type,
                        const unsigned char *m, size_t m_len)
{
    size_t di_prefix_len, dig_info_len;
    const unsigned char *di_prefix;
    unsigned char *dig_info;

    if (type == NID_undef) {
        RSAerr(RSA_F_ENCODE_PKCS1, RSA_R_UNKNOWN_ALGORITHM_TYPE);
        return 0;
    }
    di_prefix = digestinfo_encoding(type, &di_prefix_len);
    if (di_prefix == NULL) {
        RSAerr(RSA_F_ENCODE_PKCS1,
               RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD);
        return 0;
    }
    dig_info_len = di_prefix_len + m_len;
    dig_info = OPENSSL_malloc(dig_info_len);
    if (dig_info == NULL) {
        RSAerr(RSA_F_ENCODE_PKCS1, ERR_R_MALLOC_FAILURE);
        return 0;
    }
    memcpy(dig_info, di_prefix, di_prefix_len);
    memcpy(dig_info + di_prefix_len, m, m_len);

    *out = dig_info;
    *out_len = dig_info_len;
    return 1;
}

int RSA_sign(int type, const unsigned char *m, unsigned int m_len,
             unsigned char *sigret, unsigned int *siglen, RSA *rsa)
{
    int encrypt_len, ret = 0;
    size_t encoded_len = 0;
    unsigned char *tmps = NULL;
    const unsigned char *encoded = NULL;

    if (rsa->meth->rsa_sign != NULL)
        return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa);

    /* Compute the encoded digest. */
    if (type == NID_md5_sha1) {
        /*
         * NID_md5_sha1 corresponds to the MD5/SHA1 combination in TLS 1.1 and
         * earlier. It has no DigestInfo wrapper but otherwise is
         * RSASSA-PKCS1-v1_5.
         */
        if (m_len != SSL_SIG_LENGTH) {
            RSAerr(RSA_F_RSA_SIGN, RSA_R_INVALID_MESSAGE_LENGTH);
            return 0;
        }
        encoded_len = SSL_SIG_LENGTH;
        encoded = m;
    } else {
        if (!encode_pkcs1(&tmps, &encoded_len, type, m, m_len))
            goto err;
        encoded = tmps;
    }

    if (encoded_len + RSA_PKCS1_PADDING_SIZE > (size_t)RSA_size(rsa)) {
        RSAerr(RSA_F_RSA_SIGN, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
        goto err;
    }
    encrypt_len = RSA_private_encrypt((int)encoded_len, encoded, sigret, rsa,
                                      RSA_PKCS1_PADDING);
    if (encrypt_len <= 0)
        goto err;

    *siglen = encrypt_len;
    ret = 1;

err:
    OPENSSL_clear_free(tmps, encoded_len);
    return ret;
}

/*
 * Verify an RSA signature in |sigbuf| using |rsa|.
 * |type| is the NID of the digest algorithm to use.
 * If |rm| is NULL, it verifies the signature for digest |m|, otherwise
 * it recovers the digest from the signature, writing the digest to |rm| and
 * the length to |*prm_len|.
 *
 * It returns one on successful verification or zero otherwise.
 */
int int_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
                   unsigned char *rm, size_t *prm_len,
                   const unsigned char *sigbuf, size_t siglen, RSA *rsa)
{
    int len, ret = 0;
    size_t decrypt_len, encoded_len = 0;
    unsigned char *decrypt_buf = NULL, *encoded = NULL;

    if (siglen != (size_t)RSA_size(rsa)) {
        RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_WRONG_SIGNATURE_LENGTH);
        return 0;
    }

    /* Recover the encoded digest. */
    decrypt_buf = OPENSSL_malloc(siglen);
    if (decrypt_buf == NULL) {
        RSAerr(RSA_F_INT_RSA_VERIFY, ERR_R_MALLOC_FAILURE);
        goto err;
    }

    len = RSA_public_decrypt((int)siglen, sigbuf, decrypt_buf, rsa,
                             RSA_PKCS1_PADDING);
    if (len <= 0)
        goto err;
    decrypt_len = len;

    if (type == NID_md5_sha1) {
        /*
         * NID_md5_sha1 corresponds to the MD5/SHA1 combination in TLS 1.1 and
         * earlier. It has no DigestInfo wrapper but otherwise is
         * RSASSA-PKCS1-v1_5.
         */
        if (decrypt_len != SSL_SIG_LENGTH) {
            RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
            goto err;
        }

        if (rm != NULL) {
            memcpy(rm, decrypt_buf, SSL_SIG_LENGTH);
            *prm_len = SSL_SIG_LENGTH;
        } else {
            if (m_len != SSL_SIG_LENGTH) {
                RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_MESSAGE_LENGTH);
                goto err;
            }

            if (memcmp(decrypt_buf, m, SSL_SIG_LENGTH) != 0) {
                RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
                goto err;
            }
        }
    } else if (type == NID_mdc2 && decrypt_len == 2 + 16
               && decrypt_buf[0] == 0x04 && decrypt_buf[1] == 0x10) {
        /*
         * Oddball MDC2 case: signature can be OCTET STRING. check for correct
         * tag and length octets.
         */
        if (rm != NULL) {
            memcpy(rm, decrypt_buf + 2, 16);
            *prm_len = 16;
        } else {
            if (m_len != 16) {
                RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_MESSAGE_LENGTH);
                goto err;
            }

            if (memcmp(m, decrypt_buf + 2, 16) != 0) {
                RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
                goto err;
            }
        }
    } else {
        /*
         * If recovering the digest, extract a digest-sized output from the end
         * of |decrypt_buf| for |encode_pkcs1|, then compare the decryption
         * output as in a standard verification.
         */
        if (rm != NULL) {
            const EVP_MD *md = EVP_get_digestbynid(type);
            if (md == NULL) {
                RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
                goto err;
            }

            len = EVP_MD_size(md);
            if (len <= 0)
                goto err;
            m_len = (unsigned int)len;
            if (m_len > decrypt_len) {
                RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_DIGEST_LENGTH);
                goto err;
            }
            m = decrypt_buf + decrypt_len - m_len;
        }

        /* Construct the encoded digest and ensure it matches. */
        if (!encode_pkcs1(&encoded, &encoded_len, type, m, m_len))
            goto err;

        if (encoded_len != decrypt_len
                || memcmp(encoded, decrypt_buf, encoded_len) != 0) {
            RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
            goto err;
        }

        /* Output the recovered digest. */
        if (rm != NULL) {
            memcpy(rm, m, m_len);
            *prm_len = m_len;
        }
    }

    ret = 1;

err:
    OPENSSL_clear_free(encoded, encoded_len);
    OPENSSL_clear_free(decrypt_buf, siglen);
    return ret;
}

int RSA_verify(int type, const unsigned char *m, unsigned int m_len,
               const unsigned char *sigbuf, unsigned int siglen, RSA *rsa)
{

    if (rsa->meth->rsa_verify != NULL)
        return rsa->meth->rsa_verify(type, m, m_len, sigbuf, siglen, rsa);

    return int_rsa_verify(type, m, m_len, NULL, NULL, sigbuf, siglen, rsa);
}
Commit	Line	Data
2039c421	1	/*
169e422e	2	* Copyright 1995-2019 The OpenSSL Project Authors. All Rights Reserved.
d02b48c6	3	*
2a7b6f39	4	* Licensed under the Apache License 2.0 (the "License"). You may not use
2039c421 RS	5	* this file except in compliance with the License. You can obtain a copy
	6	* in the file LICENSE in the source distribution or at
	7	* https://www.openssl.org/source/license.html
d02b48c6 RE	8	*/
	9
	10	#include <stdio.h>
b39fc560	11	#include "internal/cryptlib.h"
ec577822 BM	12	#include <openssl/bn.h>
	13	#include <openssl/rsa.h>
	14	#include <openssl/objects.h>
	15	#include <openssl/x509.h>
25f2138b	16	#include "crypto/x509.h"
169e422e SL	17	#ifndef OPENSSL_NO_MD2
	18	# include <openssl/md2.h> /* uses MD2_DIGEST_LENGTH */
	19	#endif
	20	#ifndef OPENSSL_NO_MD5
	21	# include <openssl/md5.h> /* uses MD5_DIGEST_LENGTH */
	22	#endif
	23	#ifndef OPENSSL_NO_MDC2
	24	# include <openssl/mdc2.h> /* uses MDC2_DIGEST_LENGTH */
	25	#endif
	26	#include <openssl/sha.h> /* uses SHA???_DIGEST_LENGTH */
706457b7	27	#include "rsa_local.h"
d02b48c6	28
169e422e SL	29	/*
	30	* The general purpose ASN1 code is not available inside the FIPS provider.
	31	* To remove the dependency RSASSA-PKCS1-v1_5 DigestInfo encodings can be
	32	* treated as a special case by pregenerating the required ASN1 encoding.
	33	* This encoding will also be shared by the default provider.
	34	*
	35	* The EMSA-PKCS1-v1_5 encoding method includes an ASN.1 value of type
	36	* DigestInfo, where the type DigestInfo has the syntax
	37	*
	38	* DigestInfo ::= SEQUENCE {
	39	* digestAlgorithm DigestAlgorithm,
	40	* digest OCTET STRING
	41	* }
	42	*
	43	* DigestAlgorithm ::= AlgorithmIdentifier {
	44	* {PKCS1-v1-5DigestAlgorithms}
	45	* }
	46	*
	47	* The AlgorithmIdentifier is a sequence containing the digest OID and
	48	* parameters (a value of type NULL).
	49	*
	50	* The ENCODE_DIGESTINFO_SHA() and ENCODE_DIGESTINFO_MD() macros define an
	51	* initialized array containing the DER encoded DigestInfo for the specified
	52	* SHA or MD digest. The content of the OCTET STRING is not included.
	53	* \|name\| is the digest name.
	54	* \|n\| is last byte in the encoded OID for the digest.
	55	* \|sz\| is the digest length in bytes. It must not be greater than 110.
	56	*/
	57
	58	#define ASN1_SEQUENCE 0x30
	59	#define ASN1_OCTET_STRING 0x04
	60	#define ASN1_NULL 0x05
	61	#define ASN1_OID 0x06
	62
	63	/* SHA OIDs are of the form: (2 16 840 1 101 3 4 2 \|n\|) */
	64	#define ENCODE_DIGESTINFO_SHA(name, n, sz) \
	65	static const unsigned char digestinfo_##name##_der[] = { \
	66	ASN1_SEQUENCE, 0x11 + sz, \
	67	ASN1_SEQUENCE, 0x0d, \
	68	ASN1_OID, 0x09, 2 * 40 + 16, 0x86, 0x48, 1, 101, 3, 4, 2, n, \
	69	ASN1_NULL, 0x00, \
	70	ASN1_OCTET_STRING, sz \
	71	};
	72
	73	/* MD2 and MD5 OIDs are of the form: (1 2 840 113549 2 \|n\|) */
	74	#define ENCODE_DIGESTINFO_MD(name, n, sz) \
	75	static const unsigned char digestinfo_##name##_der[] = { \
	76	ASN1_SEQUENCE, 0x10 + sz, \
	77	ASN1_SEQUENCE, 0x0c, \
	78	ASN1_OID, 0x08, 1 * 40 + 2, 0x86, 0x48, 0x86, 0xf7, 0x0d, 2, n, \
	79	ASN1_NULL, 0x00, \
	80	ASN1_OCTET_STRING, sz \
	81	};
	82
	83	#ifndef FIPS_MODE
	84	# ifndef OPENSSL_NO_MD2
	85	ENCODE_DIGESTINFO_MD(md2, 0x02, MD2_DIGEST_LENGTH)
	86	# endif
	87	# ifndef OPENSSL_NO_MD5
	88	ENCODE_DIGESTINFO_MD(md5, 0x05, MD5_DIGEST_LENGTH)
	89	# endif
	90	# ifndef OPENSSL_NO_MDC2
	91	/* MDC-2 (2 5 8 3 101) */
	92	static const unsigned char digestinfo_mdc2_der[] = {
93	ASN1_SEQUENCE, 0x0c + MDC2_DIGEST_LENGTH,
94	ASN1_SEQUENCE, 0x08,
95	ASN1_OID, 0x04, 2 * 40 + 5, 8, 3, 101,
96	ASN1_NULL, 0x00,
97	ASN1_OCTET_STRING, MDC2_DIGEST_LENGTH
98	};
99	# endif
100	/* SHA-1 (1 3 14 3 2 26) */
101	static const unsigned char digestinfo_sha1_der[] = {
102	ASN1_SEQUENCE, 0x0d + SHA_DIGEST_LENGTH,
103	ASN1_SEQUENCE, 0x09,
104	ASN1_OID, 0x05, 1 * 40 + 3, 14, 3, 2, 26,
105	ASN1_NULL, 0x00,
106	ASN1_OCTET_STRING, SHA_DIGEST_LENGTH
107	};
108
109	#endif /* FIPS_MODE */
110
111	ENCODE_DIGESTINFO_SHA(sha256, 0x01, SHA256_DIGEST_LENGTH)
112	ENCODE_DIGESTINFO_SHA(sha384, 0x02, SHA384_DIGEST_LENGTH)
113	ENCODE_DIGESTINFO_SHA(sha512, 0x03, SHA512_DIGEST_LENGTH)
114	ENCODE_DIGESTINFO_SHA(sha224, 0x04, SHA224_DIGEST_LENGTH)
115	ENCODE_DIGESTINFO_SHA(sha512_224, 0x05, SHA224_DIGEST_LENGTH)
116	ENCODE_DIGESTINFO_SHA(sha512_256, 0x06, SHA256_DIGEST_LENGTH)
117	ENCODE_DIGESTINFO_SHA(sha3_224, 0x07, SHA224_DIGEST_LENGTH)
118	ENCODE_DIGESTINFO_SHA(sha3_256, 0x08, SHA256_DIGEST_LENGTH)
119	ENCODE_DIGESTINFO_SHA(sha3_384, 0x09, SHA384_DIGEST_LENGTH)
120	ENCODE_DIGESTINFO_SHA(sha3_512, 0x0a, SHA512_DIGEST_LENGTH)
121
122	#define MD_CASE(name) \
123	case NID_##name: \
124	*len = sizeof(digestinfo_##name##_der); \
125	return digestinfo_##name##_der;
126
127	static const unsigned char digestinfo_encoding(int nid, size_t len)
128	{
129	switch (nid) {
130	#ifndef FIPS_MODE
131	# ifndef OPENSSL_NO_MDC2
132	MD_CASE(mdc2)
133	# endif
134	# ifndef OPENSSL_NO_MD2
135	MD_CASE(md2)
136	# endif
137	# ifndef OPENSSL_NO_MD5
138	MD_CASE(md5)
139	# endif
140	MD_CASE(sha1)
141	#endif /* FIPS_MODE */
142	MD_CASE(sha224)
143	MD_CASE(sha256)
144	MD_CASE(sha384)
145	MD_CASE(sha512)
146	MD_CASE(sha512_224)
147	MD_CASE(sha512_256)
148	MD_CASE(sha3_224)
149	MD_CASE(sha3_256)
150	MD_CASE(sha3_384)
151	MD_CASE(sha3_512)
152	default:
153	return NULL;
154	}
155	}
156
1c80019a	157	/* Size of an SSL signature: MD5+SHA1 */
0f113f3e	158	#define SSL_SIG_LENGTH 36
1c80019a	159
608a0264	160	/*
169e422e	161	* Encodes a DigestInfo prefix of hash \|type\| and digest \|m\|, as
608a0264 DB	162	* described in EMSA-PKCS1-v1_5-ENCODE, RFC 3447 section 9.2 step 2. This
	163	* encodes the DigestInfo (T and tLen) but does not add the padding.
	164	*
	165	* On success, it returns one and sets \|*out\| to a newly allocated buffer
	166	* containing the result and \|*out_len\| to its length. The caller must free
169e422e	167	* \|*out\| with OPENSSL_free(). Otherwise, it returns zero.
608a0264	168	*/
169e422e SL	169	static int encode_pkcs1(unsigned char *out, size_t out_len, int type,
169e422e SL	170	const unsigned char *m, size_t m_len)
0f113f3e	171	{
169e422e SL	172	size_t di_prefix_len, dig_info_len;
	173	const unsigned char *di_prefix;
	174	unsigned char *dig_info;
	175
	176	if (type == NID_undef) {
608a0264 DB	177	RSAerr(RSA_F_ENCODE_PKCS1, RSA_R_UNKNOWN_ALGORITHM_TYPE);
	178	return 0;
	179	}
169e422e SL	180	di_prefix = digestinfo_encoding(type, &di_prefix_len);
169e422e SL	181	if (di_prefix == NULL) {
608a0264 DB	182	RSAerr(RSA_F_ENCODE_PKCS1,
	183	RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD);
	184	return 0;
	185	}
169e422e SL	186	dig_info_len = di_prefix_len + m_len;
	187	dig_info = OPENSSL_malloc(dig_info_len);
	188	if (dig_info == NULL) {
	189	RSAerr(RSA_F_ENCODE_PKCS1, ERR_R_MALLOC_FAILURE);
608a0264	190	return 0;
169e422e SL	191	}
	192	memcpy(dig_info, di_prefix, di_prefix_len);
	193	memcpy(dig_info + di_prefix_len, m, m_len);
608a0264	194
169e422e SL	195	*out = dig_info;
169e422e SL	196	*out_len = dig_info_len;
608a0264 DB	197	return 1;
	198	}
	199
	200	int RSA_sign(int type, const unsigned char *m, unsigned int m_len,
	201	unsigned char sigret, unsigned int siglen, RSA *rsa)
	202	{
169e422e SL	203	int encrypt_len, ret = 0;
169e422e SL	204	size_t encoded_len = 0;
608a0264 DB	205	unsigned char *tmps = NULL;
	206	const unsigned char *encoded = NULL;
	207
169e422e	208	if (rsa->meth->rsa_sign != NULL)
0f113f3e	209	return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa);
608a0264 DB	210
608a0264 DB	211	/* Compute the encoded digest. */
0f113f3e	212	if (type == NID_md5_sha1) {
608a0264 DB	213	/*
	214	* NID_md5_sha1 corresponds to the MD5/SHA1 combination in TLS 1.1 and
	215	* earlier. It has no DigestInfo wrapper but otherwise is
	216	* RSASSA-PKCS1-v1_5.
	217	*/
0f113f3e MC	218	if (m_len != SSL_SIG_LENGTH) {
0f113f3e MC	219	RSAerr(RSA_F_RSA_SIGN, RSA_R_INVALID_MESSAGE_LENGTH);
608a0264	220	return 0;
0f113f3e	221	}
608a0264 DB	222	encoded_len = SSL_SIG_LENGTH;
608a0264 DB	223	encoded = m;
0f113f3e	224	} else {
608a0264 DB	225	if (!encode_pkcs1(&tmps, &encoded_len, type, m, m_len))
	226	goto err;
	227	encoded = tmps;
0f113f3e	228	}
608a0264	229
169e422e	230	if (encoded_len + RSA_PKCS1_PADDING_SIZE > (size_t)RSA_size(rsa)) {
0f113f3e	231	RSAerr(RSA_F_RSA_SIGN, RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
608a0264	232	goto err;
0f113f3e	233	}
169e422e	234	encrypt_len = RSA_private_encrypt((int)encoded_len, encoded, sigret, rsa,
608a0264 DB	235	RSA_PKCS1_PADDING);
	236	if (encrypt_len <= 0)
	237	goto err;
d02b48c6	238
608a0264 DB	239	*siglen = encrypt_len;
	240	ret = 1;
	241
	242	err:
169e422e	243	OPENSSL_clear_free(tmps, encoded_len);
0f113f3e MC	244	return ret;
0f113f3e MC	245	}
1cfd255c	246
608a0264	247	/*
169e422e SL	248	* Verify an RSA signature in \|sigbuf\| using \|rsa\|.
	249	* \|type\| is the NID of the digest algorithm to use.
	250	* If \|rm\| is NULL, it verifies the signature for digest \|m\|, otherwise
	251	* it recovers the digest from the signature, writing the digest to \|rm\| and
	252	* the length to \|*prm_len\|.
	253	*
	254	* It returns one on successful verification or zero otherwise.
608a0264 DB	255	*/
608a0264 DB	256	int int_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
0f113f3e MC	257	unsigned char rm, size_t prm_len,
	258	const unsigned char sigbuf, size_t siglen, RSA rsa)
	259	{
169e422e SL	260	int len, ret = 0;
169e422e SL	261	size_t decrypt_len, encoded_len = 0;
608a0264	262	unsigned char decrypt_buf = NULL, encoded = NULL;
d02b48c6	263
608a0264	264	if (siglen != (size_t)RSA_size(rsa)) {
0f113f3e	265	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_WRONG_SIGNATURE_LENGTH);
608a0264	266	return 0;
0f113f3e	267	}
d02b48c6	268
608a0264 DB	269	/* Recover the encoded digest. */
	270	decrypt_buf = OPENSSL_malloc(siglen);
	271	if (decrypt_buf == NULL) {
0f113f3e MC	272	RSAerr(RSA_F_INT_RSA_VERIFY, ERR_R_MALLOC_FAILURE);
	273	goto err;
	274	}
d02b48c6	275
169e422e SL	276	len = RSA_public_decrypt((int)siglen, sigbuf, decrypt_buf, rsa,
	277	RSA_PKCS1_PADDING);
	278	if (len <= 0)
0f113f3e	279	goto err;
169e422e	280	decrypt_len = len;
608a0264 DB	281
	282	if (type == NID_md5_sha1) {
	283	/*
	284	* NID_md5_sha1 corresponds to the MD5/SHA1 combination in TLS 1.1 and
	285	* earlier. It has no DigestInfo wrapper but otherwise is
	286	* RSASSA-PKCS1-v1_5.
	287	*/
	288	if (decrypt_len != SSL_SIG_LENGTH) {
0f113f3e	289	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
608a0264	290	goto err;
dffe5109	291	}
b79aa05e	292
608a0264 DB	293	if (rm != NULL) {
	294	memcpy(rm, decrypt_buf, SSL_SIG_LENGTH);
	295	*prm_len = SSL_SIG_LENGTH;
	296	} else {
	297	if (m_len != SSL_SIG_LENGTH) {
	298	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_MESSAGE_LENGTH);
	299	goto err;
	300	}
b79aa05e	301
608a0264 DB	302	if (memcmp(decrypt_buf, m, SSL_SIG_LENGTH) != 0) {
	303	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
	304	goto err;
	305	}
0f113f3e	306	}
608a0264 DB	307	} else if (type == NID_mdc2 && decrypt_len == 2 + 16
	308	&& decrypt_buf[0] == 0x04 && decrypt_buf[1] == 0x10) {
	309	/*
	310	* Oddball MDC2 case: signature can be OCTET STRING. check for correct
	311	* tag and length octets.
	312	*/
	313	if (rm != NULL) {
	314	memcpy(rm, decrypt_buf + 2, 16);
	315	*prm_len = 16;
	316	} else {
	317	if (m_len != 16) {
	318	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_MESSAGE_LENGTH);
	319	goto err;
	320	}
b79aa05e	321
608a0264 DB	322	if (memcmp(m, decrypt_buf + 2, 16) != 0) {
	323	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
	324	goto err;
	325	}
	326	}
	327	} else {
0f113f3e	328	/*
608a0264 DB	329	* If recovering the digest, extract a digest-sized output from the end
	330	* of \|decrypt_buf\| for \|encode_pkcs1\|, then compare the decryption
	331	* output as in a standard verification.
0f113f3e	332	*/
608a0264 DB	333	if (rm != NULL) {
	334	const EVP_MD *md = EVP_get_digestbynid(type);
	335	if (md == NULL) {
	336	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_UNKNOWN_ALGORITHM_TYPE);
	337	goto err;
	338	}
	339
169e422e SL	340	len = EVP_MD_size(md);
	341	if (len <= 0)
	342	goto err;
	343	m_len = (unsigned int)len;
	344	if (m_len > decrypt_len) {
608a0264 DB	345	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_INVALID_DIGEST_LENGTH);
	346	goto err;
	347	}
	348	m = decrypt_buf + decrypt_len - m_len;
0f113f3e	349	}
d02b48c6	350
608a0264 DB	351	/* Construct the encoded digest and ensure it matches. */
	352	if (!encode_pkcs1(&encoded, &encoded_len, type, m, m_len))
	353	goto err;
dfeab068	354
608a0264	355	if (encoded_len != decrypt_len
169e422e	356	\|\| memcmp(encoded, decrypt_buf, encoded_len) != 0) {
608a0264	357	RSAerr(RSA_F_INT_RSA_VERIFY, RSA_R_BAD_SIGNATURE);
3d0cf918	358	goto err;
0f113f3e	359	}
608a0264 DB	360
	361	/* Output the recovered digest. */
	362	if (rm != NULL) {
	363	memcpy(rm, m, m_len);
	364	*prm_len = m_len;
	365	}
0f113f3e	366	}
608a0264 DB	367
	368	ret = 1;
	369
	370	err:
169e422e	371	OPENSSL_clear_free(encoded, encoded_len);
608a0264 DB	372	OPENSSL_clear_free(decrypt_buf, siglen);
608a0264 DB	373	return ret;
0f113f3e	374	}
d02b48c6	375
608a0264	376	int RSA_verify(int type, const unsigned char *m, unsigned int m_len,
0f113f3e MC	377	const unsigned char sigbuf, unsigned int siglen, RSA rsa)
0f113f3e MC	378	{
b2a97be7	379
169e422e	380	if (rsa->meth->rsa_verify != NULL)
608a0264	381	return rsa->meth->rsa_verify(type, m, m_len, sigbuf, siglen, rsa);
b2a97be7	382
608a0264	383	return int_rsa_verify(type, m, m_len, NULL, NULL, sigbuf, siglen, rsa);
0f113f3e	384	}