From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 23 Jan 2017 15:40:01 +0000 (+0200)
Subject: lib-dcrypt: Add padding parameter for RSA encrypt/decrypt
X-Git-Tag: 2.3.8~45
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b5ae6819adb539f50170cf34bb862a08b275d06c;p=thirdparty%2Fdovecot%2Fcore.git

lib-dcrypt: Add padding parameter for RSA encrypt/decrypt
---

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index f2b5dce4d7..d1ce9a6c41 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -164,12 +164,15 @@ dcrypt_openssl_private_key_id_old(struct dcrypt_private_key *key,
 static void
 dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key,
 				     struct dcrypt_public_key **pub_key_r);
-static void dcrypt_openssl_unref_private_key(struct dcrypt_private_key **key);
-static void dcrypt_openssl_unref_public_key(struct dcrypt_public_key **key);
+static void
+dcrypt_openssl_unref_private_key(struct dcrypt_private_key **key);
+static void
+dcrypt_openssl_unref_public_key(struct dcrypt_public_key **key);
 static bool
 dcrypt_openssl_rsa_decrypt(struct dcrypt_private_key *key,
 			   const unsigned char *data, size_t data_len,
-			   buffer_t *result, const char **error_r);
+			   buffer_t *result, enum dcrypt_padding padding,
+			   const char **error_r);
 static bool
 dcrypt_openssl_key_string_get_info(const char *key_data,
 	enum dcrypt_key_format *format_r, enum dcrypt_key_version *version_r,
@@ -192,6 +195,30 @@ static bool dcrypt_openssl_error(const char **error_r)
 	return FALSE;
 }
 
+static int
+dcrypt_openssl_padding_mode(enum dcrypt_padding padding,
+			    bool sig, const char **error_r)
+{
+	switch (padding) {
+	case DCRYPT_PADDING_DEFAULT:
+		if (sig) return RSA_PKCS1_PSS_PADDING;
+		else return RSA_PKCS1_OAEP_PADDING;
+	case DCRYPT_PADDING_RSA_PKCS1_OAEP:
+		return RSA_PKCS1_OAEP_PADDING;
+	case DCRYPT_PADDING_RSA_PKCS1_PSS:
+		return RSA_PKCS1_PSS_PADDING;
+	case DCRYPT_PADDING_RSA_PKCS1:
+		return RSA_PKCS1_PADDING;
+	case DCRYPT_PADDING_RSA_NO:
+		return RSA_NO_PADDING;
+	default:
+		if (*error_r != NULL)
+			*error_r = "Unsupported padding mode";
+		return -1;
+	}
+	i_unreached();
+}
+
 static bool dcrypt_openssl_initialize(const struct dcrypt_settings *set,
 				      const char **error_r)
 {
@@ -1283,7 +1310,7 @@ dcrypt_openssl_load_private_key_dovecot_v2(struct dcrypt_private_key **key_r,
 		if (EVP_PKEY_base_id((EVP_PKEY*)dec_key) == EVP_PKEY_RSA) {
 			if (!dcrypt_openssl_rsa_decrypt(dec_key,
 				peer_key->data, peer_key->used, secret,
-				error_r))
+				DCRYPT_PADDING_RSA_PKCS1_OAEP, error_r))
 				return FALSE;
 		} else {
 			/* perform ECDH */
@@ -2171,6 +2198,7 @@ dcrypt_openssl_encrypt_private_key_dovecot(buffer_t *key, int enctype,
 			buffer_set_used_size(secret, used+16);
 			if (!dcrypt_rsa_encrypt(enc_key, secret->data,
 						secret->used, peer_key,
+						DCRYPT_PADDING_RSA_PKCS1_OAEP,
 						error_r)) {
 				return FALSE;
 			}
@@ -2859,17 +2887,20 @@ static void dcrypt_openssl_unref_keypair(struct dcrypt_keypair *keypair)
 static bool
 dcrypt_openssl_rsa_encrypt(struct dcrypt_public_key *key,
 			   const unsigned char *data, size_t data_len,
-			   buffer_t *result, const char **error_r)
+			   buffer_t *result, enum dcrypt_padding padding,
+			   const char **error_r)
 {
-	int ec;
 	i_assert(key != NULL && key->key != NULL);
+	int ec, pad = dcrypt_openssl_padding_mode(padding, FALSE, error_r);
+	if (pad == -1)
+		return FALSE;
 	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(key->key, NULL);
 	size_t outl = EVP_PKEY_size(key->key);
 	unsigned char buf[outl];
 
 	if (ctx == NULL ||
 	    EVP_PKEY_encrypt_init(ctx) < 1 ||
-	    EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) < 1 ||
+	    EVP_PKEY_CTX_set_rsa_padding(ctx, pad) < 1 ||
 	    EVP_PKEY_encrypt(ctx, buf, &outl, data, data_len) < 1) {
 		dcrypt_openssl_error(error_r);
 		ec = -1;
@@ -2886,17 +2917,20 @@ dcrypt_openssl_rsa_encrypt(struct dcrypt_public_key *key,
 static bool
 dcrypt_openssl_rsa_decrypt(struct dcrypt_private_key *key,
 			   const unsigned char *data, size_t data_len,
-			   buffer_t *result, const char **error_r)
+			   buffer_t *result, enum dcrypt_padding padding,
+			   const char **error_r)
 {
-	int ec;
 	i_assert(key != NULL && key->key != NULL);
+	int ec, pad = dcrypt_openssl_padding_mode(padding, FALSE, error_r);
+	if (pad == -1)
+		return FALSE;
 	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(key->key, NULL);
 	size_t outl = EVP_PKEY_size(key->key);
 	unsigned char buf[outl];
 
 	if (ctx == NULL ||
 	    EVP_PKEY_decrypt_init(ctx) < 1 ||
-	    EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) < 1 ||
+	    EVP_PKEY_CTX_set_rsa_padding(ctx, pad) < 1 ||
 	    EVP_PKEY_decrypt(ctx, buf, &outl, data, data_len) < 1) {
 		dcrypt_openssl_error(error_r);
 		ec = -1;
diff --git a/src/lib-dcrypt/dcrypt-private.h b/src/lib-dcrypt/dcrypt-private.h
index bf3a5f192d..96fa8d3c96 100644
--- a/src/lib-dcrypt/dcrypt-private.h
+++ b/src/lib-dcrypt/dcrypt-private.h
@@ -129,10 +129,12 @@ struct dcrypt_vfs {
 
 	bool (*rsa_encrypt)(struct dcrypt_public_key *key,
 			    const unsigned char *data, size_t data_len,
-			    buffer_t *result, const char **error_r);
+			    buffer_t *result, enum dcrypt_padding padding,
+			    const char **error_r);
 	bool (*rsa_decrypt)(struct dcrypt_private_key *key,
 			    const unsigned char *data, size_t data_len,
-			    buffer_t *result, const char **error_r);
+			    buffer_t *result, enum dcrypt_padding padding,
+			    const char **error_r);
 
 	const char *(*oid2name)(const unsigned char *oid,
 				size_t oid_len, const char **error_r);
diff --git a/src/lib-dcrypt/dcrypt.c b/src/lib-dcrypt/dcrypt.c
index 0d8dbf53ef..6b2abc9b0c 100644
--- a/src/lib-dcrypt/dcrypt.c
+++ b/src/lib-dcrypt/dcrypt.c
@@ -408,18 +408,22 @@ void dcrypt_key_unref_private(struct dcrypt_private_key **key)
 
 bool dcrypt_rsa_encrypt(struct dcrypt_public_key *key,
 			const unsigned char *data, size_t data_len,
-			buffer_t *result, const char **error_r)
+			buffer_t *result, enum dcrypt_padding padding,
+			const char **error_r)
 {
 	i_assert(dcrypt_vfs != NULL);
-	return dcrypt_vfs->rsa_encrypt(key, data, data_len, result, error_r);
+	return dcrypt_vfs->rsa_encrypt(key, data, data_len, result,
+				       padding, error_r);
 }
 
 bool dcrypt_rsa_decrypt(struct dcrypt_private_key *key,
 			const unsigned char *data, size_t data_len,
-			buffer_t *result, const char **error_r)
+			buffer_t *result, enum dcrypt_padding padding,
+			const char **error_r)
 {
 	i_assert(dcrypt_vfs != NULL);
-	return dcrypt_vfs->rsa_decrypt(key, data, data_len, result, error_r);
+	return dcrypt_vfs->rsa_decrypt(key, data, data_len, result,
+				       padding, error_r);
 }
 
 const char *dcrypt_oid2name(const unsigned char *oid, size_t oid_len,
diff --git a/src/lib-dcrypt/dcrypt.h b/src/lib-dcrypt/dcrypt.h
index dbb6bef943..ab22d9b4f3 100644
--- a/src/lib-dcrypt/dcrypt.h
+++ b/src/lib-dcrypt/dcrypt.h
@@ -376,10 +376,12 @@ void dcrypt_key_set_usage_private(struct dcrypt_private_key *key,
 /* RSA stuff */
 bool dcrypt_rsa_encrypt(struct dcrypt_public_key *key,
 			const unsigned char *data, size_t data_len,
-			buffer_t *result, const char **error_r);
+			buffer_t *result, enum dcrypt_padding padding,
+			const char **error_r);
 bool dcrypt_rsa_decrypt(struct dcrypt_private_key *key,
 			const unsigned char *data, size_t data_len,
-			buffer_t *result, const char **error_r);
+			buffer_t *result, enum dcrypt_padding padding,
+			const char **error_r);
 
 /* OID stuff */
 const char *dcrypt_oid2name(const unsigned char *oid, size_t oid_len,
diff --git a/src/lib-dcrypt/istream-decrypt.c b/src/lib-dcrypt/istream-decrypt.c
index db0ea708eb..96fcc97832 100644
--- a/src/lib-dcrypt/istream-decrypt.c
+++ b/src/lib-dcrypt/istream-decrypt.c
@@ -435,8 +435,9 @@ i_stream_decrypt_key(struct decrypt_istream *stream, const char *malg,
 
 	/* decrypt the seed */
 	if (ktype == DCRYPT_KEY_RSA) {
-		if (!dcrypt_rsa_decrypt(stream->priv_key, encrypted_key,
-					eklen, key, &error)) {
+		if (!dcrypt_rsa_decrypt(stream->priv_key, encrypted_key, eklen,
+					key, DCRYPT_PADDING_RSA_PKCS1_OAEP,
+					&error)) {
 			io_stream_set_error(&stream->istream.iostream,
 					    "key decryption error: %s", error);
 			return -1;
diff --git a/src/lib-dcrypt/ostream-encrypt.c b/src/lib-dcrypt/ostream-encrypt.c
index 585e2a530a..3264725ebf 100644
--- a/src/lib-dcrypt/ostream-encrypt.c
+++ b/src/lib-dcrypt/ostream-encrypt.c
@@ -277,8 +277,9 @@ o_stream_encrypt_key_for_pubkey_v2(struct encrypt_ostream *stream,
 
 	if (ktype == DCRYPT_KEY_RSA) {
 		/* encrypt key as R (as we don't need DH with RSA)*/
-		if (!dcrypt_rsa_encrypt(pubkey, key, key_len,
-					encrypted_key, &error)) {
+		if (!dcrypt_rsa_encrypt(pubkey, key, key_len, encrypted_key,
+					DCRYPT_PADDING_RSA_PKCS1_OAEP,
+					&error)) {
 			io_stream_set_error(&stream->ostream.iostream,
 					    "Cannot encrypt key data: %s",
 					    error);