From: Stefan Metzmacher Date: Tue, 9 Mar 2021 09:40:04 +0000 (+0100) Subject: gnutls: allow gnutls_aead_cipher_encryptv2 with gcm before 3.6.15 X-Git-Tag: ldb-2.5.0~1064 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5512416a8fbe00a7a5343afe0d50846e0a8f342b;p=thirdparty%2Fsamba.git gnutls: allow gnutls_aead_cipher_encryptv2 with gcm before 3.6.15 The memory leak bug up to 3.6.14 was only related to ccm, but gcm was fine. This avoids talloc+memcpy on more systems, e.g. ubuntu 20.04, and brings ~ 20% less cpu overhead, see: https://hackmd.io/@asn/samba_crypto_benchmarks BUG: https://bugzilla.samba.org/show_bug.cgi?id=14764 Signed-off-by: Stefan Metzmacher Reviewed-by: Andreas Schneider --- diff --git a/libcli/smb/smb2_signing.c b/libcli/smb/smb2_signing.c index 830f3bf1570..fdb69e90a07 100644 --- a/libcli/smb/smb2_signing.c +++ b/libcli/smb/smb2_signing.c @@ -324,7 +324,7 @@ static NTSTATUS smb2_signing_gmac(gnutls_aead_cipher_hd_t cipher_hnd, { size_t tag_size = _tag_size; int rc; -#if defined(HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2) +#ifdef ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_GCM rc = gnutls_aead_cipher_encryptv2(cipher_hnd, iv, iv_size, @@ -336,7 +336,7 @@ static NTSTATUS smb2_signing_gmac(gnutls_aead_cipher_hd_t cipher_hnd, } return NT_STATUS_OK; -#else /* HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2 */ +#else /* ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_GCM */ TALLOC_CTX *tmp_ctx = NULL; size_t atext_size = 0; uint8_t *atext = NULL; @@ -387,7 +387,7 @@ static NTSTATUS smb2_signing_gmac(gnutls_aead_cipher_hd_t cipher_hnd, } return NT_STATUS_OK; -#endif /* HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2 */ +#endif /* ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_GCM */ } static NTSTATUS smb2_signing_calc_signature(struct smb2_signing_key *signing_key, @@ -808,6 +808,9 @@ NTSTATUS smb2_signing_encrypt_pdu(struct smb2_signing_key *encryption_key, struct iovec *vector, int count) { +#ifdef HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2 + bool use_encryptv2 = false; +#endif uint16_t cipher_id; uint8_t *tf; size_t a_total; @@ -851,18 +854,30 @@ NTSTATUS smb2_signing_encrypt_pdu(struct smb2_signing_key *encryption_key, case SMB2_ENCRYPTION_AES128_CCM: algo = GNUTLS_CIPHER_AES_128_CCM; iv_size = SMB2_AES_128_CCM_NONCE_SIZE; +#ifdef ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_CCM + use_encryptv2 = true; +#endif break; case SMB2_ENCRYPTION_AES128_GCM: algo = GNUTLS_CIPHER_AES_128_GCM; iv_size = gnutls_cipher_get_iv_size(algo); +#ifdef ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_GCM + use_encryptv2 = true; +#endif break; case SMB2_ENCRYPTION_AES256_CCM: algo = GNUTLS_CIPHER_AES_256_CCM; iv_size = SMB2_AES_128_CCM_NONCE_SIZE; +#ifdef ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_CCM + use_encryptv2 = true; +#endif break; case SMB2_ENCRYPTION_AES256_GCM: algo = GNUTLS_CIPHER_AES_256_GCM; iv_size = gnutls_cipher_get_iv_size(algo); +#ifdef ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_GCM + use_encryptv2 = true; +#endif break; default: return NT_STATUS_INVALID_PARAMETER; @@ -903,8 +918,8 @@ NTSTATUS smb2_signing_encrypt_pdu(struct smb2_signing_key *encryption_key, 0, 16 - iv_size); -#if defined(HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2) - { +#ifdef HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2 + if (use_encryptv2) { uint8_t tag[tag_size]; giovec_t auth_iov[1]; @@ -928,8 +943,8 @@ NTSTATUS smb2_signing_encrypt_pdu(struct smb2_signing_key *encryption_key, } memcpy(tf + SMB2_TF_SIGNATURE, tag, tag_size); - } -#else /* HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2 */ + } else +#endif /* HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2 */ { size_t ptext_size = m_total; uint8_t *ptext = NULL; @@ -1007,7 +1022,6 @@ NTSTATUS smb2_signing_encrypt_pdu(struct smb2_signing_key *encryption_key, TALLOC_FREE(ptext); TALLOC_FREE(ctext); } -#endif /* HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2 */ DBG_INFO("Encrypted SMB2 message\n"); @@ -1020,6 +1034,9 @@ NTSTATUS smb2_signing_decrypt_pdu(struct smb2_signing_key *decryption_key, struct iovec *vector, int count) { +#ifdef HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2 + bool use_encryptv2 = false; +#endif uint16_t cipher_id; uint8_t *tf; uint16_t flags; @@ -1073,18 +1090,30 @@ NTSTATUS smb2_signing_decrypt_pdu(struct smb2_signing_key *decryption_key, case SMB2_ENCRYPTION_AES128_CCM: algo = GNUTLS_CIPHER_AES_128_CCM; iv_size = SMB2_AES_128_CCM_NONCE_SIZE; +#ifdef ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_CCM + use_encryptv2 = true; +#endif break; case SMB2_ENCRYPTION_AES128_GCM: algo = GNUTLS_CIPHER_AES_128_GCM; iv_size = gnutls_cipher_get_iv_size(algo); +#ifdef ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_GCM + use_encryptv2 = true; +#endif break; case SMB2_ENCRYPTION_AES256_CCM: algo = GNUTLS_CIPHER_AES_256_CCM; iv_size = SMB2_AES_128_CCM_NONCE_SIZE; +#ifdef ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_CCM + use_encryptv2 = true; +#endif break; case SMB2_ENCRYPTION_AES256_GCM: algo = GNUTLS_CIPHER_AES_256_GCM; iv_size = gnutls_cipher_get_iv_size(algo); +#ifdef ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_GCM + use_encryptv2 = true; +#endif break; default: return NT_STATUS_INVALID_PARAMETER; @@ -1122,8 +1151,8 @@ NTSTATUS smb2_signing_decrypt_pdu(struct smb2_signing_key *decryption_key, } /* gnutls_aead_cipher_encryptv2() has a bug in version 3.6.10 */ -#if defined(HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2) - { +#ifdef HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2 + if (use_encryptv2) { giovec_t auth_iov[1]; auth_iov[0] = (giovec_t) { @@ -1144,8 +1173,8 @@ NTSTATUS smb2_signing_decrypt_pdu(struct smb2_signing_key *decryption_key, status = gnutls_error_to_ntstatus(rc, NT_STATUS_INTERNAL_ERROR); goto out; } - } -#else /* HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2 */ + } else +#endif /* HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2 */ { size_t ctext_size = m_total + tag_size; uint8_t *ctext = NULL; @@ -1229,7 +1258,6 @@ NTSTATUS smb2_signing_decrypt_pdu(struct smb2_signing_key *decryption_key, TALLOC_FREE(ptext); TALLOC_FREE(ctext); } -#endif /* HAVE_GNUTLS_AEAD_CIPHER_ENCRYPTV2 */ DBG_INFO("Decrypted SMB2 message\n"); diff --git a/wscript_configure_system_gnutls b/wscript_configure_system_gnutls index 28abd29f964..9864235d17e 100644 --- a/wscript_configure_system_gnutls +++ b/wscript_configure_system_gnutls @@ -13,7 +13,8 @@ conf.CHECK_CFG(package='gnutls', msg='Checking for GnuTLS >= %s' % gnutls_required_version, mandatory=True) -gnutls_version = conf.cmd_and_log(conf.env.PKGCONFIG + ['--modversion', 'gnutls']).strip() +gnutls_version_str = conf.cmd_and_log(conf.env.PKGCONFIG + ['--modversion', 'gnutls']).strip() +gnutls_version = parse_version(gnutls_version_str) # Define gnutls as a system library conf.SET_TARGET_TYPE('gnutls', 'SYSLIB') @@ -33,8 +34,11 @@ conf.CHECK_FUNCS_IN('gnutls_set_default_priority_append', 'gnutls') # # 3.6.10 - 3.6.14 have a severe memory leak with AES-CCM # https://gitlab.com/gnutls/gnutls/-/merge_requests/1278 -if (parse_version(gnutls_version) > parse_version('3.6.14')): - conf.CHECK_FUNCS_IN('gnutls_aead_cipher_encryptv2', 'gnutls') +if (gnutls_version > parse_version('3.6.10')): + if conf.CHECK_FUNCS_IN('gnutls_aead_cipher_encryptv2', 'gnutls'): + conf.DEFINE('ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_GCM', 1) + if (gnutls_version > parse_version('3.6.14')): + conf.DEFINE('ALLOW_GNUTLS_AEAD_CIPHER_ENCRYPTV2_AES_CCM', 1) # Check if gnutls has fips mode support # gnutls_fips140_mode_enabled() is available since 3.3.0