Handle private keys with lowercase hex digits in DEK-Info

Some tools, for example win-acme, create encrypted private keys in OpenSSL's
traditional format containing lowercase hex digits in the IV part of the
DEK-Info PEM header. These key files are accepted by OpenSSL. Prior to this
patch, GnuTLS did reject these keys with GNUTLS_E_INVALID_REQUEST.

Signed-off-by: Tim Kosse <tim.kosse@filezilla-project.org>
Co-authored-by: Daiki Ueno <ueno@gnu.org>

diff --git a/lib/x509/privkey_openssl.c b/lib/x509/privkey_openssl.c
index 9fc70e03221f59077edaf135de015207a00c5cd3..4ac1ad26726f95e8c172fa1e0001a87c3d57cbba 100644 (file)
--- a/lib/x509/privkey_openssl.c
+++ b/lib/x509/privkey_openssl.c
@@ -142,7 +142,7 @@ gnutls_x509_privkey_import_openssl(gnutls_x509_privkey_t key,
        gnutls_cipher_hd_t handle;
        gnutls_cipher_algorithm_t cipher = GNUTLS_CIPHER_UNKNOWN;
        gnutls_datum_t b64_data;
-       gnutls_datum_t salt, enc_key;
+       gnutls_datum_t salt, enc_key, hex_data;
        unsigned char *key_data;
        size_t key_data_size;
        const char *pem_header = (void *) data->data;
@@ -150,6 +150,7 @@ gnutls_x509_privkey_import_openssl(gnutls_x509_privkey_t key,
        ssize_t pem_header_size;
        int ret;
        unsigned int i, iv_size, l;
+       size_t salt_size;
 
        pem_header_size = data->size;
 
@@ -196,27 +197,21 @@ gnutls_x509_privkey_import_openssl(gnutls_x509_privkey_t key,
        if (!salt.data)
                return gnutls_assert_val(GNUTLS_E_MEMORY_ERROR);
 
-       for (i = 0; i < salt.size * 2; i++) {
-               unsigned char x;
-               const char *c = &pem_header[i];
+       hex_data.data = (unsigned char *)pem_header;
+       hex_data.size = salt.size * 2;
+       salt_size = salt.size;
 
-               if (*c >= '0' && *c <= '9')
-                       x = (*c) - '0';
-               else if (*c >= 'A' && *c <= 'F')
-                       x = (*c) - 'A' + 10;
-               else {
-                       gnutls_assert();
+       ret = gnutls_hex_decode(&hex_data, salt.data, &salt_size);
+       if (ret < 0) {
+               gnutls_assert();
+               if (ret == GNUTLS_E_PARSING_ERROR) {
                        /* Invalid salt in encrypted PEM file */
                        ret = GNUTLS_E_INVALID_REQUEST;
-                       goto out_salt;
                }
-               if (i & 1)
-                       salt.data[i / 2] |= x;
-               else
-                       salt.data[i / 2] = x << 4;
+               goto out_salt;
        }
 
-       pem_header += salt.size * 2;
+       pem_header += hex_data.size;
        if (*pem_header != '\r' && *pem_header != '\n') {
                gnutls_assert();
                ret = GNUTLS_E_INVALID_REQUEST;

diff --git a/tests/key-openssl.c b/tests/key-openssl.c
index 7800f23be6fa8289ea23f3c1a5459223b6dbd861..f58a8db685012cfef112e3472033c91589f93829 100644 (file)
--- a/tests/key-openssl.c
+++ b/tests/key-openssl.c
@@ -95,6 +95,20 @@ const char key2[] =
     "F3bDyqlxSOm7uxF/K3YzI44v8/D8GGnLBTpN+ANBdiY=\n"
     "-----END RSA PRIVATE KEY-----\n";
 
+const char key_lowercase_iv[] =
+    "-----BEGIN RSA PRIVATE KEY-----\n"
+    "Proc-Type: 4,ENCRYPTED\n"
+    "DEK-Info: AES-256-CBC,c1967f64f92d5c4ef302537b7b50b98f\n"
+    "\n"
+    "iRHj/BbuyHodcEt/cPduzhxOUCwe+o77j7DOepAb7rasr7uRDjFcB2DeB4yvog5q\n"
+    "M46pB5NVqegJCTcFht/90OKXprt2m04ntsCEXXfJ/NQIYP3NsLM+aNiWUL1cxPiZ\n"
+    "6fWp6uaR165F+T5vBRmo6dS3wowHeiHZMiSGuM6CbW+AO5R31og9cUuP2e02GPbq\n"
+    "ZCGyU8RnA6c1caCql/T/5WOIjyaFpJhigBnQc6EoVi3C6XULBQ1Ut9A8gw3gVWda\n"
+    "NBF8sHfwXyKuPhWLZwOM7ZewIOvnesezwW7Tpf2LfMBIe1YQixdsBgM1RvhEN2bl\n"
+    "mYR1L1zfr94z/fWDztq1MYtCBPUJcgrjNLb80xv1qq5hZorTM9gjAeLfT4x9I6/m\n"
+    "57ohSPIR3bXgRZuefjxBhQYthUPcZ+qktrbURcvHNLs=\n"
+    "-----END RSA PRIVATE KEY-----\n";
+
 static int good_pwd_cb(void* userdata, int attempt, const char* token_url,
                       const char* token_label, unsigned int flags,
                       char* pin, size_t pin_max) {
@@ -199,6 +213,22 @@ void doit(void)
        }
        gnutls_x509_privkey_deinit(pkey);
 
+       /* import a key with lowercase hex digits in the iv part of DEK-Info */
+       ret = gnutls_x509_privkey_init(&pkey);
+       if (ret < 0)
+               fail("gnutls_x509_privkey_init: %d\n", ret);
+
+       key.data = (void *) key_lowercase_iv;
+       key.size = sizeof(key_lowercase_iv);
+       ret =
+           gnutls_x509_privkey_import2(pkey, &key, GNUTLS_X509_FMT_PEM,
+                                       "123456", 0);
+       if (ret < 0) {
+               fail("gnutls_x509_privkey_import2: %s\n",
+                    gnutls_strerror(ret));
+       }
+       gnutls_x509_privkey_deinit(pkey);
+
        /*
         * Pin callback passwords will only be used if the password supplied to
         * gnutls_x509_privkey_import2 in NULL.  Consider possible combinations