upstream: make the PKCS#11 RSA code more like the new PKCS#11

ECDSA code: use a single custom RSA_METHOD instead of a method per key

suggested by me, but markus@ did all the work.
ok djm@

OpenBSD-Commit-ID: 8aafcebe923dc742fc5537a995cee549d07e4b2e

diff --git a/ssh-pkcs11.c b/ssh-pkcs11.c
index c36d31604bbbd6b394fedd4211bc395d5c373f7f..d7b3a65f0a53c8151eeb9709ea6495fa5e6c4e53 100644 (file)
--- a/ssh-pkcs11.c
+++ b/ssh-pkcs11.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssh-pkcs11.c,v 1.32 2019/01/20 23:05:52 djm Exp $ */
+/* $OpenBSD: ssh-pkcs11.c,v 1.33 2019/01/20 23:08:24 djm Exp $ */
 /*
  * Copyright (c) 2010 Markus Friedl.  All rights reserved.
  * Copyright (c) 2014 Pedro Martelletto. All rights reserved.
@@ -334,8 +334,8 @@ pkcs11_rsa_private_encrypt(int flen, const u_char *from, u_char *to, RSA *rsa,
        CK_RV                   rv;
        int                     rval = -1;
 
-       if ((k11 = RSA_get_app_data(rsa)) == NULL) {
-               error("RSA_get_app_data failed for rsa %p", rsa);
+       if ((k11 = RSA_get_ex_data(rsa, 0)) == NULL) {
+               error("RSA_get_ex_data failed for rsa %p", rsa);
                return (-1);
        }
 
@@ -365,13 +365,35 @@ pkcs11_rsa_private_decrypt(int flen, const u_char *from, u_char *to, RSA *rsa,
        return (-1);
 }
 
+static RSA_METHOD *rsa_method;
+
+static int
+pkcs11_rsa_start_wrapper(void)
+{
+       if (rsa_method != NULL)
+               return (0);
+       rsa_method = RSA_meth_dup(RSA_get_default_method());
+       if (rsa_method == NULL)
+               return (-1);
+       if (!RSA_meth_set1_name(rsa_method, "pkcs11") ||
+           !RSA_meth_set_priv_enc(rsa_method, pkcs11_rsa_private_encrypt) ||
+           !RSA_meth_set_priv_dec(rsa_method, pkcs11_rsa_private_decrypt) ||
+           !RSA_meth_set_finish(rsa_method, pkcs11_rsa_finish)) {
+               error("%s: setup pkcs11 method failed", __func__);
+               return (-1);
+       }
+       return (0);
+}
+
 /* redirect private key operations for rsa key to pkcs11 token */
 static int
 pkcs11_rsa_wrap(struct pkcs11_provider *provider, CK_ULONG slotidx,
     CK_ATTRIBUTE *keyid_attrib, RSA *rsa)
 {
        struct pkcs11_key       *k11;
-       const RSA_METHOD        *def = RSA_get_default_method();
+
+       if (pkcs11_rsa_start_wrapper() == -1)
+               return (-1);
 
        k11 = xcalloc(1, sizeof(*k11));
        k11->provider = provider;
@@ -383,19 +405,10 @@ pkcs11_rsa_wrap(struct pkcs11_provider *provider, CK_ULONG slotidx,
                k11->keyid = xmalloc(k11->keyid_len);
                memcpy(k11->keyid, keyid_attrib->pValue, k11->keyid_len);
        }
-       k11->rsa_method = RSA_meth_dup(def);
-       if (k11->rsa_method == NULL)
-               fatal("%s: RSA_meth_dup failed", __func__);
-       k11->orig_finish = RSA_meth_get_finish(def);
-       if (!RSA_meth_set1_name(k11->rsa_method, "pkcs11") ||
-           !RSA_meth_set_priv_enc(k11->rsa_method,
-           pkcs11_rsa_private_encrypt) ||
-           !RSA_meth_set_priv_dec(k11->rsa_method,
-           pkcs11_rsa_private_decrypt) ||
-           !RSA_meth_set_finish(k11->rsa_method, pkcs11_rsa_finish))
-               fatal("%s: setup pkcs11 method failed", __func__);
+
+       k11->rsa_method = rsa_method;
        RSA_set_method(rsa, k11->rsa_method);
-       RSA_set_app_data(rsa, k11);
+       RSA_set_ex_data(rsa, 0, k11);
        return (0);
 }