Make ECDSA_size() use consistent asn1 encoder.

ECDSA signature lengths are calculated using i2d_ECDSA_SIG().
i2d_ECDSA_SIG() was changed in a previous PR to use a custom ASN1 encoder (using WPACKET)
so that the normal ASN1 encoder does not need to be pulled into the provider boundary.
For consistency ECDSA_size() has been changed to also use i2d_ECDSA_SIG() - this can now
be used directly inside the FIPS provider.

Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/10577)

diff --git a/crypto/ec/ec_asn1.c b/crypto/ec/ec_asn1.c
index c993821bb9130e4e079682fe8643e92a0f6e04dd..0567f2ab069d744f9f2759534b3607fed8729e8d 100644 (file)
--- a/crypto/ec/ec_asn1.c
+++ b/crypto/ec/ec_asn1.c
@@ -1342,32 +1342,27 @@ int ECDSA_SIG_set0(ECDSA_SIG *sig, BIGNUM *r, BIGNUM *s)
     return 1;
 }
 
-#ifndef FIPS_MODE
-int ECDSA_size(const EC_KEY *r)
+int ECDSA_size(const EC_KEY *ec)
 {
-    int ret, i;
-    ASN1_INTEGER bs;
-    unsigned char buf[4];
+    int ret;
+    ECDSA_SIG sig;
     const EC_GROUP *group;
+    const BIGNUM *bn;
 
-    if (r == NULL)
+    if (ec == NULL)
         return 0;
-    group = EC_KEY_get0_group(r);
+    group = EC_KEY_get0_group(ec);
     if (group == NULL)
         return 0;
 
-    i = EC_GROUP_order_bits(group);
-    if (i == 0)
+    bn = EC_GROUP_get0_order(group);
+    if (bn == NULL)
         return 0;
-    bs.length = (i + 7) / 8;
-    bs.data = buf;
-    bs.type = V_ASN1_INTEGER;
-    /* If the top bit is set the asn1 encoding is 1 larger. */
-    buf[0] = 0xff;
-
-    i = i2d_ASN1_INTEGER(&bs, NULL);
-    i += i;                     /* r and s */
-    ret = ASN1_object_size(1, i, V_ASN1_SEQUENCE);
+
+    sig.r = sig.s = (BIGNUM *)bn;
+    ret = i2d_ECDSA_SIG(&sig, NULL);
+
+    if (ret < 0)
+        ret = 0;
     return ret;
 }
-#endif