Fix default pkey(1) DER output

This is expected to be PKCS#8, but was defaulting to traditional when
possible.  Changed default DER output format to PKCS#8 and extended the
`-traditional` option to apply also to DER output.

Fixes: #21708
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/27273)

diff --git a/CHANGES.md b/CHANGES.md
index d2cc795e31b4d56df514a5875c28c86285234fc7..352c0345d36c0e7efeee58165431cfb6a93163a5 100644 (file)
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -31,6 +31,15 @@ OpenSSL 3.6
 
 ### Changes between 3.5 and 3.6 [xx XXX xxxx]
 
+ * Changed openssl-pkey(1) to match the documentation when private keys
+   are output in DER format (`-outform DER`) by producing the `PKCS#8` form by
+   default.  Previously this would output the *traditional* form for those
+   older key types (`DSA`, `RSA`, `ECDSA`) that had such a form.  The
+   `-traditional` flag has been extended to support explicit requests to output
+   that format in DER format (it was previously PEM-only).
+
+   *Viktor Dukhovni*
+
  * Support setting a free function thunk to OPENSSL_sk stack types. Using a thunk
    allows the type specific free function to be called with the correct type
    information from generic functions like OPENSSL_sk_pop_free().

diff --git a/apps/pkey.c b/apps/pkey.c
index ab0c95c2acd6805e835999cf641868afbea8057f..8ecfbb3ca5fd4d63c78227ec730147f3060693a2 100644 (file)
--- a/apps/pkey.c
+++ b/apps/pkey.c
@@ -178,9 +178,9 @@ int pkey_main(int argc, char **argv)
     if (text && text_pub)
         BIO_printf(bio_err,
                    "Warning: The -text option is ignored with -text_pub\n");
-    if (traditional && (noout || outformat != FORMAT_PEM))
+    if (traditional && (noout || pubout))
         BIO_printf(bio_err,
-                   "Warning: The -traditional is ignored since there is no PEM output\n");
+                   "Warning: -traditional is ignored with no private key output\n");
 
     /* -pubout and -text is the same as -text_pub */
     if (!text_pub && pubout && text) {
@@ -295,8 +295,14 @@ int pkey_main(int argc, char **argv)
                     goto end;
             } else {
                 assert(private);
-                if (!i2d_PrivateKey_bio(out, pkey))
-                    goto end;
+                if (traditional) {
+                    if (!i2d_PrivateKey_bio(out, pkey))
+                        goto end;
+                } else {
+                    if (!i2d_PKCS8PrivateKey_bio(out, pkey, NULL, NULL, 0,
+                                                 NULL, NULL))
+                        goto end;
+                }
             }
         } else {
             BIO_printf(bio_err, "Bad format specified for key\n");

diff --git a/test/recipes/15-test_pkey.t b/test/recipes/15-test_pkey.t
index 85b870f5a9e7c2b4ee67779e36b4d45d341040a0..967c316130a3eef6d26a8580786c808666b145dc 100644 (file)
--- a/test/recipes/15-test_pkey.t
+++ b/test/recipes/15-test_pkey.t
@@ -97,8 +97,8 @@ subtest "=== pkey handling of DER encoding ===" => sub {
 
     my $der_out = 'key.der';
     my $pem_out = 'key.pem';
-    ok(run(app([@app, '-in', $in_key, '-outform', 'DER',
-                 '-out', $der_out])),
+    ok(run(app([@app, '-in', $in_key, qw(-traditional -outform DER -out),
+                $der_out])),
        "write DER-encoded pkey");
 
     ok(run(app(['openssl', 'asn1parse', '-in', $der_out, '-inform', 'DER',