From f492649b99012b0b8ab5c83b66b20aa65bee1f1c Mon Sep 17 00:00:00 2001
From: Viktor Dukhovni <openssl-users@dukhovni.org>
Date: Wed, 9 Apr 2025 17:33:02 +1000
Subject: [PATCH] Fix default pkey(1) DER output

This is expected to be PKCS#8, but was defaulting to traditional when
possible.  Changed default DER output format to PKCS#8 and extended the
`-traditional` option to apply also to DER output.

Fixes: #21708

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/27273)
---
 CHANGES.md                  |  9 +++++++++
 apps/pkey.c                 | 14 ++++++++++----
 test/recipes/15-test_pkey.t |  4 ++--
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index d2cc795e31b..352c0345d36 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -31,6 +31,15 @@ OpenSSL 3.6
 
 ### Changes between 3.5 and 3.6 [xx XXX xxxx]
 
+ * Changed openssl-pkey(1) to match the documentation when private keys
+   are output in DER format (`-outform DER`) by producing the `PKCS#8` form by
+   default.  Previously this would output the *traditional* form for those
+   older key types (`DSA`, `RSA`, `ECDSA`) that had such a form.  The
+   `-traditional` flag has been extended to support explicit requests to output
+   that format in DER format (it was previously PEM-only).
+
+   *Viktor Dukhovni*
+
  * Support setting a free function thunk to OPENSSL_sk stack types. Using a thunk
    allows the type specific free function to be called with the correct type
    information from generic functions like OPENSSL_sk_pop_free().
diff --git a/apps/pkey.c b/apps/pkey.c
index ab0c95c2acd..8ecfbb3ca5f 100644
--- a/apps/pkey.c
+++ b/apps/pkey.c
@@ -178,9 +178,9 @@ int pkey_main(int argc, char **argv)
     if (text && text_pub)
         BIO_printf(bio_err,
                    "Warning: The -text option is ignored with -text_pub\n");
-    if (traditional && (noout || outformat != FORMAT_PEM))
+    if (traditional && (noout || pubout))
         BIO_printf(bio_err,
-                   "Warning: The -traditional is ignored since there is no PEM output\n");
+                   "Warning: -traditional is ignored with no private key output\n");
 
     /* -pubout and -text is the same as -text_pub */
     if (!text_pub && pubout && text) {
@@ -295,8 +295,14 @@ int pkey_main(int argc, char **argv)
                     goto end;
             } else {
                 assert(private);
-                if (!i2d_PrivateKey_bio(out, pkey))
-                    goto end;
+                if (traditional) {
+                    if (!i2d_PrivateKey_bio(out, pkey))
+                        goto end;
+                } else {
+                    if (!i2d_PKCS8PrivateKey_bio(out, pkey, NULL, NULL, 0,
+                                                 NULL, NULL))
+                        goto end;
+                }
             }
         } else {
             BIO_printf(bio_err, "Bad format specified for key\n");
diff --git a/test/recipes/15-test_pkey.t b/test/recipes/15-test_pkey.t
index 85b870f5a9e..967c316130a 100644
--- a/test/recipes/15-test_pkey.t
+++ b/test/recipes/15-test_pkey.t
@@ -97,8 +97,8 @@ subtest "=== pkey handling of DER encoding ===" => sub {
 
     my $der_out = 'key.der';
     my $pem_out = 'key.pem';
-    ok(run(app([@app, '-in', $in_key, '-outform', 'DER',
-                 '-out', $der_out])),
+    ok(run(app([@app, '-in', $in_key, qw(-traditional -outform DER -out),
+                $der_out])),
        "write DER-encoded pkey");
 
     ok(run(app(['openssl', 'asn1parse', '-in', $der_out, '-inform', 'DER',
-- 
2.47.2