apps: add dgst test coverage for -keyform option

The -keyform (OPT_KEYFORM) option of the dgst app was not exercised by
any test.  Add a subtest that converts the RSA test keys to DER and
then signs and verifies with -keyform DER, covering the option for both
the private and public key loading paths.

Assisted-by: Claude:claude-opus-4-8
Reviewed-by: Daniel Kubec <kubec@openssl.foundation>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Tim Hudson <tjh@openssl.org>
MergeDate: Mon Jul 20 06:40:45 2026
(Merged from https://github.com/openssl/openssl/pull/31946)

diff --git a/test/recipes/20-test_dgst.t b/test/recipes/20-test_dgst.t
index 6cabaf3be55e9c8e235806a7ba692fa6f34d7c8d..00af8cfff288b528e3e635f3d6fa76e524e8d758 100644 (file)
--- a/test/recipes/20-test_dgst.t
+++ b/test/recipes/20-test_dgst.t
@@ -18,7 +18,7 @@ use Cwd qw(abs_path);
 
 setup("test_dgst");
 
-plan tests => 26;
+plan tests => 27;
 
 sub tsignverify {
     my $testtext = shift;
@@ -437,6 +437,40 @@ subtest "Listing supported digests with `dgst` CLI" => sub {
     ok($listing =~ /-sha512\b/, "LIST: Check sha512 is listed");
 };
 
+subtest "signing and verifying with DER `-keyform` `dgst` CLI" => sub {
+    if (disabled("rsa")) {
+        plan tests => 1;
+        ok(1, "Skipped (RSA not supported)");
+        return;
+    }
+    plan tests => 4;
+
+    my $data_to_sign = srctop_file('test', 'data.bin');
+    my $privkey_pem = srctop_file("test", "testrsa.pem");
+    my $pubkey_pem = srctop_file("test", "testrsapub.pem");
+    my $privkey_der = "testrsa-keyform.der";
+    my $pubkey_der = "testrsapub-keyform.der";
+    my $sigfile = "testrsa-keyform.sig";
+
+    # Convert the keys to DER so the `-keyform DER` code path can be exercised.
+    ok(run(app(['openssl', 'pkey', '-in', $privkey_pem,
+                '-outform', 'DER', '-out', $privkey_der])),
+       "Convert private key to DER");
+    ok(run(app(['openssl', 'pkey', '-in', $pubkey_pem, '-pubin',
+                '-outform', 'DER', '-pubout', '-out', $pubkey_der])),
+       "Convert public key to DER");
+
+    ok(run(app(['openssl', 'dgst', '-sign', $privkey_der, '-keyform', 'DER',
+                '-out', $sigfile,
+                $data_to_sign])),
+       "Generating signature with DER private key via -keyform");
+
+    ok(run(app(['openssl', 'dgst', '-verify', $pubkey_der, '-keyform', 'DER',
+                '-signature', $sigfile,
+                $data_to_sign])),
+       "Verify signature with DER public key via -keyform");
+};
+
 subtest "signing using the nonce-type sigopt" => sub {
     if (disabled("ec")) {
         plan tests => 1;