add a memfail test for x509 operations

Much like our handshake test, x509 has several operations that can be
tested easily in such a way that we ensure memory failures don't cause
cascading asan failures, and increase our test coverage.

Add a test to exercise some X509 apis to do so.

Reviewed-by: Saša Nedvědický <sashan@openssl.org>
Reviewed-by: Norbert Pocs <norbertp@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/28736)

diff --git a/test/build.info b/test/build.info
index 57ee94071a82209d16fd04e71385f7c7724d421b..0fcb2c80fdd2aa446398a645c8fd402c06e7aeac 100644 (file)
--- a/test/build.info
+++ b/test/build.info
@@ -77,7 +77,7 @@ IF[{- !$disabled{tests} -}]
   ENDIF
 
   IF[{- !$disabled{'allocfail-tests'} -}]
-    PROGRAMS{noninst}=handshake-memfail
+    PROGRAMS{noninst}=handshake-memfail x509-memfail
   ENDIF
 
   IF[{- !$disabled{'deprecated-3.0'} -}]
@@ -595,6 +595,10 @@ IF[{- !$disabled{tests} -}]
   INCLUDE[handshake-memfail]=../include ../apps/include
   DEPEND[handshake-memfail]=../libcrypto.a ../libssl.a libtestutil.a
 
+  SOURCE[x509-memfail]=x509_memfail.c
+  INCLUDE[x509-memfail]=../include ../apps/include
+  DEPEND[x509-memfail]=../libcrypto.a libtestutil.a
+
   SOURCE[ssl_handshake_rtt_test]=ssl_handshake_rtt_test.c helpers/ssltestlib.c
   INCLUDE[ssl_handshake_rtt_test]=../include ../apps/include ..
   DEPEND[ssl_handshake_rtt_test]=../libcrypto.a ../libssl.a libtestutil.a

diff --git a/test/recipes/90-test_x509-memfail.t b/test/recipes/90-test_x509-memfail.t
new file mode 100644 (file)
index 0000000..cce74c7
--- /dev/null
+++ b/test/recipes/90-test_x509-memfail.t
@@ -0,0 +1,74 @@
+#! /usr/bin/env perl
+# Copyright 2025 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use OpenSSL::Test qw/:DEFAULT srctop_file result_dir/;
+use OpenSSL::Test::Utils;
+use File::Temp qw(tempfile);
+use File::Path 2.00 qw(rmtree mkpath);
+
+setup("test_x509_memfail");
+
+#
+# Don't run this test if mdebug isn't enabled, it won't work
+#
+plan skip_all => "$test_name requires allocfail-tests to be enabled"
+    if disabled("allocfail-tests");
+
+#
+# We need to know how many mallocs we plan to fail, so run the test in count mode
+# To tell us how many mallocs it executes
+# We capture the result of the test into countinfo.txt
+# and parse that to figure out what our values are
+#
+my $resultdir = result_dir();
+run(test(["x509-memfail", "count", srctop_file("test", "certs", "servercert.pem")], stderr => "$resultdir/countinfo.txt"));
+
+#
+# Read the result file into an array
+#
+open my $handle, '<', "$resultdir/countinfo.txt";
+chomp(my @lines = <$handle>);
+close $handle;
+
+#
+# some line contains our counts, find and split that into an array
+#
+my @vals;
+foreach(@lines) {
+    if ($_ =~/skip:/) {
+        @vals = split ' ', $_;
+        break;
+    }
+}
+
+#
+# The number of mallocs we need to skip is in entry two
+# The number of mallocs to test is in entry 4
+#
+my $skipcount = $vals[2];
+my $malloccount = $vals[4];
+
+#
+# Now we can plan our tests.  We plan to run malloccount iterations of this
+# test
+#
+plan tests => $malloccount;
+
+my @seq = (1..$malloccount);
+for my $idx (@seq) {
+    #
+    # We need to setup our openssl malloc failures env var to fail the target malloc
+    # the format of this string is a series of A@B;C@D tuples where A,C are the number
+    # of mallocs to consider, and B,D are the likelyhood that they should fail.
+    # We always skip the first "skip" allocations, then iteratively guarantee that 
+    # next <idx> mallocs pass, followed by the next single malloc failing, with the remainder
+    # passing
+    #
+    $ENV{OPENSSL_MALLOC_FAILURES} = "$skipcount\@0;$idx\@0;1\@100;0\@0"; 
+    ok(run(test(["x509-memfail", "run", srctop_file("test", "certs", "servercert.pem")])));
+}

diff --git a/test/x509_memfail.c b/test/x509_memfail.c
new file mode 100644 (file)
index 0000000..d906f09
--- /dev/null
+++ b/test/x509_memfail.c
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2025 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+#define OPENSSL_SUPPRESS_DEPRECATED /* EVP_PKEY_get1/set1_RSA */
+
+#include <openssl/x509.h>
+#include <openssl/asn1.h>
+#include <openssl/evp.h>
+#include <openssl/rsa.h>
+#include <openssl/pem.h>
+#include "crypto/x509.h" /* x509_st definition */
+#include "testutil.h"
+
+static char *certfile = NULL;
+static int mcount, rcount, fcount, scount;
+
+static int do_x509(int allow_failure)
+{
+    int ret = (allow_failure == 1) ? 0 : 1;
+    BIO *bio = NULL;
+    X509 *x509 = NULL;
+    const ASN1_BIT_STRING *sig = NULL;
+    const X509_ALGOR *alg = NULL;
+    EVP_PKEY *pkey;
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+    RSA *rsa = NULL;
+#endif
+
+    if (!TEST_ptr(bio = BIO_new_file(certfile, "r"))
+        || !TEST_ptr(x509 = PEM_read_bio_X509(bio, NULL, NULL, NULL))
+        || !TEST_ptr(pkey = X509_get0_pubkey(x509)))
+        goto err;
+
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+    /* Issue #24575 requires legacy key but the test is useful anyway */
+    if (!TEST_ptr(rsa = EVP_PKEY_get1_RSA(pkey)))
+        goto err;
+
+    if (!TEST_int_gt(EVP_PKEY_set1_RSA(pkey, rsa), 0))
+        goto err;
+#endif
+
+    X509_get0_signature(&sig, &alg, x509);
+
+    if (!TEST_int_gt(ASN1_item_verify(ASN1_ITEM_rptr(X509_CINF),
+                                      (X509_ALGOR *)alg, (ASN1_BIT_STRING *)sig,
+                                      &x509->cert_info, pkey), 0))
+        goto err;
+
+    if (!TEST_int_lt(ASN1_item_verify(ASN1_ITEM_rptr(X509_CINF),
+                                     (X509_ALGOR *)alg, (ASN1_BIT_STRING *)sig,
+                                     NULL, pkey), 0))
+        goto err;
+
+    X509_issuer_name_hash(x509);
+
+    ret = 1;
+
+ err:
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+    RSA_free(rsa);
+#endif
+    X509_free(x509);
+    BIO_free(bio);
+    return ret;
+}
+
+static int test_record_alloc_counts(void)
+{
+    return do_x509(1);
+}
+
+static int test_alloc_failures(void)
+{
+    return do_x509(0);
+}
+
+static int test_report_alloc_counts(void)
+{
+    CRYPTO_get_alloc_counts(&mcount, &rcount, &fcount);
+    /*
+     * Report our memory allocations from the count run
+     * NOTE: We report a number of allocations to skip here
+     * (the scount value).  These are the allocations that took
+     * place while the test harness itself was getting setup
+     * (i.e. calling OPENSSL_init_crypto/etc).  We can't fail
+     * those allocations as they will cause the test to fail before
+     * we have even run the workload.  So report them so we can
+     * allow them to function before we start doing any real testing
+     */
+    TEST_info("skip: %d count %d\n", scount, mcount - scount);
+    return 1;
+}
+
+int setup_tests(void)
+{
+    int ret = 0;
+    char *opmode = NULL;
+
+    if (!TEST_ptr(opmode = test_get_argument(0)))
+        goto err;
+
+    if (!TEST_ptr(certfile = test_get_argument(1)))
+        goto err;
+
+    if (strcmp(opmode, "count") == 0) {
+        CRYPTO_get_alloc_counts(&scount, &rcount, &fcount);
+        ADD_TEST(test_record_alloc_counts);
+        ADD_TEST(test_report_alloc_counts);
+    } else {
+        ADD_TEST(test_alloc_failures);
+    }
+    ret = 1;
+err:
+    return ret;
+}
+
+void cleanup_tests(void)
+{
+}