99-test_fuzz.t: Clean up and re-organize such that sub-tests could be split easily

Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12359)

diff --git a/fuzz/README.md b/fuzz/README.md
index a713f853254175078b19bc824242a022af2227f3..deb7a431684752fc8d19eb60ccca242051b48e89 100644 (file)
--- a/fuzz/README.md
+++ b/fuzz/README.md
@@ -99,7 +99,7 @@ Reproducing issues
 
 If a fuzzer generates a reproducible error, you can reproduce the problem using
 the fuzz/*-test binaries and the file generated by the fuzzer. They binaries
-don't need to be build for fuzzing, there is no need to set CC or the call
+don't need to be built for fuzzing, there is no need to set CC or the call
 config with enable-fuzz-* or -fsanitize-coverage, but some of the other options
 above might be needed. For instance the enable-asan or enable-ubsan option might
 be useful to show you when the problem happens. For the client and server fuzzer
@@ -110,6 +110,20 @@ To reproduce the crash you can run:
 
     fuzz/$FUZZER-test $file
 
+To do all the tests of a specific fuzzer such as asn1 you can run
+
+    fuzz/asn1-test fuzz/corpora/asn1
+or
+    make test TESTS=fuzz_test FUZZ_TESTS=asn1
+
+To run several fuzz tests you can use for instance:
+
+    make test TESTS=test_fuzz FUZZ_TESTS="cmp cms"
+
+To run all fuzz tests you can use:
+
+    make test TESTS=test_fuzz
+
 Random numbers
 --------------
 

diff --git a/test/README.md b/test/README.md
index f9058a002620afd9fff26258c1d09d88f4ec141a..f4f0574aef797626d00b725a69a22089d426d8d3 100644 (file)
--- a/test/README.md
+++ b/test/README.md
@@ -121,6 +121,10 @@ Run all tests in test groups 80 to 99 except for tests in group 90:
 
     $ make TESTS='[89]? -90' test
 
+To run specific fuzz tests you can use for instance:
+
+    $ make test TESTS=test_fuzz FUZZ_TESTS="cmp cms"
+
 To stochastically verify that the algorithm that produces uniformly distributed
 random numbers is operating correctly (with a false positive rate of 0.01%):
 

diff --git a/test/recipes/99-test_fuzz.t b/test/recipes/99-test_fuzz.t
index c9e2c961e46286327ea70fba9f61f8e714332496..8bacad47de48779a3c748bb61fc78b82b015d2ec 100644 (file)
--- a/test/recipes/99-test_fuzz.t
+++ b/test/recipes/99-test_fuzz.t
@@ -9,35 +9,30 @@
 use strict;
 use warnings;
 
-use OpenSSL::Glob;
 use OpenSSL::Test qw/:DEFAULT srctop_file/;
 use OpenSSL::Test::Utils;
 
 setup("test_fuzz");
 
-my @fuzzers = ('asn1', 'asn1parse', 'bignum', 'bndiv', 'client', 'conf', 'crl', 'server', 'x509');
-if (!disabled("cmp")) {
-    push @fuzzers, 'cmp';
+my @fuzzers = ();
+@fuzzers = split /\s+/, $ENV{FUZZ_TESTS} if $ENV{FUZZ_TESTS};
+
+if (!@fuzzers) {
+    @fuzzers = (
+        # those commented here as very slow could be moved to separate runs
+        'asn1', # very slow
+        'asn1parse', 'bignum', 'bndiv', 'conf','crl',
+        'client', # very slow
+        'server', # very slow
+        'x509'
+        );
+    push @fuzzers, 'cmp' if !disabled("cmp");
+    push @fuzzers, 'cms' if !disabled("cms");
+    push @fuzzers, 'ct' if !disabled("ct");
 }
-if (!disabled("cms")) {
-    push @fuzzers, 'cms';
-}
-if (!disabled("ct")) {
-    push @fuzzers, 'ct';
-}
-plan tests => scalar @fuzzers;
 
-foreach my $f (@fuzzers) {
-    subtest "Fuzzing $f" => sub {
-        my @dirs = glob(srctop_file('fuzz', 'corpora', $f));
-        push @dirs, glob(srctop_file('fuzz', 'corpora', "$f-*"));
+plan tests => scalar @fuzzers + 1; # one more due to below require_ok(...)
 
-        plan skip_all => "No corpora for $f-test" unless @dirs;
+require_ok(srctop_file('test','recipes','fuzz.pl'));
 
-        plan tests => scalar @dirs;
-
-        foreach (@dirs) {
-            ok(run(fuzz(["$f-test", $_])));
-        }
-    }
-}
+&fuzz_tests(@fuzzers);

diff --git a/test/recipes/fuzz.pl b/test/recipes/fuzz.pl
new file mode 100644 (file)
index 0000000..795d85c
--- /dev/null
+++ b/test/recipes/fuzz.pl
@@ -0,0 +1,31 @@
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Glob;
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+
+sub fuzz_tests {
+    my @fuzzers = @_;
+
+    foreach my $f (@fuzzers) {
+        subtest "Fuzzing $f" => sub {
+            my @dir = glob(srctop_file('fuzz', 'corpora', "$f"));
+
+            plan skip_all => "No directory fuzz/corpora/$f" unless @dir;
+            plan tests => scalar @dir; # likely 1
+
+            foreach (@dir) {
+                ok(run(fuzz(["$f-test", $_])));
+            }
+        }
+    }
+}
+
+1;