test/run_tests.pl: Enhance the semantics of HARNESS_VERBOSE_FAILURES (VF)

Make the improved semantics of VFO replace the previous VF and remove VFO
Add warnings about overriding use of HARNESS_VERBOSE* variables

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/12279)

diff --git a/INSTALL.md b/INSTALL.md
index 3b993585d23410a908d8c3d4fc8d087d7bcded3c..6989410e87c0ffb90f911c43c8d55ba8103b02f3 100644 (file)
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -1603,9 +1603,9 @@ that isn't a problem in OpenSSL itself (like an OS malfunction or a Perl issue).
 You may want increased verbosity, that can be accomplished as described in
 section [Test Failures of test/README.md](test/README.md#test-failures).
 
-You may want to selectively specify which test(s) to perform. This can be done
-sing the `make` variable `TESTS` as described in section [Running Selected Tests
-of test/README.md](test/README.md#running-selected-tests).
+You may also want to selectively specify which test(s) to perform. This can be
+done using the `make` variable `TESTS` as described in section [Running
+Selected Tests of test/README.md](test/README.md#running-selected-tests).
 
 If you find a problem with OpenSSL itself, try removing any
 compiler optimization flags from the `CFLAGS` line in the Makefile and

diff --git a/test/README.md b/test/README.md
index 76bf1b17aa508d4cef3fd9d227a72c4d6b228251..d5591dc00fc8f09cb90667b8fab23ea6cb3488fb 100644 (file)
--- a/test/README.md
+++ b/test/README.md
@@ -27,21 +27,17 @@ Full verbosity, showing full output of all successful and failed test cases
     $ mms /macro=(V=1) test                          ! OpenVMS
     $ nmake V=1 test                                 # Windows
 
-Verbosity on test failure (`VERBOSE_FAILURE` or `VF`, Unix example shown):
+Verbosity on failed (sub-)tests only (`VERBOSE_FAILURE` or `VF`):
 
     $ make test VF=1
 
-Verbosity on failed (sub-)tests only (`VERBOSE_FAILURES_ONLY` or `VFO`):
-
-    $ make test VFO=1
-
 Verbosity on failed (sub-)tests, in addition progress on succeeded (sub-)tests
-(`VERBOSE_FAILURES_PROGRESS` or `VFP`):
+(`VERBOSE_FAILURE_PROGRESS` or `VFP`):
 
     $ make test VFP=1
 
 If you want to run just one or a few specific tests, you can use
-the `make` variable `TESTS` to specify them, like this:
+the make variable TESTS to specify them, like this:
 
     $ make TESTS='test_rsa test_dsa' test            # Unix
     $ mms/macro="TESTS=test_rsa test_dsa" test       ! OpenVMS
@@ -50,7 +46,7 @@ the `make` variable `TESTS` to specify them, like this:
 And of course, you can combine (Unix examples shown):
 
     $ make test TESTS='test_rsa test_dsa' VF=1
-    $ make test TESTS="test_cmp_*" VFO=1
+    $ make test TESTS="test_cmp_*" VFP=1
 
 You can find the list of available tests like this:
 

diff --git a/test/run_tests.pl b/test/run_tests.pl
index 5eddaf846884032f1b521566bc52cf65943e9d1a..c3d0b45d738d2a99a8aca397956d0c88810f8beb 100644 (file)
--- a/test/run_tests.pl
+++ b/test/run_tests.pl
@@ -9,15 +9,14 @@
 use strict;
 use warnings;
 
-# Recognise VERBOSE and V which is common on other projects.
-# Additionally, also recognise VERBOSE_FAILURE and VF.
+# Recognise VERBOSE aka V which is common on other projects.
+# Additionally, recognise VERBOSE_FAILURE aka VF.
+# and recognise VERBOSE_FAILURE_PROGRESS aka VFP.
 BEGIN {
     $ENV{HARNESS_VERBOSE} = "yes" if $ENV{VERBOSE} || $ENV{V};
     $ENV{HARNESS_VERBOSE_FAILURE} = "yes" if $ENV{VERBOSE_FAILURE} || $ENV{VF};
-    $ENV{HARNESS_VERBOSE_FAILURES_ONLY} = "yes"
-        if $ENV{VERBOSE_FAILURES_ONLY} || $ENV{VFO};
-    $ENV{HARNESS_VERBOSE_FAILURES_PROGRESS} = "yes"
-        if $ENV{VERBOSE_FAILURES_PROGRESS} || $ENV{VFP};
+    $ENV{HARNESS_VERBOSE_FAILURE_PROGRESS} = "yes"
+        if $ENV{VERBOSE_FAILURE_PROGRESS} || $ENV{VFP};
 }
 
 use File::Spec::Functions qw/catdir catfile curdir abs2rel rel2abs/;
@@ -53,9 +52,13 @@ $tapargs{jobs} = $jobs if defined $jobs;
 my %openssl_args = ();
 
 $openssl_args{'failure_verbosity'} = $ENV{HARNESS_VERBOSE} ? 0 :
-    $ENV{HARNESS_VERBOSE_FAILURE} ? 3 :
-    $ENV{HARNESS_VERBOSE_FAILURES_PROGRESS} ? 2 :
-    $ENV{HARNESS_VERBOSE_FAILURES_ONLY} ? 1 : 0;
+    $ENV{HARNESS_VERBOSE_FAILURE_PROGRESS} ? 2 :
+    1; # $ENV{HARNESS_VERBOSE_FAILURE}
+print "Warning: HARNESS_VERBOSE overrides HARNESS_VERBOSE_FAILURE*\n"
+    if ($ENV{HARNESS_VERBOSE} && ($ENV{HARNESS_VERBOSE_FAILURE}
+                                  || $ENV{HARNESS_VERBOSE_FAILURE_PROGRESS}));
+print "Warning: HARNESS_VERBOSE_FAILURE_PROGRESS overrides HARNESS_VERBOSE_FAILURE\n"
+    if ($ENV{HARNESS_VERBOSE_FAILURE_PROGRESS} && $ENV{HARNESS_VERBOSE_FAILURE});
 
 my $outfilename = $ENV{HARNESS_TAP_COPY};
 open $openssl_args{'tap_copy'}, ">$outfilename"
@@ -153,9 +156,7 @@ $eres = eval {
                     if defined $fh;
 
                 my $failure_verbosity = $openssl_args{failure_verbosity};
-                if ($failure_verbosity == 3) {
-                    push @failure_output, $self->as_string;
-                } elsif ($failure_verbosity > 0) {
+                if ($failure_verbosity > 0) {
                     my $is_plan = $self->is_plan;
                     my $tests_planned = $is_plan && $self->tests_planned;
                     my $is_test = $self->is_test;
@@ -201,7 +202,7 @@ $eres = eval {
                     print $_, "\n" foreach (("", @failure_output));
                 }
                 # Echo any trailing comments etc.
-                print "$output_buffer" if $failure_verbosity != 3;
+                print "$output_buffer";
             };
         }