Fix common test framework options

PR#6975 added the ability to our test framework to have common options to
all tests. For example providing the option "-test 5" to one of our test
programs will just run test number 5. This can be useful when debugging
tests.

Unforuntately this does not work well for a number of tests. In particular
those tests that call test_get_argument() without first skipping over these
common test options will not get the expected value. Some tests did this
correctly but a large number did not.

A helper function is introduced, test_skip_common_options(), to make this
easier for those tests which do not have their own specialised test option
handling, but yet still need to call test_get_argument(). This function
call is then added to all those tests that need it.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/10975)

diff --git a/test/asynciotest.c b/test/asynciotest.c
index bf0a20561e7a5ec3655362db617236bd7c5c0cb9..dcdee1068dc2dbec22cde1b31d1820de8175918f 100644 (file)
--- a/test/asynciotest.c
+++ b/test/asynciotest.c
@@ -397,6 +397,11 @@ OPT_TEST_DECLARE_USAGE("certname privkey\n")
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(cert = test_get_argument(0))
             || !TEST_ptr(privkey = test_get_argument(1)))
         return 0;

diff --git a/test/clienthellotest.c b/test/clienthellotest.c
index b4563b5beb24344bc20942991d326767e38c5275..9a7537444ca1ae687b9edf81db9abd3e8566a600 100644 (file)
--- a/test/clienthellotest.c
+++ b/test/clienthellotest.c
@@ -250,6 +250,11 @@ OPT_TEST_DECLARE_USAGE("sessionfile\n")
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(sessionfile = test_get_argument(0)))
         return 0;
 

diff --git a/test/cmp_msg_test.c b/test/cmp_msg_test.c
index 463c60789ba9d7a66c43ef6b95f2dd3cf7cdc48c..7fa0619284f9383d960d1efe12416c08ba97f548 100644 (file)
--- a/test/cmp_msg_test.c
+++ b/test/cmp_msg_test.c
@@ -538,6 +538,11 @@ void cleanup_tests(void)
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(server_cert_f = test_get_argument(0))
             || !TEST_ptr(pkcs10_f = test_get_argument(1))) {
         TEST_error("usage: cmp_msg_test server.crt pkcs10.der\n");

diff --git a/test/cmp_protect_test.c b/test/cmp_protect_test.c
index b349cac2d703bdbc8c019e3accaeb049c553c2c2..8425849835f6d3bf4dce976d20a5d901f62dd972 100644 (file)
--- a/test/cmp_protect_test.c
+++ b/test/cmp_protect_test.c
@@ -458,6 +458,11 @@ int setup_tests(void)
     char *root_f;
     char *intermediate_f;
 
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     RAND_bytes(rand_data, OSSL_CMP_TRANSACTIONID_LENGTH);
     if (!TEST_ptr(server_f = test_get_argument(0))
             || !TEST_ptr(ir_protected_f = test_get_argument(1))

diff --git a/test/cmsapitest.c b/test/cmsapitest.c
index 2ea8af58b3e4b96d06fbc403257c71804f6e5a27..3ab1b82f963d01b362e6216b30ab2b06c4e7beee 100644 (file)
--- a/test/cmsapitest.c
+++ b/test/cmsapitest.c
@@ -65,6 +65,11 @@ int setup_tests(void)
     char *certin = NULL, *privkeyin = NULL;
     BIO *certbio = NULL, *privkeybio = NULL;
 
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(certin = test_get_argument(0))
             || !TEST_ptr(privkeyin = test_get_argument(1)))
         return 0;

diff --git a/test/d2i_test.c b/test/d2i_test.c
index 3ce38485bd22137231c89ac57ee0eed2def5cdf6..84ba30fbcc626b220172d62c913453134c59f053 100644 (file)
--- a/test/d2i_test.c
+++ b/test/d2i_test.c
@@ -127,6 +127,11 @@ int setup_tests(void)
         {"compare", ASN1_COMPARE}
     };
 
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(test_type_name = test_get_argument(0))
             || !TEST_ptr(expected_error_string = test_get_argument(1))
             || !TEST_ptr(test_file = test_get_argument(2)))

diff --git a/test/danetest.c b/test/danetest.c
index 2c228ecc52b53208847480580c03ab823e86dd47..9b61699999b1fb0d538902481cb763b4b6eff1c1 100644 (file)
--- a/test/danetest.c
+++ b/test/danetest.c
@@ -413,6 +413,11 @@ OPT_TEST_DECLARE_USAGE("basedomain CAfile tlsafile\n")
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(basedomain = test_get_argument(0))
             || !TEST_ptr(CAfile = test_get_argument(1))
             || !TEST_ptr(tlsafile = test_get_argument(2)))

diff --git a/test/dtlstest.c b/test/dtlstest.c
index 022b4c014966592cbadbdc02702cc022abc7860f..607768832b19c2fa7369b7649a33a3cb7a4bceae 100644 (file)
--- a/test/dtlstest.c
+++ b/test/dtlstest.c
@@ -332,6 +332,11 @@ OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(cert = test_get_argument(0))
             || !TEST_ptr(privkey = test_get_argument(1)))
         return 0;

diff --git a/test/evp_test.c b/test/evp_test.c
index a6335ac93ecc48c7ad1dc22a7a9839fba8f9e7fd..87e901fdb43cf95e580bbb2c4b730a252db88869 100644 (file)
--- a/test/evp_test.c
+++ b/test/evp_test.c
@@ -3247,8 +3247,14 @@ OPT_TEST_DECLARE_USAGE("file...\n")
 
 int setup_tests(void)
 {
-    size_t n = test_get_argument_count();
+    size_t n;
 
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
+    n = test_get_argument_count();
     if (n == 0)
         return 0;
 

diff --git a/test/fatalerrtest.c b/test/fatalerrtest.c
index 0f18c1b67bc67c2adb5ccaeb2c9da4fc1d55d5d1..184392cff260cb32eeb569407f8a479c9a7d3b49 100644 (file)
--- a/test/fatalerrtest.c
+++ b/test/fatalerrtest.c
@@ -86,6 +86,11 @@ OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(cert = test_get_argument(0))
             || !TEST_ptr(privkey = test_get_argument(1)))
         return 0;

diff --git a/test/gosttest.c b/test/gosttest.c
index a03521d8a339ab7c65ea5690c18bdc05c9e0a6be..7737a68f5f9108386428442e3a5ddf8a519a1ba2 100644 (file)
--- a/test/gosttest.c
+++ b/test/gosttest.c
@@ -82,6 +82,11 @@ OPT_TEST_DECLARE_USAGE("certfile1 privkeyfile1 certfile2 privkeyfile2\n")
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(cert1 = test_get_argument(0))
             || !TEST_ptr(privkey1 = test_get_argument(1))
             || !TEST_ptr(cert2 = test_get_argument(2))

diff --git a/test/ocspapitest.c b/test/ocspapitest.c
index 355bd448aebfa0563f7800296c5b2ca8e21bf797..358eb54fadfd03e204fbea73d569c44d58b3d7e4 100644 (file)
--- a/test/ocspapitest.c
+++ b/test/ocspapitest.c
@@ -215,6 +215,11 @@ OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(certstr = test_get_argument(0))
         || !TEST_ptr(privkeystr = test_get_argument(1)))
         return 0;

diff --git a/test/params_conversion_test.c b/test/params_conversion_test.c
index d6490833a9775257b4c8f5ba0fe40cc9e335e9ce..838eefa2123cc97326e91d76bb37485584287f37 100644 (file)
--- a/test/params_conversion_test.c
+++ b/test/params_conversion_test.c
@@ -328,8 +328,14 @@ OPT_TEST_DECLARE_USAGE("file...\n")
 
 int setup_tests(void)
 {
-    size_t n = test_get_argument_count();
+    size_t n;
 
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
+    n = test_get_argument_count();
     if (n == 0)
         return 0;
 

diff --git a/test/recordlentest.c b/test/recordlentest.c
index 9be354bea81fbb8dd46ad0fc45e84cb25eba6cf3..01cc53469b8099ea4206412b6034e0d78ee3213f 100644 (file)
--- a/test/recordlentest.c
+++ b/test/recordlentest.c
@@ -185,6 +185,11 @@ OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(cert = test_get_argument(0))
             || !TEST_ptr(privkey = test_get_argument(1)))
         return 0;

diff --git a/test/servername_test.c b/test/servername_test.c
index 3d19265db93b1f98f9e33e16442fed2fabc21f04..33e62a169bf2eec3ac1c3418c93d4d6cd44fdda5 100644 (file)
--- a/test/servername_test.c
+++ b/test/servername_test.c
@@ -239,6 +239,11 @@ static int test_servername(int test)
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(cert = test_get_argument(0))
             || !TEST_ptr(privkey = test_get_argument(1)))
         return 0;

diff --git a/test/ssl_test.c b/test/ssl_test.c
index e54e84182793fc25475a02ae23bc44cb6ca89ef0..e6ce9a674151cff1502d9985a457035c73f354bb 100644 (file)
--- a/test/ssl_test.c
+++ b/test/ssl_test.c
@@ -506,6 +506,11 @@ int setup_tests(void)
 {
     long num_tests;
 
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(conf = NCONF_new(NULL))
             /* argv[1] should point to the test conf file */
             || !TEST_int_gt(NCONF_load(conf, test_get_argument(0), NULL), 0)

diff --git a/test/ssl_test_ctx_test.c b/test/ssl_test_ctx_test.c
index 5f54d1ef24800bbc1f39d2b9476edc413dfd9d6a..fc7ec68f29d16d2308a192ca5e161a85904c4e30 100644 (file)
--- a/test/ssl_test_ctx_test.c
+++ b/test/ssl_test_ctx_test.c
@@ -244,6 +244,11 @@ OPT_TEST_DECLARE_USAGE("conf_file\n")
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(conf = NCONF_new(NULL)))
         return 0;
     /* argument should point to test/ssl_test_ctx_test.conf */

diff --git a/test/sslapitest.c b/test/sslapitest.c
index 2e4d07eecfdb72601df2f1ee5c139d0967f934ba..0f90db92ba1b0ffed28e8d80d5e1936cf8e92984 100644 (file)
--- a/test/sslapitest.c
+++ b/test/sslapitest.c
@@ -7127,6 +7127,11 @@ OPT_TEST_DECLARE_USAGE("certfile privkeyfile srpvfile tmpfile\n")
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(certsdir = test_get_argument(0))
             || !TEST_ptr(srpvfile = test_get_argument(1))
             || !TEST_ptr(tmpfilename = test_get_argument(2)))

diff --git a/test/sslbuffertest.c b/test/sslbuffertest.c
index 163beafb2f81b3294f7d2e28225ecd8bd0516e06..3bce667b462be001af4627361b95b480a9eaffb9 100644 (file)
--- a/test/sslbuffertest.c
+++ b/test/sslbuffertest.c
@@ -156,6 +156,11 @@ int setup_tests(void)
 {
     char *cert, *pkey;
 
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(cert = test_get_argument(0))
             || !TEST_ptr(pkey = test_get_argument(1)))
         return 0;

diff --git a/test/sslcorrupttest.c b/test/sslcorrupttest.c
index 66f8cd142c2a7334cc8ad58624dc071e6036bdd3..d201cd2b27220255ba814ceacd0868abc4acbfb8 100644 (file)
--- a/test/sslcorrupttest.c
+++ b/test/sslcorrupttest.c
@@ -250,6 +250,11 @@ int setup_tests(void)
 {
     int n;
 
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(cert = test_get_argument(0))
             || !TEST_ptr(privkey = test_get_argument(1)))
         return 0;

diff --git a/test/testutil.h b/test/testutil.h
index 57ab15356c1ce22ea4959a59f06c3e2206f42a2f..2a2857a741d4c87e21a9e3c3047115bfdbad83d6 100644 (file)
--- a/test/testutil.h
+++ b/test/testutil.h
@@ -193,6 +193,12 @@ char *test_get_argument(size_t n);
 /* Return the number of additional non optional command line arguments */
 size_t test_get_argument_count(void);
 
+/*
+ * Skip over common test options. Should be called before calling
+ * test_get_argument()
+ */
+int test_skip_common_options(void);
+
 /*
  * Internal helpers. Test programs shouldn't use these directly, but should
  * rather link to one of the helper main() methods.

diff --git a/test/testutil/options.c b/test/testutil/options.c
index 9a32d1fb9441f9fd0c408c0bb37db38e593f02d7..b5c0739db72a4862c52489dad4052bf4a60d19ab 100644 (file)
--- a/test/testutil/options.c
+++ b/test/testutil/options.c
@@ -15,6 +15,21 @@
 
 static int used[100] = { 0 };
 
+int test_skip_common_options(void)
+{
+    OPTION_CHOICE_DEFAULT o;
+
+    while ((o = (OPTION_CHOICE_DEFAULT)opt_next()) != OPT_EOF) {
+        switch (o) {
+        case OPT_TEST_CASES:
+           break;
+        default:
+        case OPT_ERR:
+            return 0;
+        }
+    }
+    return 1;
+}
 
 size_t test_get_argument_count(void)
 {

diff --git a/test/tls13ccstest.c b/test/tls13ccstest.c
index 999ca5700a25d3a18232ae2a08b55a07fa4d2d9c..2820a0e8fa387bcd026650ca633eb124318455d7 100644 (file)
--- a/test/tls13ccstest.c
+++ b/test/tls13ccstest.c
@@ -492,6 +492,11 @@ OPT_TEST_DECLARE_USAGE("certfile privkeyfile\n")
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(cert = test_get_argument(0))
             || !TEST_ptr(privkey = test_get_argument(1)))
         return 0;

diff --git a/test/v3ext.c b/test/v3ext.c
index 2c8ac6bb20d9ec15d67accf2382518c93e8ff41b..5cf7581c51691fc379ccd173a1d8bcaf1fbbb527 100644 (file)
--- a/test/v3ext.c
+++ b/test/v3ext.c
@@ -41,6 +41,11 @@ OPT_TEST_DECLARE_USAGE("cert.pem\n")
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(infile = test_get_argument(0)))
         return 0;
 

diff --git a/test/verify_extra_test.c b/test/verify_extra_test.c
index d1da7303b72cef5437ed7ebf7e4e431b2f0519cd..91ed31b374809b0fcc1c5dee06d3aad98a62e38d 100644 (file)
--- a/test/verify_extra_test.c
+++ b/test/verify_extra_test.c
@@ -263,6 +263,11 @@ static int test_req_sm2_id(void)
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(roots_f = test_get_argument(0))
             || !TEST_ptr(untrusted_f = test_get_argument(1))
             || !TEST_ptr(bad_f = test_get_argument(2))

diff --git a/test/x509_check_cert_pkey_test.c b/test/x509_check_cert_pkey_test.c
index 6c1587b847a3b44516741659afd747e0f14ad21d..b1b6b9c9bfd4c303cbe8ba9b543dc34203fea9bf 100644 (file)
--- a/test/x509_check_cert_pkey_test.c
+++ b/test/x509_check_cert_pkey_test.c
@@ -122,6 +122,11 @@ const OPTIONS *test_get_options(void)
 
 int setup_tests(void)
 {
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
     if (!TEST_ptr(c = test_get_argument(0))
             || !TEST_ptr(k = test_get_argument(1))
             || !TEST_ptr(t = test_get_argument(2))

diff --git a/test/x509_dup_cert_test.c b/test/x509_dup_cert_test.c
index ebea488b10ea260fe2cc6d78a2c1741770803386..b0bf1b9598c0f904678c8eac68d95d82c4997851 100644 (file)
--- a/test/x509_dup_cert_test.c
+++ b/test/x509_dup_cert_test.c
@@ -37,8 +37,14 @@ OPT_TEST_DECLARE_USAGE("cert.pem...\n")
 
 int setup_tests(void)
 {
-    size_t n = test_get_argument_count();
+    size_t n;
 
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
+    n = test_get_argument_count();
     if (!TEST_int_gt(n, 0))
         return 0;
 

diff --git a/test/x509aux.c b/test/x509aux.c
index 1489428d5256ffb9a2fe0987fd6a001b2617dade..49b6745d5525191637b161e849af2ef0333a59b8 100644 (file)
--- a/test/x509aux.c
+++ b/test/x509aux.c
@@ -165,7 +165,14 @@ OPT_TEST_DECLARE_USAGE("certfile...\n")
 
 int setup_tests(void)
 {
-    size_t n = test_get_argument_count();
+    size_t n;
+
+    if (!test_skip_common_options()) {
+        TEST_error("Error parsing test options\n");
+        return 0;
+    }
+
+    n = test_get_argument_count();
     if (n == 0)
         return 0;