upstream: add some knobs:

UNITTEST_FAST?= no     # Skip slow tests (e.g. less intensive fuzzing).
UNITTEST_SLOW?= no     # Include slower tests (e.g. more intensive fuzzing).
UNITTEST_VERBOSE?= no  # Verbose test output (inc. per-test names).

useful if you want to run the tests as a smoke test to exercise the
functionality without waiting for all the fuzzers to run.

OpenBSD-Regress-ID: e04d82ebec86068198cd903acf1c67563c57315e

diff --git a/regress/unittests/Makefile.inc b/regress/unittests/Makefile.inc
index d662a46bc0d50d1fbd25f9ee1a0e791a2dc3ead0..428ef683698190473eab65ee921e015130334ff2 100644 (file)
--- a/regress/unittests/Makefile.inc
+++ b/regress/unittests/Makefile.inc
@@ -1,10 +1,15 @@
-#      $OpenBSD: Makefile.inc,v 1.12 2017/12/21 00:41:22 djm Exp $
+#      $OpenBSD: Makefile.inc,v 1.13 2018/10/17 23:28:05 djm Exp $
 
 REGRESS_FAIL_EARLY?=   yes
 
 .include <bsd.own.mk>
 .include <bsd.obj.mk>
 
+# User-settable options
+UNITTEST_FAST?= no     # Skip slow tests (e.g. less intensive fuzzing).
+UNITTEST_SLOW?= no     # Include slower tests (e.g. more intensive fuzzing).
+UNITTEST_VERBOSE?= no  # Verbose test output (inc. per-test names).
+
 MALLOC_OPTIONS?=       CFGJRSUX
 TEST_ENV?=             MALLOC_OPTIONS=${MALLOC_OPTIONS}
 
@@ -68,3 +73,14 @@ DPADD+= ${LIBUTIL}
 LDADD+= -lcrypto
 DPADD+= ${LIBCRYPTO}
 .endif
+
+UNITTEST_ARGS?=
+
+.if (${UNITTEST_VERBOSE:L} != "no")
+UNITTEST_ARGS+= -v
+.endif
+.if (${UNITTEST_FAST:L} != "no")
+UNITTEST_ARGS+= -f
+.elif (${UNITTEST_SLOW:L} != "no")
+UNITTEST_ARGS+= -F
+.endif

diff --git a/regress/unittests/sshbuf/Makefile b/regress/unittests/sshbuf/Makefile
index 81d4f27a6132646301a79b21c3619ccc0963c965..0e8e9fd1010e8d39ffccd62d2f573f114654767e 100644 (file)
--- a/regress/unittests/sshbuf/Makefile
+++ b/regress/unittests/sshbuf/Makefile
@@ -1,4 +1,4 @@
-#      $OpenBSD: Makefile,v 1.6 2017/12/21 00:41:22 djm Exp $
+#      $OpenBSD: Makefile,v 1.7 2018/10/17 23:28:05 djm Exp $
 
 .include <bsd.regress.mk>
 
@@ -17,6 +17,5 @@ SRCS+=sshbuf-getput-basic.c sshbuf-getput-crypto.c sshbuf-misc.c sshbuf.c
 SRCS+=atomicio.c
 
 run-regress-${PROG}: ${PROG}
-       env ${TEST_ENV} ./${PROG}
- 
+       env ${TEST_ENV} ./${PROG} ${UNITTEST_ARGS}
 

diff --git a/regress/unittests/sshbuf/test_sshbuf_fuzz.c b/regress/unittests/sshbuf/test_sshbuf_fuzz.c
index c52376b531a36f98cbb7e54bb5930c3da1fee651..e236c82f96fc77c638ed929fde9423914029d7fb 100644 (file)
--- a/regress/unittests/sshbuf/test_sshbuf_fuzz.c
+++ b/regress/unittests/sshbuf/test_sshbuf_fuzz.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: test_sshbuf_fuzz.c,v 1.1 2014/04/30 05:32:00 djm Exp $ */
+/*     $OpenBSD: test_sshbuf_fuzz.c,v 1.2 2018/10/17 23:28:05 djm Exp $ */
 /*
  * Regress test for sshbuf.h buffer API
  *
@@ -30,10 +30,15 @@ sshbuf_fuzz_tests(void)
 {
        struct sshbuf *p1;
        u_char *dp;
-       size_t sz, sz2, i;
+       size_t sz, sz2, i, ntests = NUM_FUZZ_TESTS;
        u_int32_t r;
        int ret;
 
+       if (test_is_fast())
+               ntests >>= 2;
+       if (test_is_slow())
+               ntests <<= 2;
+
        /* NB. uses sshbuf internals */
        TEST_START("fuzz alloc/dealloc");
        p1 = sshbuf_new();

diff --git a/regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c b/regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c
index c6b5c29d176b708fada046429474a6d5a532cb29..7c7cb2bfdf40442d4cce81cff658a5773aabf8de 100644 (file)
--- a/regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c
+++ b/regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: test_sshbuf_getput_fuzz.c,v 1.2 2014/05/02 02:54:00 djm Exp $ */
+/*     $OpenBSD: test_sshbuf_getput_fuzz.c,v 1.3 2018/10/17 23:28:05 djm Exp $ */
 /*
  * Regress test for sshbuf.h buffer API
  *
@@ -115,11 +115,15 @@ sshbuf_getput_fuzz_tests(void)
                0x55, 0x0f, 0x69, 0xd8, 0x0e, 0xc2, 0x3c, 0xd4,
        };
        struct fuzz *fuzz;
+       u_int fuzzers = FUZZ_1_BIT_FLIP | FUZZ_2_BIT_FLIP |
+           FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP |
+           FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END;
+
+       if (test_is_fast())
+               fuzzers &= ~(FUZZ_2_BYTE_FLIP|FUZZ_2_BIT_FLIP);
 
        TEST_START("fuzz blob parsing");
-       fuzz = fuzz_begin(FUZZ_1_BIT_FLIP | FUZZ_2_BIT_FLIP |
-           FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP |
-           FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END, blob, sizeof(blob));
+       fuzz = fuzz_begin(fuzzers, blob, sizeof(blob));
        TEST_ONERROR(onerror, fuzz);
        for(; !fuzz_done(fuzz); fuzz_next(fuzz))
                attempt_parse_blob(blob, sizeof(blob));

diff --git a/regress/unittests/sshkey/Makefile b/regress/unittests/sshkey/Makefile
index 1c940bec640b59f52d0d105ca6b43e6df1669f80..aa731df1c2d47506780756bc8b3de083ab1e1145 100644 (file)
--- a/regress/unittests/sshkey/Makefile
+++ b/regress/unittests/sshkey/Makefile
@@ -1,4 +1,4 @@
-#      $OpenBSD: Makefile,v 1.5 2017/12/21 00:41:22 djm Exp $
+#      $OpenBSD: Makefile,v 1.6 2018/10/17 23:28:05 djm Exp $
 
 PROG=test_sshkey
 SRCS=tests.c test_sshkey.c test_file.c test_fuzz.c common.c
@@ -18,7 +18,7 @@ SRCS+=digest-openssl.c
 REGRESS_TARGETS=run-regress-${PROG}
 
 run-regress-${PROG}: ${PROG}
-       env ${TEST_ENV} ./${PROG} -d ${.CURDIR}/testdata
+       env ${TEST_ENV} ./${PROG} ${UNITTEST_ARGS} -d ${.CURDIR}/testdata
 
 .include <bsd.regress.mk>
 

diff --git a/regress/unittests/sshkey/test_fuzz.c b/regress/unittests/sshkey/test_fuzz.c
index d3b0c92b47a0d222d0808d560b8dfe7f615e4484..5953de5958529bb571279578477892c124a37c57 100644 (file)
--- a/regress/unittests/sshkey/test_fuzz.c
+++ b/regress/unittests/sshkey/test_fuzz.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: test_fuzz.c,v 1.8 2017/12/21 00:41:22 djm Exp $ */
+/*     $OpenBSD: test_fuzz.c,v 1.9 2018/10/17 23:28:05 djm Exp $ */
 /*
  * Fuzz tests for key parsing
  *
@@ -51,14 +51,16 @@ public_fuzz(struct sshkey *k)
        struct sshkey *k1;
        struct sshbuf *buf;
        struct fuzz *fuzz;
+       u_int fuzzers = FUZZ_1_BIT_FLIP | FUZZ_1_BYTE_FLIP |
+           FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END;
 
+       if (test_is_fast())
+               fuzzers &= ~FUZZ_1_BIT_FLIP;
+       if (test_is_slow())
+               fuzzers |= FUZZ_2_BIT_FLIP | FUZZ_2_BYTE_FLIP;
        ASSERT_PTR_NE(buf = sshbuf_new(), NULL);
        ASSERT_INT_EQ(sshkey_putb(k, buf), 0);
-       /* XXX need a way to run the tests in "slow, but complete" mode */
-       fuzz = fuzz_begin(FUZZ_1_BIT_FLIP | /* XXX too slow FUZZ_2_BIT_FLIP | */
-           FUZZ_1_BYTE_FLIP | /* XXX too slow FUZZ_2_BYTE_FLIP | */
-           FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END,
-           sshbuf_mutable_ptr(buf), sshbuf_len(buf));
+       fuzz = fuzz_begin(fuzzers, sshbuf_mutable_ptr(buf), sshbuf_len(buf));
        ASSERT_INT_EQ(sshkey_from_blob(sshbuf_ptr(buf), sshbuf_len(buf),
            &k1), 0);
        sshkey_free(k1);
@@ -77,12 +79,17 @@ sig_fuzz(struct sshkey *k, const char *sig_alg)
        struct fuzz *fuzz;
        u_char *sig, c[] = "some junk to be signed";
        size_t l;
+       u_int fuzzers = FUZZ_1_BIT_FLIP | FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP |
+           FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END;
+
+       if (test_is_fast())
+               fuzzers &= ~FUZZ_2_BYTE_FLIP;
+       if (test_is_slow())
+               fuzzers |= FUZZ_2_BIT_FLIP;
 
        ASSERT_INT_EQ(sshkey_sign(k, &sig, &l, c, sizeof(c), sig_alg, 0), 0);
        ASSERT_SIZE_T_GT(l, 0);
-       fuzz = fuzz_begin(FUZZ_1_BIT_FLIP | /* too slow FUZZ_2_BIT_FLIP | */
-           FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP |
-           FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END, sig, l);
+       fuzz = fuzz_begin(fuzzers, sig, l);
        ASSERT_INT_EQ(sshkey_verify(k, sig, l, c, sizeof(c), NULL, 0), 0);
        free(sig);
        TEST_ONERROR(onerror, fuzz);
@@ -96,13 +103,15 @@ sig_fuzz(struct sshkey *k, const char *sig_alg)
        fuzz_cleanup(fuzz);
 }
 
+#define NUM_FAST_BASE64_TESTS  1024
+
 void
 sshkey_fuzz_tests(void)
 {
        struct sshkey *k1;
        struct sshbuf *buf, *fuzzed;
        struct fuzz *fuzz;
-       int r;
+       int r, i;
 
 
        TEST_START("fuzz RSA private");
@@ -114,12 +123,14 @@ sshkey_fuzz_tests(void)
        sshbuf_free(buf);
        ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
        TEST_ONERROR(onerror, fuzz);
-       for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+       for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                ASSERT_INT_EQ(r, 0);
                if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                        sshkey_free(k1);
                sshbuf_reset(fuzzed);
+               if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                       break;
        }
        sshbuf_free(fuzzed);
        fuzz_cleanup(fuzz);
@@ -134,12 +145,14 @@ sshkey_fuzz_tests(void)
        sshbuf_free(buf);
        ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
        TEST_ONERROR(onerror, fuzz);
-       for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+       for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                ASSERT_INT_EQ(r, 0);
                if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                        sshkey_free(k1);
                sshbuf_reset(fuzzed);
+               if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                       break;
        }
        sshbuf_free(fuzzed);
        fuzz_cleanup(fuzz);
@@ -154,12 +167,14 @@ sshkey_fuzz_tests(void)
        sshbuf_free(buf);
        ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
        TEST_ONERROR(onerror, fuzz);
-       for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+       for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                ASSERT_INT_EQ(r, 0);
                if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                        sshkey_free(k1);
                sshbuf_reset(fuzzed);
+               if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                       break;
        }
        sshbuf_free(fuzzed);
        fuzz_cleanup(fuzz);
@@ -174,12 +189,14 @@ sshkey_fuzz_tests(void)
        sshbuf_free(buf);
        ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
        TEST_ONERROR(onerror, fuzz);
-       for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+       for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                ASSERT_INT_EQ(r, 0);
                if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                        sshkey_free(k1);
                sshbuf_reset(fuzzed);
+               if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                       break;
        }
        sshbuf_free(fuzzed);
        fuzz_cleanup(fuzz);
@@ -195,12 +212,14 @@ sshkey_fuzz_tests(void)
        sshbuf_free(buf);
        ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
        TEST_ONERROR(onerror, fuzz);
-       for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+       for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                ASSERT_INT_EQ(r, 0);
                if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                        sshkey_free(k1);
                sshbuf_reset(fuzzed);
+               if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                       break;
        }
        sshbuf_free(fuzzed);
        fuzz_cleanup(fuzz);
@@ -215,12 +234,14 @@ sshkey_fuzz_tests(void)
        sshbuf_free(buf);
        ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
        TEST_ONERROR(onerror, fuzz);
-       for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+       for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                ASSERT_INT_EQ(r, 0);
                if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                        sshkey_free(k1);
                sshbuf_reset(fuzzed);
+               if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                       break;
        }
        sshbuf_free(fuzzed);
        fuzz_cleanup(fuzz);
@@ -236,12 +257,14 @@ sshkey_fuzz_tests(void)
        sshbuf_free(buf);
        ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
        TEST_ONERROR(onerror, fuzz);
-       for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+       for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
                r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
                ASSERT_INT_EQ(r, 0);
                if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
                        sshkey_free(k1);
                sshbuf_reset(fuzzed);
+               if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+                       break;
        }
        sshbuf_free(fuzzed);
        fuzz_cleanup(fuzz);

diff --git a/regress/unittests/test_helper/test_helper.c b/regress/unittests/test_helper/test_helper.c
index 4cc70852c0445e91e1c648d72ab09efe7d50ad27..6200ccd589fd4f819f6c0b441efd11cb25641c1f 100644 (file)
--- a/regress/unittests/test_helper/test_helper.c
+++ b/regress/unittests/test_helper/test_helper.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: test_helper.c,v 1.8 2018/02/08 08:46:20 djm Exp $     */
+/*     $OpenBSD: test_helper.c,v 1.9 2018/10/17 23:28:05 djm Exp $     */
 /*
  * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
  *
@@ -115,6 +115,8 @@ static test_onerror_func_t *test_onerror = NULL;
 static void *onerror_ctx = NULL;
 static const char *data_dir = NULL;
 static char subtest_info[512];
+static int fast = 0;
+static int slow = 0;
 
 int
 main(int argc, char **argv)
@@ -134,8 +136,14 @@ main(int argc, char **argv)
                }
        }
 
-       while ((ch = getopt(argc, argv, "vqd:")) != -1) {
+       while ((ch = getopt(argc, argv, "Ffvqd:")) != -1) {
                switch (ch) {
+               case 'F':
+                       slow = 1;
+                       break;
+               case 'f':
+                       fast = 1;
+                       break;
                case 'd':
                        data_dir = optarg;
                        break;
@@ -167,17 +175,29 @@ main(int argc, char **argv)
 }
 
 int
-test_is_verbose()
+test_is_verbose(void)
 {
        return verbose_mode;
 }
 
 int
-test_is_quiet()
+test_is_quiet(void)
 {
        return quiet_mode;
 }
 
+int
+test_is_fast(void)
+{
+       return fast;
+}
+
+int
+test_is_slow(void)
+{
+       return slow;
+}
+
 const char *
 test_data_file(const char *name)
 {

diff --git a/regress/unittests/test_helper/test_helper.h b/regress/unittests/test_helper/test_helper.h
index 6da0066e907ac9b4eb70c0fd9e0c4506523c58cd..1f893c8dd22ece34a52b565add909c66e9107656 100644 (file)
--- a/regress/unittests/test_helper/test_helper.h
+++ b/regress/unittests/test_helper/test_helper.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: test_helper.h,v 1.8 2018/02/08 08:46:20 djm Exp $     */
+/*     $OpenBSD: test_helper.h,v 1.9 2018/10/17 23:28:05 djm Exp $     */
 /*
  * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
  *
@@ -45,6 +45,8 @@ void set_onerror_func(test_onerror_func_t *f, void *ctx);
 void test_done(void);
 int test_is_verbose(void);
 int test_is_quiet(void);
+int test_is_fast(void);
+int test_is_slow(void);
 void test_subtest_info(const char *fmt, ...)
     __attribute__((format(printf, 1, 2)));
 void ssl_err_check(const char *file, int line);