testsuite: allow for expected failure of tests

Adds a bool to the test struct called 'expected_fail' which can be set
to flip the logic used to determine success and failure. Messaging is
also changed to reflect an unexpected pass or expected fail. This can be
used to write tests which may represent functionality desirable for a
future release.

diff --git a/testsuite/testsuite.c b/testsuite/testsuite.c
index 60dfff202e57cd09a344059560457b58d56a94f3..56e73ee8f1e357e3ac59b796d23b5ff280112def 100644 (file)
--- a/testsuite/testsuite.c
+++ b/testsuite/testsuite.c
@@ -407,21 +407,40 @@ static inline int test_run_parent(const struct test *t, int fdout[2],
                                WTERMSIG(err), strsignal(WTERMSIG(err)));
        }
 
-       if (err == 0) {
-               if (matchout)
-                       LOG("%sPASSED%s: %s\n",
-                               ANSI_HIGHLIGHT_GREEN_ON, ANSI_HIGHLIGHT_OFF,
-                               t->name);
-               else {
-                       ERR("%sFAILED%s: exit ok but outputs do not match: %s\n",
-                               ANSI_HIGHLIGHT_RED_ON, ANSI_HIGHLIGHT_OFF,
-                               t->name);
-                       err = EXIT_FAILURE;
+       if (t->expected_fail == false) {
+               if (err == 0) {
+                       if (matchout)
+                               LOG("%sPASSED%s: %s\n",
+                                       ANSI_HIGHLIGHT_GREEN_ON, ANSI_HIGHLIGHT_OFF,
+                                       t->name);
+                       else {
+                               ERR("%sFAILED%s: exit ok but outputs do not match: %s\n",
+                                       ANSI_HIGHLIGHT_RED_ON, ANSI_HIGHLIGHT_OFF,
+                                       t->name);
+                               err = EXIT_FAILURE;
+                       }
+               } else
+                       ERR("%sFAILED%s: %s\n",
+                                       ANSI_HIGHLIGHT_RED_ON, ANSI_HIGHLIGHT_OFF,
+                                       t->name);
+       } else {
+               if (err == 0) {
+                       if (matchout) {
+                               LOG("%sUNEXPECTED PASS%s: %s\n",
+                                       ANSI_HIGHLIGHT_GREEN_ON, ANSI_HIGHLIGHT_OFF,
+                                       t->name);
+                               err = EXIT_FAILURE;
+                       } else
+                               LOG("%sEXPECTED FAIL%s: exit ok but outputs do not match: %s\n",
+                                       ANSI_HIGHLIGHT_GREEN_ON, ANSI_HIGHLIGHT_OFF,
+                                       t->name);
+               } else {
+                       ERR("%sEXPECTED FAIL%s: %s\n",
+                                       ANSI_HIGHLIGHT_GREEN_ON, ANSI_HIGHLIGHT_OFF,
+                                       t->name);
+                       err = EXIT_SUCCESS;
                }
-       } else
-               ERR("%sFAILED%s: %s\n",
-                               ANSI_HIGHLIGHT_RED_ON, ANSI_HIGHLIGHT_OFF,
-                               t->name);
+       }
 
        return err;
 }

diff --git a/testsuite/testsuite.h b/testsuite/testsuite.h
index b02d73d9e990ef36fae46c910d61b05b575991ad..9081d84d3d4d5e9933f6a62bd4f944bf23934e9b 100644 (file)
--- a/testsuite/testsuite.h
+++ b/testsuite/testsuite.h
@@ -48,6 +48,7 @@ struct test {
        testfunc func;
        const char *config[_TC_LAST];
        bool need_spawn;
+       bool expected_fail;
 };