lib/torture: Don't overwrite test outcomes

If a test fails an assertion, and later calls torture_skip() to skip
part of the test, the TORTURE_SKIP result will overwrite the
TORTURE_FAIL result, and the overall outcome will be successful.

To avoid this, we now arrange possible outcomes in order of priority,
and ensure we always keep the higher priority one.

This reveals some failing tests.

Signed-off-by: Joseph Sutton <josephsutton@catalyst.net.nz>
Reviewed-by: Andreas Schneider <asn@samba.org>

diff --git a/lib/torture/torture.c b/lib/torture/torture.c
index b8fb28f669fe3abc1af1ace95800e5587141e532..41226ef75047ebc0de3a4cc5cedfafa318ca155a 100644 (file)
--- a/lib/torture/torture.c
+++ b/lib/torture/torture.c
@@ -212,18 +212,22 @@ void torture_warning(struct torture_context *context, const char *comment, ...)
 void torture_result(struct torture_context *context, 
                    enum torture_result result, const char *fmt, ...)
 {
-       va_list ap;
+       /* Of the two outcomes, keep that with the higher priority. */
+       if (result >= context->last_result) {
+               va_list ap;
 
-       va_start(ap, fmt);
+               va_start(ap, fmt);
 
-       if (context->last_reason) {
-               torture_warning(context, "%s", context->last_reason);
-               talloc_free(context->last_reason);
-       }
+               if (context->last_reason) {
+                       torture_warning(context, "%s", context->last_reason);
+                       talloc_free(context->last_reason);
+               }
 
-       context->last_result = result;
-       context->last_reason = talloc_vasprintf(context, fmt, ap);
-       va_end(ap);
+               context->last_result = result;
+               context->last_reason = talloc_vasprintf(context, fmt, ap);
+
+               va_end(ap);
+       }
 }
 
 /**

diff --git a/lib/torture/torture.h b/lib/torture/torture.h
index f6089684afd0f64d5d25b3623a4edd0bfddca95d..6818084ea9612f4ed1edaaf3c6c7b10e56fc15c7 100644 (file)
--- a/lib/torture/torture.h
+++ b/lib/torture/torture.h
@@ -27,11 +27,15 @@ struct torture_suite;
 struct torture_tcase;
 struct torture_results;
 
+/*
+ * Arranged in precedence order. TORTURE_ERROR has the highest priority;
+ * TORTURE_OK the lowest.
+ */
 enum torture_result { 
        TORTURE_OK=0, 
-       TORTURE_FAIL=1,
-       TORTURE_ERROR=2,
-       TORTURE_SKIP=3
+       TORTURE_SKIP=1,
+       TORTURE_FAIL=2,
+       TORTURE_ERROR=3
 };
 
 enum torture_progress_whence {

diff --git a/selftest/knownfail.d/samba3.rpc.samr b/selftest/knownfail.d/samba3.rpc.samr
new file mode 100644 (file)
index 0000000..1bcb89e
--- /dev/null
+++ b/selftest/knownfail.d/samba3.rpc.samr
@@ -0,0 +1,5 @@
+# TestPrivateFunctionsDomain is not implemented.
+# QueryDisplayInfo does not return all domain groups on the builtin domain handle.
+^samba3.rpc.samr.samr\(nt4_dc\)
+
+^samba3.rpc.samr.users.samr.users\(nt4_dc\)

diff --git a/selftest/knownfail.d/samba4.rpc.netlogon-s3 b/selftest/knownfail.d/samba4.rpc.netlogon-s3
new file mode 100644 (file)
index 0000000..a7d6732
--- /dev/null
+++ b/selftest/knownfail.d/samba4.rpc.netlogon-s3
@@ -0,0 +1,4 @@
+# ServerAuthenticate2 returns NT_STATUS_DOWNGRADE_DETECTED, but we expect NT_STATUS_OK
+^samba4.rpc.netlogon-s3\ with\ .netlogon.SetPassword_with_flags\(ad_dc_default\)
+^samba4.rpc.netlogon-s3\ with\ validate.netlogon.SetPassword_with_flags\(ad_dc_default\)
+^samba4.rpc.netlogon-s3\ with\ bigendian.netlogon.SetPassword_with_flags\(ad_dc_default\)

diff --git a/selftest/knownfail.d/samba4.rpc.samr b/selftest/knownfail.d/samba4.rpc.samr
new file mode 100644 (file)
index 0000000..5bb04bd
--- /dev/null
+++ b/selftest/knownfail.d/samba4.rpc.samr
@@ -0,0 +1,2 @@
+^samba4.rpc.samr.users\ on\ ncacn_np\ with\ .samr.users\(ad_dc_slowtests\)
+^samba4.rpc.samr.users\ against\ s4member\ server\ with\ local\ creds.samr.users\(s4member\)