From: Joseph Sutton <josephsutton@catalyst.net.nz>
Date: Tue, 11 Apr 2023 21:41:29 +0000 (+1200)
Subject: lib/torture: Don't overwrite test outcomes
X-Git-Tag: talloc-2.4.1~920
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=44d03bf47930e81679feede7a7719feb7aa77c95;p=thirdparty%2Fsamba.git

lib/torture: Don't overwrite test outcomes

If a test fails an assertion, and later calls torture_skip() to skip
part of the test, the TORTURE_SKIP result will overwrite the
TORTURE_FAIL result, and the overall outcome will be successful.

To avoid this, we now arrange possible outcomes in order of priority,
and ensure we always keep the higher priority one.

This reveals some failing tests.

Signed-off-by: Joseph Sutton <josephsutton@catalyst.net.nz>
Reviewed-by: Andreas Schneider <asn@samba.org>
---

diff --git a/lib/torture/torture.c b/lib/torture/torture.c
index b8fb28f669f..41226ef7504 100644
--- a/lib/torture/torture.c
+++ b/lib/torture/torture.c
@@ -212,18 +212,22 @@ void torture_warning(struct torture_context *context, const char *comment, ...)
 void torture_result(struct torture_context *context, 
 		    enum torture_result result, const char *fmt, ...)
 {
-	va_list ap;
+	/* Of the two outcomes, keep that with the higher priority. */
+	if (result >= context->last_result) {
+		va_list ap;
 
-	va_start(ap, fmt);
+		va_start(ap, fmt);
 
-	if (context->last_reason) {
-		torture_warning(context, "%s", context->last_reason);
-		talloc_free(context->last_reason);
-	}
+		if (context->last_reason) {
+			torture_warning(context, "%s", context->last_reason);
+			talloc_free(context->last_reason);
+		}
 
-	context->last_result = result;
-	context->last_reason = talloc_vasprintf(context, fmt, ap);
-	va_end(ap);
+		context->last_result = result;
+		context->last_reason = talloc_vasprintf(context, fmt, ap);
+
+		va_end(ap);
+	}
 }
 
 /**
diff --git a/lib/torture/torture.h b/lib/torture/torture.h
index f6089684afd..6818084ea96 100644
--- a/lib/torture/torture.h
+++ b/lib/torture/torture.h
@@ -27,11 +27,15 @@ struct torture_suite;
 struct torture_tcase;
 struct torture_results;
 
+/*
+ * Arranged in precedence order. TORTURE_ERROR has the highest priority;
+ * TORTURE_OK the lowest.
+ */
 enum torture_result { 
 	TORTURE_OK=0, 
-	TORTURE_FAIL=1,
-	TORTURE_ERROR=2,
-	TORTURE_SKIP=3
+	TORTURE_SKIP=1,
+	TORTURE_FAIL=2,
+	TORTURE_ERROR=3
 };
 
 enum torture_progress_whence {
diff --git a/selftest/knownfail.d/samba3.rpc.samr b/selftest/knownfail.d/samba3.rpc.samr
new file mode 100644
index 00000000000..1bcb89e6421
--- /dev/null
+++ b/selftest/knownfail.d/samba3.rpc.samr
@@ -0,0 +1,5 @@
+# TestPrivateFunctionsDomain is not implemented.
+# QueryDisplayInfo does not return all domain groups on the builtin domain handle.
+^samba3.rpc.samr.samr\(nt4_dc\)
+
+^samba3.rpc.samr.users.samr.users\(nt4_dc\)
diff --git a/selftest/knownfail.d/samba4.rpc.netlogon-s3 b/selftest/knownfail.d/samba4.rpc.netlogon-s3
new file mode 100644
index 00000000000..a7d6732d091
--- /dev/null
+++ b/selftest/knownfail.d/samba4.rpc.netlogon-s3
@@ -0,0 +1,4 @@
+# ServerAuthenticate2 returns NT_STATUS_DOWNGRADE_DETECTED, but we expect NT_STATUS_OK
+^samba4.rpc.netlogon-s3\ with\ .netlogon.SetPassword_with_flags\(ad_dc_default\)
+^samba4.rpc.netlogon-s3\ with\ validate.netlogon.SetPassword_with_flags\(ad_dc_default\)
+^samba4.rpc.netlogon-s3\ with\ bigendian.netlogon.SetPassword_with_flags\(ad_dc_default\)
diff --git a/selftest/knownfail.d/samba4.rpc.samr b/selftest/knownfail.d/samba4.rpc.samr
new file mode 100644
index 00000000000..5bb04bdab13
--- /dev/null
+++ b/selftest/knownfail.d/samba4.rpc.samr
@@ -0,0 +1,2 @@
+^samba4.rpc.samr.users\ on\ ncacn_np\ with\ .samr.users\(ad_dc_slowtests\)
+^samba4.rpc.samr.users\ against\ s4member\ server\ with\ local\ creds.samr.users\(s4member\)