Support expected failures in .test-result files.

This patch, an updated version of
<https://sourceware.org/ml/libc-alpha/2014-01/msg00195.html>, makes it
possible for .test-result files for individual tests to contain XPASS
and XFAIL rather than PASS and FAIL in cases where failure is
expected.  This replaces the marking of two individual tests with "-"
to cause them to be expected at makefile level to fail;
evaluate-test.sh will ensure it exits with status 0 for an expected
failure.

Tested x86_64.

	* scripts/evaluate-test.sh: Take new argument indicating whether
	failure is expected.
	* Makeconfig (evaluate-test): Pass argument to evaluate-test.sh
	indicating whether failure is expected.
	* conform/Makefile (test-xfail-run-conformtest): New variable.
	($(objpfx)run-conformtest.out): Don't expect to fail at makefile
	level.
	* posix/Makefile (test-xfail-annexc): New variable.
	($(objpfx)annexc.out): Don't expect to fail at makefile level.

diff --git a/ChangeLog b/ChangeLog
index 4ef5ccc31775e2c061cd7c388a0261f4433c1f75..7a64c5e6af5eddd3eeb02f1f0e4523efa20693ef 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,15 @@
+2014-02-27  Joseph Myers  <joseph@codesourcery.com>
+
+       * scripts/evaluate-test.sh: Take new argument indicating whether
+       failure is expected.
+       * Makeconfig (evaluate-test): Pass argument to evaluate-test.sh
+       indicating whether failure is expected.
+       * conform/Makefile (test-xfail-run-conformtest): New variable.
+       ($(objpfx)run-conformtest.out): Don't expect to fail at makefile
+       level.
+       * posix/Makefile (test-xfail-annexc): New variable.
+       ($(objpfx)annexc.out): Don't expect to fail at makefile level.
+
 2014-02-26  Joseph Myers  <joseph@codesourcery.com>
 
        * argp/Makefile: Include Makeconfig immediately after defining

diff --git a/Makeconfig b/Makeconfig
index 21a7d6ea65368c22543359cc8f53f1c9f3489bfe..3709c88a45170b50c5e725e7ea97b04a3243b3f0 100644 (file)
--- a/Makeconfig
+++ b/Makeconfig
@@ -1083,8 +1083,11 @@ endif
 # The name to give to a test in test results summaries.
 test-name = $(strip $(patsubst %.out, %, $(patsubst $(common-objpfx)%, %, $@)))
 
-# Command to output a test status line (such as PASS: test-name).
+# Command to output a test status line (such as PASS: test-name).  If
+# test-xfail-$(@F:.out=) has a nonempty value, the status will be
+# XPASS or XFAIL rather than PASS or FAIL.
 evaluate-test = $(..)scripts/evaluate-test.sh $(test-name) $$? \
+                 $(if $(test-xfail-$(@F:.out=)),true,false) \
                  > $(common-objpfx)$(test-name).test-result
 
 endif # Makeconfig not yet included

diff --git a/conform/Makefile b/conform/Makefile
index 41f0bb3f57efedc0ab32a0da173ccfa64f063f8a..40081f304ba550e33b6ed7f29a118162320e4078 100644 (file)
--- a/conform/Makefile
+++ b/conform/Makefile
@@ -30,10 +30,11 @@ tests: $(objpfx)run-conformtest.out
 endif
 endif
 
+test-xfail-run-conformtest = yes
 $(objpfx)run-conformtest.out: run-conformtest.sh conformtest.pl \
                              $(wildcard data/*.h-data) \
                              $(wildcard data/*/*.h-data)
-       -$(BASH) -e $< $(objpfx) $(PERL) '$(CC)' \
+       $(BASH) -e $< $(objpfx) $(PERL) '$(CC)' \
          '-I../include $(+sysdep-includes) $(sysincludes) -I..'; \
        $(evaluate-test)
 

diff --git a/posix/Makefile b/posix/Makefile
index 304238ffe850e003ed2c1319b4c24035010f4e83..b41055a509588014edf0322aca49eab39438f5c7 100644 (file)
--- a/posix/Makefile
+++ b/posix/Makefile
@@ -233,8 +233,9 @@ tests: $(objpfx)bug-regex2-mem $(objpfx)bug-regex14-mem \
 xtests: $(objpfx)bug-ga2-mem
 endif
 
+test-xfail-annexc = yes
 $(objpfx)annexc.out: $(objpfx)annexc
-       -$(dir $<)$(notdir $<) '$(CC)' \
+       $(dir $<)$(notdir $<) '$(CC)' \
          '$(patsubst %,-I../%,$(sorted-subdirs)) -I../include $(+sysdep-includes) $(sysincludes) -I..' > $@; \
        $(evaluate-test)
 

diff --git a/scripts/evaluate-test.sh b/scripts/evaluate-test.sh
index 9cb6435de2a8ee9337f6b145c1eae05625af484b..c8f5012c5accdb1c5e24e8f1ce371b53f8563670 100755 (executable)
--- a/scripts/evaluate-test.sh
+++ b/scripts/evaluate-test.sh
@@ -17,10 +17,12 @@
 # License along with the GNU C Library; if not, see
 # <http://www.gnu.org/licenses/>.
 
-# usage: evaluate-test.sh test_name rc
+# usage: evaluate-test.sh test_name rc xfail
 
 test_name=$1
 rc=$2
+orig_rc=$rc
+xfail=$3
 
 if [ $rc -eq 0 ]; then
   result="PASS"
@@ -28,6 +30,11 @@ else
   result="FAIL"
 fi
 
+if $xfail; then
+  result="X$result"
+  rc=0
+fi
+
 echo "$result: $test_name"
-echo "original exit status $rc"
+echo "original exit status $orig_rc"
 exit $rc