Generate .test-result files for ordinary tests.

This patch, an updated version of
<https://sourceware.org/ml/libc-alpha/2014-01/msg00193.html>, starts
the process of generating explicit PASS or FAIL status for individual
glibc tests.  It's based on Tomas Dohnalek's patch
<https://sourceware.org/ml/libc-alpha/2012-10/msg00278.html>, but is
deliberately more minimal: it doesn't try to cover any tests outside
of $(tests) / $(xtests) (that's for a later patch), nor does it put
the result together in an overall summary file (again, a later patch):
it just generates the .test-result files.

Thus, this patch keeps the overall logic for when a testsuite run
finishes completely unchanged: a test failing will terminate the run.
I think we *should* move to a more conventional approach where plain
"make check" does not terminate for an individual test failure, unless
e.g. you say "make stop-on-test-failure=y check", but that sort of
policy change is best done as a separate patch once the infrastructure
is in place to generate summary files for completed test runs (which
will entirely consist of PASS and XFAIL lines if the testsuite run
reaches the point of generating them, until such a policy change is
made).

Tested x86_64.

2014-02-14  Tomas Dohnalek  <tdohnale@redhat.com>
	    Joseph Myers  <joseph@codesourcery.com>

	* Makeconfig (test-name): New variable.
	(evaluate-test): Likewise.
	* Makerules (do-test-clean): Remove .test-result files.
	(common-mostlyclean): Likewise.
	* Rules ($(objpfx)%.out): Use $(evaluate-test) in both rules.
	* scripts/evaluate-test.sh: New file.

diff --git a/ChangeLog b/ChangeLog
index ed9f24c6fabb60a20230436af2c1e53219e7f91f..f625d07efc4802382c85f5e8a0dd444dd7ed7a5f 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2014-02-14  Tomas Dohnalek  <tdohnale@redhat.com>
+           Joseph Myers  <joseph@codesourcery.com>
+
+       * Makeconfig (test-name): New variable.
+       (evaluate-test): Likewise.
+       * Makerules (do-test-clean): Remove .test-result files.
+       (common-mostlyclean): Likewise.
+       * Rules ($(objpfx)%.out): Use $(evaluate-test) in both rules.
+       * scripts/evaluate-test.sh: New file.
+
 2014-02-14  Joseph Myers  <joseph@codesourcery.com>
 
        * libio/Makefile ($(objpfx)tst-fopenloc.check): Split into

diff --git a/Makeconfig b/Makeconfig
index 1908f275a9b7d8b6dd16de9e1c4e63d817ed8fe3..21a7d6ea65368c22543359cc8f53f1c9f3489bfe 100644 (file)
--- a/Makeconfig
+++ b/Makeconfig
@@ -1080,6 +1080,13 @@ defines += -D_LIBC_REENTRANT
 libio-mtsafe = -D_IO_MTSAFE_IO
 endif
 
+# The name to give to a test in test results summaries.
+test-name = $(strip $(patsubst %.out, %, $(patsubst $(common-objpfx)%, %, $@)))
+
+# Command to output a test status line (such as PASS: test-name).
+evaluate-test = $(..)scripts/evaluate-test.sh $(test-name) $$? \
+                 > $(common-objpfx)$(test-name).test-result
+
 endif # Makeconfig not yet included
 
 # Local Variables:

diff --git a/Makerules b/Makerules
index 13c3faec3f456d81f687405bc81c72ee91de0d59..54a347104f0cbde608922254d4f1557307509415 100644 (file)
--- a/Makerules
+++ b/Makerules
@@ -1332,7 +1332,10 @@ mostlyclean: common-mostlyclean
 
 do-tests-clean:
        -rm -f $(addprefix $(objpfx),$(addsuffix .out,$(tests) $(xtests) \
-                                                     $(test-srcs)))
+                                                     $(test-srcs)) \
+                                    $(addsuffix .test-result,$(tests) \
+                                                             $(xtests) \
+                                                             $(test-srcs)))
 
 # Remove the object files.
 common-mostlyclean:
@@ -1342,7 +1345,10 @@ common-mostlyclean:
                                                    $(test-srcs) $(others) \
                                                    $(sysdep-others)) \
                                     $(addsuffix .out,$(tests) $(xtests) \
-                                                     $(test-srcs)))
+                                                     $(test-srcs)) \
+                                    $(addsuffix .test-result,$(tests) \
+                                                             $(xtests) \
+                                                             $(test-srcs)))
        -rm -f $(addprefix $(objpfx),$(extra-objs) $(extra-test-objs) \
                                     $(install-lib) $(install-lib.so) \
                                     $(install-lib.so:%.so=%_pic.a))

diff --git a/Rules b/Rules
index 49ca8ea7a2539453e016184a712b7397dd66d298..3e305ddaa551a84054fc143fd76614c711337aeb 100644 (file)
--- a/Rules
+++ b/Rules
@@ -183,9 +183,11 @@ make-test-out = $(test-wrapper-env) \
                GCONV_PATH=$(common-objpfx)iconvdata LC_ALL=C \
                $($*-ENV) $(host-test-program-cmd) $($*-ARGS)
 $(objpfx)%.out: %.input $(objpfx)%
-       $(make-test-out) > $@ < $(word 1,$^)
+       $(make-test-out) > $@ < $(word 1,$^); \
+       $(evaluate-test)
 $(objpfx)%.out: /dev/null $(objpfx)%   # Make it 2nd arg for canned sequence.
-       $(make-test-out) > $@
+       $(make-test-out) > $@; \
+       $(evaluate-test)
 
 endif  # tests
 

diff --git a/scripts/evaluate-test.sh b/scripts/evaluate-test.sh
new file mode 100755 (executable)
index 0000000..9cb6435
--- /dev/null
+++ b/scripts/evaluate-test.sh
@@ -0,0 +1,33 @@
+#! /bin/sh
+# Output a test status line.
+# Copyright (C) 2012-2014 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+# usage: evaluate-test.sh test_name rc
+
+test_name=$1
+rc=$2
+
+if [ $rc -eq 0 ]; then
+  result="PASS"
+else
+  result="FAIL"
+fi
+
+echo "$result: $test_name"
+echo "original exit status $rc"
+exit $rc