Collect failure details when the regression tests fail

* README.in: Add a section on running regression tests.
* Makefile.am (check-regression): Capture the test run output, and
on failure collect configure and test results into a tar file.

diff --git a/.gitignore b/.gitignore
index 5c558ea4a78f2b249be6666814e5422e570694d6..e93c46b9ea08650215a99b207e0fef1939619565 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -60,6 +60,10 @@ WinRel/
 GccDebug/
 GccRel/
 
+# Test artifacts
+makeerror-*.tar.gz
+test-suite.log
+
 # Distribution artifacts
 .dep_segment
 .check-git-HEAD

diff --git a/Makefile.am b/Makefile.am
index 5a474613f1e7056d6d6636ef87827cf5d47cd006..5912a396cea88697e542d9afe880b9f54a59cfff 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -137,8 +137,8 @@ src/gmk-default.h: $(top_srcdir)/src/gmk-default.scm
 dist-hook:
        (cd $(top_srcdir); \
         sub=`find tests/scripts -follow \( -name .git -o -name .deps -o -name work -o -name .gitignore -o -name \*.orig -o -name \*.rej -o -name \*~ -o -name \*.out -o -name Makefile \) -prune -o -type f -print`; \
-        tar chf - $$sub) \
-       | (cd $(distdir); tar xfBp -)
+        $(AMTAR) chf - $$sub) \
+       | (cd $(distdir); $(AMTAR) xfBp -)
 
 
 # --------------- Local CHECK Section
@@ -162,9 +162,19 @@ MAKETESTFLAGS =
 
 .PHONY: check-regression
 
-GMK_OUTDIR=..
+GMK_OUTDIR = ..
+
+testlog = test-suite.log
+testresult = tests/.test-result
+errorfile = makeerror-$(host_triplet).tar.gz
+
+testfiles = $(testlog) $(testresult) $(errorfile)
+MOSTLYCLEANFILES = $(testfiles)
+
+errordetails = config.status config.log src/config.h $(testlog) tests/work
 
 check-regression: tests/config-flags.pm
+       @rm -f $(testfiles)
        @if test -f '$(top_srcdir)/tests/run_make_tests'; then \
          ulimit -n 128; \
          if $(PERL) -v >/dev/null 2>&1; then \
@@ -177,7 +187,13 @@ check-regression: tests/config-flags.pm
                   done; fi ;; \
            esac; \
            echo "cd tests && $(PERL) $(PERLFLAGS) ./run_make_tests.pl -srcdir $(abs_top_srcdir) -make $(GMK_OUTDIR)/make$(EXEEXT) $(MAKETESTFLAGS)"; \
-           cd tests && $(PERL) $(PERLFLAGS) ./run_make_tests.pl -srcdir '$(abs_top_srcdir)' -make '$(GMK_OUTDIR)/make$(EXEEXT)' $(MAKETESTFLAGS); \
+           (cd tests && $(PERL) $(PERLFLAGS) ./run_make_tests.pl -srcdir '$(abs_top_srcdir)' -make '$(GMK_OUTDIR)/make$(EXEEXT)' $(MAKETESTFLAGS); echo $$? >.test-result) 2>&1 | tee $(testlog); \
+           er=$$(cat $(testresult)); if test "$$er" -ne 0; then \
+             $(AMTAR) chf - $(errordetails) | eval GZIP= gzip $(GZIP_ENV) -c >$(errorfile); \
+             echo '*** Testing failed!  Details saved in: $(errorfile)'; \
+             echo '*** Please report to <$(PACKAGE_BUGREPORT)>'; echo; \
+             exit $$er; \
+           fi; \
          else \
            echo "Can't find a working Perl ($(PERL)); the test suite requires Perl."; \
          fi; \

diff --git a/README.in b/README.in
index 4ef14a6ad99e33bc8884643d8750b3490a5f9036..f144caf27c07d1e296fa20d998241312259f6962 100644 (file)
--- a/README.in
+++ b/README.in
@@ -66,6 +66,19 @@ You can find most information concerning the development of GNU Make at
 this site.
 
 
+Regression Tests
+----------------
+
+GNU Make contains a suite of regression tests.  To run them use "make check"
+after building GNU Make.  If they fail a tar package will be created
+containing useful information, which can be emailed (as an attachment) to
+the <bug-make@gnu.org> mailing list.
+
+Please note that since these tests rely on known-good-output comparisons,
+they can show spurious failures on some systems (particularly non-POSIX systems
+such as Windows).
+
+
 Bug Reporting
 -------------
 

diff --git a/tests/.gitignore b/tests/.gitignore
index a30a689584a2d1cbb37829de71afd6f4534d2943..0550104683390291095b056f0da998b865362739 100644 (file)
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -1,2 +1,3 @@
+.test-result
 config-flags.pm
 work