runtime(syntax-tests): Do not ignore failed screendumps

The process of preparing and submitting syntax tests is
fraught with challenges that can turn away many aspiring
contributors from ever attempting it.  (Out of 69 languages
introduced since v9.0.1627, there are only syntax tests for
Tera.)

After v9.1.1176~1, one visual clue for admitting syntax test
failures previously available with e.g. "git status" is gone
after all files under "failed/" have been made ignored for
Git and Mercurial.  There isn't a single way to go about it:
some people may move files from "failed/" to "dumps/" after
each iteration; some people may only move "good" iteration
files; when a test file is refactored to a great extent,
some people may prefer deleting all test-related files under
"dumps/" before moving files from "failed/".  The usability
of reporting, at any time, that there are some _untracked_
files under "failed/" cannot be overstated.  Without it, the
chances are greater for pushing mismatched changesets.  And
when tests fail then everyone but the author will be kept in
the dark about the cause: were some updated screendumps not
committed _or_ was a wrong version of the syntax plugin
committed?

Another file, "testdir/Xfilter" (v9.1.0763), that will be
created to establish communication from Make to Vim about
what subset of syntax tests is requested for running, should
also be not ignored but rather deleted once its contents are
read.  Unless it is explicitly deleted _after test failure_,
the file may contain new *and* old test names when another
testing attempt is under way.  And by virtue of it being
ignored, the reason for also running not requested tests
will be as ever puzzling.

Both Git and Mercurial support per-user configuration; such
wide-reaching settings hardly belong to clonable defaults.

Also, match literal dots in testname filters.

Also, discover and report _some_ disused screendump files
tracked under "dumps/".

References:
- https://git-scm.com/docs/gitignore
- https://www.mercurial-scm.org/help/topics/config#ui

closes: #16917

Signed-off-by: Aliaksei Budavei <0x000c70@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>

diff --git a/.gitignore b/.gitignore
index 420b3d90e9fd2a5677b64edb8f2940252c003e73..aee4e9944c4a6ab0534dcff2dd2523be40ef36d2 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -96,9 +96,7 @@ src/json_test
 src/message_test
 src/kword_test
 
-runtime/syntax/testdir/Xfilter
 runtime/syntax/testdir/done/
-runtime/syntax/testdir/failed/
 runtime/syntax/testdir/messages
 runtime/syntax/testdir/testdeps.mk
 runtime/syntax/testdir/vimcmd

diff --git a/.hgignore b/.hgignore
index d7677be5d83a9cdb095138ec6ab81981d080f78c..38388a53098d3459c95a8c9972116fdbb52538ce 100644 (file)
--- a/.hgignore
+++ b/.hgignore
@@ -98,9 +98,7 @@ src/json_test
 src/message_test
 src/kword_test
 
-runtime/syntax/testdir/Xfilter
 runtime/syntax/testdir/done/
-runtime/syntax/testdir/failed/
 runtime/syntax/testdir/messages
 runtime/syntax/testdir/testdeps.mk
 runtime/syntax/testdir/vimcmd

diff --git a/runtime/syntax/Makefile b/runtime/syntax/Makefile
index b081cde78aa165256008ced0c482d1aeeb768c1c..9392b8dc98c189377904015e0b35ef0d9758795a 100644 (file)
--- a/runtime/syntax/Makefile
+++ b/runtime/syntax/Makefile
@@ -43,7 +43,6 @@ test:
        @#mkdir -p testdir/failed
        @#touch "$(VIM_SYNTAX_TEST_LOG)"
        VIMRUNTIME=$(VIMRUNTIME) $(ENVVARS) $(VIMPROG) --clean --not-a-term $(DEBUGLOG) -u testdir/runtest.vim > /dev/null
-       @rm -f testdir/Xfilter
        @# FIXME: Temporarily show the whole file to find out what goes wrong
        @#if [ -f testdir/messages ]; then tail -n 6 testdir/messages; fi
        @if [ -f testdir/messages ]; then cat testdir/messages; fi

diff --git a/runtime/syntax/testdir/runtest.vim b/runtime/syntax/testdir/runtest.vim
index 4152702c2076855fd67d0a58a131a0d561992759..767510cb1d35f6befce6bba82bb411e826be5165 100644 (file)
--- a/runtime/syntax/testdir/runtest.vim
+++ b/runtime/syntax/testdir/runtest.vim
@@ -389,18 +389,21 @@ func RunTest()
       \ : {}
   lockvar DUMP_OPTS MAX_FAILED_COUNT XTESTSCRIPT
   let ok_count = 0
+  let disused_pages = []
   let failed_tests = []
   let skipped_count = 0
   let last_test_status = 'invalid'
+  let filter = ''
   " Create a map of setup configuration filenames with their basenames as keys.
   let setup = glob('input/setup/*.vim', 1, 1)
     \ ->reduce({d, f -> extend(d, {fnamemodify(f, ':t:r'): f})}, {})
   " Turn a subset of filenames etc. requested for testing into a pattern.
-  let filter = filereadable('../testdir/Xfilter')
-    \ ? readfile('../testdir/Xfilter')
-       \ ->map({_, v -> '^' .. substitute(v, '_$', '', '')})
+  if filereadable('../testdir/Xfilter')
+    let filter = readfile('../testdir/Xfilter')
+       \ ->map({_, v -> '^' .. escape(substitute(v, '_$', '', ''), '.')})
        \ ->join('\|')
-    \ : ''
+    call delete('../testdir/Xfilter')
+  endif
 
   " Treat "^self-testing" as a string NOT as a regexp.
   if filter ==# '^self-testing'
@@ -430,8 +433,8 @@ func RunTest()
       let filetype = substitute(root, '\([^_.]*\)[_.].*', '\1', '')
       let failed_root = 'failed/' .. root
 
-      for dumpname in glob(failed_root .. '_\d*\.dump', 1, 1)
-       call delete(dumpname)
+      for pagename in glob(failed_root .. '_\d*\.dump', 1, 1)
+       call delete(pagename)
       endfor
       call delete('done/' .. root)
       call writefile(XTESTSCRIPT, 'Xtestscript')
@@ -477,11 +480,11 @@ func RunTest()
       call ch_log('First screendump for ' .. in_name_and_out_name)
       " Make a screendump at the start of the file: failed/root_00.dump
       let fail = VerifyScreenDump(buf, root_00, DUMP_OPTS)
+      let nr = 0
 
       " Accommodate the next code block to "buf"'s contingency for self
       " wipe-out.
       try
-       let nr = 0
        let keys_a = ":call ScrollToSecondPage((18 * 75 + 1), 19, 5) | redraw!\<CR>"
        let keys_b = ":call ScrollToNextPage((18 * 75 + 1), 19, 5) | redraw!\<CR>"
        while s:CannotSeeLastLine(ruler)
@@ -503,6 +506,15 @@ func RunTest()
        call delete('Xtestscript')
       endtry
 
+      let nr += 1
+      let pagename = printf('dumps/%s_%02d.dump', root, nr)
+
+      while filereadable(pagename)
+       call add(disused_pages, pagename)
+       let nr += 1
+       let pagename = printf('dumps/%s_%02d.dump', root, nr)
+      endwhile
+
       " redraw here to avoid the following messages to get mixed up with screen
       " output.
       redraw
@@ -573,6 +585,10 @@ func RunTest()
   call Message('FAILED: ' .. len(failed_tests) .. ': ' .. string(failed_tests))
   call Message('skipped: ' .. skipped_count)
 
+  for pagename in disused_pages
+    call Message(printf('No input page found for "%s"', pagename))
+  endfor
+
   if !empty(failed_tests)
     call Message('')
     call Message('View generated screendumps with "../../src/vim --clean -S testdir/viewdumps.vim"')