]> git.ipfire.org Git - thirdparty/vim.git/commitdiff
Revert "CI: Manage multibyte characters in syntax tests"
authorAliaksei Budavei <0x000c70@gmail.com>
Sat, 7 Feb 2026 15:48:33 +0000 (15:48 +0000)
committerChristian Brabandt <cb@256bit.org>
Sat, 7 Feb 2026 15:48:33 +0000 (15:48 +0000)
This reverts commit 0fde6aebddef5cb0428e85040994ba45e55cba99.

With "v9.1.2134" applied, arbitrary multibyte characters are
not replaced with spurious U+FFFD characters (typically when
the host system is under load); U+FFFD characters that are
intentionally written in an input file continue to remain
present for later output and comparison.  The workaround of
"v9.1.1592~3" is no longer necessary.

Also prefer "page_nr" to "nr" in syntax/testdir/runtest.vim

closes: #19348

Signed-off-by: Aliaksei Budavei <0x000c70@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>
runtime/syntax/Makefile
runtime/syntax/testdir/README.txt
runtime/syntax/testdir/runtest.vim
src/testdir/util/screendump.vim

index c18369a0bfc07fa8c9b0789291a38ffec914bd7d..9392b8dc98c189377904015e0b35ef0d9758795a 100644 (file)
@@ -13,7 +13,7 @@ VIMRUNTIME = ../..
 # Uncomment this line to use valgrind for memory leaks and extra warnings.
 # VALGRIND = valgrind --tool=memcheck --leak-check=yes --num-callers=45 --log-file=valgrind.$*
 
-# Trace liveness on demand.
+# Trace ruler liveness on demand.
 # VIM_SYNTAX_TEST_LOG = `pwd`/testdir/failed/00-TRACE_LOG
 
 # ENVVARS = LC_ALL=C VIM_SYNTAX_TEST_LOG="$(VIM_SYNTAX_TEST_LOG)"
@@ -39,7 +39,7 @@ test:
        @# the "vimcmd" file is used by the screendump utils
        @echo "../$(VIMPROG)" > testdir/vimcmd
        @echo "$(RUN_VIMTEST)" >> testdir/vimcmd
-       @# Trace liveness on demand.
+       @# Trace ruler liveness on demand.
        @#mkdir -p testdir/failed
        @#touch "$(VIM_SYNTAX_TEST_LOG)"
        VIMRUNTIME=$(VIMRUNTIME) $(ENVVARS) $(VIMPROG) --clean --not-a-term $(DEBUGLOG) -u testdir/runtest.vim > /dev/null
index 98e59129e6584fd095f77c495776270064546b7f..e4f3945c8f8f6c0184241bba0ea307b6d358a843 100644 (file)
@@ -61,6 +61,8 @@ an "input/setup/java.vim" script file with the following lines:
 Both inline setup commands and setup scripts may be used at the same time, the
 script file will be sourced before any VIM_TEST_SETUP commands are executed.
 
+Every line of a source file must not be longer than 1425 (19 x 75) characters.
+
 If there is no further setup required, you can now run all tests:
 
        make test
@@ -110,20 +112,6 @@ If they look OK, move them to the "dumps" directory:
 If you now run the test again, it will succeed.
 
 
-Limitations for syntax plugin tests
------------------------------------
-
-Do not compose ASCII lines that do not fit a 19 by 75 window (1425 columns).
-
-Use multibyte characters, if at all, sparingly (see #16559).  When possible,
-move multibyte characters closer to the end of a line and keep the line short:
-no more than a 75-byte total of displayed characters.  A poorly rendered line
-may otherwise become wrapped when enough of spurious U+FFFD (0xEF 0xBF 0xBD)
-characters claim more columns than are available (75) and then invalidate line
-correspondence under test.  Refrain from mixing non-spurious U+FFFD characters
-with other multibyte characters in the same line.
-
-
 Adjusting a syntax plugin test
 ------------------------------
 
index 9392a4667177a787b8225db8ae805755742fdec1..f71ccf1638b9df07580f26bdfac63ae01cffcf63 100644 (file)
@@ -116,54 +116,15 @@ func HandleSwapExists()
   endif
 endfunc
 
+" Trace ruler liveness on demand.
 if !empty($VIM_SYNTAX_TEST_LOG) && filewritable($VIM_SYNTAX_TEST_LOG)
-  " Trace liveness.
-  def s:TraceLiveness(context: string, times: number, tail: string)
+  def s:TraceRulerLiveness(context: string, times: number, tail: string)
     writefile([printf('%s: %4d: %s', context, times, tail)],
        $VIM_SYNTAX_TEST_LOG,
        'a')
   enddef
-
-  " Anticipate rendering idiosyncrasies (see #16559).
-  def s:CanFindRenderedFFFDChars(
-         buf: number,
-         in_name_and_out_name: string,
-         times: number): bool
-    if CannotUseRealEstate(in_name_and_out_name)
-      return false
-    endif
-    # Expect a 20-line terminal buffer (see "term_util#RunVimInTerminal()"),
-    # where the bottom, reserved line is of the default "&cmdheight".
-    var lines: list<number> = []
-    for lnum: number in range(1, 19)
-      if stridx(term_getline(buf, lnum), "\xef\xbf\xbd") >= 0
-       add(lines, lnum)
-      endif
-    endfor
-    TraceLiveness('F', times, string(lines))
-    return !empty(lines)
-  enddef
 else
-  " Do not trace liveness.
-  def s:TraceLiveness(_: string, _: number, _: string)
-  enddef
-
-  " Anticipate rendering idiosyncrasies (see #16559).
-  def s:CanFindRenderedFFFDChars(
-         buf: number,
-         in_name_and_out_name: string,
-         _: number): bool
-    if CannotUseRealEstate(in_name_and_out_name)
-      return false
-    endif
-    # Expect a 20-line terminal buffer (see "term_util#RunVimInTerminal()"),
-    # where the bottom, reserved line is of the default "&cmdheight".
-    for lnum: number in range(1, 19)
-      if stridx(term_getline(buf, lnum), "\xef\xbf\xbd") >= 0
-       return true
-      endif
-    endfor
-    return false
+  def s:TraceRulerLiveness(_: string, _: number, _: string)
   enddef
 endif
 
@@ -191,68 +152,6 @@ def s:CannotDumpShellFirstPage(buf: number, _: list<string>, ruler: list<string>
       get(term_getcursor(buf), 0) != 20)
 enddef
 
-def s:CannotUseRealEstate(in_name_and_out_name: string): bool
-  # Expect defaults from "term_util#RunVimInTerminal()".
-  if winwidth(1) != 75 || winheight(1) != 20
-    ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)',
-       in_name_and_out_name,
-       winwidth(1),
-       winheight(1)))
-    return true
-  endif
-  return false
-enddef
-
-" Throw an "FFFD" string if U+FFFD characters are found in the terminal buffer
-" during a non-last test round; otherwise, generate a screendump and proceed
-" with its verification.
-def s:VerifyScreenDumpOrThrowFFFD(
-       buf: number,
-       which_page: string,
-       in_name_and_out_name: string,
-       aborted_count: number,
-       max_aborted_count: number,
-       basename: string,
-       opts: dict<any>,
-       page_quota: dict<number>,
-       seen_pages: list<number>,
-       page_nr: number): number
-  if !has_key(page_quota, page_nr)
-    # Constrain management of unseen pages to the product of "wait" times
-    # "max_aborted_count" (see "opts" below).  When _test repetition_ and
-    # _line rewriting_ FAIL page verification, the page gets to keep its
-    # unseen mark; when _test repetition_ is FAILING for a later page, all
-    # earlier unseen pages get another chance at _test repetition_ etc. before
-    # further progress can be made for the later page.
-    page_quota[page_nr] = max_aborted_count
-  endif
-  const with_fffd: bool = CanFindRenderedFFFDChars(
-      buf,
-      in_name_and_out_name,
-      (max_aborted_count - aborted_count + 1))
-  if with_fffd && aborted_count > 1
-    throw 'FFFD'
-  endif
-  ch_log(which_page .. ' screendump for ' .. in_name_and_out_name)
-  # Generate a screendump of every 19 lines of "buf", reusing the bottom line
-  # (or the bottom six or so lines for "*_01.dump") from the previous dump as
-  # the top line(s) in the next dump for continuity.  Constrain generation of
-  # unseen pages for the last test round (via "wait").
-  const status: number = g:VerifyScreenDump(
-      buf,
-      basename,
-      (aborted_count != max_aborted_count)
-         ? extend({wait: max_aborted_count}, opts, 'keep')
-         : opts)
-  if !with_fffd || (!status || !page_quota[page_nr])
-    add(seen_pages, page_nr)
-  else
-    TraceLiveness('Q', (max_aborted_count - aborted_count + 1), string(page_quota))
-  endif
-  page_quota[page_nr] -= 1
-  return status
-enddef
-
 " Poll for updates of the cursor position in the terminal buffer occupying the
 " first window.  (ALWAYS call the function or its equivalent before calling
 " "VerifyScreenDump()" *and* after calling any number of "term_sendkeys()".)
@@ -260,7 +159,12 @@ def s:TermPollRuler(
        CannotDumpPage: func,   # (TYPE FOR LEGACY CONTEXT CALL SITES.)
        buf: number,
        in_name_and_out_name: string): list<string>
-  if CannotUseRealEstate(in_name_and_out_name)
+  # Expect defaults from "term_util#RunVimInTerminal()".
+  if winwidth(1) != 75 || winheight(1) != 20
+    ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)',
+      in_name_and_out_name,
+      winwidth(1),
+      winheight(1)))
     return ['0,0-1', 'All']
   endif
   # A two-fold role for redrawing:
@@ -283,7 +187,7 @@ def s:TermPollRuler(
       redraw
     endif
   endwhile
-  TraceLiveness('P', (2048 - times), in_name_and_out_name)
+  TraceRulerLiveness('P', (2048 - times), in_name_and_out_name)
   return ruler
 enddef
 
@@ -292,7 +196,12 @@ enddef
 " the terminal buffer.  (Call the function before calling "VerifyScreenDump()"
 " for the first time.)
 def s:TermWaitAndPollRuler(buf: number, in_name_and_out_name: string): list<string>
-  if CannotUseRealEstate(in_name_and_out_name)
+  # Expect defaults from "term_util#RunVimInTerminal()".
+  if winwidth(1) != 75 || winheight(1) != 20
+    ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)',
+      in_name_and_out_name,
+      winwidth(1),
+      winheight(1)))
     return ['0,0-1', 'All']
   endif
   # The contents of "ruler".
@@ -311,7 +220,7 @@ def s:TermWaitAndPollRuler(buf: number, in_name_and_out_name: string): list<stri
       redraw
     endif
   endwhile
-  TraceLiveness('W', (32768 - times), in_name_and_out_name)
+  TraceRulerLiveness('W', (32768 - times), in_name_and_out_name)
   if strpart(ruler, 0, 8) !=# 'is_nonce'
     # Retain any of "b:is_(bash|dash|kornshell|posix|sh)" entries and let
     # "CannotDumpShellFirstPage()" win the cursor race.
@@ -384,34 +293,6 @@ func RunTest()
       redraw!
     endfunc
 
-    def CollectFFFDChars()
-      const fffd: string = "\xef\xbf\xbd"
-      const flags: string = 'eW'
-      const pos: list<number> = getpos('.')
-      var fffds: list<list<number>> = []
-      try
-       cursor(1, 1)
-       var prev: list<number> = [0, 0]
-       var next: list<number> = [0, 0]
-       next = searchpos(fffd, 'c' .. flags)
-       while next[0] > 0 && prev != next
-         add(fffds, next)
-         prev = next
-         next = searchpos(fffd, flags)
-       endwhile
-      finally
-       setpos('.', pos)
-      endtry
-      if !empty(fffds)
-       # Use "actions/upload-artifact@v4" of ci.yml for delivery.
-       writefile(
-         [printf('%s: %s', bufname('%'), string(fffds))],
-         'failed/10-FFFDS',
-         'a')
-      endif
-      redraw!
-    enddef
-
     def s:AssertCursorForwardProgress(): bool
       const curnum: number = line('.')
       if curnum <= cursor
@@ -512,18 +393,12 @@ func RunTest()
       return AssertCursorForwardProgress()
     enddef
   END
-  let MAX_ABORTED_COUNT = 5
   let MAX_FAILED_COUNT = 5
-  let DUMP_OPTS = extend(
-         \ exists("$VIM_SYNTAX_TEST_WAIT_TIME") &&
+  let DUMP_OPTS = exists("$VIM_SYNTAX_TEST_WAIT_TIME") &&
          \ !empty($VIM_SYNTAX_TEST_WAIT_TIME)
-             \ ? {'wait': max([1, str2nr($VIM_SYNTAX_TEST_WAIT_TIME)])}
-             \ : {},
-      \ {'FileComparisonPreAction':
-         \ function('g:ScreenDumpDiscardFFFDChars'),
-      \ 'NonEqualLineComparisonPostAction':
-         \ function('g:ScreenDumpLookForFFFDChars')})
-  lockvar DUMP_OPTS MAX_FAILED_COUNT MAX_ABORTED_COUNT XTESTSCRIPT
+      \ ? {'wait': max([1, str2nr($VIM_SYNTAX_TEST_WAIT_TIME)])}
+      \ : {}
+  lockvar DUMP_OPTS MAX_FAILED_COUNT XTESTSCRIPT
   let ok_count = 0
   let disused_pages = []
   let failed_tests = []
@@ -591,107 +466,53 @@ func RunTest()
       let args = !empty(path)
        \ ? prefix .. ' -S ' .. path
        \ : prefix
-      let fail = 0
+      let buf = RunVimInTerminal(args, {})
+      " edit the file only after catching the SwapExists event
+      call term_sendkeys(buf, ":edit " .. fname .. "\<CR>")
+      " set up the testing environment
+      call term_sendkeys(buf, ":call SetUpVim()\<CR>")
+      " load filetype specific settings
+      call term_sendkeys(buf, ":call LoadFiletype('" .. filetype .. "')\<CR>")
+
+      " Make a synchronisation point between buffers by requesting to echo
+      " a known token in the terminal buffer and asserting its availability
+      " with "s:TermWaitAndPollRuler()".
+      if filetype == 'sh'
+       call term_sendkeys(buf, ":call ShellInfo()\<CR>")
+      else
+       call term_sendkeys(buf, ":echo 'is_nonce'\<CR>")
+      endif
 
+      let root_00 = root .. '_00'
+      let in_name_and_out_name = fname .. ': failed/' .. root_00 .. '.dump'
+      " Queue up all "term_sendkeys()"es and let them finish before returning
+      " from "s:TermWaitAndPollRuler()".
+      let ruler = s:TermWaitAndPollRuler(buf, in_name_and_out_name)
+      call ch_log('First screendump for ' .. in_name_and_out_name)
+      " Make a screendump at the start of the file: failed/root_00.dump
+      let fail = VerifyScreenDump(buf, root_00, DUMP_OPTS)
+      let page_nr = 0
+
+      " Accommodate the next code block to "buf"'s contingency for self
+      " wipe-out.
       try
-       let aborted_count = MAX_ABORTED_COUNT
-       let collected_count = 0
-       let seen_pages = []
-       let page_quota = {}
-
-       " See #16559.  For each processed page, repeat pre-verification steps
-       " from scratch (subject to page cacheing) whenever U+FFFD characters
-       " are found in the terminal buffer with "term_getline()", i.e. treat
-       " these pages as if they were distinct test files.  U+FFFD characters
-       " found at the last attempt (see "MAX_ABORTED_COUNT") will be ignored
-       " and "VerifyScreenDump()" will take over with own filtering.
-       while aborted_count > 0
-         let buf = RunVimInTerminal(args, {})
-         try
-           " edit the file only after catching the SwapExists event
-           call term_sendkeys(buf, ":edit " .. fname .. "\<CR>")
-           " set up the testing environment
-           call term_sendkeys(buf, ":call SetUpVim()\<CR>")
-           " load filetype specific settings
-           call term_sendkeys(buf, ":call LoadFiletype('" .. filetype .. "')\<CR>")
-
-           " Collect all *non-spurious* U+FFFD characters for scrutiny.
-           if aborted_count == 1 && collected_count != 1
-             let collected_count = 1
-             call term_sendkeys(buf, ":call CollectFFFDChars()\<CR>")
-           endif
-
-           " Make a synchronisation point between a terminal buffer and
-           " another buffer by requesting to echo a known token in the former
-           " and asserting its availability with "s:TermWaitAndPollRuler()"
-           " from the latter.
-           if filetype == 'sh'
-             call term_sendkeys(buf, ":call ShellInfo()\<CR>")
-           else
-             call term_sendkeys(buf, ":echo 'is_nonce'\<CR>")
-           endif
-
-           let page_nr = 0
-           let root_00 = root .. '_00'
-           let in_name_and_out_name = fname .. ': failed/' .. root_00 .. '.dump'
-           " Queue up all "term_sendkeys()"es and let them finish before
-           " returning from "s:TermWaitAndPollRuler()".
-           let ruler = s:TermWaitAndPollRuler(buf, in_name_and_out_name)
-           if index(seen_pages, page_nr) < 0
-             let fail += s:VerifyScreenDumpOrThrowFFFD(
-                 \ buf,
-                 \ 'First',
-                 \ in_name_and_out_name,
-                 \ aborted_count,
-                 \ MAX_ABORTED_COUNT,
-                 \ root_00,
-                 \ DUMP_OPTS,
-                 \ page_quota,
-                 \ seen_pages,
-                 \ page_nr)
-             " Reset "aborted_count" for another page.
-             let aborted_count = MAX_ABORTED_COUNT
-           endif
-           let keys_a = ":call ScrollToSecondPage((18 * 75 + 1), 19, 5) | redraw!\<CR>"
-           let keys_b = ":call ScrollToNextPage((18 * 75 + 1), 19, 5) | redraw!\<CR>"
-
-           while s:CannotSeeLastLine(ruler)
-             call term_sendkeys(buf, keys_a)
-             let keys_a = keys_b
-             let page_nr += 1
-             let root_next = printf('%s_%02d', root, page_nr)
-             let in_name_and_out_name = fname .. ': failed/' .. root_next .. '.dump'
-             let ruler = s:TermPollRuler(
-                 \ function('s:CannotDumpNextPage', [buf, ruler]),
-                 \ buf,
-                 \ in_name_and_out_name)
-             if index(seen_pages, page_nr) < 0
-               let fail += s:VerifyScreenDumpOrThrowFFFD(
-                   \ buf,
-                   \ 'Next',
-                   \ in_name_and_out_name,
-                   \ aborted_count,
-                   \ MAX_ABORTED_COUNT,
-                   \ root_next,
-                   \ DUMP_OPTS,
-                   \ page_quota,
-                   \ seen_pages,
-                   \ page_nr)
-               " Reset "aborted_count" for another page.
-               let aborted_count = MAX_ABORTED_COUNT
-             endif
-           endwhile
-           call StopVimInTerminal(buf)
-           break
-         catch /^FFFD$/
-           " Clear out.
-           call StopVimInTerminal(buf)
-           while winnr('$') > 1
-             close
-           endwhile
-           let aborted_count -= 1
-         endtry
+       let keys_a = ":call ScrollToSecondPage((18 * 75 + 1), 19, 5) | redraw!\<CR>"
+       let keys_b = ":call ScrollToNextPage((18 * 75 + 1), 19, 5) | redraw!\<CR>"
+       while s:CannotSeeLastLine(ruler)
+         call term_sendkeys(buf, keys_a)
+         let keys_a = keys_b
+         let page_nr += 1
+         let root_next = printf('%s_%02d', root, page_nr)
+         let in_name_and_out_name = fname .. ': failed/' .. root_next .. '.dump'
+         let ruler = s:TermPollRuler(
+             \ function('s:CannotDumpNextPage', [buf, ruler]),
+             \ buf,
+             \ in_name_and_out_name)
+         call ch_log('Next screendump for ' .. in_name_and_out_name)
+         " Make a screendump of every 18 lines of the file: failed/root_NN.dump
+         let fail += VerifyScreenDump(buf, root_next, DUMP_OPTS)
        endwhile
+       call StopVimInTerminal(buf)
       finally
        call delete('Xtestscript')
       endtry
index d9e90737988223168b28a8dafc927e38b9d47b96..c881eb96367007a481ed797c09f9ead879a8bdbb 100644 (file)
@@ -105,7 +105,6 @@ enddef
 "          some dictionary with some state entries;
 "          the file contents of the newly generated screen dump;
 "          the zero-based number of the line whose copies are not equal.
-" (See an example in runtime/syntax/testdir/runtest.vim.)
 "
 " The file name used is "dumps/{filename}.dump".
 "