From f0b8b2c32ad055ee66845d2554be7c7b1d461e50 Mon Sep 17 00:00:00 2001 From: Aliaksei Budavei <0x000c70@gmail.com> Date: Sat, 7 Feb 2026 15:48:33 +0000 Subject: [PATCH] Revert "CI: Manage multibyte characters in syntax tests" This reverts commit 0fde6aebddef5cb0428e85040994ba45e55cba99. With "v9.1.2134" applied, arbitrary multibyte characters are not replaced with spurious U+FFFD characters (typically when the host system is under load); U+FFFD characters that are intentionally written in an input file continue to remain present for later output and comparison. The workaround of "v9.1.1592~3" is no longer necessary. Also prefer "page_nr" to "nr" in syntax/testdir/runtest.vim closes: #19348 Signed-off-by: Aliaksei Budavei <0x000c70@gmail.com> Signed-off-by: Christian Brabandt --- runtime/syntax/Makefile | 4 +- runtime/syntax/testdir/README.txt | 16 +- runtime/syntax/testdir/runtest.vim | 309 ++++++----------------------- src/testdir/util/screendump.vim | 1 - 4 files changed, 69 insertions(+), 261 deletions(-) diff --git a/runtime/syntax/Makefile b/runtime/syntax/Makefile index c18369a0bf..9392b8dc98 100644 --- a/runtime/syntax/Makefile +++ b/runtime/syntax/Makefile @@ -13,7 +13,7 @@ VIMRUNTIME = ../.. # Uncomment this line to use valgrind for memory leaks and extra warnings. # VALGRIND = valgrind --tool=memcheck --leak-check=yes --num-callers=45 --log-file=valgrind.$* -# Trace liveness on demand. +# Trace ruler liveness on demand. # VIM_SYNTAX_TEST_LOG = `pwd`/testdir/failed/00-TRACE_LOG # ENVVARS = LC_ALL=C VIM_SYNTAX_TEST_LOG="$(VIM_SYNTAX_TEST_LOG)" @@ -39,7 +39,7 @@ test: @# the "vimcmd" file is used by the screendump utils @echo "../$(VIMPROG)" > testdir/vimcmd @echo "$(RUN_VIMTEST)" >> testdir/vimcmd - @# Trace liveness on demand. + @# Trace ruler liveness on demand. @#mkdir -p testdir/failed @#touch "$(VIM_SYNTAX_TEST_LOG)" VIMRUNTIME=$(VIMRUNTIME) $(ENVVARS) $(VIMPROG) --clean --not-a-term $(DEBUGLOG) -u testdir/runtest.vim > /dev/null diff --git a/runtime/syntax/testdir/README.txt b/runtime/syntax/testdir/README.txt index 98e59129e6..e4f3945c8f 100644 --- a/runtime/syntax/testdir/README.txt +++ b/runtime/syntax/testdir/README.txt @@ -61,6 +61,8 @@ an "input/setup/java.vim" script file with the following lines: Both inline setup commands and setup scripts may be used at the same time, the script file will be sourced before any VIM_TEST_SETUP commands are executed. +Every line of a source file must not be longer than 1425 (19 x 75) characters. + If there is no further setup required, you can now run all tests: make test @@ -110,20 +112,6 @@ If they look OK, move them to the "dumps" directory: If you now run the test again, it will succeed. -Limitations for syntax plugin tests ------------------------------------ - -Do not compose ASCII lines that do not fit a 19 by 75 window (1425 columns). - -Use multibyte characters, if at all, sparingly (see #16559). When possible, -move multibyte characters closer to the end of a line and keep the line short: -no more than a 75-byte total of displayed characters. A poorly rendered line -may otherwise become wrapped when enough of spurious U+FFFD (0xEF 0xBF 0xBD) -characters claim more columns than are available (75) and then invalidate line -correspondence under test. Refrain from mixing non-spurious U+FFFD characters -with other multibyte characters in the same line. - - Adjusting a syntax plugin test ------------------------------ diff --git a/runtime/syntax/testdir/runtest.vim b/runtime/syntax/testdir/runtest.vim index 9392a46671..f71ccf1638 100644 --- a/runtime/syntax/testdir/runtest.vim +++ b/runtime/syntax/testdir/runtest.vim @@ -116,54 +116,15 @@ func HandleSwapExists() endif endfunc +" Trace ruler liveness on demand. if !empty($VIM_SYNTAX_TEST_LOG) && filewritable($VIM_SYNTAX_TEST_LOG) - " Trace liveness. - def s:TraceLiveness(context: string, times: number, tail: string) + def s:TraceRulerLiveness(context: string, times: number, tail: string) writefile([printf('%s: %4d: %s', context, times, tail)], $VIM_SYNTAX_TEST_LOG, 'a') enddef - - " Anticipate rendering idiosyncrasies (see #16559). - def s:CanFindRenderedFFFDChars( - buf: number, - in_name_and_out_name: string, - times: number): bool - if CannotUseRealEstate(in_name_and_out_name) - return false - endif - # Expect a 20-line terminal buffer (see "term_util#RunVimInTerminal()"), - # where the bottom, reserved line is of the default "&cmdheight". - var lines: list = [] - for lnum: number in range(1, 19) - if stridx(term_getline(buf, lnum), "\xef\xbf\xbd") >= 0 - add(lines, lnum) - endif - endfor - TraceLiveness('F', times, string(lines)) - return !empty(lines) - enddef else - " Do not trace liveness. - def s:TraceLiveness(_: string, _: number, _: string) - enddef - - " Anticipate rendering idiosyncrasies (see #16559). - def s:CanFindRenderedFFFDChars( - buf: number, - in_name_and_out_name: string, - _: number): bool - if CannotUseRealEstate(in_name_and_out_name) - return false - endif - # Expect a 20-line terminal buffer (see "term_util#RunVimInTerminal()"), - # where the bottom, reserved line is of the default "&cmdheight". - for lnum: number in range(1, 19) - if stridx(term_getline(buf, lnum), "\xef\xbf\xbd") >= 0 - return true - endif - endfor - return false + def s:TraceRulerLiveness(_: string, _: number, _: string) enddef endif @@ -191,68 +152,6 @@ def s:CannotDumpShellFirstPage(buf: number, _: list, ruler: list get(term_getcursor(buf), 0) != 20) enddef -def s:CannotUseRealEstate(in_name_and_out_name: string): bool - # Expect defaults from "term_util#RunVimInTerminal()". - if winwidth(1) != 75 || winheight(1) != 20 - ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)', - in_name_and_out_name, - winwidth(1), - winheight(1))) - return true - endif - return false -enddef - -" Throw an "FFFD" string if U+FFFD characters are found in the terminal buffer -" during a non-last test round; otherwise, generate a screendump and proceed -" with its verification. -def s:VerifyScreenDumpOrThrowFFFD( - buf: number, - which_page: string, - in_name_and_out_name: string, - aborted_count: number, - max_aborted_count: number, - basename: string, - opts: dict, - page_quota: dict, - seen_pages: list, - page_nr: number): number - if !has_key(page_quota, page_nr) - # Constrain management of unseen pages to the product of "wait" times - # "max_aborted_count" (see "opts" below). When _test repetition_ and - # _line rewriting_ FAIL page verification, the page gets to keep its - # unseen mark; when _test repetition_ is FAILING for a later page, all - # earlier unseen pages get another chance at _test repetition_ etc. before - # further progress can be made for the later page. - page_quota[page_nr] = max_aborted_count - endif - const with_fffd: bool = CanFindRenderedFFFDChars( - buf, - in_name_and_out_name, - (max_aborted_count - aborted_count + 1)) - if with_fffd && aborted_count > 1 - throw 'FFFD' - endif - ch_log(which_page .. ' screendump for ' .. in_name_and_out_name) - # Generate a screendump of every 19 lines of "buf", reusing the bottom line - # (or the bottom six or so lines for "*_01.dump") from the previous dump as - # the top line(s) in the next dump for continuity. Constrain generation of - # unseen pages for the last test round (via "wait"). - const status: number = g:VerifyScreenDump( - buf, - basename, - (aborted_count != max_aborted_count) - ? extend({wait: max_aborted_count}, opts, 'keep') - : opts) - if !with_fffd || (!status || !page_quota[page_nr]) - add(seen_pages, page_nr) - else - TraceLiveness('Q', (max_aborted_count - aborted_count + 1), string(page_quota)) - endif - page_quota[page_nr] -= 1 - return status -enddef - " Poll for updates of the cursor position in the terminal buffer occupying the " first window. (ALWAYS call the function or its equivalent before calling " "VerifyScreenDump()" *and* after calling any number of "term_sendkeys()".) @@ -260,7 +159,12 @@ def s:TermPollRuler( CannotDumpPage: func, # (TYPE FOR LEGACY CONTEXT CALL SITES.) buf: number, in_name_and_out_name: string): list - if CannotUseRealEstate(in_name_and_out_name) + # Expect defaults from "term_util#RunVimInTerminal()". + if winwidth(1) != 75 || winheight(1) != 20 + ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)', + in_name_and_out_name, + winwidth(1), + winheight(1))) return ['0,0-1', 'All'] endif # A two-fold role for redrawing: @@ -283,7 +187,7 @@ def s:TermPollRuler( redraw endif endwhile - TraceLiveness('P', (2048 - times), in_name_and_out_name) + TraceRulerLiveness('P', (2048 - times), in_name_and_out_name) return ruler enddef @@ -292,7 +196,12 @@ enddef " the terminal buffer. (Call the function before calling "VerifyScreenDump()" " for the first time.) def s:TermWaitAndPollRuler(buf: number, in_name_and_out_name: string): list - if CannotUseRealEstate(in_name_and_out_name) + # Expect defaults from "term_util#RunVimInTerminal()". + if winwidth(1) != 75 || winheight(1) != 20 + ch_log(printf('Aborting for %s: (75 x 20) != (%d x %d)', + in_name_and_out_name, + winwidth(1), + winheight(1))) return ['0,0-1', 'All'] endif # The contents of "ruler". @@ -311,7 +220,7 @@ def s:TermWaitAndPollRuler(buf: number, in_name_and_out_name: string): list = getpos('.') - var fffds: list> = [] - try - cursor(1, 1) - var prev: list = [0, 0] - var next: list = [0, 0] - next = searchpos(fffd, 'c' .. flags) - while next[0] > 0 && prev != next - add(fffds, next) - prev = next - next = searchpos(fffd, flags) - endwhile - finally - setpos('.', pos) - endtry - if !empty(fffds) - # Use "actions/upload-artifact@v4" of ci.yml for delivery. - writefile( - [printf('%s: %s', bufname('%'), string(fffds))], - 'failed/10-FFFDS', - 'a') - endif - redraw! - enddef - def s:AssertCursorForwardProgress(): bool const curnum: number = line('.') if curnum <= cursor @@ -512,18 +393,12 @@ func RunTest() return AssertCursorForwardProgress() enddef END - let MAX_ABORTED_COUNT = 5 let MAX_FAILED_COUNT = 5 - let DUMP_OPTS = extend( - \ exists("$VIM_SYNTAX_TEST_WAIT_TIME") && + let DUMP_OPTS = exists("$VIM_SYNTAX_TEST_WAIT_TIME") && \ !empty($VIM_SYNTAX_TEST_WAIT_TIME) - \ ? {'wait': max([1, str2nr($VIM_SYNTAX_TEST_WAIT_TIME)])} - \ : {}, - \ {'FileComparisonPreAction': - \ function('g:ScreenDumpDiscardFFFDChars'), - \ 'NonEqualLineComparisonPostAction': - \ function('g:ScreenDumpLookForFFFDChars')}) - lockvar DUMP_OPTS MAX_FAILED_COUNT MAX_ABORTED_COUNT XTESTSCRIPT + \ ? {'wait': max([1, str2nr($VIM_SYNTAX_TEST_WAIT_TIME)])} + \ : {} + lockvar DUMP_OPTS MAX_FAILED_COUNT XTESTSCRIPT let ok_count = 0 let disused_pages = [] let failed_tests = [] @@ -591,107 +466,53 @@ func RunTest() let args = !empty(path) \ ? prefix .. ' -S ' .. path \ : prefix - let fail = 0 + let buf = RunVimInTerminal(args, {}) + " edit the file only after catching the SwapExists event + call term_sendkeys(buf, ":edit " .. fname .. "\") + " set up the testing environment + call term_sendkeys(buf, ":call SetUpVim()\") + " load filetype specific settings + call term_sendkeys(buf, ":call LoadFiletype('" .. filetype .. "')\") + + " Make a synchronisation point between buffers by requesting to echo + " a known token in the terminal buffer and asserting its availability + " with "s:TermWaitAndPollRuler()". + if filetype == 'sh' + call term_sendkeys(buf, ":call ShellInfo()\") + else + call term_sendkeys(buf, ":echo 'is_nonce'\") + endif + let root_00 = root .. '_00' + let in_name_and_out_name = fname .. ': failed/' .. root_00 .. '.dump' + " Queue up all "term_sendkeys()"es and let them finish before returning + " from "s:TermWaitAndPollRuler()". + let ruler = s:TermWaitAndPollRuler(buf, in_name_and_out_name) + call ch_log('First screendump for ' .. in_name_and_out_name) + " Make a screendump at the start of the file: failed/root_00.dump + let fail = VerifyScreenDump(buf, root_00, DUMP_OPTS) + let page_nr = 0 + + " Accommodate the next code block to "buf"'s contingency for self + " wipe-out. try - let aborted_count = MAX_ABORTED_COUNT - let collected_count = 0 - let seen_pages = [] - let page_quota = {} - - " See #16559. For each processed page, repeat pre-verification steps - " from scratch (subject to page cacheing) whenever U+FFFD characters - " are found in the terminal buffer with "term_getline()", i.e. treat - " these pages as if they were distinct test files. U+FFFD characters - " found at the last attempt (see "MAX_ABORTED_COUNT") will be ignored - " and "VerifyScreenDump()" will take over with own filtering. - while aborted_count > 0 - let buf = RunVimInTerminal(args, {}) - try - " edit the file only after catching the SwapExists event - call term_sendkeys(buf, ":edit " .. fname .. "\") - " set up the testing environment - call term_sendkeys(buf, ":call SetUpVim()\") - " load filetype specific settings - call term_sendkeys(buf, ":call LoadFiletype('" .. filetype .. "')\") - - " Collect all *non-spurious* U+FFFD characters for scrutiny. - if aborted_count == 1 && collected_count != 1 - let collected_count = 1 - call term_sendkeys(buf, ":call CollectFFFDChars()\") - endif - - " Make a synchronisation point between a terminal buffer and - " another buffer by requesting to echo a known token in the former - " and asserting its availability with "s:TermWaitAndPollRuler()" - " from the latter. - if filetype == 'sh' - call term_sendkeys(buf, ":call ShellInfo()\") - else - call term_sendkeys(buf, ":echo 'is_nonce'\") - endif - - let page_nr = 0 - let root_00 = root .. '_00' - let in_name_and_out_name = fname .. ': failed/' .. root_00 .. '.dump' - " Queue up all "term_sendkeys()"es and let them finish before - " returning from "s:TermWaitAndPollRuler()". - let ruler = s:TermWaitAndPollRuler(buf, in_name_and_out_name) - if index(seen_pages, page_nr) < 0 - let fail += s:VerifyScreenDumpOrThrowFFFD( - \ buf, - \ 'First', - \ in_name_and_out_name, - \ aborted_count, - \ MAX_ABORTED_COUNT, - \ root_00, - \ DUMP_OPTS, - \ page_quota, - \ seen_pages, - \ page_nr) - " Reset "aborted_count" for another page. - let aborted_count = MAX_ABORTED_COUNT - endif - let keys_a = ":call ScrollToSecondPage((18 * 75 + 1), 19, 5) | redraw!\" - let keys_b = ":call ScrollToNextPage((18 * 75 + 1), 19, 5) | redraw!\" - - while s:CannotSeeLastLine(ruler) - call term_sendkeys(buf, keys_a) - let keys_a = keys_b - let page_nr += 1 - let root_next = printf('%s_%02d', root, page_nr) - let in_name_and_out_name = fname .. ': failed/' .. root_next .. '.dump' - let ruler = s:TermPollRuler( - \ function('s:CannotDumpNextPage', [buf, ruler]), - \ buf, - \ in_name_and_out_name) - if index(seen_pages, page_nr) < 0 - let fail += s:VerifyScreenDumpOrThrowFFFD( - \ buf, - \ 'Next', - \ in_name_and_out_name, - \ aborted_count, - \ MAX_ABORTED_COUNT, - \ root_next, - \ DUMP_OPTS, - \ page_quota, - \ seen_pages, - \ page_nr) - " Reset "aborted_count" for another page. - let aborted_count = MAX_ABORTED_COUNT - endif - endwhile - call StopVimInTerminal(buf) - break - catch /^FFFD$/ - " Clear out. - call StopVimInTerminal(buf) - while winnr('$') > 1 - close - endwhile - let aborted_count -= 1 - endtry + let keys_a = ":call ScrollToSecondPage((18 * 75 + 1), 19, 5) | redraw!\" + let keys_b = ":call ScrollToNextPage((18 * 75 + 1), 19, 5) | redraw!\" + while s:CannotSeeLastLine(ruler) + call term_sendkeys(buf, keys_a) + let keys_a = keys_b + let page_nr += 1 + let root_next = printf('%s_%02d', root, page_nr) + let in_name_and_out_name = fname .. ': failed/' .. root_next .. '.dump' + let ruler = s:TermPollRuler( + \ function('s:CannotDumpNextPage', [buf, ruler]), + \ buf, + \ in_name_and_out_name) + call ch_log('Next screendump for ' .. in_name_and_out_name) + " Make a screendump of every 18 lines of the file: failed/root_NN.dump + let fail += VerifyScreenDump(buf, root_next, DUMP_OPTS) endwhile + call StopVimInTerminal(buf) finally call delete('Xtestscript') endtry diff --git a/src/testdir/util/screendump.vim b/src/testdir/util/screendump.vim index d9e9073798..c881eb9636 100644 --- a/src/testdir/util/screendump.vim +++ b/src/testdir/util/screendump.vim @@ -105,7 +105,6 @@ enddef " some dictionary with some state entries; " the file contents of the newly generated screen dump; " the zero-based number of the line whose copies are not equal. -" (See an example in runtime/syntax/testdir/runtest.vim.) " " The file name used is "dumps/{filename}.dump". " -- 2.47.3