runtime(syntax-tests): Filter out non-Latin-1 characters for syntax tests (#14767)

Syntax tests are run with the LC_ALL=C environment variable
passed to "make".  Occasionally, there are CI failures for
such test files containing non-Latin-1 characters with error
messages pointing to multi-byte characters:

https://github.com/vim/vim/actions/runs/8824925004/job/24228298023#step:10:16370 ,
https://github.com/vim/vim/actions/runs/8840856619/job/24276935260#step:10:16347 ,
https://github.com/vim/vim/actions/runs/8854043458/job/24316210645#step:10:16362 ,
https://github.com/vim/vim/actions/runs/8856501136/job/24322848765#step:10:16354 ,
https://github.com/vim/vim/actions/runs/9038417238/job/24839482152#step:11:16980 .

But since the very same unchanged tests pass at other times:

https://github.com/vim/vim/actions/runs/8827593571/job/24235935458#step:10:16353 ,
https://github.com/vim/vim/actions/runs/9065214647/job/24905321661#step:11:17002 ;

these failures are unrelated to the nature of syntax tests
and should be considered false positives.

As a temporary workaround, all bytes of known non-Latin-1
characters can be replaced in memory with an arbitrary ASCII
byte (?) by applying a filter

> " To ignore part of the dump, provide a "dumps/{filename}.vim" file with
> " Vim commands to be applied to both the reference and the current dump, so
> " that parts that are irrelevant are not used for the comparison.  The result
> " is NOT written, thus "term_dumpdiff()" shows the difference anyway.

before lines are compared between files.

Signed-off-by: Aliaksei Budavei <0x000c70@gmail.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>

diff --git a/Filelist b/Filelist
index 771854812fbeb9735720a5fdf4b2312328c90df8..cd787ef64d320aca4a4024bd2cbf80088732420e 100644 (file)
--- a/Filelist
+++ b/Filelist
@@ -834,6 +834,7 @@ RT_SCRIPTS =        \
                runtime/syntax/testdir/input/*.* \
                runtime/syntax/testdir/input/setup/*.* \
                runtime/syntax/testdir/dumps/*.dump \
+               runtime/syntax/testdir/dumps/*.vim \
                runtime/syntax/generator/Makefile \
                runtime/syntax/generator/README.md \
                runtime/syntax/generator/gen_syntax_vim.vim \

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent2_00.vim b/runtime/syntax/testdir/dumps/java_methods_indent2_00.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent2_00.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent2_01.vim b/runtime/syntax/testdir/dumps/java_methods_indent2_01.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent2_01.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent2_02.vim b/runtime/syntax/testdir/dumps/java_methods_indent2_02.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent2_02.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent2_03.vim b/runtime/syntax/testdir/dumps/java_methods_indent2_03.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent2_03.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent2_04.vim b/runtime/syntax/testdir/dumps/java_methods_indent2_04.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent2_04.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent2_99.vim b/runtime/syntax/testdir/dumps/java_methods_indent2_99.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent2_99.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent4_00.vim b/runtime/syntax/testdir/dumps/java_methods_indent4_00.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent4_00.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent4_01.vim b/runtime/syntax/testdir/dumps/java_methods_indent4_01.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent4_01.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent4_02.vim b/runtime/syntax/testdir/dumps/java_methods_indent4_02.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent4_02.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent4_03.vim b/runtime/syntax/testdir/dumps/java_methods_indent4_03.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent4_03.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent4_04.vim b/runtime/syntax/testdir/dumps/java_methods_indent4_04.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent4_04.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent4_99.vim b/runtime/syntax/testdir/dumps/java_methods_indent4_99.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent4_99.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent8_00.vim b/runtime/syntax/testdir/dumps/java_methods_indent8_00.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent8_00.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent8_01.vim b/runtime/syntax/testdir/dumps/java_methods_indent8_01.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent8_01.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent8_02.vim b/runtime/syntax/testdir/dumps/java_methods_indent8_02.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent8_02.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent8_03.vim b/runtime/syntax/testdir/dumps/java_methods_indent8_03.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent8_03.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent8_04.vim b/runtime/syntax/testdir/dumps/java_methods_indent8_04.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent8_04.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_indent8_99.vim b/runtime/syntax/testdir/dumps/java_methods_indent8_99.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_indent8_99.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_style_00.vim b/runtime/syntax/testdir/dumps/java_methods_style_00.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_style_00.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_style_01.vim b/runtime/syntax/testdir/dumps/java_methods_style_01.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_style_01.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_style_02.vim b/runtime/syntax/testdir/dumps/java_methods_style_02.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_style_02.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_style_03.vim b/runtime/syntax/testdir/dumps/java_methods_style_03.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_style_03.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_style_04.vim b/runtime/syntax/testdir/dumps/java_methods_style_04.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_style_04.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/java_methods_style_99.vim b/runtime/syntax/testdir/dumps/java_methods_style_99.vim
new file mode 100644 (file)
index 0000000..1509558
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/java_methods_style_99.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[ƒɐɘʬʭΑ-Ωα-ω�]+?+Ige

diff --git a/runtime/syntax/testdir/dumps/vim9_keymap_01.vim b/runtime/syntax/testdir/dumps/vim9_keymap_01.vim
new file mode 100644 (file)
index 0000000..546837c
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/vim9_keymap_01.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[“�]+?+ge

diff --git a/runtime/syntax/testdir/dumps/vim9_keymap_99.vim b/runtime/syntax/testdir/dumps/vim9_keymap_99.vim
new file mode 100644 (file)
index 0000000..546837c
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/vim9_keymap_99.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[“�]+?+ge

diff --git a/runtime/syntax/testdir/dumps/vim_keymap_99.vim b/runtime/syntax/testdir/dumps/vim_keymap_99.vim
new file mode 100644 (file)
index 0000000..546837c
--- /dev/null
+++ b/runtime/syntax/testdir/dumps/vim_keymap_99.vim
@@ -0,0 +1,2 @@
+" Replace known non-Latin-1 characters.
+%s+[“�]+?+ge