patch 9.1.2134: Terminal doesn't handle split UTF-8 sequence after ASCII

Problem:  Terminal doesn't handle split UTF-8 sequence after ASCII.
Solution: Only use one UTF-8 encoding state per vterm state (zeertzjq).

fixes:  #16559
closes: #18640

Signed-off-by: zeertzjq <zeertzjq@outlook.com>
Signed-off-by: Christian Brabandt <cb@256bit.org>

diff --git a/runtime/doc/version9.txt b/runtime/doc/version9.txt
index 5b43fa91bf8da2e73b6875aa0edcd0a003fbbe37..73083398b31e0f1a8b4d86acbf925e8248cda767 100644 (file)
--- a/runtime/doc/version9.txt
+++ b/runtime/doc/version9.txt
@@ -52486,4 +52486,8 @@ Patch 9.1.2133
 Problem:  Another case of buffer overflow with 'helpfile'.
 Solution: Leave room for "tags" in the buffer (zeertzjq).
 
+Patch 9.1.2134
+Problem:  Terminal doesn't handle split UTF-8 sequence after ASCII.
+Solution: Only use one UTF-8 encoding state per vterm state (zeertzjq).
+
  vim:tw=78:ts=8:noet:ft=help:norl:fdm=manual:nofoldenable

diff --git a/src/libvterm/src/state.c b/src/libvterm/src/state.c
index ee36ad4c2d388807cab42724b7d65d7ea0f714f5..aff97b076893e2ac0ddf5fa5c3e631909378dd49 100644 (file)
--- a/src/libvterm/src/state.c
+++ b/src/libvterm/src/state.c
@@ -297,6 +297,8 @@ static int on_text(const char bytes[], size_t len, void *user)
     !(bytes[eaten] & 0x80) ? &state->encoding[state->gl_set] :
     state->vt->mode.utf8   ? &state->encoding_utf8 :
                              &state->encoding[state->gr_set];
+  if (encoding->enc == state->encoding_utf8.enc)
+    encoding = &state->encoding_utf8;  // Only use one UTF-8 encoding state.
 
   (*encoding->enc->decode)(encoding->enc, encoding->data,
       codepoints, &npoints, state->gsingle_set ? 1 : (int)maxpoints,

diff --git a/src/libvterm/t/14state_encoding.test b/src/libvterm/t/14state_encoding.test
index b1f5d6985305c8217e8bdd68585331b6236064f2..2fd90e4386dc8bcbb344195187eab0500e8a2901 100644 (file)
--- a/src/libvterm/t/14state_encoding.test
+++ b/src/libvterm/t/14state_encoding.test
@@ -103,3 +103,12 @@ PUSH "AB\xc4\x88D"
   putglyph 0x0042 1 0,1
   putglyph 0x0108 1 0,2
   putglyph 0x0044 1 0,3
+
+!Split UTF-8 after US-ASCII
+RESET
+PUSH "AB\xc4"
+  putglyph 0x0041 1 0,0
+  putglyph 0x0042 1 0,1
+PUSH "\x88D"
+  putglyph 0x0108 1 0,2
+  putglyph 0x0044 1 0,3

diff --git a/src/testdir/test_terminal3.vim b/src/testdir/test_terminal3.vim
index ff8f8c206bcad5489056365c6f224290ad7fd5ac..b637a665282f6ee70dca22ae7d97b18d65ff613d 100644 (file)
--- a/src/testdir/test_terminal3.vim
+++ b/src/testdir/test_terminal3.vim
@@ -1124,4 +1124,28 @@ func Test_terminal_backspace_on_windows()
   let $PROMPT = save_prompt
 endfunc
 
+func Test_terminal_split_utf8()
+  CheckUnix
+
+  let buf = term_start('cat', {})
+  let chan = buf->term_getjob()->job_getchannel()
+  call ch_sendraw(chan, "1: \xc3")
+  call WaitForAssert({-> assert_equal('1: ', term_getline(buf, 1))})
+  call ch_sendraw(chan, "\xa5\xcc\xb2\n")
+  call WaitForAssert({-> assert_equal('1: å̲', term_getline(buf, 1))})
+  call WaitForAssert({-> assert_equal('1: å̲', term_getline(buf, 2))})
+  call ch_sendraw(chan, "2: \xc3\xa5")
+  call WaitForAssert({-> assert_equal('2: å', term_getline(buf, 3))})
+  call ch_sendraw(chan, "\xcc\xb2\n")
+  call WaitForAssert({-> assert_equal('2: å̲', term_getline(buf, 3))})
+  call WaitForAssert({-> assert_equal('2: å̲', term_getline(buf, 4))})
+  call ch_sendraw(chan, "3: \xc3\xa5\xcc")
+  call WaitForAssert({-> assert_equal('3: å', term_getline(buf, 5))})
+  call ch_sendraw(chan, "\xb2\n")
+  call WaitForAssert({-> assert_equal('3: å̲', term_getline(buf, 5))})
+  call WaitForAssert({-> assert_equal('3: å̲', term_getline(buf, 6))})
+
+  exe buf .. "bwipe!"
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab

diff --git a/src/version.c b/src/version.c
index fea5c51edfe7ac056173e56183810a16f674dbb7..a9967e2bb5aea06fb65ea7fbdf5247e4a6c3224d 100644 (file)
--- a/src/version.c
+++ b/src/version.c
@@ -734,6 +734,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    2134,
 /**/
     2133,
 /**/