Add extra test for handling of embedded nul characters in the fts4 unicode61 tokenizer.

FossilOrigin-Name: c2c2c7e945f5d5700d91b8e779117e70e388ffc613912a434885ae27f5fe4e22

diff --git a/manifest b/manifest
index 92e502e71ec081b85a4fddd233661b713966377a..eee9dbcaa68f5407ab845447da830aa6d4431c66 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sLemon,\sfactor\sthe\sparser\sstack\soverflow\sdetection\slogic\sout\sof\sthe\nyy_reduce()\ssubroutine\sand\sinto\sthe\smain\sparser\sroutine,\sso\sthat\swhen\soverflow\nis\sdetected,\sit\scan\sexit\simmediately.\s\sThis\ssaves\sa\ssingle\sconditional\sin\nthe\smain\sloop\sof\sthe\sparser.
-D 2021-01-02T23:56:37.261
+C Add\sextra\stest\sfor\shandling\sof\sembedded\snul\scharacters\sin\sthe\sfts4\sunicode61\stokenizer.
+D 2021-01-04T18:28:29.171
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -1018,7 +1018,7 @@ F test/fts4opt.test 0fd0cc84000743ff2a883b9b84b4a5be07249f0ba790c8848a757164cdd4
 F test/fts4record.test a48508f69a84c9287c8019d3a1ae712f5730d8335ffaf8e2101e691d078950bb
 F test/fts4rename.test 15fd9985c2bce6dea20da2245b22029ec89bd4710ed317c4c53abbe3cfd0c880
 F test/fts4umlaut.test fcaca4471de7e78c9d1f7e8976e3e8704d7d8ad979d57a739d00f3f757380429
-F test/fts4unicode.test ceca76422abc251818cb25dabe33d3c3970da5f7c90e1540f190824e6b3a7c95
+F test/fts4unicode.test 82a9c16b68ba2f358a856226bb2ee02f81583797bc4744061c54401bf1a0f4c9
 F test/fts4upfrom.test 8df5acb6e10ad73f393d1add082b042ab1db72567888847d098152121e507b34
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test f673822636fb8ed618dd2b80230d16e495d19c8f2e2e7d6c22e93e2b3de097ad
@@ -1895,7 +1895,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d01e9f2d00dc439c529cd8885a219fcddbaad73b9f471b020e2a0c18e2add69b
-R 30317f78042211a4eaf8fdf4d0539bb2
-U drh
-Z 16a40d4bd4688bf574b37b863bb32871
+P 203c049c662380411522d0c7c493201331bbb2792a7c5b12684f04f532a0695d
+R 13e46d62ad91a9f054084b2f8c17f303
+U dan
+Z 21909ae0d43a0fafde3bfee513c0e7b6

diff --git a/manifest.uuid b/manifest.uuid
index 076424831b8025c34cefa96b0464cf124de2c82d..b74500a2daa81c313d25060d7226d631065544d2 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-203c049c662380411522d0c7c493201331bbb2792a7c5b12684f04f532a0695d
\ No newline at end of file
+c2c2c7e945f5d5700d91b8e779117e70e388ffc613912a434885ae27f5fe4e22
\ No newline at end of file

diff --git a/test/fts4unicode.test b/test/fts4unicode.test
index f4d1a33486ffb9a3b0c8e837e50582638c970863..facf2bf9c41b20bab9f36983c959ee91d7ecd513 100644 (file)
--- a/test/fts4unicode.test
+++ b/test/fts4unicode.test
@@ -567,4 +567,22 @@ do_execsql_test 11.1 {
   berlin@street sydney.road
 }
 
+# Test for embedded nul characters in fts4 unicode index.
+#
+do_execsql_test 12.0 {
+  CREATE VIRTUAL TABLE t12 USING fts4(tokenize=unicode61);
+  INSERT INTO t12 VALUES('abc' || char(0) || 'def');
+  SELECT hex(CAST(content AS blob)) FROM t12;
+} {61626300646566}
+do_execsql_test 12.1 {
+  INSERT INTO t12(t12) VALUES('integrity-check');
+} {}
+do_execsql_test 12.2 { 
+  CREATE VIRTUAL TABLE t12aux USING fts4aux(t12);
+  SELECT * FROM t12aux;
+} {abc * 1 1 abc 0 1 1}
+do_execsql_test 12.3 { 
+  SELECT hex(CAST(content AS blob)) FROM t12 WHERE t12 MATCH 'abc'
+} {61626300646566}
+
 finish_test