Allow the subsitute character (codepoint 26 - 0x1A) to appear in fts5 barewords.

FossilOrigin-Name: baec1b96cb64a6a6317143a8df841e502afe3914

diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c
index 0cbade21909b412659e9cc79238e0bf14226ac06..e7e8d6d663c4bef65f864a7d4a818e9061931c83 100644 (file)
--- a/ext/fts5/fts5_buffer.c
+++ b/ext/fts5/fts5_buffer.c
@@ -292,11 +292,12 @@ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){
 **   * The 52 upper and lower case ASCII characters, and
 **   * The 10 integer ASCII characters.
 **   * The underscore character "_" (0x5F).
+**   * The unicode "subsitute" character (0x1A).
 */
 int sqlite3Fts5IsBareword(char t){
   u8 aBareword[128] = {
     0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x00 .. 0x0F */
-    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x10 .. 0x1F */
+    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 1, 0, 0, 0, 0, 0,   /* 0x10 .. 0x1F */
     0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x20 .. 0x2F */
     1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 0, 0, 0, 0, 0, 0,   /* 0x30 .. 0x3F */
     0, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1,   /* 0x40 .. 0x4F */

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index e1d13f1215a5a39bee075e0d5f64647afe5dfce8..5b2914ab49f3c7ef6f15907114e0aa0ed2052cf6 100644 (file)
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4687,10 +4687,10 @@ int sqlite3Fts5IterPoslist(
 */
 int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){
   Fts5Index *p = pIter->pIndex;
-
+  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
   assert( p->rc==SQLITE_OK );
   fts5BufferZero(pBuf);
-  fts5MultiIterPoslist(p, pIter, 0, 0, pBuf);
+  fts5SegiterPoslist(p, pSeg, 0, pBuf);
   return fts5IndexReturn(p);
 }
 

diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test
index 77407f5e2c6ba46a193ee5a6ea898219ec867096..67fc494435cc65db61145ed3f1a9b890b48d0223 100644 (file)
--- a/ext/fts5/test/fts5simple.test
+++ b/ext/fts5/test/fts5simple.test
@@ -251,6 +251,36 @@ do_execsql_test 10.1 {
   SELECT rowid FROM t3('c: c*');
 } {2}
 
+#-------------------------------------------------------------------------
+# Test that character 0x1A is allowed in fts5 barewords.
+#
+do_test 11.0 {
+  execsql "CREATE VIRTUAL TABLE t4 USING fts5(x, tokenize=\"ascii tokenchars '\x1A'\")"
+  execsql "
+    INSERT INTO t4 VALUES('a b c \x1A');
+    INSERT INTO t4 VALUES('a b c d\x1A');
+    INSERT INTO t4 VALUES('a b c \x1Ad');
+    INSERT INTO t4 VALUES('a b c d');
+  "
+} {}
+
+do_test 11.1 {
+  execsql "SELECT rowid FROM t4('\x1A')"
+} {1}
+do_test 11.2 {
+  execsql "SELECT rowid FROM t4('\x1A*')"
+} {1 3}
+do_test 11.3 {
+  execsql "SELECT rowid FROM t4('d\x1A')"
+} {2}
+
+do_test 11.4 {
+  catchsql "SELECT rowid FROM t4('d\x1B')"
+} {/fts5: syntax error/}
+do_test 11.5 {
+  catchsql "SELECT rowid FROM t4('d\x19')"
+} {/fts5: syntax error/}
+
 
 finish_test
 

diff --git a/manifest b/manifest
index de5762bf9dcf5dc01e4be4550ec8d7a5aa96ff35..364d23070bb73408985d0def9cfd984e4daf838b 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Adjustments\sto\sspellfix2.test\sso\sthat\sit\sworks\sreliably\son\smac.
-D 2015-10-07T16:14:18.747
+C Allow\sthe\ssubsitute\scharacter\s(codepoint\s26\s-\s0x1A)\sto\sappear\sin\sfts5\sbarewords.
+D 2015-10-07T17:01:22.560
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 2143eeef6d0cc26006ae5fc4bb242a4a8b973412
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -108,11 +108,11 @@ F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 98f802fe41481f9d797fce496f0fefcad72c7782
 F ext/fts5/fts5Int.h ed6c05b803e0bacf85228a8d255853e89796f6f5
 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e
-F ext/fts5/fts5_buffer.c 195f6f05599129aa8a2d88257f474c39edbc27c8
+F ext/fts5/fts5_buffer.c b2fb69c1ee3378956c0d9ee964d61b59d296afaf
 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695
 F ext/fts5/fts5_expr.c 2054e550e75cffa117557c9416210c425934436d
 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
-F ext/fts5/fts5_index.c 3bc8a522f2e0c7e588dfc6da78dd11a5869005e3
+F ext/fts5/fts5_index.c 6b818dd5b68237d5d0f7723795a329f0fb2e9987
 F ext/fts5/fts5_main.c fe5243d6bbb79217394f0ec7f4f5199ddbc9e7e8
 F ext/fts5/fts5_storage.c df061a5caf9e50fbbd43113009b5b248362f4995
 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37
@@ -175,7 +175,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821
-F ext/fts5/test/fts5simple.test 84d22123e0a7584f1ffb6efcd37eee46f317ab90
+F ext/fts5/test/fts5simple.test 85bbb268e01d2e3527d70a7fa511ddc3bba2ccc0
 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
@@ -1390,7 +1390,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P f20f9f813f00cefcd427e506a09b0b973c782e3f
-R 5c72368b779571d25566b43efcab6857
-U drh
-Z 9286de7475387911171425dc1c43ba95
+P d591e860d3d7c61637b4b07989bf176cb337db2c
+R 4054948498db600c620d7cf47f4149f2
+U dan
+Z db95020d457f0e85944d65c1e34b0155

diff --git a/manifest.uuid b/manifest.uuid
index fbc538a962bfa37c1edbf1983631bf3ccdcce538..08cc59d84d3b9622f041a78fa0393608318b6d15 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d591e860d3d7c61637b4b07989bf176cb337db2c
\ No newline at end of file
+baec1b96cb64a6a6317143a8df841e502afe3914
\ No newline at end of file