Fix a problem with fts5 synonyms and phrase queries. Also fix an OOM handling bug...

FossilOrigin-Name: a4c35fa2c94fe34b376670244fe72303c99868c1

diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index cc24179561bd727fb7fe4f556faafdd875ac56ee..8796ba5e2dbd00bd5a8f5dd1628f106cc1660fbd 100644 (file)
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -288,8 +288,8 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){
 ** Argument pTerm must be a synonym iterator. Return the current rowid
 ** that it points to.
 */
-static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){
-  i64 iRet;
+static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
+  i64 iRet = 0;
   int bRetValid = 0;
   Fts5ExprTerm *p;
 
@@ -305,7 +305,7 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){
     }
   }
 
-  assert( bRetValid );
+  if( pbEof && bRetValid==0 ) *pbEof = 1;
   return iRet;
 }
 
@@ -346,9 +346,9 @@ static int fts5ExprSynonymPoslist(
         nAlloc = nAlloc*2;
         aIter = aNew;
       }
-      if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){
-        nIter++;
-      }
+      sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter]);
+      assert( aIter[nIter].bEof==0 );
+      nIter++;
     }
   }
 
@@ -659,7 +659,7 @@ static int fts5ExprNearAdvanceFirst(
     Fts5ExprTerm *p;
 
     /* Find the firstest rowid any synonym points to. */
-    i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc);
+    i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0);
 
     /* Advance each iterator that currently points to iRowid. Or, if iFrom
       ** is valid - each iterator that points to a rowid before iFrom.  */
@@ -738,6 +738,35 @@ static int fts5ExprAdvanceto(
   return 0;
 }
 
+static int fts5ExprSynonymAdvanceto(
+  Fts5ExprTerm *pTerm,            /* Term iterator to advance */
+  int bDesc,                      /* True if iterator is "rowid DESC" */
+  i64 *piLast,                    /* IN/OUT: Lastest rowid seen so far */
+  int *pRc,                       /* OUT: Error code */
+  int *pbEof                      /* OUT: Set to true if EOF */
+){
+  int rc = SQLITE_OK;
+  i64 iLast = *piLast;
+  Fts5ExprTerm *p;
+
+  for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){
+    if( sqlite3Fts5IterEof(p->pIter)==0 ){
+      i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
+      if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
+        rc = sqlite3Fts5IterNextFrom(p->pIter, iLast);
+      }
+    }
+  }
+
+  if( rc!=SQLITE_OK ){
+    *pbEof = 1;
+  }else{
+    *piLast = fts5ExprSynonymRowid(pTerm, bDesc, pbEof);
+  }
+
+  return rc;
+}
+
 /*
 ** IN/OUT parameter (*pa) points to a position list n bytes in size. If
 ** the position list contains entries for column iCol, then (*pa) is set
@@ -906,7 +935,7 @@ static int fts5ExprNearNextMatch(
   ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
   ** means the minimum rowid.  */
   if( pLeft->aTerm[0].pSynonym ){
-    iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc);
+    iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0);
   }else{
     iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter);
   }
@@ -920,21 +949,13 @@ static int fts5ExprNearNextMatch(
         if( pTerm->pSynonym ){
           Fts5ExprTerm *p;
           int bEof = 1;
-          i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc);
+          i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
           if( iRowid==iLast ) continue;
-          for(p=pTerm; p; p=p->pSynonym){
-            Fts5IndexIter *pIter = p->pIter;
-            int dummy;
-            if( 0==sqlite3Fts5IterEof(pIter)
-             && 0==fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &dummy)==0 
-            ){
-              bEof = 0;
-            }
-          }
-          if( bEof || rc ){
-            pNode->bEof = 1;
+          bMatch = 0;
+          if( fts5ExprSynonymAdvanceto(pTerm,bDesc,&iLast,&rc,&pNode->bEof) ){
             return rc;
           }
+
         }else{
           Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
           i64 iRowid = sqlite3Fts5IterRowid(pIter);

diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test
index b92df3ce79bb00c0915be1b4d4e9c7be44db6ab0..b739e874185cacde8aaa430f2f64836fce6bcf89 100644 (file)
--- a/ext/fts5/test/fts5fault6.test
+++ b/ext/fts5/test/fts5fault6.test
@@ -22,6 +22,8 @@ ifcapable !fts5 {
   return
 }
 
+if 1 {
+
 #-------------------------------------------------------------------------
 # OOM while rebuilding an FTS5 table.
 #
@@ -148,8 +150,14 @@ do_faultsim_test 4.1 -faults oom-t* -prep {
   faultsim_test_result {0 {}}
 }
 
+}
+
 #-------------------------------------------------------------------------
-# OOM while running a query that includes synonyms and matchinfo().
+#
+# 5.2.* OOM while running a query that includes synonyms and matchinfo().
+#
+# 5.3.* OOM while running a query that returns a row containing instances
+#       of more than 4 synonyms for a single term.
 #
 proc mit {blob} {
   set scan(littleEndian) i*
@@ -174,6 +182,7 @@ db func mit mit
 sqlite3_fts5_register_matchinfo db
 do_test 5.0 {
   execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl) }
+  execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) }
   foreach {rowid text} {
     1 {aaaa cc b aaaaa cc aa} 
     2 {aa aa bb a bbb}
@@ -184,6 +193,37 @@ do_test 5.0 {
     7 {aaaaa aa aa ccccc bb}
     8 {ccc bbbbb ccccc bbb c}
     9 {cccccc bbbb a aaa cccc c}
+
+    20 {ddd f ddd eeeee fff ffff eeee ddd fff eeeee dddddd eeee}
+    21 {fffff eee dddd fffff dd ee ee eeeee eee eeeeee ee dd e}
+    22 {fffff d eeee dddd fffff dddddd ffff ddddd eeeee ee eee dddd ddddd}
+    23 {ddddd fff ddd eeeee ffff eeee ddd ff ff ffffff eeeeee dddd ffffff}
+    24 {eee dd ee dddd dddd eeeeee e eee fff ffff}
+    25 {ddddd ffffff dddddd fff ddd ddddd ddd f eeee fff dddd f}
+    26 {f ffff fff fff eeeeee dddd d dddddd ddddd eee ff eeeee}
+    27 {eee fff dddddd eeeee eeeee dddd ddddd ffff f eeeee eee dddddd ddddd d}
+    28 {dd ddddd d ddd d fff d dddd ee dddd ee ddd dddddd dddddd}
+    29 {eeee dddd ee dddd eeee dddd dd fffff f ddd eeeee ddd ee}
+    30 {ff ffffff eeeeee eeeee eee ffffff ff ffff f fffff eeeee}
+    31 {fffff eeeeee dddd eeee eeee eeeeee eee fffff d ddddd ffffff ffff dddddd}
+    32 {dddddd fffff ee eeeeee eeee ee fff dddd fff eeee ffffff eeeeee ffffff}
+    33 {ddddd eeee dd ffff dddddd fff eeee ddddd ffff eeee ddd}
+    34 {ee dddd ddddd dddddd eeee eeeeee f dd ee dddddd ffffff}
+    35 {ee dddd dd eeeeee ddddd eee d eeeeee dddddd eee dddd fffff}
+    36 {eee ffffff ffffff e fffff eeeee ff dddddd dddddd fff}
+    37 {eeeee fffff dddddd dddd ffffff fff f dd ee dd dd eeeee}
+    38 {eeeeee ee d ff eeeeee eeeeee eee eeeee ee ffffff dddd eeee dddddd ee}
+    39 {eeeeee ddd fffff e dddd ee eee eee ffffff ee f d dddd}
+    40 {ffffff dddddd eee ee ffffff eee eeee ddddd ee eeeeee f}
+    41 {ddd ddd fff fffff ee fffff f fff ddddd fffff}
+    42 {dddd ee ff d f ffffff fff ffffff ff dd dddddd f eeee}
+    43 {d dd fff fffff d f fff e dddd ee ee}
+    44 {ff ffff eee ddd d dd ffff dddd d eeee d eeeeee}
+    45 {eeee f eeeee ee e ffff f ddd e fff}
+    46 {ffff d ffff eeee ffff eeeee f ffff ddddd eee}
+    47 {dd dd dddddd ddddd fffff dddddd ddd ddddd eeeeee ffff eeee eee ee}
+    48 {ffff ffff e dddd ffffff dd dd dddd f fffff}
+    49 {ffffff d dddddd ffff eeeee f ffff ffff d dd fffff eeeee}
   } {
     execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) }
   }
@@ -196,9 +236,12 @@ set res [list {*}{
   7 {3 24 8 1 12 6}
   9 {2 24 8 3 12 6}
 }]
-do_execsql_test 5.1 {
+do_execsql_test 5.1.1 {
   SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
 } $res
+do_execsql_test 5.1.2 {
+  SELECT count(*) FROM t1 WHERE t1 MATCH 'd e f'
+} 29
 
 faultsim_save_and_close
 do_faultsim_test 5.2 -faults oom* -prep {
@@ -214,5 +257,16 @@ do_faultsim_test 5.2 -faults oom* -prep {
   faultsim_test_result [list 0 $::res]
 }
 
+do_faultsim_test 5.3 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  sqlite3_fts5_create_tokenizer db tcl tcl_create
+} -body {
+  db eval { 
+    SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f'
+  }
+} -test {
+  faultsim_test_result {0 29}
+}
+
 finish_test
 

diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test
index 6acc5617f06087658bafe319413b6722af13064a..33d20a014ea71d0c97fc6031bc481d6c66d8e014 100644 (file)
--- a/ext/fts5/test/fts5synonym.test
+++ b/ext/fts5/test/fts5synonym.test
@@ -93,6 +93,7 @@ foreach {tn expr res} {
   2 "eight OR 8 OR 5" {2 3}
   3 "10" {}
   4 "1*" {1}
+  5 "1 + 2" {1}
 } {
   do_execsql_test 2.1.$tn {
     SELECT rowid FROM ft WHERE ft MATCH $expr
@@ -233,6 +234,7 @@ do_test 5.0 {
   }
 } {}
 
+
 foreach {tn q res} {
   1 {one} {
     1 {four v 4 [i] three} {[1] 3 five five 4 [one]}
@@ -265,6 +267,23 @@ foreach {tn q res} {
     8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]}
     9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]}
   }
+
+  4 {5 + 1} {
+    2 {[5 1] 3 4 i} {2 2 v two 4} 
+    3 {[5 i] 5 2 four 4 1} {iii ii five two 1} 
+    4 {ii four 4 one 5 three five} {one [5 1] iii 4 3} 
+    5 {three i [v i] four 4 1} {ii five five five iii}
+  }
+
+  5 {one + two + three} {
+    7 {ii ii two three 2 5} {iii [i ii iii] iii one one}
+    8 {2 ii [i two 3] three 2} {two iv v iii 3 five}
+  }
+
+  6 {"v v"} {
+    1 {four v 4 i three} {1 3 [five five] 4 one}
+    5 {three i v i four 4 1} {ii [five five five] iii}
+  }
 } {
   do_execsql_test 5.1.$tn {
     SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']')

diff --git a/main.mk b/main.mk
index 9988fe2775031be43adbe1e091ae8c2bfc5c35ab..828a824eeeb8d7364aa160288dd77dc6fbb802b2 100644 (file)
--- a/main.mk
+++ b/main.mk
@@ -47,6 +47,7 @@
 TCCX =  $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) 
 TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3
 TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth
+TCCX += -I$(TOP)/ext/fts5
 
 # Object files for the SQLite library.
 #
@@ -230,6 +231,29 @@ SRC += \
   $(TOP)/ext/rbu/sqlite3rbu.h
 
 
+# FTS5 things
+#
+FTS5_HDR = \
+   $(TOP)/ext/fts5/fts5.h \
+   $(TOP)/ext/fts5/fts5Int.h \
+   fts5parse.h
+          
+FTS5_SRC = \
+   $(TOP)/ext/fts5/fts5_aux.c \
+   $(TOP)/ext/fts5/fts5_buffer.c \
+   $(TOP)/ext/fts5/fts5_main.c \
+   $(TOP)/ext/fts5/fts5_config.c \
+   $(TOP)/ext/fts5/fts5_expr.c \
+   $(TOP)/ext/fts5/fts5_hash.c \
+   $(TOP)/ext/fts5/fts5_index.c \
+   fts5parse.c \
+   $(TOP)/ext/fts5/fts5_storage.c \
+   $(TOP)/ext/fts5/fts5_tokenize.c \
+   $(TOP)/ext/fts5/fts5_unicode2.c \
+   $(TOP)/ext/fts5/fts5_varint.c \
+   $(TOP)/ext/fts5/fts5_vocab.c  \
+
+
 # Generated source code files
 #
 SRC += \
@@ -636,25 +660,6 @@ fts3_write.o:      $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
 rtree.o:       $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
        $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
 
-# FTS5 things
-#
-FTS5_SRC = \
-   $(TOP)/ext/fts5/fts5.h \
-   $(TOP)/ext/fts5/fts5Int.h \
-   $(TOP)/ext/fts5/fts5_aux.c \
-   $(TOP)/ext/fts5/fts5_buffer.c \
-   $(TOP)/ext/fts5/fts5_main.c \
-   $(TOP)/ext/fts5/fts5_config.c \
-   $(TOP)/ext/fts5/fts5_expr.c \
-   $(TOP)/ext/fts5/fts5_hash.c \
-   $(TOP)/ext/fts5/fts5_index.c \
-   fts5parse.c fts5parse.h \
-   $(TOP)/ext/fts5/fts5_storage.c \
-   $(TOP)/ext/fts5/fts5_tokenize.c \
-   $(TOP)/ext/fts5/fts5_unicode2.c \
-   $(TOP)/ext/fts5/fts5_varint.c \
-   $(TOP)/ext/fts5/fts5_vocab.c  \
-
 fts5parse.c:   $(TOP)/ext/fts5/fts5parse.y lemon 
        cp $(TOP)/ext/fts5/fts5parse.y .
        rm -f fts5parse.h
@@ -662,11 +667,10 @@ fts5parse.c:      $(TOP)/ext/fts5/fts5parse.y lemon
 
 fts5parse.h: fts5parse.c
 
-fts5.c: $(FTS5_SRC)
+fts5.c: $(FTS5_SRC) $(FTS5_HDR)
        tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl
        cp $(TOP)/ext/fts5/fts5.h .
 
-
 userauth.o:    $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
        $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c
 

diff --git a/manifest b/manifest
index 01f188aef69778ff81d213e34e170a4dce8128d2..30f63e1524f323fa4868517ce1dbb00c32c00afd 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\shandling\sOOM\sconditions\swithin\sfts5\squeries\sthat\sfeature\ssynonyms.
-D 2015-09-02T14:17:38.670
+C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sphrase\squeries.\sAlso\sfix\san\sOOM\shandling\sbug\sin\sfts5.
+D 2015-09-02T17:34:22.663
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in e2218eb228374422969de7b1680eda6864affcef
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b
 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e
 F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384
-F ext/fts5/fts5_expr.c 5ae9bae1a36583a1e85f742f972c65bffd7e9d3b
+F ext/fts5/fts5_expr.c 408384b2f540dcf72a363e1cde4c3e9896e8d41d
 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374
 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e
@@ -156,7 +156,7 @@ F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341
 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7
 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875
-F ext/fts5/test/fts5fault6.test 233a29d4a086cd09cfae262bbb15c709a438d010
+F ext/fts5/test/fts5fault6.test 93e9d43ab1e05ae5efa30cfde17649b3bb6a82c3
 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6
 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145
@@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460
-F ext/fts5/test/fts5synonym.test dd3c9016345e1bbd7384ec25bbcd3de1f1700475
+F ext/fts5/test/fts5synonym.test 88efc3de51a2c46aff984d74f06c833e4def0493
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
@@ -259,7 +259,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 04840e8277ab5159af16172eafd214dae7cffff5
+F main.mk 8a8468564b2e387116614fcda2a6d4f5f754b955
 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
 F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P cf3e45e76d23e10ee06296c3561a341591597a04
-R 0e89c166acdebcc1b36d66580652787d
+P 11fa980897c6c7be218bbd9c4cd8253272d2c300
+R 6df2cc151a0891be96a1f7bf13aeaf04
 U dan
-Z 84847a00fa2f61f76712fa9a6aa07879
+Z 72b6e5381768fc37d01c74eff21e6a40

diff --git a/manifest.uuid b/manifest.uuid
index 985c544ce9150fb34cf7fbacdbfe970f69bb2890..2675a32e17017abd42eff65662c20a57b979cbbf 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-11fa980897c6c7be218bbd9c4cd8253272d2c300
\ No newline at end of file
+a4c35fa2c94fe34b376670244fe72303c99868c1
\ No newline at end of file