From: dan <dan@noemail.net>
Date: Wed, 3 Jun 2015 11:23:30 +0000 (+0000)
Subject: Fix an fts5 problem in extracting columns from position lists containing large varints.
X-Git-Tag: version-3.8.11~114^2~17
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=6f277eb6bcaf42f939dc67774b1e108e090b6055;p=thirdparty%2Fsqlite.git

Fix an fts5 problem in extracting columns from position lists containing large varints.

FossilOrigin-Name: 4ea015ab983300d420ef104cca550b22a6395866
---

diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 24e2c95dc3..6af3b84f31 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -668,7 +668,7 @@ static int fts5ExprExtractCol(
   while( iCol!=iCurrent ){
     /* Advance pointer p until it points to pEnd or an 0x01 byte that is
     ** not part of a varint */
-    while( !(prev & 0x80) && *p!=0x01 ){
+    while( (prev & 0x80) || *p!=0x01 ){
       prev = *p++;
       if( p==pEnd ) return 0;
     }
@@ -678,7 +678,8 @@ static int fts5ExprExtractCol(
 
   /* Advance pointer p until it points to pEnd or an 0x01 byte that is
   ** not part of a varint */
-  while( p<pEnd && !(prev & 0x80) && *p!=0x01 ){
+  assert( (prev & 0x80)==0 );
+  while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){
     prev = *p++;
   }
   return p - (*pa);
@@ -710,65 +711,27 @@ static int fts5ExprNearTest(
 ){
   Fts5ExprNearset *pNear = pNode->pNear;
   int rc = *pRc;
+  int i;
 
-  if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){
-    /* If this "NEAR" object is actually a single phrase that consists 
-    ** of a single term only, then grab pointers into the poslist
-    ** managed by the fts5_index.c iterator object. This is much faster 
-    ** than synthesizing a new poslist the way we have to for more
-    ** complicated phrase or NEAR expressions.  */
-    Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
-    Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
-    Fts5ExprColset *pColset = pNear->pColset;
-    const u8 *pPos;
-    int nPos;
-
-    if( rc!=SQLITE_OK ) return 0;
-    rc = sqlite3Fts5IterPoslist(pIter, &pPos, &nPos, &pNode->iRowid);
-
-    /* If the term may match any column, then this must be a match. 
-    ** Return immediately in this case. Otherwise, try to find the
-    ** part of the poslist that corresponds to the required column.
-    ** If it can be found, return. If it cannot, the next iteration
-    ** of the loop will test the next rowid in the database for this
-    ** term.  */
-    if( pColset==0 ){
-      assert( pPhrase->poslist.nSpace==0 );
-      pPhrase->poslist.p = (u8*)pPos;
-      pPhrase->poslist.n = nPos;
-    }else if( pColset->nCol==1 ){
-      assert( pPhrase->poslist.nSpace==0 );
-      pPhrase->poslist.n = fts5ExprExtractCol(&pPos, nPos, pColset->aiCol[0]);
-      pPhrase->poslist.p = (u8*)pPos;
-    }else if( rc==SQLITE_OK ){
-      rc = fts5ExprExtractColset(pColset, pPos, nPos, &pPhrase->poslist);
-    }
-
-    *pRc = rc;
-    return (pPhrase->poslist.n>0);
-  }else{
-    int i;
-
-    /* Check that each phrase in the nearset matches the current row.
-    ** Populate the pPhrase->poslist buffers at the same time. If any
-    ** phrase is not a match, break out of the loop early.  */
-    for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
-      Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
-      if( pPhrase->nTerm>1 || pNear->pColset ){
-        int bMatch = 0;
-        rc = fts5ExprPhraseIsMatch(pExpr, pNear->pColset, pPhrase, &bMatch);
-        if( bMatch==0 ) break;
-      }else{
-        rc = sqlite3Fts5IterPoslistBuffer(
-            pPhrase->aTerm[0].pIter, &pPhrase->poslist
-        );
-      }
+  /* Check that each phrase in the nearset matches the current row.
+  ** Populate the pPhrase->poslist buffers at the same time. If any
+  ** phrase is not a match, break out of the loop early.  */
+  for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
+    Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
+    if( pPhrase->nTerm>1 || pNear->pColset ){
+      int bMatch = 0;
+      rc = fts5ExprPhraseIsMatch(pExpr, pNear->pColset, pPhrase, &bMatch);
+      if( bMatch==0 ) break;
+    }else{
+      rc = sqlite3Fts5IterPoslistBuffer(
+          pPhrase->aTerm[0].pIter, &pPhrase->poslist
+      );
     }
+  }
 
-    *pRc = rc;
-    if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
-      return 1;
-    }
+  *pRc = rc;
+  if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
+    return 1;
   }
 
   return 0;
@@ -939,12 +902,10 @@ static int fts5RowidCmp(
 }
 
 static void fts5ExprSetEof(Fts5ExprNode *pNode){
-  if( pNode ){
-    int i;
-    pNode->bEof = 1;
-    for(i=0; i<pNode->nChild; i++){
-      fts5ExprSetEof(pNode->apChild[i]);
-    }
+  int i;
+  pNode->bEof = 1;
+  for(i=0; i<pNode->nChild; i++){
+    fts5ExprSetEof(pNode->apChild[i]);
   }
 }
 
@@ -1562,34 +1523,26 @@ Fts5ExprColset *sqlite3Fts5ParseColset(
   Fts5Token *p
 ){
   Fts5ExprColset *pRet = 0;
+  int iCol;
+  char *z;                        /* Dequoted copy of token p */
 
+  z = sqlite3Fts5Strndup(&pParse->rc, p->p, p->n);
   if( pParse->rc==SQLITE_OK ){
-    int iCol;
-    char *z = 0;
-    int rc = fts5ParseStringFromToken(p, &z);
-    if( rc==SQLITE_OK ){
-      Fts5Config *pConfig = pParse->pConfig;
-      sqlite3Fts5Dequote(z);
-      for(iCol=0; iCol<pConfig->nCol; iCol++){
-        if( 0==sqlite3_stricmp(pConfig->azCol[iCol], z) ){
-          break;
-        }
-      }
-      if( iCol==pConfig->nCol ){
-        sqlite3Fts5ParseError(pParse, "no such column: %s", z);
-      }
-      sqlite3_free(z);
-    }else{
-      pParse->rc = rc;
+    Fts5Config *pConfig = pParse->pConfig;
+    sqlite3Fts5Dequote(z);
+    for(iCol=0; iCol<pConfig->nCol; iCol++){
+      if( 0==sqlite3_stricmp(pConfig->azCol[iCol], z) ) break;
     }
-
-    if( pParse->rc==SQLITE_OK ){
+    if( iCol==pConfig->nCol ){
+      sqlite3Fts5ParseError(pParse, "no such column: %s", z);
+    }else{
       pRet = fts5ParseColset(pParse, pColset, iCol);
     }
+    sqlite3_free(z);
   }
 
-  if( pParse->rc!=SQLITE_OK ){
-    assert( pRet==0 );
+  if( pRet==0 ){
+    assert( pParse->rc!=SQLITE_OK );
     sqlite3_free(pColset);
   }
 
@@ -1771,8 +1724,6 @@ static char *fts5ExprPrintTcl(
       if( zRet==0 ) return 0;
     }
 
-    if( zRet==0 ) return 0;
-
   }else{
     char const *zOp = 0;
     int i;
diff --git a/ext/fts5/test/fts5auto.test b/ext/fts5/test/fts5auto.test
index 52a54fc04e..771a0b64d8 100644
--- a/ext/fts5/test/fts5auto.test
+++ b/ext/fts5/test/fts5auto.test
@@ -232,33 +232,43 @@ do_execsql_test 1.0 {
 
 fts5_aux_test_functions db
 
-proc matchdata {expr {order ASC}} {
-  set tclexpr [db one {
+proc matchdata {expr tbl collist {order ASC}} {
+
+  set cols ""
+  foreach e $collist {
+    append cols ", '$e'"
+  }
+
+  set tclexpr [db one [subst -novar {
     SELECT fts5_expr_tcl(
-      $expr, 'nearset $cols -pc ::pc', 'a','b','c','d','e','f'
+      $expr, 'nearset $cols -pc ::pc' [set cols]
     )
-  }]
+  }]]
   set res [list]
 
-  db eval "SELECT rowid, * FROM tt ORDER BY rowid $order" {
-    set cols [list $a $b $c $d $e $f]
+  db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
+    set cols [list]
+    foreach col $x(*) {
+      if {$col != "rowid"} { lappend cols $x($col) }
+    }
+    # set cols [list $a $b $c $d $e $f]
     set ::pc 0
     set rowdata [eval $tclexpr]
-    if {$rowdata != ""} { lappend res $rowid $rowdata }
+    if {$rowdata != ""} { lappend res $x(rowid) $rowdata }
   }
 
   set res
 }
 
-proc do_auto_test {tn expr} { 
+proc do_auto_test {tn tbl cols expr} { 
   foreach order {asc desc} {
-    set res [matchdata $expr $order]
-    set testname "3.$tn.[string range $order 0 0].rows=[expr [llength $res]/2]"
+    set res [matchdata $expr $tbl $cols $order]
+    set testname "$tn.[string range $order 0 0].rows=[expr [llength $res]/2]"
 
     set ::autotest_expr $expr
     do_execsql_test $testname [subst -novar {
-      SELECT rowid, fts5_test_poslist(tt) FROM tt 
-      WHERE tt MATCH $::autotest_expr ORDER BY rowid [set order]
+      SELECT rowid, fts5_test_poslist([set tbl]) FROM [set tbl] 
+      WHERE [set tbl] MATCH $::autotest_expr ORDER BY rowid [set order]
     }] $res
   }
 
@@ -310,6 +320,7 @@ for {set fold 0} {$fold < 3} {incr fold} {
     A.7 { {c a b} : x + c }
     A.8 { {c d} : "l m" }
     A.9 { {c e} : "l m" }
+    A.10 { {a b c a b c a b c f f e} : "l m" }
 
     B.1 { a NOT b }
     B.2 { a NOT a:b }
@@ -321,9 +332,48 @@ for {set fold 0} {$fold < 3} {incr fold} {
     C.1 { a OR (b AND "b c") }
     C.2 { a OR (b AND "z c") }
   } {
-    do_auto_test 3.$fold.$tn $expr
+    do_auto_test 3.$fold.$tn tt {a b c d e f} $expr
+  }
+}
+
+proc replace_elems {list args} {
+  set ret $list
+  foreach {idx elem} $args {
+    set ret [lreplace $ret $idx $idx $elem]
+  }
+  set ret
+}
+
+#-------------------------------------------------------------------------
+#
+set bigdoc [string trim [string repeat "a " 1000]]
+do_test 4.0 {
+  set a [replace_elems $bigdoc  50 x  950 x]
+  set b [replace_elems $bigdoc  20 y   21 x  887 x 888 y]
+  set c [replace_elems $bigdoc   1 z  444 z  789 z]
+  execsql {
+    CREATE VIRTUAL TABLE yy USING fts5(c1, c2, c3);
+    INSERT INTO yy(rowid, c1, c2, c3) VALUES(-56789, $a, $b, $c);
+    INSERT INTO yy(rowid, c1, c2, c3) VALUES(250, $a, $b, $c);
   }
+} {}
+
+foreach {tn expr} {
+  1 x    
+  2 y    
+  3 z
+
+  4 {c1 : x} 5 {c2 : x} 6 {c3 : x}
+  7 {c1 : y} 8 {c2 : y} 9 {c3 : y}
+  10 {c1 : z} 11 {c2 : z} 12 {c3 : z}
+
+
+} {
+breakpoint
+  do_auto_test 4.$tn yy {c1 c2 c3} $expr
 }
 
+
+
 finish_test
 
diff --git a/ext/fts5/test/fts5fault4.test b/ext/fts5/test/fts5fault4.test
index 6a37fcffc1..f224df40cd 100644
--- a/ext/fts5/test/fts5fault4.test
+++ b/ext/fts5/test/fts5fault4.test
@@ -371,5 +371,33 @@ do_faultsim_test 13.1 -faults oom-t* -prep {
   faultsim_test_result {0 {a 1 1 b 1 1}} 
 }
 
+#-------------------------------------------------------------------------
+# OOM in multi-column token query.
+#
+reset_db
+do_execsql_test 13.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, y, z);
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 32);
+  INSERT INTO ft VALUES(
+      'x x x x x x x x x x x x x x x x',
+      'y y y y y y y y y y y y y y y y',
+      'z z z z z z z z x x x x x x x x'
+  );
+  INSERT INTO ft SELECT * FROM ft;
+  INSERT INTO ft SELECT * FROM ft;
+  INSERT INTO ft SELECT * FROM ft;
+  INSERT INTO ft SELECT * FROM ft;
+}
+faultsim_save_and_close
+do_faultsim_test 13.1 -faults oom-t* -prep {
+  faultsim_restore_and_reopen
+  db eval { SELECT * FROM ft }
+} -body {
+  db eval { SELECT rowid FROM ft WHERE ft MATCH '{x z}: x' }
+} -test {
+  faultsim_test_result {0 {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}}
+}
+
+
 finish_test
 
diff --git a/manifest b/manifest
index 49ba67b417..b7b3fe17f8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sthe\sfts5\smulti-column\ssyntax\sto\suse\sparenthesis\sinstead\sof\ssquare\sbrackets.
-D 2015-06-02T19:38:15.157
+C Fix\san\sfts5\sproblem\sin\sextracting\scolumns\sfrom\sposition\slists\scontaining\slarge\svarints.
+D 2015-06-03T11:23:30.476
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in d272f8755b464f20e02dd7799bfe16794c9574c4
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -111,7 +111,7 @@ F ext/fts5/fts5Int.h 4c677f3b797acde90ba1b7730eca6a32e7def742
 F ext/fts5/fts5_aux.c d53f00f31ad615ca4f139dd8751f9041afa00971
 F ext/fts5/fts5_buffer.c 9ec57c75c81e81dca118568876b1caead0aadadf
 F ext/fts5/fts5_config.c 11f969ed711a0a8b611d47431d74c372ad78c713
-F ext/fts5/fts5_expr.c b28917bc2ec08eca4c8395f2d2b61adeae489462
+F ext/fts5/fts5_expr.c 78a498ba149fbcfbd95c9630054c27955253309d
 F ext/fts5/fts5_hash.c c1cfdb2cae0fad00b06fae38a40eaf9261563ccc
 F ext/fts5/fts5_index.c 7cea402924cd3d8cd5943a7f9514c9153696571b
 F ext/fts5/fts5_storage.c 04e6717656b78eb230a1c730cac3b935eb94889b
@@ -135,7 +135,7 @@ F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37
 F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8
 F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592
 F ext/fts5/test/fts5al.test fc60ebeac9d8e366e71309d4c31fa72199d711d7
-F ext/fts5/test/fts5auto.test 3cef6f63c306bac05b95f47a94c3e87de71e61e3
+F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca
 F ext/fts5/test/fts5aux.test e5631607bbc05ac1c38cf7d691000509aca71ef3
 F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
 F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b
@@ -151,7 +151,7 @@ F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
 F ext/fts5/test/fts5fault1.test b42d3296be8a75f557cf2cbce0d8b483fc9db45b
 F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341
 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
-F ext/fts5/test/fts5fault4.test 25306f396d239fd2ef35b2cc273a7f40fab80173
+F ext/fts5/test/fts5fault4.test 8671f534136aa1c80a102e8fd25b4921885e6667
 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875
 F ext/fts5/test/fts5fault6.test 234dc6355f8d3f8b5be2763f30699d770247c215
 F ext/fts5/test/fts5full.test 0924bdca5416a242103239ace79c6f5aa34bab8d
@@ -1357,7 +1357,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P c9ffda4abb4390bbc5719e269196e2807b254f97
-R 29c0ea9b35d0dc8e0559ad6dbdc1731d
+P ab85a6fc4f7580278fc9d1f0090fdcf0a90d065b
+R a504500377faead9f927e5b8c0ebee20
 U dan
-Z 95711865cf9f19b5a5f7ef868f1de2a7
+Z 68530452f05ae148227ba366408c8cfc
diff --git a/manifest.uuid b/manifest.uuid
index 0a6bf7e949..8efcca17c3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ab85a6fc4f7580278fc9d1f0090fdcf0a90d065b
\ No newline at end of file
+4ea015ab983300d420ef104cca550b22a6395866
\ No newline at end of file