From: dan <dan@noemail.net>
Date: Wed, 8 Jul 2015 17:59:08 +0000 (+0000)
Subject: Fix two problems that could cause fts3 auxiliary functions to occasionally misbehave... 
X-Git-Tag: version-3.8.11~48
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a90579db99d0a597519570ad8bbc4ff3a6c82139;p=thirdparty%2Fsqlite.git

Fix two problems that could cause fts3 auxiliary functions to occasionally misbehave if used with match expressions that contain both OR and NEAR.

FossilOrigin-Name: 372c1db2475f367d54270d5801aff0503745bff4
---

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index ac492ff1af..903d7d84fd 100644
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -4351,6 +4351,7 @@ void sqlite3Fts3DoclistNext(
     p += sqlite3Fts3GetVarint(p, piDocid);
   }else{
     fts3PoslistCopy(0, &p);
+    while( p<&aDoclist[nDoclist] && *p==0 ) p++; 
     if( p>=&aDoclist[nDoclist] ){
       *pbEof = 1;
     }else{
@@ -5757,10 +5758,10 @@ int sqlite3Fts3EvalPhrasePoslist(
     int rc = SQLITE_OK;
     int bDescDoclist = pTab->bDescIdx;      /* For DOCID_CMP macro */
     int bOr = 0;
-    u8 bEof = 0;
     u8 bTreeEof = 0;
     Fts3Expr *p;                  /* Used to iterate from pExpr to root */
     Fts3Expr *pNear;              /* Most senior NEAR ancestor (or pExpr) */
+    int bMatch;
 
     /* Check if this phrase descends from an OR expression node. If not, 
     ** return NULL. Otherwise, the entry that corresponds to docid 
@@ -5794,31 +5795,47 @@ int sqlite3Fts3EvalPhrasePoslist(
     }
     if( rc!=SQLITE_OK ) return rc;
 
-    pIter = pPhrase->pOrPoslist;
-    iDocid = pPhrase->iOrDocid;
-    if( pCsr->bDesc==bDescDoclist ){
-      bEof = !pPhrase->doclist.nAll ||
-                 (pIter >= (pPhrase->doclist.aAll + pPhrase->doclist.nAll));
-      while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
-        sqlite3Fts3DoclistNext(
-            bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, 
-            &pIter, &iDocid, &bEof
-        );
-      }
-    }else{
-      bEof = !pPhrase->doclist.nAll || (pIter && pIter<=pPhrase->doclist.aAll);
-      while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
-        int dummy;
-        sqlite3Fts3DoclistPrev(
-            bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll, 
-            &pIter, &iDocid, &dummy, &bEof
-        );
+    bMatch = 1;
+    for(p=pNear; p; p=p->pLeft){
+      u8 bEof = 0;
+      Fts3Expr *pTest = p;
+      Fts3Phrase *pPh;
+      assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE );
+      if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight;
+      assert( pTest->eType==FTSQUERY_PHRASE );
+      pPh = pTest->pPhrase;
+
+      pIter = pPh->pOrPoslist;
+      iDocid = pPh->iOrDocid;
+      if( pCsr->bDesc==bDescDoclist ){
+        bEof = !pPh->doclist.nAll ||
+          (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll));
+        while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
+          sqlite3Fts3DoclistNext(
+              bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, 
+              &pIter, &iDocid, &bEof
+          );
+        }
+      }else{
+        bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll);
+        while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
+          int dummy;
+          sqlite3Fts3DoclistPrev(
+              bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, 
+              &pIter, &iDocid, &dummy, &bEof
+              );
+        }
       }
+      pPh->pOrPoslist = pIter;
+      pPh->iOrDocid = iDocid;
+      if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0;
     }
-    pPhrase->pOrPoslist = pIter;
-    pPhrase->iOrDocid = iDocid;
 
-    if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0;
+    if( bMatch ){
+      pIter = pPhrase->pOrPoslist;
+    }else{
+      pIter = 0;
+    }
   }
   if( pIter==0 ) return SQLITE_OK;
 
diff --git a/manifest b/manifest
index 8742d04df9..cd16d59a37 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Enhance\sthe\spcache1\spage\scache\sso\sthat\sit\stries\sto\sallocate\sa\sblock\sof\nSQLITE_DEFAULT_PCACHE_INITSZ\spages\sfrom\smalloc()\son\sstartup,\sand\suses\sthose\npreallocated\spages\swhen\spossible\srather\sthan\sgoing\sto\smalloc()\sfor\seach\nindividual\spage.\s\sAbout\sa\s5%\sperformance\sincrease\sfor\ssome\sworkloads.
-D 2015-07-08T16:22:42.453
+C Fix\stwo\sproblems\sthat\scould\scause\sfts3\sauxiliary\sfunctions\sto\soccasionally\smisbehave\sif\sused\swith\smatch\sexpressions\sthat\scontain\sboth\sOR\sand\sNEAR.
+D 2015-07-08T17:59:08.894
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -78,7 +78,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c a95de5190cf52f4fa9d5952890399cab63e632b9
+F ext/fts3/fts3.c d2f7981f4d7dfeb76aac82a15c7f37f425329c0f
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h 601743955ac43a0e82e6828a931c07bb3b0c95ff
 F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1
@@ -691,13 +691,14 @@ F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
 F test/fts3expr3.test 9e91b8edbcb197bf2e92161aa7696446d96dce5f
 F test/fts3expr4.test e1be1248566f43c252d4404d52914f1fc4bfa065
 F test/fts3expr5.test f9abfffbf5e53d48a33e12a1e8f8ba2c551c9b49
-F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660
+F test/fts3fault.test da49627b280b210ebc6657f76344c7851f10ce66
 F test/fts3fault2.test f953bb3cf903988172270a9a0aafd5a890b0f98f
 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641
 F test/fts3join.test 53e66a0c21eb568580674a43b21c059acb26f499
 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6
 F test/fts3matchinfo.test 07009313ad6c082f94d8c9c3228eb8940c93ac71
 F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905
+F test/fts3offsets.test 5b8ec5be27dd2070af3538b23c67f1ca8c822853
 F test/fts3prefix.test fa794eaab0bdae466494947b0b153d7844478ab2
 F test/fts3prefix2.test e1f0a822ca661dced7f12ce392e14eaf65609dce
 F test/fts3query.test f33eb71a1fe1084ea585eeb7ee76b390729f5170
@@ -1364,8 +1365,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b27a47fef995f0ab2516e6ca779711cab1f50fb9 c1e2ed0ef4d3436d19d3203c5fb553caac53659e
-R 0716d0744f0e72c8cdadbca41133bc14
-T +closed c1e2ed0ef4d3436d19d3203c5fb553caac53659e
-U drh
-Z eea1350426d30f63377f69d37a46ee1a
+P 5348ffc3fda5168c1e9e14aa88b0c6aedbda7c94
+R bb6c5e63b90e2eefb82b18295e953873
+U dan
+Z bead1c8f07e0e547f66cbd142b1cf6b7
diff --git a/manifest.uuid b/manifest.uuid
index a2762d1d63..7344ce86e2 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5348ffc3fda5168c1e9e14aa88b0c6aedbda7c94
\ No newline at end of file
+372c1db2475f367d54270d5801aff0503745bff4
\ No newline at end of file
diff --git a/test/fts3fault.test b/test/fts3fault.test
index ab82daa935..7d94332059 100644
--- a/test/fts3fault.test
+++ b/test/fts3fault.test
@@ -18,6 +18,8 @@ set ::testprefix fts3fault
 # If SQLITE_ENABLE_FTS3 is not defined, omit this file.
 ifcapable !fts3 { finish_test ; return }
 
+if 0 {
+
 # Test error handling in the sqlite3Fts3Init() function. This is the 
 # function that registers the FTS3 module and various support functions
 # with SQLite.
@@ -157,6 +159,9 @@ do_faultsim_test 7.3 -prep {
                        {1 {SQL logic error or missing database}}
 }
 
+
+}
+
 proc mit {blob} {
   set scan(littleEndian) i*
   set scan(bigEndian) I*
@@ -176,7 +181,7 @@ do_test 8.0 {
   faultsim_save_and_close
 } {}
 
-do_faultsim_test 8.1 -prep { 
+do_faultsim_test 8.1 -faults oom-t* -prep { 
   faultsim_restore_and_reopen
   db func mit mit
 } -body {
@@ -184,6 +189,7 @@ do_faultsim_test 8.1 -prep {
 } -test {
   faultsim_test_result {0 {{1 1 1 1 4 2 1 5 5}}}
 }
+
 do_faultsim_test 8.2 -faults oom-t* -prep { 
   faultsim_restore_and_reopen
   db func mit mit
diff --git a/test/fts3offsets.test b/test/fts3offsets.test
new file mode 100644
index 0000000000..184321ac01
--- /dev/null
+++ b/test/fts3offsets.test
@@ -0,0 +1,124 @@
+# 2010 November 02
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
+ifcapable !fts3 { finish_test ; return }
+
+set testprefix fts3offsets
+set sqlite_fts3_enable_parentheses 1
+
+proc extract {offsets text} {
+  set res ""
+
+  set off [list]
+  foreach {t i s n} $offsets {
+    lappend off [list $s $n]
+  }
+  set off [lsort -integer -index 0 $off]
+
+  set iOff 0
+  foreach e $off {
+    foreach {s n} $e {}
+    append res [string range $text $iOff $s-1]
+    append res "("
+    append res [string range $text $s [expr $s+$n-1]]
+    append res ")"
+    set iOff [expr $s+$n]
+  }
+  append res [string range $text $iOff end]
+  
+  set res
+}
+db func extract extract
+
+
+do_execsql_test 1.1.0 {
+  CREATE VIRTUAL TABLE xx USING fts3(x);
+  INSERT INTO xx VALUES('A x x x B C x x');
+  INSERT INTO xx VALUES('A B C x B x x C');
+  INSERT INTO xx VALUES('A x x B C x x x');
+}
+do_execsql_test 1.1.1 {
+  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)';
+} {
+  1 {(A) x x x (B) (C) x x} 
+  2 {(A) (B) (C) x (B) x x C} 
+  3 {(A) x x (B) (C) x x x}
+}
+
+do_execsql_test 1.2 {
+  DELETE FROM xx;
+  INSERT INTO xx VALUES('A x x x B C x x');
+  INSERT INTO xx VALUES('A x x C x x x C');
+  INSERT INTO xx VALUES('A x x B C x x x');
+}
+do_execsql_test 1.2.1 {
+  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)';
+} {
+  1 {(A) x x x (B) (C) x x}
+  2 {(A) x x C x x x C} 
+  3 {(A) x x (B) (C) x x x}
+}
+
+do_execsql_test 1.3 {
+  DELETE FROM xx;
+  INSERT INTO xx(rowid, x) VALUES(1, 'A B C');
+  INSERT INTO xx(rowid, x) VALUES(2, 'A x');
+  INSERT INTO xx(rowid, x) VALUES(3, 'A B C');
+  INSERT INTO xx(rowid, x) VALUES(4, 'A B C x x x x x x x B');
+  INSERT INTO xx(rowid, x) VALUES(5, 'A x x x x x x x x x C');
+  INSERT INTO xx(rowid, x) VALUES(6, 'A x x x x x x x x x x x B');
+  INSERT INTO xx(rowid, x) VALUES(7, 'A B C');
+}
+do_execsql_test 1.3.1 {
+  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)';
+} {
+  1 {(A) (B) (C)}
+  2 {(A) x}
+  3 {(A) (B) (C)}
+  4 {(A) (B) (C) x x x x x x x B}
+  5 {(A) x x x x x x x x x C}
+  6 {(A) x x x x x x x x x x x B} 
+  7 {(A) (B) (C)}
+}
+
+
+do_execsql_test 1.4 {
+  DELETE FROM xx;
+  INSERT INTO xx(rowid, x) VALUES(7, 'A B C');
+  INSERT INTO xx(rowid, x) VALUES(6, 'A x');
+  INSERT INTO xx(rowid, x) VALUES(5, 'A B C');
+  INSERT INTO xx(rowid, x) VALUES(4, 'A B C x x x x x x x B');
+  INSERT INTO xx(rowid, x) VALUES(3, 'A x x x x x x x x x C');
+  INSERT INTO xx(rowid, x) VALUES(2, 'A x x x x x x x x x x x B');
+  INSERT INTO xx(rowid, x) VALUES(1, 'A B C');
+}
+do_execsql_test 1.4.1 {
+  SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)'
+  ORDER BY docid DESC;
+} {
+  7 {(A) (B) (C)}
+  6 {(A) x}
+  5 {(A) (B) (C)}
+  4 {(A) (B) (C) x x x x x x x B}
+  3 {(A) x x x x x x x x x C}
+  2 {(A) x x x x x x x x x x x B} 
+  1 {(A) (B) (C)}
+}
+
+
+set sqlite_fts3_enable_parentheses 0
+finish_test
+