Fix problems to do with using both OR and NEAR operators in a single expression.

FossilOrigin-Name: 4e8dd19eef04777d800977faf1859a405e396f30

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index 5536a684ae63b4e05adce5b984f3c04c757f1d0f..18a87e73ce54d2e8093c91c6fb7436db19ec2017 100644 (file)
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -3763,13 +3763,13 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){
   return rc;
 }
 
-static void fts3EvalFreeDeferredDoclist(Fts3Phrase *pPhrase){
+static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){
   if( pPhrase->doclist.bFreeList ){
     sqlite3_free(pPhrase->doclist.pList);
-    pPhrase->doclist.pList = 0;
-    pPhrase->doclist.nList = 0;
-    pPhrase->doclist.bFreeList = 0;
   }
+  pPhrase->doclist.pList = 0;
+  pPhrase->doclist.nList = 0;
+  pPhrase->doclist.bFreeList = 0;
 }
 
 static int fts3EvalNearTrim2(
@@ -3786,6 +3786,8 @@ static int fts3EvalNearTrim2(
   char *pOut; 
   int res;
 
+  assert( pPhrase->doclist.pList );
+
   p2 = pOut = pPhrase->doclist.pList;
   res = fts3PoslistNearMerge(
     &pOut, aTmp, nParam1, nParam2, paPoslist, &p2
@@ -3959,16 +3961,16 @@ static void fts3EvalNext(
           fts3EvalNext(pCsr, pRight, pRc);
           assert( *pRc!=SQLITE_OK || pRight->bStart );
         }
-        do {
-          fts3EvalNext(pCsr, pLeft, pRc);
-          if( pLeft->bEof ) break;
+
+        fts3EvalNext(pCsr, pLeft, pRc);
+        if( pLeft->bEof==0 ){
           while( !*pRc 
               && !pRight->bEof 
               && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 
           ){
             fts3EvalNext(pCsr, pRight, pRc);
           }
-        }while( !pRight->bEof && pRight->iDocid==pLeft->iDocid && !*pRc );
+        }
         pExpr->iDocid = pLeft->iDocid;
         pExpr->bEof = pLeft->bEof;
         break;
@@ -3976,7 +3978,7 @@ static void fts3EvalNext(
 
       default: {
         Fts3Phrase *pPhrase = pExpr->pPhrase;
-        fts3EvalFreeDeferredDoclist(pPhrase);
+        fts3EvalZeroPoslist(pPhrase);
         *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
         pExpr->iDocid = pPhrase->doclist.iDocid;
         break;
@@ -3997,11 +3999,11 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){
          && fts3EvalNearTest(pExpr, pRc)
         );
 
-        /* If this is a NEAR node and the NEAR expression does not match
-        ** any rows, zero the doclist for all phrases involved in the NEAR.
-        ** This is because the snippet(), offsets() and matchinfo() functions
-        ** are not supposed to recognize any instances of phrases that are
-        ** part of unmatched NEAR queries. For example if this expression:
+        /* If the NEAR expression does not match any rows, zero the doclist for 
+        ** all phrases involved in the NEAR. This is because the snippet(),
+        ** offsets() and matchinfo() functions are not supposed to recognize 
+        ** any instances of phrases that are part of unmatched NEAR queries. 
+        ** For example if this expression:
         **
         **    ... MATCH 'a OR (b NEAR c)'
         **
@@ -4012,12 +4014,19 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){
         ** then any snippet() should ony highlight the "a" term, not the "b"
         ** (as "b" is part of a non-matching NEAR clause).
         */
-        if( pExpr->eType==FTSQUERY_NEAR && bHit==0 ){
+        if( bHit==0 
+         && pExpr->eType==FTSQUERY_NEAR 
+         && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
+        ){
           Fts3Expr *p;
           for(p=pExpr; p->pPhrase==0; p=p->pLeft){
-            p->pRight->pPhrase->doclist.pList = 0;
+            if( p->pRight->iDocid==pCsr->iPrevId ){
+              fts3EvalZeroPoslist(p->pRight->pPhrase);
+            }
+          }
+          if( p->iDocid==pCsr->iPrevId ){
+            fts3EvalZeroPoslist(p->pPhrase);
           }
-          p->pPhrase->doclist.pList = 0;
         }
 
         break;
@@ -4037,9 +4046,14 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){
         break;
 
       default: {
-        if( pCsr->pDeferred ){
+        if( pCsr->pDeferred 
+         && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
+        ){
           Fts3Phrase *pPhrase = pExpr->pPhrase;
-          fts3EvalFreeDeferredDoclist(pPhrase);
+          assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
+          if( pExpr->bDeferred ){
+            fts3EvalZeroPoslist(pPhrase);
+          }
           *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
           bHit = (pPhrase->doclist.pList!=0);
           pExpr->iDocid = pCsr->iPrevId;
@@ -4116,7 +4130,7 @@ static void fts3EvalRestart(
     Fts3Phrase *pPhrase = pExpr->pPhrase;
 
     if( pPhrase ){
-      fts3EvalFreeDeferredDoclist(pPhrase);
+      fts3EvalZeroPoslist(pPhrase);
       if( pPhrase->bIncr ){
         sqlite3Fts3EvalPhraseCleanup(pPhrase);
         memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
@@ -4172,7 +4186,7 @@ static void fts3EvalUpdateCounts(
   }
 }
 
-static int fts3EvalNearStats(
+static int fts3EvalGatherStats(
   Fts3Cursor *pCsr,
   Fts3Expr *pExpr
 ){
@@ -4195,6 +4209,7 @@ static int fts3EvalNearStats(
     }
     iDocid = pRoot->iDocid;
     bEof = pRoot->bEof;
+    assert( pRoot->bStart );
 
     /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */
     for(p=pRoot; p; p=p->pLeft){
@@ -4236,11 +4251,17 @@ static int fts3EvalNearStats(
     if( bEof ){
       pRoot->bEof = bEof;
     }else{
+      /* Caution: pRoot may iterate through docids in ascending or descending
+      ** order. For this reason, even though it seems more defensive, the 
+      ** do loop can not be written:
+      **
+      **   do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
+      */
       fts3EvalRestart(pCsr, pRoot, &rc);
-      while( pRoot->iDocid<iDocid && rc==SQLITE_OK ){
+      do {
         fts3EvalNext(pCsr, pRoot, &rc);
         assert( pRoot->bEof==0 );
-      }
+      }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
       fts3EvalLoadDeferred(pCsr, &rc);
     }
   }
@@ -4293,7 +4314,7 @@ int sqlite3Fts3EvalPhraseStats(
       aiOut[iCol*3 + 2] = pCsr->nDoc;
     }
   }else{
-    rc = fts3EvalNearStats(pCsr, pExpr);
+    rc = fts3EvalGatherStats(pCsr, pExpr);
     if( rc==SQLITE_OK ){
       assert( pExpr->aMI );
       for(iCol=0; iCol<pTab->nColumn; iCol++){
@@ -4372,7 +4393,7 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
   if( pPhrase ){
     int i;
     sqlite3_free(pPhrase->doclist.aAll);
-    fts3EvalFreeDeferredDoclist(pPhrase);
+    fts3EvalZeroPoslist(pPhrase);
     memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
     for(i=0; i<pPhrase->nToken; i++){
       fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);

diff --git a/manifest b/manifest
index c0c0a4d2ec3fbe8e966fb5b73787e957a4166215..4398869eb310958b556ea0d25cdf9987dc8372e8 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\svarious\sissues\sto\sdo\swith\sdeferred\stokens,\sNEAR\sexpressions\sand\smatchinfo().
-D 2011-06-08T18:39:07.487
+C Fix\sproblems\sto\sdo\swith\susing\sboth\sOR\sand\sNEAR\soperators\sin\sa\ssingle\sexpression.
+D 2011-06-09T10:48:02.352
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c b44083cafb9840be0927f8b9fb2ab4f373167f77
+F ext/fts3/fts3.c 5df3b5797522d3d17949ee12d5918d6d213b5114
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6
 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe
@@ -475,7 +475,7 @@ F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f
 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844
 F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676
 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2
-F test/fts3rnd.test d88ec3dbe52e81e410cd1a39554d15941f86333c
+F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0
 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2
 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd
 F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402
@@ -943,7 +943,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
 F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d
-P 9d10a6846b12a9cc8fd4fdc3affd931a27218b5a
-R 37e4da2cb9907d0ccf1d8076445165fd
+P 3972a787df5ec253b99b148385655e7b68d851fa
+R 9f88ec7fe13b9820fcb74aa4a46dd50c
 U dan
-Z 147c4bbcabf01e6d99dff7a301984a70
+Z 78f71a9a33e1daee77ff848d72b67bf4

diff --git a/manifest.uuid b/manifest.uuid
index f3e1134a0aa67a0ebad72b31aaf930813467d418..f91e3008ee41d90cb04600c1071946caaff63f28 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3972a787df5ec253b99b148385655e7b68d851fa
\ No newline at end of file
+4e8dd19eef04777d800977faf1859a405e396f30
\ No newline at end of file

diff --git a/test/fts3rnd.test b/test/fts3rnd.test
index 0ad41d8ce40f1a4c65949af0701582787d2f06b1..97af54925f3953e866468ddac7cc3afa8736b471 100644 (file)
--- a/test/fts3rnd.test
+++ b/test/fts3rnd.test
@@ -273,6 +273,8 @@ proc do_orderbydocid_test {tn sql res} {
   ]
 }
 
+set NUM_TRIALS 100
+
 foreach {nodesize order} {
   50    DESC
   50    ASC
@@ -292,7 +294,7 @@ foreach {nodesize order} {
     for {set i 0} {$i < 100} {incr i} { insert_row $i }
   }
   
-  for {set iTest 1} {$iTest <= 100} {incr iTest} {
+  for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} {
     catchsql COMMIT
 
     set DO_MALLOC_TEST 0
@@ -334,11 +336,11 @@ foreach {nodesize order} {
     #
     for {set i 0} {$i < 10} {incr i} {
       set term [random_term]
-      do_select_test 1.$i {
+      do_select_test 1.$i.asc {
         SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
         ORDER BY docid ASC
       } [simple_token_matchinfo $term 0]
-      do_select_test 1.$i {
+      do_select_test 1.$i.desc {
         SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
         ORDER BY docid DESC
       } [simple_token_matchinfo $term 1]
@@ -432,9 +434,9 @@ foreach {nodesize order} {
     # Set operations on NEAR queries.
     #
     foreach {tn op proc} {
-      8  OR  setop_or
-      9  NOT setop_not
-      10 AND setop_and
+      11 OR  setop_or
+      12 NOT setop_not
+      13 AND setop_and
     } {
       for {set i 0} {$i < $nRep} {incr i} {
         set term1 [random_term]