Fix a problem with fts3 prefix terms within phrase queries on "order=DESC" tables...

FossilOrigin-Name: 0cdf502885ea7e5805d7ba3719f055f5d48fc78d

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index 46b499e96bc3fc65657094e1bb721c848876d103..7a15379b83e815aadb0bd2ea3384365967beae0c 100644 (file)
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -2502,26 +2502,33 @@ static int fts3DoclistOrMerge(
 **
 ** The right-hand input doclist is overwritten by this function.
 */
-static void fts3DoclistPhraseMerge(
+static int fts3DoclistPhraseMerge(
   int bDescDoclist,               /* True if arguments are desc */
   int nDist,                      /* Distance from left to right (1=adjacent) */
   char *aLeft, int nLeft,         /* Left doclist */
-  char *aRight, int *pnRight      /* IN/OUT: Right/output doclist */
+  char **paRight, int *pnRight    /* IN/OUT: Right/output doclist */
 ){
   sqlite3_int64 i1 = 0;
   sqlite3_int64 i2 = 0;
   sqlite3_int64 iPrev = 0;
+  char *aRight = *paRight;
   char *pEnd1 = &aLeft[nLeft];
   char *pEnd2 = &aRight[*pnRight];
   char *p1 = aLeft;
   char *p2 = aRight;
   char *p;
   int bFirstOut = 0;
-  char *aOut = aRight;
+  char *aOut;
 
   assert( nDist>0 );
-
+  if( bDescDoclist ){
+    aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX);
+    if( aOut==0 ) return SQLITE_NOMEM;
+  }else{
+    aOut = aRight;
+  }
   p = aOut;
+
   fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
   fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
 
@@ -2550,6 +2557,12 @@ static void fts3DoclistPhraseMerge(
   }
 
   *pnRight = (int)(p - aOut);
+  if( bDescDoclist ){
+    sqlite3_free(aRight);
+    *paRight = aOut;
+  }
+
+  return SQLITE_OK;
 }
 
 /*
@@ -2674,8 +2687,22 @@ static int fts3TermSelectMerge(
 ){
   if( pTS->aaOutput[0]==0 ){
     /* If this is the first term selected, copy the doclist to the output
-    ** buffer using memcpy(). */
-    pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
+    ** buffer using memcpy(). 
+    **
+    ** Add FTS3_VARINT_MAX bytes of unused space to the end of the 
+    ** allocation. This is so as to ensure that the buffer is big enough
+    ** to hold the current doclist AND'd with any other doclist. If the
+    ** doclists are stored in order=ASC order, this padding would not be
+    ** required (since the size of [doclistA AND doclistB] is always less
+    ** than or equal to the size of [doclistA] in that case). But this is
+    ** not true for order=DESC. For example, a doclist containing (1, -1) 
+    ** may be smaller than (-1), as in the first example the -1 may be stored
+    ** as a single-byte delta, whereas in the second it must be stored as a
+    ** FTS3_VARINT_MAX byte varint.
+    **
+    ** Similar padding is added in the fts3DoclistOrMerge() function.
+    */
+    pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1);
     pTS->anOutput[0] = nDoclist;
     if( pTS->aaOutput[0] ){
       memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
@@ -3931,14 +3958,17 @@ static void fts3EvalAllocateReaders(
 ** This function assumes that pList points to a buffer allocated using
 ** sqlite3_malloc(). This function takes responsibility for eventually
 ** freeing the buffer.
+**
+** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs.
 */
-static void fts3EvalPhraseMergeToken(
+static int fts3EvalPhraseMergeToken(
   Fts3Table *pTab,                /* FTS Table pointer */
   Fts3Phrase *p,                  /* Phrase to merge pList/nList into */
   int iToken,                     /* Token pList/nList corresponds to */
   char *pList,                    /* Pointer to doclist */
   int nList                       /* Number of bytes in pList */
 ){
+  int rc = SQLITE_OK;
   assert( iToken!=p->iDoclistToken );
 
   if( pList==0 ){
@@ -3977,13 +4007,16 @@ static void fts3EvalPhraseMergeToken(
       nDiff = p->iDoclistToken - iToken;
     }
 
-    fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight);
+    rc = fts3DoclistPhraseMerge(
+        pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight
+    );
     sqlite3_free(pLeft);
     p->doclist.aAll = pRight;
     p->doclist.nAll = nRight;
   }
 
   if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken;
+  return rc;
 }
 
 /*
@@ -4009,7 +4042,7 @@ static int fts3EvalPhraseLoad(
       char *pThis = 0;
       rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis);
       if( rc==SQLITE_OK ){
-        fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis);
+        rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis);
       }
     }
     assert( pToken->pSegcsr==0 );
@@ -4811,9 +4844,13 @@ static int fts3EvalSelectDeferred(
         char *pList = 0;
         rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList);
         assert( rc==SQLITE_OK || pList==0 );
+        if( rc==SQLITE_OK ){
+          rc = fts3EvalPhraseMergeToken(
+              pTab, pTC->pPhrase, pTC->iToken,pList,nList
+          );
+        }
         if( rc==SQLITE_OK ){
           int nCount;
-          fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList);
           nCount = fts3DoclistCountDocids(
               pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll
           );

diff --git a/manifest b/manifest
index 762233f67693f44c867d04c15377d89c552a6fdf..12c20e5b8a9fe14e4b4b0ae127735b4bce0d23ac 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\swith\sresolving\sORDER\sBY\sclauses\sthat\sfeature\sCOLLATE\sclauses\sattached\sto\scompound\sSELECT\sstatements.
-D 2015-04-04T16:49:04.697
+C Fix\sa\sproblem\swith\sfts3\sprefix\sterms\swithin\sphrase\squeries\son\s"order=DESC"\stables\swith\sa\smix\sof\snegative\sand\spositive\srowids.
+D 2015-04-06T09:05:29.052
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 00d12636df7a5b08af09116bcd6c7bfd49b8b3b4
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -78,7 +78,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c d3f6f0e95d366f3c2028d916c36a0844bf805840
+F ext/fts3/fts3.c 23bd9d37a777342f5c22a648e9b4b005dde9e58f
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h 394858c12a17740f7a1f6bd372c4606d4425a8d1
 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365
@@ -594,7 +594,7 @@ F test/fts3join.test 53e66a0c21eb568580674a43b21c059acb26f499
 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6
 F test/fts3matchinfo.test 58544fa4d254000fa4e7f494b0a832f7ba61d45e
 F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905
-F test/fts3prefix.test 9f68e3598a139c23ec47d09299420e0fc4c72a83
+F test/fts3prefix.test fa794eaab0bdae466494947b0b153d7844478ab2
 F test/fts3prefix2.test e1f0a822ca661dced7f12ce392e14eaf65609dce
 F test/fts3query.test c838b18f2b859e15fd31c64be3d79ef1556803ca
 F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0
@@ -1249,7 +1249,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 4ae9a3acc4eeeb7998769eb856c97c2233476f72 427b50fba7362e5b447e79d39050f25ed2ef10af
-R 0a683c8bd67cbd2be5ea29e95266b636
+P edc1de2a588fd50c0049bb2be76d3f6783443165
+R a8252a4468bea13d638722c58d3ce9cc
+T *branch * fts3-prefix-query-fix
+T *sym-fts3-prefix-query-fix *
+T -sym-trunk *
 U dan
-Z c9ac74415bdb7893d918eacf5a95862c
+Z 7c1c374a4c4efe674ae583b0f9d983d2

diff --git a/manifest.uuid b/manifest.uuid
index ff36ec293f64947148a4772e5ba61f67010a840e..2a42d14f3316fb44ae95df02ee743fa3ec28ff7f 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-edc1de2a588fd50c0049bb2be76d3f6783443165
\ No newline at end of file
+0cdf502885ea7e5805d7ba3719f055f5d48fc78d
\ No newline at end of file

diff --git a/test/fts3prefix.test b/test/fts3prefix.test
index 8ffabe8d65fdcbddecf5ddb0c9b4361b35e88373..e8d2b2b5fb25113b53ef5f31cbc4b637b027f2a1 100644 (file)
--- a/test/fts3prefix.test
+++ b/test/fts3prefix.test
@@ -274,4 +274,22 @@ do_execsql_test 6.5.2 {
   SELECT md5sum(quote(root)) FROM t1_segdir;
 } [db eval {SELECT md5sum(quote(root)) FROM t2_segdir}]
 
+
+do_execsql_test 7.0 {
+  CREATE VIRTUAL TABLE t6 USING fts4(x,order=DESC);
+  INSERT INTO t6(docid, x) VALUES(-1,'a b');
+  INSERT INTO t6(docid, x) VALUES(1, 'b');
+}
+do_execsql_test 7.1 {
+  SELECT docid FROM t6 WHERE t6 MATCH '"a* b"';
+} {-1}
+do_execsql_test 7.2 {
+  SELECT docid FROM t6 WHERE t6 MATCH 'a*';
+} {-1}
+do_execsql_test 7.3 {
+  SELECT docid FROM t6 WHERE t6 MATCH 'a* b';
+} {-1}
+
+
+
 finish_test