Fix some problems with FTS3 and 3-way NEAR queries.

FossilOrigin-Name: 23dc6fb5b28712d1ba18dc7ddb3f2ef3b469d611

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index 0ca6d41503cf86b19e8456832e38ee163e91d138..d41f10ea30f500d6f0e6003e2646b1b035e44360 100644 (file)
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -1064,6 +1064,54 @@ static void fts3ColumnlistCopy(char **pp, char **ppPoslist){
   *ppPoslist = pEnd;
 }
 
+/*
+** Value used to signify the end of an offset-list. This is safe because
+** it is not possible to have a document with 2^31 terms.
+*/
+#define OFFSET_LIST_END 0x7fffffff
+
+/*
+** This function is used to help parse offset-lists. When this function is
+** called, *pp may point to the start of the next varint in the offset-list
+** being parsed, or it may point to 1 byte past the end of the offset-list
+** (in which case **pp will be 0x00 or 0x01).
+**
+** If *pp points past the end of the current offset list, set *pi to 
+** OFFSET_LIST_END and return. Otherwise, read the next varint from *pp,
+** increment the current value of *pi by the value read, and set *pp to
+** point to the next value before returning.
+*/
+static void fts3ReadNextPos(
+  char **pp,                      /* IN/OUT: Pointer into offset-list buffer */
+  sqlite3_int64 *pi               /* IN/OUT: Value read from offset-list */
+){
+  if( **pp&0xFE ){
+    fts3GetDeltaVarint(pp, pi);
+    *pi -= 2;
+  }else{
+    *pi = OFFSET_LIST_END;
+  }
+}
+
+/*
+** If parameter iCol is not 0, write an 0x01 byte followed by the value of
+** iCol encoded as a varint to *pp. 
+**
+** Set *pp to point to the byte just after the last byte written before 
+** returning (do not modify it if iCol==0). Return the total number of bytes
+** written (0 if iCol==0).
+*/
+static int fts3PutColNumber(char **pp, int iCol){
+  int n = 0;                      /* Number of bytes written */
+  if( iCol ){
+    char *p = *pp;                /* Output pointer */
+    n = 1 + sqlite3Fts3PutVarint(&p[1], iCol);
+    *p = 0x01;
+    *pp = &p[n];
+  }
+  return n;
+}
+
 /*
 **
 */
@@ -1086,42 +1134,37 @@ static void fts3PoslistMerge(
       sqlite3_int64 i1 = 0;
       sqlite3_int64 i2 = 0;
       sqlite3_int64 iPrev = 0;
-      if( iCol1!=0 ){
-        int n;
-        *p++ = 0x01;
-        n = sqlite3Fts3PutVarint(p, iCol1);
-        p += n;
-        p1 += 1 + n;
-        p2 += 1 + n;
-      }
-      while( (*p1&0xFE) || (*p2&0xFE) ){
+      int n = fts3PutColNumber(&p, iCol1);
+      p1 += n;
+      p2 += n;
+
+      /* At this point, both p1 and p2 point to the start of offset-lists.
+      ** An offset-list is a list of non-negative delta-encoded varints, each 
+      ** incremented by 2 before being stored. Each list is terminated by a 0 
+      ** or 1 value (0x00 or 0x01). The following block merges the two lists
+      ** and writes the results to buffer p. p is left pointing to the byte
+      ** after the list written. No terminator (0x00 or 0x01) is written to
+      ** the output.
+      */
+      fts3GetDeltaVarint(&p1, &i1);
+      fts3GetDeltaVarint(&p2, &i2);
+      do {
+        fts3PutDeltaVarint(&p, &iPrev, (i1<i2) ? i1 : i2); 
+        iPrev -= 2;
         if( i1==i2 ){
-          fts3GetDeltaVarint(&p1, &i1); i1 -= 2;
-          fts3GetDeltaVarint(&p2, &i2); i2 -= 2;
+          fts3ReadNextPos(&p1, &i1);
+          fts3ReadNextPos(&p2, &i2);
         }else if( i1<i2 ){
-          fts3GetDeltaVarint(&p1, &i1); i1 -= 2;
+          fts3ReadNextPos(&p1, &i1);
         }else{
-          fts3GetDeltaVarint(&p2, &i2); i2 -= 2;
+          fts3ReadNextPos(&p2, &i2);
         }
-        fts3PutDeltaVarint(&p, &iPrev, (i1<i2 ? i1 : i2) + 2); iPrev -= 2;
-        if( 0==(*p1&0xFE) ) i1 = 0x7FFFFFFF;
-        if( 0==(*p2&0xFE) ) i2 = 0x7FFFFFFF;
-      }
+      }while( i1!=OFFSET_LIST_END || i2!=OFFSET_LIST_END );
     }else if( iCol1<iCol2 ){
-      if( iCol1 ){
-        int n = sqlite3Fts3PutVarint(&p[1], iCol1);
-        *p = 0x01;
-        p += n+1;
-        p1 += n+1;
-      }
+      p1 += fts3PutColNumber(&p, iCol1);
       fts3ColumnlistCopy(&p, &p1);
     }else{
-      if( iCol2 ){
-        int n = sqlite3Fts3PutVarint(&p[1], iCol2);
-        *p = 0x01;
-        p += n+1;
-        p2 += n+1;
-      }
+      p2 += fts3PutColNumber(&p, iCol2);
       fts3ColumnlistCopy(&p, &p2);
     }
   }
@@ -1188,13 +1231,14 @@ static int fts3PoslistPhraseMerge(
           fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2;
           pSave = 0;
         }
-        if( iPos2<=iPos1 ){
+        if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){
           if( (*p2&0xFE)==0 ) break;
           fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
         }else{
           if( (*p1&0xFE)==0 ) break;
           fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
         }
+
       }
       if( pSave && pp ){
         p = pSave;
@@ -1241,13 +1285,16 @@ static int fts3PoslistPhraseMerge(
   return 1;
 }
 
+/*
+** Merge two position-lists as required by the NEAR operator.
+*/
 static int fts3PoslistNearMerge(
   char **pp,                      /* Output buffer */
   char *aTmp,                     /* Temporary buffer space */
   int nRight,                     /* Maximum difference in token positions */
   int nLeft,                      /* Maximum difference in token positions */
-  char **pp1,                     /* Left input list */
-  char **pp2                      /* Right input list */
+  char **pp1,                     /* IN/OUT: Left input list */
+  char **pp2                      /* IN/OUT: Right input list */
 ){
   char *p1 = *pp1;
   char *p2 = *pp2;

diff --git a/manifest b/manifest
index 29711c9436ec818a50738bb0aff043ee203533af..80a34d4a1e6a9384f526962c11e3d93bca7c1c82 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,8 +1,5 @@
------BEGIN PGP SIGNED MESSAGE-----
-Hash: SHA1
-
-C Add\sthe\sSQLITE_4_BYTE_ALIGNED_MALLOC\scompile-time\soption\swhich\stells\ssome\nassert()\sstatements\sthat\sthe\sunderlying\ssystem\sonly\srequires\s4-byte\salignment\nof\s8-byte\sdata\sobjects\slike\sdouble\sor\sint64\sand\sthat\ssystem\smalloc()\sonly\nguarantees\s4-byte\salignment\sof\sreturned\spointers.
-D 2009-12-04T23:10:13
+C Fix\ssome\sproblems\swith\sFTS3\sand\s3-way\sNEAR\squeries.
+D 2009-12-05T11:37:19
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -59,7 +56,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 54ad3e8f0374148768ad65724cff89a8a5a241a7
+F ext/fts3/fts3.c 7d344410bffdddf98c1af623993dd284e156a89f
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h cc716c74afa7da8e0f8ef39404f33ea62a823eb3
 F ext/fts3/fts3_expr.c c18794a62c257d3456d3314c5a18e348ae0d84bd
@@ -379,7 +376,7 @@ F test/fts2p.test 4b48c35c91e6a7dbf5ac8d1e5691823cc999aafb
 F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682
 F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e
 F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a
-F test/fts3.test f4f380d3717493605270dfa3b0fa893ea0afb18d
+F test/fts3.test ae0433b09b12def08105640e57693726c4949338
 F test/fts3_common.tcl 31935839b1b601a5955572cb4e8060513c96bde0
 F test/fts3aa.test 5327d4c1d9b6c61021696746cc9a6cdc5bf159c0
 F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9
@@ -779,14 +776,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 04fc9c7661dd24d080f965e7eae9010a2d346e6a
-R 012380c4025d7401ec31300c24d1a49d
-U drh
-Z 3a60f929898567f03b13abb7e9741c72
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.6 (GNU/Linux)
-
-iD8DBQFLGZbXoxKgR168RlERAgzLAKCGX83GaoDdnZtQhPjMOrZGC9L8TwCcC1Yk
-V482pOMKKDD0UFm2zbfBJVE=
-=H8qI
------END PGP SIGNATURE-----
+P 08faee686eb2fabe0dde51231ee55880e78541e8
+R b8a65915c3bbacc7a62e1d6f19ca7f03
+U dan
+Z bdcaf8016833561b09539543f7f27b88

diff --git a/manifest.uuid b/manifest.uuid
index 86acda274a5c6192cecf8b82985067c5eb331064..61482edf96492253c0aff0d9e0a288dda3232b5a 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-08faee686eb2fabe0dde51231ee55880e78541e8
\ No newline at end of file
+23dc6fb5b28712d1ba18dc7ddb3f2ef3b469d611
\ No newline at end of file

diff --git a/test/fts3.test b/test/fts3.test
index 38a08229acddd89026769466d8638ea6829da693..b22ec887235417d427388af5da4ef0cc0b538e6c 100644 (file)
--- a/test/fts3.test
+++ b/test/fts3.test
@@ -43,6 +43,7 @@ set ISQUICK 1
 set EXCLUDE {
   fts3.test
   fts3malloc.test
+  fts3rnd.test
 }
 
 # Files to include in the test.  If this list is empty then everything