Fix an FTS problem triggered by querying for an N character prefix using an N+1 chara...

FossilOrigin-Name: c05c3fd20d93f430140d762ead23bacd337ffb4d

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index 88a4148acb711f8f10f9dc17ac3c1c82dd5d86ca..d17809fe545b2f9d1e951af832b7717083545b49 100644 (file)
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -2600,7 +2600,9 @@ static int fts3SegReaderCursor(
       }
  
       rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, 
-          iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg
+          (isPrefix==0 && isScan==0),
+          iStartBlock, iLeavesEndBlock, 
+          iEndBlock, zRoot, nRoot, &pSeg
       );
       if( rc!=SQLITE_OK ) goto finished;
       rc = fts3SegReaderCursorAppend(pCsr, pSeg);

diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h
index 78392ec3f815c06eb1e3659b05183993152a0435..16c3de05786855a7ca37d654826e2aa610908e07 100644 (file)
--- a/ext/fts3/fts3Int.h
+++ b/ext/fts3/fts3Int.h
@@ -401,7 +401,7 @@ int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
 int sqlite3Fts3PendingTermsFlush(Fts3Table *);
 void sqlite3Fts3PendingTermsClear(Fts3Table *);
 int sqlite3Fts3Optimize(Fts3Table *);
-int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
+int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64,
   sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
 int sqlite3Fts3SegReaderPending(
   Fts3Table*,int,const char*,int,int,Fts3SegReader**);

diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index 0ee7ea81f1331725feb581aff84480fc40287ad8..8ca54a571d185507eb1816bc639ce89be3f3ec01 100644 (file)
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -110,6 +110,7 @@ struct Fts3DeferredToken {
 */
 struct Fts3SegReader {
   int iIdx;                       /* Index within level, or 0x7FFFFFFF for PT */
+  int bLookup;                    /* True for a lookup only */
 
   sqlite3_int64 iStartBlock;      /* Rowid of first leaf block to traverse */
   sqlite3_int64 iLeafEndBlock;    /* Rowid of final leaf block to traverse */
@@ -1088,6 +1089,18 @@ static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){
   return rc;
 }
 
+/*
+** Set an Fts3SegReader cursor to point at EOF.
+*/
+static void fts3SegReaderSetEof(Fts3SegReader *pSeg){
+  if( !fts3SegReaderIsRootOnly(pSeg) ){
+    sqlite3_free(pSeg->aNode);
+    sqlite3_blob_close(pSeg->pBlob);
+    pSeg->pBlob = 0;
+  }
+  pSeg->aNode = 0;
+}
+
 /*
 ** Move the iterator passed as the first argument to the next term in the
 ** segment. If successful, SQLITE_OK is returned. If there is no next term,
@@ -1127,12 +1140,7 @@ static int fts3SegReaderNext(
       return SQLITE_OK;
     }
 
-    if( !fts3SegReaderIsRootOnly(pReader) ){
-      sqlite3_free(pReader->aNode);
-      sqlite3_blob_close(pReader->pBlob);
-      pReader->pBlob = 0;
-    }
-    pReader->aNode = 0;
+    fts3SegReaderSetEof(pReader);
 
     /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf 
     ** blocks have already been traversed.  */
@@ -1379,6 +1387,7 @@ void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
 */
 int sqlite3Fts3SegReaderNew(
   int iAge,                       /* Segment "age". */
+  int bLookup,                    /* True for a lookup only */
   sqlite3_int64 iStartLeaf,       /* First leaf to traverse */
   sqlite3_int64 iEndLeaf,         /* Final leaf to traverse */
   sqlite3_int64 iEndBlock,        /* Final block of segment */
@@ -1400,6 +1409,7 @@ int sqlite3Fts3SegReaderNew(
   }
   memset(pReader, 0, sizeof(Fts3SegReader));
   pReader->iIdx = iAge;
+  pReader->bLookup = bLookup;
   pReader->iStartBlock = iStartLeaf;
   pReader->iLeafEndBlock = iEndLeaf;
   pReader->iEndBlock = iEndBlock;
@@ -2402,11 +2412,16 @@ static int fts3SegReaderStart(
   ** b-tree leaf nodes contain more than one term.
   */
   for(i=0; pCsr->bRestart==0 && i<pCsr->nSegment; i++){
+    int res;
     Fts3SegReader *pSeg = pCsr->apSegment[i];
     do {
       int rc = fts3SegReaderNext(p, pSeg, 0);
       if( rc!=SQLITE_OK ) return rc;
-    }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
+    }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 );
+
+    if( pSeg->bLookup && res!=0 ){
+      fts3SegReaderSetEof(pSeg);
+    }
   }
   fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp);
 
@@ -2527,7 +2542,12 @@ int sqlite3Fts3SegReaderStep(
     ** forward. Then sort the list in order of current term again.  
     */
     for(i=0; i<pCsr->nAdvance; i++){
-      rc = fts3SegReaderNext(p, apSegment[i], 0);
+      Fts3SegReader *pSeg = apSegment[i];
+      if( pSeg->bLookup ){
+        fts3SegReaderSetEof(pSeg);
+      }else{
+        rc = fts3SegReaderNext(p, pSeg, 0);
+      }
       if( rc!=SQLITE_OK ) return rc;
     }
     fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp);

diff --git a/manifest b/manifest
index 3c525f8dcb76d145e4ec9754f774c9aefcaa9049..89d526200f1b56923f6aaca8a0f15f9fd548059c 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Changes\sto\sthe\sasync-io\smodule\sso\sthat\sthe\sxFileControl\smethod\sreturns\sSQLITE_NOTFOUND\swhen\sa\sfile-control\sis\snot\srecognized\sand\sso\sthat\sit\sadds\sthe\ssecond\snul-terminator\sbyte\sto\sfilenames\spassed\sto\sthe\sxOpen\smethod\sof\sthe\sunderlying\sVFS.
-D 2012-01-24T10:08:26.737
+C Fix\san\sFTS\sproblem\striggered\sby\squerying\sfor\san\sN\scharacter\sprefix\susing\san\sN+1\scharacter\sprefix\sindex\safter\srows\shave\sbeen\sdeleted\sfrom\sthe\sFTS\stable.
+D 2012-01-25T16:29:45.749
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -63,9 +63,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c bd570b99f1f65b17d587361a421d7f2f28082aa0
+F ext/fts3/fts3.c 4cf7b8e5bbb6667f5d7818fa0bf064fbbb72b086
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h def7a900f98c5ab5fa4772e922bfa219d5097f05
+F ext/fts3/fts3Int.h ce958a6fa92a95462853aa3acc0b69bcda39102f
 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691
 F ext/fts3/fts3_expr.c f5df26bddf46a5916b2a5f80c4027996e92b7b15
 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914
@@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894
 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106
 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68
-F ext/fts3/fts3_write.c fdf0c99830360146ec7128150271c8c014a8fef7
+F ext/fts3/fts3_write.c 632fe0afc51a4755efb414cb0cc8cc757b9ffa18
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@@ -465,7 +465,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
 F test/fts3ao.test e7b80272efcced57d1d087a9da5c690dd7c21fd9
 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3
-F test/fts3auto.test c1a30b37002b7c764a96937fbc71065b73d69494
+F test/fts3auto.test 868a2afea308d7d8b45ef29fcf022644a9e6d662
 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0
 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
@@ -488,6 +488,7 @@ F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be
 F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6
 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844
 F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1
+F test/fts3prefix2.test 477ca96e67f60745b7ac931cfa6e9b080c562da5
 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2
 F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0
 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2
@@ -987,7 +988,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P ab80f2c3b2f1eea6e35989d3e9b6137b51091754
-R 3216920d2e27b11857b2b54fad4907a8
+P 7036886e83fccad32187668306ee2ae3f950dfce
+R 57813aba0c543953696b45a4603bd663
 U dan
-Z f6abf7311320f3c384ed2a6d12cef7c5
+Z c267c1375aaf1973adf1dbf05cf88f8d

diff --git a/manifest.uuid b/manifest.uuid
index 3e9cd2ece8b9a9ae36e64d5fad37112bfc9a959e..7e71e75c35f7ec3d63bb0e41574c07aec4fc7d75 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7036886e83fccad32187668306ee2ae3f950dfce
\ No newline at end of file
+c05c3fd20d93f430140d762ead23bacd337ffb4d
\ No newline at end of file

diff --git a/test/fts3auto.test b/test/fts3auto.test
index 1c58a172041948d857f9e4921f11dbf513a3f1df..e5c46a0af4146e75c58be3960e0659bd0d59b6d6 100644 (file)
--- a/test/fts3auto.test
+++ b/test/fts3auto.test
@@ -573,10 +573,10 @@ set chunkconfig [fts3_configure_incr_load 1 1]
 foreach {tn create pending} {
   1    "fts4(a, b)"                                  1
   2    "fts4(a, b, order=ASC, prefix=1)"             1
-  3    "fts4(a, b, order=ASC,  prefix=1,3)"          0
-  4    "fts4(a, b, order=DESC, prefix=2,4)"          0
-  5    "fts4(a, b, order=DESC, prefix=1)"            0
-  6    "fts4(a, b, order=ASC,  prefix=1,3)"          0
+  3    "fts4(a, b, order=ASC,  prefix=\"1,3\")"      0
+  4    "fts4(a, b, order=DESC, prefix=\"2,4\")"      0
+  5    "fts4(a, b, order=DESC, prefix=\"1\")"        0
+  6    "fts4(a, b, order=ASC,  prefix=\"1,3\")"      0
 } {
 
   execsql [subst {

diff --git a/test/fts3prefix2.test b/test/fts3prefix2.test
new file mode 100644 (file)
index 0000000..e3da3b7
--- /dev/null
+++ b/test/fts3prefix2.test
@@ -0,0 +1,62 @@
+# 2012 January 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS3 module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts3prefix2
+
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 { PRAGMA page_size = 512 }
+do_execsql_test 1.1 { 
+  CREATE VIRTUAL TABLE t1 USING fts4(x, prefix="2,3");
+
+  BEGIN;
+    INSERT INTO t1 VALUES('T TX T TX T TX T TX T TX');
+    INSERT INTO t1 SELECT * FROM t1;                       -- 2
+    INSERT INTO t1 SELECT * FROM t1;                       -- 4
+    INSERT INTO t1 SELECT * FROM t1;                       -- 8
+    INSERT INTO t1 SELECT * FROM t1;                       -- 16
+    INSERT INTO t1 SELECT * FROM t1;                       -- 32
+    INSERT INTO t1 SELECT * FROM t1;                       -- 64
+    INSERT INTO t1 SELECT * FROM t1;                       -- 128
+    INSERT INTO t1 SELECT * FROM t1;                       -- 256
+    INSERT INTO t1 SELECT * FROM t1;                       -- 512
+    INSERT INTO t1 SELECT * FROM t1;                       -- 1024
+    INSERT INTO t1 SELECT * FROM t1;                       -- 2048
+  COMMIT;
+}
+
+do_execsql_test 1.2 {
+  INSERT INTO t1 SELECT * FROM t1 LIMIT 10;
+  INSERT INTO t1 SELECT * FROM t1 LIMIT 10;
+  INSERT INTO t1 SELECT * FROM t1 LIMIT 10;
+  DELETE FROM t1 WHERE docid > 5;
+}
+
+do_execsql_test 1.3 {
+  SELECT * FROM t1 WHERE t1 MATCH 'T*';
+} {
+  {T TX T TX T TX T TX T TX}
+  {T TX T TX T TX T TX T TX}
+  {T TX T TX T TX T TX T TX}
+  {T TX T TX T TX T TX T TX}
+  {T TX T TX T TX T TX T TX}
+}
+
+finish_test
+