Further streamlining of fts5 prefix query code.

FossilOrigin-Name: ca91bd8ac70a5b3fef127364f73ec675e58bb92c

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index fc7158490910b55d64a906bf34a0edd0c84cdc60..6318af900f74ded4a6cce49a2c6e6d73ac8054a0 100644 (file)
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -282,6 +282,7 @@ struct Fts5PoslistWriter {
   i64 iPrev;
 };
 int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
+void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
 
 int sqlite3Fts5PoslistNext64(
   const u8 *a, int n,             /* Buffer containing poslist */

diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c
index d69adf7d240320012c839c42cc202aa40265cae2..878240af4f5a445c58bf21ef7a1ea736195d0103 100644 (file)
--- a/ext/fts5/fts5_buffer.c
+++ b/ext/fts5/fts5_buffer.c
@@ -16,18 +16,20 @@
 #include "fts5Int.h"
 
 int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
-  u32 nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
-  u8 *pNew;
-  while( nNew<nByte ){
-    nNew = nNew * 2;
-  }
-  pNew = sqlite3_realloc(pBuf->p, nNew);
-  if( pNew==0 ){
-    *pRc = SQLITE_NOMEM;
-    return 1;
-  }else{
-    pBuf->nSpace = nNew;
-    pBuf->p = pNew;
+  if( pBuf->nSpace<nByte ){
+    u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
+    u8 *pNew;
+    while( nNew<nByte ){
+      nNew = nNew * 2;
+    }
+    pNew = sqlite3_realloc(pBuf->p, nNew);
+    if( pNew==0 ){
+      *pRc = SQLITE_NOMEM;
+      return 1;
+    }else{
+      pBuf->nSpace = nNew;
+      pBuf->p = pNew;
+    }
   }
   return 0;
 }
@@ -208,23 +210,37 @@ int sqlite3Fts5PoslistReaderInit(
   return pIter->bEof;
 }
 
+/*
+** Append position iPos to the position list being accumulated in buffer
+** pBuf, which must be already be large enough to hold the new data.
+** The previous position written to this list is *piPrev. *piPrev is set
+** to iPos before returning.
+*/
+void sqlite3Fts5PoslistSafeAppend(
+  Fts5Buffer *pBuf, 
+  i64 *piPrev, 
+  i64 iPos
+){
+  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
+  if( (iPos & colmask) != (*piPrev & colmask) ){
+    pBuf->p[pBuf->n++] = 1;
+    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
+    *piPrev = (iPos & colmask);
+  }
+  pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
+  *piPrev = iPos;
+}
+
 int sqlite3Fts5PoslistWriterAppend(
   Fts5Buffer *pBuf, 
   Fts5PoslistWriter *pWriter,
   i64 iPos
 ){
   static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
-  int rc = SQLITE_OK;
-  if( 0==fts5BufferGrow(&rc, pBuf, 5+5+5) ){
-    if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
-      pBuf->p[pBuf->n++] = 1;
-      pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
-      pWriter->iPrev = (iPos & colmask);
-    }
-    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2);
-    pWriter->iPrev = iPos;
-  }
-  return rc;
+  int rc;
+  if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc;
+  sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
+  return SQLITE_OK;
 }
 
 void *sqlite3Fts5MallocZero(int *pRc, int nByte){

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 0461f3a778459199ed23d65475921ed4f5c6dd24..3dd9a9c92dd5c6dd9a76e3ac779988166fa9d080 100644 (file)
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4704,6 +4704,7 @@ static void fts5MergePrefixLists(
   Fts5Buffer *p1,                 /* First list to merge */
   Fts5Buffer *p2                  /* Second list to merge */
 ){
+
   if( p2->n ){
     if( p1->n==0 ){
       fts5BufferSwap(p1, p2);
@@ -4719,7 +4720,6 @@ static void fts5MergePrefixLists(
       fts5DoclistIterInit(p2, &i2);
 
       while( 1 ){
-
         if( i1.iRowid<i2.iRowid ){
           /* Copy entry from i1 */
           fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
@@ -4742,32 +4742,57 @@ static void fts5MergePrefixLists(
           u8 *a1 = &i1.aPoslist[i1.nSize];
           u8 *a2 = &i2.aPoslist[i2.nSize];
 
+          i64 iPrev = 0;
           Fts5PoslistWriter writer;
           memset(&writer, 0, sizeof(writer));
 
           /* Merge the two position lists. */ 
           fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
           fts5BufferZero(&tmp);
+          sqlite3Fts5BufferSize(&p->rc, &tmp, i1.nPoslist + i2.nPoslist);
+          if( p->rc ) break;
 
           sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
           sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
+          assert( iPos1>=0 && iPos2>=0 );
 
-          while( iPos1>=0 || iPos2>=0 ){
-            i64 iNew;
-            if( iPos2<0 || (iPos1>=0 && iPos1<iPos2) ){
-              iNew = iPos1;
-              sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
-            }else{
-              iNew = iPos2;
-              sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
-              if( iPos1==iPos2 ){
-                sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1);
+          if( iPos1<iPos2 ){
+            sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
+            sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
+          }else{
+            sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
+            sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
+          }
+
+          if( iPos1>=0 && iPos2>=0 ){
+            while( 1 ){
+              if( iPos1<iPos2 ){
+                if( iPos1!=iPrev ){
+                  sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
+                }
+                sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
+                if( iPos1<0 ) break;
+              }else{
+                if( iPos2!=iPrev ){
+                  sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
+                }
+                sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
+                if( iPos2<0 ) break;
               }
             }
-            if( iNew!=writer.iPrev || tmp.n==0 ){
-              p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
-              if( p->rc ) goto error_out;
+          }
+
+          if( iPos1>=0 ){
+            if( iPos1!=iPrev ){
+              sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
+            }
+            fts5BufferSafeAppendBlob(&tmp, &a1[iOff1], i1.nPoslist-iOff1);
+          }
+          else if( iPos2>=0 ){
+            if( iPos2!=iPrev ){
+              sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
             }
+            fts5BufferSafeAppendBlob(&tmp, &a2[iOff2], i2.nPoslist-iOff2);
           }
 
           /* WRITEPOSLISTSIZE */
@@ -4936,7 +4961,7 @@ int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
 int sqlite3Fts5IndexRollback(Fts5Index *p){
   fts5CloseReader(p);
   fts5IndexDiscardData(p);
-  assert( p->rc==SQLITE_OK );
+  /* assert( p->rc==SQLITE_OK ); */
   return SQLITE_OK;
 }
 

diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 9ef8d9a01a33d0be22d304143c78ce6c85dacd13..4b798314c033af622f6f323aef6c8cf25c849208 100644 (file)
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -803,7 +803,7 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
 */
 static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
   Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
-  int rc = SQLITE_OK;
+  int rc;
 
   assert( (pCsr->ePlan<3)==
           (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) 
@@ -820,6 +820,7 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
     switch( pCsr->ePlan ){
       case FTS5_PLAN_SPECIAL: {
         CsrFlagSet(pCsr, FTS5CSR_EOF);
+        rc = SQLITE_OK;
         break;
       }
   

diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test
index c6b09d8121b899fde72d8efc6e7f41b2549ba348..974aa781aa6a24455dac255aaeafc897fbac4d80 100644 (file)
--- a/ext/fts5/test/fts5ad.test
+++ b/ext/fts5/test/fts5ad.test
@@ -11,6 +11,8 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is testing the FTS5 module.
 #
+# More specifically, the focus is on testing prefix queries, both with and
+# without prefix indexes.
 #
 
 source [file join [file dirname [info script]] fts5_common.tcl]

diff --git a/ext/fts5/test/fts5detail.test b/ext/fts5/test/fts5detail.test
index 5bdd14424ee0ee4e663af49d15ba7c2e67acf9f2..58fda3e995483f056cb8b2f93ac008909744d5ae 100644 (file)
--- a/ext/fts5/test/fts5detail.test
+++ b/ext/fts5/test/fts5detail.test
@@ -85,6 +85,10 @@ do_execsql_test 2.2 {
   SELECT fts5_test_poslist(t2) FROM t2('aa');
 } {0.0.0}
 
+do_execsql_test 2.3 {
+  SELECT fts5_test_collist(t2) FROM t2('aa');
+} {0.0}
+
 set ::pc 0
 #puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*]
 #exit

diff --git a/manifest b/manifest
index 35ca92466bda9491a9baf9c022fea0ed261b8561..76600b81905c16c3e887f7ce3314e8075f9988f5 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Make\ssure\sthe\s"bak.db"\sdatabase\sfile\sdoes\snot\sactually\sexist\sbefore\nstarting\sthe\s"quota.test"\stesting.
-D 2016-02-05T17:49:43.402
+C Further\sstreamlining\sof\sfts5\sprefix\squery\scode.
+D 2016-02-05T19:18:02.811
 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 72b7858f02017611c3ac1ddc965251017fed0845
@@ -98,14 +98,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
-F ext/fts5/fts5Int.h 2095cc38e776f19cc083ca90e00772ea0b204ab3
+F ext/fts5/fts5Int.h efb02807dbe5a2bfb0ea592a472d1171cb553d53
 F ext/fts5/fts5_aux.c b9bcce753ef5b451267b2232f0ca153ddeb3951d
-F ext/fts5/fts5_buffer.c f6e0c6018ffc8e39fc0b333b5daa8b8d528ae6e4
+F ext/fts5/fts5_buffer.c ad4bb545c866eea6add1b0f84c2c5029cd689092
 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238
 F ext/fts5/fts5_expr.c ff5c451a6d025909639ac0f0d0af0cc595b50feb
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
-F ext/fts5/fts5_index.c 614c54544f844387ba6728c613a884a1aa151f06
-F ext/fts5/fts5_main.c 7e8a5f27d504bc04e3de7f1cba8867f0332aee9d
+F ext/fts5/fts5_index.c 9706959f2188d97d72df750519fee7baccef9964
+F ext/fts5/fts5_main.c 6e23df904049edb498538bd3e22e53ec1ab6f4f7
 F ext/fts5/fts5_storage.c 2a1f44deae090cd711f02cec0c2af8e660360d24
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
 F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070
@@ -120,7 +120,7 @@ F ext/fts5/test/fts5_common.tcl 61ff0d1a29d98a91c4553b20b3f410d858834ee9
 F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084
 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
-F ext/fts5/test/fts5ad.test 0ddaa5b692ff220100ee396228838f4331399eaa
+F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
 F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c
 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
@@ -142,7 +142,7 @@ F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c
 F ext/fts5/test/fts5corrupt3.test a2b537c120bdd43c79c42fe2438d7b8c81fe5599
-F ext/fts5/test/fts5detail.test 4e971d28e7336c61ab916fc287900355dab7054d
+F ext/fts5/test/fts5detail.test ef5c690535a797413acaf5ad9b8ab5d49972df69
 F ext/fts5/test/fts5dlidx.test 13871a14641017ae42f6f1055a8067bafd44cb3d
 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
 F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
@@ -1426,7 +1426,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P a6b35273239669189f33402144f58328b133d182
-R a53abbe37c3df437dd159edcdce10b4b
-U drh
-Z b28b558e8ce57b27d1473527e7eaafc5
+P 1cac6c45eeb7dbbcda0c76ba7d3a2ca77bc725cd
+R 82ed27d217d52be944f2d4bee3dfe639
+U dan
+Z ad1bc1867d7f59ff72d365283a40efbf

diff --git a/manifest.uuid b/manifest.uuid
index 25bc3f5bbcc28d253146def139ea6964c47635c4..8902cb7ad95588fe64d78c55cbf6ae0daac578dd 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1cac6c45eeb7dbbcda0c76ba7d3a2ca77bc725cd
\ No newline at end of file
+ca91bd8ac70a5b3fef127364f73ec675e58bb92c
\ No newline at end of file