Improve performance of prefix queries without a prefix index on fts5 tables.

FossilOrigin-Name: f2f0184e9e1c9f121ee2ac864cd28c8cd8efecb5

diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c
index 1a7c0d0f8ac066f878bf5940cb2461705578d678..e2dbcbf141abffabf9e0a7f6d4285056dfc7650c 100644 (file)
--- a/ext/fts5/fts5_buffer.c
+++ b/ext/fts5/fts5_buffer.c
@@ -233,13 +233,15 @@ int sqlite3Fts5PoslistWriterAppend(
 ){
   static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
   int rc = SQLITE_OK;
-  if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
-    fts5BufferAppendVarint(&rc, pBuf, 1);
-    fts5BufferAppendVarint(&rc, pBuf, (iPos >> 32));
-    pWriter->iPrev = (iPos & colmask);
+  if( 0==sqlite3Fts5BufferGrow(&rc, pBuf, 5+5+5) ){
+    if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
+      pBuf->p[pBuf->n++] = 1;
+      pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
+      pWriter->iPrev = (iPos & colmask);
+    }
+    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2);
+    pWriter->iPrev = iPos;
   }
-  fts5BufferAppendVarint(&rc, pBuf, (iPos - pWriter->iPrev) + 2);
-  pWriter->iPrev = iPos;
   return rc;
 }
 

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 418127a51cc432a94706c8bfcda537d7bf75e004..490dbd7cb78bf5c1edd24a2fb5ae775f20ffe5e3 100644 (file)
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -307,9 +307,7 @@ struct Fts5Index {
 };
 
 struct Fts5DoclistIter {
-  u8 *a;
-  int n;
-  int i;
+  u8 *aEof;                       /* Pointer to 1 byte past end of doclist */
 
   /* Output variables. aPoslist==0 at EOF */
   i64 iRowid;
@@ -3706,10 +3704,15 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){
   return ret;
 }
 
-#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \
-  assert( pBuf->nSpace>=(pBuf->n+nBlob) );             \
-  memcpy(&pBuf->p[pBuf->n], pBlob, nBlob);             \
-  pBuf->n += nBlob;                                    \
+#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) {     \
+  assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) );             \
+  memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob);             \
+  (pBuf)->n += nBlob;                                      \
+}
+
+#define fts5BufferSafeAppendVarint(pBuf, iVal) {                \
+  (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal));  \
+  assert( (pBuf)->nSpace>=(pBuf)->n );                          \
 }
 
 /*
@@ -3989,22 +3992,28 @@ static void fts5MultiIterPoslist(
 }
 
 static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
-  if( pIter->i<pIter->n ){
-    int bDummy;
-    if( pIter->i ){
-      i64 iDelta;
-      pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&iDelta);
-      pIter->iRowid += iDelta;
+  u8 *p = pIter->aPoslist + pIter->nPoslist;
+
+  assert( pIter->aPoslist );
+  if( p>=pIter->aEof ){
+    pIter->aPoslist = 0;
+  }else{
+    i64 iDelta;
+
+    p += fts5GetVarint(p, (u64*)&iDelta);
+    pIter->iRowid += iDelta;
+
+    /* Read position list size */
+    if( p[0] & 0x80 ){
+      int nPos;
+      p += fts5GetVarint32(p, nPos);
+      pIter->nPoslist = (nPos>>1);
     }else{
-      pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
+      pIter->nPoslist = ((int)(p[0])) >> 1;
+      p++;
     }
-    pIter->i += fts5GetPoslistSize(
-        &pIter->a[pIter->i], &pIter->nPoslist, &bDummy
-    );
-    pIter->aPoslist = &pIter->a[pIter->i];
-    pIter->i += pIter->nPoslist;
-  }else{
-    pIter->aPoslist = 0;
+
+    pIter->aPoslist = p;
   }
 }
 
@@ -4013,27 +4022,34 @@ static void fts5DoclistIterInit(
   Fts5DoclistIter *pIter
 ){
   memset(pIter, 0, sizeof(*pIter));
-  pIter->a = pBuf->p;
-  pIter->n = pBuf->n;
+  pIter->aPoslist = pBuf->p;
+  pIter->aEof = &pBuf->p[pBuf->n];
   fts5DoclistIterNext(pIter);
 }
 
+#if 0
 /*
 ** Append a doclist to buffer pBuf.
+**
+** This function assumes that space within the buffer has already been
+** allocated.
 */
 static void fts5MergeAppendDocid(
-  int *pRc,                       /* IN/OUT: Error code */
   Fts5Buffer *pBuf,               /* Buffer to write to */
   i64 *piLastRowid,               /* IN/OUT: Previous rowid written (if any) */
   i64 iRowid                      /* Rowid to append */
 ){
-  if( pBuf->n==0 ){
-    fts5BufferAppendVarint(pRc, pBuf, iRowid);
-  }else{
-    fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid);
-  }
+  assert( pBuf->n!=0 || (*piLastRowid)==0 );
+  fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid);
   *piLastRowid = iRowid;
 }
+#endif
+
+#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) {       \
+  assert( (pBuf)->n!=0 || (iLastRowid)==0 );                   \
+  fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \
+  (iLastRowid) = (iRowid);                                     \
+}
 
 /*
 ** Buffers p1 and p2 contain doclists. This function merges the content
@@ -4057,53 +4073,60 @@ static void fts5MergePrefixLists(
     memset(&out, 0, sizeof(out));
     memset(&tmp, 0, sizeof(tmp));
 
+    sqlite3Fts5BufferGrow(&p->rc, &out, p1->n + p2->n);
     fts5DoclistIterInit(p1, &i1);
     fts5DoclistIterInit(p2, &i2);
     while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){
       if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowid<i2.iRowid) ){
         /* Copy entry from i1 */
-        fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i1.iRowid);
+        fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
         /* WRITEPOSLISTSIZE */
-        fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist * 2);
-        fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist);
+        fts5BufferSafeAppendVarint(&out, i1.nPoslist * 2);
+        fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist);
         fts5DoclistIterNext(&i1);
       }
       else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){
         /* Copy entry from i2 */
-        fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid);
+        fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
         /* WRITEPOSLISTSIZE */
-        fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist * 2);
-        fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist);
+        fts5BufferSafeAppendVarint(&out, i2.nPoslist * 2);
+        fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist);
         fts5DoclistIterNext(&i2);
       }
       else{
-        Fts5PoslistReader r1;
-        Fts5PoslistReader r2;
-        Fts5PoslistWriter writer;
+        i64 iPos1 = 0;
+        i64 iPos2 = 0;
+        int iOff1 = 0;
+        int iOff2 = 0;
 
+        Fts5PoslistWriter writer;
         memset(&writer, 0, sizeof(writer));
 
         /* Merge the two position lists. */ 
-        fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid);
+        fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
         fts5BufferZero(&tmp);
-        sqlite3Fts5PoslistReaderInit(-1, i1.aPoslist, i1.nPoslist, &r1);
-        sqlite3Fts5PoslistReaderInit(-1, i2.aPoslist, i2.nPoslist, &r2);
-        while( p->rc==SQLITE_OK && (r1.bEof==0 || r2.bEof==0) ){
+
+        sqlite3Fts5PoslistNext64(i1.aPoslist, i1.nPoslist, &iOff1, &iPos1);
+        sqlite3Fts5PoslistNext64(i2.aPoslist, i2.nPoslist, &iOff2, &iPos2);
+
+        while( p->rc==SQLITE_OK && (iPos1>=0 || iPos2>=0) ){
           i64 iNew;
-          if( r2.bEof || (r1.bEof==0 && r1.iPos<r2.iPos) ){
-            iNew = r1.iPos;
-            sqlite3Fts5PoslistReaderNext(&r1);
+          if( iPos2<0 || (iPos1>=0 && iPos1<iPos2) ){
+            iNew = iPos1;
+            sqlite3Fts5PoslistNext64(i1.aPoslist, i1.nPoslist, &iOff1, &iPos1);
           }else{
-            iNew = r2.iPos;
-            sqlite3Fts5PoslistReaderNext(&r2);
-            if( r1.iPos==r2.iPos ) sqlite3Fts5PoslistReaderNext(&r1);
+            iNew = iPos2;
+            sqlite3Fts5PoslistNext64(i2.aPoslist, i2.nPoslist, &iOff2, &iPos2);
+            if( iPos1==iPos2 ){
+              sqlite3Fts5PoslistNext64(i1.aPoslist, i1.nPoslist, &iOff1,&iPos1);
+            }
           }
           p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
         }
 
         /* WRITEPOSLISTSIZE */
-        fts5BufferAppendVarint(&p->rc, &out, tmp.n * 2);
-        fts5BufferAppendBlob(&p->rc, &out, tmp.n, tmp.p);
+        fts5BufferSafeAppendVarint(&out, tmp.n * 2);
+        fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n);
         fts5DoclistIterNext(&i1);
         fts5DoclistIterNext(&i2);
       }
@@ -4165,14 +4188,19 @@ static void fts5SetupPrefixIter(
             fts5BufferZero(&aBuf[i]);
           }
         }
+        iLastRowid = 0;
       }
 
-      fts5MergeAppendDocid(&p->rc, &doclist, &iLastRowid, iRowid);
-      fts5MultiIterPoslist(p, p1, 1, &doclist);
+      if( 0==sqlite3Fts5BufferGrow(&p->rc, &doclist, 9) ){
+        fts5MergeAppendDocid(&doclist, iLastRowid, iRowid);
+        fts5MultiIterPoslist(p, p1, 1, &doclist);
+      }
     }
 
     for(i=0; i<nBuf; i++){
-      fts5MergePrefixLists(p, &doclist, &aBuf[i]);
+      if( p->rc==SQLITE_OK ){
+        fts5MergePrefixLists(p, &doclist, &aBuf[i]);
+      }
       fts5BufferFree(&aBuf[i]);
     }
     fts5MultiIterFree(p, p1);

diff --git a/manifest b/manifest
index a6a5c2b408dc71e6e950e2d6dda3546544118b7b..5081c72572a2b753b371a55acbccaf5e7fda38e3 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sfts3\sso\sthat\sexpressions\sto\sthe\sleft\sand\sright\sof\sa\sNOT\soperator\sare\sbalanced.\sThis\sprevents\srelatively\ssmall\sexpressions\s(a\sdozen\sterms\sor\sso)\sthat\sare\schildren\sof\sNOT\soperators\sfrom\striggering\sthe\s"expression\stree\sis\stoo\slarge"\serror.
-D 2015-10-05T15:39:45.681
+C Improve\sperformance\sof\sprefix\squeries\swithout\sa\sprefix\sindex\son\sfts5\stables.
+D 2015-10-05T19:41:16.348
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 2143eeef6d0cc26006ae5fc4bb242a4a8b973412
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -108,11 +108,11 @@ F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 98f802fe41481f9d797fce496f0fefcad72c7782
 F ext/fts5/fts5Int.h ff78a77d819a7fc04a7f8b08b0e1ce361a3395e4
 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e
-F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4
+F ext/fts5/fts5_buffer.c 54b18497395a19dfe1d00f63a3b403e5f93d4fd1
 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695
 F ext/fts5/fts5_expr.c 667faaf14a69a5683ac383acdc8d942cf32c3f93
 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
-F ext/fts5/fts5_index.c 00d2593f94ede440ea274f8db21864cf41632aa3
+F ext/fts5/fts5_index.c ca3912a44ef5a173ef098f3454465519bd4b8e88
 F ext/fts5/fts5_main.c fe5243d6bbb79217394f0ec7f4f5199ddbc9e7e8
 F ext/fts5/fts5_storage.c df061a5caf9e50fbbd43113009b5b248362f4995
 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37
@@ -1391,7 +1391,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 44f1ce30d1e446c9ee5f8bd8a62119e00356aa0e
-R 0954493ffc80fb341b0df1cab997ace8
+P d6b66cd7b89fbd964f798d160a34caac0ba7347a
+R 0053d2d2b912a0e9b79250c67681b097
 U dan
-Z 52900cdc8d31460ed8eb8d6276be0e92
+Z c981c12e0ed92ecd6400cdfa9ecf4084

diff --git a/manifest.uuid b/manifest.uuid
index a2a1d63f6e5ed0f23c0bfb220274df2d3dfdd6ef..fcb70f59075e2715b15a2d341cd4e12b11173fed 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d6b66cd7b89fbd964f798d160a34caac0ba7347a
\ No newline at end of file
+f2f0184e9e1c9f121ee2ac864cd28c8cd8efecb5
\ No newline at end of file