Improve performance of fts5 queries.

FossilOrigin-Name: 2334e88244afe6387208be5d527aba9b5ddf4e4c

diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 3a1497f05de3bde1b762750007d493d3f445c09f..7c8e3c87c1c998eaa923f279ca58e83789e3fa25 100644 (file)
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -62,6 +62,9 @@ struct Fts5ExprNode {
   int bEof;                       /* True at EOF */
   int bNomatch;                   /* True if entry is not a match */
 
+  /* Next method for this node. */
+  int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64);
+
   i64 iRowid;                     /* Current rowid */
   Fts5ExprNearset *pNear;         /* For FTS5_STRING - cluster of phrases */
 
@@ -73,6 +76,12 @@ struct Fts5ExprNode {
 
 #define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING)
 
+/*
+** Invoke the xNext method of an Fts5ExprNode object. This macro should be
+** used as if it has the same signature as the xNext() methods themselves.
+*/
+#define fts5ExprNodeNext(a,b,c,d) (b)->xNext((a), (b), (c), (d))
+
 /*
 ** An instance of the following structure represents a single search term
 ** or term prefix.
@@ -234,7 +243,15 @@ int sqlite3Fts5ExprNew(
       sParse.rc = SQLITE_NOMEM;
       sqlite3Fts5ParseNodeFree(sParse.pExpr);
     }else{
-      pNew->pRoot = sParse.pExpr;
+      if( !sParse.pExpr ){
+        const int nByte = sizeof(Fts5ExprNode);
+        pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte);
+        if( pNew->pRoot ){
+          pNew->pRoot->bEof = 1;
+        }
+      }else{
+        pNew->pRoot = sParse.pExpr;
+      }
       pNew->pIndex = 0;
       pNew->pConfig = pConfig;
       pNew->apExprPhrase = sParse.apPhrase;
@@ -500,12 +517,6 @@ static int fts5LookaheadReaderInit(
   return fts5LookaheadReaderNext(p);
 }
 
-#if 0
-static int fts5LookaheadReaderEof(Fts5LookaheadReader *p){
-  return (p->iPos==FTS5_LOOKAHEAD_EOF);
-}
-#endif
-
 typedef struct Fts5NearTrimmer Fts5NearTrimmer;
 struct Fts5NearTrimmer {
   Fts5LookaheadReader reader;     /* Input iterator */
@@ -636,6 +647,7 @@ static int fts5ExprNearAdvanceFirst(
   Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0];
   int rc = SQLITE_OK;
 
+  pNode->bNomatch = 0;
   if( pTerm->pSynonym ){
     int bEof = 1;
     Fts5ExprTerm *p;
@@ -765,17 +777,6 @@ static int fts5ExprNearTest(
     for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
       Fts5IndexIter *pIter = pTerm->pIter;
       if( sqlite3Fts5IterEof(pIter)==0 ){
-#if 0
-        int n;
-        i64 iRowid;
-        rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid);
-        if( rc!=SQLITE_OK ){
-          *pRc = rc;
-          return 0;
-        }else if( iRowid==pNode->iRowid && n>0 ){
-          pPhrase->poslist.n = 1;
-        }
-#endif
         if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){
           pPhrase->poslist.n = 1;
         }
@@ -797,11 +798,6 @@ static int fts5ExprNearTest(
       }else{
         Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
         fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData);
-#if 0
-        rc = sqlite3Fts5IterPoslistBuffer(
-            pPhrase->aTerm[0].pIter, &pPhrase->poslist
-        );
-#endif
       }
     }
 
@@ -888,6 +884,7 @@ static int fts5ExprNearNextMatch(
           if( iRowid==iLast ) continue;
           bMatch = 0;
           if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){
+            pNode->bNomatch = 0;
             pNode->bEof = 1;
             return rc;
           }
@@ -905,7 +902,8 @@ static int fts5ExprNearNextMatch(
   }while( bMatch==0 );
 
   pNode->iRowid = iLast;
-  pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode));
+  pNode->bNomatch = ((0==fts5ExprNearTest(&rc, pExpr, pNode)) && rc==SQLITE_OK);
+  assert( pNode->bEof==0 || pNode->bNomatch==0 );
 
   return rc;
 }
@@ -923,6 +921,7 @@ static int fts5ExprNearInitAll(
   int i, j;
   int rc = SQLITE_OK;
 
+  assert( pNode->bNomatch==0 );
   for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
     Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
     for(j=0; j<pPhrase->nTerm; j++){
@@ -990,6 +989,7 @@ static int fts5RowidCmp(
 static void fts5ExprSetEof(Fts5ExprNode *pNode){
   int i;
   pNode->bEof = 1;
+  pNode->bNomatch = 0;
   for(i=0; i<pNode->nChild; i++){
     fts5ExprSetEof(pNode->apChild[i]);
   }
@@ -1012,8 +1012,6 @@ static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){
 }
 
 
-static int fts5ExprNodeNext(Fts5Expr*, Fts5ExprNode*, int, i64);
-
 /*
 ** Argument pNode is an FTS5_AND node.
 */
@@ -1093,6 +1091,33 @@ static int fts5NodeCompare(
   return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid);
 }
 
+/*
+** xNext() method for a node of type FTS5_TERM.
+*/
+static int fts5ExprNodeNext_Term(
+  Fts5Expr *pExpr, 
+  Fts5ExprNode *pNode,
+  int bFromValid,
+  i64 iFrom
+){
+  int rc;
+  Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter;
+
+  assert( pNode->bEof==0 );
+  if( bFromValid ){
+    rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
+  }else{
+    rc = sqlite3Fts5IterNext(pIter);
+  }
+  if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){
+    rc = fts5ExprTokenTest(pExpr, pNode);
+  }else{
+    pNode->bEof = 1;
+    pNode->bNomatch = 0;
+  }
+  return rc;
+}
+
 /*
 ** Advance node iterator pNode, part of expression pExpr. If argument
 ** bFromValid is zero, then pNode is advanced exactly once. Or, if argument
@@ -1100,7 +1125,7 @@ static int fts5NodeCompare(
 ** rowid value iFrom. Whether "past" means "less than" or "greater than"
 ** depends on whether this is an ASC or DESC iterator.
 */
-static int fts5ExprNodeNext(
+static int fts5ExprNodeNext_Fallback(
   Fts5Expr *pExpr, 
   Fts5ExprNode *pNode,
   int bFromValid,
@@ -1127,6 +1152,7 @@ static int fts5ExprNodeNext(
           rc = fts5ExprTokenTest(pExpr, pNode);
         }else{
           pNode->bEof = 1;
+          pNode->bNomatch = 0;
         }
         return rc;
       };
@@ -1180,6 +1206,7 @@ static int fts5ExprNodeNext(
       || pNode->iRowid==iFrom || pExpr->bDesc==(pNode->iRowid<iFrom)    /* c */
   );
 
+  assert( pNode->bNomatch==0 || rc==SQLITE_OK );
   return rc;
 }
 
@@ -1246,6 +1273,7 @@ static int fts5ExprNodeNextMatch(
           rc = fts5ExprNodeNext(pExpr, p1, 0, 0);
         }
         pNode->bEof = p1->bEof;
+        pNode->bNomatch = p1->bNomatch;
         pNode->iRowid = p1->iRowid;
         if( p1->bEof ){
           fts5ExprNodeZeroPoslist(p2);
@@ -1268,16 +1296,36 @@ static int fts5ExprNodeNextMatch(
 static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
   int rc = SQLITE_OK;
   pNode->bEof = 0;
+  pNode->bNomatch = 0;
 
   if( Fts5NodeIsString(pNode) ){
     /* Initialize all term iterators in the NEAR object. */
     rc = fts5ExprNearInitAll(pExpr, pNode);
   }else{
     int i;
+    int nEof = 0;
     for(i=0; i<pNode->nChild && rc==SQLITE_OK; i++){
+      Fts5ExprNode *pChild = pNode->apChild[i];
       rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]);
+      assert( pChild->bEof==0 || pChild->bEof==1 );
+      nEof += pChild->bEof;
     }
     pNode->iRowid = pNode->apChild[0]->iRowid;
+
+    switch( pNode->eType ){
+      case FTS5_AND:
+        if( nEof>0 ) fts5ExprSetEof(pNode);
+        break;
+
+      case FTS5_OR:
+        if( pNode->nChild==nEof ) fts5ExprSetEof(pNode);
+        break;
+
+      default:
+        assert( pNode->eType==FTS5_NOT );
+        pNode->bEof = pNode->apChild[0]->bEof;
+        break;
+    }
   }
 
   if( rc==SQLITE_OK ){
@@ -1305,7 +1353,7 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
 int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){
   Fts5ExprNode *pRoot = p->pRoot;
   int rc = SQLITE_OK;
-  if( pRoot ){
+  if( pRoot->xNext ){
     p->pIndex = pIdx;
     p->bDesc = bDesc;
     rc = fts5ExprNodeFirst(p, pRoot);
@@ -1333,9 +1381,11 @@ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){
 int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){
   int rc;
   Fts5ExprNode *pRoot = p->pRoot;
+  assert( pRoot->bEof==0 && pRoot->bNomatch==0 );
   do {
     rc = fts5ExprNodeNext(p, pRoot, 0, 0);
-  }while( pRoot->bNomatch && pRoot->bEof==0 && rc==SQLITE_OK );
+    assert( pRoot->bNomatch==0 || (rc==SQLITE_OK && pRoot->bEof==0) );
+  }while( pRoot->bNomatch );
   if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){
     pRoot->bEof = 1;
   }
@@ -1343,7 +1393,7 @@ int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){
 }
 
 int sqlite3Fts5ExprEof(Fts5Expr *p){
-  return (p->pRoot==0 || p->pRoot->bEof);
+  return p->pRoot->bEof;
 }
 
 i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
@@ -1640,6 +1690,7 @@ int sqlite3Fts5ExprClonePhrase(
     pNew->apExprPhrase[0] = sCtx.pPhrase;
     pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
     pNew->pRoot->pNear->nPhrase = 1;
+    pNew->pRoot->xNext = fts5ExprNodeNext_Fallback;
     sCtx.pPhrase->pNode = pNew->pRoot;
 
     if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
@@ -1840,6 +1891,7 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
     pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte);
 
     if( pRet ){
+      pRet->xNext = fts5ExprNodeNext_Fallback;
       pRet->eType = eType;
       pRet->pNear = pNear;
       if( eType==FTS5_STRING ){
@@ -1850,6 +1902,7 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
         if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){
           if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){
             pRet->eType = FTS5_TERM;
+            pRet->xNext = fts5ExprNodeNext_Term;
           }
         }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
           assert( pParse->rc==SQLITE_OK );
@@ -2144,7 +2197,7 @@ static void fts5ExprFunction(
   }
   if( rc==SQLITE_OK ){
     char *zText;
-    if( pExpr->pRoot==0 ){
+    if( pExpr->pRoot->xNext==0 ){
       zText = sqlite3_mprintf("");
     }else if( bTcl ){
       zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);

diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index ec045da6bf229f775d915872b1d9117aaa9e0f5c..9ef8d9a01a33d0be22d304143c78ce6c85dacd13 100644 (file)
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -220,10 +220,10 @@ struct Fts5Cursor {
 /*
 ** Values for Fts5Cursor.csrflags
 */
-#define FTS5CSR_REQUIRE_CONTENT   0x01
-#define FTS5CSR_REQUIRE_DOCSIZE   0x02
-#define FTS5CSR_REQUIRE_INST      0x04
-#define FTS5CSR_EOF               0x08
+#define FTS5CSR_EOF               0x01
+#define FTS5CSR_REQUIRE_CONTENT   0x02
+#define FTS5CSR_REQUIRE_DOCSIZE   0x04
+#define FTS5CSR_REQUIRE_INST      0x08
 #define FTS5CSR_FREE_ZRANK        0x10
 #define FTS5CSR_REQUIRE_RESEEK    0x20
 #define FTS5CSR_REQUIRE_POSLIST   0x40
@@ -778,7 +778,7 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
     i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
 
     rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, iRowid, bDesc);
-    if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){
+    if( rc==SQLITE_OK &&  iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){
       *pbSkip = 1;
     }
 
@@ -786,6 +786,7 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
     fts5CsrNewrow(pCsr);
     if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
       CsrFlagSet(pCsr, FTS5CSR_EOF);
+      *pbSkip = 1;
     }
   }
   return rc;
@@ -807,14 +808,13 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
   assert( (pCsr->ePlan<3)==
           (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) 
   );
+  assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) );
 
   if( pCsr->ePlan<3 ){
     int bSkip = 0;
     if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
     rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
-    if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
-      CsrFlagSet(pCsr, FTS5CSR_EOF);
-    }
+    CsrFlagSet(pCsr, sqlite3Fts5ExprEof(pCsr->pExpr));
     fts5CsrNewrow(pCsr);
   }else{
     switch( pCsr->ePlan ){

diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test
index e29c13117c00ba9963e88665276df971373bbdb5..b6563fd9d01aead35a5796ed518b7b7087e28d64 100644 (file)
--- a/ext/fts5/test/fts5simple.test
+++ b/ext/fts5/test/fts5simple.test
@@ -19,7 +19,7 @@ ifcapable !fts5 {
   return
 }
 
- if 1 {
+if 1 {
 
 #-------------------------------------------------------------------------
 #
@@ -363,8 +363,6 @@ do_execsql_test 15.1 {
   INSERT INTO x2(x2) VALUES('integrity-check');
 }
 
-}
-
 #-------------------------------------------------------------------------
 foreach_detail_mode $testprefix {
   reset_db
@@ -382,5 +380,24 @@ foreach_detail_mode $testprefix {
   } {{0.0.0 1.0.2}}
 }
 
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 17.0 {
+  CREATE VIRTUAL TABLE x3 USING fts5(x);
+  INSERT INTO x3 VALUES('a b c');
+}
+
+do_execsql_test 17.1 {
+  SELECT rowid FROM x3('b AND d');
+}
+
+#-------------------------------------------------------------------------
+do_execsql_test 18.1 {
+  CREATE VIRTUAL TABLE x4 USING fts5(x);
+  SELECT rowid FROM x4('""');
+}
+
 finish_test
 

diff --git a/manifest b/manifest
index 5ae7fd9d24261bb47abd4f501371434d41887aab..f5e3ab6b6df4a5792c543102f1718cf420a847de 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Number\sVDBE\sopcodes\sstarting\swith\s0\sinstead\sof\s1,\sas\sthis\sobviates\sthe\nlower-bound\stest\son\s"switch(opcode){...}",\smaking\sthe\scode\ssmaller\sand\sfaster.
-D 2016-02-01T17:20:08.707
+C Improve\sperformance\sof\sfts5\squeries.
+D 2016-02-01T20:12:41.276
 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 72b7858f02017611c3ac1ddc965251017fed0845
@@ -102,10 +102,10 @@ F ext/fts5/fts5Int.h 6e0f90eb4872654a5b98130dec16965716525c9a
 F ext/fts5/fts5_aux.c b9bcce753ef5b451267b2232f0ca153ddeb3951d
 F ext/fts5/fts5_buffer.c f6e0c6018ffc8e39fc0b333b5daa8b8d528ae6e4
 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238
-F ext/fts5/fts5_expr.c a66b9694519d9c336d9bdbd46ea22e7e14aef412
+F ext/fts5/fts5_expr.c 768d221e592df03b26f46da56aa0a561f00fa4e0
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
 F ext/fts5/fts5_index.c cd1e4faca8b9adc2d89b367075bf93a7f50c406b
-F ext/fts5/fts5_main.c 3886bbfc5ac1d9df29979823ddf2b68241e1127e
+F ext/fts5/fts5_main.c 7e8a5f27d504bc04e3de7f1cba8867f0332aee9d
 F ext/fts5/fts5_storage.c 2a1f44deae090cd711f02cec0c2af8e660360d24
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
 F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070
@@ -175,7 +175,7 @@ F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6
-F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9
+F ext/fts5/test/fts5simple.test 7fcacfa473a37355af2e60096650c87b5ba8f3ba
 F ext/fts5/test/fts5simple2.test 98377ae1ff7749a42c21fe1a139c1ed312522c46
 F ext/fts5/test/fts5simple3.test e671b36bc4dbd4f5095e66cb04473cba9f680f53
 F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48
@@ -1422,7 +1422,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 5d7c092869137a0ba69f93324fe4ed56a05b5985
-R c689c18fc524ba4b0a7bb357f3467ad8
-U drh
-Z 4a66c8beaa1a93ec6bbf8e1c0f940158
+P 4c9222f75bfac47f5422fff86b2d69a61933b3a2
+R 267ad0dad7a7346f85182e52086f7b51
+U dan
+Z ee79d8b7c590b68c4d6e9bbdc4c83095

diff --git a/manifest.uuid b/manifest.uuid
index cc32153f401c26d8aa4c5f8bd23e7d6a41d91629..3b468df8c833b9b2dd84459495d913aed501c4d3 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4c9222f75bfac47f5422fff86b2d69a61933b3a2
\ No newline at end of file
+2334e88244afe6387208be5d527aba9b5ddf4e4c
\ No newline at end of file