When gathering a doclist for querying, don't discard empty position lists until the...

FossilOrigin-Name: 111ca616713dd89b5d1e114de29c83256731c482

diff --git a/ext/fts1/fts1.c b/ext/fts1/fts1.c
index bfbc483a2485aee6ee0ecbd94ab311d64a7ea684..77878ac5549f7266882aedabe96f60fff7c646cd 100644 (file)
--- a/ext/fts1/fts1.c
+++ b/ext/fts1/fts1.c
@@ -339,7 +339,7 @@ static sqlite_int64 peekDocid(DocListReader *pReader){
   return ret;
 }
 
-/* Read the next docid.   See also nextValidDocid().
+/* Read the next docid.   See also nextDocid().
 */
 static sqlite_int64 readDocid(DocListReader *pReader){
   sqlite_int64 ret;
@@ -459,8 +459,8 @@ static void printDoclist(DocList *p){
 }
 #endif /* SQLITE_DEBUG */
 
-/* Trim the given doclist to contain only positions in column [iRestrictColumn],
- * discarding any docids without any remaining positions. */
+/* Trim the given doclist to contain only positions in column
+ * [iRestrictColumn]. */
 static void docListRestrictColumn(DocList *in, int iRestrictColumn){
   DocListReader r;
   DocList out;
@@ -471,14 +471,11 @@ static void docListRestrictColumn(DocList *in, int iRestrictColumn){
 
   while( !atEnd(&r) ){
     sqlite_int64 iDocid = readDocid(&r);
-    int match = 0;
     int iPos, iColumn;
+
+    docListAddDocid(&out, iDocid);
     while( (iPos = readPosition(&r, &iColumn)) != -1 ){
       if( iColumn==iRestrictColumn ){
-        if( !match ){
-          docListAddDocid(&out, iDocid);
-          match = 1;
-        }
         docListAddPos(&out, iColumn, iPos);
       }
     }
@@ -488,6 +485,35 @@ static void docListRestrictColumn(DocList *in, int iRestrictColumn){
   *in = out;
 }
 
+/* Trim the given doclist by discarding any docids without any remaining
+ * positions. */
+static void docListDiscardEmpty(DocList *in) {
+  DocListReader r;
+  DocList out;
+
+  /* TODO: It would be nice to implement this operation in place; that
+   * could save a significant amount of memory in queries with long doclists. */
+  assert( in->iType>=DL_POSITIONS );
+  readerInit(&r, in);
+  docListInit(&out, DL_POSITIONS, NULL, 0);
+
+  while( !atEnd(&r) ){
+    sqlite_int64 iDocid = readDocid(&r);
+    int match = 0;
+    int iPos, iColumn;
+    while( (iPos = readPosition(&r, &iColumn)) != -1 ){
+      if( !match ){
+        docListAddDocid(&out, iDocid);
+        match = 1;
+      }
+      docListAddPos(&out, iColumn, iPos);
+    }
+  }
+
+  docListDestroy(in);
+  *in = out;
+}
+
 /* Helper function for docListUpdate() and docListAccumulate().
 ** Splices a doclist element into the doclist represented by r,
 ** leaving r pointing after the newly spliced element.
@@ -571,16 +597,14 @@ static void docListAccumulate(DocList *pAcc, DocList *pUpdate){
 }
 
 /*
-** Read the next non-deleted docid off of pIn.  Return
-** 0 if we reach the end of pDoclist.
+** Read the next docid off of pIn.  Return 0 if we reach the end.
+*
+* TODO: This assumes that docids are never 0, but they may actually be 0 since
+* users can choose docids when inserting into a full-text table.  Fix this.
 */
-static sqlite_int64 nextValidDocid(DocListReader *pIn){
-  sqlite_int64 docid = 0;
+static sqlite_int64 nextDocid(DocListReader *pIn){
   skipPositionList(pIn);
-  while( !atEnd(pIn) && (docid = readDocid(pIn))==0 ){
-    skipPositionList(pIn);
-  }
-  return docid;
+  return atEnd(pIn) ? 0 : readDocid(pIn);
 }
 
 /*
@@ -651,18 +675,18 @@ static void docListPhraseMerge(
 
   readerInit(&left, pLeft);
   readerInit(&right, pRight);
-  docidLeft = nextValidDocid(&left);
-  docidRight = nextValidDocid(&right);
+  docidLeft = nextDocid(&left);
+  docidRight = nextDocid(&right);
 
   while( docidLeft>0 && docidRight>0 ){
     if( docidLeft<docidRight ){
-      docidLeft = nextValidDocid(&left);
+      docidLeft = nextDocid(&left);
     }else if( docidRight<docidLeft ){
-      docidRight = nextValidDocid(&right);
+      docidRight = nextDocid(&right);
     }else{
       mergePosList(&left, &right, docidLeft, pOut);
-      docidLeft = nextValidDocid(&left);
-      docidRight = nextValidDocid(&right);
+      docidLeft = nextDocid(&left);
+      docidRight = nextDocid(&right);
     }
   }
 }
@@ -685,18 +709,18 @@ static void docListAndMerge(
 
   readerInit(&left, pLeft);
   readerInit(&right, pRight);
-  docidLeft = nextValidDocid(&left);
-  docidRight = nextValidDocid(&right);
+  docidLeft = nextDocid(&left);
+  docidRight = nextDocid(&right);
 
   while( docidLeft>0 && docidRight>0 ){
     if( docidLeft<docidRight ){
-      docidLeft = nextValidDocid(&left);
+      docidLeft = nextDocid(&left);
     }else if( docidRight<docidLeft ){
-      docidRight = nextValidDocid(&right);
+      docidRight = nextDocid(&right);
     }else{
       docListAddDocid(pOut, docidLeft);
-      docidLeft = nextValidDocid(&left);
-      docidRight = nextValidDocid(&right);
+      docidLeft = nextDocid(&left);
+      docidRight = nextDocid(&right);
     }
   }
 }
@@ -717,8 +741,8 @@ static void docListOrMerge(
 
   readerInit(&left, pLeft);
   readerInit(&right, pRight);
-  docidLeft = nextValidDocid(&left);
-  docidRight = nextValidDocid(&right);
+  docidLeft = nextDocid(&left);
+  docidRight = nextDocid(&right);
 
   while( docidLeft>0 && docidRight>0 ){
     if( docidLeft<=docidRight ){
@@ -728,19 +752,19 @@ static void docListOrMerge(
     }
     priorLeft = docidLeft;
     if( docidLeft<=docidRight ){
-      docidLeft = nextValidDocid(&left);
+      docidLeft = nextDocid(&left);
     }
     if( docidRight>0 && docidRight<=priorLeft ){
-      docidRight = nextValidDocid(&right);
+      docidRight = nextDocid(&right);
     }
   }
   while( docidLeft>0 ){
     docListAddDocid(pOut, docidLeft);
-    docidLeft = nextValidDocid(&left);
+    docidLeft = nextDocid(&left);
   }
   while( docidRight>0 ){
     docListAddDocid(pOut, docidRight);
-    docidRight = nextValidDocid(&right);
+    docidRight = nextDocid(&right);
   }
 }
 
@@ -762,8 +786,8 @@ static void docListExceptMerge(
 
   readerInit(&left, pLeft);
   readerInit(&right, pRight);
-  docidLeft = nextValidDocid(&left);
-  docidRight = nextValidDocid(&right);
+  docidLeft = nextDocid(&left);
+  docidRight = nextDocid(&right);
 
   while( docidLeft>0 && docidRight>0 ){
     priorLeft = docidLeft;
@@ -771,15 +795,15 @@ static void docListExceptMerge(
       docListAddDocid(pOut, docidLeft);
     }
     if( docidLeft<=docidRight ){
-      docidLeft = nextValidDocid(&left);
+      docidLeft = nextDocid(&left);
     }
     if( docidRight>0 && docidRight<=priorLeft ){
-      docidRight = nextValidDocid(&right);
+      docidRight = nextDocid(&right);
     }
   }
   while( docidLeft>0 ){
     docListAddDocid(pOut, docidLeft);
-    docidLeft = nextValidDocid(&left);
+    docidLeft = nextDocid(&left);
   }
 }
 
@@ -1283,6 +1307,7 @@ static int term_select_all(
     return rc;
   }
 
+  docListDiscardEmpty(&doclist);
   *out = doclist;
   return SQLITE_OK;
 }
@@ -2370,7 +2395,7 @@ static int fulltextNext(sqlite3_vtab_cursor *pCursor){
     rc = sqlite3_reset(c->pStmt);
     if( rc!=SQLITE_OK ) return rc;
 
-    iDocid = nextValidDocid(&c->result);
+    iDocid = nextDocid(&c->result);
     if( iDocid==0 ){
       c->eof = 1;
       return SQLITE_OK;

diff --git a/manifest b/manifest
index 2cc15396d4fb4cd5a0d097d5eba297ca2f731429..976db60d93f5eb05fffc283583e331f0ea52bbcd 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Be\smore\saggressive\swith\sthe\sSQLITE_OMIT_VACUUM\smacro.\s\sSaves\sabout\s150\nbytes\sof\scode\sspace.\s(CVS\s3432)
-D 2006-09-21T11:02:17
+C When\sgathering\sa\sdoclist\sfor\squerying,\sdon't\sdiscard\sempty\sposition\slists\suntil\sthe\send;\sthis\sallows\sempty\sposition\slists\sto\soverride\snon-empty\slists\sencountered\slater\sin\sthe\sgathering\sprocess.\s\sThis\sfixes\s#1982,\swhich\swas\scaused\sby\sthe\sfact\sthat\sfor\sall-column\squeries\swe\sweren't\sdiscarding\sempty\sposition\slists\sat\sall.\s(CVS\s3433)
+D 2006-09-21T20:56:52
 F Makefile.in cabd42d34340f49260bc2a7668c38eba8d4cfd99
 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -21,7 +21,7 @@ F ext/README.txt 913a7bd3f4837ab14d7e063304181787658b14e1
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
 F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
 F ext/fts1/ft_hash.h 1a35e654a235c2c662d3ca0dfc3138ad60b8b7d5
-F ext/fts1/fts1.c 02c5b614ff8055b374b88acaf5cae3a834da3150
+F ext/fts1/fts1.c 16f58a0c6b7d714d83e5f0ca4a520b530f8118e4
 F ext/fts1/fts1.h 6060b8f62c1d925ea8356cb1a6598073eb9159a6
 F ext/fts1/fts1_hash.c 3196cee866edbebb1c0521e21672e6d599965114
 F ext/fts1/fts1_hash.h 957d378355ed29f672cd5add012ce8b088a5e089
@@ -399,7 +399,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513
-P c7ee60d00976efab25a830e7416538010c734129
-R 4a4f7b1d165466da2e1372b56afee139
-U drh
-Z 2d27498809c9282eb1d2a169b6b639c4
+P 7e618db4579d752cc6d775c664c93e141217948f
+R 9812eeaf3b8a5ffaeac13843969bbadf
+U adamd
+Z a38ec8fb44bc3c26289a40e8b94f4ec3

diff --git a/manifest.uuid b/manifest.uuid
index 5fe311116395dc670298f323801e78fd05849847..d469e90cb9899120fa998cb16dfec1e77b9dce1e 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7e618db4579d752cc6d775c664c93e141217948f
\ No newline at end of file
+111ca616713dd89b5d1e114de29c83256731c482
\ No newline at end of file