From: adamd Date: Thu, 21 Sep 2006 20:56:52 +0000 (+0000) Subject: When gathering a doclist for querying, don't discard empty position lists until the... X-Git-Tag: version-3.6.10~2734 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f40a504164243464fa80893f369290c3d52308e0;p=thirdparty%2Fsqlite.git When gathering a doclist for querying, don't discard empty position lists until the end; this allows empty position lists to override non-empty lists encountered later in the gathering process. This fixes #1982, which was caused by the fact that for all-column queries we weren't discarding empty position lists at all. (CVS 3433) FossilOrigin-Name: 111ca616713dd89b5d1e114de29c83256731c482 --- diff --git a/ext/fts1/fts1.c b/ext/fts1/fts1.c index bfbc483a24..77878ac554 100644 --- a/ext/fts1/fts1.c +++ b/ext/fts1/fts1.c @@ -339,7 +339,7 @@ static sqlite_int64 peekDocid(DocListReader *pReader){ return ret; } -/* Read the next docid. See also nextValidDocid(). +/* Read the next docid. See also nextDocid(). */ static sqlite_int64 readDocid(DocListReader *pReader){ sqlite_int64 ret; @@ -459,8 +459,8 @@ static void printDoclist(DocList *p){ } #endif /* SQLITE_DEBUG */ -/* Trim the given doclist to contain only positions in column [iRestrictColumn], - * discarding any docids without any remaining positions. */ +/* Trim the given doclist to contain only positions in column + * [iRestrictColumn]. */ static void docListRestrictColumn(DocList *in, int iRestrictColumn){ DocListReader r; DocList out; @@ -471,14 +471,11 @@ static void docListRestrictColumn(DocList *in, int iRestrictColumn){ while( !atEnd(&r) ){ sqlite_int64 iDocid = readDocid(&r); - int match = 0; int iPos, iColumn; + + docListAddDocid(&out, iDocid); while( (iPos = readPosition(&r, &iColumn)) != -1 ){ if( iColumn==iRestrictColumn ){ - if( !match ){ - docListAddDocid(&out, iDocid); - match = 1; - } docListAddPos(&out, iColumn, iPos); } } @@ -488,6 +485,35 @@ static void docListRestrictColumn(DocList *in, int iRestrictColumn){ *in = out; } +/* Trim the given doclist by discarding any docids without any remaining + * positions. */ +static void docListDiscardEmpty(DocList *in) { + DocListReader r; + DocList out; + + /* TODO: It would be nice to implement this operation in place; that + * could save a significant amount of memory in queries with long doclists. */ + assert( in->iType>=DL_POSITIONS ); + readerInit(&r, in); + docListInit(&out, DL_POSITIONS, NULL, 0); + + while( !atEnd(&r) ){ + sqlite_int64 iDocid = readDocid(&r); + int match = 0; + int iPos, iColumn; + while( (iPos = readPosition(&r, &iColumn)) != -1 ){ + if( !match ){ + docListAddDocid(&out, iDocid); + match = 1; + } + docListAddPos(&out, iColumn, iPos); + } + } + + docListDestroy(in); + *in = out; +} + /* Helper function for docListUpdate() and docListAccumulate(). ** Splices a doclist element into the doclist represented by r, ** leaving r pointing after the newly spliced element. @@ -571,16 +597,14 @@ static void docListAccumulate(DocList *pAcc, DocList *pUpdate){ } /* -** Read the next non-deleted docid off of pIn. Return -** 0 if we reach the end of pDoclist. +** Read the next docid off of pIn. Return 0 if we reach the end. +* +* TODO: This assumes that docids are never 0, but they may actually be 0 since +* users can choose docids when inserting into a full-text table. Fix this. */ -static sqlite_int64 nextValidDocid(DocListReader *pIn){ - sqlite_int64 docid = 0; +static sqlite_int64 nextDocid(DocListReader *pIn){ skipPositionList(pIn); - while( !atEnd(pIn) && (docid = readDocid(pIn))==0 ){ - skipPositionList(pIn); - } - return docid; + return atEnd(pIn) ? 0 : readDocid(pIn); } /* @@ -651,18 +675,18 @@ static void docListPhraseMerge( readerInit(&left, pLeft); readerInit(&right, pRight); - docidLeft = nextValidDocid(&left); - docidRight = nextValidDocid(&right); + docidLeft = nextDocid(&left); + docidRight = nextDocid(&right); while( docidLeft>0 && docidRight>0 ){ if( docidLeft0 && docidRight>0 ){ if( docidLeft0 && docidRight>0 ){ if( docidLeft<=docidRight ){ @@ -728,19 +752,19 @@ static void docListOrMerge( } priorLeft = docidLeft; if( docidLeft<=docidRight ){ - docidLeft = nextValidDocid(&left); + docidLeft = nextDocid(&left); } if( docidRight>0 && docidRight<=priorLeft ){ - docidRight = nextValidDocid(&right); + docidRight = nextDocid(&right); } } while( docidLeft>0 ){ docListAddDocid(pOut, docidLeft); - docidLeft = nextValidDocid(&left); + docidLeft = nextDocid(&left); } while( docidRight>0 ){ docListAddDocid(pOut, docidRight); - docidRight = nextValidDocid(&right); + docidRight = nextDocid(&right); } } @@ -762,8 +786,8 @@ static void docListExceptMerge( readerInit(&left, pLeft); readerInit(&right, pRight); - docidLeft = nextValidDocid(&left); - docidRight = nextValidDocid(&right); + docidLeft = nextDocid(&left); + docidRight = nextDocid(&right); while( docidLeft>0 && docidRight>0 ){ priorLeft = docidLeft; @@ -771,15 +795,15 @@ static void docListExceptMerge( docListAddDocid(pOut, docidLeft); } if( docidLeft<=docidRight ){ - docidLeft = nextValidDocid(&left); + docidLeft = nextDocid(&left); } if( docidRight>0 && docidRight<=priorLeft ){ - docidRight = nextValidDocid(&right); + docidRight = nextDocid(&right); } } while( docidLeft>0 ){ docListAddDocid(pOut, docidLeft); - docidLeft = nextValidDocid(&left); + docidLeft = nextDocid(&left); } } @@ -1283,6 +1307,7 @@ static int term_select_all( return rc; } + docListDiscardEmpty(&doclist); *out = doclist; return SQLITE_OK; } @@ -2370,7 +2395,7 @@ static int fulltextNext(sqlite3_vtab_cursor *pCursor){ rc = sqlite3_reset(c->pStmt); if( rc!=SQLITE_OK ) return rc; - iDocid = nextValidDocid(&c->result); + iDocid = nextDocid(&c->result); if( iDocid==0 ){ c->eof = 1; return SQLITE_OK; diff --git a/manifest b/manifest index 2cc15396d4..976db60d93 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Be\smore\saggressive\swith\sthe\sSQLITE_OMIT_VACUUM\smacro.\s\sSaves\sabout\s150\nbytes\sof\scode\sspace.\s(CVS\s3432) -D 2006-09-21T11:02:17 +C When\sgathering\sa\sdoclist\sfor\squerying,\sdon't\sdiscard\sempty\sposition\slists\suntil\sthe\send;\sthis\sallows\sempty\sposition\slists\sto\soverride\snon-empty\slists\sencountered\slater\sin\sthe\sgathering\sprocess.\s\sThis\sfixes\s#1982,\swhich\swas\scaused\sby\sthe\sfact\sthat\sfor\sall-column\squeries\swe\sweren't\sdiscarding\sempty\sposition\slists\sat\sall.\s(CVS\s3433) +D 2006-09-21T20:56:52 F Makefile.in cabd42d34340f49260bc2a7668c38eba8d4cfd99 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -21,7 +21,7 @@ F ext/README.txt 913a7bd3f4837ab14d7e063304181787658b14e1 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b F ext/fts1/ft_hash.h 1a35e654a235c2c662d3ca0dfc3138ad60b8b7d5 -F ext/fts1/fts1.c 02c5b614ff8055b374b88acaf5cae3a834da3150 +F ext/fts1/fts1.c 16f58a0c6b7d714d83e5f0ca4a520b530f8118e4 F ext/fts1/fts1.h 6060b8f62c1d925ea8356cb1a6598073eb9159a6 F ext/fts1/fts1_hash.c 3196cee866edbebb1c0521e21672e6d599965114 F ext/fts1/fts1_hash.h 957d378355ed29f672cd5add012ce8b088a5e089 @@ -399,7 +399,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513 -P c7ee60d00976efab25a830e7416538010c734129 -R 4a4f7b1d165466da2e1372b56afee139 -U drh -Z 2d27498809c9282eb1d2a169b6b639c4 +P 7e618db4579d752cc6d775c664c93e141217948f +R 9812eeaf3b8a5ffaeac13843969bbadf +U adamd +Z a38ec8fb44bc3c26289a40e8b94f4ec3 diff --git a/manifest.uuid b/manifest.uuid index 5fe3111163..d469e90cb9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e618db4579d752cc6d775c664c93e141217948f \ No newline at end of file +111ca616713dd89b5d1e114de29c83256731c482 \ No newline at end of file