From: dan Date: Thu, 9 Jun 2011 10:48:02 +0000 (+0000) Subject: Fix problems to do with using both OR and NEAR operators in a single expression. X-Git-Tag: version-3.7.7~62^2~8 X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=786b06896713494d7913976a590abe1c1420c30b;p=thirdparty%2Fsqlite.git Fix problems to do with using both OR and NEAR operators in a single expression. FossilOrigin-Name: 4e8dd19eef04777d800977faf1859a405e396f30 --- diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 5536a684ae..18a87e73ce 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3763,13 +3763,13 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ return rc; } -static void fts3EvalFreeDeferredDoclist(Fts3Phrase *pPhrase){ +static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){ if( pPhrase->doclist.bFreeList ){ sqlite3_free(pPhrase->doclist.pList); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - pPhrase->doclist.bFreeList = 0; } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; } static int fts3EvalNearTrim2( @@ -3786,6 +3786,8 @@ static int fts3EvalNearTrim2( char *pOut; int res; + assert( pPhrase->doclist.pList ); + p2 = pOut = pPhrase->doclist.pList; res = fts3PoslistNearMerge( &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 @@ -3959,16 +3961,16 @@ static void fts3EvalNext( fts3EvalNext(pCsr, pRight, pRc); assert( *pRc!=SQLITE_OK || pRight->bStart ); } - do { - fts3EvalNext(pCsr, pLeft, pRc); - if( pLeft->bEof ) break; + + fts3EvalNext(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ while( !*pRc && !pRight->bEof && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 ){ fts3EvalNext(pCsr, pRight, pRc); } - }while( !pRight->bEof && pRight->iDocid==pLeft->iDocid && !*pRc ); + } pExpr->iDocid = pLeft->iDocid; pExpr->bEof = pLeft->bEof; break; @@ -3976,7 +3978,7 @@ static void fts3EvalNext( default: { Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalFreeDeferredDoclist(pPhrase); + fts3EvalZeroPoslist(pPhrase); *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); pExpr->iDocid = pPhrase->doclist.iDocid; break; @@ -3997,11 +3999,11 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ && fts3EvalNearTest(pExpr, pRc) ); - /* If this is a NEAR node and the NEAR expression does not match - ** any rows, zero the doclist for all phrases involved in the NEAR. - ** This is because the snippet(), offsets() and matchinfo() functions - ** are not supposed to recognize any instances of phrases that are - ** part of unmatched NEAR queries. For example if this expression: + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: ** ** ... MATCH 'a OR (b NEAR c)' ** @@ -4012,12 +4014,19 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ ** then any snippet() should ony highlight the "a" term, not the "b" ** (as "b" is part of a non-matching NEAR clause). */ - if( pExpr->eType==FTSQUERY_NEAR && bHit==0 ){ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ Fts3Expr *p; for(p=pExpr; p->pPhrase==0; p=p->pLeft){ - p->pRight->pPhrase->doclist.pList = 0; + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pPhrase); } - p->pPhrase->doclist.pList = 0; } break; @@ -4037,9 +4046,14 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ break; default: { - if( pCsr->pDeferred ){ + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalFreeDeferredDoclist(pPhrase); + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalZeroPoslist(pPhrase); + } *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); bHit = (pPhrase->doclist.pList!=0); pExpr->iDocid = pCsr->iPrevId; @@ -4116,7 +4130,7 @@ static void fts3EvalRestart( Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ - fts3EvalFreeDeferredDoclist(pPhrase); + fts3EvalZeroPoslist(pPhrase); if( pPhrase->bIncr ){ sqlite3Fts3EvalPhraseCleanup(pPhrase); memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); @@ -4172,7 +4186,7 @@ static void fts3EvalUpdateCounts( } } -static int fts3EvalNearStats( +static int fts3EvalGatherStats( Fts3Cursor *pCsr, Fts3Expr *pExpr ){ @@ -4195,6 +4209,7 @@ static int fts3EvalNearStats( } iDocid = pRoot->iDocid; bEof = pRoot->bEof; + assert( pRoot->bStart ); /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ for(p=pRoot; p; p=p->pLeft){ @@ -4236,11 +4251,17 @@ static int fts3EvalNearStats( if( bEof ){ pRoot->bEof = bEof; }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidiDocidbEof==0 ); - } + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); fts3EvalLoadDeferred(pCsr, &rc); } } @@ -4293,7 +4314,7 @@ int sqlite3Fts3EvalPhraseStats( aiOut[iCol*3 + 2] = pCsr->nDoc; } }else{ - rc = fts3EvalNearStats(pCsr, pExpr); + rc = fts3EvalGatherStats(pCsr, pExpr); if( rc==SQLITE_OK ){ assert( pExpr->aMI ); for(iCol=0; iColnColumn; iCol++){ @@ -4372,7 +4393,7 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ if( pPhrase ){ int i; sqlite3_free(pPhrase->doclist.aAll); - fts3EvalFreeDeferredDoclist(pPhrase); + fts3EvalZeroPoslist(pPhrase); memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); for(i=0; inToken; i++){ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); diff --git a/manifest b/manifest index c0c0a4d2ec..4398869eb3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\svarious\sissues\sto\sdo\swith\sdeferred\stokens,\sNEAR\sexpressions\sand\smatchinfo(). -D 2011-06-08T18:39:07.487 +C Fix\sproblems\sto\sdo\swith\susing\sboth\sOR\sand\sNEAR\soperators\sin\sa\ssingle\sexpression. +D 2011-06-09T10:48:02.352 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c b44083cafb9840be0927f8b9fb2ab4f373167f77 +F ext/fts3/fts3.c 5df3b5797522d3d17949ee12d5918d6d213b5114 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -475,7 +475,7 @@ F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 -F test/fts3rnd.test d88ec3dbe52e81e410cd1a39554d15941f86333c +F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402 @@ -943,7 +943,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 9d10a6846b12a9cc8fd4fdc3affd931a27218b5a -R 37e4da2cb9907d0ccf1d8076445165fd +P 3972a787df5ec253b99b148385655e7b68d851fa +R 9f88ec7fe13b9820fcb74aa4a46dd50c U dan -Z 147c4bbcabf01e6d99dff7a301984a70 +Z 78f71a9a33e1daee77ff848d72b67bf4 diff --git a/manifest.uuid b/manifest.uuid index f3e1134a0a..f91e3008ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3972a787df5ec253b99b148385655e7b68d851fa \ No newline at end of file +4e8dd19eef04777d800977faf1859a405e396f30 \ No newline at end of file diff --git a/test/fts3rnd.test b/test/fts3rnd.test index 0ad41d8ce4..97af54925f 100644 --- a/test/fts3rnd.test +++ b/test/fts3rnd.test @@ -273,6 +273,8 @@ proc do_orderbydocid_test {tn sql res} { ] } +set NUM_TRIALS 100 + foreach {nodesize order} { 50 DESC 50 ASC @@ -292,7 +294,7 @@ foreach {nodesize order} { for {set i 0} {$i < 100} {incr i} { insert_row $i } } - for {set iTest 1} {$iTest <= 100} {incr iTest} { + for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} { catchsql COMMIT set DO_MALLOC_TEST 0 @@ -334,11 +336,11 @@ foreach {nodesize order} { # for {set i 0} {$i < 10} {incr i} { set term [random_term] - do_select_test 1.$i { + do_select_test 1.$i.asc { SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term ORDER BY docid ASC } [simple_token_matchinfo $term 0] - do_select_test 1.$i { + do_select_test 1.$i.desc { SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term ORDER BY docid DESC } [simple_token_matchinfo $term 1] @@ -432,9 +434,9 @@ foreach {nodesize order} { # Set operations on NEAR queries. # foreach {tn op proc} { - 8 OR setop_or - 9 NOT setop_not - 10 AND setop_and + 11 OR setop_or + 12 NOT setop_not + 13 AND setop_and } { for {set i 0} {$i < $nRep} {incr i} { set term1 [random_term]