From: dan Date: Mon, 11 Jan 2016 17:30:28 +0000 (+0000) Subject: Fix bugs in fts5 synonym processing for detail=col and other modes. X-Git-Tag: version-3.11.0~157^2~3 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=aaf17c41ec558e2c0c174055c6dcbbd1908aa290;p=thirdparty%2Fsqlite.git Fix bugs in fts5 synonym processing for detail=col and other modes. FossilOrigin-Name: 0e3c545423246231ddac8fed2f103d71d556e17f --- diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 3804a2abfe..16209bf4ba 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -40,6 +40,7 @@ void sqlite3Fts5ParserTrace(FILE*, char*); struct Fts5Expr { Fts5Index *pIndex; + Fts5Config *pConfig; Fts5ExprNode *pRoot; int bDesc; /* Iterate in descending rowid order */ int nPhrase; /* Number of phrases in expression */ @@ -235,6 +236,7 @@ int sqlite3Fts5ExprNew( }else{ pNew->pRoot = sParse.pExpr; pNew->pIndex = 0; + pNew->pConfig = pConfig; pNew->apExprPhrase = sParse.apPhrase; pNew->nPhrase = sParse.nPhrase; sParse.apPhrase = 0; @@ -299,8 +301,9 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ /* ** Argument pTerm must be a synonym iterator. */ -static int fts5ExprSynonymPoslist( +static int fts5ExprSynonymList( Fts5ExprTerm *pTerm, + int bCollist, Fts5Colset *pColset, i64 iRowid, int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ @@ -319,9 +322,16 @@ static int fts5ExprSynonymPoslist( if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ const u8 *a; int n; - i64 dummy; - rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); + + if( bCollist ){ + rc = sqlite3Fts5IterCollist(pIter, &a, &n); + }else{ + i64 dummy; + rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); + } + if( rc!=SQLITE_OK ) goto synonym_poslist_out; + if( n==0 ) continue; if( nIter==nAlloc ){ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); @@ -422,8 +432,8 @@ static int fts5ExprPhraseIsMatch( int bFlag = 0; const u8 *a = 0; if( pTerm->pSynonym ){ - rc = fts5ExprSynonymPoslist( - pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n + rc = fts5ExprSynonymList( + pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n ); }else{ rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy); @@ -757,30 +767,51 @@ static int fts5ExprNearTest( ){ Fts5ExprNearset *pNear = pNode->pNear; int rc = *pRc; - int i; - /* Check that each phrase in the nearset matches the current row. - ** Populate the pPhrase->poslist buffers at the same time. If any - ** phrase is not a match, break out of the loop early. */ - for(i=0; rc==SQLITE_OK && inPhrase; i++){ - Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; - if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ - int bMatch = 0; - rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); - if( bMatch==0 ) break; - }else{ - rc = sqlite3Fts5IterPoslistBuffer( - pPhrase->aTerm[0].pIter, &pPhrase->poslist - ); + if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5ExprTerm *pTerm; + Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; + pPhrase->poslist.n = 0; + for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ + Fts5IndexIter *pIter = pTerm->pIter; + if( sqlite3Fts5IterEof(pIter)==0 ){ + int n; + i64 iRowid; + rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return 0; + }else if( iRowid==pNode->iRowid && n>0 ){ + pPhrase->poslist.n = 1; + } + } } - } + return pPhrase->poslist.n; + }else{ + int i; - *pRc = rc; - if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ - return 1; - } + /* Check that each phrase in the nearset matches the current row. + ** Populate the pPhrase->poslist buffers at the same time. If any + ** phrase is not a match, break out of the loop early. */ + for(i=0; rc==SQLITE_OK && inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ + int bMatch = 0; + rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); + if( bMatch==0 ) break; + }else{ + rc = sqlite3Fts5IterPoslistBuffer( + pPhrase->aTerm[0].pIter, &pPhrase->poslist + ); + } + } - return 0; + *pRc = rc; + if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ + return 1; + } + return 0; + } } static int fts5ExprTokenTest( @@ -1606,6 +1637,7 @@ int sqlite3Fts5ExprClonePhrase( if( rc==SQLITE_OK ){ /* All the allocations succeeded. Put the expression object together. */ pNew->pIndex = pExpr->pIndex; + pNew->pConfig = pExpr->pConfig; pNew->nPhrase = 1; pNew->apExprPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; @@ -2453,17 +2485,35 @@ int sqlite3Fts5ExprPhraseCollist( ){ Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; Fts5ExprNode *pNode = pPhrase->pNode; - assert( iPhrase>=0 && iPhrasenPhrase ); + int rc = SQLITE_OK; + assert( iPhrase>=0 && iPhrasenPhrase ); if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid && pPhrase->poslist.n>0 ){ - sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist); + Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; + if( pTerm->pSynonym ){ + int bDel = 0; + u8 *a; + rc = fts5ExprSynonymList( + pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist + ); + if( bDel ){ + sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a); + *ppCollist = pPhrase->poslist.p; + sqlite3_free(a); + }else{ + *ppCollist = a; + } + }else{ + sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist); + } }else{ *ppCollist = 0; *pnCollist = 0; } - return SQLITE_OK; + + return rc; } diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test index e3a8003758..96cacb293f 100644 --- a/ext/fts5/test/fts5synonym2.test +++ b/ext/fts5/test/fts5synonym2.test @@ -13,7 +13,7 @@ # source [file join [file dirname [info script]] fts5_common.tcl] -set testprefix fts5synonym +set testprefix fts5synonym2 # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { @@ -47,7 +47,7 @@ proc tcl_tokenize {tflags text} { foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd if {$tflags == "query"} { - foreach s $::SYNDICT($w) { sqlite3_fts5_token -colo $s $iStart $iEnd } + foreach s $::syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd } } } } @@ -67,7 +67,6 @@ fts5_aux_test_functions db do_execsql_test 1.0 { CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%); - INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i'); INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi'); INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3'); @@ -120,14 +119,30 @@ do_execsql_test 1.0 { } foreach {tn expr} { - 1 "eight" + 1.1 "one" 1.2 "two" 1.3 "three" 1.4 "four" + 1.5 "v" 1.6 "vi" 1.7 "vii" 1.8 "viii" + 1.9 "9" 1.10 "0" 1.11 "1" 1.12 "2" + + 2.1 "one OR two OR three OR four" + 2.2 "(one AND two) OR (three AND four)" + 2.3 "(one AND two) OR (three AND four) NOT five" + 2.4 "(one AND two) NOT 6" + + 3.1 "b:one AND a:two" + 3.2 "b:one OR a:two" + 3.3 "a:one OR b:1 OR {a b} : i" + + 4.1 "NEAR(one two, 2)" + 4.2 "NEAR(one two three, 2)" + 4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)" } { if {[fts5_expr_ok $expr ss]==0} { do_test 1.$tn.OMITTED { list } [list] continue } - set res [fts5_query_data $expr ss ASC ::SYNDICT] + set res [fts5_query_data $expr ss ASC ::syn] + breakpoint do_execsql_test 1.$tn.[llength $res].asc { SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr) } $res diff --git a/manifest b/manifest index 48daf058c3..e4ef860b2c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\sadding\sfts5\stests\sinvolving\ssynonyms\sand\sdetail=none/col\stables. -D 2016-01-08T17:21:18.901 +C Fix\sbugs\sin\sfts5\ssynonym\sprocessing\sfor\sdetail=col\sand\sother\smodes. +D 2016-01-11T17:30:28.056 F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042 @@ -101,7 +101,7 @@ F ext/fts5/fts5Int.h 3918045ebceb1d600c9c1c1b460489ff0c788e96 F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 -F ext/fts5/fts5_expr.c 913ab9f9a7651d76f055f9847ca0ae0e2f5da78d +F ext/fts5/fts5_expr.c 6eba2220747ea1b20a358fb3b34b2ab78323e285 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 F ext/fts5/fts5_index.c 04fe098ffab66d9424ba4e7c0b88ce7c98748cb1 F ext/fts5/fts5_main.c 03bd44e4bd0ba16213ca9259ad5df1d4d743fd7e @@ -173,7 +173,7 @@ F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821 F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9 F ext/fts5/test/fts5simple2.test 843f1f7fe439ff32bf74f4fd6430632f9636ef3a F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 -F ext/fts5/test/fts5synonym2.test 6aa842d0e5bd019db0c3597e0860eb68eb2867e4 +F ext/fts5/test/fts5synonym2.test d2d9099d9d105b55ea03fd52d61ae2847d534129 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1410,7 +1410,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d9135cc723fc4227aace6dcf4ffa4630c9d23aa0 -R c443735bd0d019a3a4798f3f3350cb78 +P b3e6f15ec2d9a834e2c80b91ffd7097553816228 +R f239c4ba67c1753046748953bb2cde92 U dan -Z 48c62f9fa751846d5897bddd7cc5158f +Z 915aa1f2ec9df261dd260c37cdbace05 diff --git a/manifest.uuid b/manifest.uuid index dc0f491a48..3b1a59f4b6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b3e6f15ec2d9a834e2c80b91ffd7097553816228 \ No newline at end of file +0e3c545423246231ddac8fed2f103d71d556e17f \ No newline at end of file