From: dan Date: Tue, 1 Sep 2015 18:08:36 +0000 (+0000) Subject: Add tests for fts5 synonyms implemented by adding extra terms to queries. And fixes... X-Git-Tag: version-3.9.0~153^2~15 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3faa07ee0474436575a6cb81d56e80f6c47c3bd9;p=thirdparty%2Fsqlite.git Add tests for fts5 synonyms implemented by adding extra terms to queries. And fixes for the same. FossilOrigin-Name: dbcb73802b88f76be17f09f3eb83ffac542de633 --- diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index e8d997b060..1ad61617d6 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -235,6 +235,8 @@ struct Fts5PoslistReader { int n; /* Size of buffer at a[] in bytes */ int i; /* Current offset in a[] */ + u8 bFlag; /* For client use (any custom purpose) */ + /* Output variables */ u8 bEof; /* Set to true at EOF */ i64 iPos; /* (iCol<<32) + iPos */ diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index bba9932daf..24d63c1cab 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -22,6 +22,8 @@ */ #define FTS5_EOF 0 +#define FTS5_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) + typedef struct Fts5ExprTerm Fts5ExprTerm; /* @@ -386,26 +388,66 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){ static int fts5ExprSynonymPoslist( Fts5ExprTerm *pTerm, i64 iRowid, + int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ u8 **pa, int *pn ){ Fts5PoslistWriter writer = {0}; Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int nIter = 0; + int rc = SQLITE_OK; Fts5ExprTerm *p; assert( pTerm->pSynonym ); for(p=pTerm; p; p=p->pSynonym){ Fts5IndexIter *pIter = p->pIter; if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ + const u8 *a; + int n; i64 dummy; - int rc = sqlite3Fts5IterPoslist(pIter, (const u8**)pa, pn, &dummy); - return rc; + rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy); + if( rc!=SQLITE_OK ) return rc; + if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){ + nIter++; + } } } - assert( 0 ); - return SQLITE_ERROR; + assert( *pbDel==0 ); + if( nIter==1 ){ + *pa = (u8*)aIter[0].a; + *pn = aIter[0].n; + }else{ + Fts5PoslistWriter writer = {0}; + Fts5Buffer buf = {0,0,0}; + i64 iPrev = -1; + while( 1 ){ + int i; + i64 iMin = FTS5_LARGEST_INT64; + for(i=0; inTerm); /* Initialize a term iterator for each term in the phrase */ for(i=0; inTerm; i++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; i64 dummy; - int n; - const u8 *a; + int n = 0; + int bFlag = 0; + const u8 *a = 0; if( pTerm->pSynonym ){ - rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, (u8**)&a, &n); + rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, &bFlag, (u8**)&a, &n); }else{ rc = sqlite3Fts5IterPoslist(pTerm->pIter, &a, &n, &dummy); } - if( rc || sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ){ - goto ismatch_out; - } + if( rc!=SQLITE_OK ) goto ismatch_out; + sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]); + aIter[i].bFlag = bFlag; + if( aIter[i].bEof ) goto ismatch_out; } while( 1 ){ @@ -495,6 +540,9 @@ static int fts5ExprPhraseIsMatch( ismatch_out: *pbMatch = (pPhrase->poslist.n>0); + for(i=0; inTerm; i++){ + if( aIter[i].bFlag ) sqlite3_free((u8*)aIter[i].a); + } if( aIter!=aStatic ) sqlite3_free(aIter); return rc; } @@ -672,13 +720,25 @@ static int fts5ExprNearAdvanceFirst( /* Find the firstest rowid any synonym points to. */ i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc); - /* Advance each iterator that currently points to iRowid */ + /* Advance each iterator that currently points to iRowid. Or, if iFrom + ** is valid - each iterator that points to a rowid before iFrom. */ for(p=pTerm; p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ - bEof = 0; - if( sqlite3Fts5IterRowid(p->pIter)==iRowid ){ - rc = sqlite3Fts5IterNext(p->pIter); + i64 ii = sqlite3Fts5IterRowid(p->pIter); + if( ii==iRowid + || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) + ){ + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(p->pIter); + } if( rc!=SQLITE_OK ) break; + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + bEof = 0; + } + }else{ + bEof = 0; } } } @@ -807,7 +867,7 @@ static int fts5ExprNearTest( ** phrase is not a match, break out of the loop early. */ for(i=0; rc==SQLITE_OK && inPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; - if( pPhrase->nTerm>1 || pNear->pColset ){ + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ int bMatch = 0; rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); if( bMatch==0 ) break; @@ -985,7 +1045,7 @@ static int fts5ExprNearInitAll( &p->pIter ); assert( rc==SQLITE_OK || p->pIter==0 ); - if( p->pIter && 0==sqlite3Fts5IterEof(pTerm->pIter) ){ + if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){ bEof = 0; } } diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index a607b7e29c..5aa29e003a 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -23,15 +23,15 @@ ifcapable !fts5 { foreach S { {zero 0} - {one 1} - {two 2} + {one 1 i} + {two 2 ii} {three 3 iii} - {four 4} - {five 5} - {six 6} - {seven 7} - {eight 8} - {nine 9} + {four 4 iv} + {five 5 v} + {six 6 vi} + {seven 7 vii} + {eight 8 viii} + {nine 9 ix} } { foreach s $S { set o [list] @@ -186,16 +186,23 @@ do_execsql_test 3.2.0 { one 1 4 three 1 2 two 1 2 } do_execsql_test 3.2.1 { + SELECT rowid FROM ft WHERE ft MATCH 'one'; +} {1} +do_execsql_test 3.2.2 { SELECT rowid FROM ft WHERE ft MATCH 'one two three'; +} {1} +do_execsql_test 3.2.3 { SELECT rowid FROM ft WHERE ft MATCH 'one + one + two + three'; -} {1 1} -do_execsql_test 3.2.2 { +} {1} +do_execsql_test 3.2.4 { SELECT rowid FROM ft WHERE ft MATCH 'one two two three'; - SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three'; } {1} +do_execsql_test 3.2.5 { + SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three'; +} {} #------------------------------------------------------------------------- -# Check that expressions with synonyms can be parsed. +# Check that expressions with synonyms can be parsed and executed. # reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create @@ -212,11 +219,72 @@ proc tcl_tokenize {tflags text} { foreach {tn expr res} { 1 {abc} {"abc"} - 2 {one} {"one"|"1"} + 2 {one} {"one"|"i"|"1"} 3 {3} {"3"|"iii"|"three"} 4 {3*} {"3"|"iii"|"three" *} } { - do_execsql_test 4.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res] + do_execsql_test 4.1.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res] +} + +do_execsql_test 4.2.1 { + CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tcl); + INSERT INTO xx VALUES('one two'); + INSERT INTO xx VALUES('three four'); +} + +do_execsql_test 4.2.2 { + SELECT rowid FROM xx WHERE xx MATCH '2' +} {1} + +do_execsql_test 4.2.3 { + SELECT rowid FROM xx WHERE xx MATCH '3' +} {2} + +do_test 5.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tcl) + } + foreach {rowid a b} { + 1 {four v 4 i three} {1 3 five five 4 one} + 2 {5 1 3 4 i} {2 2 v two 4} + 3 {5 i 5 2 four 4 1} {iii ii five two 1} + 4 {ii four 4 one 5 three five} {one 5 1 iii 4 3} + 5 {three i v i four 4 1} {ii five five five iii} + 6 {4 2 ii two 2 iii} {three 1 four 4 iv 1 iv} + 7 {ii ii two three 2 5} {iii i ii iii iii one one} + 8 {2 ii i two 3 three 2} {two iv v iii 3 five} + 9 {i 2 iv 3 five four v} {iii 4 three i three ii 1} + } { + execsql { INSERT INTO t1(rowid, a, b) VALUES($rowid, $a, $b) } + } +} {} + +foreach {tn q res} { + 1 {one} { + 1 {four v 4 [i] three} {[1] 3 five five 4 [one]} + 2 {5 [1] 3 4 [i]} {2 2 v two 4} + 3 {5 [i] 5 2 four 4 [1]} {iii ii five two [1]} + 4 {ii four 4 [one] 5 three five} {[one] 5 [1] iii 4 3} + 5 {three [i] v [i] four 4 [1]} {ii five five five iii} + 6 {4 2 ii two 2 iii} {three [1] four 4 iv [1] iv} + 7 {ii ii two three 2 5} {iii [i] ii iii iii [one] [one]} + 8 {2 ii [i] two 3 three 2} {two iv v iii 3 five} + 9 {[i] 2 iv 3 five four v} {iii 4 three [i] three ii [1]} + } + 2 {five four} { + 1 {[four] [v] [4] i three} {1 3 [five] [five] [4] one} + 2 {[5] 1 3 [4] i} {2 2 [v] two [4]} + 3 {[5] i [5] 2 [four] [4] 1} {iii ii [five] two 1} + 4 {ii [four] [4] one [5] three [five]} {one [5] 1 iii [4] 3} + 5 {three i [v] i [four] [4] 1} {ii [five] [five] [five] iii} + 8 {2 ii i two 3 three 2} {two [iv] [v] iii 3 [five]} + 9 {i 2 [iv] 3 [five] [four] [v]} {iii [4] three i three ii 1} + } +} { + do_execsql_test 5.1.$tn { + SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') + FROM t1 WHERE t1 MATCH $q + } $res } finish_test diff --git a/manifest b/manifest index 4d5891206e..4a0414c4b0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\schanges\sto\sallow\ssynonym\ssupport\sby\sadding\smultiple\sterms\sto\sa\squery\s(an\salternative\sto\sadding\smultiple\sterms\sto\sthe\sFTS\sindex). -D 2015-08-31T20:06:06.235 +C Add\stests\sfor\sfts5\ssynonyms\simplemented\sby\sadding\sextra\sterms\sto\squeries.\sAnd\sfixes\sfor\sthe\ssame. +D 2015-09-01T18:08:36.324 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,11 +106,11 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 -F ext/fts5/fts5Int.h 9c538f0fcc1c3bc2fa12f7199d1326bd2362ce9c +F ext/fts5/fts5Int.h d46f89aeb357fbcf5b268d71b0d2c5000cd27bd9 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 1458c3e33c1ec3ad99284f9692edfc49c44afd7c +F ext/fts5/fts5_expr.c 99560f28339b635ba0e0b13f80586c0be58fc680 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 3343a05f7b1d3fc6fa4559bd3bd0ee979928c30d +F ext/fts5/test/fts5synonym.test a2b0fb9a584417a9c02554aa465ed9084653cdde F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 98d07d16cab92f1e7001afbe370df3ec6343fc1f -R e2a3883c4d0440bf5b53c5a71ba46260 +P ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 +R 7ab2a9e80bd7422fd7466024fe59c669 U dan -Z 7c01c61f2a729597f3365ce7db7a6ce6 +Z 679d87b3074cd417fe8a5b64a65c64bd diff --git a/manifest.uuid b/manifest.uuid index b0ebea4bef..2ddf214da5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 \ No newline at end of file +dbcb73802b88f76be17f09f3eb83ffac542de633 \ No newline at end of file