From: dan Date: Wed, 2 Sep 2015 17:34:22 +0000 (+0000) Subject: Fix a problem with fts5 synonyms and phrase queries. Also fix an OOM handling bug... X-Git-Tag: version-3.9.0~153^2~11 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=bea34fc53af825712a983dd3f1a717fcb9e6c03d;p=thirdparty%2Fsqlite.git Fix a problem with fts5 synonyms and phrase queries. Also fix an OOM handling bug in fts5. FossilOrigin-Name: a4c35fa2c94fe34b376670244fe72303c99868c1 --- diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index cc24179561..8796ba5e2d 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -288,8 +288,8 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){ ** Argument pTerm must be a synonym iterator. Return the current rowid ** that it points to. */ -static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){ - i64 iRet; +static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ + i64 iRet = 0; int bRetValid = 0; Fts5ExprTerm *p; @@ -305,7 +305,7 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){ } } - assert( bRetValid ); + if( pbEof && bRetValid==0 ) *pbEof = 1; return iRet; } @@ -346,9 +346,9 @@ static int fts5ExprSynonymPoslist( nAlloc = nAlloc*2; aIter = aNew; } - if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){ - nIter++; - } + sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter]); + assert( aIter[nIter].bEof==0 ); + nIter++; } } @@ -659,7 +659,7 @@ static int fts5ExprNearAdvanceFirst( Fts5ExprTerm *p; /* Find the firstest rowid any synonym points to. */ - i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc); + i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); /* Advance each iterator that currently points to iRowid. Or, if iFrom ** is valid - each iterator that points to a rowid before iFrom. */ @@ -738,6 +738,35 @@ static int fts5ExprAdvanceto( return 0; } +static int fts5ExprSynonymAdvanceto( + Fts5ExprTerm *pTerm, /* Term iterator to advance */ + int bDesc, /* True if iterator is "rowid DESC" */ + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ + int *pRc, /* OUT: Error code */ + int *pbEof /* OUT: Set to true if EOF */ +){ + int rc = SQLITE_OK; + i64 iLast = *piLast; + Fts5ExprTerm *p; + + for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + i64 iRowid = sqlite3Fts5IterRowid(p->pIter); + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLastpIter, iLast); + } + } + } + + if( rc!=SQLITE_OK ){ + *pbEof = 1; + }else{ + *piLast = fts5ExprSynonymRowid(pTerm, bDesc, pbEof); + } + + return rc; +} + /* ** IN/OUT parameter (*pa) points to a position list n bytes in size. If ** the position list contains entries for column iCol, then (*pa) is set @@ -906,7 +935,7 @@ static int fts5ExprNearNextMatch( ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it ** means the minimum rowid. */ if( pLeft->aTerm[0].pSynonym ){ - iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc); + iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0); }else{ iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter); } @@ -920,21 +949,13 @@ static int fts5ExprNearNextMatch( if( pTerm->pSynonym ){ Fts5ExprTerm *p; int bEof = 1; - i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc); + i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); if( iRowid==iLast ) continue; - for(p=pTerm; p; p=p->pSynonym){ - Fts5IndexIter *pIter = p->pIter; - int dummy; - if( 0==sqlite3Fts5IterEof(pIter) - && 0==fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &dummy)==0 - ){ - bEof = 0; - } - } - if( bEof || rc ){ - pNode->bEof = 1; + bMatch = 0; + if( fts5ExprSynonymAdvanceto(pTerm,bDesc,&iLast,&rc,&pNode->bEof) ){ return rc; } + }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; i64 iRowid = sqlite3Fts5IterRowid(pIter); diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test index b92df3ce79..b739e87418 100644 --- a/ext/fts5/test/fts5fault6.test +++ b/ext/fts5/test/fts5fault6.test @@ -22,6 +22,8 @@ ifcapable !fts5 { return } +if 1 { + #------------------------------------------------------------------------- # OOM while rebuilding an FTS5 table. # @@ -148,8 +150,14 @@ do_faultsim_test 4.1 -faults oom-t* -prep { faultsim_test_result {0 {}} } +} + #------------------------------------------------------------------------- -# OOM while running a query that includes synonyms and matchinfo(). +# +# 5.2.* OOM while running a query that includes synonyms and matchinfo(). +# +# 5.3.* OOM while running a query that returns a row containing instances +# of more than 4 synonyms for a single term. # proc mit {blob} { set scan(littleEndian) i* @@ -174,6 +182,7 @@ db func mit mit sqlite3_fts5_register_matchinfo db do_test 5.0 { execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl) } + execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) } foreach {rowid text} { 1 {aaaa cc b aaaaa cc aa} 2 {aa aa bb a bbb} @@ -184,6 +193,37 @@ do_test 5.0 { 7 {aaaaa aa aa ccccc bb} 8 {ccc bbbbb ccccc bbb c} 9 {cccccc bbbb a aaa cccc c} + + 20 {ddd f ddd eeeee fff ffff eeee ddd fff eeeee dddddd eeee} + 21 {fffff eee dddd fffff dd ee ee eeeee eee eeeeee ee dd e} + 22 {fffff d eeee dddd fffff dddddd ffff ddddd eeeee ee eee dddd ddddd} + 23 {ddddd fff ddd eeeee ffff eeee ddd ff ff ffffff eeeeee dddd ffffff} + 24 {eee dd ee dddd dddd eeeeee e eee fff ffff} + 25 {ddddd ffffff dddddd fff ddd ddddd ddd f eeee fff dddd f} + 26 {f ffff fff fff eeeeee dddd d dddddd ddddd eee ff eeeee} + 27 {eee fff dddddd eeeee eeeee dddd ddddd ffff f eeeee eee dddddd ddddd d} + 28 {dd ddddd d ddd d fff d dddd ee dddd ee ddd dddddd dddddd} + 29 {eeee dddd ee dddd eeee dddd dd fffff f ddd eeeee ddd ee} + 30 {ff ffffff eeeeee eeeee eee ffffff ff ffff f fffff eeeee} + 31 {fffff eeeeee dddd eeee eeee eeeeee eee fffff d ddddd ffffff ffff dddddd} + 32 {dddddd fffff ee eeeeee eeee ee fff dddd fff eeee ffffff eeeeee ffffff} + 33 {ddddd eeee dd ffff dddddd fff eeee ddddd ffff eeee ddd} + 34 {ee dddd ddddd dddddd eeee eeeeee f dd ee dddddd ffffff} + 35 {ee dddd dd eeeeee ddddd eee d eeeeee dddddd eee dddd fffff} + 36 {eee ffffff ffffff e fffff eeeee ff dddddd dddddd fff} + 37 {eeeee fffff dddddd dddd ffffff fff f dd ee dd dd eeeee} + 38 {eeeeee ee d ff eeeeee eeeeee eee eeeee ee ffffff dddd eeee dddddd ee} + 39 {eeeeee ddd fffff e dddd ee eee eee ffffff ee f d dddd} + 40 {ffffff dddddd eee ee ffffff eee eeee ddddd ee eeeeee f} + 41 {ddd ddd fff fffff ee fffff f fff ddddd fffff} + 42 {dddd ee ff d f ffffff fff ffffff ff dd dddddd f eeee} + 43 {d dd fff fffff d f fff e dddd ee ee} + 44 {ff ffff eee ddd d dd ffff dddd d eeee d eeeeee} + 45 {eeee f eeeee ee e ffff f ddd e fff} + 46 {ffff d ffff eeee ffff eeeee f ffff ddddd eee} + 47 {dd dd dddddd ddddd fffff dddddd ddd ddddd eeeeee ffff eeee eee ee} + 48 {ffff ffff e dddd ffffff dd dd dddd f fffff} + 49 {ffffff d dddddd ffff eeeee f ffff ffff d dd fffff eeeee} } { execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) } } @@ -196,9 +236,12 @@ set res [list {*}{ 7 {3 24 8 1 12 6} 9 {2 24 8 3 12 6} }] -do_execsql_test 5.1 { +do_execsql_test 5.1.1 { SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c' } $res +do_execsql_test 5.1.2 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'd e f' +} 29 faultsim_save_and_close do_faultsim_test 5.2 -faults oom* -prep { @@ -214,5 +257,16 @@ do_faultsim_test 5.2 -faults oom* -prep { faultsim_test_result [list 0 $::res] } +do_faultsim_test 5.3 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create +} -body { + db eval { + SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f' + } +} -test { + faultsim_test_result {0 29} +} + finish_test diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 6acc5617f0..33d20a014e 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -93,6 +93,7 @@ foreach {tn expr res} { 2 "eight OR 8 OR 5" {2 3} 3 "10" {} 4 "1*" {1} + 5 "1 + 2" {1} } { do_execsql_test 2.1.$tn { SELECT rowid FROM ft WHERE ft MATCH $expr @@ -233,6 +234,7 @@ do_test 5.0 { } } {} + foreach {tn q res} { 1 {one} { 1 {four v 4 [i] three} {[1] 3 five five 4 [one]} @@ -265,6 +267,23 @@ foreach {tn q res} { 8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]} 9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]} } + + 4 {5 + 1} { + 2 {[5 1] 3 4 i} {2 2 v two 4} + 3 {[5 i] 5 2 four 4 1} {iii ii five two 1} + 4 {ii four 4 one 5 three five} {one [5 1] iii 4 3} + 5 {three i [v i] four 4 1} {ii five five five iii} + } + + 5 {one + two + three} { + 7 {ii ii two three 2 5} {iii [i ii iii] iii one one} + 8 {2 ii [i two 3] three 2} {two iv v iii 3 five} + } + + 6 {"v v"} { + 1 {four v 4 i three} {1 3 [five five] 4 one} + 5 {three i v i four 4 1} {ii [five five five] iii} + } } { do_execsql_test 5.1.$tn { SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') diff --git a/main.mk b/main.mk index 9988fe2775..828a824eee 100644 --- a/main.mk +++ b/main.mk @@ -47,6 +47,7 @@ TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth +TCCX += -I$(TOP)/ext/fts5 # Object files for the SQLite library. # @@ -230,6 +231,29 @@ SRC += \ $(TOP)/ext/rbu/sqlite3rbu.h +# FTS5 things +# +FTS5_HDR = \ + $(TOP)/ext/fts5/fts5.h \ + $(TOP)/ext/fts5/fts5Int.h \ + fts5parse.h + +FTS5_SRC = \ + $(TOP)/ext/fts5/fts5_aux.c \ + $(TOP)/ext/fts5/fts5_buffer.c \ + $(TOP)/ext/fts5/fts5_main.c \ + $(TOP)/ext/fts5/fts5_config.c \ + $(TOP)/ext/fts5/fts5_expr.c \ + $(TOP)/ext/fts5/fts5_hash.c \ + $(TOP)/ext/fts5/fts5_index.c \ + fts5parse.c \ + $(TOP)/ext/fts5/fts5_storage.c \ + $(TOP)/ext/fts5/fts5_tokenize.c \ + $(TOP)/ext/fts5/fts5_unicode2.c \ + $(TOP)/ext/fts5/fts5_varint.c \ + $(TOP)/ext/fts5/fts5_vocab.c \ + + # Generated source code files # SRC += \ @@ -636,25 +660,6 @@ fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c -# FTS5 things -# -FTS5_SRC = \ - $(TOP)/ext/fts5/fts5.h \ - $(TOP)/ext/fts5/fts5Int.h \ - $(TOP)/ext/fts5/fts5_aux.c \ - $(TOP)/ext/fts5/fts5_buffer.c \ - $(TOP)/ext/fts5/fts5_main.c \ - $(TOP)/ext/fts5/fts5_config.c \ - $(TOP)/ext/fts5/fts5_expr.c \ - $(TOP)/ext/fts5/fts5_hash.c \ - $(TOP)/ext/fts5/fts5_index.c \ - fts5parse.c fts5parse.h \ - $(TOP)/ext/fts5/fts5_storage.c \ - $(TOP)/ext/fts5/fts5_tokenize.c \ - $(TOP)/ext/fts5/fts5_unicode2.c \ - $(TOP)/ext/fts5/fts5_varint.c \ - $(TOP)/ext/fts5/fts5_vocab.c \ - fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h @@ -662,11 +667,10 @@ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon fts5parse.h: fts5parse.c -fts5.c: $(FTS5_SRC) +fts5.c: $(FTS5_SRC) $(FTS5_HDR) tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl cp $(TOP)/ext/fts5/fts5.h . - userauth.o: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c diff --git a/manifest b/manifest index 01f188aef6..30f63e1524 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\shandling\sOOM\sconditions\swithin\sfts5\squeries\sthat\sfeature\ssynonyms. -D 2015-09-02T14:17:38.670 +C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sphrase\squeries.\sAlso\sfix\san\sOOM\shandling\sbug\sin\sfts5. +D 2015-09-02T17:34:22.663 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 5ae9bae1a36583a1e85f742f972c65bffd7e9d3b +F ext/fts5/fts5_expr.c 408384b2f540dcf72a363e1cde4c3e9896e8d41d F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e @@ -156,7 +156,7 @@ F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875 -F ext/fts5/test/fts5fault6.test 233a29d4a086cd09cfae262bbb15c709a438d010 +F ext/fts5/test/fts5fault6.test 93e9d43ab1e05ae5efa30cfde17649b3bb6a82c3 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test dd3c9016345e1bbd7384ec25bbcd3de1f1700475 +F ext/fts5/test/fts5synonym.test 88efc3de51a2c46aff984d74f06c833e4def0493 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -259,7 +259,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 04840e8277ab5159af16172eafd214dae7cffff5 +F main.mk 8a8468564b2e387116614fcda2a6d4f5f754b955 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cf3e45e76d23e10ee06296c3561a341591597a04 -R 0e89c166acdebcc1b36d66580652787d +P 11fa980897c6c7be218bbd9c4cd8253272d2c300 +R 6df2cc151a0891be96a1f7bf13aeaf04 U dan -Z 84847a00fa2f61f76712fa9a6aa07879 +Z 72b6e5381768fc37d01c74eff21e6a40 diff --git a/manifest.uuid b/manifest.uuid index 985c544ce9..2675a32e17 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11fa980897c6c7be218bbd9c4cd8253272d2c300 \ No newline at end of file +a4c35fa2c94fe34b376670244fe72303c99868c1 \ No newline at end of file