From: dan Date: Tue, 1 Sep 2015 18:44:33 +0000 (+0000) Subject: Fix a problem that occurs when more than 4 synonyms for a term appear within a single... X-Git-Tag: version-3.9.0~153^2~14 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=d917ad019c446dedc74abf76cd49da512ed0fcbb;p=thirdparty%2Fsqlite.git Fix a problem that occurs when more than 4 synonyms for a term appear within a single row. FossilOrigin-Name: cd359550bdc2bf7be4c294b60130c9fc3f583902 --- diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 24d63c1cab..a44c4e4427 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -395,6 +395,7 @@ static int fts5ExprSynonymPoslist( Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int nIter = 0; + int nAlloc = 4; int rc = SQLITE_OK; Fts5ExprTerm *p; @@ -406,7 +407,18 @@ static int fts5ExprSynonymPoslist( int n; i64 dummy; rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ) goto synonym_poslist_out; + if( nIter==nAlloc ){ + int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; + Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + goto synonym_poslist_out; + } + memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); + nAlloc = nAlloc*2; + aIter = aNew; + } if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){ nIter++; } @@ -447,6 +459,8 @@ static int fts5ExprSynonymPoslist( } } + synonym_poslist_out: + if( aIter!=aStatic ) sqlite3_free(aIter); return rc; } diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 5aa29e003a..ddd685c155 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -280,6 +280,17 @@ foreach {tn q res} { 8 {2 ii i two 3 three 2} {two [iv] [v] iii 3 [five]} 9 {i 2 [iv] 3 [five] [four] [v]} {iii [4] three i three ii 1} } + 3 {one OR two OR iii OR 4 OR v} { + 1 {[four] [v] [4] [i] [three]} {[1] [3] [five] [five] [4] [one]} + 2 {[5] [1] [3] [4] [i]} {[2] [2] [v] [two] [4]} + 3 {[5] [i] [5] [2] [four] [4] [1]} {[iii] [ii] [five] [two] [1]} + 4 {[ii] [four] [4] [one] [5] [three] [five]} {[one] [5] [1] [iii] [4] [3]} + 5 {[three] [i] [v] [i] [four] [4] [1]} {[ii] [five] [five] [five] [iii]} + 6 {[4] [2] [ii] [two] [2] [iii]} {[three] [1] [four] [4] [iv] [1] [iv]} + 7 {[ii] [ii] [two] [three] [2] [5]} {[iii] [i] [ii] [iii] [iii] [one] [one]} + 8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]} + 9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]} + } } { do_execsql_test 5.1.$tn { SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') @@ -287,5 +298,82 @@ foreach {tn q res} { } $res } +#------------------------------------------------------------------------- +# Test terms with more than 4 synonyms. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="query" && [string length $w]==1} { + for {set i 2} {$i<=10} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} + +do_test 6.0 { + execsql { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl) + } + foreach {rowid a b} { + 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} + 3 {zzzz llll gggggg cccc uu} {hhhhhh aaaa ppppp rr ee jjjj} + 4 {r f i rrrrrr ww hhh} {aa yyy t x aaaaa ii} + 5 {fffff mm vvvv ooo ffffff kkkk tttt} {cccccc bb e zzz d n} + 6 {iii dddd hh qqqq ddd ooo} {ttt d c b aaaaaa qqqq} + 7 {jjjj rrrr v zzzzz u tt t} {ppppp pp dddd mm hhh uuu} + 8 {gggg rrrrrr kkkk vvvv gggg jjjjjj b} {dddddd jj r w cccc wwwwww ss} + 9 {kkkkk qqq oooo e tttttt mmm} {e ss qqqqqq hhhh llllll gg} + } { + execsql { INSERT INTO t2(rowid, a, b) VALUES($rowid, $a, $b) } + } +} {} + +foreach {tn q res} { + 1 {a} { + 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq [aaaa]} + 3 {zzzz llll gggggg cccc uu} {hhhhhh [aaaa] ppppp rr ee jjjj} + 4 {r f i rrrrrr ww hhh} {[aa] yyy t x [aaaaa] ii} + 6 {iii dddd hh qqqq ddd ooo} {ttt d c b [aaaaaa] qqqq} + } + + 2 {a AND q} { + 1 {yyyy vvvvv [qq] oo yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]} + 6 {iii dddd hh [qqqq] ddd ooo} {ttt d c b [aaaaaa] [qqqq]} + } + + 3 {o OR (q AND a)} { + 1 {yyyy vvvvv [qq] [oo] yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]} + 2 {ww [oooooo] bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} + 5 {fffff mm vvvv [ooo] ffffff kkkk tttt} {cccccc bb e zzz d n} + 6 {iii dddd hh [qqqq] ddd [ooo]} {ttt d c b [aaaaaa] [qqqq]} + 9 {kkkkk qqq [oooo] e tttttt mmm} {e ss qqqqqq hhhh llllll gg} + } +} { + do_execsql_test 6.1.$tn { + SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') + FROM t2 WHERE t2 MATCH $q + } $res +} + +do_execsql_test 6.2.1 { + INSERT INTO t2(rowid, a, b) VALUES(13, + 'x xx xxx xxxx xxxxx xxxxxx xxxxxxx', 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy' + ); + SELECT rowid, highlight(t2, 0, '<', '>'), highlight(t2, 1, '(', ')') + FROM t2 WHERE t2 MATCH 'x OR y' +} { + 1 { vvvvv qq oo vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff (yy) iiii rr s ccc qqqqq} + 4 {r f i rrrrrr ww hhh} {aa (yyy) t (x) aaaaa ii} + 13 { } + {(y) (yy) (yyy) (yyyy) (yyyyy) (yyyyyy) (yyyyyyy)} +} + + finish_test diff --git a/manifest b/manifest index 4a0414c4b0..55a78b93ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sfts5\ssynonyms\simplemented\sby\sadding\sextra\sterms\sto\squeries.\sAnd\sfixes\sfor\sthe\ssame. -D 2015-09-01T18:08:36.324 +C Fix\sa\sproblem\sthat\soccurs\swhen\smore\sthan\s4\ssynonyms\sfor\sa\sterm\sappear\swithin\sa\ssingle\srow. +D 2015-09-01T18:44:33.194 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h d46f89aeb357fbcf5b268d71b0d2c5000cd27bd9 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 99560f28339b635ba0e0b13f80586c0be58fc680 +F ext/fts5/fts5_expr.c 44caa0ccd7e9a392864fda0c14e9b9829a395a84 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test a2b0fb9a584417a9c02554aa465ed9084653cdde +F ext/fts5/test/fts5synonym.test 4321e59c29186d9187cf8ab576e70530176eae49 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 -R 7ab2a9e80bd7422fd7466024fe59c669 +P dbcb73802b88f76be17f09f3eb83ffac542de633 +R 07c5ba5d7a2806d5caa7dc123b3f32ad U dan -Z 679d87b3074cd417fe8a5b64a65c64bd +Z fc32f7df10846658b7b8e62d59624854 diff --git a/manifest.uuid b/manifest.uuid index 2ddf214da5..6c61644fa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dbcb73802b88f76be17f09f3eb83ffac542de633 \ No newline at end of file +cd359550bdc2bf7be4c294b60130c9fc3f583902 \ No newline at end of file