From: dan Date: Mon, 13 Jun 2011 17:00:12 +0000 (+0000) Subject: Add tests for deferred tokens to fts3auto.test. Fix a problem with OR queries and... X-Git-Tag: version-3.7.7~62^2~5 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=2f779343854941321b64995cd608e6fe7346ad2d;p=thirdparty%2Fsqlite.git Add tests for deferred tokens to fts3auto.test. Fix a problem with OR queries and deferred tokens. FossilOrigin-Name: b9fb69e55bb05a8819688ee63987f2a05c33d59b --- diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 4b37c4e645..17a7b2ccaa 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3572,11 +3572,16 @@ static void fts3EvalTokenCosts( } }else if( pExpr->eType!=FTSQUERY_NOT ){ if( pExpr->eType==FTSQUERY_OR ){ - pRoot = pExpr; - **ppOr = pExpr; + pRoot = pExpr->pLeft; + **ppOr = pRoot; (*ppOr)++; } fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); } } @@ -3736,7 +3741,7 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ Fts3Expr **apOr; aTC = (Fts3TokenAndCost *)sqlite3_malloc( sizeof(Fts3TokenAndCost) * nToken - + sizeof(Fts3Expr *) * nOr + + sizeof(Fts3Expr *) * nOr * 2 ); apOr = (Fts3Expr **)&aTC[nToken]; @@ -4310,7 +4315,7 @@ int sqlite3Fts3EvalPhraseStats( int rc = SQLITE_OK; int iCol; - if( pExpr->bDeferred ){ + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ assert( pCsr->nDoc>0 ); for(iCol=0; iColnColumn; iCol++){ aiOut[iCol*3 + 1] = pCsr->nDoc; diff --git a/manifest b/manifest index 8bdca7e965..15429e6135 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sfts3auto.test\sto\stest\sOR,\sAND\sand\sNOT\soperations. -D 2011-06-13T13:48:36.083 +C Add\stests\sfor\sdeferred\stokens\sto\sfts3auto.test.\sFix\sa\sproblem\swith\sOR\squeries\sand\sdeferred\stokens. +D 2011-06-13T17:00:12.630 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c e71dafb1f324358d12fd02ea12644d8c6cea577a +F ext/fts3/fts3.c 2a48945d199d5d0fe66a06c8cf73ad2f0e7a4b5f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test 4aca0c0631fd40e79a45d8490828152b46ad4559 +F test/fts3auto.test bf01a1d793155d744d23e4092871ca116f1db64c F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -466,7 +466,7 @@ F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35 -F test/fts3defer2.test ad5bd3ae616fb719eb06b0a73eb5c8fb1c411989 +F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 5f6b87f420f21749aa7c72e020c50aca74890086 -R ca51faeb6c7aef68bca6b7079ff7ebda +P e4ab6cdb101bbeb804820425cf569ee7dc2397fb +R 7fcdbf104822a42776783c2cd9e89101 U dan -Z 9a71ae3656c163debcbb9f929d0e7d22 +Z 0999cf6b26be67f673d9db0918d86701 diff --git a/manifest.uuid b/manifest.uuid index 45225befd7..41624e6daa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4ab6cdb101bbeb804820425cf569ee7dc2397fb \ No newline at end of file +b9fb69e55bb05a8819688ee63987f2a05c33d59b \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index 97b4304f0c..68c82c8a85 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -93,12 +93,45 @@ proc get_single_near_results {tbl expr arrayvar nullvar} { set iCol [lindex [split $key ,] 1] set iPhrase 0 foreach c $counts($key) { - if {$c>0} { incr nHit($iPhrase,$iCol) 1 } - incr nDoc($iPhrase,$iCol) $c + if {$c>0} { incr nDoc($iPhrase,$iCol) 1 } + incr nHit($iPhrase,$iCol) $c incr iPhrase } } + if {[info exists ::fts3_deferred] && [llength $expr]==1} { + set phrase [lindex $expr 0] + set rewritten [list] + set partial 0 + foreach tok $phrase { + if {[lsearch $::fts3_deferred $tok]>=0} { + lappend rewritten * + } else { + lappend rewritten $tok + set partial 1 + } + } + if {$partial==0} { + set tblsize [db one "SELECT count(*) FROM $tbl"] + for {set iCol 0} {$iCol < $nCol} {incr iCol} { + set nHit(0,$iCol) $tblsize + set nDoc(0,$iCol) $tblsize + } + } elseif {$rewritten != $phrase} { + while {[lindex $rewritten end] == "*"} { + set rewritten [lrange $rewritten 0 end-1] + } + while {[lindex $rewritten 0] == "*"} { + set rewritten [lrange $rewritten 1 end] + } + get_single_near_results $tbl [list $rewritten] aRewrite nullentry + foreach docid [array names hits] { + set aMatchinfo($docid) $aRewrite($docid) + } + return + } + } + # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to # contain the output of matchinfo('x') for the document. # @@ -107,8 +140,8 @@ proc get_single_near_results {tbl expr arrayvar nullvar} { for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { for {set iCol 0} {$iCol<$nCol} {incr iCol} { lappend mi [lindex $counts($docid,$iCol) $iPhrase] - lappend mi $nDoc($iPhrase,$iCol) lappend mi $nHit($iPhrase,$iCol) + lappend mi $nDoc($iPhrase,$iCol) } } set aMatchinfo($docid) $mi @@ -119,7 +152,7 @@ proc get_single_near_results {tbl expr arrayvar nullvar} { set nullentry [list] for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { for {set iCol 0} {$iCol<$nCol} {incr iCol} { - lappend nullentry 0 $nDoc($iPhrase,$iCol) $nHit($iPhrase,$iCol) + lappend nullentry 0 $nHit($iPhrase,$iCol) $nDoc($iPhrase,$iCol) } } } @@ -219,7 +252,7 @@ proc get_near_results {tbl expr arrayvar {nullvar ""}} { } } -proc do_near_test {tn tbl expr} { +proc do_fts3query_test {tn tbl expr} { get_near_results $tbl $expr aMatchinfo set match $expr @@ -286,19 +319,20 @@ test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1} #-------------------------------------------------------------------------- # Test cases fts3auto-2.* run some simple tests using the -# [do_near_test] proc. +# [do_fts3query_test] proc. # foreach {tn create} { - 1 "CREATE VIRTUAL TABLE t1 USING fts4(a, b)" - 2 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC)" - 3 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC)" - 4 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, prefix=1)" - 5 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC, prefix=1)" - 6 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC, prefix=1)" + 1 "fts4(a, b)" + 2 "fts4(a, b, order=DESC)" + 3 "fts4(a, b, order=ASC)" + 4 "fts4(a, b, prefix=1)" + 5 "fts4(a, b, order=DESC, prefix=1)" + 6 "fts4(a, b, order=ASC, prefix=1)" } { +break do_test 2.$tn.1 { catchsql { DROP TABLE t1 } - execsql $create + execsql "CREATE VIRTUAL TABLE t1 USING $create" for {set i 0} {$i<32} {incr i} { set doc [list] if {$i&0x01} {lappend doc one} @@ -309,6 +343,7 @@ foreach {tn create} { execsql { INSERT INTO t1 VALUES($doc, null) } } } {} + foreach {tn2 expr} { 1 {one} 2 {one NEAR/1 five} @@ -327,7 +362,69 @@ foreach {tn create} { 15 {"one two"} 16 {"one two" NOT "three four"} } { - do_near_test 2.$tn.2.$tn2 t1 $expr + do_fts3query_test 2.$tn.2.$tn2 t1 $expr + } +} + +#-------------------------------------------------------------------------- +# Some test cases involving deferred tokens. +# +proc make_token_deferrable {tbl token} { + set nRow [db one "SELECT count(*) FROM $tbl"] + set pgsz [db one "PRAGMA page_size"] + + execsql "INSERT INTO $tbl ($tbl) VALUES('maxpending=100000000')" + execsql BEGIN + for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { + set doc [string repeat "$token " 100] + execsql "INSERT INTO $tbl VALUES(\$doc)" + } + execsql "INSERT INTO $tbl VALUES('aaaaaaa ${token}aaaaa')" + execsql COMMIT +} + +foreach {tn create} { + 1 "fts4(x)" + 2 "fts4(x, order=DESC)" +} { + catchsql { DROP TABLE t1 } + execsql "CREATE VIRTUAL TABLE t1 USING $create" + do_execsql_test 3.$tn.1 { + INSERT INTO t1 VALUES('a b c d e f g h i j k'); + INSERT INTO t1 VALUES('b c d e f g h i j k a'); + INSERT INTO t1 VALUES('c d e f g h i j k a b'); + INSERT INTO t1 VALUES('d e f g h i j k a b c'); + INSERT INTO t1 VALUES('e f g h i j k a b c d'); + INSERT INTO t1 VALUES('f g h i j k a b c d e'); + INSERT INTO t1 VALUES('a c e g i k'); + INSERT INTO t1 VALUES('a d g j'); + INSERT INTO t1 VALUES('c a b'); + } + + make_token_deferrable t1 c + + set ::fts3_deferred [list] + foreach {tn2 expr} { + 1 {a OR c} + } { + do_fts3query_test 3.$tn.2.$tn2 t1 $expr + } + + set ::fts3_deferred [list c] + execsql { + UPDATE t1_segments + SET block = zeroblob(length(block)) + WHERE length(block)>10000 AND 0 + } + foreach {tn2 expr} { + 1 {a NEAR c} + 2 {a AND c} + 3 {"a c"} + 4 {"c a"} + 5 {"a c" NEAR/1 g} + 6 {"a c" NEAR/0 g} + } { + do_fts3query_test 3.$tn.2.$tn2 t1 $expr } } diff --git a/test/fts3defer2.test b/test/fts3defer2.test index b7bf923550..92a4491efd 100644 --- a/test/fts3defer2.test +++ b/test/fts3defer2.test @@ -56,14 +56,13 @@ do_execsql_test 1.2.1 { SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } {{a b c d e f a x y}} -breakpoint do_execsql_test 1.2.2 { SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal')) FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } [list \ {a b c d [e] [f] [a] x y} \ {0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 1 1 1 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 1 1 1 8 5001 9] ] do_execsql_test 1.2.3 { @@ -72,7 +71,7 @@ do_execsql_test 1.2.3 { } [list \ {[a] b c d [e] [f] [a] x y} \ {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 1 1 2 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 2 2 1 8 5001 9] ] do_execsql_test 1.3.1 { DROP TABLE t1 }