From: dan Date: Mon, 7 Mar 2016 20:14:27 +0000 (+0000) Subject: Update fts3/4 so that the 'merge=X,0' command merges X pages from all segments of... X-Git-Tag: version-3.12.0~98 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=108b7953ed1ec8b434edcd3764bed457fedab4bd;p=thirdparty%2Fsqlite.git Update fts3/4 so that the 'merge=X,0' command merges X pages from all segments of the first level in the fts index that contains 2 or more segments. FossilOrigin-Name: cddf69dbc46f10ee7e87538dd850e086386b544c --- diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index d5a408222e..6a2299fccf 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -333,7 +333,8 @@ static int fts3SqlStmt( ** of the oldest level in the db that contains at least ? segments. Or, ** if no level in the FTS index contains more than ? segments, the statement ** returns zero rows. */ -/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" +/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " + " GROUP BY level HAVING cnt>=?" " ORDER BY (level %% 1024) ASC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment @@ -4836,10 +4837,14 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** set nSeg to -1. */ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); - sqlite3_bind_int(pFindLevel, 1, nMin); + sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin)); if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ iAbsLevel = sqlite3_column_int64(pFindLevel, 0); - nSeg = nMin; + if( nMin<2 ){ + nSeg = sqlite3_column_int(pFindLevel, 1); + }else{ + nSeg = nMin; + } }else{ nSeg = -1; } @@ -4991,7 +4996,7 @@ static int fts3DoIncrmerge( nMin = fts3Getint(&z); } - if( z[0]!='\0' || nMin<2 ){ + if( z[0]!='\0' || nMin<0 || nMin==1 ){ rc = SQLITE_ERROR; }else{ rc = SQLITE_OK; diff --git a/manifest b/manifest index 381efdf956..c6b7f7d8a4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\sNULL\spointer\sdereference\sfollowing\san\sOOM\swhile\sgenerating\scode\nfor\sIN\soperators\son\svirtual\stables. -D 2016-03-07T19:08:27.802 +C Update\sfts3/4\sso\sthat\sthe\s'merge=X,0'\scommand\smerges\sX\spages\sfrom\sall\ssegments\sof\sthe\sfirst\slevel\sin\sthe\sfts\sindex\sthat\scontains\s2\sor\smore\ssegments. +D 2016-03-07T20:14:27.338 F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66 @@ -88,7 +88,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c a93f5edc0aff44ef8b06d7cb55b52026541ca145 F ext/fts3/fts3_unicode2.c c3d01968d497bd7001e7dc774ba75b372738c057 -F ext/fts3/fts3_write.c f442223e4a1914dc1fc12b65af7e4f2c255fa47c +F ext/fts3/fts3_write.c f56345c64387ca30bf91b91511082fa83d9b5178 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 5d78b668f4e9598af9147f8999632599fb0d9dd5 @@ -748,12 +748,13 @@ F test/fts4growth.test 60d6bb3f78e25b34f533797dd9f2f9402310a13a F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269 F test/fts4incr.test 4e353a0bd886ea984e56fce9e77724fc923b8d0d F test/fts4langid.test 9794addcc8faaee85ac60eceecdb52feb0c70f68 -F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee +F test/fts4merge.test 146563df4a90ccb7f111a6e11ab29bbfe9e6ce2c F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test 5553d7bb2e20bf4a06b23e849352efc022ce6309 F test/fts4onepass.test 7319d61a2ed1325fc54afd0c060a0513b462303a +F test/fts4opt.test f15c84305c4b78c665fc476d8569e27ab9a0d418 F test/fts4unicode.test 27378af76394542cf490cf001d8d1505fe55f6a9 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef @@ -1453,7 +1454,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b6c4202432dc96f8f1740f52d0bf872116357fcc -R e48be4f1fe6a73cf9fce9701a7a4b585 -U drh -Z 61bc8adb3df986a31e200efbd3ce6ae3 +P c924008692e35f1f5144830af08d6de051dd21dd +R f3add94e73abe6caae33e7ec37099121 +U dan +Z db7aad48334dff0e6979b8c94cd633a5 diff --git a/manifest.uuid b/manifest.uuid index c96048b4e5..9c4f38bdba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c924008692e35f1f5144830af08d6de051dd21dd \ No newline at end of file +cddf69dbc46f10ee7e87538dd850e086386b544c \ No newline at end of file diff --git a/test/fts4merge.test b/test/fts4merge.test index fabb651e64..41f91fdc8f 100644 --- a/test/fts4merge.test +++ b/test/fts4merge.test @@ -89,7 +89,6 @@ foreach mod {fts3 fts4} { 5 {merge=6,%} 6 {merge=6,six} 7 {merge=6,1} - 8 {merge=6,0} } { do_catchsql_test 2.$tn { INSERT INTO t2(t2) VALUES($arg); diff --git a/test/fts4opt.test b/test/fts4opt.test new file mode 100644 index 0000000000..5638aff1b1 --- /dev/null +++ b/test/fts4opt.test @@ -0,0 +1,169 @@ +# 2016 March 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl +set ::testprefix fts4opt + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +# Create the fts_kjv_genesis procedure which fills and FTS3/4 table +# with the complete text of the Book of Genesis. +# +source $testdir/genesis.tcl + +do_execsql_test 1.0 { CREATE TABLE t1(docid, words) } +fts_kjv_genesis + +#------------------------------------------------------------------------- +# Argument $db is an open database handle. $tbl is the name of an FTS3/4 +# table with the database. This command rearranges the contents of the +# %_segdir table so that all segments within each index are on the same +# level. This means that the 'merge' command can then be used for an +# incremental optimize routine. +# +proc prepare_for_optimize {db tbl} { + $db eval [string map [list % $tbl] { + BEGIN; + CREATE TEMP TABLE tmp_segdir( + level, idx, start_block, leaves_end_block, end_block, root + ); + + INSERT INTO temp.tmp_segdir + SELECT + 1024*(o.level / 1024) + 32, -- level + sum(o.leveli.idx)), -- idx + o.start_block, o.leaves_end_block, o.end_block, o.root -- other + FROM %_segdir o, %_segdir i + WHERE (o.level / 1024) = (i.level / 1024) + GROUP BY o.level, o.idx; + + DELETE FROM %_segdir; + INSERT INTO %_segdir SELECT * FROM temp.tmp_segdir; + DROP TABLE temp.tmp_segdir; + + COMMIT; + }] +} + +do_test 1.1 { + execsql { CREATE VIRTUAL TABLE t2 USING fts4(words, prefix="1,2,3") } + foreach {docid words} [db eval { SELECT * FROM t1 }] { + execsql { INSERT INTO t2(docid, words) VALUES($docid, $words) } + } +} {} + +do_execsql_test 1.2 { + SELECT level, count(*) FROM t2_segdir GROUP BY level +} { + 0 13 1 15 2 5 + 1024 13 1025 15 1026 5 + 2048 13 2049 15 2050 5 + 3072 13 3073 15 3074 5 +} + +do_execsql_test 1.3 { INSERT INTO t2(t2) VALUES('integrity-check') } +prepare_for_optimize db t2 +do_execsql_test 1.4 { INSERT INTO t2(t2) VALUES('integrity-check') } + +do_execsql_test 1.5 { + SELECT level, count(*) FROM t2_segdir GROUP BY level +} { + 32 33 + 1056 33 + 2080 33 + 3104 33 +} + +do_test 1.6 { + while 1 { + set tc1 [db total_changes] + execsql { INSERT INTO t2(t2) VALUES('merge=5,0') } + set tc2 [db total_changes] + if {($tc2 - $tc1) < 2} break + } + execsql { SELECT level, count(*) FROM t2_segdir GROUP BY level } +} {33 1 1057 1 2081 1 3105 1} +do_execsql_test 1.7 { INSERT INTO t2(t2) VALUES('integrity-check') } + +do_execsql_test 1.8 { + INSERT INTO t2(words) SELECT words FROM t1; + SELECT level, count(*) FROM t2_segdir GROUP BY level; +} {0 2 1024 2 2048 2 3072 2} + +#------------------------------------------------------------------------- + +do_execsql_test 2.0 { + DELETE FROM t2; +} +do_test 2.1 { + foreach {docid words} [db eval { SELECT * FROM t1 }] { + execsql { INSERT INTO t2(docid, words) VALUES($docid, $words) } + } + + set i 0 + foreach {docid words} [db eval { SELECT * FROM t1 }] { + if {[incr i] % 2} { execsql { DELETE FROM t2 WHERE docid = $docid } } + } + + set i 0 + foreach {docid words} [db eval { SELECT * FROM t1 }] { + if {[incr i] % 3} { + execsql { INSERT OR REPLACE INTO t2(docid, words) VALUES($docid, $words) } + } + } +} {} + +do_execsql_test 2.2 { + SELECT level, count(*) FROM t2_segdir GROUP BY level +} { + 0 10 1 15 2 12 + 1024 10 1025 15 1026 12 + 2048 10 2049 15 2050 12 + 3072 10 3073 15 3074 12 +} + +do_execsql_test 2.3 { INSERT INTO t2(t2) VALUES('integrity-check') } +prepare_for_optimize db t2 +do_execsql_test 2.4 { INSERT INTO t2(t2) VALUES('integrity-check') } + +do_execsql_test 2.5 { + SELECT level, count(*) FROM t2_segdir GROUP BY level +} { + 32 37 + 1056 37 + 2080 37 + 3104 37 +} + +do_test 2.6 { + while 1 { + set tc1 [db total_changes] + execsql { INSERT INTO t2(t2) VALUES('merge=5,0') } + set tc2 [db total_changes] + if {($tc2 - $tc1) < 2} break + } + execsql { SELECT level, count(*) FROM t2_segdir GROUP BY level } +} {33 1 1057 1 2081 1 3105 1} +do_execsql_test 2.7 { INSERT INTO t2(t2) VALUES('integrity-check') } + +do_execsql_test 2.8 { + INSERT INTO t2(words) SELECT words FROM t1; + SELECT level, count(*) FROM t2_segdir GROUP BY level; +} {0 2 1024 2 2048 2 3072 2} + +finish_test