"ORDER BY level DESC, idx ASC",
/* Update statements used while promoting segments */
-/* 38 */ "UPDATE %Q.'%q_segdir' SET level=-1,idx=? WHERE level=? AND idx=?",
-/* 39 */ "UPDATE %Q.'%q_segdir' SET level=? WHERE level=-1"
+/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? "
+ "WHERE level=? AND idx=?",
+/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1"
};
int rc = SQLITE_OK;
Fts3SegFilter filter; /* Segment term filter condition */
Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */
int bIgnoreEmpty = 0; /* True to ignore empty segments */
+ i64 iMaxLevel = 0; /* Max level number for this index/langid */
assert( iLevel==FTS3_SEGCURSOR_ALL
|| iLevel==FTS3_SEGCURSOR_PENDING
rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr);
if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
+ if( iLevel!=FTS3_SEGCURSOR_PENDING ){
+ rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel);
+ if( rc!=SQLITE_OK ) goto finished;
+ }
+
if( iLevel==FTS3_SEGCURSOR_ALL ){
/* This call is to merge all segments in the database to a single
** segment. The level of the new segment is equal to the numerically
rc = SQLITE_DONE;
goto finished;
}
- rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel);
+ iNewLevel = iMaxLevel;
bIgnoreEmpty = 1;
- }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
- iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0);
- rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx);
}else{
/* This call is to merge all segments at level iLevel. find the next
** available segment index at level iLevel+1. The call to
** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
** a single iLevel+2 segment if necessary. */
- rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx);
+ assert( FTS3_SEGCURSOR_PENDING==-1 );
iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1);
+ rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx);
+ bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel);
}
if( rc!=SQLITE_OK ) goto finished;
+
assert( csr.nSegment>0 );
assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) );
assert( iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL) );
csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
}
if( rc!=SQLITE_OK ) goto finished;
- assert( pWriter );
+ assert( pWriter || bIgnoreEmpty );
if( iLevel!=FTS3_SEGCURSOR_PENDING ){
rc = fts3DeleteSegdir(
);
if( rc!=SQLITE_OK ) goto finished;
}
- rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
- if( rc==SQLITE_OK ){
- rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData);
+ if( pWriter ){
+ rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
+ if( rc==SQLITE_OK ){
+ rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData);
+ }
}
finished:
const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex;
sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */
int bUseHint = 0; /* True if attempting to append */
+ int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */
/* Search the %_segdir table for the absolute level with the smallest
** relative level number that contains at least nMin segments, if any.
** to start work on some other level. */
memset(pWriter, 0, nAlloc);
pFilter->flags = FTS3_SEGMENT_REQUIRE_POS;
+
+ if( rc==SQLITE_OK ){
+ rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
+ assert( bUseHint==1 || bUseHint==0 );
+ if( (iIdx-bUseHint)==0 ) pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY;
+ }
if( rc==SQLITE_OK ){
rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr);
}
&& SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter))
&& SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr))
){
- int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */
- rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
- if( rc==SQLITE_OK ){
- if( bUseHint && iIdx>0 ){
- const char *zKey = pCsr->zTerm;
- int nKey = pCsr->nTerm;
- rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter);
- }else{
- rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter);
- }
+ if( bUseHint && iIdx>0 ){
+ const char *zKey = pCsr->zTerm;
+ int nKey = pCsr->nTerm;
+ rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter);
+ }else{
+ rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter);
}
if( rc==SQLITE_OK && pWriter->nLeafEst ){
-C Experimental\scode\sto\sprevent\sFTS\sindexes\sfrom\sgrowing\sindefinitely\sas\sthe\stable\sis\supdated.
-D 2014-05-12T20:04:48.571
+C Fix\sa\sproblem\spreventing\sdelete\smarkers\sfrom\sever\sbeing\sremoved\sfrom\sthe\sFTS\sindex.
+D 2014-05-13T20:11:37.423
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9
F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d
-F ext/fts3/fts3_write.c 5fd2aa9d1812387c6254304e20d9ac2b29e16700
+F ext/fts3/fts3_write.c 283b24477729129a210d91b48f7c53181583a848
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb
F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
-F test/fts3d.test 597b0b76e41f0d672e2731c4d7b631d628efd13f
+F test/fts3d.test 95c17d1b67b33a5eac0bf5a0d11116a0c0ac7a3a
F test/fts3defer.test 0be4440b73a2e651fc1e472066686d6ada4b9963
F test/fts3defer2.test e880e3b65bdf999f4746cdaefa65f14a98b9b724
F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd
F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8
F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06
F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01
-F test/fts4growth.test f7eac9fadfe67765c4a0d6202c85f7272766fb9e
+F test/fts4growth.test 3b1f8c98b603b38dc9fe4a266f4f5ddb0c73f092
F test/fts4incr.test 361960ed3550e781f3f313e17e2182ef9cefc0e9
F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7
F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 0a4f59676bd0ab33b2c86c9a35a2ebbdbaf09ee7
-R de45a14af3a90f6b390273a9a2d8cfa9
-T *branch * fts4-experimental
-T *sym-fts4-experimental *
-T -sym-trunk *
+P b3b505a4dd0c679437a4272109f1188175088cd1
+R d1a058fc899c4ecbd09fd7fceab935e7
U dan
-Z a03d85f9bfa278d813164b1e97a88ff7
+Z f678f48fb654227badac5ce98e23d62c
-b3b505a4dd0c679437a4272109f1188175088cd1
\ No newline at end of file
+7f47ae5c5ddb1227484ddae7c6960183932a052a
\ No newline at end of file
{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \
{0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}]
-check_terms_all fts3d-4.1 {a four is one test that this three two was}
+puts [db eval {SELECT c FROM t1 } ]
+check_terms_all fts3d-4.1 {a four is test that this was}
check_doclist_all fts3d-4.1.1 a {[1 0[2]] [2 0[2]] [3 0[2]]}
check_doclist_all fts3d-4.1.2 four {}
check_doclist_all fts3d-4.1.3 is {[1 0[1]] [3 0[1]]}
-check_doclist_all fts3d-4.1.4 one {}
+#check_doclist_all fts3d-4.1.4 one {}
check_doclist_all fts3d-4.1.5 test {[1 0[3]] [2 0[3]] [3 0[3]]}
check_doclist_all fts3d-4.1.6 that {[2 0[0]]}
check_doclist_all fts3d-4.1.7 this {[1 0[0]] [3 0[0]]}
-check_doclist_all fts3d-4.1.8 three {}
-check_doclist_all fts3d-4.1.9 two {}
+#check_doclist_all fts3d-4.1.8 three {}
+#check_doclist_all fts3d-4.1.9 two {}
check_doclist_all fts3d-4.1.10 was {[2 0[1]]}
check_terms fts3d-4.2 0 0 {a four test that was}
check_doclist fts3d-4.3.4 0 1 test {[3 0[3]]}
check_doclist fts3d-4.3.5 0 1 this {[3 0[0]]}
-check_terms fts3d-4.4 1 0 {a four is one test that this three two was}
+check_terms fts3d-4.4 1 0 {a four is test that this was}
check_doclist fts3d-4.4.1 1 0 a {[1 0[2]] [2 0[2]] [3 0[2]]}
-check_doclist fts3d-4.4.2 1 0 four {[1] [2 0[4]] [3 0[4]]}
+check_doclist fts3d-4.4.2 1 0 four {[2 0[4]] [3 0[4]]}
check_doclist fts3d-4.4.3 1 0 is {[1 0[1]] [3 0[1]]}
-check_doclist fts3d-4.4.4 1 0 one {[1] [2] [3]}
+#check_doclist fts3d-4.4.4 1 0 one {[1] [2] [3]}
check_doclist fts3d-4.4.5 1 0 test {[1 0[3]] [2 0[3]] [3 0[3]]}
check_doclist fts3d-4.4.6 1 0 that {[2 0[0]]}
check_doclist fts3d-4.4.7 1 0 this {[1 0[0]] [3 0[0]]}
-check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]}
-check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]}
+#check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]}
+#check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]}
check_doclist fts3d-4.4.10 1 0 was {[2 0[1]]}
# Optimize should leave the result in the level of the highest-level
} {86120}
#--------------------------------------------------------------------------
+# Test that delete markers are removed from FTS segments when possible.
+# It is only possible to remove delete markers when the output of the
+# merge operation will become the oldest segment in the index.
+#
+# 3.1 - when the oldest segment is created by an 'optimize'.
+# 3.2 - when the oldest segment is created by an incremental merge.
+# 3.3 - by a crisis merge.
+#
+
+proc insert_doc {args} {
+ foreach iDoc $args {
+ set L [lindex {
+ {In your eagerness to engage the Trojans,}
+ {don’t any of you charge ahead of others,}
+ {trusting in your strength and horsemanship.}
+ {And don’t lag behind. That will hurt our charge.}
+ {Any man whose chariot confronts an enemy’s}
+ {should thrust with his spear at him from there.}
+ {That’s the most effective tactic, the way}
+ {men wiped out city strongholds long ago —}
+ {their chests full of that style and spirit.}
+ } [expr $iDoc%9]]
+ execsql { REPLACE INTO x3(docid, content) VALUES($iDoc, $L) }
+ }
+}
+
+proc delete_doc {args} {
+ foreach iDoc $args {
+ execsql { DELETE FROM x3 WHERE docid = $iDoc }
+ }
+}
+
+proc second {x} { lindex $x 1 }
+db func second second
+
+do_execsql_test 3.0 { CREATE VIRTUAL TABLE x3 USING fts4 }
+
+do_test 3.1.1 {
+ db transaction { insert_doc 1 2 3 4 5 6 }
+ execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
+} {0 0 412}
+do_test 3.1.2 {
+ delete_doc 1 2 3 4 5 6
+ execsql { SELECT count(*) FROM x3_segdir }
+} {0}
+do_test 3.1.3 {
+ db transaction {
+ insert_doc 1 2 3 4 5 6 7 8 9
+ delete_doc 9 8 7
+ }
+ execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
+} {0 0 591 0 1 65 0 2 72 0 3 76}
+do_test 3.1.4 {
+ execsql { INSERT INTO x3(x3) VALUES('optimize') }
+ execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
+} {0 0 412}
+
+do_test 3.2.1 {
+ execsql { DELETE FROM x3 }
+ insert_doc 8 7 6 5 4 3 2 1
+ delete_doc 7 8
+ execsql { SELECT count(*) FROM x3_segdir }
+} {10}
+do_test 3.2.2 {
+ execsql { INSERT INTO x3(x3) VALUES('merge=500,10') }
+ execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
+} {1 0 412}
+
+# This assumes the crisis merge happens when there are already 16
+# segments and one more is added.
+#
+do_test 3.3.1 {
+ execsql { DELETE FROM x3 }
+ insert_doc 1 2 3 4 5 6 7 8 9 10 11
+ delete_doc 11 10 9 8 7
+ execsql { SELECT count(*) FROM x3_segdir }
+} {16}
+do_test 3.3.2 {
+ insert_doc 12
+ execsql { SELECT level, idx, second(end_block) FROM x3_segdir WHERE level=1 }
+} {1 0 412}
+
+#--------------------------------------------------------------------------
+do_execsql_test 4.1 {
+ DROP TABLE IF EXISTS x4;
+ DROP TABLE IF EXISTS t1;
+ CREATE TABLE t1(docid, words);
+ CREATE VIRTUAL TABLE x4 USING fts4(words);
+}
+do_test 4.2 {
+ fts_kjv_genesis
+ execsql { INSERT INTO x4 SELECT words FROM t1 }
+ execsql { INSERT INTO x4 SELECT words FROM t1 }
+} {}
+
+do_execsql_test 4.3 {
+ SELECT level, idx, second(end_block) FROM x4_segdir
+} {0 0 117483 0 1 118006}
+
+do_execsql_test 4.4 {
+ INSERT INTO x4(x4) VALUES('merge=10,2');
+ SELECT count(*) FROM x4_segdir;
+} {3}
+
+breakpoint
+do_execsql_test 4.5 {
+ INSERT INTO x4(x4) VALUES('merge=10,2');
+ SELECT count(*) FROM x4_segdir;
+} {3}
+
+if 0 {
do_execsql_test 3.1 {
DROP TABLE IF EXISTS x2;
DROP TABLE IF EXISTS t1;
#execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level}
#} {0 13 1 15 2 5}
-#proc second {x} { lindex $x 1 }
-#db func second second
-#for {set i 0} {$i <1000} {incr i} {
-# t1_to_x2
-# db eval {
-# SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level;
-# } {
-# puts "$i.$level: $c"
-# }
-#}
+proc second {x} { lindex $x 1 }
+db func second second
+for {set i 0} {$i <1000} {incr i} {
+ t1_to_x2
+ db eval {
+ SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level;
+ } {
+ puts "$i.$level: $c"
+ }
+}
+}
finish_test
projects / thirdparty / sqlite.git / commitdiff
