From: dan Date: Tue, 27 Mar 2012 11:48:02 +0000 (+0000) Subject: Allow multiple incremental merges to proceed concurrently. This is required to preven... X-Git-Tag: mountain-lion~3^2~9^2~7^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5da0aa1603f5003ca7a02ad93009b0ac3b6ab794;p=thirdparty%2Fsqlite.git Allow multiple incremental merges to proceed concurrently. This is required to prevent a large crisis-merge from occuring while an even larger incremental-merge is underway. FossilOrigin-Name: 7ed9d2f24a650b424b97dfc19b8042c4cf09c82c --- diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index bb5e515da6..640c2078cc 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -328,7 +328,7 @@ static int fts3SqlStmt( ** if no level in the FTS index contains more than ? segments, the statement ** returns zero rows. */ /* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" - " ORDER BY (level %% 1024) DESC LIMIT 1", + " ORDER BY (level %% 1024) ASC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment ** created by merging the oldest :2 segments from absolute level :1. See @@ -4448,24 +4448,14 @@ static int fts3IncrmergeChomp( /* ** Store an incr-merge hint in the database. */ -static int fts3IncrmergeHintStore( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level to read input data from */ - int nMerge /* Number of segments to merge */ -){ - char aBlob[FTS3_VARINT_MAX * 2]; - int nBlob = 0; - int rc; +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ sqlite3_stmt *pReplace = 0; - - assert( p->bHasStat ); - nBlob += sqlite3Fts3PutVarint(&aBlob[nBlob], iAbsLevel); - nBlob += sqlite3Fts3PutVarint(&aBlob[nBlob], nMerge); + int rc; /* Return code */ rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); - sqlite3_bind_blob(pReplace, 2, aBlob, nBlob, SQLITE_TRANSIENT); + sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); } @@ -4474,42 +4464,87 @@ static int fts3IncrmergeHintStore( } /* -** Load an incr-merge hint from the database. +** Load an incr-merge hint from the database. The incr-merge hint, if one +** exists, is stored in the rowid==1 row of the %_stat table. ** -** The incr-merge hint, if one exists, is stored in the rowid==1 row of -** the %_stat table. +** If successful, populate blob *pHint with the value read from the %_stat +** table and return SQLITE_OK. Otherwise, if an error occurs, return an +** SQLite error code. */ -static int fts3IncrmergeHintLoad( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 *piAbsLevel, /* Absolute level to read input data from */ - int *pnMerge /* Number of segments to merge */ -){ +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ sqlite3_stmt *pSelect = 0; int rc; - *pnMerge = 0; - *piAbsLevel = 0; - + pHint->n = 0; rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); if( rc==SQLITE_OK ){ + int rc2; sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); if( SQLITE_ROW==sqlite3_step(pSelect) ){ const char *aHint = sqlite3_column_blob(pSelect, 0); int nHint = sqlite3_column_bytes(pSelect, 0); if( aHint ){ - int i; - char aBlob[FTS3_VARINT_MAX * 2]; - memcpy(aBlob, aHint, MAX(sizeof(aBlob), nHint)); - i = sqlite3Fts3GetVarint(aBlob, piAbsLevel); - sqlite3Fts3GetVarint32(&aBlob[i], pnMerge); + blobGrowBuffer(pHint, nHint, &rc); + if( rc==SQLITE_OK ){ + memcpy(pHint->a, aHint, nHint); + pHint->n = nHint; + } } } - rc = sqlite3_reset(pSelect); + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; } return rc; } +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry +** consists of two varints, the absolute level number of the input segments +** and the number of input segments. +** +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, +** set *pRc to an SQLite error code before returning. +*/ +static void fts3IncrmergeHintPush( + Blob *pHint, /* Hint blob to append to */ + i64 iAbsLevel, /* First varint to store in hint */ + int nInput, /* Second varint to store in hint */ + int *pRc /* IN/OUT: Error code */ +){ + blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); + if( *pRc==SQLITE_OK ){ + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); + } +} + +/* +** Read the last entry (most recently pushed) from the hint blob *pHint +** and then remove the entry. Write the two values read to *piAbsLevel and +** *pnInput before returning. +** +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. +*/ +static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ + const int nHint = pHint->n; + int i; + + i = pHint->n-2; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + + pHint->n = i; + i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); + i += sqlite3Fts3GetVarint32(&pHint->a[i], pnInput); + if( i!=nHint ) return SQLITE_CORRUPT_VTAB; + + return SQLITE_OK; +} + + /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** @@ -4522,53 +4557,70 @@ static int fts3IncrmergeHintLoad( int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ int nRem = nMerge; /* Number of leaf pages yet to be written */ - int bUseHint = 1; /* True if hint has not yet been attempted */ - sqlite3_int64 iHintAbsLevel = 0;/* Hint level */ - int nHintSeg = 0; /* Hint number of segments */ + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ int nSeg = 0; /* Number of input segments */ sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ + Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ + int bDirtyHint = 0; /* True if blob 'hint' has been modified */ - assert( nMin>=2 ); - - rc = fts3IncrmergeHintLoad(p, &iHintAbsLevel, &nHintSeg); - if( nHintSeg==0 ) bUseHint = 0; + /* Allocate space for the cursor, filter and writer objects */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; + rc = fts3IncrmergeHintLoad(p, &hint); while( rc==SQLITE_OK && nRem>0 ){ - Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ - Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ - IncrmergeWriter *pWriter; /* Writer object */ - const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); - - if( bUseHint ){ - iAbsLevel = iHintAbsLevel; - nSeg = nHintSeg; - }else{ - sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ - - /* Determine which level to merge segments from. Any level, from any - ** prefix or language index may be selected. Stack variable iAbsLevel - ** is set to the absolute level number of the level to merge from. */ - rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); - sqlite3_bind_int(pFindLevel, 1, nMin); - if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){ - /* There are no levels with nMin or more segments. Or an error has - ** occurred. Either way, exit early. */ - rc = sqlite3_reset(pFindLevel); - iAbsLevel = 0; - nSeg = 0; - break; - } + const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + int bUseHint = 0; /* True if attempting to append */ + + /* Search the %_segdir table for the absolute level with the smallest + ** relative level number that contains at least nMin segments, if any. + ** If one is found, set iAbsLevel to the absolute level number and + ** nSeg to nMin. If no level with at least nMin segments can be found, + ** set nSeg to -1. + */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, nMin); + if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ iAbsLevel = sqlite3_column_int64(pFindLevel, 0); nSeg = nMin; - sqlite3_reset(pFindLevel); + }else{ + nSeg = -1; } + rc = sqlite3_reset(pFindLevel); - /* Allocate space for the cursor, filter and writer objects */ - pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); - if( !pWriter ) return SQLITE_NOMEM; - memset(pWriter, 0, nAlloc); - pFilter = (Fts3SegFilter *)&pWriter[1]; - pCsr = (Fts3MultiSegReader *)&pFilter[1]; + /* If the hint read from the %_stat table is not empty, check if the + ** last entry in it specifies a relative level smaller than or equal + ** to the level identified by the block above (if any). If so, this + ** iteration of the loop will work on merging at the hinted level. + */ + if( rc==SQLITE_OK && hint.n ){ + int nHint = hint.n; + sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ + + rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); + if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + iAbsLevel = iHintAbsLevel; + nSeg = nHintSeg; + bUseHint = 1; + bDirtyHint = 1; + }else{ + /* This undoes the effect of the HintPop() above - so that no entry + ** is removed from the hint blob. */ + hint.n = nHint; + } + } + + /* If nSeg is less that zero, then there is no level with at least + ** nMin segments and no hint in the %_stat table. No work to do. + ** Exit early in this case. */ + if( nSeg<0 ) break; /* Open a cursor to iterate through the contents of the oldest nSeg ** indexes of absolute level iAbsLevel. If this cursor is opened using @@ -4576,10 +4628,13 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** segments available in level iAbsLevel. In this case, no work is ** done on iAbsLevel - fall through to the next iteration of the loop ** to start work on some other level. */ + memset(pWriter, 0, nAlloc); pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; - rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); - if( pCsr->nSegment==nSeg && SQLITE_OK==rc - && SQLITE_OK ==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + } + if( SQLITE_OK==rc && pCsr->nSegment==nSeg + && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ @@ -4606,6 +4661,10 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ if( rc==SQLITE_OK ){ nRem -= (1 + pWriter->nWork); rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + if( nSeg!=0 ){ + bDirtyHint = 1; + fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); + } } } @@ -4613,15 +4672,15 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ } sqlite3Fts3SegReaderFinish(pCsr); - sqlite3_free(pWriter); - bUseHint = 0; } /* Write the hint values into the %_stat table for the next incr-merger */ - if( rc==SQLITE_OK && (iAbsLevel!=iHintAbsLevel || nHintSeg!=nSeg) ){ - rc = fts3IncrmergeHintStore(p, iAbsLevel, nSeg); + if( bDirtyHint && rc==SQLITE_OK ){ + rc = fts3IncrmergeHintStore(p, &hint); } + sqlite3_free(pWriter); + sqlite3_free(hint.a); return rc; } diff --git a/manifest b/manifest index 80bdaf0fc4..533d49a69a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scorrect\serrors\sin\sthe\sfile\sformat\sdescription\sfor\sFTS3/4\scontained\sin\nthe\sfts3.c\sheader\scomment. -D 2012-03-27T00:38:33.806 +C Allow\smultiple\sincremental\smerges\sto\sproceed\sconcurrently.\sThis\sis\srequired\sto\sprevent\sa\slarge\scrisis-merge\sfrom\soccuring\swhile\san\seven\slarger\sincremental-merge\sis\sunderway. +D 2012-03-27T11:48:02.399 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_write.c 6014014cf0257d314d29d7eb50e0c88d85356d65 +F ext/fts3/fts3_write.c a9990753ba132cd79b666c61ec58ae15eb032387 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 71d6149a268e8375cd9e5c8224868fa7460614c3 @@ -501,7 +501,7 @@ F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4check.test 72134071f4e9f8bed76af1f2375fd5aff0c5ea48 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7 -F test/fts4merge.test 16ba38960dc06ffd0c47c5487ec1060b5130661f +F test/fts4merge.test c7ee5fbb3b316d1496f1d2fa861c53840b84cef9 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test e0e21332f592fc003fcab112928ea891407d83cb F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca @@ -1000,7 +1000,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 6d09de231b68dd9520d99c65d133f26e90eb784f -R 52fac8e81373757809153ddbb5601b3a -U drh -Z 7768dc9c0ca55bddb36c9095acc2e9ee +P fb8aacdd8fbdc946cb271cc589f76b806387937d +R a481c2564099a87723843f2e40a259a7 +T *branch * fts4-incr-merge-exp +T *sym-fts4-incr-merge-exp * +T -sym-fts4-incr-merge * +U dan +Z 42d22ddb577089e47eaff1e8771c43d5 diff --git a/manifest.uuid b/manifest.uuid index 461c127272..334f6758db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb8aacdd8fbdc946cb271cc589f76b806387937d \ No newline at end of file +7ed9d2f24a650b424b97dfc19b8042c4cf09c82c \ No newline at end of file diff --git a/test/fts4merge.test b/test/fts4merge.test index 95910ea654..0a4f6b6184 100644 --- a/test/fts4merge.test +++ b/test/fts4merge.test @@ -196,47 +196,49 @@ do_execsql_test 5.2 { } do_execsql_test 5.3 { - INSERT INTO t1(t1) VALUES('merge=1,4'); + INSERT INTO t1(t1) VALUES('merge=1,5'); + INSERT INTO t1(t1) VALUES('merge=1,5'); SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; } { - 0 {0 1 2 3 4 5 6 7} - 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 0 {0 1 2} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} 2 {0 1 2 3} } -do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0104'} +do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'} do_test 5.5 { foreach docid [execsql {SELECT docid FROM t1}] { execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid} } } {} -do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0104'} +do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'} do_execsql_test 5.7 { SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; SELECT quote(value) from t1_stat WHERE rowid=1; } { - 0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15} - 1 {0 1 2 3 4 5 6 7 8 9 10 11} + 0 {0 1 2 3 4 5 6 7 8 9 10} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12} 2 {0 1 2 3 4 5 6 7} - X'0104' + X'0105' } do_execsql_test 5.8 { - INSERT INTO t1(t1) VALUES('merge=1,4'); + INSERT INTO t1(t1) VALUES('merge=1,6'); + INSERT INTO t1(t1) VALUES('merge=1,6'); SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; SELECT quote(value) from t1_stat WHERE rowid=1; } { - 0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15} - 1 {0 1 2 3 4 5 6 7 8 9 10 11} - 2 {0 1 2 3 4 5 6 7} - 3 {0} - X'0204' + 0 {0 1 2 3 4} + 1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} + 2 {0 1 2 3 4 5 6 7 8} X'0106' } +do_test 5.8.1 { fts3_integrity_check t1 } ok + do_test 5.9 { - set L [expr 16*16*8 + 16*4 + 1] + set L [expr 16*16*7 + 16*3 + 12] foreach docid [execsql { SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L }] { @@ -248,17 +250,15 @@ do_execsql_test 5.10 { SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; SELECT quote(value) from t1_stat WHERE rowid=1; } { - 0 0 1 0 2 0 3 {0 1} - X'0204' + 0 0 1 {0 1} 2 0 3 0 X'0106' } do_execsql_test 5.11 { - INSERT INTO t1(t1) VALUES('merge=10,4'); + INSERT INTO t1(t1) VALUES('merge=1,6'); SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level; SELECT quote(value) from t1_stat WHERE rowid=1; } { - 0 0 1 0 2 0 3 {0 1} - X'0000' + 0 0 1 {0 1} 2 0 3 0 X'' } #-------------------------------------------------------------------------