int pgsz; /* Approximate page size used in %_data */
int nAutomerge; /* 'automerge' setting */
int nCrisisMerge; /* Maximum allowed segments per level */
+ int nUsermerge; /* 'usermerge' setting */
int nHashSize; /* Bytes of memory for in-memory hash */
char *zRank; /* Name of rank function */
char *zRankArgs; /* Arguments to rank function */
#define FTS5_DEFAULT_PAGE_SIZE 4050
#define FTS5_DEFAULT_AUTOMERGE 4
+#define FTS5_DEFAULT_USERMERGE 4
#define FTS5_DEFAULT_CRISISMERGE 16
#define FTS5_DEFAULT_HASHSIZE (1024*1024)
}
}
+ else if( 0==sqlite3_stricmp(zKey, "usermerge") ){
+ int nUsermerge = -1;
+ if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+ nUsermerge = sqlite3_value_int(pVal);
+ }
+ if( nUsermerge<2 || nUsermerge>16 ){
+ *pbBadkey = 1;
+ }else{
+ pConfig->nUsermerge = nUsermerge;
+ }
+ }
+
else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){
int nCrisisMerge = -1;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
/* Set default values */
pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
+ pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
/*
** Do up to nPg pages of automerge work on the index.
+**
+** Return true if any changes were actually made, or false otherwise.
*/
-static void fts5IndexMerge(
+static int fts5IndexMerge(
Fts5Index *p, /* FTS5 backend object */
Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */
- int nPg /* Pages of work to do */
+ int nPg, /* Pages of work to do */
+ int nMin /* Minimum number of segments to merge */
){
int nRem = nPg;
+ int bRet = 0;
Fts5Structure *pStruct = *ppStruct;
while( nRem>0 && p->rc==SQLITE_OK ){
int iLvl; /* To iterate through levels */
}
#endif
- if( nBest<p->pConfig->nAutomerge
- && pStruct->aLevel[iBestLvl].nMerge==0
- ){
+ if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){
break;
}
+ bRet = 1;
fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
fts5StructurePromote(p, iBestLvl+1, pStruct);
}
}
*ppStruct = pStruct;
+ return bRet;
}
/*
pStruct->nWriteCounter += nLeaf;
nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel);
- fts5IndexMerge(p, ppStruct, nRem);
+ fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge);
}
}
}
}
-
-int sqlite3Fts5IndexOptimize(Fts5Index *p){
- Fts5Structure *pStruct;
+static Fts5Structure *fts5IndexOptimizeStruct(
+ Fts5Index *p,
+ Fts5Structure *pStruct
+){
Fts5Structure *pNew = 0;
- int nSeg = 0;
-
- assert( p->rc==SQLITE_OK );
- fts5IndexFlush(p);
- pStruct = fts5StructureRead(p);
+ int nByte = sizeof(Fts5Structure);
+ int nSeg = pStruct->nSegment;
+ int i;
- if( pStruct ){
- assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
- nSeg = pStruct->nSegment;
- if( nSeg>1 ){
- int nByte = sizeof(Fts5Structure);
- nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
- pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
+ /* Figure out if this structure requires optimization. A structure does
+ ** not require optimization if either:
+ **
+ ** + it consists of fewer than two segments, or
+ ** + all segments are on the same level, or
+ ** + all segments except one are currently inputs to a merge operation.
+ **
+ ** In the first case, return NULL. In the second, increment the ref-count
+ ** on *pStruct and return a copy of the pointer to it.
+ */
+ if( nSeg<2 ) return 0;
+ for(i=0; i<pStruct->nLevel; i++){
+ int nThis = pStruct->aLevel[i].nSeg;
+ if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){
+ fts5StructureRef(pStruct);
+ return pStruct;
}
+ assert( pStruct->aLevel[i].nMerge<=nThis );
}
+
+ nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
+ pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
+
if( pNew ){
Fts5StructureLevel *pLvl;
int nByte = nSeg * sizeof(Fts5StructureSegment);
}
}
- if( pNew ){
- int iLvl = pNew->nLevel-1;
+ return pNew;
+}
+
+int sqlite3Fts5IndexOptimize(Fts5Index *p){
+ Fts5Structure *pStruct;
+ Fts5Structure *pNew = 0;
+ int nSeg = 0;
+
+ assert( p->rc==SQLITE_OK );
+ fts5IndexFlush(p);
+ pStruct = fts5StructureRead(p);
+
+ if( pStruct ){
+ pNew = fts5IndexOptimizeStruct(p, pStruct);
+ }
+ fts5StructureRelease(pStruct);
+
+ if( pNew && pNew->nSegment>0 ){
+ int iLvl;
+ for(iLvl=0; iLvl<pNew->nLevel; iLvl++){
+ if( pNew->aLevel[iLvl].nSeg ) break;
+ }
while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){
int nRem = FTS5_OPT_WORK_UNIT;
fts5IndexMergeLevel(p, &pNew, iLvl, &nRem);
fts5StructureRelease(pNew);
}
- fts5StructureRelease(pStruct);
return fts5IndexReturn(p);
}
+/*
+** This is called to implement the special "VALUES('merge', $nMerge)"
+** INSERT command.
+*/
int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
- Fts5Structure *pStruct;
-
- pStruct = fts5StructureRead(p);
- if( pStruct && pStruct->nLevel ){
- fts5IndexMerge(p, &pStruct, nMerge);
- fts5StructureWrite(p, pStruct);
+ Fts5Structure *pStruct = fts5StructureRead(p);
+ if( pStruct ){
+ int nMin = p->pConfig->nUsermerge;
+ if( nMerge<0 ){
+ Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
+ fts5StructureRelease(pStruct);
+ pStruct = pNew;
+ nMin = 2;
+ nMerge = nMerge*-1;
+ }
+ if( pStruct && pStruct->nLevel ){
+ if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){
+ fts5StructureWrite(p, pStruct);
+ }
+ }
+ fts5StructureRelease(pStruct);
}
- fts5StructureRelease(pStruct);
-
return fts5IndexReturn(p);
}
rc = SQLITE_ERROR;
}
- /* Case 1: DELETE */
+ /* DELETE */
else if( nArg==1 ){
i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */
rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
}
- /* Case 2: INSERT */
+ /* INSERT */
else if( eType0!=SQLITE_INTEGER ){
/* If this is a REPLACE, first remove the current entry (if any) */
if( eConflict==SQLITE_REPLACE
fts5StorageInsert(&rc, pTab, apVal, pRowid);
}
- /* Case 2: UPDATE */
+ /* UPDATE */
else{
i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */
i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */
** If an error occurs, return NULL and leave an error in the database
** handle (accessible using sqlite3_errcode()/errmsg()).
*/
-static fts5_api *fts5_api_from_db(sqlite3 *db){
- fts5_api *pRet = 0;
+static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
sqlite3_stmt *pStmt = 0;
+ int rc;
- if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0)
- && SQLITE_ROW==sqlite3_step(pStmt)
- && sizeof(pRet)==sqlite3_column_bytes(pStmt, 0)
- ){
- memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet));
+ *ppApi = 0;
+ rc = sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ if( SQLITE_ROW==sqlite3_step(pStmt)
+ && sizeof(fts5_api*)==sqlite3_column_bytes(pStmt, 0)
+ ){
+ memcpy(ppApi, sqlite3_column_blob(pStmt, 0), sizeof(fts5_api*));
+ }
+ rc = sqlite3_finalize(pStmt);
}
- sqlite3_finalize(pStmt);
- return pRet;
+
+ return rc;
}
/* Extract the FTS5 API pointer from the database handle. The
** fts5_api_from_db() function above is copied verbatim from the
** FTS5 documentation. Refer there for details. */
- pApi = fts5_api_from_db(db);
+ rc = fts5_api_from_db(db, &pApi);
+ if( rc!=SQLITE_OK ) return rc;
/* If fts5_api_from_db() returns NULL, then either FTS5 is not registered
** with this database handle, or an error (OOM perhaps?) has occurred.
}
}
+proc fts5_segcount {tbl} {
+ set N 0
+ foreach n [fts5_level_segs $tbl] { incr N $n }
+ set N
+}
+
proc fts5_level_segs {tbl} {
set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
set ret [list]
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<$::nRowPerSeg)
INSERT INTO x8 SELECT repeat('x y ', i % 16) FROM ii;
- INSERT INTO x8(x8, rank) VALUES('automerge', 2);
+ INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
}
for {set tn 1} {[lindex [fts5_level_segs x8] 0]>0} {incr tn} {
execsql { INSERT INTO x8 VALUES( rnddoc(($i%16) + 5) ) }
while {[not_merged x8]} {
execsql {
- INSERT INTO x8(x8, rank) VALUES('automerge', 2);
+ INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
INSERT INTO x8(x8, rank) VALUES('merge', 1);
- INSERT INTO x8(x8, rank) VALUES('automerge', 16);
+ INSERT INTO x8(x8, rank) VALUES('usermerge', 16);
INSERT INTO x8(x8) VALUES('integrity-check');
}
}
do_merge2_test 2.3 20
#-------------------------------------------------------------------------
-# Test that an auto-merge will complete any merge that has already been
+# Test that a merge will complete any merge that has already been
# started, even if the number of input segments is less than the current
-# value of the 'automerge' configuration parameter.
+# value of the 'usermerge' configuration parameter.
#
db func rnddoc fts5_rnddoc
}
do_test 3.2 {
execsql {
- INSERT INTO x8(x8, rank) VALUES('automerge', 4);
+ INSERT INTO x8(x8, rank) VALUES('usermerge', 4);
INSERT INTO x8(x8, rank) VALUES('merge', 1);
}
fts5_level_segs x8
do_test 3.3 {
execsql {
- INSERT INTO x8(x8, rank) VALUES('automerge', 2);
+ INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
INSERT INTO x8(x8, rank) VALUES('merge', 1);
}
fts5_level_segs x8
} {2 1}
do_test 3.4 {
- execsql { INSERT INTO x8(x8, rank) VALUES('automerge', 4) }
+ execsql { INSERT INTO x8(x8, rank) VALUES('usermerge', 4) }
while {[not_merged x8]} {
execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1) }
}
INSERT INTO x8 SELECT mydoc() FROM ii;
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
INSERT INTO x8 SELECT mydoc() FROM ii;
- INSERT INTO x8(x8, rank) VALUES('automerge', 2);
+ INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
}
set expect [mycount]
# db eval {SELECT fts5_decode(rowid, block) AS r FROM x8_data} { puts $r }
}
+#-------------------------------------------------------------------------
+# Test that the 'merge' command does not modify the database if there is
+# no work to do.
+
+do_execsql_test 5.1 {
+ CREATE VIRTUAL TABLE x9 USING fts5(one, two);
+ INSERT INTO x9(x9, rank) VALUES('pgsz', 32);
+ INSERT INTO x9(x9, rank) VALUES('automerge', 2);
+ INSERT INTO x9(x9, rank) VALUES('usermerge', 2);
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+}
+
+do_test 5.2 {
+ while 1 {
+ set nChange [db total_changes]
+ execsql { INSERT INTO x9(x9, rank) VALUES('merge', 1); }
+ set nChange [expr [db total_changes] - $nChange]
+ #puts $nChange
+ if {$nChange<2} break
+ }
+} {}
+
+
+
finish_test
3 50
4 500
} {
-if {$tn!=4} continue
reset_db
db func rnddoc rnddoc
do_execsql_test 1.$tn.1 {
do_execsql_test 1.$tn.5 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
+
+ do_test 1.$tn.6 { fts5_segcount t1 } 1
+}
+
+foreach {tn nStep} {
+ 1 2
+ 2 10
+ 3 50
+ 4 500
+} {
+ reset_db
+ db func rnddoc rnddoc
+ do_execsql_test 1.$tn.1 {
+ CREATE VIRTUAL TABLE t1 USING fts5(x, y);
+ }
+ do_test 2.$tn.2 {
+ for {set i 0} {$i < $nStep} {incr i} {
+ execsql { INSERT INTO t1 VALUES( rnddoc(5), rnddoc(5) ) }
+ }
+ } {}
+
+ do_execsql_test 2.$tn.3 {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+ }
+
+ do_test 2.$tn.4 {
+ execsql { INSERT INTO t1(t1, rank) VALUES('merge', -1) }
+ while 1 {
+ set c [db total_changes]
+ execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) }
+ set c [expr [db total_changes]-$c]
+ if {$c<2} break
+ }
+ } {}
+
+ do_execsql_test 2.$tn.5 {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+ }
+
+ do_test 2.$tn.6 { fts5_segcount t1 } 1
}
finish_test
-C Fix\sa\sproblem\sin\sfts3/4\sthat\swas\scausing\sit\sto\sdiscard\sdata\scached\sin-memory\sif\san\s'optimize'\scommand\sis\srun\swhen\sthere\sis\sno\sdata\son\sdisk.\sThe\susual\sway\sthis\swould\shappen\sis\sif\sthe\svery\sfirst\stransaction\sthat\swrites\sto\sthe\sfts3/4\stable\salso\sincludes\san\s'optimize'\scommand.
-D 2016-03-09T18:17:42.821
+C Add\san\sincremental\soptimize\scapability\sto\sfts5.\sMake\sthe\s'merge'\scommand\sindependent\sof\sthe\s'automerge'\ssettings.
+D 2016-03-09T20:54:14.606
F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66
F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
-F ext/fts5/fts5Int.h fa7c17e5c3ec9c8690387ff962f9dc6aee75e114
+F ext/fts5/fts5Int.h 59e13423371512df1992532a08fe80518244f96b
F ext/fts5/fts5_aux.c daa57fb45216491814520bbb587e97bf81ced458
F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
-F ext/fts5/fts5_config.c 35c5173cae4eb17e82164a7f5aeef56a48903079
+F ext/fts5/fts5_config.c abd3ae1107ca8b8def7445a0a3b70c2e46d05986
F ext/fts5/fts5_expr.c 8e8e4635f655133eb39018072fc0f0942a2c4337
F ext/fts5/fts5_hash.c f3a7217c86eb8f272871be5f6aa1b6798960a337
-F ext/fts5/fts5_index.c 26a4a6112864feb599a6f6144d06a78bb179736a
-F ext/fts5/fts5_main.c db24ac714c6c4a1b3c24a1f8c25889f2952148c1
+F ext/fts5/fts5_index.c f7f3c042edf3e6a8984efa95ce1d5305d73bb535
+F ext/fts5/fts5_main.c b8501e1a6a11591c53b18ce7aea7e5386cfb0421
F ext/fts5/fts5_storage.c f8343db90d8c95a4d4b52f6676e354b4649ffd6e
F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
-F ext/fts5/fts5_test_mi.c b8d04816428202b2898d4ca38deb1739ac0110ae
+F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc
F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be
F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8
F ext/fts5/fts5_unicode2.c b450b209b157d598f7b9df9f837afb75a14c24bf
F ext/fts5/fts5_vocab.c dba72ca393d71c2588548b51380387f6b44c77a8
F ext/fts5/fts5parse.y 86fe6ba094a47e02fe8be2571539e6833d197764
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl b9b1fed811c0390511cef8b254826ea15d380f4d
+F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841
F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084
F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
F ext/fts5/test/fts5hash.test 06f9309ccb4d5050a131594e9e47d0b21456837d
F ext/fts5/test/fts5integrity.test f5e4f8d284385875068ad0f3e894ce43e9de835d
F ext/fts5/test/fts5matchinfo.test f7dde99697bcb310ea8faa8eb2714d9f4dfc0e1b
-F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367
+F ext/fts5/test/fts5merge.test 9e04a16963e32bf7c51835ce23e58325bbdfb35f
F ext/fts5/test/fts5merge2.test a6da3c16d694235938d1939f503cfa53f0943d75
F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc
F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c
-F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5
+F ext/fts5/test/fts5optimize.test 0f25ce4cc1f78a4cf5dd1247d30135b3f7180a19
F ext/fts5/test/fts5phrase.test f6d1d464da5beb25dc56277aa4f1d6102f0d9a2f
F ext/fts5/test/fts5plan.test 6a55ecbac9890765b0e16f8c421c7e0888cfe436
F ext/fts5/test/fts5porter.test 7cdc07bef301d70eebbfa75dcaf45c3680e1d0e1
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2c55c3c2950cafdc256ab540f60dc4609b9c354b
-R b4dbf73ab3865664cce5c88a4214186d
+P 79338b991bf01e81d336790ca87a0fa747da4ff3
+R a3916c8b307d01845fad235da028d539
U dan
-Z a31e8746a8bf8afd08aed0fea525a7d3
+Z 1a31d386083951a3ddfabf8f6c0df92f
-79338b991bf01e81d336790ca87a0fa747da4ff3
\ No newline at end of file
+556671444c03e3afca072d0f5e9bea2657de6fd3
\ No newline at end of file
projects / thirdparty / sqlite.git / commitdiff
