From: dan Date: Mon, 18 Aug 2014 19:30:01 +0000 (+0000) Subject: Add an "automerge=0" mode that disables auto-merging and falls back to fts4-style... X-Git-Tag: version-3.8.11~114^2~133 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=6885bbc71382ce75a126c51574330fdc1a6cf3f0;p=thirdparty%2Fsqlite.git Add an "automerge=0" mode that disables auto-merging and falls back to fts4-style crisis merges. FossilOrigin-Name: 2397404e152b908d838e6491294b263b05943b3f --- diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c index 8b07047aec..ade9c87ceb 100644 --- a/ext/fts5/fts5.c +++ b/ext/fts5/fts5.c @@ -872,6 +872,12 @@ static int fts5SpecialCommand(Fts5Table *pTab, sqlite3_value *pVal){ if( pgsz<32 ) pgsz = 32; sqlite3Fts5IndexPgsz(pTab->pIndex, pgsz); rc = SQLITE_OK; + }else + + if( n>10 && 0==sqlite3_strnicmp("automerge=", z, 10) ){ + int nAutomerge = atoi(&z[10]); + sqlite3Fts5IndexAutomerge(pTab->pIndex, nAutomerge); + rc = SQLITE_OK; } return rc; diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 82d4884dc7..b2865d6609 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -26,7 +26,7 @@ /************************************************************************* ** CUSTOM AUXILIARY FUNCTIONS ** -** Virtual table implemenations may overload SQL functions by implementing +** Virtual table implementations may overload SQL functions by implementing ** the sqlite3_module.xFindFunction() method. */ @@ -42,7 +42,6 @@ typedef void (*fts5_extension_function)( ); /* -** ** xUserData(pFts): ** ** Return a copy of the context pointer the extension function was diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 602f293097..9512d3505f 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -273,6 +273,8 @@ int sqlite3Fts5IndexInit(sqlite3*); */ void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz); +void sqlite3Fts5IndexAutomerge(Fts5Index *p, int nMerge); + /* ** Return the total number of entries read from the %_data table by ** this connection since it was created. diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 9676ad4b45..214cc13931 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -45,6 +45,7 @@ #define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ #define FTS5_MIN_MERGE 4 /* Minimum number of segments to merge */ +#define FTS5_CRISIS_MERGE 16 /* Maximum number of segments to merge */ #define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ @@ -291,6 +292,7 @@ struct Fts5Index { char *zDataTbl; /* Name of %_data table */ int pgsz; /* Target page size for this index */ int nMinMerge; /* Minimum input segments in a merge */ + int nCrisisMerge; /* Maximum allowed segments per level */ int nWorkUnit; /* Leaf pages in a "unit" of work */ /* @@ -2987,14 +2989,15 @@ static void fts5TrimSegments(Fts5Index *p, Fts5MultiSegIter *pIter){ static void fts5IndexMergeLevel( Fts5Index *p, /* FTS5 backend object */ int iIdx, /* Index to work on */ - Fts5Structure *pStruct, /* Stucture of index iIdx */ + Fts5Structure **ppStruct, /* IN/OUT: Stucture of index iIdx */ int iLvl, /* Level to read input from */ int *pnRem /* Write up to this many output leaves */ ){ + Fts5Structure *pStruct = *ppStruct; Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; - Fts5StructureLevel *pLvlOut = &pStruct->aLevel[iLvl+1]; + Fts5StructureLevel *pLvlOut; Fts5MultiSegIter *pIter = 0; /* Iterator to read input data */ - int nRem = *pnRem; /* Output leaf pages left to write */ + int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ int nInput; /* Number of input segments */ Fts5SegWriter writer; /* Writer object */ Fts5StructureSegment *pSeg; /* Output segment */ @@ -3009,12 +3012,24 @@ static void fts5IndexMergeLevel( memset(&term, 0, sizeof(Fts5Buffer)); writer.iIdx = iIdx; if( pLvl->nMerge ){ + pLvlOut = &pStruct->aLevel[iLvl+1]; assert( pLvlOut->nSeg>0 ); nInput = pLvl->nMerge; fts5WriteInitForAppend(p, &writer, iIdx, &pLvlOut->aSeg[pLvlOut->nSeg-1]); pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1]; }else{ int iSegid = fts5AllocateSegid(p, pStruct); + + /* Extend the Fts5Structure object as required to ensure the output + ** segment exists. */ + if( iLvl==pStruct->nLevel-1 ){ + fts5StructureAddLevel(&p->rc, ppStruct); + pStruct = *ppStruct; + } + fts5StructureExtendLevel(&p->rc, pStruct, iLvl+1, 1, 0); + pLvl = &pStruct->aLevel[iLvl]; + pLvlOut = &pStruct->aLevel[iLvl+1]; + fts5WriteInit(p, &writer, iIdx, iSegid); /* Add the new segment to the output level */ @@ -3049,7 +3064,7 @@ fflush(stdout); int nTerm; const u8 *pTerm = fts5MultiIterTerm(pIter, &nTerm); if( nTerm!=term.n || memcmp(pTerm, term.p, nTerm) ){ - if( writer.nLeafWritten>nRem ){ + if( pnRem && writer.nLeafWritten>nRem ){ fts5ChunkIterRelease(&sPos); break; } @@ -3106,7 +3121,7 @@ fflush(stdout); fts5MultiIterFree(p, pIter); fts5BufferFree(&term); - *pnRem -= writer.nLeafWritten; + if( pnRem ) *pnRem -= writer.nLeafWritten; } /* @@ -3165,17 +3180,31 @@ static void fts5IndexWork( #endif if( nBestnMinMerge && pStruct->aLevel[iBestLvl].nMerge==0 ) break; - if( iBestLvl==pStruct->nLevel-1 ){ - fts5StructureAddLevel(&p->rc, &pStruct); - } - fts5StructureExtendLevel(&p->rc, pStruct, iBestLvl+1, 1, 0); - fts5IndexMergeLevel(p, iIdx, pStruct, iBestLvl, &nRem); + fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem); fts5StructurePromote(p, iBestLvl+1, pStruct); assert( nRem==0 || p->rc==SQLITE_OK ); *ppStruct = pStruct; } } +static void fts5IndexCrisisMerge( + Fts5Index *p, /* FTS5 backend object */ + int iIdx, /* Index to work on */ + Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ +){ + Fts5Structure *pStruct = *ppStruct; + int iLvl = 0; + while( p->rc==SQLITE_OK + && iLvlnLevel + && pStruct->aLevel[iLvl].nSeg>=p->nCrisisMerge + ){ + fts5IndexMergeLevel(p, iIdx, &pStruct, iLvl, 0); + fts5StructurePromote(p, iLvl+1, pStruct); + iLvl++; + } + *ppStruct = pStruct; +} + typedef struct Fts5FlushCtx Fts5FlushCtx; struct Fts5FlushCtx { Fts5Index *pIdx; @@ -3203,7 +3232,6 @@ static int fts5FlushNewEntry( const u8 *aPoslist, int nPoslist ){ - Fts5Buffer *pBuf; Fts5FlushCtx *p = (Fts5FlushCtx*)pCtx; int rc = SQLITE_OK; @@ -3250,7 +3278,8 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){ if( p->rc==SQLITE_OK ) p->rc = rc; fts5WriteFinish(p, &ctx.writer, &nHeight, &pgnoLast); - /* Edit the Fts5Structure and write it back to the database. */ + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ if( pStruct->nLevel==0 ){ fts5StructureAddLevel(&p->rc, &pStruct); } @@ -3264,7 +3293,8 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){ } } - fts5IndexWork(p, iHash, &pStruct, pgnoLast); + if( p->nMinMerge>0 ) fts5IndexWork(p, iHash, &pStruct, pgnoLast); + fts5IndexCrisisMerge(p, iHash, &pStruct); fts5StructureWrite(p, iHash, pStruct); fts5StructureRelease(pStruct); } @@ -3343,6 +3373,7 @@ int sqlite3Fts5IndexOpen( p->pConfig = pConfig; p->pgsz = 1000; p->nMinMerge = FTS5_MIN_MERGE; + p->nCrisisMerge = FTS5_CRISIS_MERGE; p->nWorkUnit = FTS5_WORK_UNIT; p->nMaxPendingData = 1024*1024; p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName); @@ -3962,6 +3993,19 @@ void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz){ p->pgsz = pgsz; } +/* +** Set the minimum number of segments that an auto-merge operation should +** attempt to merge together. A value of 1 sets the object to use the +** compile time default. Zero or less disables auto-merge altogether. +*/ +void sqlite3Fts5IndexAutomerge(Fts5Index *p, int nMinMerge){ + if( nMinMerge==1 ){ + p->nMinMerge = FTS5_MIN_MERGE; + }else{ + p->nMinMerge = nMinMerge; + } +} + /* ** Iterator pMulti currently points to a valid entry (not EOF). This ** function appends a copy of the position-list of the entry pMulti diff --git a/manifest b/manifest index 1f03c49271..0edf60c0aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cache\sthe\svalue\sof\sthe\s"totals"\srecord\sin\smemory\sduring\stransactions. -D 2014-08-12T16:07:35.119 +C Add\san\s"automerge=0"\smode\sthat\sdisables\sauto-merging\sand\sfalls\sback\sto\sfts4-style\scrisis\smerges. +D 2014-08-18T19:30:01.020 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -103,15 +103,15 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368 -F ext/fts5/fts5.c 31db0b90774201820915db17916a9a4d9ac1c80b -F ext/fts5/fts5.h 8ace10d5b249a3baa983c79e7a1306d2a79cfd6a -F ext/fts5/fts5Int.h b0eb5cd422ba74148b30753f01031d546ffb98e4 +F ext/fts5/fts5.c dd56525d45b354218b86c9accab2ed12ea4b4f4f +F ext/fts5/fts5.h 1c501ea7c5c686b8aa7fba0382badc5df6026aa7 +F ext/fts5/fts5Int.h bc6fa374a42c6121ae8276b20f141d6cd6d8d9f9 F ext/fts5/fts5_aux.c 31e581413ecab0962ce2b37468f9f658f36f4b0e F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00 F ext/fts5/fts5_config.c f4ebf143e141b8c77355e3b15aba81b7be51d710 F ext/fts5/fts5_expr.c 7b8e380233176053841904a86006696ee8f6cd24 F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279 -F ext/fts5/fts5_index.c 0453bb593fe0ef6245762b6823e88839757fdc75 +F ext/fts5/fts5_index.c 3f4d84a1762e4284319739d4672b90b18b91060a F ext/fts5/fts5_storage.c 5913aa01a1dada1c5e1a39e4cbb44e84c5f7f350 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 @@ -1163,7 +1163,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 3ff0fec22f92dfb54e62eeb48772eddffdbeb0d6 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc -F tool/loadfts.c b5b3206ddd58d89ec8d54038c784bcadd6195915 +F tool/loadfts.c 76b6589ab5efcdc9cfe16d43ab5a6c2618e44bd4 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 @@ -1202,7 +1202,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f1cb48f412a5f200f1fe04f91072864f379db08f -R 6321f5990f75eb3b76570e3e341050f0 +P 05dfdad445b22f375b71abe0b1fa1bf7ca331be7 +R a4a2c1b7e4d79c30cafb117d4f31d356 U dan -Z f483f9b1471761b27d0fa8b15d969ed4 +Z a836ac39870a35d2f8436dd5e99c8845 diff --git a/manifest.uuid b/manifest.uuid index dc8ddf389d..7ae37fa218 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -05dfdad445b22f375b71abe0b1fa1bf7ca331be7 \ No newline at end of file +2397404e152b908d838e6491294b263b05943b3f \ No newline at end of file diff --git a/tool/loadfts.c b/tool/loadfts.c index 7da07b15dd..5b2ed5dc6a 100644 --- a/tool/loadfts.c +++ b/tool/loadfts.c @@ -163,6 +163,9 @@ int main(int argc, char **argv){ char *zSql; VisitContext sCtx; + int nCmd = 0; + char **aCmd = 0; + if( argc % 2 ) showHelp(argv[0]); for(i=1; i<(argc-1); i+=2){ @@ -172,7 +175,7 @@ int main(int argc, char **argv){ iFts = atoi(zArg); if( iFts!=3 && iFts!=4 && iFts!= 5) showHelp(argv[0]); } - if( strcmp(zOpt, "-trans")==0 ){ + else if( strcmp(zOpt, "-trans")==0 ){ nRowPerTrans = atoi(zArg); } else if( strcmp(zOpt, "-idx")==0 ){ @@ -182,6 +185,14 @@ int main(int argc, char **argv){ else if( strcmp(zOpt, "-dir")==0 ){ zDir = zArg; } + else if( strcmp(zOpt, "-special")==0 ){ + nCmd++; + aCmd = sqlite3_realloc(aCmd, sizeof(char*) * nCmd); + aCmd[nCmd-1] = zArg; + } + else{ + showHelp(argv[0]); + } } /* Open the database file */ @@ -198,6 +209,13 @@ int main(int argc, char **argv){ if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_exec(1)", db); sqlite3_free(zSql); + for(i=0; i