From: dan Date: Fri, 23 Jan 2015 06:50:33 +0000 (+0000) Subject: Remove some redundant code from fts5. X-Git-Tag: version-3.8.11~114^2~95 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=626d9e306298bc2fa401268f8c917322504d3246;p=thirdparty%2Fsqlite.git Remove some redundant code from fts5. FossilOrigin-Name: 939b7a5de25e064bdf08e03864c35ab718da6f6f --- diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 7b7a86d8eb..d7172e315e 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -86,6 +86,7 @@ struct Fts5Config { int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ + int nCrisisMerge; /* Maximum allowed segments per level */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ }; diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index ecf24dcd48..438cdf370a 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -17,6 +17,7 @@ #define FTS5_DEFAULT_PAGE_SIZE 1000 #define FTS5_DEFAULT_AUTOMERGE 4 +#define FTS5_DEFAULT_CRISISMERGE 16 /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (128*1024) @@ -717,6 +718,19 @@ int sqlite3Fts5ConfigSetValue( } } + else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){ + int nCrisisMerge = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nCrisisMerge = sqlite3_value_int(pVal); + } + if( nCrisisMerge<0 ){ + if( pbBadkey ) *pbBadkey = 1; + }else{ + if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; + pConfig->nCrisisMerge = nCrisisMerge; + } + } + else if( 0==sqlite3_stricmp(zKey, "rank") ){ const char *zIn = (const char*)sqlite3_value_text(pVal); char *zRank; @@ -749,6 +763,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ /* Set default values */ pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; + pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; zSql = sqlite3_mprintf(zSelect, pConfig->zDb, pConfig->zName); if( zSql==0 ){ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index d7cc3da1db..3cd4892a59 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -43,7 +43,6 @@ #define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */ #define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ -#define FTS5_CRISIS_MERGE 16 /* Maximum number of segments to merge */ #define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ @@ -293,7 +292,6 @@ typedef struct Fts5StructureSegment Fts5StructureSegment; struct Fts5Index { Fts5Config *pConfig; /* Virtual table configuration */ char *zDataTbl; /* Name of %_data table */ - int nCrisisMerge; /* Maximum allowed segments per level */ int nWorkUnit; /* Leaf pages in a "unit" of work */ /* @@ -1105,8 +1103,8 @@ static Fts5Structure *fts5StructureRead(Fts5Index *p, int iIdx){ if( !pData ) return 0; p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet); - if( p->rc==SQLITE_OK && p->pConfig->iCookie!=iCookie ){ - p->rc = sqlite3Fts5ConfigLoad(p->pConfig, iCookie); + if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); } fts5DataRelease(pData); @@ -1250,7 +1248,6 @@ static void fts5StructurePromote( int iPromote = -1; int szPromote; /* Promote anything this size or smaller */ Fts5StructureSegment *pSeg; /* Segment just written */ - Fts5StructureLevel *pTst; int szSeg; /* Size of segment just written */ @@ -1259,11 +1256,11 @@ static void fts5StructurePromote( /* Check for condition (a) */ for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--); - pTst = &pStruct->aLevel[iTst]; - assert( pTst->nMerge==0 ); if( iTst>=0 ){ int i; int szMax = 0; + Fts5StructureLevel *pTst = &pStruct->aLevel[iTst]; + assert( pTst->nMerge==0 ); for(i=0; inSeg; i++){ int sz = pTst->aSeg[i].pgnoLast - pTst->aSeg[i].pgnoFirst + 1; if( sz>szMax ) szMax = sz; @@ -2483,28 +2480,6 @@ static int fts5PosIterEof(Fts5Index *p, Fts5PosIter *pIter){ return (p->rc || pIter->chunk.pLeaf==0); } -/* -** Add an entry for (iRowid/iCol/iPos) to the doclist for (pToken/nToken) -** in hash table for index iIdx. If iIdx is zero, this is the main terms -** index. Values of 1 and greater for iIdx are prefix indexes. -** -** If an OOM error is encountered, set the Fts5Index.rc error code -** accordingly. -*/ -static void fts5AddTermToHash( - Fts5Index *p, /* Index object to write to */ - int iIdx, /* Entry in p->aHash[] to update */ - int iCol, /* Column token appears in (-ve -> delete) */ - int iPos, /* Position of token within column */ - const char *pToken, int nToken /* Token to add or remove to or from index */ -){ - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3Fts5HashWrite( - p->apHash[iIdx], p->iWriteRowid, iCol, iPos, pToken, nToken - ); - } -} - /* ** Allocate a new segment-id for the structure pStruct. ** @@ -3228,7 +3203,7 @@ static void fts5IndexCrisisMerge( int iLvl = 0; while( p->rc==SQLITE_OK && iLvlnLevel - && pStruct->aLevel[iLvl].nSeg>=p->nCrisisMerge + && pStruct->aLevel[iLvl].nSeg>=p->pConfig->nCrisisMerge ){ fts5IndexMergeLevel(p, iIdx, &pStruct, iLvl, 0); fts5StructurePromote(p, iLvl+1, pStruct); @@ -4000,6 +3975,29 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ */ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){ assert( p->rc==SQLITE_OK ); + + /* Allocate hash tables if they have not already been allocated */ + if( p->apHash==0 ){ + int i; + int rc = SQLITE_OK; + int nHash = p->pConfig->nPrefix + 1; + Fts5Hash **apNew; + + apNew = (Fts5Hash**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Hash*)*nHash); + for(i=0; rc==SQLITE_OK && inPendingData); + } + if( rc==SQLITE_OK ){ + p->apHash = apNew; + }else{ + for(i=0; iiWriteRowid || (p->nPendingData > p->nMaxPendingData) ){ fts5IndexFlush(p); } @@ -4071,7 +4069,6 @@ int sqlite3Fts5IndexOpen( memset(p, 0, sizeof(Fts5Index)); p->pConfig = pConfig; - p->nCrisisMerge = FTS5_CRISIS_MERGE; p->nWorkUnit = FTS5_WORK_UNIT; p->nMaxPendingData = 1024*1024; p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName); @@ -4196,29 +4193,26 @@ int sqlite3Fts5IndexWrite( const char *pToken, int nToken /* Token to add or remove to or from index */ ){ int i; /* Used to iterate through indexes */ + int rc; /* Return code */ Fts5Config *pConfig = p->pConfig; - assert( p->rc==SQLITE_OK ); - /* Allocate hash tables if they have not already been allocated */ - if( p->apHash==0 ){ - int nHash = pConfig->nPrefix + 1; - p->apHash = (Fts5Hash**)fts5IdxMalloc(p, sizeof(Fts5Hash*) * nHash); - for(i=0; p->rc==SQLITE_OK && irc = sqlite3Fts5HashNew(&p->apHash[i], &p->nPendingData); - } - } + assert( p->rc==SQLITE_OK ); /* Add the new token to the main terms hash table. And to each of the ** prefix hash tables that it is large enough for. */ - fts5AddTermToHash(p, 0, iCol, iPos, pToken, nToken); - for(i=0; inPrefix; i++){ + rc = sqlite3Fts5HashWrite( + p->apHash[0], p->iWriteRowid, iCol, iPos, pToken, nToken + ); + for(i=0; inPrefix && rc==SQLITE_OK; i++){ int nByte = fts5IndexCharlenToBytelen(pToken, nToken, pConfig->aPrefix[i]); if( nByte ){ - fts5AddTermToHash(p, i+1, iCol, iPos, pToken, nByte); + rc = sqlite3Fts5HashWrite( + p->apHash[i+1], p->iWriteRowid, iCol, iPos, pToken, nByte + ); } } - return fts5IndexReturn(p); + return rc; } /* diff --git a/ext/fts5/tool/loadfts5.tcl b/ext/fts5/tool/loadfts5.tcl index c63564cd0a..034286bc60 100644 --- a/ext/fts5/tool/loadfts5.tcl +++ b/ext/fts5/tool/loadfts5.tcl @@ -29,6 +29,7 @@ proc usage {} { puts stderr " -porter (use porter tokenizer)" puts stderr " -limit N (load no more than N documents)" puts stderr " -automerge N (set the automerge parameter to N)" + puts stderr " -crisismerge N (set the crisismerge parameter to N)" exit 1 } @@ -36,6 +37,7 @@ set O(vtab) fts5 set O(tok) "" set O(limit) 0 set O(automerge) -1 +set O(crisismerge) -1 if {[llength $argv]<2} usage set nOpt [expr {[llength $argv]-2}] @@ -64,6 +66,11 @@ for {set i 0} {$i < $nOpt} {incr i} { set O(automerge) [lindex $argv $i] } + -crisismerge { + if { [incr i]>=$nOpt } usage + set O(crisismerge) [lindex $argv $i] + } + default { usage } @@ -82,6 +89,12 @@ db transaction { db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) } } } + if {$O(crisismerge)>=0} { + if {$O(vtab) == "fts5"} { + db eval {INSERT INTO t1(t1, rank) VALUES('crisismerge', $O(crisismerge))} + } else { + } + } load_hierachy [lindex $argv end] } diff --git a/manifest b/manifest index ae6189124b..5ed6b12d4c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sand\sfixes\sfor\sfts5. -D 2015-01-22T19:13:08.439 +C Remove\ssome\sredundant\scode\sfrom\sfts5. +D 2015-01-23T06:50:33.338 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,13 +106,13 @@ F ext/fts3/unicode/mkunicode.tcl 4199cb887040ee3c3cd59a5171ddb0566904586e F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a F ext/fts5/fts5.c 0ba5a8f27e1aa4deab82f0fc295d55f67dfe7f34 F ext/fts5/fts5.h f931954065693898d26c51f23f1d27200184a69a -F ext/fts5/fts5Int.h 99da8551098bb23fd94d0aa3f4ae1a411ee630b4 +F ext/fts5/fts5Int.h da4ad7558c2284fdf3297f907e2c5454a2237e15 F ext/fts5/fts5_aux.c 549aef152b0fd46020f5595d861b1fd60b3f9b4f F ext/fts5/fts5_buffer.c 32dd3c950392346ca69a0f1803501766c5c954f9 -F ext/fts5/fts5_config.c 33534ca25198cc62c54ff7d285d455c57ad19399 +F ext/fts5/fts5_config.c e3421a76c2abd33a05ac09df0c97c64952d1e700 F ext/fts5/fts5_expr.c 8a0e643768666dc2bffe74104141274809699808 F ext/fts5/fts5_hash.c 7a87f9f2eae2216c710064821fa0621ac6a8ce7b -F ext/fts5/fts5_index.c ee7b141adde3dbdaa56f1e198c06a0786d298126 +F ext/fts5/fts5_index.c 604e346f7a04b87f11090b91a80afa50bc74f88b F ext/fts5/fts5_storage.c d56722960982d0c48ba1b88d9001fefed8cff1a4 F ext/fts5/fts5_tcl.c 1293fac2bb26903fd3d5cdee59c5885ba7e620d5 F ext/fts5/fts5_tokenize.c 7c61d5c35c3449597bdeaa54dd48afe26852c7b0 @@ -146,7 +146,7 @@ F ext/fts5/test/fts5rowid.test db482328fe9bf78bb6a09f2dbf055e2caeaac00a F ext/fts5/test/fts5tokenizer.test b34ae592db66f6e89546d791ce1f905ba0b3395c F ext/fts5/test/fts5unicode.test 79b3e34eb29ce4929628aa514a40cb467fdabe4d F ext/fts5/test/fts5unicode2.test 64a5267fd6082fcb46439892ebd0cbaa5c38acee -F ext/fts5/tool/loadfts5.tcl 17c9771fb225b6b7ddd02a698fc7f320eadd7b15 +F ext/fts5/tool/loadfts5.tcl 0d39b916550097a3b714060bfc1164a4a9b73f4c F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 @@ -1282,7 +1282,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c020a291ed293a66d21c5885e50a7fee04aa6366 -R 244beb886a9d1f5f10328b67a9ad3f5b +P 5b295897153e9b26cd0d2e7ea112a4d461d0a665 +R 02a6ea9937d0b3e40d0a3982c5f888d6 U dan -Z 4d510e0e441ea9491ac2e8425faae5c2 +Z 8b5a907ba25d2b7d394f8c2162c154cb diff --git a/manifest.uuid b/manifest.uuid index 75fced3560..63b8637677 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b295897153e9b26cd0d2e7ea112a4d461d0a665 \ No newline at end of file +939b7a5de25e064bdf08e03864c35ab718da6f6f \ No newline at end of file