Add an "automerge=0" mode that disables auto-merging and falls back to fts4-style...

FossilOrigin-Name: 2397404e152b908d838e6491294b263b05943b3f

diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c
index 8b07047aec9af013a4aeaae7338da796e63d96ba..ade9c87ceb9923aa6133777bcccd6eca6cdcd461 100644 (file)
--- a/ext/fts5/fts5.c
+++ b/ext/fts5/fts5.c
@@ -872,6 +872,12 @@ static int fts5SpecialCommand(Fts5Table *pTab, sqlite3_value *pVal){
     if( pgsz<32 ) pgsz = 32;
     sqlite3Fts5IndexPgsz(pTab->pIndex, pgsz);
     rc = SQLITE_OK;
+  }else
+  
+  if( n>10 && 0==sqlite3_strnicmp("automerge=", z, 10) ){
+    int nAutomerge = atoi(&z[10]);
+    sqlite3Fts5IndexAutomerge(pTab->pIndex, nAutomerge);
+    rc = SQLITE_OK;
   }
 
   return rc;

diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h
index 82d4884dc7ed37f69026c0cd37b404c51e799237..b2865d66099dfb981cee474c6713dbae787abf37 100644 (file)
--- a/ext/fts5/fts5.h
+++ b/ext/fts5/fts5.h
@@ -26,7 +26,7 @@
 /*************************************************************************
 ** CUSTOM AUXILIARY FUNCTIONS
 **
-** Virtual table implemenations may overload SQL functions by implementing
+** Virtual table implementations may overload SQL functions by implementing
 ** the sqlite3_module.xFindFunction() method.
 */
 
@@ -42,7 +42,6 @@ typedef void (*fts5_extension_function)(
 );
 
 /*
-**
 ** xUserData(pFts):
 **
 **   Return a copy of the context pointer the extension function was 

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 602f293097f62bf4ca16fb3aa80f6e2237344baa..9512d3505f7ecca7ba0fa06d360f0098521f1140 100644 (file)
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -273,6 +273,8 @@ int sqlite3Fts5IndexInit(sqlite3*);
 */
 void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz);
 
+void sqlite3Fts5IndexAutomerge(Fts5Index *p, int nMerge);
+
 /*
 ** Return the total number of entries read from the %_data table by 
 ** this connection since it was created.

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 9676ad4b458dbcceccc05b130dd9d106008acead..214cc13931d7eaacfe2822972cb77d277a767d29 100644 (file)
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -45,6 +45,7 @@
 
 #define FTS5_WORK_UNIT      64    /* Number of leaf pages in unit of work */
 #define FTS5_MIN_MERGE       4    /* Minimum number of segments to merge */
+#define FTS5_CRISIS_MERGE   16    /* Maximum number of segments to merge */
 
 #define FTS5_MIN_DLIDX_SIZE  4    /* Add dlidx if this many empty pages */
 
@@ -291,6 +292,7 @@ struct Fts5Index {
   char *zDataTbl;                 /* Name of %_data table */
   int pgsz;                       /* Target page size for this index */
   int nMinMerge;                  /* Minimum input segments in a merge */
+  int nCrisisMerge;               /* Maximum allowed segments per level */
   int nWorkUnit;                  /* Leaf pages in a "unit" of work */
 
   /*
@@ -2987,14 +2989,15 @@ static void fts5TrimSegments(Fts5Index *p, Fts5MultiSegIter *pIter){
 static void fts5IndexMergeLevel(
   Fts5Index *p,                   /* FTS5 backend object */
   int iIdx,                       /* Index to work on */
-  Fts5Structure *pStruct,         /* Stucture of index iIdx */
+  Fts5Structure **ppStruct,       /* IN/OUT: Stucture of index iIdx */
   int iLvl,                       /* Level to read input from */
   int *pnRem                      /* Write up to this many output leaves */
 ){
+  Fts5Structure *pStruct = *ppStruct;
   Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
-  Fts5StructureLevel *pLvlOut = &pStruct->aLevel[iLvl+1];
+  Fts5StructureLevel *pLvlOut;
   Fts5MultiSegIter *pIter = 0;    /* Iterator to read input data */
-  int nRem = *pnRem;              /* Output leaf pages left to write */
+  int nRem = pnRem ? *pnRem : 0;  /* Output leaf pages left to write */
   int nInput;                     /* Number of input segments */
   Fts5SegWriter writer;           /* Writer object */
   Fts5StructureSegment *pSeg;     /* Output segment */
@@ -3009,12 +3012,24 @@ static void fts5IndexMergeLevel(
   memset(&term, 0, sizeof(Fts5Buffer));
   writer.iIdx = iIdx;
   if( pLvl->nMerge ){
+    pLvlOut = &pStruct->aLevel[iLvl+1];
     assert( pLvlOut->nSeg>0 );
     nInput = pLvl->nMerge;
     fts5WriteInitForAppend(p, &writer, iIdx, &pLvlOut->aSeg[pLvlOut->nSeg-1]);
     pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1];
   }else{
     int iSegid = fts5AllocateSegid(p, pStruct);
+
+    /* Extend the Fts5Structure object as required to ensure the output
+    ** segment exists. */
+    if( iLvl==pStruct->nLevel-1 ){
+      fts5StructureAddLevel(&p->rc, ppStruct);
+      pStruct = *ppStruct;
+    }
+    fts5StructureExtendLevel(&p->rc, pStruct, iLvl+1, 1, 0);
+    pLvl = &pStruct->aLevel[iLvl];
+    pLvlOut = &pStruct->aLevel[iLvl+1];
+
     fts5WriteInit(p, &writer, iIdx, iSegid);
 
     /* Add the new segment to the output level */
@@ -3049,7 +3064,7 @@ fflush(stdout);
       int nTerm;
       const u8 *pTerm = fts5MultiIterTerm(pIter, &nTerm);
       if( nTerm!=term.n || memcmp(pTerm, term.p, nTerm) ){
-        if( writer.nLeafWritten>nRem ){
+        if( pnRem && writer.nLeafWritten>nRem ){
           fts5ChunkIterRelease(&sPos);
           break;
         }
@@ -3106,7 +3121,7 @@ fflush(stdout);
 
   fts5MultiIterFree(p, pIter);
   fts5BufferFree(&term);
-  *pnRem -= writer.nLeafWritten;
+  if( pnRem ) *pnRem -= writer.nLeafWritten;
 }
 
 /*
@@ -3165,17 +3180,31 @@ static void fts5IndexWork(
 #endif
 
     if( nBest<p->nMinMerge && pStruct->aLevel[iBestLvl].nMerge==0 ) break;
-    if( iBestLvl==pStruct->nLevel-1 ){
-      fts5StructureAddLevel(&p->rc, &pStruct);
-    }
-    fts5StructureExtendLevel(&p->rc, pStruct, iBestLvl+1, 1, 0);
-    fts5IndexMergeLevel(p, iIdx, pStruct, iBestLvl, &nRem);
+    fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem);
     fts5StructurePromote(p, iBestLvl+1, pStruct);
     assert( nRem==0 || p->rc==SQLITE_OK );
     *ppStruct = pStruct;
   }
 }
 
+static void fts5IndexCrisisMerge(
+  Fts5Index *p,                   /* FTS5 backend object */
+  int iIdx,                       /* Index to work on */
+  Fts5Structure **ppStruct        /* IN/OUT: Current structure of index */
+){
+  Fts5Structure *pStruct = *ppStruct;
+  int iLvl = 0;
+  while( p->rc==SQLITE_OK 
+      && iLvl<pStruct->nLevel
+      && pStruct->aLevel[iLvl].nSeg>=p->nCrisisMerge 
+  ){
+    fts5IndexMergeLevel(p, iIdx, &pStruct, iLvl, 0);
+    fts5StructurePromote(p, iLvl+1, pStruct);
+    iLvl++;
+  }
+  *ppStruct = pStruct;
+}
+
 typedef struct Fts5FlushCtx Fts5FlushCtx;
 struct Fts5FlushCtx {
   Fts5Index *pIdx;
@@ -3203,7 +3232,6 @@ static int fts5FlushNewEntry(
   const u8 *aPoslist, 
   int nPoslist
 ){
-  Fts5Buffer *pBuf;
   Fts5FlushCtx *p = (Fts5FlushCtx*)pCtx;
   int rc = SQLITE_OK;
 
@@ -3250,7 +3278,8 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
     if( p->rc==SQLITE_OK ) p->rc = rc;
     fts5WriteFinish(p, &ctx.writer, &nHeight, &pgnoLast);
 
-    /* Edit the Fts5Structure and write it back to the database. */
+    /* Update the Fts5Structure. It is written back to the database by the
+    ** fts5StructureRelease() call below.  */
     if( pStruct->nLevel==0 ){
       fts5StructureAddLevel(&p->rc, &pStruct);
     }
@@ -3264,7 +3293,8 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
     }
   }
 
-  fts5IndexWork(p, iHash, &pStruct, pgnoLast);
+  if( p->nMinMerge>0 ) fts5IndexWork(p, iHash, &pStruct, pgnoLast);
+  fts5IndexCrisisMerge(p, iHash, &pStruct);
   fts5StructureWrite(p, iHash, pStruct);
   fts5StructureRelease(pStruct);
 }
@@ -3343,6 +3373,7 @@ int sqlite3Fts5IndexOpen(
   p->pConfig = pConfig;
   p->pgsz = 1000;
   p->nMinMerge = FTS5_MIN_MERGE;
+  p->nCrisisMerge = FTS5_CRISIS_MERGE;
   p->nWorkUnit = FTS5_WORK_UNIT;
   p->nMaxPendingData = 1024*1024;
   p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName);
@@ -3962,6 +3993,19 @@ void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz){
   p->pgsz = pgsz;
 }
 
+/*
+** Set the minimum number of segments that an auto-merge operation should
+** attempt to merge together. A value of 1 sets the object to use the 
+** compile time default. Zero or less disables auto-merge altogether.
+*/
+void sqlite3Fts5IndexAutomerge(Fts5Index *p, int nMinMerge){
+  if( nMinMerge==1 ){
+    p->nMinMerge = FTS5_MIN_MERGE;
+  }else{
+    p->nMinMerge = nMinMerge;
+  }
+}
+
 /*
 ** Iterator pMulti currently points to a valid entry (not EOF). This
 ** function appends a copy of the position-list of the entry pMulti 

diff --git a/manifest b/manifest
index 1f03c49271cdba9639776d5632c0f8244165dcfb..0edf60c0aa9be028278d5d1b73a410a58d8121fc 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Cache\sthe\svalue\sof\sthe\s"totals"\srecord\sin\smemory\sduring\stransactions.
-D 2014-08-12T16:07:35.119
+C Add\san\s"automerge=0"\smode\sthat\sdisables\sauto-merging\sand\sfalls\sback\sto\sfts4-style\scrisis\smerges.
+D 2014-08-18T19:30:01.020
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -103,15 +103,15 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
-F ext/fts5/fts5.c 31db0b90774201820915db17916a9a4d9ac1c80b
-F ext/fts5/fts5.h 8ace10d5b249a3baa983c79e7a1306d2a79cfd6a
-F ext/fts5/fts5Int.h b0eb5cd422ba74148b30753f01031d546ffb98e4
+F ext/fts5/fts5.c dd56525d45b354218b86c9accab2ed12ea4b4f4f
+F ext/fts5/fts5.h 1c501ea7c5c686b8aa7fba0382badc5df6026aa7
+F ext/fts5/fts5Int.h bc6fa374a42c6121ae8276b20f141d6cd6d8d9f9
 F ext/fts5/fts5_aux.c 31e581413ecab0962ce2b37468f9f658f36f4b0e
 F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00
 F ext/fts5/fts5_config.c f4ebf143e141b8c77355e3b15aba81b7be51d710
 F ext/fts5/fts5_expr.c 7b8e380233176053841904a86006696ee8f6cd24
 F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279
-F ext/fts5/fts5_index.c 0453bb593fe0ef6245762b6823e88839757fdc75
+F ext/fts5/fts5_index.c 3f4d84a1762e4284319739d4672b90b18b91060a
 F ext/fts5/fts5_storage.c 5913aa01a1dada1c5e1a39e4cbb44e84c5f7f350
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
@@ -1163,7 +1163,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/lemon.c 3ff0fec22f92dfb54e62eeb48772eddffdbeb0d6
 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
-F tool/loadfts.c b5b3206ddd58d89ec8d54038c784bcadd6195915
+F tool/loadfts.c 76b6589ab5efcdc9cfe16d43ab5a6c2618e44bd4
 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6
 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383
 F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670
@@ -1202,7 +1202,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P f1cb48f412a5f200f1fe04f91072864f379db08f
-R 6321f5990f75eb3b76570e3e341050f0
+P 05dfdad445b22f375b71abe0b1fa1bf7ca331be7
+R a4a2c1b7e4d79c30cafb117d4f31d356
 U dan
-Z f483f9b1471761b27d0fa8b15d969ed4
+Z a836ac39870a35d2f8436dd5e99c8845

diff --git a/manifest.uuid b/manifest.uuid
index dc8ddf389dace3fe76ce2b7d3bf3bb679de6554b..7ae37fa21819004120658d3b4d282ceb7fa2d9bc 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-05dfdad445b22f375b71abe0b1fa1bf7ca331be7
\ No newline at end of file
+2397404e152b908d838e6491294b263b05943b3f
\ No newline at end of file

diff --git a/tool/loadfts.c b/tool/loadfts.c
index 7da07b15ddcd6333f6e4dd20dc995aa4148d65c0..5b2ed5dc6a25b050faf8a93ca8662e351b7219db 100644 (file)
--- a/tool/loadfts.c
+++ b/tool/loadfts.c
@@ -163,6 +163,9 @@ int main(int argc, char **argv){
   char *zSql;
   VisitContext sCtx;
 
+  int nCmd = 0;
+  char **aCmd = 0;
+
   if( argc % 2 ) showHelp(argv[0]);
 
   for(i=1; i<(argc-1); i+=2){
@@ -172,7 +175,7 @@ int main(int argc, char **argv){
       iFts = atoi(zArg);
       if( iFts!=3 && iFts!=4 && iFts!= 5) showHelp(argv[0]);
     }
-    if( strcmp(zOpt, "-trans")==0 ){
+    else if( strcmp(zOpt, "-trans")==0 ){
       nRowPerTrans = atoi(zArg);
     }
     else if( strcmp(zOpt, "-idx")==0 ){
@@ -182,6 +185,14 @@ int main(int argc, char **argv){
     else if( strcmp(zOpt, "-dir")==0 ){
       zDir = zArg;
     }
+    else if( strcmp(zOpt, "-special")==0 ){
+      nCmd++;
+      aCmd = sqlite3_realloc(aCmd, sizeof(char*) * nCmd);
+      aCmd[nCmd-1] = zArg;
+    }
+    else{
+      showHelp(argv[0]);
+    }
   }
 
   /* Open the database file */
@@ -198,6 +209,13 @@ int main(int argc, char **argv){
   if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_exec(1)", db);
   sqlite3_free(zSql);
 
+  for(i=0; i<nCmd; i++){
+    zSql = sqlite3_mprintf("INSERT INTO fts(fts) VALUES(%Q)", aCmd[i]);
+    rc = sqlite3_exec(db, zSql, 0, 0, 0);
+    if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_exec(1)", db);
+    sqlite3_free(zSql);
+  }
+
   /* Compile the INSERT statement to write data to the FTS table. */
   memset(&sCtx, 0, sizeof(VisitContext));
   sCtx.db = db;
@@ -215,5 +233,6 @@ int main(int argc, char **argv){
   /* Clean up and exit. */
   sqlite3_finalize(sCtx.pInsert);
   sqlite3_close(db);
+  sqlite3_free(aCmd);
   return 0;
 }