An attempt at automatic incremental merging for FTS4.

FossilOrigin-Name: ed69434cd89084f4b57bd2cc4f5cc558904af565

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index ad0cb36da9de91a985569b8e43215ae835b84ab0..6a68d45e457c7d1844708a954e78cd038e0d865c 100644 (file)
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -1276,6 +1276,7 @@ static int fts3InitVtab(
   p->bHasDocsize = (isFts4 && bNoDocsize==0);
   p->bHasStat = isFts4;
   p->bDescIdx = bDescIdx;
+  p->bAutoincrmerge = 0xff;   /* 0xff means setting unknown */
   p->zContentTbl = zContent;
   p->zLanguageid = zLanguageid;
   zContent = 0;
@@ -3102,8 +3103,14 @@ static int fts3UpdateMethod(
 ** hash-table to the database.
 */
 static int fts3SyncMethod(sqlite3_vtab *pVtab){
-  int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
-  sqlite3Fts3SegmentsClose((Fts3Table *)pVtab);
+  Fts3Table *p = (Fts3Table*)pVtab;
+  int rc = sqlite3Fts3PendingTermsFlush(p);
+  if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>0 ){
+    int A = p->nLeafAdd * p->mxLevel;
+    A += A/2;
+    rc = sqlite3Fts3Incrmerge(p, A, 8);
+  }
+  sqlite3Fts3SegmentsClose(p);
   return rc;
 }
 
@@ -3118,6 +3125,7 @@ static int fts3BeginMethod(sqlite3_vtab *pVtab){
   assert( p->inTransaction!=1 );
   TESTONLY( p->inTransaction = 1 );
   TESTONLY( p->mxSavepoint = -1; );
+  p->nLeafAdd = 0;
   return SQLITE_OK;
 }
 

diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h
index f84d15d196e2d6e774e650a27054d041c5b07818..ddf3a6b13364e89ab295ef53a7334295a1c75cc0 100644 (file)
--- a/ext/fts3/fts3Int.h
+++ b/ext/fts3/fts3Int.h
@@ -196,6 +196,9 @@ struct Fts3Table {
   sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
   char *zContentTbl;              /* content=xxx option, or NULL */
   char *zLanguageid;              /* languageid=xxx option, or NULL */
+  u8 bAutoincrmerge;              /* True if automerge=1 */
+  int mxLevel;                    /* Maximum level seen on this transaction */
+  u32 nLeafAdd;                   /* Number of leaf blocks added this trans */
 
   /* Precompiled statements used by the implementation. Each of these 
   ** statements is run and reset within a single virtual table API call. 
@@ -479,6 +482,8 @@ struct Fts3MultiSegReader {
   int nDoclist;                   /* Size of aDoclist[] in bytes */
 };
 
+int sqlite3Fts3Incrmerge(Fts3Table*,int,int);
+
 /* fts3.c */
 int sqlite3Fts3PutVarint(char *, sqlite3_int64);
 int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);

diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index 30f5e666eb5fc1f436dec849430b653e46928b06..e9fe87c492a85288b324c5fccae061ea500825ac 100644 (file)
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -72,6 +72,7 @@ int test_fts3_node_chunk_threshold = (4*1024)*4;
 */
 #define FTS_STAT_DOCTOTAL      0
 #define FTS_STAT_INCRMERGEHINT 1
+#define FTS_STAT_AUTOINCRMERGE 2
 
 /*
 ** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic
@@ -330,7 +331,7 @@ static int fts3SqlStmt(
 
 /* Estimate the upper limit on the number of leaf nodes in a new segment
 ** created by merging the oldest :2 segments from absolute level :1. See 
-** function fts3Incrmerge() for details.  */
+** function sqlite3Fts3Incrmerge() for details.  */
 /* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) "
          "  FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?",
 
@@ -1868,6 +1869,14 @@ static int fts3WriteSegment(
   return rc;
 }
 
+/*
+** Update the Fts3Table.mxLevel field, if appropriate
+*/
+static void fts3UpdateMaxLevel(Fts3Table *p, sqlite3_int64 iLevel){
+  iLevel %= FTS3_SEGDIR_MAXLEVEL;
+  if( iLevel>p->mxLevel ) p->mxLevel = iLevel;
+}
+
 /* 
 ** Insert a record into the %_segdir table.
 */
@@ -1885,6 +1894,7 @@ static int fts3WriteSegdir(
   int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
   if( rc==SQLITE_OK ){
     sqlite3_bind_int64(pStmt, 1, iLevel);
+    fts3UpdateMaxLevel(p, iLevel);
     sqlite3_bind_int(pStmt, 2, iIdx);
     sqlite3_bind_int64(pStmt, 3, iStartBlock);
     sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
@@ -2369,6 +2379,7 @@ static int fts3SegmentMaxLevel(
   );
   if( SQLITE_ROW==sqlite3_step(pStmt) ){
     *pnMax = sqlite3_column_int64(pStmt, 0);
+    fts3UpdateMaxLevel(p, *pnMax);
   }
   return sqlite3_reset(pStmt);
 }
@@ -2999,11 +3010,29 @@ static int fts3SegmentMerge(
 int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
   int rc = SQLITE_OK;
   int i;
+        
+  p->nLeafAdd += (p->nPendingData + p->nNodeSize - 1)/p->nNodeSize;
   for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
     rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING);
     if( rc==SQLITE_DONE ) rc = SQLITE_OK;
   }
   sqlite3Fts3PendingTermsClear(p);
+
+  /* Determine the auto-incr-merge setting if unknown.  If enabled,
+  ** estimate the number of leaf blocks of content to be written
+  */
+  if( rc==SQLITE_OK && p->bHasStat
+   && p->bAutoincrmerge==0xff && p->nLeafAdd>0
+  ){
+    sqlite3_stmt *pStmt = 0;
+    rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0);
+    if( rc==SQLITE_OK ){
+      sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
+      rc = sqlite3_step(pStmt);
+      p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0));
+      rc = sqlite3_reset(pStmt);
+    }
+  }
   return rc;
 }
 
@@ -4472,7 +4501,7 @@ static int fts3IncrmergeHintLoad(
 ** nMin segments. Multiple merges might occur in an attempt to write the 
 ** quota of nMerge leaf blocks.
 */
-static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
+int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
   int rc;                         /* Return code */
   int nRem = nMerge;              /* Number of leaf pages yet to  be written */
   int bUseHint = 1;               /* True if hint has not yet been attempted */
@@ -4578,6 +4607,19 @@ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
   return rc;
 }
 
+/*
+** Convert the text beginning at *pz into an integer and return
+** its value.  Advance *pz to point to the first character past
+** the integer.
+*/
+static int fts3Getint(const char **pz){
+  const char *z = *pz;
+  int i = 0;
+  while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0';
+  *pz = z;
+  return i;
+}
+
 /*
 ** Process statements of the form:
 **
@@ -4597,30 +4639,49 @@ static int fts3DoIncrmerge(
   const char *z = zParam;
 
   /* Read the first integer value */
-  for(z=zParam; z[0]>='0' && z[0]<='9'; z++){
-    nMerge = nMerge * 10 + (z[0] - '0');
-  }
+  nMerge = fts3Getint(&z);
 
   /* If the first integer value is followed by a ',',  read the second
   ** integer value. */
   if( z[0]==',' && z[1]!='\0' ){
     z++;
-    nMin = 0;
-    while( z[0]>='0' && z[0]<='9' ){
-      nMin = nMin * 10 + (z[0] - '0');
-      z++;
-    }
+    nMin = fts3Getint(&z);
   }
 
   if( z[0]!='\0' || nMin<2 ){
     rc = SQLITE_ERROR;
   }else{
-    rc = fts3Incrmerge(p, nMerge, nMin);
+    rc = sqlite3Fts3Incrmerge(p, nMerge, nMin);
     sqlite3Fts3SegmentsClose(p);
   }
   return rc;
 }
 
+/*
+** Process statements of the form:
+**
+**    INSERT INTO table(table) VALUES('automerge=X');
+**
+** where X is an integer.  X==0 means to turn automerge off.  X!=0 means
+** turn it on.  The setting is persistent.
+*/
+static int fts3DoAutoincrmerge(
+  Fts3Table *p,                   /* FTS3 table handle */
+  const char *zParam              /* Nul-terminated string containing boolean */
+){
+  int rc;
+  sqlite3_stmt *pStmt = 0;
+  p->bAutoincrmerge = fts3Getint(&zParam)!=0;
+  rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0);
+  if( rc ) return rc;;
+  sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
+  sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge);
+  sqlite3_step(pStmt);
+  rc = sqlite3_reset(pStmt);
+  return rc;
+}
+
+
 /*
 ** Handle a 'special' INSERT of the form:
 **
@@ -4642,6 +4703,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
     rc = fts3DoRebuild(p);
   }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){
     rc = fts3DoIncrmerge(p, &zVal[6]);
+  }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
+    rc = fts3DoAutoincrmerge(p, &zVal[10]);
 #ifdef SQLITE_TEST
   }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
     p->nNodeSize = atoi(&zVal[9]);

diff --git a/manifest b/manifest
index a6dafc26fe8f4fd63749d7b5de37fd19a4a21265..b62cac476e4768add580e0678670525b4eaf52c4 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sspurious\sSQLITE_CONSTRAINT\serror\sthat\smay\sbe\sreturned\sby\san\sincr-merge\soperation.
-D 2012-03-23T18:26:11.608
+C An\sattempt\sat\sautomatic\sincremental\smerging\sfor\sFTS4.
+D 2012-03-24T02:20:43.242
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -63,9 +63,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c c47bb6fe7a6bde82cbdf4c504302221710699d64
+F ext/fts3/fts3.c ef3d7af4a635beacf5546ea6c8c03329e263fd30
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h 5fe1651db88206ca68f421cf62bb8501395677a3
+F ext/fts3/fts3Int.h 1e635c9ba3a384496a4524f7b94d7d0525930552
 F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e
 F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551
 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914
@@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4
 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce
 F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
-F ext/fts3/fts3_write.c e3b750530132b3237aed3e5f008845cc4d7865f9
+F ext/fts3/fts3_write.c 663f2fa51e7096d8f496f94db5f215c6d8d22820
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@@ -295,7 +295,7 @@ F test/backup_ioerr.test 40d208bc9224b666ee3ed423f49bc9062a36a9d0
 F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450
 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f
 F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f
-F test/bc_common.tcl df4c51ae0de0cb414a18027be2abb2bd7693ce7a
+F test/bc_common.tcl 5c8689cc6d2fb44b7c0968ae4f85eb26d50022fa
 F test/between.test 16b1776c6323faadb097a52d673e8e3d8be7d070
 F test/bigfile.test 82dfe93ee7eb9e2e05641afa2b39ffd947a92ff1
 F test/bigfile2.test 852f948cb492aadab45b58f4d2f3b0832a115cb0
@@ -997,7 +997,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P 1c72cecc6bf5be2a5c04ad6214a6bac22a29f860
-R c0998f3b8bc74543bb7322ba65e1257e
-U dan
-Z 3e48c8e93d6fd57a390b8a97532e70ea
+P ed7c17ea165f6348506bd23ebc58c427bb65d697
+R c65796e564760bf2b3e7eed574e1eb1f
+T *branch * fts4-auto-incr-merge
+T *sym-fts4-auto-incr-merge *
+T -sym-fts4-incr-merge *
+U drh
+Z 9b979efabd6b6c201cc4798b465233ce

diff --git a/manifest.uuid b/manifest.uuid
index 0886bc8716761c149bb244d2b10e10dd6c0a040e..4f3a06edafce12c5d5f99eae017ce356140fc9dc 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ed7c17ea165f6348506bd23ebc58c427bb65d697
\ No newline at end of file
+ed69434cd89084f4b57bd2cc4f5cc558904af565
\ No newline at end of file

diff --git a/test/bc_common.tcl b/test/bc_common.tcl
index 8bba0245a5e3129b398c487dd268588c11a8a4ad..eb9b6db9d3536ef9bbbe7d4eb7e03d2042967c3b 100644 (file)
--- a/test/bc_common.tcl
+++ b/test/bc_common.tcl
@@ -17,7 +17,7 @@ proc bc_find_binaries {zCaption} {
 
   if {[llength $binaries]==0} {
     puts "WARNING: No historical binaries to test against."
-    puts "WARNING: Omitting backwards-compatibility tests $zFile"
+    puts "WARNING: Omitting backwards-compatibility tests"
   }
 
   foreach bin $binaries {
@@ -70,7 +70,3 @@ proc do_all_bc_test {script} {
     uplevel [list do_bc_test $bin $script]
   }
 }
-
-
-
-