From: dan <dan@noemail.net>
Date: Wed, 7 Jan 2015 19:33:11 +0000 (+0000)
Subject: Add the fts5 'optimize' command.
X-Git-Tag: version-3.8.11~114^2~109
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=d6b1c880d2f9486c0a92f1f2f709de8203378fc6;p=thirdparty%2Fsqlite.git

Add the fts5 'optimize' command.

FossilOrigin-Name: e749be563d8e738af113bd301770e2f22763ab77
---

diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c
index c52b80f0ea..1dd026bb4b 100644
--- a/ext/fts5/fts5.c
+++ b/ext/fts5/fts5.c
@@ -1080,6 +1080,8 @@ static int fts5SpecialInsert(
     }else{
       rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
     }
+  }else if( 0==sqlite3_stricmp("optimize", z) ){
+    rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
   }else if( 0==sqlite3_stricmp("integrity-check", z) ){
     rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
   }else{
diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 893d743cda..6148ec7f6c 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -341,6 +341,7 @@ int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
 int sqlite3Fts5IndexReads(Fts5Index *p);
 
 int sqlite3Fts5IndexReinit(Fts5Index *p);
+int sqlite3Fts5IndexOptimize(Fts5Index *p);
 
 /*
 ** End of interface to code in fts5_index.c.
@@ -425,6 +426,7 @@ int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
 
 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
+int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 
 /*
 ** End of interface to code in fts5_storage.c.
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 8552a35792..4d22c41467 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -41,6 +41,7 @@
 **
 */
 
+#define FTS5_OPT_WORK_UNIT  1000  /* Number of leaf pages per optimize step */
 #define FTS5_WORK_UNIT      64    /* Number of leaf pages in unit of work */
 #define FTS5_CRISIS_MERGE   16    /* Maximum number of segments to merge */
 
@@ -3164,6 +3165,12 @@ static void fts5IndexCrisisMerge(
   *ppStruct = pStruct;
 }
 
+static int fts5IndexReturn(Fts5Index *p){
+  int rc = p->rc;
+  p->rc = SQLITE_OK;
+  return rc;
+}
+
 typedef struct Fts5FlushCtx Fts5FlushCtx;
 struct Fts5FlushCtx {
   Fts5Index *pIdx;
@@ -3277,6 +3284,66 @@ static void fts5IndexFlush(Fts5Index *p){
   p->nPendingData = 0;
 }
 
+
+int sqlite3Fts5IndexOptimize(Fts5Index *p){
+  Fts5Config *pConfig = p->pConfig;
+  int i;
+
+  fts5IndexFlush(p);
+  for(i=0; i<=pConfig->nPrefix; i++){
+    Fts5Structure *pStruct = fts5StructureRead(p, i);
+    Fts5Structure *pNew = 0;
+    int nSeg = 0;
+    if( pStruct ){
+      nSeg = fts5StructureCountSegments(pStruct);
+      if( nSeg>1 ){
+        int nByte = sizeof(Fts5Structure);
+        nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
+        pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
+      }
+    }
+    if( pNew ){
+      Fts5StructureLevel *pLvl;
+      int nByte = nSeg * sizeof(Fts5StructureSegment);
+      pNew->nLevel = pStruct->nLevel+1;
+      pNew->nWriteCounter = pStruct->nWriteCounter;
+      pLvl = &pNew->aLevel[pStruct->nLevel];
+      pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
+      if( pLvl->aSeg ){
+        int iLvl, iSeg;
+        int iSegOut = 0;
+        for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
+          for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
+            pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg];
+            iSegOut++;
+          }
+        }
+        pLvl->nSeg = nSeg;
+      }else{
+        sqlite3_free(pNew);
+        pNew = 0;
+      }
+    }
+
+    if( pNew ){
+      int iLvl = pNew->nLevel-1;
+      while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){
+        int nRem = FTS5_OPT_WORK_UNIT;
+        fts5IndexMergeLevel(p, i, &pNew, iLvl, &nRem);
+      }
+
+      fts5StructureWrite(p, i, pNew);
+      fts5StructureRelease(pNew);
+    }
+
+    fts5StructureRelease(pStruct);
+  }
+
+  return fts5IndexReturn(p); 
+}
+
+
+
 /*
 ** Return a simple checksum value based on the arguments.
 */
@@ -3788,12 +3855,6 @@ static void fts5SetupPrefixIter(
   sqlite3_free(aBuf);
 }
 
-static int fts5IndexReturn(Fts5Index *p){
-  int rc = p->rc;
-  p->rc = SQLITE_OK;
-  return rc;
-}
-
 /*
 ** Run internal checks to ensure that the FTS index (a) is internally 
 ** consistent and (b) contains entries for which the XOR of the checksums
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index 8431f6dd4c..b82db3e4a4 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -585,6 +585,10 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
   return rc;
 }
 
+int sqlite3Fts5StorageOptimize(Fts5Storage *p){
+  return sqlite3Fts5IndexOptimize(p->pIndex);
+}
+
 /*
 ** Allocate a new rowid. This is used for "external content" tables when
 ** a NULL value is inserted into the rowid column. The new rowid is allocated
diff --git a/ext/fts5/test/fts5optimize.test b/ext/fts5/test/fts5optimize.test
new file mode 100644
index 0000000000..068cf4c225
--- /dev/null
+++ b/ext/fts5/test/fts5optimize.test
@@ -0,0 +1,60 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5optimize
+
+proc rnddoc {nWord} {
+  set vocab {a b c d e f g h i j k l m n o p q r s t u v w x y z}
+  set nVocab [llength $vocab]
+  set ret [list]
+  for {set i 0} {$i < $nWord} {incr i} {
+    lappend ret [lindex $vocab [expr {int(rand() * $nVocab)}]]
+  }
+  return $ret
+}
+
+
+foreach {tn nStep} {
+  1 2
+  2 10
+  3 50
+  4 500
+} {
+if {$tn!=4} continue
+  reset_db
+  db func rnddoc rnddoc
+  do_execsql_test 1.$tn.1 {
+    CREATE VIRTUAL TABLE t1 USING fts5(x, y);
+  }
+  do_test 1.$tn.2 {
+    for {set i 0} {$i < $nStep} {incr i} {
+      execsql { INSERT INTO t1 VALUES( rnddoc(5), rnddoc(5) ) }
+    }
+  } {}
+
+  do_execsql_test 1.$tn.3 {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+
+  do_execsql_test 1.$tn.4 {
+    INSERT INTO t1(t1) VALUES('optimize');
+  }
+
+  do_execsql_test 1.$tn.5 {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+}
+
+finish_test
+
diff --git a/manifest b/manifest
index a7855d306f..adf58144e8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s'rebuild'\sand\s'delete-all'\scommands.
-D 2015-01-07T17:11:11.301
+C Add\sthe\sfts5\s'optimize'\scommand.
+D 2015-01-07T19:33:11.551
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -104,16 +104,16 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl 4199cb887040ee3c3cd59a5171ddb0566904586e
 F ext/fts5/extract_api_docs.tcl 6320db4a1d0722a4e2069e661381ad75e9889786
-F ext/fts5/fts5.c 66ca4324ea89dc727f01ea77eb48e5ba311be032
+F ext/fts5/fts5.c c90004f4a91ce4f4dfad2fc980ade0d9314ebb10
 F ext/fts5/fts5.h 0f8563e21ffa69cb87be4c2e24652fc41b441850
-F ext/fts5/fts5Int.h 00a8770e34b56f3db7eb29e5b110d2f7623ca959
+F ext/fts5/fts5Int.h 0142ba4c3c70e1976578604c0e738670f7689726
 F ext/fts5/fts5_aux.c 549aef152b0fd46020f5595d861b1fd60b3f9b4f
 F ext/fts5/fts5_buffer.c 32dd3c950392346ca69a0f1803501766c5c954f9
 F ext/fts5/fts5_config.c 33534ca25198cc62c54ff7d285d455c57ad19399
 F ext/fts5/fts5_expr.c 0320ae948e82cf7dca800463de7f5b6a808ba7c3
 F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279
-F ext/fts5/fts5_index.c 4e612b2c91a57ec770869b6cc89caeec0f658107
-F ext/fts5/fts5_storage.c 844b9667030370e9bb1daf3f9e862716cddb1a22
+F ext/fts5/fts5_index.c ea36c1e42aaf8038b6139be95575eb7fe01f34e4
+F ext/fts5/fts5_storage.c 8bc9e5b6654e1545e9513def277ef3f025921664
 F ext/fts5/fts5_tcl.c 1293fac2bb26903fd3d5cdee59c5885ba7e620d5
 F ext/fts5/fts5_tokenize.c 4c30cf32c63e59bec5b38533e0a65987df262851
 F ext/fts5/fts5_unicode2.c 9c7dd640d1f014bf5c3ee029759adfbb4d7e95a9
@@ -135,6 +135,7 @@ F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
 F ext/fts5/test/fts5content.test 4234e0b11e003fe1e80472aa637f70464396fdd0
 F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
 F ext/fts5/test/fts5fault1.test f3f4c6ed15cc7a4dc8d517c0d1969d8e5a35a65c
+F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54
 F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e
 F ext/fts5/test/fts5rebuild.test 2a5e98205393487b4a732c8290999af7c0b907b4
 F ext/fts5/test/fts5tokenizer.test f951bb9be29232bd057b0ac4d535b879d9cd9a89
@@ -1272,7 +1273,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 65f0262fb82dbfd9f80233ac7c3108e2f2716c0a
-R 3ced0a5a3507e3fd1f8ff1a5ee996e45
+P 0cb2fed525778d96237b5b0943047665e1f636d1
+R b413984e0011c860316df7bca0fa936a
 U dan
-Z 06bf9328f4305b86890a4adcb16c3d2f
+Z ad35ce36f519fcc615b0ece9f543df9d
diff --git a/manifest.uuid b/manifest.uuid
index 5026d1fa16..0a08358e38 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0cb2fed525778d96237b5b0943047665e1f636d1
\ No newline at end of file
+e749be563d8e738af113bd301770e2f22763ab77
\ No newline at end of file