From: dan <dan@noemail.net>
Date: Wed, 16 Mar 2016 19:48:10 +0000 (+0000)
Subject: Have fts5 cache the decoded structure of fts5 indexes in memory. Use "PRAGMA data_ver... 
X-Git-Tag: version-3.12.0~28^2~4
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f55fb6615be38fc3e19e8fa3bef352374cd9e02d;p=thirdparty%2Fsqlite.git

Have fts5 cache the decoded structure of fts5 indexes in memory. Use "PRAGMA data_version" to detect stale caches.

FossilOrigin-Name: 33ef2210ef19e55c8d460bfe9d3dc146034c8acc
---

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 35f15abbaf..961206f2b3 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -480,6 +480,7 @@ int sqlite3Fts5IndexReads(Fts5Index *p);
 int sqlite3Fts5IndexReinit(Fts5Index *p);
 int sqlite3Fts5IndexOptimize(Fts5Index *p);
 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
+int sqlite3Fts5IndexReset(Fts5Index *p);
 
 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 
@@ -622,6 +623,7 @@ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
 int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
+int sqlite3Fts5StorageReset(Fts5Storage *p);
 
 /*
 ** End of interface to code in fts5_storage.c.
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 323e6cefdc..78003396d6 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -304,6 +304,10 @@ struct Fts5Index {
   sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=? */
   sqlite3_stmt *pIdxSelect;
   int nRead;                      /* Total number of blocks read */
+
+  sqlite3_stmt *pDataVersion;
+  i64 iStructVersion;             /* data_version when pStruct read */
+  Fts5Structure *pStruct;         /* Current db structure (or NULL) */
 };
 
 struct Fts5DoclistIter {
@@ -959,6 +963,48 @@ static void fts5StructureExtendLevel(
   }
 }
 
+static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){
+  Fts5Structure *pRet = 0;
+  Fts5Config *pConfig = p->pConfig;
+  int iCookie;                    /* Configuration cookie */
+  Fts5Data *pData;
+
+  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
+  if( p->rc==SQLITE_OK ){
+    /* TODO: Do we need this if the leaf-index is appended? Probably... */
+    memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
+    p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
+    if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
+      p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
+    }
+    fts5DataRelease(pData);
+    if( p->rc!=SQLITE_OK ){
+      fts5StructureRelease(pRet);
+      pRet = 0;
+    }
+  }
+
+  return pRet;
+}
+
+static i64 fts5IndexDataVersion(Fts5Index *p){
+  i64 iVersion = 0;
+
+  if( p->pDataVersion==0 ){
+    p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, 
+        sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb)
+    );
+    if( p->rc ) return 0;
+  }
+
+  if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){
+    iVersion = sqlite3_column_int64(p->pDataVersion, 0);
+  }
+  p->rc = sqlite3_reset(p->pDataVersion);
+
+  return iVersion;
+}
+
 /*
 ** Read, deserialize and return the structure record.
 **
@@ -971,28 +1017,51 @@ static void fts5StructureExtendLevel(
 ** is called, it is a no-op.
 */
 static Fts5Structure *fts5StructureRead(Fts5Index *p){
-  Fts5Config *pConfig = p->pConfig;
-  Fts5Structure *pRet = 0;        /* Object to return */
-  int iCookie;                    /* Configuration cookie */
-  Fts5Data *pData;
+  Fts5Structure *pRet;            /* Object to return */
 
-  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
-  if( p->rc ) return 0;
-  /* TODO: Do we need this if the leaf-index is appended? Probably... */
-  memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
-  p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
-  if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
-    p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
+  if( p->pStruct ){
+    pRet = p->pStruct;
+#ifdef SQLITE_DEBUG
+    {
+      Fts5Structure *pTest = fts5StructureReadUncached(p);
+      if( pTest ){
+        int i, j;
+        assert_nc( pRet->nSegment==pTest->nSegment );
+        assert_nc( pRet->nLevel==pTest->nLevel );
+        for(i=0; i<pTest->nLevel; i++){
+          assert_nc( pRet->aLevel[i].nMerge==pTest->aLevel[i].nMerge );
+          assert_nc( pRet->aLevel[i].nSeg==pTest->aLevel[i].nSeg );
+          for(j=0; j<pTest->aLevel[i].nSeg; j++){
+            Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j];
+            Fts5StructureSegment *p2 = &pRet->aLevel[i].aSeg[j];
+            assert_nc( p1->iSegid==p2->iSegid );
+            assert_nc( p1->pgnoFirst==p2->pgnoFirst );
+            assert_nc( p1->pgnoLast==p2->pgnoLast );
+          }
+        }
+        fts5StructureRelease(pTest);
+      }
+    }
+#endif
+  }else{
+    pRet = fts5StructureReadUncached(p);
   }
 
-  fts5DataRelease(pData);
-  if( p->rc!=SQLITE_OK ){
-    fts5StructureRelease(pRet);
-    pRet = 0;
+  if( pRet ){
+    fts5StructureRef(pRet);
+    p->pStruct = pRet;
+    p->iStructVersion = fts5IndexDataVersion(p);
   }
   return pRet;
 }
 
+static void fts5StructureInvalidate(Fts5Index *p){
+  if( p->pStruct ){
+    fts5StructureRelease(p->pStruct);
+    p->pStruct = 0;
+  }
+}
+
 /*
 ** Return the total number of segments in index structure pStruct. This
 ** function is only ever used as part of assert() conditions.
@@ -4328,6 +4397,7 @@ static void fts5FlushOneHash(Fts5Index *p){
   ** for the new level-0 segment.  */
   pStruct = fts5StructureRead(p);
   iSegid = fts5AllocateSegid(p, pStruct);
+  fts5StructureInvalidate(p);
 
   if( iSegid ){
     const int pgsz = p->pConfig->pgsz;
@@ -4547,6 +4617,7 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
   assert( p->rc==SQLITE_OK );
   fts5IndexFlush(p);
   pStruct = fts5StructureRead(p);
+  fts5StructureInvalidate(p);
 
   if( pStruct ){
     pNew = fts5IndexOptimizeStruct(p, pStruct);
@@ -4577,6 +4648,7 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
   Fts5Structure *pStruct = fts5StructureRead(p);
   if( pStruct ){
     int nMin = p->pConfig->nUsermerge;
+    fts5StructureInvalidate(p);
     if( nMerge<0 ){
       Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
       fts5StructureRelease(pStruct);
@@ -5004,6 +5076,7 @@ int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
 int sqlite3Fts5IndexRollback(Fts5Index *p){
   fts5CloseReader(p);
   fts5IndexDiscardData(p);
+  fts5StructureInvalidate(p);
   /* assert( p->rc==SQLITE_OK ); */
   return SQLITE_OK;
 }
@@ -5015,6 +5088,7 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){
 */
 int sqlite3Fts5IndexReinit(Fts5Index *p){
   Fts5Structure s;
+  fts5StructureInvalidate(p);
   memset(&s, 0, sizeof(Fts5Structure));
   fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
   fts5StructureWrite(p, &s);
@@ -5073,11 +5147,13 @@ int sqlite3Fts5IndexClose(Fts5Index *p){
   int rc = SQLITE_OK;
   if( p ){
     assert( p->pReader==0 );
+    fts5StructureInvalidate(p);
     sqlite3_finalize(p->pWriter);
     sqlite3_finalize(p->pDeleter);
     sqlite3_finalize(p->pIdxWriter);
     sqlite3_finalize(p->pIdxDeleter);
     sqlite3_finalize(p->pIdxSelect);
+    sqlite3_finalize(p->pDataVersion);
     sqlite3Fts5HashFree(p->pHash);
     sqlite3_free(p->zDataTbl);
     sqlite3_free(p);
@@ -6333,3 +6409,12 @@ int sqlite3Fts5IndexInit(sqlite3 *db){
   }
   return rc;
 }
+
+
+int sqlite3Fts5IndexReset(Fts5Index *p){
+  assert( p->pStruct==0 || p->iStructVersion!=0 );
+  if( fts5IndexDataVersion(p)!=p->iStructVersion ){
+    fts5StructureInvalidate(p);
+  }
+  return fts5IndexReturn(p);
+}
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index dcd131c74a..4feb3e1a2f 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -597,27 +597,38 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   return SQLITE_OK;
 }
 
+static int fts5NewTransaction(Fts5Table *pTab){
+  Fts5Cursor *pCsr;
+  for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
+    if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
+  }
+  return sqlite3Fts5StorageReset(pTab->pStorage);
+}
+
 /*
 ** Implementation of xOpen method.
 */
 static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
   Fts5Table *pTab = (Fts5Table*)pVTab;
   Fts5Config *pConfig = pTab->pConfig;
-  Fts5Cursor *pCsr;               /* New cursor object */
+  Fts5Cursor *pCsr = 0;           /* New cursor object */
   int nByte;                      /* Bytes of space to allocate */
-  int rc = SQLITE_OK;             /* Return code */
+  int rc;                         /* Return code */
 
-  nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
-  pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
-  if( pCsr ){
-    Fts5Global *pGlobal = pTab->pGlobal;
-    memset(pCsr, 0, nByte);
-    pCsr->aColumnSize = (int*)&pCsr[1];
-    pCsr->pNext = pGlobal->pCsr;
-    pGlobal->pCsr = pCsr;
-    pCsr->iCsrId = ++pGlobal->iNextId;
-  }else{
-    rc = SQLITE_NOMEM;
+  rc = fts5NewTransaction(pTab);
+  if( rc==SQLITE_OK ){
+    nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
+    pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
+    if( pCsr ){
+      Fts5Global *pGlobal = pTab->pGlobal;
+      memset(pCsr, 0, nByte);
+      pCsr->aColumnSize = (int*)&pCsr[1];
+      pCsr->pNext = pGlobal->pCsr;
+      pGlobal->pCsr = pCsr;
+      pCsr->iCsrId = ++pGlobal->iNextId;
+    }else{
+      rc = SQLITE_NOMEM;
+    }
   }
   *ppCsr = (sqlite3_vtab_cursor*)pCsr;
   return rc;
@@ -1578,8 +1589,8 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){
 ** Implementation of xBegin() method. 
 */
 static int fts5BeginMethod(sqlite3_vtab *pVtab){
-  UNUSED_PARAM(pVtab);  /* Call below is a no-op for NDEBUG builds */
   fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0);
+  fts5NewTransaction((Fts5Table*)pVtab);
   return SQLITE_OK;
 }
 
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index e4f5dd8f51..bd1c1f88e9 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -639,6 +639,10 @@ int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
   return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
 }
 
+int sqlite3Fts5StorageReset(Fts5Storage *p){
+  return sqlite3Fts5IndexReset(p->pIndex);
+}
+
 /*
 ** Allocate a new rowid. This is used for "external content" tables when
 ** a NULL value is inserted into the rowid column. The new rowid is allocated
diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test
index 7a8cb5c465..9653bcaac2 100644
--- a/ext/fts5/test/fts5corrupt3.test
+++ b/ext/fts5/test/fts5corrupt3.test
@@ -179,6 +179,10 @@ for {set i 1} {1} {incr i} {
   if {$end<=$i} break
   lset var end [expr $end - $i]
   set struct [binary format c* $var]
+
+  db close
+  sqlite3 db test.db
+
   db eval {
     BEGIN;
     UPDATE t1_data SET block = $struct WHERE id=10;
diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test
index 232b5021f1..655beb984b 100644
--- a/ext/fts5/test/fts5dlidx.test
+++ b/ext/fts5/test/fts5dlidx.test
@@ -178,7 +178,7 @@ do_execsql_test 3.2 {
   ORDER BY rowid DESC;
 } {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1}
 
-do_execsql_test 3.2 {
+do_execsql_test 3.3 {
   INSERT INTO abc(abc) VALUES('integrity-check');
   INSERT INTO abc(abc) VALUES('optimize');
   INSERT INTO abc(abc) VALUES('integrity-check');
@@ -187,7 +187,7 @@ do_execsql_test 3.2 {
 set v [lindex $vocab 0]
 set i 0
 foreach v $vocab {
-  do_execsql_test 3.3.[incr i] {
+  do_execsql_test 3.4.[incr i] {
     SELECT rowid FROM abc WHERE abc MATCH $v
   } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
 }
diff --git a/manifest b/manifest
index bf413d574c..7c682009be 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sSQLITE_OMIT_CODEC_FROM_TCL\scompile-time\soption.
-D 2016-03-16T01:03:10.251
+C Have\sfts5\scache\sthe\sdecoded\sstructure\sof\sfts5\sindexes\sin\smemory.\sUse\s"PRAGMA\sdata_version"\sto\sdetect\sstale\scaches.
+D 2016-03-16T19:48:10.873
 F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66
@@ -98,15 +98,15 @@ F ext/fts3/unicode/mkunicode.tcl 2debed3f582d77b3fdd0b8830880250021571fd8
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
-F ext/fts5/fts5Int.h 4060504b7979601d99e1385c2b5713036854979a
+F ext/fts5/fts5Int.h 4e507abebae0d7d3ac9b8daebf049d5153d00961
 F ext/fts5/fts5_aux.c daa57fb45216491814520bbb587e97bf81ced458
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c be309fb227003c931107bfcc12d5be4f2fd2bb8c
 F ext/fts5/fts5_hash.c f3a7217c86eb8f272871be5f6aa1b6798960a337
-F ext/fts5/fts5_index.c d4f0c12e4f04bbc3a06b6da052039f2ce3e45438
-F ext/fts5/fts5_main.c b8501e1a6a11591c53b18ce7aea7e5386cfb0421
-F ext/fts5/fts5_storage.c f8343db90d8c95a4d4b52f6676e354b4649ffd6e
+F ext/fts5/fts5_index.c 317040cb17cdad05e973376b39239b29c75f90b5
+F ext/fts5/fts5_main.c b4a0fc5bf17f2f1f056ee76cdd7d2af08b360f55
+F ext/fts5/fts5_storage.c a3361410422e69639ca2bcd5a56a0933dadf84d2
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
 F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc
 F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be
@@ -141,9 +141,9 @@ F ext/fts5/test/fts5conflict.test 26f4e46c4d31e16221794832a990dc4e30e18de5
 F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c
-F ext/fts5/test/fts5corrupt3.test b9558d5b0ca44a8b6247fbb5d4a47592a8976892
+F ext/fts5/test/fts5corrupt3.test f77f65e386231daf62902466b40ff998b2c8ce4f
 F ext/fts5/test/fts5detail.test ef5c690535a797413acaf5ad9b8ab5d49972df69
-F ext/fts5/test/fts5dlidx.test 13871a14641017ae42f6f1055a8067bafd44cb3d
+F ext/fts5/test/fts5dlidx.test 007e9390c94638760797dbec2990c97c3fa08dfe
 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
 F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
 F ext/fts5/test/fts5eb.test c516ae0c934be6fd29ec95ea8b5f11f461311535
@@ -1456,7 +1456,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e0b116edd64a55c971c368685aa343cb6beed0f1
-R 1b0c52bbc749be5b9a732d29d3d099e5
-U drh
-Z 5dddb7cbfc6ef15baab9797e73067d60
+P 45f7f0c80bd91a0c7ff859c27fd9e82e551bd83e
+R 7875ababc9def91b1b2c4626dd7ef0ad
+T *branch * fts5-data-version
+T *sym-fts5-data-version *
+T -sym-trunk *
+U dan
+Z 015dc4987d82758ea8c0d324359dc366
diff --git a/manifest.uuid b/manifest.uuid
index 3aa4fcd861..3d6ceeb2ed 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-45f7f0c80bd91a0c7ff859c27fd9e82e551bd83e
\ No newline at end of file
+33ef2210ef19e55c8d460bfe9d3dc146034c8acc
\ No newline at end of file