From: dan <dan@noemail.net>
Date: Thu, 23 Apr 2015 18:14:21 +0000 (+0000)
Subject: Have OTA maintain a small LRU cache of UPDATE statements. This reduces the amount... 
X-Git-Tag: version-3.8.11~252^2~8
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1cdd523a506f1471eb7bdf9807df1d1632619f2f;p=thirdparty%2Fsqlite.git

Have OTA maintain a small LRU cache of UPDATE statements. This reduces the amount of time it spends compiling UPDATE if a single data_xxx table contains many different ota_control strings.

FossilOrigin-Name: baee3556ea10d96f1623cf4dce112fa1a1070820
---

diff --git a/ext/ota/ota13.test b/ext/ota/ota13.test
new file mode 100644
index 0000000000..42911a0605
--- /dev/null
+++ b/ext/ota/ota13.test
@@ -0,0 +1,65 @@
+# 2015 February 16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test an OTA update that features lots of different ota_control strings
+# for UPDATE statements. This tests OTA's internal UPDATE statement cache.
+#
+
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set ::testprefix ota13
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a PRIMARY KEY, b, c, d, e, f, g, h);
+  WITH ii(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM ii WHERE i<127)
+  INSERT INTO t1 SELECT i, 0, 0, 0, 0, 0, 0, 0 FROM ii;
+}
+
+forcedelete ota.db
+do_execsql_test 1.1 {
+  ATTACH 'ota.db' AS ota;
+  CREATE TABLE ota.data_t1(a, b, c, d, e, f, g, h, ota_control);
+}
+
+do_test 1.2 {
+  for {set i 0} {$i<128} {incr i} {
+    set control "."
+    for {set bit 6} {$bit>=0} {incr bit -1} {
+      if { $i & (1<<$bit) } {
+        append control "x"
+      } else {
+        append control "."
+      }
+    }
+    execsql { INSERT INTO data_t1 VALUES($i, 1, 1, 1, 1, 1, 1, 1, $control) }
+  }
+} {}
+
+do_test 1.3 {
+  sqlite3ota ota test.db ota.db
+  while 1 {
+    set rc [ota step]
+    if {$rc!="SQLITE_OK"} break
+  }
+  ota close
+} {SQLITE_DONE}
+
+do_execsql_test 1.4 {
+  SELECT count(*) FROM t1 WHERE
+  a == ( (b<<6) + (c<<5) + (d<<4) + (e<<3) + (f<<2) + (g<<1) + (h<<0) )
+} {128}
+
+
+finish_test
+
diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c
index c2fa993d44..9f74e561db 100644
--- a/ext/ota/sqlite3ota.c
+++ b/ext/ota/sqlite3ota.c
@@ -90,6 +90,9 @@
 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_OTA)
 #include "sqlite3ota.h"
 
+/* Maximum number of prepared UPDATE statements held by this module */
+#define SQLITE_OTA_UPDATE_CACHESIZE 16
+
 /*
 ** Swap two objects of type TYPE.
 */
@@ -165,6 +168,7 @@ typedef struct OtaObjIter OtaObjIter;
 typedef struct OtaState OtaState;
 typedef struct ota_vfs ota_vfs;
 typedef struct ota_file ota_file;
+typedef struct OtaUpdateStmt OtaUpdateStmt;
 
 #if !defined(SQLITE_AMALGAMATION)
 typedef unsigned int u32;
@@ -195,6 +199,12 @@ struct OtaState {
   i64 iOalSz;
 };
 
+struct OtaUpdateStmt {
+  char *zMask;                    /* Copy of update mask used with pUpdate */
+  sqlite3_stmt *pUpdate;          /* Last update statement (or NULL) */
+  OtaUpdateStmt *pNext;
+};
+
 /*
 ** An iterator of this type is used to iterate through all objects in
 ** the target database that require updating. For each such table, the
@@ -239,8 +249,7 @@ struct OtaObjIter {
   sqlite3_stmt *pTmpInsert;       /* Insert into ota_tmp_$zTbl */
 
   /* Last UPDATE used (for PK b-tree updates only), or NULL. */
-  char *zMask;                    /* Copy of update mask used with pUpdate */
-  sqlite3_stmt *pUpdate;          /* Last update statement (or NULL) */
+  OtaUpdateStmt *pOtaUpdate;
 };
 
 /*
@@ -436,8 +445,6 @@ static void otaObjIterFreeCols(OtaObjIter *pIter){
   pIter->abTblPk = 0;
   pIter->abNotNull = 0;
   pIter->nTblCol = 0;
-  sqlite3_free(pIter->zMask);
-  pIter->zMask = 0;
   pIter->eType = 0;               /* Invalid value */
 }
 
@@ -446,15 +453,24 @@ static void otaObjIterFreeCols(OtaObjIter *pIter){
 ** the current object (table/index pair).
 */
 static void otaObjIterClearStatements(OtaObjIter *pIter){
+  OtaUpdateStmt *pUp;
+
   sqlite3_finalize(pIter->pSelect);
   sqlite3_finalize(pIter->pInsert);
   sqlite3_finalize(pIter->pDelete);
-  sqlite3_finalize(pIter->pUpdate);
   sqlite3_finalize(pIter->pTmpInsert);
+  pUp = pIter->pOtaUpdate;
+  while( pUp ){
+    OtaUpdateStmt *pTmp = pUp->pNext;
+    sqlite3_finalize(pUp->pUpdate);
+    sqlite3_free(pUp);
+    pUp = pTmp;
+  }
+  
   pIter->pSelect = 0;
   pIter->pInsert = 0;
   pIter->pDelete = 0;
-  pIter->pUpdate = 0;
+  pIter->pOtaUpdate = 0;
   pIter->pTmpInsert = 0;
   pIter->nCol = 0;
 }
@@ -1709,9 +1725,6 @@ static int otaObjIterPrepareAll(
         otaObjIterPrepareTmpInsert(p, pIter, zCollist, zOtaRowid);
       }
 
-      /* Allocate space required for the zMask field. */
-      pIter->zMask = (char*)otaMalloc(p, pIter->nTblCol+1);
-
       sqlite3_free(zWhere);
       sqlite3_free(zOldlist);
       sqlite3_free(zNewlist);
@@ -1729,6 +1742,9 @@ static int otaObjIterPrepareAll(
 ** be used to update the imposter table for the main table b-tree of the
 ** table object that pIter currently points to, assuming that the 
 ** ota_control column of the data_xyz table contains zMask.
+** 
+** If the zMask string does not specify any columns to update, then this
+** is not an error. Output variable *ppStmt is set to NULL in this case.
 */
 static int otaGetUpdateStmt(
   sqlite3ota *p,                  /* OTA handle */
@@ -1736,15 +1752,50 @@ static int otaGetUpdateStmt(
   const char *zMask,              /* ota_control value ('x.x.') */
   sqlite3_stmt **ppStmt           /* OUT: UPDATE statement handle */
 ){
-  if( pIter->pUpdate && strcmp(zMask, pIter->zMask)==0 ){
-    *ppStmt = pIter->pUpdate;
+  OtaUpdateStmt **pp;
+  OtaUpdateStmt *pUp = 0;
+  int nUp = 0;
+
+  /* In case an error occurs */
+  *ppStmt = 0;
+
+  /* Search for an existing statement. If one is found, shift it to the front
+  ** of the LRU queue and return immediately. Otherwise, leave nUp pointing
+  ** to the number of statements currently in the cache and pUp to the
+  ** last object in the list.  */
+  for(pp=&pIter->pOtaUpdate; *pp; pp=&((*pp)->pNext)){
+    pUp = *pp;
+    if( strcmp(pUp->zMask, zMask)==0 ){
+      *pp = pUp->pNext;
+      pUp->pNext = pIter->pOtaUpdate;
+      pIter->pOtaUpdate = pUp;
+      *ppStmt = pUp->pUpdate; 
+      return SQLITE_OK;
+    }
+    nUp++;
+  }
+  assert( pUp==0 || pUp->pNext==0 );
+
+  if( nUp>=SQLITE_OTA_UPDATE_CACHESIZE ){
+    for(pp=&pIter->pOtaUpdate; *pp!=pUp; pp=&((*pp)->pNext));
+    *pp = 0;
+    sqlite3_finalize(pUp->pUpdate);
+    pUp->pUpdate = 0;
   }else{
+    pUp = (OtaUpdateStmt*)otaMalloc(p, sizeof(OtaUpdateStmt)+pIter->nTblCol+1);
+  }
+
+  if( pUp ){
     char *zWhere = otaObjIterGetWhere(p, pIter);
     char *zSet = otaObjIterGetSetlist(p, pIter, zMask);
     char *zUpdate = 0;
-    sqlite3_finalize(pIter->pUpdate);
-    pIter->pUpdate = 0;
-    if( p->rc==SQLITE_OK ){
+
+    pUp->zMask = (char*)&pUp[1];
+    memcpy(pUp->zMask, zMask, pIter->nTblCol);
+    pUp->pNext = pIter->pOtaUpdate;
+    pIter->pOtaUpdate = pUp;
+
+    if( zSet ){
       const char *zPrefix = "";
 
       if( pIter->eType!=OTA_PK_VTAB ) zPrefix = "ota_imp_";
@@ -1752,16 +1803,14 @@ static int otaGetUpdateStmt(
           zPrefix, pIter->zTbl, zSet, zWhere
       );
       p->rc = prepareFreeAndCollectError(
-          p->dbMain, &pIter->pUpdate, &p->zErrmsg, zUpdate
+          p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate
       );
-      *ppStmt = pIter->pUpdate;
-    }
-    if( p->rc==SQLITE_OK ){
-      memcpy(pIter->zMask, zMask, pIter->nTblCol);
+      *ppStmt = pUp->pUpdate;
     }
     sqlite3_free(zWhere);
     sqlite3_free(zSet);
   }
+
   return p->rc;
 }
 
@@ -1857,7 +1906,7 @@ static void otaFileSuffix3(const char *zBase, char *z){
 ** blob starting at byte offset 40.
 */
 static i64 otaShmChecksum(sqlite3ota *p){
-  i64 iRet;
+  i64 iRet = 0;
   if( p->rc==SQLITE_OK ){
     sqlite3_file *pDb = p->pTargetFd->pReal;
     u32 volatile *ptr;
diff --git a/manifest b/manifest
index 96272c5c22..f64a8df91a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\scomments\srelated\sto\sdatabase\slocking\sto\ssqlite3ota.h.
-D 2015-04-22T11:34:38.145
+C Have\sOTA\smaintain\sa\ssmall\sLRU\scache\sof\sUPDATE\sstatements.\sThis\sreduces\sthe\samount\sof\stime\sit\sspends\scompiling\sUPDATE\sif\sa\ssingle\sdata_xxx\stable\scontains\smany\sdifferent\sota_control\sstrings.
+D 2015-04-23T18:14:21.590
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 79b306896135a2305cfb7e6d88990fc4820fb917
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -128,6 +128,7 @@ F ext/ota/ota1.test 960418e4171a989426f8b1ad8ee31770e0f94fb8
 F ext/ota/ota10.test 85e0f6e7964db5007590c1b299e75211ed4240d4
 F ext/ota/ota11.test 2f606cd2b4af260a86b549e91b9f395450fc75cb
 F ext/ota/ota12.test 0dff44474de448fb4b0b28c20da63273a4149abb
+F ext/ota/ota13.test f7a3d73fa5d3fabf2755b569f125fce7390a874c
 F ext/ota/ota3.test 3fe3521fbdce32d0e4e116a60999c3cba47712c5
 F ext/ota/ota5.test ad0799daf8923ddebffe75ae8c5504ca90b7fadb
 F ext/ota/ota6.test 3bde7f69a894748b27206b6753462ec3b75b6bb6
@@ -138,7 +139,7 @@ F ext/ota/otaA.test ab67f7f53670b81c750dcc946c5b704f51c429a4
 F ext/ota/otacrash.test 8346192b2d46cbe7787d5d65904d81d3262a3cbf
 F ext/ota/otafault.test 8c43586c2b96ca16bbce00b5d7e7d67316126db8
 F ext/ota/otafault2.test fa202a98ca221faec318f3e5c5f39485b1256561
-F ext/ota/sqlite3ota.c b45d73607b78eba89ca224fb0d10e10d1b32b6b0
+F ext/ota/sqlite3ota.c 3b53197166b08bae980ea9b3b668f7eb0580fe8c
 F ext/ota/sqlite3ota.h c0d32e46aa52cce3b16899399639deaff1f27b2f
 F ext/ota/test_ota.c e34c801c665d64b4b9e00b71f1acf8c652404b2b
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
@@ -1269,7 +1270,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 352fa09efd8240567b8f9487944e8c3ade4e664a
-R 7514208e376f1404350994ff3e1120f2
+P 77242965e77446313a8f0a65a39fccb67fe4cabf
+R a8ed618fd0d0e5a18750e29ca6db9e19
 U dan
-Z aab2f59acbc2b3195f57173b44acb17c
+Z 00cb71982fa5d2b70e9c222c1f403ddb
diff --git a/manifest.uuid b/manifest.uuid
index 511df81b1d..a2366aee89 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-77242965e77446313a8f0a65a39fccb67fe4cabf
\ No newline at end of file
+baee3556ea10d96f1623cf4dce112fa1a1070820
\ No newline at end of file