The analyze_as_needed pragma now responds to table size growth and will

automatically rerun the analysis after each 10x size increase.

FossilOrigin-Name: bfbdd07409688fac4ccddbab3639745f6152e23d

diff --git a/manifest b/manifest
index 9d27e603a6db82b140ad98735090803d0ad04392..42a11a725c8e02a1350a6ab0f5a659ecd9a0a3bd 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s"PRAGMA\sanalyze_as_needed"\scommand.
-D 2017-02-17T16:26:34.780
+C The\sanalyze_as_needed\spragma\snow\sresponds\sto\stable\ssize\sgrowth\sand\swill\nautomatically\srerun\sthe\sanalysis\safter\seach\s10x\ssize\sincrease.
+D 2017-02-17T19:24:06.618
 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2
@@ -331,14 +331,14 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
 F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121
-F src/analyze.c 39206f31ca136a0a4d4f1d6bfd00bc81c6b3e5ec
+F src/analyze.c acef9c48eac30438e82f39eb8df6682c6e33942e
 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43
 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
-F src/btree.c 3ae66974881e74df9909093818b4c3428f8d7982
-F src/btree.h e6d352808956ec163a17f832193a3e198b3fb0ac
+F src/btree.c e7d724b02365d88920171caf0c919aa96d9b19e3
+F src/btree.h bf64dfeeddeebdb775a5eba0098bbc00d073290d
 F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0
 F src/build.c 2e05d0360568f40dc583461f2211f020ff282ee4
 F src/callback.c 2e76147783386374bf01b227f752c81ec872d730
@@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1
 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae
-F src/sqliteInt.h 85706e0963964336270590a1e508dec78540bc5c
+F src/sqliteInt.h 60c38b84a796cc9d0a9a794cca254b09898b9cc5
 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -461,7 +461,7 @@ F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58
 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c
 F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6
 F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16
-F src/vdbe.c 16f378640570c24442fd7191b136b5d6380f5c7b
+F src/vdbe.c ac8538b96af5c63d6f18484ffd981d4f44db0073
 F src/vdbe.h 59998ffd71d7caa8886bc78dafaf8caeccd4c13c
 F src/vdbeInt.h 4e4b15b2e1330e1636e4e01974eab2b0b985092f
 F src/vdbeapi.c 3e4a8893feeb78620f4aac4ac5b85d92255b97e1
@@ -521,7 +521,7 @@ F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0
 F test/auth.test c6ede04bee65637ff354b43fc1235aa560c0863e
 F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1
 F test/auth3.test 0d48b901cf111c14b4b1b5205c7d28f1a278190f
-F test/autoanalyze1.test b31c18e73cf4e97b896288d8c817fce743f23e76
+F test/autoanalyze1.test 1ba80d5e1412d46503566b70741a5eea060da929
 F test/autoinc.test 6ae8fb69c9f656962464ae4e6667045d0dfc3b46
 F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea
 F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df
@@ -1557,7 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7
-R 23ff1c8d68460fd739a12857d6287bf8
+P e93db2373127d31d33ec46ef918fa9386bb664a6
+R 484e61dbf95be1098adb3acfa069e723
 U drh
-Z 4884503796ef3cb7f91c7100ffa75ea8
+Z 81466028927592a195c5763d3460fcde

diff --git a/manifest.uuid b/manifest.uuid
index f532ee84ddc304bd5000984735cbcede3c5d457b..8be7eeb5b943d64607ac825ff7516c27f863152d 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e93db2373127d31d33ec46ef918fa9386bb664a6
\ No newline at end of file
+bfbdd07409688fac4ccddbab3639745f6152e23d
\ No newline at end of file

diff --git a/src/analyze.c b/src/analyze.c
index 34de7fc7449e400ccf15fc9ae014cdeea762311b..082713fd23a4ddad6a8c3e85cb8b92e355c2d174 100644 (file)
--- a/src/analyze.c
+++ b/src/analyze.c
@@ -958,10 +958,12 @@ static void analyzeOneTable(
   Index *pOnlyIdx, /* If not NULL, only analyze this one index */
   int iStatCur,    /* Index of VdbeCursor that writes the sqlite_stat1 table */
   int iMem,        /* Available memory locations begin here */
-  int iTab         /* Next available cursor */
+  int iTab,        /* Next available cursor */
+  LogEst szOld     /* Run the analysis if table row count is larger than this */
 ){
   sqlite3 *db = pParse->db;    /* Database handle */
   Index *pIdx;                 /* An index to being analyzed */
+  int addrSizeCk = 0;          /* Address of the IfSmaller */
   int iIdxCur;                 /* Cursor open on index being analyzed */
   int iTabCur;                 /* Table cursor */
   Vdbe *v;                     /* The virtual machine being built up */
@@ -1014,6 +1016,9 @@ static void analyzeOneTable(
   iIdxCur = iTab++;
   pParse->nTab = MAX(pParse->nTab, iTab);
   sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead);
+  if( szOld>0 ){
+    addrSizeCk = sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, 0, szOld);
+   }
   sqlite3VdbeLoadString(v, regTabname, pTab->zName);
 
   for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
@@ -1271,6 +1276,7 @@ static void analyzeOneTable(
     sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
     sqlite3VdbeJumpHere(v, jZeroRows);
   }
+  sqlite3VdbeJumpHere(v, addrSizeCk);
 }
 
 
@@ -1286,16 +1292,33 @@ static void loadAnalysis(Parse *pParse, int iDb){
 }
 
 /*
-** Return true if table pTab is in need of being reanalyzed.
+** Return true if table pTab might need to being reanalyzed.  Return
+** false if we know that pTab should not be reanalyzed.
+**
+** If returning true, also set *pThreshold to a size threshold that
+** will determine at run-time whether or not the reanalysis occurs.
+** The reanalysis will only occur if the size of the table is greater
+** than the threshold. Not that the threshold is a logarithmic LogEst
+** value.
 */
-static int analyzeNeeded(Table *pTab){
+static int analyzeNeeded(Table *pTab, LogEst *pThreshold){
   Index *pIdx;
   if( (pTab->tabFlags & TF_StatsUsed)==0 ) return 0;
-  if( (pTab->tabFlags & TF_SizeChng)!=0 ) return 1;
+
+  /* If TF_StatsUsed is true, then we might need to reanalyze.  But
+  ** only reanalyze if the table size has grown by a factor of 10 or more */
+  *pThreshold = pTab->nRowLogEst + 33;  assert( sqlite3LogEst(10)==33 );
+
+  /* Except, if any of the indexes of the table do not have valid
+  ** sqlite_stat1 entries, then set the size threshold to zero to
+  ** ensure the analysis will always occur. */
   for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
-    if( !pIdx->hasStat1 ) return 1;
+    if( !pIdx->hasStat1 ){
+      *pThreshold = 0;
+      break;
+    }
   }
-  return 0;
+  return 1;
 }
 
 /*
@@ -1329,7 +1352,8 @@ static int analyzeDatabase(Parse *pParse, int iDbReq, int onlyIfNeeded){
     cnt = 0;
     for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){
       Table *pTab = (Table*)sqliteHashData(k);
-      if( !onlyIfNeeded || analyzeNeeded(pTab) ){
+      LogEst szThreshold = 0;
+      if( !onlyIfNeeded || analyzeNeeded(pTab, &szThreshold) ){
         if( cnt==0 ){
           sqlite3BeginWriteOperation(pParse, 0, iDb);
           iStatCur = pParse->nTab;
@@ -1338,7 +1362,7 @@ static int analyzeDatabase(Parse *pParse, int iDbReq, int onlyIfNeeded){
           iMem = pParse->nMem+1;
           iTab = pParse->nTab;
         }
-        analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab);
+        analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab, szThreshold);
         cnt++;
         allCnt++;
       }
@@ -1368,7 +1392,8 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){
   }else{
     openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl");
   }
-  analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1,pParse->nTab);
+  analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1,
+                  pParse->nTab, 0);
   loadAnalysis(pParse, iDb);
 }
 

diff --git a/src/btree.c b/src/btree.c
index e78ffef1bec7e277d2c55782ba39e51084c9d64c..8aed08cb36b1b2af45adf276f0870f8a422a7b50 100644 (file)
--- a/src/btree.c
+++ b/src/btree.c
@@ -5319,6 +5319,25 @@ int sqlite3BtreeEof(BtCursor *pCur){
   return (CURSOR_VALID!=pCur->eState);
 }
 
+/*
+** Return an estimate for the number of rows in the table that pCur is
+** pointing to.  Return a negative number if no estimate is currently 
+** available.
+*/
+i64 sqlite3BtreeRowCountEst(BtCursor *pCur){
+  i64 n;
+  u8 i;
+
+  assert( cursorOwnsBtShared(pCur) );
+  assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
+  if( pCur->eState!=CURSOR_VALID ) return -1;
+  if( pCur->apPage[pCur->iPage-1]->leaf==0 ) return -1;
+  for(n=1, i=0; i<pCur->iPage; i++){
+    n *= pCur->apPage[i]->nCell;
+  }
+  return n;
+}
+
 /*
 ** Advance the cursor to the next entry in the database.  If
 ** successful then set *pRes=0.  If the cursor

diff --git a/src/btree.h b/src/btree.h
index ae57468e3fd5fec513515d97f842029ec9f9aa7c..0e017f5300b20a635bfdf7c4e3753d0c42b9f9eb 100644 (file)
--- a/src/btree.h
+++ b/src/btree.h
@@ -296,6 +296,7 @@ u32 sqlite3BtreePayloadSize(BtCursor*);
 
 char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*);
 struct Pager *sqlite3BtreePager(Btree*);
+i64 sqlite3BtreeRowCountEst(BtCursor*);
 
 #ifndef SQLITE_OMIT_INCRBLOB
 int sqlite3BtreePayloadChecked(BtCursor*, u32 offset, u32 amt, void*);

diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index f590c85f930f587e80c89624af439de6162d0fe5..d9cad11bcfade79716d55e6ddcbed95f01ddcbc9 100644 (file)
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1887,8 +1887,7 @@ struct Table {
 #define TF_WithoutRowid    0x0020    /* No rowid.  PRIMARY KEY is the key */
 #define TF_NoVisibleRowid  0x0040    /* No user-visible "rowid" column */
 #define TF_OOOHidden       0x0080    /* Out-of-Order hidden columns */
-#define TF_SizeChng        0x0100    /* nRowLogEst might be inaccurate */
-#define TF_StatsUsed       0x0200    /* Query planner decisions affected by
+#define TF_StatsUsed       0x0100    /* Query planner decisions affected by
                                      ** Index.aiRowLogEst[] values */
 
 /*

diff --git a/src/vdbe.c b/src/vdbe.c
index 7f286b2c6d3a068511d6f28a863fab7e48547c20..b595c4fb01b96abbc3c0e6d6997ee5939d95f66d 100644 (file)
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -4847,6 +4847,33 @@ case OP_Last: {        /* jump */
   break;
 }
 
+/* Opcode: IfSmaller P1 P2 P3 * *
+**
+** Estimate the number of rows in the table P1.  Jump to P2 if that
+** estimate is less than approximately 2**(0.1*P3).
+*/
+case OP_IfSmaller: {        /* jump */
+  VdbeCursor *pC;
+  BtCursor *pCrsr;
+  int res;
+  i64 sz;
+
+  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
+  pC = p->apCsr[pOp->p1];
+  assert( pC!=0 );
+  pCrsr = pC->uc.pCursor;
+  assert( pCrsr );
+  rc = sqlite3BtreeFirst(pCrsr, &res);
+  if( rc ) goto abort_due_to_error;
+  if( res==0 ){
+    sz = sqlite3BtreeRowCountEst(pCrsr);
+    if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)<pOp->p3 ) res = 1;
+  }
+  VdbeBranchTaken(res!=0,2);
+  if( res ) goto jump_to_p2;
+  break;
+}
+
 
 /* Opcode: SorterSort P1 P2 * * *
 **

diff --git a/test/autoanalyze1.test b/test/autoanalyze1.test
index c36e0fcbc704ee50e76c6118cce726f27fdc84bb..77e767a89e650b6c4c44d58d2f913469ce2cd66f 100644 (file)
--- a/test/autoanalyze1.test
+++ b/test/autoanalyze1.test
@@ -16,8 +16,7 @@
 # If any of the following constants changes:
 #
 #    define TF_HasStat1   0x0010
-#    define TF_SizeChng   0x0100
-#    define TF_StatsUsed  0x0200
+#    define TF_StatsUsed  0x0100
 #
 # then some of the magic numbers in test results below might need to be
 # adjusted.
@@ -52,7 +51,7 @@ do_execsql_test autoanalyze1-100 {
 
 # No use of stat1 recorded so far
 do_execsql_test autoanalyze1-110 {
-  SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
 } {0}
 
 # Access using a unique index does not set the TF_StatsUsed flag.
@@ -61,21 +60,21 @@ do_execsql_test autoanalyze1-200 {
   SELECT * FROM t1 WHERE a=55;
 } {55 55 55 55}
 do_execsql_test autoanalyze1-201 {
-  SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
 } {0}
 
 do_execsql_test autoanalyze1-210 {
   SELECT * FROM t1 WHERE a IN (55,199,299);
 } {55 55 55 55}
 do_execsql_test autoanalyze1-211 {
-  SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
 } {0}
 
 do_execsql_test autoanalyze1-220 {
   SELECT * FROM t1 WHERE (b,c)=(45,45);
 } {45 45 45 45}
 do_execsql_test autoanalyze1-221 {
-  SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
 } {0}
 
 # Any use of the non-unique t1d index triggers the use of stats.
@@ -85,7 +84,7 @@ do_execsql_test autoanalyze1-300 {
   SELECT * FROM t1 WHERE d=45;
 } {45 45 45 45}
 do_execsql_test autoanalyze1-301 {
-  SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
 } {1}
 
 sqlite3 db test.db
@@ -93,7 +92,7 @@ do_execsql_test autoanalyze1-310 {
   SELECT * FROM t1 WHERE d=45 AND a=45;
 } {45 45 45 45}
 do_execsql_test autoanalyze1-311 {
-  SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
 } {0}  ;# The ROWID lookup short-circuits the d=45 constraint
 
 sqlite3 db test.db
@@ -101,7 +100,7 @@ do_execsql_test autoanalyze1-320 {
   SELECT * FROM t1 WHERE d=45 AND a IN (45,46);
 } {45 45 45 45}
 do_execsql_test autoanalyze1-321 {
-  SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
 } {1}
 
 # Any use of prefix of a unique index triggers the use of stats
@@ -111,14 +110,14 @@ do_execsql_test autoanalyze1-400 {
   SELECT * FROM t1 WHERE b=45;
 } {45 45 45 45}
 do_execsql_test autoanalyze1-401 {
-  SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
 } {1}
 
 # The TF_StatsUsed flag is reset when the database is reopened
 #
 sqlite3 db test.db
 do_execsql_test autoanalyze1-500 {
-  SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
 } {0}
 
 finish_test