Background work for experiments trying to enhance ANALYZE so that it runs

off of samples of the entire index and does not need to read the entire
index.

FossilOrigin-Name: 29d1cc5c3619a88229f18c3c8131228f8a2d151ac3d9203f0c7fc538a996ecec

diff --git a/manifest b/manifest
index 9f3c8dd408a1894b95b989f7eee90f7629b99bf5..3cb4d4a72419193644d1f9743090e5efcb110130 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Cleaner\sseparation\sof\sthe\sSTAT4-specific\slogic\sin\sthe\simplementation\sof\nANALYZE.
-D 2020-03-09T18:26:11.821
+C Background\swork\sfor\sexperiments\strying\sto\senhance\sANALYZE\sso\sthat\sit\sruns\noff\sof\ssamples\sof\sthe\sentire\sindex\sand\sdoes\snot\sneed\sto\sread\sthe\sentire\nindex.
+D 2020-03-10T01:24:13.546
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -466,7 +466,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
 F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c f48a4423c8f198d7f1ae4940f74b606707d05384ac79fb219be8e3323af2a2de
-F src/analyze.c 831bb090988477a00d3b4c000746e1b0454dcc93b10b793e6ebe1c47f25d193a
+F src/analyze.c cc3401286f1ff97aba5f2d94a2e8f9108beeaa25fa67faee090c38e387db396f
 F src/attach.c fa5addce233a2bb2dfdefeee3b37000e154c47214d3269cab1bb331416e330db
 F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06
 F src/backup.c 5e617c087f1c2d6005c2ec694ce80d6e16bc68d906e1b1c556d7c7c2228b636b
@@ -1860,7 +1860,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cab1834cfc71f71bfed3c5170a0ba40a39385c3b2c50b7c6b6f09cc830dd1b1e
-R c62e646df7b5e5719c4a0602837bf1ef
+P 3df07e5a9a3781a4cf866fc6ee0e5c6f9cd7ca35ce0a6eb3aa7f5f3502e0ffae
+R e35bdeff6c928526c7218be9d4bb475b
+T *branch * approximate-analyze
+T *sym-approximate-analyze *
+T -sym-trunk *
 U drh
-Z 8147aa7ce1e35e16be35124cd1704d38
+Z 2efa3c6a2e23594b6e9a59be6c71be4b

diff --git a/manifest.uuid b/manifest.uuid
index e08451b6b24e5a2833cbebe49753a2d444c7cfef..a3157116f9c60acbfa6efa4fecd1bca60b3d069f 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3df07e5a9a3781a4cf866fc6ee0e5c6f9cd7ca35ce0a6eb3aa7f5f3502e0ffae
\ No newline at end of file
+29d1cc5c3619a88229f18c3c8131228f8a2d151ac3d9203f0c7fc538a996ecec
\ No newline at end of file

diff --git a/src/analyze.c b/src/analyze.c
index eee71fbbde481612e8e329eb492474aaf98dfb86..7db62919bd0d7fdfd5b2a5fc69f75a6648486e26 100644 (file)
--- a/src/analyze.c
+++ b/src/analyze.c
@@ -188,6 +188,11 @@ static void openStatTable(
   Vdbe *v = sqlite3GetVdbe(pParse);
   int aRoot[ArraySize(aTable)];
   u8 aCreateTbl[ArraySize(aTable)];
+#ifdef SQLITE_ENABLE_STAT4
+  const int nToOpen = OptimizationEnabled(db,SQLITE_Stat4) ? 2 : 1;
+#else
+  const int nToOpen = 1;
+#endif
 
   if( v==0 ) return;
   assert( sqlite3BtreeHoldsAllMutexes(db) );
@@ -200,8 +205,9 @@ static void openStatTable(
   for(i=0; i<ArraySize(aTable); i++){
     const char *zTab = aTable[i].zName;
     Table *pStat;
+    aCreateTbl[i] = 0;
     if( (pStat = sqlite3FindTable(db, zTab, pDb->zDbSName))==0 ){
-      if( aTable[i].zCols ){
+      if( i<nToOpen ){
         /* The sqlite_statN table does not exist. Create it. Note that a 
         ** side-effect of the CREATE TABLE statement is to leave the rootpage 
         ** of the new table in register pParse->regRoot. This is important 
@@ -217,7 +223,6 @@ static void openStatTable(
       ** associated with the table zWhere. If zWhere is NULL, delete the
       ** entire contents of the table. */
       aRoot[i] = pStat->tnum;
-      aCreateTbl[i] = 0;
       sqlite3TableLock(pParse, iDb, aRoot[i], 1, zTab);
       if( zWhere ){
         sqlite3NestedParse(pParse,
@@ -236,7 +241,7 @@ static void openStatTable(
   }
 
   /* Open the sqlite_stat[134] tables for writing. */
-  for(i=0; aTable[i].zCols; i++){
+  for(i=0; i<nToOpen; i++){
     assert( i<ArraySize(aTable) );
     sqlite3VdbeAddOp4Int(v, OP_OpenWrite, iStatCur+i, aRoot[i], iDb, 3);
     sqlite3VdbeChangeP5(v, aCreateTbl[i]);
@@ -357,10 +362,12 @@ static void sampleCopy(StatAccum *p, StatSample *pTo, StatSample *pFrom){
 static void statAccumDestructor(void *pOld){
   StatAccum *p = (StatAccum*)pOld;
 #ifdef SQLITE_ENABLE_STAT4
-  int i;
-  for(i=0; i<p->nCol; i++) sampleClear(p->db, p->aBest+i);
-  for(i=0; i<p->mxSample; i++) sampleClear(p->db, p->a+i);
-  sampleClear(p->db, &p->current);
+  if( p->mxSample ){
+    int i;
+    for(i=0; i<p->nCol; i++) sampleClear(p->db, p->aBest+i);
+    for(i=0; i<p->mxSample; i++) sampleClear(p->db, p->a+i);
+    sampleClear(p->db, &p->current);
+  }
 #endif
   sqlite3DbFree(p->db, p);
 }
@@ -400,7 +407,7 @@ static void statInit(
   int n;                          /* Bytes of space to allocate */
   sqlite3 *db;                    /* Database connection */
 #ifdef SQLITE_ENABLE_STAT4
-  int mxSample = SQLITE_STAT4_SAMPLES;
+  int mxSample = sqlite3_value_int(argv[2]) ? SQLITE_STAT4_SAMPLES : 0;
 #endif
 
   /* Decode the three function arguments */
@@ -437,7 +444,7 @@ static void statInit(
   p->current.anEq = &p->current.anDLt[nColUp];
 
 #ifdef SQLITE_ENABLE_STAT4
-  {
+  if( mxSample ){
     u8 *pSpace;                     /* Allocated space not yet assigned */
     int i;                          /* Used to iterate through p->aSample[] */
 
@@ -704,7 +711,7 @@ static void statPush(
   }else{
     /* Second and subsequent calls get processed here */
 #ifdef SQLITE_ENABLE_STAT4
-    samplePushPrevious(p, iChng);
+    if( p->mxSample ) samplePushPrevious(p, iChng);
 #endif
 
     /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply
@@ -715,26 +722,24 @@ static void statPush(
     for(i=iChng; i<p->nCol; i++){
       p->current.anDLt[i]++;
 #ifdef SQLITE_ENABLE_STAT4
-      p->current.anLt[i] += p->current.anEq[i];
+      if( p->mxSample ) p->current.anLt[i] += p->current.anEq[i];
 #endif
       p->current.anEq[i] = 1;
     }
   }
   p->nRow++;
 #ifdef SQLITE_ENABLE_STAT4
-  if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){
-    sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2]));
-  }else{
-    sampleSetRowid(p->db, &p->current, sqlite3_value_bytes(argv[2]),
-                                       sqlite3_value_blob(argv[2]));
-  }
-  p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345;
-#endif
-
-#ifdef SQLITE_ENABLE_STAT4
-  {
-    tRowcnt nLt = p->current.anLt[p->nCol-1];
+  if( p->mxSample ){
+    tRowcnt nLt;
+    if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){
+      sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2]));
+    }else{
+      sampleSetRowid(p->db, &p->current, sqlite3_value_bytes(argv[2]),
+                                         sqlite3_value_blob(argv[2]));
+    }
+    p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345;
 
+    nLt = p->current.anLt[p->nCol-1];
     /* Check if this is to be a periodic sample. If so, add it. */
     if( (nLt/p->nPSample)!=(nLt+1)/p->nPSample ){
       p->current.isPSample = 1;
@@ -804,6 +809,7 @@ static void statGet(
        || eCall==STAT_GET_ROWID || eCall==STAT_GET_NLT
        || eCall==STAT_GET_NDLT 
   );
+  assert( eCall==STAT_GET_STAT1 || p->mxSample );
   if( eCall==STAT_GET_STAT1 )
 #else
   assert( argc==1 );
@@ -1089,7 +1095,11 @@ static void analyzeOneTable(
     ** The third argument is only used for STAT4
     */
 #ifdef SQLITE_ENABLE_STAT4
-    sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3);
+    if( OptimizationEnabled(db, SQLITE_Stat4) ){
+      sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3);
+    }else{
+      sqlite3VdbeAddOp2(v, OP_Integer, 0, regStat4+3);
+    }
 #endif
     sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1);
     sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2);
@@ -1171,21 +1181,23 @@ static void analyzeOneTable(
     **   if !eof(csr) goto next_row;
     */
 #ifdef SQLITE_ENABLE_STAT4
-    assert( regRowid==(regStat4+2) );
-    if( HasRowid(pTab) ){
-      sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid);
-    }else{
-      Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable);
-      int j, k, regKey;
-      regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol);
-      for(j=0; j<pPk->nKeyCol; j++){
-        k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]);
-        assert( k>=0 && k<pIdx->nColumn );
-        sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j);
-        VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName));
+    if( OptimizationEnabled(db, SQLITE_Stat4) ){
+      assert( regRowid==(regStat4+2) );
+      if( HasRowid(pTab) ){
+        sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid);
+      }else{
+        Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable);
+        int j, k, regKey;
+        regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol);
+        for(j=0; j<pPk->nKeyCol; j++){
+          k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]);
+          assert( k>=0 && k<pIdx->nColumn );
+          sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j);
+          VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName));
+        }
+        sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid);
+        sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol);
       }
-      sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid);
-      sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol);
     }
 #endif
     assert( regChng==(regStat4+1) );
@@ -1206,7 +1218,7 @@ static void analyzeOneTable(
 
     /* Add the entries to the stat4 table. */
 #ifdef SQLITE_ENABLE_STAT4
-    {
+    if( OptimizationEnabled(db, SQLITE_Stat4) ){
       int regEq = regStat1;
       int regLt = regStat1+1;
       int regDLt = regStat1+2;