Experimental code to load stat4 data using a single pass.

FossilOrigin-Name: e0bcd30581b5b55abb0ba0f2855dbfa27f4e036b45423062402fe344db7bb070

diff --git a/manifest b/manifest
index 0ad34da1fd3e44963695b29365331e31c79ead7d..0e7738f89a9db6e246c343893333d8f4149398bb 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\sa\swriter\sinitializes\sa\snew\s*-shm\sfile\shash\stable,\szero\sonly\sthe\shash-table\sslots,\snot\sthe\sarray\sof\spage\snumbers.
-D 2024-12-11T20:42:29.249
+C Experimental\scode\sto\sload\sstat4\sdata\susing\sa\ssingle\spass.
+D 2024-12-12T19:15:46.069
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -707,7 +707,7 @@ F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2
 F src/alter.c aa93e37e4a36a0525bbb2a2aeda20d2018f0aa995542c7dc658e031375e3f532
-F src/analyze.c 7fda93b7952edc84282f57f56dd63d99afe428502b0289f571e698e0ce7b978a
+F src/analyze.c 4d49e7be20b5c58c197f53280122a46facc9821c80b26837dd12a219818ecc7b
 F src/attach.c 08235ab62ed5ccc93c22bf36e640d19effcd632319615851bccf724ec9341333
 F src/auth.c 4c1ea890e0069ad73bead5d17a5b12c34cfa4f1a24175c8147ea439b64be271c
 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523
@@ -779,7 +779,7 @@ F src/shell.c.in 40de636c1d90fb8a9ca7f49dc8f50d930f1b60736e73aca5eb37c4c7d0e47f9
 F src/sqlite.h.in b7ff496637807ae88b2557039fc940518db328bf5d5621e2f7c048dfba32a52b
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
-F src/sqliteInt.h 3fb893edbabad31aa7a32cecf3a5282cbd14391b5e1d143e23594d04fe568b36
+F src/sqliteInt.h c8f95e16884274e80dc53d491a1cf8283b6021ac2f64334c07a37eaf21b55300
 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728
 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -849,7 +849,7 @@ F src/vdbe.c fa31aa6525e34b51763702d246a69c3877436f0bbf6be5f6351cdcb770b9b7ab
 F src/vdbe.h b74bfd9cb1fa895e545a5286ee1cac6d75f706d325f89be0e3bf3c5107eb8a78
 F src/vdbeInt.h 92b7eabbaadbe8127741cd96e7e39c4834c2bb0b75987d5f6251975f47233690
 F src/vdbeapi.c 80235ac380e9467fec1cb0883354d841f2a771976e766995f7e0c77f845406df
-F src/vdbeaux.c 6f2c4e262d124f3126a3fb865c09314ae825ffefc9eca140381584968502c989
+F src/vdbeaux.c b8361c6daaa8b189774e2d2433073511311a81bb411429238fcc24d68ddc4a01
 F src/vdbeblob.c 255be187436da38b01f276c02e6a08103489bbe2a7c6c21537b7aecbe0e1f797
 F src/vdbemem.c 831a244831eaa45335f9ae276b50a7a82ee10d8c46c2c72492d4eb8c98d94d89
 F src/vdbesort.c d0a3c7056c081703c8b6d91ad60f17da5e062a5c64bf568ed0fa1b5f4cae311f
@@ -2250,8 +2250,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P df4183ace93b788b798b258274bf6b651906c9f1cf2af4983e447cdf52904523
-R 94edb7bb8062fb14ef6badf564e5e7de
+P ac5401c62b3ff03e43e01d066f4664491ce9b640ec64150a58b38839b4834afb
+R 83b28506326e6134726d20ac1da94326
 U dan
-Z 6fb05f3841c97a9cc6100a75f02a6296
+Z b392d5c3e26ddc5e80d40edd1473ba14
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index cf395e006644e4b9144b02af9ed1d22ddf845194..ae35375c63501c87c0ce832f61fc539cda129417 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ac5401c62b3ff03e43e01d066f4664491ce9b640ec64150a58b38839b4834afb
+e0bcd30581b5b55abb0ba0f2855dbfa27f4e036b45423062402fe344db7bb070

diff --git a/src/analyze.c b/src/analyze.c
index 03a0aa891a3207166edc54fadf621fa22b3e145c..1325940d76e72b32d3cafd40f64be722655f0000 100644 (file)
--- a/src/analyze.c
+++ b/src/analyze.c
@@ -1677,6 +1677,7 @@ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){
   if( db->pnBytesFreed==0 ){
     pIdx->nSample = 0;
     pIdx->aSample = 0;
+    pIdx->nSampleAlloc = 0;
   }
 #else
   UNUSED_PARAMETER(db);
@@ -1761,6 +1762,70 @@ static Index *findIndexOrPrimaryKey(
   return pIdx;
 }
 
+/*
+** Grow the pIdx->aSample[] array. Return SQLITE_OK if successful, or
+** SQLITE_NOMEM otherwise.
+*/
+static int growSampleArray(sqlite3 *db, Index *pIdx){
+  int nIdxCol = pIdx->nSampleCol;
+  int nNew = 0;
+  IndexSample *aNew = 0;
+  int nByte = 0;
+  tRowcnt *pSpace; /* Available allocated memory space */
+  u8 *pPtr;        /* Available memory as a u8 for easier manipulation */
+  int i;
+
+  /* In production set the initial allocation to SQLITE_STAT4_SAMPLES. This
+  ** means that reallocation will almost never be required. But for debug 
+  ** builds, set the initial allocation size to 6 entries so that the 
+  ** reallocation code gets tested. todo: use real tests for this. */
+  assert( pIdx->nSample==pIdx->nSampleAlloc );
+#ifdef SQLITE_DEBUG
+  nNew = 6;
+#else
+  nNew = SQLITE_STAT4_SAMPLES;
+#endif
+  if( pIdx->nSample ){
+    nNew = pIdx->nSample*2;
+  }
+
+  nByte = ROUND8(sizeof(IndexSample) * nNew);
+  nByte += sizeof(tRowcnt) * nIdxCol * 3 * nNew;
+  nByte += nIdxCol * sizeof(tRowcnt);   /* Space for Index.aAvgEq[] */
+
+  aNew = (IndexSample*)sqlite3DbMallocZero(db, nByte);
+  if( aNew==0 ) return SQLITE_NOMEM_BKPT;
+
+  pPtr = (u8*)aNew;
+  pPtr += ROUND8(nNew*sizeof(pIdx->aSample[0]));
+  pSpace = (tRowcnt*)pPtr;
+
+  pIdx->aAvgEq = pSpace; pSpace += nIdxCol;
+  assert( EIGHT_BYTE_ALIGNMENT( pSpace ) );
+
+  if( pIdx->nSample ){
+    /* Copy the contents of the anEq[], anLt[], anDLt[] arrays for all
+    ** extant samples to the new location.  */
+    int nByte = nIdxCol * 3 * sizeof(tRowcnt) * pIdx->nSample;
+    memcpy(pSpace, pIdx->aSample[0].anEq, nByte);
+  }
+  for(i=0; i<nNew; i++){
+    aNew[i].anEq = pSpace; pSpace += nIdxCol;
+    aNew[i].anLt = pSpace; pSpace += nIdxCol;
+    aNew[i].anDLt = pSpace; pSpace += nIdxCol;
+    if( i<pIdx->nSample ){
+      aNew[i].p = pIdx->aSample[i].p;
+      aNew[i].n = pIdx->aSample[i].n;
+    }
+  }
+  assert( ((u8*)pSpace)-nByte==(u8*)aNew );
+
+  sqlite3DbFree(db, pIdx->aSample);
+  pIdx->aSample = aNew;
+  pIdx->nSampleAlloc = nNew;
+  return SQLITE_OK;
+}
+
 /*
 ** Load the content from either the sqlite_stat4
 ** into the relevant Index.aSample[] arrays.
@@ -1786,6 +1851,7 @@ static int loadStatTbl(
   IndexSample *pSample;         /* A slot in pIdx->aSample[] */
 
   assert( db->lookaside.bDisable );
+#if 0
   zSql = sqlite3MPrintf(db, zSql1, zDb);
   if( !zSql ){
     return SQLITE_NOMEM_BKPT;
@@ -1852,6 +1918,9 @@ static int loadStatTbl(
   }
   rc = sqlite3_finalize(pStmt);
   if( rc ) return rc;
+#endif
+
+  sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q1);
 
   zSql = sqlite3MPrintf(db, zSql2, zDb);
   if( !zSql ){
@@ -1861,8 +1930,6 @@ static int loadStatTbl(
   sqlite3DbFree(db, zSql);
   if( rc ) return rc;
 
-  sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q1);
-
   while( sqlite3_step(pStmt)==SQLITE_ROW ){
     char *zIndex;                 /* Index name */
     Index *pIdx;                  /* Pointer to the index object */
@@ -1872,18 +1939,24 @@ static int loadStatTbl(
     if( zIndex==0 ) continue;
     pIdx = findIndexOrPrimaryKey(db, zIndex, zDb);
     if( pIdx==0 ) continue;
-    if( pIdx->nSample>=pIdx->mxSample ){
-      /* Too many slots used because the same index appears in
-      ** sqlite_stat4 using multiple names */
-      continue;
+
+    if( pIdx->nSample==pIdx->nSampleAlloc ){
+      pIdx->pTable->tabFlags |= TF_HasStat4;
+      assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 );
+      if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
+        pIdx->nSampleCol = pIdx->nKeyCol;
+      }else{
+        pIdx->nSampleCol = pIdx->nColumn;
+      }
+      if( growSampleArray(db, pIdx) ) break;
     }
-    /* This next condition is true if data has already been loaded from 
-    ** the sqlite_stat4 table. */
-    nCol = pIdx->nSampleCol;
+
     if( pIdx!=pPrevIdx ){
       initAvgEq(pPrevIdx);
       pPrevIdx = pIdx;
     }
+
+    nCol = pIdx->nSampleCol;
     pSample = &pIdx->aSample[pIdx->nSample];
     decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0);
     decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0);

diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 4676ab141391702c930b4e82fd35cf85d94d1d31..37363c63a2b30743d6bfe25a845ce2d6aaeb8f76 100644 (file)
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1878,7 +1878,7 @@ struct sqlite3 {
 #define SCHEMA_TIME_FINISH 19
 
 #define SCHEMA_TIME_N 20
-#define SCHEMA_TIME_TIMEOUT (1 * 1000 * 1000)
+#define SCHEMA_TIME_TIMEOUT (500 * 1000)
 
 
 
@@ -2882,7 +2882,7 @@ struct Index {
                            ** expression, or a reference to a VIRTUAL column */
 #ifdef SQLITE_ENABLE_STAT4
   int nSample;             /* Number of elements in aSample[] */
-  int mxSample;            /* Number of slots allocated to aSample[] */
+  int nSampleAlloc;        /* Number of slots allocated to aSample[] */
   int nSampleCol;          /* Size of IndexSample.anEq[] and so on */
   tRowcnt *aAvgEq;         /* Average nEq values for keys not in aSample */
   IndexSample *aSample;    /* Samples of the left-most key */

diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index c691d6ab9c95567c12fad80015c3dea8439235ff..4cde4aa981adfa6062fd10e707719472343a1a3e 100644 (file)
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -5499,7 +5499,7 @@ void sqlite3CommitTimeLog(u64 *aCommit){
       }
       zStr = sqlite3_mprintf("%z%s%s%d%s", zStr, (zStr?", ":""),zHash,iVal,zU);
     }
-    sqlite3_log(SQLITE_WARNING, "slow commit (v=17): (%s)", zStr);
+    sqlite3_log(SQLITE_WARNING, "slow commit (v=18): (%s)", zStr);
     sqlite3_free(zStr);
   }
 }
@@ -5527,7 +5527,7 @@ void sqlite3PrepareTimeLog(const char *zSql, int nSql, u64 *aPrepare){
     }
     if( nByte<0 ){ nByte = sqlite3Strlen30(zSql); }
     sqlite3_log(SQLITE_WARNING, 
-        "slow prepare (v=17): (%s) [%.*s]", zStr, nByte, zSql
+        "slow prepare (v=18): (%s) [%.*s]", zStr, nByte, zSql
     );
     sqlite3_free(zStr);
   }
@@ -5545,7 +5545,7 @@ void sqlite3SchemaTimeLog(u64 *aSchema, const char *zFile){
       }
       zStr = sqlite3_mprintf("%z%s%d", zStr, (zStr?", ":""), val);
     }
-    sqlite3_log(SQLITE_WARNING, "slow schema (%s) (v=17): (%s)", zFile, zStr);
+    sqlite3_log(SQLITE_WARNING, "slow schema (%s) (v=18): (%s)", zFile, zStr);
     sqlite3_free(zStr);
   }
 }