Consolidate memory allocations made while loading stat4 data in a way that may be...

FossilOrigin-Name: af65a902d10e50d827ff31f9ded7d05bc7ab0956a767366e1321b28fcb60b0bd

diff --git a/manifest b/manifest
index fd35b17d9aef1664abc3be0104aa79361636169b..062e958cd08118ffbdf0169de9c94803d0b0c1b9 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C If\sSQLITE_ENABLE_WAL2NOCKSUM\sis\sdefined,\sthen\sSQLite\scalculates\sthe\sframe\schecksums\sused\sin\swal2\smode\sbased\son\sthe\sprevious\schecksum\sand\sthe\sframe\sheader\sonly,\snot\sthe\sframe\sbody\sonly.\sThis\srisks\scorruption\sfollowing\sa\sOS\scrash\sor\spower\sfailure,\sbut\salso\sspeeds\sup\swrites\sin\swal2\smode.
-D 2025-01-29T15:11:07.802
+C Consolidate\smemory\sallocations\smade\swhile\sloading\sstat4\sdata\sin\sa\sway\sthat\smay\sbe\smore\sefficient\son\ssystems\sunder\sload.
+D 2025-01-29T17:26:44.487
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d
@@ -719,7 +719,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
 F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc
 F sqlite3.pc.in 0977c03a4da7c4204bd60e784a0efb8d51a190448aba78a4e973fe7192bdaf03
 F src/alter.c aa93e37e4a36a0525bbb2a2aeda20d2018f0aa995542c7dc658e031375e3f532
-F src/analyze.c 9a8b67239d899ac12289db5db3f5bfe7f7a0ad1277f80f87ead1d048085876eb
+F src/analyze.c c5573e4c5442bdcebd38692319fb4636c4b8c17de28be10e9ee9c2b57f7f8794
 F src/attach.c f35bb8cc1fcdde8f6815a7ef09ae413bcac71821d530796800ba24b3c7da1e80
 F src/auth.c 54ab9c6c5803b47c0d45b76ce27eff22a03b4b1f767c5945a3a4eb13aa4c78dc
 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523
@@ -729,7 +729,7 @@ F src/btree.c 9145be06761b9947d621e575d3837f9cf265242a013c86b23f28ef4be66b3f3e
 F src/btree.h df26089b055c4cffe243e5bc98edc729c4ad880bfeb8f229fd16248e4cec10ff
 F src/btreeInt.h bb28bf05e6206befd5f5fd2ed3825fc6382979fa4a83bf50f1875a0d3404111b
 F src/build.c 5a14f6593b9b0002f54d4215febceeaa1354c12f8eca53f8135d3fdf8a93f4d7
-F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490
+F src/callback.c 43c8ca52b1ecbdec43522f121126fd4e3ee10bc9ca01cdd3ae207cfa419780b6
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c dbd2d51fa2874ad54c673cb71b1f5bbd0df1b8b47b34f31d105f42500680a89a
 F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a
@@ -791,7 +791,7 @@ F src/shell.c.in beb370609906092a6810fcd9ea76737be2c91694445061c2eb05c4c0a3753de
 F src/sqlite.h.in 278a2679a4e50ebe965c5f8e3ec78d52541aa5eeb2632b6eb0a0a395f63b1cea
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
-F src/sqliteInt.h d711bbe8b255024336fb84b090fbc4a6e506f2768925da5426222594537b90c8
+F src/sqliteInt.h 1fc598d06c6c7d09fbb68791f8e876ba5a72a42421f55c3f14e066f155bee1d2
 F src/sqliteLimit.h 1bbdbf72bd0411d003267ffebc59a262f061df5653027a75627d03f48ca30523
 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -2242,9 +2242,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P fab341c8295545739cdce8b71e38ead68cb80a6f836f7ec0540b387f17b6cbe2
-Q +4d3706727005397c3c9006b9ad9a2645b09533b02735ea68974c4f2df4c2e853
-R 474ce037ad8c9757d8c8e66f75356bf2
+P dc74bd8915a9e1a915fb4ff3229a7b5e8f89486fe1df812a7738f6627d379648
+R e6eba16885e8253a0c8420a72196b2b2
 U dan
-Z 363c756be7599a910bd16bdd6751a132
+Z f498dbc7b0d6c6dc50ccede72f5ffcd3
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index 513952cd5de2b67b64b1e806dd1b7f95ac71b24b..9008a1b088e98a416357aca93166194f6d624704 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-dc74bd8915a9e1a915fb4ff3229a7b5e8f89486fe1df812a7738f6627d379648
+af65a902d10e50d827ff31f9ded7d05bc7ab0956a767366e1321b28fcb60b0bd

diff --git a/src/analyze.c b/src/analyze.c
index 9213c202b77488c86e53ad93164337fbce6344e2..b1f72b4d35e97b11e116c6b1f650985d7f47d94b 100644 (file)
--- a/src/analyze.c
+++ b/src/analyze.c
@@ -256,6 +256,13 @@ static void openStatTable(
 # define SQLITE_STAT4_SAMPLES 24
 #endif
 
+/*
+** Assumed number of of samples when loading sqlite_stat4 data. It doesn't
+** matter if there are more or fewer samples than this, but is more efficient
+** if this estimate turns out to be true.
+*/
+#define SQLITE_STAT4_EST_SAMPLES SQLITE_STAT4_SAMPLES
+
 /*
 ** Three SQL functions - stat_init(), stat_push(), and stat_get() -
 ** share an instance of the following structure to hold their state
@@ -1550,6 +1557,9 @@ static void decodeIntArray(
 #endif
     if( *z==' ' ) z++;
   }
+  if( aOut ){
+    for(/* no-op */; i<nOut; i++){ aOut[i] = 0; }
+  }
 #ifndef SQLITE_ENABLE_STAT4
   assert( pIndex!=0 ); {
 #else
@@ -1672,11 +1682,14 @@ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){
       IndexSample *p = &pIdx->aSample[j];
       sqlite3DbFree(db, p->p);
     }
-    sqlite3DbFree(db, pIdx->aSample);
+    if( pIdx->nSampleAlloc!=SQLITE_STAT4_EST_SAMPLES ){
+      sqlite3DbFree(db, pIdx->aSample);
+    }
   }
   if( db->pnBytesFreed==0 ){
     pIdx->nSample = 0;
     pIdx->aSample = 0;
+    pIdx->nSampleAlloc = 0;
   }
 #else
   UNUSED_PARAMETER(db);
@@ -1762,8 +1775,110 @@ static Index *findIndexOrPrimaryKey(
 }
 
 /*
-** Load the content from either the sqlite_stat4
-** into the relevant Index.aSample[] arrays.
+** Grow the pIdx->aSample[] array. Return SQLITE_OK if successful, or
+** SQLITE_NOMEM otherwise.
+*/
+static int growSampleArray(sqlite3 *db, Index *pIdx, int *piOff){
+  int nIdxCol = pIdx->nSampleCol;
+  int nNew = 0;
+  IndexSample *aNew = 0;
+  int nByte = 0;
+  tRowcnt *pSpace; /* Available allocated memory space */
+  u8 *pPtr;        /* Available memory as a u8 for easier manipulation */
+  int i;
+  u64 t;
+
+  assert( pIdx->nSample==pIdx->nSampleAlloc );
+  nNew = SQLITE_STAT4_EST_SAMPLES;
+  if( pIdx->nSample ){
+    nNew = pIdx->nSample*2;
+  }
+
+  /* Set nByte to the required amount of space */
+  nByte = ROUND8(sizeof(IndexSample) * nNew);
+  nByte += sizeof(tRowcnt) * nIdxCol * 3 * nNew;
+  nByte += nIdxCol * sizeof(tRowcnt);   /* Space for Index.aAvgEq[] */
+
+  if( nNew==SQLITE_STAT4_EST_SAMPLES ){
+    aNew = (IndexSample*)&((u8*)pIdx->pSchema->pStat4Space)[*piOff];
+    *piOff += nByte;
+    assert( *piOff<=sqlite3_msize(pIdx->pSchema->pStat4Space) );
+  }else{
+    aNew = (IndexSample*)sqlite3DbMallocRaw(db, nByte);
+    if( aNew==0 ) return SQLITE_NOMEM_BKPT;
+  }
+
+  pPtr = (u8*)aNew;
+  pPtr += ROUND8(nNew*sizeof(pIdx->aSample[0]));
+  pSpace = (tRowcnt*)pPtr;
+
+  pIdx->aAvgEq = pSpace; pSpace += nIdxCol;
+  assert( EIGHT_BYTE_ALIGNMENT( pSpace ) );
+
+  if( pIdx->nSample ){
+    /* Copy the contents of the anEq[], anLt[], anDLt[] arrays for all
+    ** extant samples to the new location.  */
+    int nByte = nIdxCol * 3 * sizeof(tRowcnt) * pIdx->nSample;
+    memcpy(pSpace, pIdx->aSample[0].anEq, nByte);
+  }
+  for(i=0; i<nNew; i++){
+    aNew[i].anEq = pSpace; pSpace += nIdxCol;
+    aNew[i].anLt = pSpace; pSpace += nIdxCol;
+    aNew[i].anDLt = pSpace; pSpace += nIdxCol;
+    if( i<pIdx->nSample ){
+      aNew[i].p = pIdx->aSample[i].p;
+      aNew[i].n = pIdx->aSample[i].n;
+    }
+  }
+  assert( ((u8*)pSpace)-nByte==(u8*)aNew );
+
+  if( pIdx->nSample!=SQLITE_STAT4_EST_SAMPLES ){
+    sqlite3DbFree(db, pIdx->aSample);
+  }
+  pIdx->aSample = aNew;
+  pIdx->nSampleAlloc = nNew;
+  return SQLITE_OK;
+}
+
+/*
+** Allocate the space that will likely be required for the Index.aSample[] 
+** arrays populated by loading data from the sqlite_stat4 table. Return
+** SQLITE_OK if successful, or SQLITE_NOMEM otherwise.
+*/
+static int stat4AllocSpace(sqlite3 *db, const char *zDb){
+  int iDb = sqlite3FindDbName(db, zDb);
+  Schema *pSchema = db->aDb[iDb].pSchema;
+  int nByte = 0;
+  HashElem *k;
+
+  assert( iDb>=0 );
+  assert( pSchema->pStat4Space==0 );
+  for(k=sqliteHashFirst(&pSchema->idxHash); k; k=sqliteHashNext(k)){
+    Index *pIdx = sqliteHashData(k);
+    int nIdxCol;
+    if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
+      nIdxCol = pIdx->nKeyCol;
+    }else{
+      nIdxCol = pIdx->nColumn;
+    }
+    nByte += ROUND8(sizeof(IndexSample) * SQLITE_STAT4_EST_SAMPLES);
+    nByte += sizeof(tRowcnt) * nIdxCol * 3 * SQLITE_STAT4_EST_SAMPLES;
+    nByte += nIdxCol * sizeof(tRowcnt);   /* Space for Index.aAvgEq[] */
+  }
+
+  if( nByte>0 ){
+    pSchema->pStat4Space = sqlite3_malloc(nByte);
+    if( pSchema->pStat4Space==0 ){
+      return SQLITE_NOMEM_BKPT;
+    }
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Load the content from the sqlite_stat4 into the relevant Index.aSample[]
+** arrays.
 **
 ** Arguments zSql1 and zSql2 must point to SQL statements that return
 ** data equivalent to the following:
@@ -1784,69 +1899,14 @@ static int loadStatTbl(
   char *zSql;                   /* Text of the SQL statement */
   Index *pPrevIdx = 0;          /* Previous index in the loop */
   IndexSample *pSample;         /* A slot in pIdx->aSample[] */
+  int iBlockOff = 0;            /* Offset into Schema.pStat4Space */
 
   assert( db->lookaside.bDisable );
-  zSql = sqlite3MPrintf(db, zSql1, zDb);
-  if( !zSql ){
-    return SQLITE_NOMEM_BKPT;
-  }
-  rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
-  sqlite3DbFree(db, zSql);
-  if( rc ) return rc;
-
-  while( sqlite3_step(pStmt)==SQLITE_ROW ){
-    int nIdxCol = 1;              /* Number of columns in stat4 records */
 
-    char *zIndex;    /* Index name */
-    Index *pIdx;     /* Pointer to the index object */
-    int nSample;     /* Number of samples */
-    i64 nByte;       /* Bytes of space required */
-    i64 i;           /* Bytes of space required */
-    tRowcnt *pSpace; /* Available allocated memory space */
-    u8 *pPtr;        /* Available memory as a u8 for easier manipulation */
-
-    zIndex = (char *)sqlite3_column_text(pStmt, 0);
-    if( zIndex==0 ) continue;
-    nSample = sqlite3_column_int(pStmt, 1);
-    pIdx = findIndexOrPrimaryKey(db, zIndex, zDb);
-    assert( pIdx==0 || pIdx->nSample==0 );
-    if( pIdx==0 ) continue;
-    if( pIdx->aSample!=0 ){
-      /* The same index appears in sqlite_stat4 under multiple names */
-      continue;
-    }
-    assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 );
-    if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
-      nIdxCol = pIdx->nKeyCol;
-    }else{
-      nIdxCol = pIdx->nColumn;
-    }
-    pIdx->nSampleCol = nIdxCol;
-    pIdx->mxSample = nSample;
-    nByte = ROUND8(sizeof(IndexSample) * nSample);
-    nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample;
-    nByte += nIdxCol * sizeof(tRowcnt);     /* Space for Index.aAvgEq[] */
-
-    pIdx->aSample = sqlite3DbMallocZero(db, nByte);
-    if( pIdx->aSample==0 ){
-      sqlite3_finalize(pStmt);
-      return SQLITE_NOMEM_BKPT;
-    }
-    pPtr = (u8*)pIdx->aSample;
-    pPtr += ROUND8(nSample*sizeof(pIdx->aSample[0]));
-    pSpace = (tRowcnt*)pPtr;
-    assert( EIGHT_BYTE_ALIGNMENT( pSpace ) );
-    pIdx->aAvgEq = pSpace; pSpace += nIdxCol;
-    pIdx->pTable->tabFlags |= TF_HasStat4;
-    for(i=0; i<nSample; i++){
-      pIdx->aSample[i].anEq = pSpace; pSpace += nIdxCol;
-      pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol;
-      pIdx->aSample[i].anDLt = pSpace; pSpace += nIdxCol;
-    }
-    assert( ((u8*)pSpace)-nByte==(u8*)(pIdx->aSample) );
-  }
-  rc = sqlite3_finalize(pStmt);
-  if( rc ) return rc;
+  /* Allocate the Schema.pStat4Space block that will be used for the
+  ** Index.aSample[] arrays populated by this call.  */
+  rc = stat4AllocSpace(db, zDb);
+  if( rc!=SQLITE_OK ) return rc;
 
   zSql = sqlite3MPrintf(db, zSql2, zDb);
   if( !zSql ){
@@ -1865,18 +1925,23 @@ static int loadStatTbl(
     if( zIndex==0 ) continue;
     pIdx = findIndexOrPrimaryKey(db, zIndex, zDb);
     if( pIdx==0 ) continue;
-    if( pIdx->nSample>=pIdx->mxSample ){
-      /* Too many slots used because the same index appears in
-      ** sqlite_stat4 using multiple names */
-      continue;
+
+    if( pIdx->nSample==pIdx->nSampleAlloc ){
+      pIdx->pTable->tabFlags |= TF_HasStat4;
+      assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 );
+      if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
+        pIdx->nSampleCol = pIdx->nKeyCol;
+      }else{
+        pIdx->nSampleCol = pIdx->nColumn;
+      }
+      if( growSampleArray(db, pIdx, &iBlockOff) ) break;
     }
-    /* This next condition is true if data has already been loaded from 
-    ** the sqlite_stat4 table. */
-    nCol = pIdx->nSampleCol;
+
     if( pIdx!=pPrevIdx ){
       initAvgEq(pPrevIdx);
       pPrevIdx = pIdx;
     }
+    nCol = pIdx->nSampleCol;
     pSample = &pIdx->aSample[pIdx->nSample];
     decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0);
     decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0);
@@ -1973,6 +2038,10 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
     pIdx->aSample = 0;
 #endif
   }
+#ifdef SQLITE_ENABLE_STAT4
+  sqlite3_free(pSchema->pStat4Space);
+  pSchema->pStat4Space = 0;
+#endif
 
   /* Load new statistics out of the sqlite_stat1 table */
   sInfo.db = db;

diff --git a/src/callback.c b/src/callback.c
index c36d51a4ecf31e65a2a8f93cc576ea786c00b2de..f78abe049b370a3e70e560f2ad4abc985c86d835 100644 (file)
--- a/src/callback.c
+++ b/src/callback.c
@@ -514,6 +514,10 @@ void sqlite3SchemaClear(void *p){
     pSchema->iGeneration++;
   }
   pSchema->schemaFlags &= ~(DB_SchemaLoaded|DB_ResetWanted);
+#ifdef SQLITE_ENABLE_STAT4
+  sqlite3_free(pSchema->pStat4Space);
+  pSchema->pStat4Space = 0;
+#endif
 }
 
 /*

diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index a38938bca6d8daddb04f286bb402c6df440251e9..255ffb5f2fceb31c9318fa900a707aa4cd7a58d3 100644 (file)
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1487,6 +1487,9 @@ struct Schema {
   u8 enc;              /* Text encoding used by this database */
   u16 schemaFlags;     /* Flags associated with this schema */
   int cache_size;      /* Number of pages to use in the cache */
+#ifdef SQLITE_ENABLE_STAT4
+  void *pStat4Space;   /* Memory for stat4 Index.aSample[] arrays */
+#endif
 };
 
 /*
@@ -2779,7 +2782,7 @@ struct Index {
                            ** expression, or a reference to a VIRTUAL column */
 #ifdef SQLITE_ENABLE_STAT4
   int nSample;             /* Number of elements in aSample[] */
-  int mxSample;            /* Number of slots allocated to aSample[] */
+  int nSampleAlloc;        /* Allocated size of aSample[] */
   int nSampleCol;          /* Size of IndexSample.anEq[] and so on */
   tRowcnt *aAvgEq;         /* Average nEq values for keys not in aSample */
   IndexSample *aSample;    /* Samples of the left-most key */