From: dan Date: Wed, 29 Jan 2025 17:26:44 +0000 (+0000) Subject: Consolidate memory allocations made while loading stat4 data in a way that may be... X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=bbda3d68da513ee7317a83aa197036e4faec85cb;p=thirdparty%2Fsqlite.git Consolidate memory allocations made while loading stat4 data in a way that may be more efficient on systems under load. FossilOrigin-Name: af65a902d10e50d827ff31f9ded7d05bc7ab0956a767366e1321b28fcb60b0bd --- diff --git a/manifest b/manifest index fd35b17d9a..062e958cd0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sSQLITE_ENABLE_WAL2NOCKSUM\sis\sdefined,\sthen\sSQLite\scalculates\sthe\sframe\schecksums\sused\sin\swal2\smode\sbased\son\sthe\sprevious\schecksum\sand\sthe\sframe\sheader\sonly,\snot\sthe\sframe\sbody\sonly.\sThis\srisks\scorruption\sfollowing\sa\sOS\scrash\sor\spower\sfailure,\sbut\salso\sspeeds\sup\swrites\sin\swal2\smode. -D 2025-01-29T15:11:07.802 +C Consolidate\smemory\sallocations\smade\swhile\sloading\sstat4\sdata\sin\sa\sway\sthat\smay\sbe\smore\sefficient\son\ssystems\sunder\sload. +D 2025-01-29T17:26:44.487 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -719,7 +719,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 0977c03a4da7c4204bd60e784a0efb8d51a190448aba78a4e973fe7192bdaf03 F src/alter.c aa93e37e4a36a0525bbb2a2aeda20d2018f0aa995542c7dc658e031375e3f532 -F src/analyze.c 9a8b67239d899ac12289db5db3f5bfe7f7a0ad1277f80f87ead1d048085876eb +F src/analyze.c c5573e4c5442bdcebd38692319fb4636c4b8c17de28be10e9ee9c2b57f7f8794 F src/attach.c f35bb8cc1fcdde8f6815a7ef09ae413bcac71821d530796800ba24b3c7da1e80 F src/auth.c 54ab9c6c5803b47c0d45b76ce27eff22a03b4b1f767c5945a3a4eb13aa4c78dc F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 @@ -729,7 +729,7 @@ F src/btree.c 9145be06761b9947d621e575d3837f9cf265242a013c86b23f28ef4be66b3f3e F src/btree.h df26089b055c4cffe243e5bc98edc729c4ad880bfeb8f229fd16248e4cec10ff F src/btreeInt.h bb28bf05e6206befd5f5fd2ed3825fc6382979fa4a83bf50f1875a0d3404111b F src/build.c 5a14f6593b9b0002f54d4215febceeaa1354c12f8eca53f8135d3fdf8a93f4d7 -F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 +F src/callback.c 43c8ca52b1ecbdec43522f121126fd4e3ee10bc9ca01cdd3ae207cfa419780b6 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c dbd2d51fa2874ad54c673cb71b1f5bbd0df1b8b47b34f31d105f42500680a89a F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a @@ -791,7 +791,7 @@ F src/shell.c.in beb370609906092a6810fcd9ea76737be2c91694445061c2eb05c4c0a3753de F src/sqlite.h.in 278a2679a4e50ebe965c5f8e3ec78d52541aa5eeb2632b6eb0a0a395f63b1cea F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h d711bbe8b255024336fb84b090fbc4a6e506f2768925da5426222594537b90c8 +F src/sqliteInt.h 1fc598d06c6c7d09fbb68791f8e876ba5a72a42421f55c3f14e066f155bee1d2 F src/sqliteLimit.h 1bbdbf72bd0411d003267ffebc59a262f061df5653027a75627d03f48ca30523 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2242,9 +2242,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fab341c8295545739cdce8b71e38ead68cb80a6f836f7ec0540b387f17b6cbe2 -Q +4d3706727005397c3c9006b9ad9a2645b09533b02735ea68974c4f2df4c2e853 -R 474ce037ad8c9757d8c8e66f75356bf2 +P dc74bd8915a9e1a915fb4ff3229a7b5e8f89486fe1df812a7738f6627d379648 +R e6eba16885e8253a0c8420a72196b2b2 U dan -Z 363c756be7599a910bd16bdd6751a132 +Z f498dbc7b0d6c6dc50ccede72f5ffcd3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 513952cd5d..9008a1b088 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc74bd8915a9e1a915fb4ff3229a7b5e8f89486fe1df812a7738f6627d379648 +af65a902d10e50d827ff31f9ded7d05bc7ab0956a767366e1321b28fcb60b0bd diff --git a/src/analyze.c b/src/analyze.c index 9213c202b7..b1f72b4d35 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -256,6 +256,13 @@ static void openStatTable( # define SQLITE_STAT4_SAMPLES 24 #endif +/* +** Assumed number of of samples when loading sqlite_stat4 data. It doesn't +** matter if there are more or fewer samples than this, but is more efficient +** if this estimate turns out to be true. +*/ +#define SQLITE_STAT4_EST_SAMPLES SQLITE_STAT4_SAMPLES + /* ** Three SQL functions - stat_init(), stat_push(), and stat_get() - ** share an instance of the following structure to hold their state @@ -1550,6 +1557,9 @@ static void decodeIntArray( #endif if( *z==' ' ) z++; } + if( aOut ){ + for(/* no-op */; iaSample[j]; sqlite3DbFree(db, p->p); } - sqlite3DbFree(db, pIdx->aSample); + if( pIdx->nSampleAlloc!=SQLITE_STAT4_EST_SAMPLES ){ + sqlite3DbFree(db, pIdx->aSample); + } } if( db->pnBytesFreed==0 ){ pIdx->nSample = 0; pIdx->aSample = 0; + pIdx->nSampleAlloc = 0; } #else UNUSED_PARAMETER(db); @@ -1762,8 +1775,110 @@ static Index *findIndexOrPrimaryKey( } /* -** Load the content from either the sqlite_stat4 -** into the relevant Index.aSample[] arrays. +** Grow the pIdx->aSample[] array. Return SQLITE_OK if successful, or +** SQLITE_NOMEM otherwise. +*/ +static int growSampleArray(sqlite3 *db, Index *pIdx, int *piOff){ + int nIdxCol = pIdx->nSampleCol; + int nNew = 0; + IndexSample *aNew = 0; + int nByte = 0; + tRowcnt *pSpace; /* Available allocated memory space */ + u8 *pPtr; /* Available memory as a u8 for easier manipulation */ + int i; + u64 t; + + assert( pIdx->nSample==pIdx->nSampleAlloc ); + nNew = SQLITE_STAT4_EST_SAMPLES; + if( pIdx->nSample ){ + nNew = pIdx->nSample*2; + } + + /* Set nByte to the required amount of space */ + nByte = ROUND8(sizeof(IndexSample) * nNew); + nByte += sizeof(tRowcnt) * nIdxCol * 3 * nNew; + nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + + if( nNew==SQLITE_STAT4_EST_SAMPLES ){ + aNew = (IndexSample*)&((u8*)pIdx->pSchema->pStat4Space)[*piOff]; + *piOff += nByte; + assert( *piOff<=sqlite3_msize(pIdx->pSchema->pStat4Space) ); + }else{ + aNew = (IndexSample*)sqlite3DbMallocRaw(db, nByte); + if( aNew==0 ) return SQLITE_NOMEM_BKPT; + } + + pPtr = (u8*)aNew; + pPtr += ROUND8(nNew*sizeof(pIdx->aSample[0])); + pSpace = (tRowcnt*)pPtr; + + pIdx->aAvgEq = pSpace; pSpace += nIdxCol; + assert( EIGHT_BYTE_ALIGNMENT( pSpace ) ); + + if( pIdx->nSample ){ + /* Copy the contents of the anEq[], anLt[], anDLt[] arrays for all + ** extant samples to the new location. */ + int nByte = nIdxCol * 3 * sizeof(tRowcnt) * pIdx->nSample; + memcpy(pSpace, pIdx->aSample[0].anEq, nByte); + } + for(i=0; inSample ){ + aNew[i].p = pIdx->aSample[i].p; + aNew[i].n = pIdx->aSample[i].n; + } + } + assert( ((u8*)pSpace)-nByte==(u8*)aNew ); + + if( pIdx->nSample!=SQLITE_STAT4_EST_SAMPLES ){ + sqlite3DbFree(db, pIdx->aSample); + } + pIdx->aSample = aNew; + pIdx->nSampleAlloc = nNew; + return SQLITE_OK; +} + +/* +** Allocate the space that will likely be required for the Index.aSample[] +** arrays populated by loading data from the sqlite_stat4 table. Return +** SQLITE_OK if successful, or SQLITE_NOMEM otherwise. +*/ +static int stat4AllocSpace(sqlite3 *db, const char *zDb){ + int iDb = sqlite3FindDbName(db, zDb); + Schema *pSchema = db->aDb[iDb].pSchema; + int nByte = 0; + HashElem *k; + + assert( iDb>=0 ); + assert( pSchema->pStat4Space==0 ); + for(k=sqliteHashFirst(&pSchema->idxHash); k; k=sqliteHashNext(k)){ + Index *pIdx = sqliteHashData(k); + int nIdxCol; + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nIdxCol = pIdx->nKeyCol; + }else{ + nIdxCol = pIdx->nColumn; + } + nByte += ROUND8(sizeof(IndexSample) * SQLITE_STAT4_EST_SAMPLES); + nByte += sizeof(tRowcnt) * nIdxCol * 3 * SQLITE_STAT4_EST_SAMPLES; + nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + } + + if( nByte>0 ){ + pSchema->pStat4Space = sqlite3_malloc(nByte); + if( pSchema->pStat4Space==0 ){ + return SQLITE_NOMEM_BKPT; + } + } + + return SQLITE_OK; +} + +/* +** Load the content from the sqlite_stat4 into the relevant Index.aSample[] +** arrays. ** ** Arguments zSql1 and zSql2 must point to SQL statements that return ** data equivalent to the following: @@ -1784,69 +1899,14 @@ static int loadStatTbl( char *zSql; /* Text of the SQL statement */ Index *pPrevIdx = 0; /* Previous index in the loop */ IndexSample *pSample; /* A slot in pIdx->aSample[] */ + int iBlockOff = 0; /* Offset into Schema.pStat4Space */ assert( db->lookaside.bDisable ); - zSql = sqlite3MPrintf(db, zSql1, zDb); - if( !zSql ){ - return SQLITE_NOMEM_BKPT; - } - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - sqlite3DbFree(db, zSql); - if( rc ) return rc; - - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - int nIdxCol = 1; /* Number of columns in stat4 records */ - char *zIndex; /* Index name */ - Index *pIdx; /* Pointer to the index object */ - int nSample; /* Number of samples */ - i64 nByte; /* Bytes of space required */ - i64 i; /* Bytes of space required */ - tRowcnt *pSpace; /* Available allocated memory space */ - u8 *pPtr; /* Available memory as a u8 for easier manipulation */ - - zIndex = (char *)sqlite3_column_text(pStmt, 0); - if( zIndex==0 ) continue; - nSample = sqlite3_column_int(pStmt, 1); - pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); - assert( pIdx==0 || pIdx->nSample==0 ); - if( pIdx==0 ) continue; - if( pIdx->aSample!=0 ){ - /* The same index appears in sqlite_stat4 under multiple names */ - continue; - } - assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); - if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ - nIdxCol = pIdx->nKeyCol; - }else{ - nIdxCol = pIdx->nColumn; - } - pIdx->nSampleCol = nIdxCol; - pIdx->mxSample = nSample; - nByte = ROUND8(sizeof(IndexSample) * nSample); - nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; - nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ - - pIdx->aSample = sqlite3DbMallocZero(db, nByte); - if( pIdx->aSample==0 ){ - sqlite3_finalize(pStmt); - return SQLITE_NOMEM_BKPT; - } - pPtr = (u8*)pIdx->aSample; - pPtr += ROUND8(nSample*sizeof(pIdx->aSample[0])); - pSpace = (tRowcnt*)pPtr; - assert( EIGHT_BYTE_ALIGNMENT( pSpace ) ); - pIdx->aAvgEq = pSpace; pSpace += nIdxCol; - pIdx->pTable->tabFlags |= TF_HasStat4; - for(i=0; iaSample[i].anEq = pSpace; pSpace += nIdxCol; - pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol; - pIdx->aSample[i].anDLt = pSpace; pSpace += nIdxCol; - } - assert( ((u8*)pSpace)-nByte==(u8*)(pIdx->aSample) ); - } - rc = sqlite3_finalize(pStmt); - if( rc ) return rc; + /* Allocate the Schema.pStat4Space block that will be used for the + ** Index.aSample[] arrays populated by this call. */ + rc = stat4AllocSpace(db, zDb); + if( rc!=SQLITE_OK ) return rc; zSql = sqlite3MPrintf(db, zSql2, zDb); if( !zSql ){ @@ -1865,18 +1925,23 @@ static int loadStatTbl( if( zIndex==0 ) continue; pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); if( pIdx==0 ) continue; - if( pIdx->nSample>=pIdx->mxSample ){ - /* Too many slots used because the same index appears in - ** sqlite_stat4 using multiple names */ - continue; + + if( pIdx->nSample==pIdx->nSampleAlloc ){ + pIdx->pTable->tabFlags |= TF_HasStat4; + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + pIdx->nSampleCol = pIdx->nKeyCol; + }else{ + pIdx->nSampleCol = pIdx->nColumn; + } + if( growSampleArray(db, pIdx, &iBlockOff) ) break; } - /* This next condition is true if data has already been loaded from - ** the sqlite_stat4 table. */ - nCol = pIdx->nSampleCol; + if( pIdx!=pPrevIdx ){ initAvgEq(pPrevIdx); pPrevIdx = pIdx; } + nCol = pIdx->nSampleCol; pSample = &pIdx->aSample[pIdx->nSample]; decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); @@ -1973,6 +2038,10 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ pIdx->aSample = 0; #endif } +#ifdef SQLITE_ENABLE_STAT4 + sqlite3_free(pSchema->pStat4Space); + pSchema->pStat4Space = 0; +#endif /* Load new statistics out of the sqlite_stat1 table */ sInfo.db = db; diff --git a/src/callback.c b/src/callback.c index c36d51a4ec..f78abe049b 100644 --- a/src/callback.c +++ b/src/callback.c @@ -514,6 +514,10 @@ void sqlite3SchemaClear(void *p){ pSchema->iGeneration++; } pSchema->schemaFlags &= ~(DB_SchemaLoaded|DB_ResetWanted); +#ifdef SQLITE_ENABLE_STAT4 + sqlite3_free(pSchema->pStat4Space); + pSchema->pStat4Space = 0; +#endif } /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a38938bca6..255ffb5f2f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1487,6 +1487,9 @@ struct Schema { u8 enc; /* Text encoding used by this database */ u16 schemaFlags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ +#ifdef SQLITE_ENABLE_STAT4 + void *pStat4Space; /* Memory for stat4 Index.aSample[] arrays */ +#endif }; /* @@ -2779,7 +2782,7 @@ struct Index { ** expression, or a reference to a VIRTUAL column */ #ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ - int mxSample; /* Number of slots allocated to aSample[] */ + int nSampleAlloc; /* Allocated size of aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */