From: dan Date: Fri, 20 Dec 2024 19:37:41 +0000 (+0000) Subject: When loading stat4 data, make a single large allocation for all Index.aSample[] array... X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f46f925962925eac7981203210201aea2f84ada8;p=thirdparty%2Fsqlite.git When loading stat4 data, make a single large allocation for all Index.aSample[] arrays, instead of a separate allocation for each. FossilOrigin-Name: b40cd7395c44b1f2d019d8e809e03de0e083c93693322a72ddb250a85640528f --- diff --git a/manifest b/manifest index df987ee7fa..893f0e4095 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\smemset()\swhen\sloading\sstat4\sdata.\sAdd\sinstrumentation\sto\sthe\ssame\scode. -D 2024-12-13T18:13:51.565 +C When\sloading\sstat4\sdata,\smake\sa\ssingle\slarge\sallocation\sfor\sall\sIndex.aSample[]\sarrays,\sinstead\sof\sa\sseparate\sallocation\sfor\seach. +D 2024-12-20T19:37:41.637 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -707,7 +707,7 @@ F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c aa93e37e4a36a0525bbb2a2aeda20d2018f0aa995542c7dc658e031375e3f532 -F src/analyze.c b927896d0982b91632030982cc727608f8c682ab0279c1f6b33f01e3928e7714 +F src/analyze.c c011b69c903197f45688799d81e9f216be84f6eb4b3faa23a3d7638111859da2 F src/attach.c 08235ab62ed5ccc93c22bf36e640d19effcd632319615851bccf724ec9341333 F src/auth.c 4c1ea890e0069ad73bead5d17a5b12c34cfa4f1a24175c8147ea439b64be271c F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 @@ -717,7 +717,7 @@ F src/btree.c 2664c81f217a42afadc7c010bb4a175057d5e53b99e9512234eb74817f2ad59c F src/btree.h bdeeb35614caa33526b603138f04c8d07a3f90a1300b5ade76848b755edf2027 F src/btreeInt.h caa893e74d2261fb0ff1681fce998533c0552858e882bd04fc6805075f5f6e75 F src/build.c 08697d6a4df78f8e289582eb58473445492319676f81cc4794ef4056d36ae5fd -F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 +F src/callback.c 43c8ca52b1ecbdec43522f121126fd4e3ee10bc9ca01cdd3ae207cfa419780b6 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 193f6f9a75204274b7e7f45ac6d6517c12c70b55a5dfb39312dfc3a52e2a8138 F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a @@ -779,7 +779,7 @@ F src/shell.c.in 40de636c1d90fb8a9ca7f49dc8f50d930f1b60736e73aca5eb37c4c7d0e47f9 F src/sqlite.h.in b7ff496637807ae88b2557039fc940518db328bf5d5621e2f7c048dfba32a52b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h d3964f3cdf1006e10864226d8698575e18a4cc2072052684083a1ce127c97046 +F src/sqliteInt.h 4281b9ed8b660fe703a58ff8aa70247df07ad6b75f47eb7f45d1380021720f81 F src/sqliteLimit.h da2cffdffa7d71b035f9e59668fbaad74b5939300dbfa9915304e6d8f72b0761 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -849,7 +849,7 @@ F src/vdbe.c fa31aa6525e34b51763702d246a69c3877436f0bbf6be5f6351cdcb770b9b7ab F src/vdbe.h b74bfd9cb1fa895e545a5286ee1cac6d75f706d325f89be0e3bf3c5107eb8a78 F src/vdbeInt.h 75373833a8178a78126625dda269bcd41db40547092114d1edff12e7bcbb1606 F src/vdbeapi.c 80235ac380e9467fec1cb0883354d841f2a771976e766995f7e0c77f845406df -F src/vdbeaux.c a5250f3c1b4a7add56a1d0f7343a41ede2c0d4042f5135ee75cf628cf050fe7d +F src/vdbeaux.c c9c01a58fa3c591220f06f4e45486da60231284008679ed72cf2344de19af990 F src/vdbeblob.c 255be187436da38b01f276c02e6a08103489bbe2a7c6c21537b7aecbe0e1f797 F src/vdbemem.c 831a244831eaa45335f9ae276b50a7a82ee10d8c46c2c72492d4eb8c98d94d89 F src/vdbesort.c d0a3c7056c081703c8b6d91ad60f17da5e062a5c64bf568ed0fa1b5f4cae311f @@ -2250,8 +2250,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3c25c69c93e55738cdbfdd87fa3c879b878674973955490770f5e274da1ca9a4 -R 6dd6638c622e4fc0acf34660a1e4b6d7 +P fa87355f6286be1e92f22a71cbfbfb13d1a478d5fb5b38abedbd78bf903171fa +R 9097e0f64b77340a961f20f33cb504fb U dan -Z 5de3a82d9a785cefee51329773e29e7c +Z f620b0a1456c2c860e9486abd7f7bdfb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 10e07e6645..5f9bc9a5c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa87355f6286be1e92f22a71cbfbfb13d1a478d5fb5b38abedbd78bf903171fa +b40cd7395c44b1f2d019d8e809e03de0e083c93693322a72ddb250a85640528f diff --git a/src/analyze.c b/src/analyze.c index c1c3b27b89..58f29ef717 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -256,6 +256,13 @@ static void openStatTable( # define SQLITE_STAT4_SAMPLES 24 #endif +/* +** Assumed number of of samples when loading sqlite_stat4 data. It doesn't +** matter if there are more or fewer samples than this, but is more efficient +** if this estimate turns out to be true. +*/ +#define SQLITE_STAT4_EST_SAMPLES SQLITE_STAT4_SAMPLES + /* ** Three SQL functions - stat_init(), stat_push(), and stat_get() - ** share an instance of the following structure to hold their state @@ -1675,7 +1682,9 @@ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ IndexSample *p = &pIdx->aSample[j]; sqlite3DbFree(db, p->p); } - sqlite3DbFree(db, pIdx->aSample); + if( pIdx->nSampleAlloc!=SQLITE_STAT4_EST_SAMPLES ){ + sqlite3DbFree(db, pIdx->aSample); + } } if( db->pnBytesFreed==0 ){ pIdx->nSample = 0; @@ -1769,7 +1778,7 @@ static Index *findIndexOrPrimaryKey( ** Grow the pIdx->aSample[] array. Return SQLITE_OK if successful, or ** SQLITE_NOMEM otherwise. */ -static int growSampleArray(sqlite3 *db, Index *pIdx){ +static int growSampleArray(sqlite3 *db, Index *pIdx, int *piOff){ int nIdxCol = pIdx->nSampleCol; int nNew = 0; IndexSample *aNew = 0; @@ -1779,31 +1788,24 @@ static int growSampleArray(sqlite3 *db, Index *pIdx){ int i; u64 t; - /* In production set the initial allocation to SQLITE_STAT4_SAMPLES. This - ** means that reallocation will almost never be required. But for debug - ** builds, set the initial allocation size to 6 entries so that the - ** reallocation code gets tested. todo: use real tests for this. */ assert( pIdx->nSample==pIdx->nSampleAlloc ); -#ifdef SQLITE_DEBUG - nNew = 6; -#else - nNew = SQLITE_STAT4_SAMPLES; -#endif + nNew = SQLITE_STAT4_EST_SAMPLES; if( pIdx->nSample ){ nNew = pIdx->nSample*2; } + /* Set nByte to the required amount of space */ nByte = ROUND8(sizeof(IndexSample) * nNew); nByte += sizeof(tRowcnt) * nIdxCol * 3 * nNew; nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ - if( db->aSchemaTime ){ - t = sqlite3STimeNow(); - } - aNew = (IndexSample*)sqlite3DbMallocRaw(db, nByte); - if( aNew==0 ) return SQLITE_NOMEM_BKPT; - if( db->aSchemaTime ){ - db->aSchemaTime[SCHEMA_TIME_STAT4_SAMPLE_MALLOC] += (sqlite3STimeNow() - t); + if( nNew==SQLITE_STAT4_EST_SAMPLES ){ + aNew = (IndexSample*)&((u8*)pIdx->pSchema->pStat4Space)[*piOff]; + *piOff += nByte; + assert( *piOff<=sqlite3_msize(pIdx->pSchema->pStat4Space) ); + }else{ + aNew = (IndexSample*)sqlite3DbMallocRaw(db, nByte); + if( aNew==0 ) return SQLITE_NOMEM_BKPT; } pPtr = (u8*)aNew; @@ -1830,15 +1832,53 @@ static int growSampleArray(sqlite3 *db, Index *pIdx){ } assert( ((u8*)pSpace)-nByte==(u8*)aNew ); - sqlite3DbFree(db, pIdx->aSample); + if( pIdx->nSample!=SQLITE_STAT4_EST_SAMPLES ){ + sqlite3DbFree(db, pIdx->aSample); + } pIdx->aSample = aNew; pIdx->nSampleAlloc = nNew; return SQLITE_OK; } /* -** Load the content from either the sqlite_stat4 -** into the relevant Index.aSample[] arrays. +** Allocate the space that will likely be required for the Index.aSample[] +** arrays populated by loading data from the sqlite_stat4 table. Return +** SQLITE_OK if successful, or SQLITE_NOMEM otherwise. +*/ +static int stat4AllocSpace(sqlite3 *db, const char *zDb){ + int iDb = sqlite3FindDbName(db, zDb); + Schema *pSchema = db->aDb[iDb].pSchema; + int nByte = 0; + HashElem *k; + + assert( iDb>=0 ); + assert( pSchema->pStat4Space==0 ); + for(k=sqliteHashFirst(&pSchema->idxHash); k; k=sqliteHashNext(k)){ + Index *pIdx = sqliteHashData(k); + int nIdxCol; + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nIdxCol = pIdx->nKeyCol; + }else{ + nIdxCol = pIdx->nColumn; + } + nByte += ROUND8(sizeof(IndexSample) * SQLITE_STAT4_EST_SAMPLES); + nByte += sizeof(tRowcnt) * nIdxCol * 3 * SQLITE_STAT4_EST_SAMPLES; + nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + } + + if( nByte>0 ){ + pSchema->pStat4Space = sqlite3_malloc(nByte); + if( pSchema->pStat4Space==0 ){ + return SQLITE_NOMEM_BKPT; + } + } + + return SQLITE_OK; +} + +/* +** Load the content from the sqlite_stat4 into the relevant Index.aSample[] +** arrays. ** ** Arguments zSql1 and zSql2 must point to SQL statements that return ** data equivalent to the following: @@ -1859,78 +1899,16 @@ static int loadStatTbl( char *zSql; /* Text of the SQL statement */ Index *pPrevIdx = 0; /* Previous index in the loop */ IndexSample *pSample; /* A slot in pIdx->aSample[] */ + int iBlockOff = 0; /* Offset into Schema.pStat4Space */ assert( db->lookaside.bDisable ); -#if 0 - zSql = sqlite3MPrintf(db, zSql1, zDb); - if( !zSql ){ - return SQLITE_NOMEM_BKPT; - } - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - sqlite3DbFree(db, zSql); - if( rc ) return rc; - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - int nIdxCol = 1; /* Number of columns in stat4 records */ - - char *zIndex; /* Index name */ - Index *pIdx; /* Pointer to the index object */ - int nSample; /* Number of samples */ - int nByte; /* Bytes of space required */ - int i; /* Bytes of space required */ - tRowcnt *pSpace; /* Available allocated memory space */ - u8 *pPtr; /* Available memory as a u8 for easier manipulation */ - - u64 t = sqlite3STimeNow(); - - zIndex = (char *)sqlite3_column_text(pStmt, 0); - if( zIndex==0 ) continue; - nSample = sqlite3_column_int(pStmt, 1); - pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); - assert( pIdx==0 || pIdx->nSample==0 ); - if( pIdx==0 ) continue; - if( pIdx->aSample!=0 ){ - /* The same index appears in sqlite_stat4 under multiple names */ - continue; - } - assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); - if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ - nIdxCol = pIdx->nKeyCol; - }else{ - nIdxCol = pIdx->nColumn; - } - pIdx->nSampleCol = nIdxCol; - pIdx->mxSample = nSample; - nByte = ROUND8(sizeof(IndexSample) * nSample); - nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; - nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ - - pIdx->aSample = sqlite3DbMallocZero(db, nByte); - if( pIdx->aSample==0 ){ - sqlite3_finalize(pStmt); - return SQLITE_NOMEM_BKPT; - } - pPtr = (u8*)pIdx->aSample; - pPtr += ROUND8(nSample*sizeof(pIdx->aSample[0])); - pSpace = (tRowcnt*)pPtr; - assert( EIGHT_BYTE_ALIGNMENT( pSpace ) ); - pIdx->aAvgEq = pSpace; pSpace += nIdxCol; - pIdx->pTable->tabFlags |= TF_HasStat4; - for(i=0; iaSample[i].anEq = pSpace; pSpace += nIdxCol; - pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol; - pIdx->aSample[i].anDLt = pSpace; pSpace += nIdxCol; - } - assert( ((u8*)pSpace)-nByte==(u8*)(pIdx->aSample) ); - if( db->aSchemaTime ){ - db->aSchemaTime[SCHEMA_TIME_STAT4_Q1_BODY] += (sqlite3STimeNow() - t); - } - } - rc = sqlite3_finalize(pStmt); - if( rc ) return rc; -#endif + /* Allocate the Schema.pStat4Space block that will be used for the + ** Index.aSample[] arrays populated by this call. */ + rc = stat4AllocSpace(db, zDb); + if( rc!=SQLITE_OK ) return rc; - sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q1); + sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_SPACE); zSql = sqlite3MPrintf(db, zSql2, zDb); if( !zSql ){ @@ -1940,6 +1918,8 @@ static int loadStatTbl( sqlite3DbFree(db, zSql); if( rc ) return rc; + sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_PREPARE); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ char *zIndex; /* Index name */ Index *pIdx; /* Pointer to the index object */ @@ -1952,6 +1932,7 @@ static int loadStatTbl( if( pIdx==0 ) continue; if( pIdx->nSample==pIdx->nSampleAlloc ){ + u64 t2; pIdx->pTable->tabFlags |= TF_HasStat4; assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ @@ -1959,7 +1940,11 @@ static int loadStatTbl( }else{ pIdx->nSampleCol = pIdx->nColumn; } - if( growSampleArray(db, pIdx) ) break; + t2 = sqlite3STimeNow(); + if( growSampleArray(db, pIdx, &iBlockOff) ) break; + if( db->aSchemaTime ){ + db->aSchemaTime[SCHEMA_TIME_STAT4_GROWUS] += (sqlite3STimeNow() - t); + } } if( pIdx!=pPrevIdx ){ @@ -1992,7 +1977,7 @@ static int loadStatTbl( pIdx->nSample++; if( db->aSchemaTime ){ - db->aSchemaTime[SCHEMA_TIME_STAT4_Q2_BODY] += (sqlite3STimeNow() - t); + db->aSchemaTime[SCHEMA_TIME_STAT4_Q2_BODYUS] += (sqlite3STimeNow() - t); } } rc = sqlite3_finalize(pStmt); @@ -2069,6 +2054,10 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ pIdx->aSample = 0; #endif } +#ifdef SQLITE_ENABLE_STAT4 + sqlite3_free(pSchema->pStat4Space); + pSchema->pStat4Space = 0; +#endif sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_CLEAR_STATS); diff --git a/src/callback.c b/src/callback.c index c36d51a4ec..f78abe049b 100644 --- a/src/callback.c +++ b/src/callback.c @@ -514,6 +514,10 @@ void sqlite3SchemaClear(void *p){ pSchema->iGeneration++; } pSchema->schemaFlags &= ~(DB_SchemaLoaded|DB_ResetWanted); +#ifdef SQLITE_ENABLE_STAT4 + sqlite3_free(pSchema->pStat4Space); + pSchema->pStat4Space = 0; +#endif } /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 2817de3c87..19be90f86f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1490,6 +1490,9 @@ struct Schema { u8 enc; /* Text encoding used by this database */ u16 schemaFlags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ +#ifdef SQLITE_ENABLE_STAT4 + void *pStat4Space; /* Memory for stat4 Index.aSample[] arrays */ +#endif }; /* @@ -1869,11 +1872,13 @@ struct sqlite3 { #define SCHEMA_TIME_AFTER_STAT1 12 #define SCHEMA_TIME_AFTER_DEFAULTS 13 -#define SCHEMA_TIME_STAT4_Q1_BODY 14 -#define SCHEMA_TIME_AFTER_STAT4_Q1 15 -#define SCHEMA_TIME_STAT4_Q2_BODY 16 -#define SCHEMA_TIME_STAT4_SAMPLE_MALLOC 17 -#define SCHEMA_TIME_AFTER_STAT4_Q2 18 +#define SCHEMA_TIME_AFTER_STAT4_SPACE 14 +#define SCHEMA_TIME_AFTER_STAT4_PREPARE 15 + +#define SCHEMA_TIME_STAT4_GROWUS 16 +#define SCHEMA_TIME_STAT4_Q2_BODYUS 17 +#define SCHEMA_TIME_AFTER_STAT4_Q2 18 + #define SCHEMA_TIME_AFTER_STAT4 19 #define SCHEMA_TIME_END_ANALYZE_LOAD 20 diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 6a40156091..7e31f92b18 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -5499,7 +5499,7 @@ void sqlite3CommitTimeLog(u64 *aCommit){ } zStr = sqlite3_mprintf("%z%s%s%d%s", zStr, (zStr?", ":""),zHash,iVal,zU); } - sqlite3_log(SQLITE_WARNING, "slow commit (v=19): (%s)", zStr); + sqlite3_log(SQLITE_WARNING, "slow commit (v=20): (%s)", zStr); sqlite3_free(zStr); } } @@ -5527,7 +5527,7 @@ void sqlite3PrepareTimeLog(const char *zSql, int nSql, u64 *aPrepare){ } if( nByte<0 ){ nByte = sqlite3Strlen30(zSql); } sqlite3_log(SQLITE_WARNING, - "slow prepare (v=19): (%s) [%.*s]", zStr, nByte, zSql + "slow prepare (v=20): (%s) [%.*s]", zStr, nByte, zSql ); sqlite3_free(zStr); } @@ -5541,15 +5541,14 @@ void sqlite3SchemaTimeLog(u64 *aSchema, const char *zFile){ for(ii=1; ii