From: dan Date: Thu, 12 Dec 2024 19:15:46 +0000 (+0000) Subject: Experimental code to load stat4 data using a single pass. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=068d15924e251113d0b7a6dcf5bd58086588d34e;p=thirdparty%2Fsqlite.git Experimental code to load stat4 data using a single pass. FossilOrigin-Name: e0bcd30581b5b55abb0ba0f2855dbfa27f4e036b45423062402fe344db7bb070 --- diff --git a/manifest b/manifest index 0ad34da1fd..0e7738f89a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sa\swriter\sinitializes\sa\snew\s*-shm\sfile\shash\stable,\szero\sonly\sthe\shash-table\sslots,\snot\sthe\sarray\sof\spage\snumbers. -D 2024-12-11T20:42:29.249 +C Experimental\scode\sto\sload\sstat4\sdata\susing\sa\ssingle\spass. +D 2024-12-12T19:15:46.069 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -707,7 +707,7 @@ F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c aa93e37e4a36a0525bbb2a2aeda20d2018f0aa995542c7dc658e031375e3f532 -F src/analyze.c 7fda93b7952edc84282f57f56dd63d99afe428502b0289f571e698e0ce7b978a +F src/analyze.c 4d49e7be20b5c58c197f53280122a46facc9821c80b26837dd12a219818ecc7b F src/attach.c 08235ab62ed5ccc93c22bf36e640d19effcd632319615851bccf724ec9341333 F src/auth.c 4c1ea890e0069ad73bead5d17a5b12c34cfa4f1a24175c8147ea439b64be271c F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 @@ -779,7 +779,7 @@ F src/shell.c.in 40de636c1d90fb8a9ca7f49dc8f50d930f1b60736e73aca5eb37c4c7d0e47f9 F src/sqlite.h.in b7ff496637807ae88b2557039fc940518db328bf5d5621e2f7c048dfba32a52b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 3fb893edbabad31aa7a32cecf3a5282cbd14391b5e1d143e23594d04fe568b36 +F src/sqliteInt.h c8f95e16884274e80dc53d491a1cf8283b6021ac2f64334c07a37eaf21b55300 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -849,7 +849,7 @@ F src/vdbe.c fa31aa6525e34b51763702d246a69c3877436f0bbf6be5f6351cdcb770b9b7ab F src/vdbe.h b74bfd9cb1fa895e545a5286ee1cac6d75f706d325f89be0e3bf3c5107eb8a78 F src/vdbeInt.h 92b7eabbaadbe8127741cd96e7e39c4834c2bb0b75987d5f6251975f47233690 F src/vdbeapi.c 80235ac380e9467fec1cb0883354d841f2a771976e766995f7e0c77f845406df -F src/vdbeaux.c 6f2c4e262d124f3126a3fb865c09314ae825ffefc9eca140381584968502c989 +F src/vdbeaux.c b8361c6daaa8b189774e2d2433073511311a81bb411429238fcc24d68ddc4a01 F src/vdbeblob.c 255be187436da38b01f276c02e6a08103489bbe2a7c6c21537b7aecbe0e1f797 F src/vdbemem.c 831a244831eaa45335f9ae276b50a7a82ee10d8c46c2c72492d4eb8c98d94d89 F src/vdbesort.c d0a3c7056c081703c8b6d91ad60f17da5e062a5c64bf568ed0fa1b5f4cae311f @@ -2250,8 +2250,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P df4183ace93b788b798b258274bf6b651906c9f1cf2af4983e447cdf52904523 -R 94edb7bb8062fb14ef6badf564e5e7de +P ac5401c62b3ff03e43e01d066f4664491ce9b640ec64150a58b38839b4834afb +R 83b28506326e6134726d20ac1da94326 U dan -Z 6fb05f3841c97a9cc6100a75f02a6296 +Z b392d5c3e26ddc5e80d40edd1473ba14 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cf395e0066..ae35375c63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac5401c62b3ff03e43e01d066f4664491ce9b640ec64150a58b38839b4834afb +e0bcd30581b5b55abb0ba0f2855dbfa27f4e036b45423062402fe344db7bb070 diff --git a/src/analyze.c b/src/analyze.c index 03a0aa891a..1325940d76 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1677,6 +1677,7 @@ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ if( db->pnBytesFreed==0 ){ pIdx->nSample = 0; pIdx->aSample = 0; + pIdx->nSampleAlloc = 0; } #else UNUSED_PARAMETER(db); @@ -1761,6 +1762,70 @@ static Index *findIndexOrPrimaryKey( return pIdx; } +/* +** Grow the pIdx->aSample[] array. Return SQLITE_OK if successful, or +** SQLITE_NOMEM otherwise. +*/ +static int growSampleArray(sqlite3 *db, Index *pIdx){ + int nIdxCol = pIdx->nSampleCol; + int nNew = 0; + IndexSample *aNew = 0; + int nByte = 0; + tRowcnt *pSpace; /* Available allocated memory space */ + u8 *pPtr; /* Available memory as a u8 for easier manipulation */ + int i; + + /* In production set the initial allocation to SQLITE_STAT4_SAMPLES. This + ** means that reallocation will almost never be required. But for debug + ** builds, set the initial allocation size to 6 entries so that the + ** reallocation code gets tested. todo: use real tests for this. */ + assert( pIdx->nSample==pIdx->nSampleAlloc ); +#ifdef SQLITE_DEBUG + nNew = 6; +#else + nNew = SQLITE_STAT4_SAMPLES; +#endif + if( pIdx->nSample ){ + nNew = pIdx->nSample*2; + } + + nByte = ROUND8(sizeof(IndexSample) * nNew); + nByte += sizeof(tRowcnt) * nIdxCol * 3 * nNew; + nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + + aNew = (IndexSample*)sqlite3DbMallocZero(db, nByte); + if( aNew==0 ) return SQLITE_NOMEM_BKPT; + + pPtr = (u8*)aNew; + pPtr += ROUND8(nNew*sizeof(pIdx->aSample[0])); + pSpace = (tRowcnt*)pPtr; + + pIdx->aAvgEq = pSpace; pSpace += nIdxCol; + assert( EIGHT_BYTE_ALIGNMENT( pSpace ) ); + + if( pIdx->nSample ){ + /* Copy the contents of the anEq[], anLt[], anDLt[] arrays for all + ** extant samples to the new location. */ + int nByte = nIdxCol * 3 * sizeof(tRowcnt) * pIdx->nSample; + memcpy(pSpace, pIdx->aSample[0].anEq, nByte); + } + for(i=0; inSample ){ + aNew[i].p = pIdx->aSample[i].p; + aNew[i].n = pIdx->aSample[i].n; + } + } + assert( ((u8*)pSpace)-nByte==(u8*)aNew ); + + sqlite3DbFree(db, pIdx->aSample); + pIdx->aSample = aNew; + pIdx->nSampleAlloc = nNew; + return SQLITE_OK; +} + /* ** Load the content from either the sqlite_stat4 ** into the relevant Index.aSample[] arrays. @@ -1786,6 +1851,7 @@ static int loadStatTbl( IndexSample *pSample; /* A slot in pIdx->aSample[] */ assert( db->lookaside.bDisable ); +#if 0 zSql = sqlite3MPrintf(db, zSql1, zDb); if( !zSql ){ return SQLITE_NOMEM_BKPT; @@ -1852,6 +1918,9 @@ static int loadStatTbl( } rc = sqlite3_finalize(pStmt); if( rc ) return rc; +#endif + + sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q1); zSql = sqlite3MPrintf(db, zSql2, zDb); if( !zSql ){ @@ -1861,8 +1930,6 @@ static int loadStatTbl( sqlite3DbFree(db, zSql); if( rc ) return rc; - sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q1); - while( sqlite3_step(pStmt)==SQLITE_ROW ){ char *zIndex; /* Index name */ Index *pIdx; /* Pointer to the index object */ @@ -1872,18 +1939,24 @@ static int loadStatTbl( if( zIndex==0 ) continue; pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); if( pIdx==0 ) continue; - if( pIdx->nSample>=pIdx->mxSample ){ - /* Too many slots used because the same index appears in - ** sqlite_stat4 using multiple names */ - continue; + + if( pIdx->nSample==pIdx->nSampleAlloc ){ + pIdx->pTable->tabFlags |= TF_HasStat4; + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + pIdx->nSampleCol = pIdx->nKeyCol; + }else{ + pIdx->nSampleCol = pIdx->nColumn; + } + if( growSampleArray(db, pIdx) ) break; } - /* This next condition is true if data has already been loaded from - ** the sqlite_stat4 table. */ - nCol = pIdx->nSampleCol; + if( pIdx!=pPrevIdx ){ initAvgEq(pPrevIdx); pPrevIdx = pIdx; } + + nCol = pIdx->nSampleCol; pSample = &pIdx->aSample[pIdx->nSample]; decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4676ab1413..37363c63a2 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1878,7 +1878,7 @@ struct sqlite3 { #define SCHEMA_TIME_FINISH 19 #define SCHEMA_TIME_N 20 -#define SCHEMA_TIME_TIMEOUT (1 * 1000 * 1000) +#define SCHEMA_TIME_TIMEOUT (500 * 1000) @@ -2882,7 +2882,7 @@ struct Index { ** expression, or a reference to a VIRTUAL column */ #ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ - int mxSample; /* Number of slots allocated to aSample[] */ + int nSampleAlloc; /* Number of slots allocated to aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index c691d6ab9c..4cde4aa981 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -5499,7 +5499,7 @@ void sqlite3CommitTimeLog(u64 *aCommit){ } zStr = sqlite3_mprintf("%z%s%s%d%s", zStr, (zStr?", ":""),zHash,iVal,zU); } - sqlite3_log(SQLITE_WARNING, "slow commit (v=17): (%s)", zStr); + sqlite3_log(SQLITE_WARNING, "slow commit (v=18): (%s)", zStr); sqlite3_free(zStr); } } @@ -5527,7 +5527,7 @@ void sqlite3PrepareTimeLog(const char *zSql, int nSql, u64 *aPrepare){ } if( nByte<0 ){ nByte = sqlite3Strlen30(zSql); } sqlite3_log(SQLITE_WARNING, - "slow prepare (v=17): (%s) [%.*s]", zStr, nByte, zSql + "slow prepare (v=18): (%s) [%.*s]", zStr, nByte, zSql ); sqlite3_free(zStr); } @@ -5545,7 +5545,7 @@ void sqlite3SchemaTimeLog(u64 *aSchema, const char *zFile){ } zStr = sqlite3_mprintf("%z%s%d", zStr, (zStr?", ":""), val); } - sqlite3_log(SQLITE_WARNING, "slow schema (%s) (v=17): (%s)", zFile, zStr); + sqlite3_log(SQLITE_WARNING, "slow schema (%s) (v=18): (%s)", zFile, zStr); sqlite3_free(zStr); } }