From: drh Date: Sat, 13 Aug 2011 00:58:05 +0000 (+0000) Subject: The ANALYZE command picks for 15 samples for sqlite_stat3 with the largest X-Git-Tag: version-3.7.9~67^2~11^2~10 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ade3addfb541b6db23619af99763732bca468dd6;p=thirdparty%2Fsqlite.git The ANALYZE command picks for 15 samples for sqlite_stat3 with the largest nEq fields, plus 5 other evenly spaced samples. FossilOrigin-Name: 8225924ea015a0c331b69134139922ec83f989f8 --- diff --git a/manifest b/manifest index 13b45de240..7193488362 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\sa\sbranch\sthat\sexperimentally\sreplaces\ssqlite_stat2\swith\sa\snew\stable\ncalled\ssqlite_stat3\sthat\swill\shopefully\sfacilitate\sbetter\squery\nplanning\sdecisions. -D 2011-08-12T01:51:45.485 +C The\sANALYZE\scommand\spicks\sfor\s15\ssamples\sfor\ssqlite_stat3\swith\sthe\slargest\nnEq\sfields,\splus\s5\sother\sevenly\sspaced\ssamples. +D 2011-08-13T00:58:05.748 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1e6988b3c11dee9bd5edc0c804bd4468d74a9cdc F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,7 +118,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c ac80a0f31189f8b4a524ebf661e47e84536ee7f5 -F src/analyze.c da6661dbe12f71d37e81c1138cd7b3175fa60a4f +F src/analyze.c 31a1ea5a5a355097aa7a5fce09bbd9ae2a2c7672 F src/attach.c 12c6957996908edc31c96d7c68d4942c2474405f F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c @@ -958,10 +958,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 6b236069e1ea3c99ff0a007a790d4baebda70b13 -R 28c5bf799a1842cffe08b02c7be0270b -T *branch * stat3-enhancement -T *sym-stat3-enhancement * -T -sym-trunk * +P 52e1d7e8ddd4bb5ef3a9d00fd2d719a8a784f807 +R d14fbf4d209dccbd0b61f66b6e37c6c9 U drh -Z 1615a89a47ce6302aa7ba89aa7dcb40d +Z 59baacb653226a018ea530dc8e60b319 diff --git a/manifest.uuid b/manifest.uuid index 1903efcb65..e910470b5f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -52e1d7e8ddd4bb5ef3a9d00fd2d719a8a784f807 \ No newline at end of file +8225924ea015a0c331b69134139922ec83f989f8 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 7ccddfb115..848a939576 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -207,9 +207,214 @@ static void openStatTable( ** Recommended number of samples for sqlite_stat3 */ #ifndef SQLITE_STAT3_SAMPLES -# define SQLITE_STAT3_SAMPLES 16 +# define SQLITE_STAT3_SAMPLES 20 #endif +/* +** Three SQL functions - stat3_init(), stat3_push(), and stat3_pop() - +** share an instance of the following structure to hold their state +** information. +*/ +typedef struct Stat3Accum Stat3Accum; +struct Stat3Accum { + tRowcnt nRow; /* Number of rows in the entire table */ + tRowcnt nPSample; /* How often to do a periodic sample */ + int iMin; /* Index of entry with minimum nEq and hash */ + int mxSample; /* Maximum number of samples to accumulate */ + int nSample; /* Current number of samples */ + struct Stat3Sample { + i64 iRowid; /* Rowid in main table of the key */ + tRowcnt nEq; /* sqlite_stat3.nEq */ + tRowcnt nLt; /* sqlite_stat3.nLt */ + u8 isPSample; /* True if a periodic sample */ + u32 iHash; /* Tiebreaker hash */ + } *a; /* An array of samples */ +}; + +#ifdef SQLITE_ENABLE_STAT3 +/* +** Implementation of the stat3_init(C,S) SQL function. The two parameters +** are the number of rows in the table or index (C) and the number of samples +** to accumulate (S). +** +** This routine allocates the Stat3Accum object. +** +** The return value is the Stat3Accum object (P). +*/ +static void stat3Init( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Stat3Accum *p; + tRowcnt nRow; + int mxSample; + int n; + + nRow = (tRowcnt)sqlite3_value_int64(argv[0]); + mxSample = sqlite3_value_int(argv[1]); + n = sizeof(*p) + sizeof(p->a[0])*mxSample; + p = sqlite3_malloc( n ); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + memset(p, 0, n); + p->a = (struct Stat3Sample*)&p[1]; + p->nRow = nRow; + p->mxSample = mxSample; + p->nPSample = p->nRow/6 + 1; + sqlite3_result_blob(context, p, sizeof(p), sqlite3_free); +} +static const FuncDef stat3InitFuncdef = { + 2, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + stat3Init, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat3_init", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ +}; + + +/* +** Implementation of the stat3_push(nEq,nLt,rowid,P) SQL function. The +** arguments describe a single key instance. This routine makes the +** decision about whether or not to retain this key for the sqlite_stat3 +** table. +** +** The return value is NULL. +*/ +static void stat3Push( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Stat3Accum *p = (Stat3Accum*)sqlite3_value_blob(argv[3]); + tRowcnt nEq = sqlite3_value_int64(argv[0]); + tRowcnt nLt = sqlite3_value_int64(argv[1]); + i64 rowid = sqlite3_value_int64(argv[2]); + u8 isPSample = 0; + u8 doInsert = 0; + int iMin = p->iMin; + struct Stat3Sample *pSample; + int i; + u32 h, h1, h2, h3; + if( nEq==0 ) return; + + h1 = (unsigned)(rowid&0xffff); + h2 = (unsigned)nEq; + h3 = (unsigned)(nLt+1); + h = h1*h2*h3*0x10010001; + + if( (nLt/p->nPSample)!=((nEq+nLt)/p->nPSample) ){ + doInsert = isPSample = 1; + }else if( p->nSamplemxSample ){ + doInsert = 1; + }else{ + if( nEq>p->a[iMin].nEq || (nEq==p->a[iMin].nEq && h>p->a[iMin].iHash) ){ + doInsert = 1; + } + } + if( !doInsert ) return; + if( p->nSample==p->mxSample ){ + pSample = &p->a[iMin]; + }else{ + pSample = &p->a[p->nSample++]; + } + pSample->iRowid = rowid; + pSample->nEq = nEq; + pSample->nLt = nLt; + pSample->iHash = h; + pSample->isPSample = isPSample; + + /* Find the new minimum */ + if( p->nSample==p->mxSample ){ + pSample = p->a; + i = 0; + while( pSample->isPSample ){ + i++; + pSample++; + assert( inSample ); + } + nEq = pSample->nEq; + h = pSample->iHash; + iMin = i; + for(i++, pSample++; inSample; i++, pSample++){ + if( pSample->isPSample ) continue; + if( pSample->nEqnEq==nEq && pSample->iHashnEq; + h = pSample->iHash; + } + } + p->iMin = iMin; + } +} +static const FuncDef stat3PushFuncdef = { + 3, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + stat3Push, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat3_push", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ +}; + +/* +** Implementation of the stat3_get(P,N,...) SQL function. This routine is +** used to query the results. Content is returned for the Nth sqlite_stat3 +** row where N is between 0 and S-1 and S is the number of samples. The +** value returned depends on the number of arguments. +** +** argc==2 result: rowid +** argc==3 result: nEq +** argc==4 result: nLt +*/ +static void stat3Get( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int n = sqlite3_value_int(argv[1]); + Stat3Accum *p = (Stat3Accum*)sqlite3_value_blob(argv[0]); + + assert( p!=0 ); + if( p->nSample<=n ) return; + switch( argc ){ + case 2: sqlite3_result_int64(context, p->a[n].iRowid); break; + case 3: sqlite3_result_int64(context, p->a[n].nEq); break; + case 4: sqlite3_result_int64(context, p->a[n].nLt); break; + } +} +static const FuncDef stat3GetFuncdef = { + -1, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + stat3Get, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat3_get", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ +}; +#endif /* SQLITE_ENABLE_STAT3 */ + + + + /* ** Generate code to do an analysis of all indices associated with ** a single table. @@ -234,23 +439,23 @@ static void analyzeOneTable( int regIdxname = iMem++; /* Register containing index name */ int regStat1 = iMem++; /* The stat column of sqlite_stat1 */ #ifdef SQLITE_ENABLE_STAT3 - int regNumEq = iMem-1; /* Number of instances. Same as regStat1 */ + int regNumEq = regStat1; /* Number of instances. Same as regStat1 */ int regNumLt = iMem++; /* Number of keys less than regSample */ int regSample = iMem++; /* The next sample value */ - int regNext = iMem++; /* Index of next sample to record */ - int regSpacing = iMem++; /* Spacing between samples */ - int regBigSize = iMem++; /* Always save entries with nEq >= this */ - int regTemp1 = iMem++; /* Intermediate register */ + int regRowid = regSample; /* Rowid of a sample */ + int regAccum = iMem++; /* Register to hold Stat3Accum object */ + int regLoop = iMem++; /* Loop counter */ int regCount = iMem++; /* Number of rows in the table or index */ - int regGosub = iMem++; /* Register holding subroutine return addr */ + int regTemp1 = iMem++; /* Intermediate register */ + int regTemp2 = iMem++; /* Intermediate register */ int once = 1; /* One-time initialization */ int shortJump = 0; /* Instruction address */ - int addrStoreStat3 = 0; /* Address of subroutine to wrote to stat3 */ + int iTabCur = pParse->nTab++; /* Table cursor */ #endif int regCol = iMem++; /* Content of a column in analyzed table */ int regRec = iMem++; /* Register holding completed record */ int regTemp = iMem++; /* Temporary use register */ - int regRowid = iMem++; /* Rowid for the inserted record */ + int regNewRowid = iMem++; /* Rowid for the inserted record */ v = sqlite3GetVdbe(pParse); @@ -307,41 +512,17 @@ static void analyzeOneTable( sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, pIdx->zName, 0); #ifdef SQLITE_ENABLE_STAT3 - - /* If this iteration of the loop is generating code to analyze the - ** first index in the pTab->pIndex list, then register regLast has - ** not been populated. In this case populate it now. */ if( once ){ once = 0; - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regCount); - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_STAT3_SAMPLES, regTemp1); - sqlite3VdbeAddOp3(v, OP_Divide, regTemp1, regCount, regSpacing); - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_STAT3_SAMPLES/2, regTemp1); - sqlite3VdbeAddOp3(v, OP_Divide, regTemp1, regCount, regBigSize); - - /* Generate code for a subroutine that store the most recent sample - ** in the sqlite_stat3 table - */ - shortJump = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 5, regRec, "bbbbb", 0); - VdbeComment((v, "begin stat3 write subroutine")); - sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regRec, regRowid); - sqlite3VdbeAddOp3(v, OP_Add, regNext, regSpacing, regNext); - sqlite3VdbeAddOp1(v, OP_Return, regGosub); - addrStoreStat3 = - sqlite3VdbeAddOp3(v, OP_Ge, regBigSize, shortJump+1, regNumEq); - sqlite3VdbeAddOp3(v, OP_Add, regNumEq, regNumLt, regTemp1); - sqlite3VdbeAddOp3(v, OP_Ge, regNext, shortJump+1, regTemp1); - sqlite3VdbeAddOp1(v, OP_Return, regGosub); - VdbeComment((v, "end stat3 write subroutine")); - sqlite3VdbeJumpHere(v, shortJump); + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); } - /* Reset state registers */ - sqlite3VdbeAddOp2(v, OP_Copy, regSpacing, regNext); + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regCount); + sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_STAT3_SAMPLES, regTemp1); sqlite3VdbeAddOp2(v, OP_Integer, 0, regNumEq); sqlite3VdbeAddOp2(v, OP_Integer, 0, regNumLt); - + sqlite3VdbeAddOp4(v, OP_Function, 1, regCount, regAccum, + (char*)&stat3InitFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 2); #endif /* SQLITE_ENABLE_STAT3 */ /* The block of memory cells initialized here is used as follows. @@ -389,7 +570,7 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); VdbeComment((v, "jump if column %d changed", i)); #ifdef SQLITE_ENABLE_STAT3 - if( i==0 && addrStoreStat3 ){ + if( i==0 ){ sqlite3VdbeAddOp2(v, OP_AddImm, regNumEq, 1); VdbeComment((v, "incr repeat count")); } @@ -401,8 +582,10 @@ static void analyzeOneTable( if( i==0 ){ sqlite3VdbeJumpHere(v, addrIfNot); /* Jump dest for OP_IfNot */ #ifdef SQLITE_ENABLE_STAT3 - sqlite3VdbeAddOp2(v, OP_Gosub, regGosub, addrStoreStat3); - sqlite3VdbeAddOp2(v, OP_Copy, regCol, regSample); + sqlite3VdbeAddOp4(v, OP_Function, 1, regNumEq, regTemp2, + (char*)&stat3PushFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 4); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, pIdx->nColumn, regRowid); sqlite3VdbeAddOp3(v, OP_Add, regNumEq, regNumLt, regNumLt); sqlite3VdbeAddOp2(v, OP_Integer, 1, regNumEq); #endif @@ -418,7 +601,30 @@ static void analyzeOneTable( sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, topOfLoop); sqlite3VdbeAddOp1(v, OP_Close, iIdxCur); #ifdef SQLITE_ENABLE_STAT3 - sqlite3VdbeAddOp2(v, OP_Gosub, regGosub, addrStoreStat3); + sqlite3VdbeAddOp4(v, OP_Function, 1, regNumEq, regTemp2, + (char*)&stat3PushFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 4); + sqlite3VdbeAddOp2(v, OP_Integer, -1, regLoop); + shortJump = + sqlite3VdbeAddOp2(v, OP_AddImm, regLoop, 1); + sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regTemp1, + (char*)&stat3GetFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 2); + sqlite3VdbeAddOp1(v, OP_IsNull, regTemp1); + sqlite3VdbeAddOp3(v, OP_NotExists, iTabCur, shortJump, regTemp1); + sqlite3VdbeAddOp3(v, OP_Column, iTabCur, pIdx->aiColumn[0], regSample); + sqlite3ColumnDefault(v, pTab, pIdx->aiColumn[0], regSample); + sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regNumEq, + (char*)&stat3GetFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 3); + sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regNumLt, + (char*)&stat3GetFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 4); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 5, regRec, "bbbbb", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regRec, regNewRowid); + sqlite3VdbeAddOp2(v, OP_Goto, 0, shortJump); + sqlite3VdbeJumpHere(v, shortJump+2); #endif /* Store the results in sqlite_stat1. @@ -453,8 +659,8 @@ static void analyzeOneTable( sqlite3VdbeAddOp3(v, OP_Concat, regTemp, regStat1, regStat1); } sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regNewRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); } @@ -473,8 +679,8 @@ static void analyzeOneTable( } sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regNewRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); if( pParse->nMemnMem = regRec; sqlite3VdbeJumpHere(v, jZeroRows); @@ -504,7 +710,7 @@ static void analyzeDatabase(Parse *pParse, int iDb){ sqlite3BeginWriteOperation(pParse, 0, iDb); iStatCur = pParse->nTab; - pParse->nTab += 2; + pParse->nTab += 3; openStatTable(pParse, iDb, iStatCur, 0, 0); iMem = pParse->nMem+1; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); @@ -529,7 +735,7 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); sqlite3BeginWriteOperation(pParse, 0, iDb); iStatCur = pParse->nTab; - pParse->nTab += 2; + pParse->nTab += 3; if( pOnlyIdx ){ openStatTable(pParse, iDb, iStatCur, pOnlyIdx->zName, "idx"); }else{