-C Fix\san\soff-by-one\serror\sin\sthe\sWhereCost\sto\sinteger\sconversion.
-D 2013-06-13T14:51:53.839
+C Make\sthe\sMIN()\sand\sMAX()\smacros\savailable\sin\ssqliteInt.h.\s\sAdd\sTUNING\ncomments\sto\sthe\sNGQP\sand\sadjust\scosts\sslightly.
+D 2013-06-13T15:16:53.047
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F src/analyze.c 27e541b9b5b48b41eb899b22a49ff42384899151
F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704
F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34
-F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03
+F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165
F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb
F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
F src/btree.c 7fba377c29573adfc6091832e27ee1fcbefb51d0
F src/mem2.c e307323e86b5da1853d7111b68fd6b84ad6f09cf
F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534
F src/mem5.c c2c63b7067570b00bf33d751c39af24182316f7f
-F src/memjournal.c 41a598445c8f249bd9b26ecae700c60dcf34cbf3
+F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785
F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc
F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea
F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553
F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f
F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04
F src/os_unix.c 42c9b8b7c61c9fa3561258f523be5749e52ed0e0
-F src/os_win.c 5f018dbd4cec25c5b47e11432b946a7d2ccee60b
+F src/os_win.c 074cb2b9bca6a1c2bd72acf04666cdc554bfaa9b
F src/pager.c 79df56da9dd49aceaa4cac207484a9a82cba40ae
F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1
F src/parse.y 9acfcc83ddbf0cf82f0ed9582ccf0ad6c366ff37
F src/sqlite.h.in 5b390ca5d94e09e56e7fee6a51ddde4721b89f8e
F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0
F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5
-F src/sqliteInt.h 026a52f3e24c15603fddb78025e786b96c4c0d2c
+F src/sqliteInt.h 82a0f3e7c3410cc748c80268e0831dba5ea98cb4
F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9
F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d
F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73
-F src/where.c c6944d98be4dfd0f080c2e205909ca6c25c21224
+F src/where.c 2e8e0a18dcef3a56a5874a33ba5ec4339f6911ce
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
F test/auth2.test a2a371aa6df15f8b0c8109b33d3d7f0f73e4c9aa
F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5
F test/autoinc.test bd30d372d00045252f6c2e41b5f41455e1975acf
-F test/autoindex1.test fac4207316e4e954abd086eafdd571ef093c5f5d
+F test/autoindex1.test 6a22a566b2dcbc968a62f8282af81574055d1bbf
F test/autovacuum.test 9f22a7733f39c56ef6a5665d10145ac25d8cb574
F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4
F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85
F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae
F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P e120c558a5bafc0f0d2cc12ee5c9d36e20cc642d
-R 70bdc5e3664356410e482d1133202557
+P b5ca80d924f8c6d31f036247ba6e20d234f4482e
+R 224bd6bce7ee1e3846617c8c2c148fe1
U drh
-Z 22748fc276d8e873b764764d1effa482
+Z df43f94251ce9f1e894782fd34764619
** 10*log2(X) exactly. Instead, a close estimate is used. Any value of
** X<=1 is stored as 0. X=2 is 10. X=3 is 16. X=1000 is 99. etc.
**
+** The tool/wherecosttest.c source file implements a command-line program
+** that will convert between WhereCost to integers and do addition and
+** multiplication on WhereCost values. That command-line program is a
+** useful utility to have around when working with this module.
*/
typedef unsigned short int WhereCost;
** Convert an integer into a WhereCost. In other words, compute a
** good approximatation for 10*log2(x).
*/
-static WhereCost whereCostFromInt(tRowcnt x){
+static WhereCost whereCost(tRowcnt x){
static WhereCost a[] = { 0, 2, 3, 5, 6, 7, 8, 9 };
WhereCost y = 40;
if( x<8 ){
WhereCost e;
assert( sizeof(x)==8 && sizeof(a)==8 );
if( x<=1 ) return 0;
- if( x<=2000000000 ) return whereCostFromInt((tRowcnt)x);
+ if( x<=2000000000 ) return whereCost((tRowcnt)x);
memcpy(&a, &x, 8);
e = (a>>52) - 1022;
return e*10;
** Estimate the logarithm of the input value to base 2.
*/
static WhereCost estLog(WhereCost N){
- WhereCost x = whereCostFromInt(N);
+ WhereCost x = whereCost(N);
return x>33 ? x - 33 : 0;
}
sqlite3ValueFree(pRangeVal);
}
if( rc==SQLITE_OK ){
- WhereCost iBase = whereCostFromInt(p->aiRowEst[0]);
+ WhereCost iBase = whereCost(p->aiRowEst[0]);
if( iUpper>iLower ){
- iBase -= whereCostFromInt(iUpper - iLower);
+ iBase -= whereCost(iUpper - iLower);
}
*pRangeDiv = iBase;
WHERETRACE(0x100, ("range scan regions: %u..%u div=%d\n",
#endif
assert( pLower || pUpper );
*pRangeDiv = 0;
+ /* TUNING: Each inequality constraint reduces the search space 4-fold.
+ ** A BETWEEN operator, therefore, reduces the search space 16-fold */
if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ){
- *pRangeDiv += 20; assert( 20==whereCostFromInt(4) );
+ *pRangeDiv += 20; assert( 20==whereCost(4) );
}
if( pUpper ){
- *pRangeDiv += 20; assert( 20==whereCostFromInt(4) );
+ *pRangeDiv += 20; assert( 20==whereCost(4) );
}
return rc;
}
if( pNew->u.btree.nEq < pProbe->nColumn ){
iCol = pProbe->aiColumn[pNew->u.btree.nEq];
- nRowEst = whereCostFromInt(pProbe->aiRowEst[pNew->u.btree.nEq+1]);
+ nRowEst = whereCost(pProbe->aiRowEst[pNew->u.btree.nEq+1]);
}else{
iCol = -1;
nRowEst = 0;
saved_prereq = pNew->prereq;
saved_nOut = pNew->nOut;
pNew->rSetup = 0;
- rLogSize = estLog(whereCostFromInt(pProbe->aiRowEst[0]));
+ rLogSize = estLog(whereCost(pProbe->aiRowEst[0]));
for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){
int nIn = 0;
if( pTerm->prereqRight & pNew->maskSelf ) continue;
if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
pNew->aLTerm[pNew->nLTerm++] = pTerm;
pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf;
- pNew->rRun = rLogSize;
+ pNew->rRun = rLogSize; /* Baseline cost is log2(N). Adjustments below */
if( pTerm->eOperator & WO_IN ){
Expr *pExpr = pTerm->pExpr;
pNew->wsFlags |= WHERE_COLUMN_IN;
if( ExprHasProperty(pExpr, EP_xIsSelect) ){
- /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */
- nIn = 46; assert( 46==whereCostFromInt(25) );
+ /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */
+ nIn = 46; assert( 46==whereCost(25) );
}else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
/* "x IN (value, value, ...)" */
- nIn = whereCostFromInt(pExpr->x.pList->nExpr);
+ nIn = whereCost(pExpr->x.pList->nExpr);
}
pNew->rRun += nIn;
pNew->u.btree.nEq++;
}else if( pTerm->eOperator & (WO_ISNULL) ){
pNew->wsFlags |= WHERE_COLUMN_NULL;
pNew->u.btree.nEq++;
- nIn = 10; assert( 10==whereCostFromInt(2) );
+ /* TUNING: IS NULL selects 2 rows */
+ nIn = 10; assert( 10==whereCost(2) );
pNew->nOut = nRowEst + nInMul + nIn;
}else if( pTerm->eOperator & (WO_GT|WO_GE) ){
pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
&& !ExprHasProperty(pTerm->pExpr, EP_xIsSelect) ){
rc = whereInScanEst(pParse, pProbe, pTerm->pExpr->x.pList, &nOut);
}
- pNew->nOut = whereCostFromInt(nOut);
+ pNew->nOut = whereCost(nOut);
}
#endif
if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){
}
pProbe = &sPk;
}
- rSize = whereCostFromInt(pSrc->pTab->nRowEst);
+ rSize = whereCost(pSrc->pTab->nRowEst);
rLogSize = estLog(rSize);
/* Automatic indexes */
pNew->u.btree.pIndex = 0;
pNew->nLTerm = 1;
pNew->aLTerm[0] = pTerm;
- assert( 43==whereCostFromInt(20) );
- pNew->rSetup = 43 + rLogSize + rSize;
- pNew->nOut = 33; assert( 33==whereCostFromInt(10) );
+ /* TUNING: One-time cost for computing the automatic index is
+ ** approximately 6*N*log2(N) where N is the number of rows in
+ ** the table being indexed. */
+ pNew->rSetup = rLogSize + rSize + 26; assert( 26==whereCost(6) );
+ /* TUNING: Each index lookup yields 10 rows in the table */
+ pNew->nOut = 33; assert( 33==whereCost(10) );
pNew->rRun = whereCostAdd(rLogSize,pNew->nOut);
pNew->wsFlags = WHERE_TEMP_INDEX;
pNew->prereq = mExtra | pTerm->prereqRight;
/* Full table scan */
pNew->iSortIdx = b ? iSortIdx : 0;
pNew->nOut = rSize;
- pNew->rRun = whereCostAdd(rSize,rLogSize) + 16 - b;
+ /* TUNING: Cost of full table scan is 3*(N + log2(N)).
+ ** + The extra 3 factor is to encourage the use of indexed lookups
+ ** over full scans. A smaller constant 2 is used for covering
+ ** index scans so that a covering index scan will be favored over
+ ** a table scan. */
+ pNew->rRun = whereCostAdd(rSize,rLogSize) + 16;
rc = whereLoopInsert(pBuilder, pNew);
if( rc ) break;
}else{
){
pNew->iSortIdx = b ? iSortIdx : 0;
pNew->nOut = rSize;
- pNew->rRun = whereCostAdd(rSize,rLogSize);
- pNew->rRun += ((m!=0) ? rLogSize : 10) - b;
+ if( m==0 ){
+ /* TUNING: Cost of a covering index scan is 2*(N + log2(N)).
+ ** + The extra 2 factor is to encourage the use of indexed lookups
+ ** over index scans. A table scan uses a factor of 3 so that
+ ** index scans are favored over table scans.
+ ** + If this covering index might also help satisfy the ORDER BY
+ ** clause, then the cost is fudged down slightly so that this
+ ** index is favored above other indices that have no hope of
+ ** helping with the ORDER BY. */
+ pNew->rRun = 10 + whereCostAdd(rSize,rLogSize) - b;
+ }else{
+ assert( b!=0 );
+ /* TUNING: Cost of scanning a non-covering index is (N+1)*log2(N)
+ ** which we will simplify to just N*log2(N) */
+ pNew->rRun = rSize + rLogSize;
+ }
rc = whereLoopInsert(pBuilder, pNew);
if( rc ) break;
}
&& pIdxInfo->orderByConsumed);
pNew->rSetup = 0;
pNew->rRun = whereCostFromDouble(pIdxInfo->estimatedCost);
- pNew->nOut = 46; assert( 46 == whereCostFromInt(25) );
+ /* TUNING: Every virtual table query returns 25 rows */
+ pNew->nOut = 46; assert( 46==whereCost(25) );
whereLoopInsert(pBuilder, pNew);
if( pNew->u.vtab.needFree ){
sqlite3_free(pNew->u.vtab.idxStr);
static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){
int mxChoice; /* Maximum number of simultaneous paths tracked */
int nLoop; /* Number of terms in the join */
+ Parse *pParse; /* Parsing context */
sqlite3 *db; /* The database connection */
int iLoop; /* Loop counter over the terms of the join */
int ii, jj; /* Loop counters */
WhereLoop **pX; /* Used to divy up the pSpace memory */
char *pSpace; /* Temporary memory used by this routine */
- db = pWInfo->pParse->db;
+ pParse = pWInfo->pParse;
+ db = pParse->db;
nLoop = pWInfo->nLevel;
+ /* TUNING: For simple queries, only the best path is tracked.
+ ** For 2-way joins, the 5 best paths are followed.
+ ** For joins of 3 or more tables, track the 10 best paths */
mxChoice = (nLoop==1) ? 1 : (nLoop==2 ? 5 : 10);
assert( nLoop<=pWInfo->pTabList->nSrc );
WHERETRACE(0x002, ("---- begin solver\n"));
pFrom->aLoop = pX;
}
- /* Seed the search with a single WherePath containing zero WhereLoops */
- aFrom[0].nRow = pWInfo->pParse->nQueryLoop;
+ /* Seed the search with a single WherePath containing zero WhereLoops.
+ **
+ ** TUNING: Do not let the number of iterations go above 25. If the cost
+ ** of computing an automatic index is not paid back within the first 25
+ ** rows, then do not use the automatic index. */
+ aFrom[0].nRow = MIN(pParse->nQueryLoop, 46); assert( 46==whereCost(25) );
nFrom = 1;
/* Precompute the cost of sorting the final result set, if the caller
if( pWInfo->pOrderBy==0 || nRowEst==0 ){
aFrom[0].isOrderedValid = 1;
}else{
- /* Compute an estimate on the cost to sort the entire result set */
+ /* TUNING: Estimated cost of sorting is N*log2(N) where N is the
+ ** number of output rows. */
rSortCost = nRowEst + estLog(nRowEst);
WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost));
}
}
if( nFrom==0 ){
- sqlite3ErrorMsg(pWInfo->pParse, "no query solution");
+ sqlite3ErrorMsg(pParse, "no query solution");
sqlite3DbFree(db, pSpace);
return SQLITE_ERROR;
}
pLoop->aLTerm[0] = pTerm;
pLoop->nLTerm = 1;
pLoop->u.btree.nEq = 1;
- pLoop->rRun = 33; /* 33 == whereCostFromInt(10) */
+ /* TUNING: Cost of a rowid lookup is 10 */
+ pLoop->rRun = 33; /* 33==whereCost(10) */
}else{
for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
if( pIdx->onError==OE_None ) continue;
pLoop->nLTerm = j;
pLoop->u.btree.nEq = j;
pLoop->u.btree.pIndex = pIdx;
- pLoop->rRun = 39; /* 39 == whereCostFromInt(15) */
+ /* TUNING: Cost of a unique index lookup is 15 */
+ pLoop->rRun = 39; /* 39==whereCost(15) */
break;
}
}