*/
#include "sqliteInt.h"
+
/*
** Trace output macros
*/
#define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */
#define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */
#define TERM_OR_OK 0x40 /* Used during OR-clause processing */
+#ifdef SQLITE_ENABLE_STAT2
+# define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */
+#else
+# define TERM_VNULL 0x00 /* Disabled if not using stat2 */
+#endif
/*
** An instance of the following structure holds all information about a
struct WhereCost {
WherePlan plan; /* The lookup strategy */
double rCost; /* Overall cost of pursuing this search strategy */
- double nRow; /* Estimated number of output rows */
Bitmask used; /* Bitmask of cursors used by this plan */
};
#define WO_ISNULL 0x080
#define WO_OR 0x100 /* Two or more OR-connected terms */
#define WO_AND 0x200 /* Two or more AND-connected terms */
+#define WO_NOOP 0x800 /* This term does not restrict search space */
#define WO_ALL 0xfff /* Mask of all possible WO_* values */
#define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */
#define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */
#define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */
#define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */
-#define WHERE_NOT_FULLSCAN 0x000f3000 /* Does not do a full table scan */
+#define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */
#define WHERE_IN_ABLE 0x000f1000 /* Able to support an IN operator */
#define WHERE_TOP_LIMIT 0x00100000 /* x<EXPR or x<=EXPR constraint */
#define WHERE_BTM_LIMIT 0x00200000 /* x>EXPR or x>=EXPR constraint */
+#define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and x<EXPR */
#define WHERE_IDX_ONLY 0x00800000 /* Use index only - omit table */
#define WHERE_ORDERBY 0x01000000 /* Output will appear in correct order */
#define WHERE_REVERSE 0x02000000 /* Scan in reverse order */
}
if( op==TK_VARIABLE ){
Vdbe *pReprepare = pParse->pReprepare;
- pVal = sqlite3VdbeGetValue(pReprepare, pRight->iColumn, SQLITE_AFF_NONE);
+ int iCol = pRight->iColumn;
+ pVal = sqlite3VdbeGetValue(pReprepare, iCol, SQLITE_AFF_NONE);
if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){
z = (char *)sqlite3_value_text(pVal);
}
- sqlite3VdbeSetVarmask(pParse->pVdbe, pRight->iColumn);
+ sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); /* IMP: R-23257-02778 */
assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER );
}else if( op==TK_STRING ){
z = pRight->u.zToken;
*ppPrefix = pPrefix;
if( op==TK_VARIABLE ){
Vdbe *v = pParse->pVdbe;
- sqlite3VdbeSetVarmask(v, pRight->iColumn);
+ sqlite3VdbeSetVarmask(v, pRight->iColumn); /* IMP: R-23257-02778 */
if( *pisComplete && pRight->u.zToken[1] ){
/* If the rhs of the LIKE expression is a variable, and the current
** value of the variable means there is no need to invoke the LIKE
}else{
sqlite3ExprListDelete(db, pList);
}
- pTerm->eOperator = 0; /* case 1 trumps case 2 */
+ pTerm->eOperator = WO_NOOP; /* case 1 trumps case 2 */
}
}
}
}
#endif /* SQLITE_OMIT_VIRTUALTABLE */
+#ifdef SQLITE_ENABLE_STAT2
+ /* When sqlite_stat2 histogram data is available an operator of the
+ ** form "x IS NOT NULL" can sometimes be evaluated more efficiently
+ ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a
+ ** virtual term of that form.
+ **
+ ** Note that the virtual term must be tagged with TERM_VNULL. This
+ ** TERM_VNULL tag will suppress the not-null check at the beginning
+ ** of the loop. Without the TERM_VNULL flag, the not-null check at
+ ** the start of the loop will prevent any results from being returned.
+ */
+ if( pExpr->op==TK_NOTNULL && pExpr->pLeft->iColumn>=0 ){
+ Expr *pNewExpr;
+ Expr *pLeft = pExpr->pLeft;
+ int idxNew;
+ WhereTerm *pNewTerm;
+
+ pNewExpr = sqlite3PExpr(pParse, TK_GT,
+ sqlite3ExprDup(db, pLeft, 0),
+ sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0);
+
+ idxNew = whereClauseInsert(pWC, pNewExpr,
+ TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL);
+ if( idxNew ){
+ pNewTerm = &pWC->a[idxNew];
+ pNewTerm->prereqRight = 0;
+ pNewTerm->leftCursor = pLeft->iTable;
+ pNewTerm->u.leftColumn = pLeft->iColumn;
+ pNewTerm->eOperator = WO_GT;
+ pNewTerm->iParent = idxTerm;
+ pTerm = &pWC->a[idxTerm];
+ pTerm->nChild = 1;
+ pTerm->wtFlags |= TERM_COPIED;
+ pNewTerm->prereqAll = pTerm->prereqAll;
+ }
+ }
+#endif /* SQLITE_ENABLE_STAT2 */
+
/* Prevent ON clause terms of a LEFT JOIN from being used to drive
** an index for tables to the left of the join.
*/
int base, /* Cursor number for the table to be sorted */
ExprList *pOrderBy, /* The ORDER BY clause */
int nEqCol, /* Number of index columns with == constraints */
+ int wsFlags, /* Index usages flags */
int *pbRev /* Set to 1 if ORDER BY is DESC */
){
int i, j; /* Loop counters */
return 1;
}
if( pIdx->onError!=OE_None && i==pIdx->nColumn
+ && (wsFlags & WHERE_COLUMN_NULL)==0
&& !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){
/* All terms of this index match some prefix of the ORDER BY clause
** and the index is UNIQUE and no terms on the tail of the ORDER BY
** clause reference other tables in a join. If this is all true then
- ** the order by clause is superfluous. */
+ ** the order by clause is superfluous. Not that if the matching
+ ** condition is IS NULL then the result is not necessarily unique
+ ** even on a UNIQUE index, so disallow those cases. */
return 1;
}
return 0;
** Required because bestIndex() is called by bestOrClauseIndex()
*/
static void bestIndex(
- Parse*, WhereClause*, struct SrcList_item*, Bitmask, ExprList*, WhereCost*);
+ Parse*, WhereClause*, struct SrcList_item*,
+ Bitmask, Bitmask, ExprList*, WhereCost*);
/*
** This routine attempts to find an scanning strategy that can be used
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
struct SrcList_item *pSrc, /* The FROM clause term to search */
- Bitmask notReady, /* Mask of cursors that are not available */
+ Bitmask notReady, /* Mask of cursors not available for indexing */
+ Bitmask notValid, /* Cursors not available for any purpose */
ExprList *pOrderBy, /* The ORDER BY clause */
WhereCost *pCost /* Lowest cost query plan */
){
WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */
WhereTerm *pTerm; /* A single term of the WHERE clause */
- /* No OR-clause optimization allowed if the NOT INDEXED clause is used */
- if( pSrc->notIndexed ){
+ /* No OR-clause optimization allowed if the INDEXED BY or NOT INDEXED clauses
+ ** are used */
+ if( pSrc->notIndexed || pSrc->pIndex!=0 ){
return;
}
));
if( pOrTerm->eOperator==WO_AND ){
WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc;
- bestIndex(pParse, pAndWC, pSrc, notReady, 0, &sTermCost);
+ bestIndex(pParse, pAndWC, pSrc, notReady, notValid, 0, &sTermCost);
}else if( pOrTerm->leftCursor==iCur ){
WhereClause tempWC;
tempWC.pParse = pWC->pParse;
tempWC.op = TK_AND;
tempWC.a = pOrTerm;
tempWC.nTerm = 1;
- bestIndex(pParse, &tempWC, pSrc, notReady, 0, &sTermCost);
+ bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost);
}else{
continue;
}
rTotal += sTermCost.rCost;
- nRow += sTermCost.nRow;
+ nRow += sTermCost.plan.nRow;
used |= sTermCost.used;
if( rTotal>=pCost->rCost ) break;
}
WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow));
if( rTotal<pCost->rCost ){
pCost->rCost = rTotal;
- pCost->nRow = nRow;
pCost->used = used;
+ pCost->plan.nRow = nRow;
pCost->plan.wsFlags = flags;
pCost->plan.u.pTerm = pTerm;
}
assert( pParse->nQueryLoop >= (double)1 );
pTable = pSrc->pTab;
- nTableRow = pTable->pIndex ? pTable->pIndex->aiRowEst[0] : 1000000;
+ nTableRow = pTable->nRowEst;
logN = estLog(nTableRow);
costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1);
if( costTempIdx>=pCost->rCost ){
pWCEnd = &pWC->a[pWC->nTerm];
for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
if( termCanDriveIndex(pTerm, pSrc, notReady) ){
- WHERETRACE(("auto-index reduces cost from %.2f to %.2f\n",
+ WHERETRACE(("auto-index reduces cost from %.1f to %.1f\n",
pCost->rCost, costTempIdx));
pCost->rCost = costTempIdx;
- pCost->nRow = logN + 1;
+ pCost->plan.nRow = logN + 1;
pCost->plan.wsFlags = WHERE_TEMP_INDEX;
pCost->used = pTerm->prereqRight;
break;
idxCols |= cMask;
pIdx->aiColumn[n] = pTerm->u.leftColumn;
pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
- pIdx->azColl[n] = pColl->zName;
+ assert( pColl!=0 || pParse->nErr>0 );
+ pIdx->azColl[n] = pColl ? pColl->zName : "BINARY";
n++;
}
}
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
struct SrcList_item *pSrc, /* The FROM clause term to search */
- Bitmask notReady, /* Mask of cursors that are not available */
+ Bitmask notReady, /* Mask of cursors not available for index */
+ Bitmask notValid, /* Cursors not valid for any purpose */
ExprList *pOrderBy, /* The order by clause */
WhereCost *pCost, /* Lowest cost query plan */
sqlite3_index_info **ppIdxInfo /* Index information passed to xBestIndex */
/* Try to find a more efficient access pattern by using multiple indexes
** to optimize an OR expression within the WHERE clause.
*/
- bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost);
+ bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
}
#endif /* SQLITE_OMIT_VIRTUALTABLE */
/*
** Argument pIdx is a pointer to an index structure that has an array of
** SQLITE_INDEX_SAMPLES evenly spaced samples of the first indexed column
-** stored in Index.aSample. The domain of values stored in said column
-** may be thought of as divided into (SQLITE_INDEX_SAMPLES+1) regions.
-** Region 0 contains all values smaller than the first sample value. Region
-** 1 contains values larger than or equal to the value of the first sample,
-** but smaller than the value of the second. And so on.
+** stored in Index.aSample. These samples divide the domain of values stored
+** the index into (SQLITE_INDEX_SAMPLES+1) regions.
+** Region 0 contains all values less than the first sample value. Region
+** 1 contains values between the first and second samples. Region 2 contains
+** values between samples 2 and 3. And so on. Region SQLITE_INDEX_SAMPLES
+** contains values larger than the last sample.
+**
+** If the index contains many duplicates of a single value, then it is
+** possible that two or more adjacent samples can hold the same value.
+** When that is the case, the smallest possible region code is returned
+** when roundUp is false and the largest possible region code is returned
+** when roundUp is true.
**
** If successful, this function determines which of the regions value
** pVal lies in, sets *piRegion to the region index (a value between 0
Parse *pParse, /* Database connection */
Index *pIdx, /* Index to consider domain of */
sqlite3_value *pVal, /* Value to consider */
+ int roundUp, /* Return largest valid region if true */
int *piRegion /* OUT: Region of domain in which value lies */
){
+ assert( roundUp==0 || roundUp==1 );
if( ALWAYS(pVal) ){
IndexSample *aSample = pIdx->aSample;
int i = 0;
double r = sqlite3_value_double(pVal);
for(i=0; i<SQLITE_INDEX_SAMPLES; i++){
if( aSample[i].eType==SQLITE_NULL ) continue;
- if( aSample[i].eType>=SQLITE_TEXT || aSample[i].u.r>r ) break;
+ if( aSample[i].eType>=SQLITE_TEXT ) break;
+ if( roundUp ){
+ if( aSample[i].u.r>r ) break;
+ }else{
+ if( aSample[i].u.r>=r ) break;
+ }
+ }
+ }else if( eType==SQLITE_NULL ){
+ i = 0;
+ if( roundUp ){
+ while( i<SQLITE_INDEX_SAMPLES && aSample[i].eType==SQLITE_NULL ) i++;
}
}else{
sqlite3 *db = pParse->db;
n = sqlite3ValueBytes(pVal, pColl->enc);
for(i=0; i<SQLITE_INDEX_SAMPLES; i++){
- int r;
+ int c;
int eSampletype = aSample[i].eType;
if( eSampletype==SQLITE_NULL || eSampletype<eType ) continue;
if( (eSampletype!=eType) ) break;
assert( db->mallocFailed );
return SQLITE_NOMEM;
}
- r = pColl->xCmp(pColl->pUser, nSample, zSample, n, z);
+ c = pColl->xCmp(pColl->pUser, nSample, zSample, n, z);
sqlite3DbFree(db, zSample);
}else
#endif
{
- r = pColl->xCmp(pColl->pUser, aSample[i].nByte, aSample[i].u.z, n, z);
+ c = pColl->xCmp(pColl->pUser, aSample[i].nByte, aSample[i].u.z, n, z);
}
- if( r>0 ) break;
+ if( c-roundUp>=0 ) break;
}
}
u8 aff,
sqlite3_value **pp
){
- /* The evalConstExpr() function will have already converted any TK_VARIABLE
- ** expression involved in an comparison into a TK_REGISTER. */
- assert( pExpr->op!=TK_VARIABLE );
- if( pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE ){
+ if( pExpr->op==TK_VARIABLE
+ || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE)
+ ){
int iVar = pExpr->iColumn;
- sqlite3VdbeSetVarmask(pParse->pVdbe, iVar);
+ sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); /* IMP: R-23257-02778 */
*pp = sqlite3VdbeGetValue(pParse->pReprepare, iVar, aff);
return SQLITE_OK;
}
** constraints.
**
** In the absence of sqlite_stat2 ANALYZE data, each range inequality
-** reduces the search space by 2/3rds. Hence a single constraint (x>?)
-** results in a return of 33 and a range constraint (x>? AND x<?) results
-** in a return of 11.
+** reduces the search space by 3/4ths. Hence a single constraint (x>?)
+** results in a return of 25 and a range constraint (x>? AND x<?) results
+** in a return of 6.
*/
static int whereRangeScanEst(
Parse *pParse, /* Parsing & code generating context */
int iEst;
int iLower = 0;
int iUpper = SQLITE_INDEX_SAMPLES;
+ int roundUpUpper;
+ int roundUpLower;
u8 aff = p->pTable->aCol[p->aiColumn[0]].affinity;
if( pLower ){
Expr *pExpr = pLower->pExpr->pRight;
rc = valueFromExpr(pParse, pExpr, aff, &pLowerVal);
+ assert( pLower->eOperator==WO_GT || pLower->eOperator==WO_GE );
+ roundUpLower = (pLower->eOperator==WO_GT) ?1:0;
}
if( rc==SQLITE_OK && pUpper ){
Expr *pExpr = pUpper->pExpr->pRight;
rc = valueFromExpr(pParse, pExpr, aff, &pUpperVal);
+ assert( pUpper->eOperator==WO_LT || pUpper->eOperator==WO_LE );
+ roundUpUpper = (pUpper->eOperator==WO_LE) ?1:0;
}
if( rc!=SQLITE_OK || (pLowerVal==0 && pUpperVal==0) ){
sqlite3ValueFree(pUpperVal);
goto range_est_fallback;
}else if( pLowerVal==0 ){
- rc = whereRangeRegion(pParse, p, pUpperVal, &iUpper);
+ rc = whereRangeRegion(pParse, p, pUpperVal, roundUpUpper, &iUpper);
if( pLower ) iLower = iUpper/2;
}else if( pUpperVal==0 ){
- rc = whereRangeRegion(pParse, p, pLowerVal, &iLower);
+ rc = whereRangeRegion(pParse, p, pLowerVal, roundUpLower, &iLower);
if( pUpper ) iUpper = (iLower + SQLITE_INDEX_SAMPLES + 1)/2;
}else{
- rc = whereRangeRegion(pParse, p, pUpperVal, &iUpper);
+ rc = whereRangeRegion(pParse, p, pUpperVal, roundUpUpper, &iUpper);
if( rc==SQLITE_OK ){
- rc = whereRangeRegion(pParse, p, pLowerVal, &iLower);
+ rc = whereRangeRegion(pParse, p, pLowerVal, roundUpLower, &iLower);
}
}
+ WHERETRACE(("range scan regions: %d..%d\n", iLower, iUpper));
iEst = iUpper - iLower;
testcase( iEst==SQLITE_INDEX_SAMPLES );
assert( iEst<=SQLITE_INDEX_SAMPLES );
if( iEst<1 ){
- iEst = 1;
+ *piEst = 50/SQLITE_INDEX_SAMPLES;
+ }else{
+ *piEst = (iEst*100)/SQLITE_INDEX_SAMPLES;
}
-
sqlite3ValueFree(pLowerVal);
sqlite3ValueFree(pUpperVal);
- *piEst = (iEst * 100)/SQLITE_INDEX_SAMPLES;
return rc;
}
range_est_fallback:
UNUSED_PARAMETER(nEq);
#endif
assert( pLower || pUpper );
- if( pLower && pUpper ){
- *piEst = 11;
+ *piEst = 100;
+ if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ) *piEst /= 4;
+ if( pUpper ) *piEst /= 4;
+ return rc;
+}
+
+#ifdef SQLITE_ENABLE_STAT2
+/*
+** Estimate the number of rows that will be returned based on
+** an equality constraint x=VALUE and where that VALUE occurs in
+** the histogram data. This only works when x is the left-most
+** column of an index and sqlite_stat2 histogram data is available
+** for that index.
+**
+** Write the estimated row count into *pnRow and return SQLITE_OK.
+** If unable to make an estimate, leave *pnRow unchanged and return
+** non-zero.
+**
+** This routine can fail if it is unable to load a collating sequence
+** required for string comparison, or if unable to allocate memory
+** for a UTF conversion required for comparison. The error is stored
+** in the pParse structure.
+*/
+int whereEqualScanEst(
+ Parse *pParse, /* Parsing & code generating context */
+ Index *p, /* The index whose left-most column is pTerm */
+ Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */
+ double *pnRow /* Write the revised row estimate here */
+){
+ sqlite3_value *pRhs = 0; /* VALUE on right-hand side of pTerm */
+ int iLower, iUpper; /* Range of histogram regions containing pRhs */
+ u8 aff; /* Column affinity */
+ int rc; /* Subfunction return code */
+ double nRowEst; /* New estimate of the number of rows */
+
+ assert( p->aSample!=0 );
+ aff = p->pTable->aCol[p->aiColumn[0]].affinity;
+ rc = valueFromExpr(pParse, pExpr, aff, &pRhs);
+ if( rc ) goto whereEqualScanEst_cancel;
+ if( pRhs==0 ) return SQLITE_NOTFOUND;
+ rc = whereRangeRegion(pParse, p, pRhs, 0, &iLower);
+ if( rc ) goto whereEqualScanEst_cancel;
+ rc = whereRangeRegion(pParse, p, pRhs, 1, &iUpper);
+ if( rc ) goto whereEqualScanEst_cancel;
+ WHERETRACE(("equality scan regions: %d..%d\n", iLower, iUpper));
+ if( iLower>=iUpper ){
+ nRowEst = p->aiRowEst[0]/(SQLITE_INDEX_SAMPLES*2);
+ if( nRowEst<*pnRow ) *pnRow = nRowEst;
}else{
- *piEst = 33;
+ nRowEst = (iUpper-iLower)*p->aiRowEst[0]/SQLITE_INDEX_SAMPLES;
+ *pnRow = nRowEst;
+ }
+
+whereEqualScanEst_cancel:
+ sqlite3ValueFree(pRhs);
+ return rc;
+}
+#endif /* defined(SQLITE_ENABLE_STAT2) */
+
+#ifdef SQLITE_ENABLE_STAT2
+/*
+** Estimate the number of rows that will be returned based on
+** an IN constraint where the right-hand side of the IN operator
+** is a list of values. Example:
+**
+** WHERE x IN (1,2,3,4)
+**
+** Write the estimated row count into *pnRow and return SQLITE_OK.
+** If unable to make an estimate, leave *pnRow unchanged and return
+** non-zero.
+**
+** This routine can fail if it is unable to load a collating sequence
+** required for string comparison, or if unable to allocate memory
+** for a UTF conversion required for comparison. The error is stored
+** in the pParse structure.
+*/
+int whereInScanEst(
+ Parse *pParse, /* Parsing & code generating context */
+ Index *p, /* The index whose left-most column is pTerm */
+ ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */
+ double *pnRow /* Write the revised row estimate here */
+){
+ sqlite3_value *pVal = 0; /* One value from list */
+ int iLower, iUpper; /* Range of histogram regions containing pRhs */
+ u8 aff; /* Column affinity */
+ int rc = SQLITE_OK; /* Subfunction return code */
+ double nRowEst; /* New estimate of the number of rows */
+ int nSpan = 0; /* Number of histogram regions spanned */
+ int nSingle = 0; /* Histogram regions hit by a single value */
+ int nNotFound = 0; /* Count of values that are not constants */
+ int i; /* Loop counter */
+ u8 aSpan[SQLITE_INDEX_SAMPLES+1]; /* Histogram regions that are spanned */
+ u8 aSingle[SQLITE_INDEX_SAMPLES+1]; /* Histogram regions hit once */
+
+ assert( p->aSample!=0 );
+ aff = p->pTable->aCol[p->aiColumn[0]].affinity;
+ memset(aSpan, 0, sizeof(aSpan));
+ memset(aSingle, 0, sizeof(aSingle));
+ for(i=0; i<pList->nExpr; i++){
+ sqlite3ValueFree(pVal);
+ rc = valueFromExpr(pParse, pList->a[i].pExpr, aff, &pVal);
+ if( rc ) break;
+ if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){
+ nNotFound++;
+ continue;
+ }
+ rc = whereRangeRegion(pParse, p, pVal, 0, &iLower);
+ if( rc ) break;
+ rc = whereRangeRegion(pParse, p, pVal, 1, &iUpper);
+ if( rc ) break;
+ if( iLower>=iUpper ){
+ aSingle[iLower] = 1;
+ }else{
+ assert( iLower>=0 && iUpper<=SQLITE_INDEX_SAMPLES );
+ while( iLower<iUpper ) aSpan[iLower++] = 1;
+ }
+ }
+ if( rc==SQLITE_OK ){
+ for(i=nSpan=0; i<=SQLITE_INDEX_SAMPLES; i++){
+ if( aSpan[i] ){
+ nSpan++;
+ }else if( aSingle[i] ){
+ nSingle++;
+ }
+ }
+ nRowEst = (nSpan*2+nSingle)*p->aiRowEst[0]/(2*SQLITE_INDEX_SAMPLES)
+ + nNotFound*p->aiRowEst[1];
+ if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0];
+ *pnRow = nRowEst;
+ WHERETRACE(("IN row estimate: nSpan=%d, nSingle=%d, nNotFound=%d, est=%g\n",
+ nSpan, nSingle, nNotFound, nRowEst));
}
+ sqlite3ValueFree(pVal);
return rc;
}
+#endif /* defined(SQLITE_ENABLE_STAT2) */
/*
-** Find the query plan for accessing a particular table. Write the
+** Find the best query plan for accessing a particular table. Write the
** best query plan and its cost into the WhereCost object supplied as the
** last parameter.
**
** The lowest cost plan wins. The cost is an estimate of the amount of
-** CPU and disk I/O need to process the request using the selected plan.
+** CPU and disk I/O needed to process the requested result.
** Factors that influence cost include:
**
** * The estimated number of rows that will be retrieved. (The
**
** If a NOT INDEXED clause (pSrc->notIndexed!=0) was attached to the table
** in the SELECT statement, then no indexes are considered. However, the
-** selected plan may still take advantage of the tables built-in rowid
+** selected plan may still take advantage of the built-in rowid primary key
** index.
*/
static void bestBtreeIndex(
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
struct SrcList_item *pSrc, /* The FROM clause term to search */
- Bitmask notReady, /* Mask of cursors that are not available */
+ Bitmask notReady, /* Mask of cursors not available for indexing */
+ Bitmask notValid, /* Cursors not available for any purpose */
ExprList *pOrderBy, /* The ORDER BY clause */
WhereCost *pCost /* Lowest cost query plan */
){
wsFlagMask = ~(WHERE_ROWID_EQ|WHERE_ROWID_RANGE);
eqTermMask = idxEqTermMask;
}else{
- /* There is no INDEXED BY clause. Create a fake Index object to
- ** represent the primary key */
- Index *pFirst; /* Any other index on the table */
+ /* There is no INDEXED BY clause. Create a fake Index object in local
+ ** variable sPk to represent the rowid primary key index. Make this
+ ** fake index the first in a chain of Index objects with all of the real
+ ** indices to follow */
+ Index *pFirst; /* First of real indices on the table */
memset(&sPk, 0, sizeof(Index));
sPk.nColumn = 1;
sPk.aiColumn = &aiColumnPk;
sPk.aiRowEst = aiRowEstPk;
- aiRowEstPk[1] = 1;
sPk.onError = OE_Replace;
sPk.pTable = pSrc->pTab;
+ aiRowEstPk[0] = pSrc->pTab->nRowEst;
+ aiRowEstPk[1] = 1;
pFirst = pSrc->pTab->pIndex;
if( pSrc->notIndexed==0 ){
+ /* The real indices of the table are only considered if the
+ ** NOT INDEXED qualifier is omitted from the FROM clause */
sPk.pNext = pFirst;
}
- /* The aiRowEstPk[0] is an estimate of the total number of rows in the
- ** table. Get this information from the ANALYZE information if it is
- ** available. If not available, assume the table 1 million rows in size.
- */
- if( pFirst ){
- assert( pFirst->aiRowEst!=0 ); /* Allocated together with pFirst */
- aiRowEstPk[0] = pFirst->aiRowEst[0];
- }else{
- aiRowEstPk[0] = 1000000;
- }
pProbe = &sPk;
wsFlagMask = ~(
WHERE_COLUMN_IN|WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_RANGE
const unsigned int * const aiRowEst = pProbe->aiRowEst;
double cost; /* Cost of using pProbe */
double nRow; /* Estimated number of rows in result set */
+ double log10N; /* base-10 logarithm of nRow (inexact) */
int rev; /* True to scan in reverse order */
int wsFlags = 0;
Bitmask used = 0;
/* The following variables are populated based on the properties of
- ** scan being evaluated. They are then used to determine the expected
+ ** index being evaluated. They are then used to determine the expected
** cost and number of rows returned.
**
** nEq:
** Number of equality terms that can be implemented using the index.
+ ** In other words, the number of initial fields in the index that
+ ** are used in == or IN or NOT NULL constraints of the WHERE clause.
**
** nInMul:
** The "in-multiplier". This is an estimate of how many seek operations
**
** bInEst:
** Set to true if there was at least one "x IN (SELECT ...)" term used
- ** in determining the value of nInMul.
+ ** in determining the value of nInMul. Note that the RHS of the
+ ** IN operator must be a SELECT, not a value list, for this variable
+ ** to be true.
**
** estBound:
** An estimate on the amount of the table that must be searched. A
** might reduce this to a value less than 100 to indicate that only
** a fraction of the table needs searching. In the absence of
** sqlite_stat2 ANALYZE data, a single inequality reduces the search
- ** space to 1/3rd its original size. So an x>? constraint reduces
- ** estBound to 33. Two constraints (x>? AND x<?) reduce estBound to 11.
+ ** space to 1/4rd its original size. So an x>? constraint reduces
+ ** estBound to 25. Two constraints (x>? AND x<?) reduce estBound to 6.
**
** bSort:
** Boolean. True if there is an ORDER BY clause that will require an
** correctly order records).
**
** bLookup:
- ** Boolean. True if for each index entry visited a lookup on the
- ** corresponding table b-tree is required. This is always false
- ** for the rowid index. For other indexes, it is true unless all the
- ** columns of the table used by the SELECT statement are present in
- ** the index (such an index is sometimes described as a covering index).
+ ** Boolean. True if a table lookup is required for each index entry
+ ** visited. In other words, true if this is not a covering index.
+ ** This is always false for the rowid primary key index of a table.
+ ** For other indexes, it is true unless all the columns of the table
+ ** used by the SELECT statement are present in the index (such an
+ ** index is sometimes described as a covering index).
** For example, given the index on (a, b), the second of the following
- ** two queries requires table b-tree lookups, but the first does not.
+ ** two queries requires table b-tree lookups in order to find the value
+ ** of column c, but the first does not because columns a and b are
+ ** both available in the index.
**
** SELECT a, b FROM tbl WHERE a = 1;
** SELECT a, b, c FROM tbl WHERE a = 1;
*/
- int nEq;
- int bInEst = 0;
- int nInMul = 1;
- int estBound = 100;
- int nBound = 0; /* Number of range constraints seen */
- int bSort = 0;
- int bLookup = 0;
- WhereTerm *pTerm; /* A single term of the WHERE clause */
+ int nEq; /* Number of == or IN terms matching index */
+ int bInEst = 0; /* True if "x IN (SELECT...)" seen */
+ int nInMul = 1; /* Number of distinct equalities to lookup */
+ int estBound = 100; /* Estimated reduction in search space */
+ int nBound = 0; /* Number of range constraints seen */
+ int bSort = 0; /* True if external sort required */
+ int bLookup = 0; /* True if not a covering index */
+ WhereTerm *pTerm; /* A single term of the WHERE clause */
+#ifdef SQLITE_ENABLE_STAT2
+ WhereTerm *pFirstTerm = 0; /* First term matching the index */
+#endif
/* Determine the values of nEq and nInMul */
for(nEq=0; nEq<pProbe->nColumn; nEq++){
Expr *pExpr = pTerm->pExpr;
wsFlags |= WHERE_COLUMN_IN;
if( ExprHasProperty(pExpr, EP_xIsSelect) ){
+ /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */
nInMul *= 25;
bInEst = 1;
- }else if( ALWAYS(pExpr->x.pList) ){
- nInMul *= pExpr->x.pList->nExpr + 1;
+ }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
+ /* "x IN (value, value, ...)" */
+ nInMul *= pExpr->x.pList->nExpr;
}
}else if( pTerm->eOperator & WO_ISNULL ){
wsFlags |= WHERE_COLUMN_NULL;
}
+#ifdef SQLITE_ENABLE_STAT2
+ if( nEq==0 && pProbe->aSample ) pFirstTerm = pTerm;
+#endif
used |= pTerm->prereqRight;
}
** in wsFlags. Otherwise, if there is an ORDER BY clause but the index
** will scan rows in a different order, set the bSort variable. */
if( pOrderBy ){
- if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0
- && isSortingIndex(pParse,pWC->pMaskSet,pProbe,iCur,pOrderBy,nEq,&rev)
+ if( (wsFlags & WHERE_COLUMN_IN)==0
+ && isSortingIndex(pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy,
+ nEq, wsFlags, &rev)
){
wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY;
wsFlags |= (rev ? WHERE_REVERSE : 0);
}
/*
- ** Estimate the number of rows of output. For an IN operator,
- ** do not let the estimate exceed half the rows in the table.
+ ** Estimate the number of rows of output. For an "x IN (SELECT...)"
+ ** constraint, do not let the estimate exceed half the rows in the table.
*/
nRow = (double)(aiRowEst[nEq] * nInMul);
if( bInEst && nRow*2>aiRowEst[0] ){
nInMul = (int)(nRow / aiRowEst[nEq]);
}
- /* Assume constant cost to access a row and logarithmic cost to
- ** do a binary search. Hence, the initial cost is the number of output
- ** rows plus log2(table-size) times the number of binary searches.
+#ifdef SQLITE_ENABLE_STAT2
+ /* If the constraint is of the form x=VALUE and histogram
+ ** data is available for column x, then it might be possible
+ ** to get a better estimate on the number of rows based on
+ ** VALUE and how common that value is according to the histogram.
*/
- cost = nRow + nInMul*estLog(aiRowEst[0]);
+ if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 ){
+ if( pFirstTerm->eOperator==WO_EQ ){
+ whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow);
+ }else if( pFirstTerm->eOperator==WO_IN && bInEst==0 ){
+ whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &nRow);
+ }
+ }
+#endif /* SQLITE_ENABLE_STAT2 */
- /* Adjust the number of rows and the cost downward to reflect rows
+ /* Adjust the number of output rows and downward to reflect rows
** that are excluded by range constraints.
*/
nRow = (nRow * (double)estBound) / (double)100;
- cost = (cost * (double)estBound) / (double)100;
-
- /* Add in the estimated cost of sorting the result
+ if( nRow<1 ) nRow = 1;
+
+ /* Experiments run on real SQLite databases show that the time needed
+ ** to do a binary search to locate a row in a table or index is roughly
+ ** log10(N) times the time to move from one row to the next row within
+ ** a table or index. The actual times can vary, with the size of
+ ** records being an important factor. Both moves and searches are
+ ** slower with larger records, presumably because fewer records fit
+ ** on one page and hence more pages have to be fetched.
+ **
+ ** The ANALYZE command and the sqlite_stat1 and sqlite_stat2 tables do
+ ** not give us data on the relative sizes of table and index records.
+ ** So this computation assumes table records are about twice as big
+ ** as index records
*/
- if( bSort ){
- cost += cost*estLog(cost);
+ if( (wsFlags & WHERE_NOT_FULLSCAN)==0 ){
+ /* The cost of a full table scan is a number of move operations equal
+ ** to the number of rows in the table.
+ **
+ ** We add an additional 4x penalty to full table scans. This causes
+ ** the cost function to err on the side of choosing an index over
+ ** choosing a full scan. This 4x full-scan penalty is an arguable
+ ** decision and one which we expect to revisit in the future. But
+ ** it seems to be working well enough at the moment.
+ */
+ cost = aiRowEst[0]*4;
+ }else{
+ log10N = estLog(aiRowEst[0]);
+ cost = nRow;
+ if( pIdx ){
+ if( bLookup ){
+ /* For an index lookup followed by a table lookup:
+ ** nInMul index searches to find the start of each index range
+ ** + nRow steps through the index
+ ** + nRow table searches to lookup the table entry using the rowid
+ */
+ cost += (nInMul + nRow)*log10N;
+ }else{
+ /* For a covering index:
+ ** nInMul index searches to find the initial entry
+ ** + nRow steps through the index
+ */
+ cost += nInMul*log10N;
+ }
+ }else{
+ /* For a rowid primary key lookup:
+ ** nInMult table searches to find the initial entry for each range
+ ** + nRow steps through the table
+ */
+ cost += nInMul*log10N;
+ }
}
- /* If all information can be taken directly from the index, we avoid
- ** doing table lookups. This reduces the cost by half. (Not really -
- ** this needs to be fixed.)
+ /* Add in the estimated cost of sorting the result. Actual experimental
+ ** measurements of sorting performance in SQLite show that sorting time
+ ** adds C*N*log10(N) to the cost, where N is the number of rows to be
+ ** sorted and C is a factor between 1.95 and 4.3. We will split the
+ ** difference and select C of 3.0.
*/
- if( pIdx && bLookup==0 ){
- cost /= (double)2;
+ if( bSort ){
+ cost += nRow*estLog(nRow)*3;
}
+
/**** Cost of using this index has now been computed ****/
/* If there are additional constraints on this table that cannot
** with this step if we already know this index will not be chosen.
** Also, never reduce the output row count below 2 using this step.
**
- ** Do not reduce the output row count if pSrc is the only table that
- ** is notReady; if notReady is a power of two. This will be the case
- ** when the main sqlite3WhereBegin() loop is scanning for a table with
- ** and "optimal" index, and on such a scan the output row count
- ** reduction is not valid because it does not update the "pCost->used"
- ** bitmap. The notReady bitmap will also be a power of two when we
- ** are scanning for the last table in a 64-way join. We are willing
- ** to bypass this optimization in that corner case.
+ ** It is critical that the notValid mask be used here instead of
+ ** the notReady mask. When computing an "optimal" index, the notReady
+ ** mask will only have one bit set - the bit for the current table.
+ ** The notValid mask, on the other hand, always has all bits set for
+ ** tables that are not in outer loops. If notReady is used here instead
+ ** of notValid, then a optimal index that depends on inner joins loops
+ ** might be selected even when there exists an optimal index that has
+ ** no such dependency.
*/
- if( nRow>2 && cost<=pCost->rCost && (notReady & (notReady-1))!=0 ){
+ if( nRow>2 && cost<=pCost->rCost ){
int k; /* Loop counter */
int nSkipEq = nEq; /* Number of == constraints to skip */
int nSkipRange = nBound; /* Number of < constraints to skip */
thisTab = getMask(pWC->pMaskSet, iCur);
for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){
if( pTerm->wtFlags & TERM_VIRTUAL ) continue;
- if( (pTerm->prereqAll & notReady)!=thisTab ) continue;
+ if( (pTerm->prereqAll & notValid)!=thisTab ) continue;
if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){
if( nSkipEq ){
/* Ignore the first nEq equality matches since the index
}
}else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GT|WO_GE) ){
if( nSkipRange ){
- /* Ignore the first nBound range constraints since the index
+ /* Ignore the first nSkipRange range constraints since the index
** has already accounted for these */
nSkipRange--;
}else{
/* Assume each additional range constraint reduces the result
- ** set size by a factor of 3 */
+ ** set size by a factor of 3. Indexed range constraints reduce
+ ** the search space by a larger factor: 4. We make indexed range
+ ** more selective intentionally because of the subjective
+ ** observation that indexed range constraints really are more
+ ** selective in practice, on average. */
nRow /= 3;
}
- }else{
+ }else if( pTerm->eOperator!=WO_NOOP ){
/* Any other expression lowers the output row count by half */
nRow /= 2;
}
WHERETRACE((
"%s(%s): nEq=%d nInMul=%d estBound=%d bSort=%d bLookup=%d wsFlags=0x%x\n"
- " notReady=0x%llx nRow=%.2f cost=%.2f used=0x%llx\n",
+ " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f used=0x%llx\n",
pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"),
nEq, nInMul, estBound, bSort, bLookup, wsFlags,
- notReady, nRow, cost, used
+ notReady, log10N, nRow, cost, used
));
/* If this index is the best we have seen so far, then record this
** index and its cost in the pCost structure.
*/
if( (!pIdx || wsFlags)
- && (cost<pCost->rCost || (cost<=pCost->rCost && nRow<pCost->nRow))
+ && (cost<pCost->rCost || (cost<=pCost->rCost && nRow<pCost->plan.nRow))
){
pCost->rCost = cost;
- pCost->nRow = nRow;
pCost->used = used;
+ pCost->plan.nRow = nRow;
pCost->plan.wsFlags = (wsFlags&wsFlagMask);
pCost->plan.nEq = nEq;
pCost->plan.u.pIdx = pIdx;
pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk")
));
- bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost);
+ bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost);
pCost->plan.wsFlags |= eqTermMask;
}
Parse *pParse, /* The parsing context */
WhereClause *pWC, /* The WHERE clause */
struct SrcList_item *pSrc, /* The FROM clause term to search */
- Bitmask notReady, /* Mask of cursors that are not available */
+ Bitmask notReady, /* Mask of cursors not available for indexing */
+ Bitmask notValid, /* Cursors not available for any purpose */
ExprList *pOrderBy, /* The ORDER BY clause */
WhereCost *pCost /* Lowest cost query plan */
){
#ifndef SQLITE_OMIT_VIRTUALTABLE
if( IsVirtual(pSrc->pTab) ){
sqlite3_index_info *p = 0;
- bestVirtualIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost, &p);
+ bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost,&p);
if( p->needToFreeIdxStr ){
sqlite3_free(p->idxStr);
}
}else
#endif
{
- bestBtreeIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost);
+ bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
}
}
return regBase;
}
+#ifndef SQLITE_OMIT_EXPLAIN
+/*
+** This routine is a helper for explainIndexRange() below
+**
+** pStr holds the text of an expression that we are building up one term
+** at a time. This routine adds a new term to the end of the expression.
+** Terms are separated by AND so add the "AND" text for second and subsequent
+** terms only.
+*/
+static void explainAppendTerm(
+ StrAccum *pStr, /* The text expression being built */
+ int iTerm, /* Index of this term. First is zero */
+ const char *zColumn, /* Name of the column */
+ const char *zOp /* Name of the operator */
+){
+ if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5);
+ sqlite3StrAccumAppend(pStr, zColumn, -1);
+ sqlite3StrAccumAppend(pStr, zOp, 1);
+ sqlite3StrAccumAppend(pStr, "?", 1);
+}
+
+/*
+** Argument pLevel describes a strategy for scanning table pTab. This
+** function returns a pointer to a string buffer containing a description
+** of the subset of table rows scanned by the strategy in the form of an
+** SQL expression. Or, if all rows are scanned, NULL is returned.
+**
+** For example, if the query:
+**
+** SELECT * FROM t1 WHERE a=1 AND b>2;
+**
+** is run and there is an index on (a, b), then this function returns a
+** string similar to:
+**
+** "a=? AND b>?"
+**
+** The returned pointer points to memory obtained from sqlite3DbMalloc().
+** It is the responsibility of the caller to free the buffer when it is
+** no longer required.
+*/
+static char *explainIndexRange(sqlite3 *db, WhereLevel *pLevel, Table *pTab){
+ WherePlan *pPlan = &pLevel->plan;
+ Index *pIndex = pPlan->u.pIdx;
+ int nEq = pPlan->nEq;
+ int i, j;
+ Column *aCol = pTab->aCol;
+ int *aiColumn = pIndex->aiColumn;
+ StrAccum txt;
+
+ if( nEq==0 && (pPlan->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){
+ return 0;
+ }
+ sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH);
+ txt.db = db;
+ sqlite3StrAccumAppend(&txt, " (", 2);
+ for(i=0; i<nEq; i++){
+ explainAppendTerm(&txt, i, aCol[aiColumn[i]].zName, "=");
+ }
+
+ j = i;
+ if( pPlan->wsFlags&WHERE_BTM_LIMIT ){
+ explainAppendTerm(&txt, i++, aCol[aiColumn[j]].zName, ">");
+ }
+ if( pPlan->wsFlags&WHERE_TOP_LIMIT ){
+ explainAppendTerm(&txt, i, aCol[aiColumn[j]].zName, "<");
+ }
+ sqlite3StrAccumAppend(&txt, ")", 1);
+ return sqlite3StrAccumFinish(&txt);
+}
+
+/*
+** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN
+** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single
+** record is added to the output to describe the table scan strategy in
+** pLevel.
+*/
+static void explainOneScan(
+ Parse *pParse, /* Parse context */
+ SrcList *pTabList, /* Table list this loop refers to */
+ WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */
+ int iLevel, /* Value for "level" column of output */
+ int iFrom, /* Value for "from" column of output */
+ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */
+){
+ if( pParse->explain==2 ){
+ u32 flags = pLevel->plan.wsFlags;
+ struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom];
+ Vdbe *v = pParse->pVdbe; /* VM being constructed */
+ sqlite3 *db = pParse->db; /* Database handle */
+ char *zMsg; /* Text to add to EQP output */
+ sqlite3_int64 nRow; /* Expected number of rows visited by scan */
+ int iId = pParse->iSelectId; /* Select id (left-most output column) */
+ int isSearch; /* True for a SEARCH. False for SCAN. */
+
+ if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return;
+
+ isSearch = (pLevel->plan.nEq>0)
+ || (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0
+ || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX));
+
+ zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN");
+ if( pItem->pSelect ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId);
+ }else{
+ zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName);
+ }
+
+ if( pItem->zAlias ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias);
+ }
+ if( (flags & WHERE_INDEXED)!=0 ){
+ char *zWhere = explainIndexRange(db, pLevel, pItem->pTab);
+ zMsg = sqlite3MAppendf(db, zMsg, "%s USING %s%sINDEX%s%s%s", zMsg,
+ ((flags & WHERE_TEMP_INDEX)?"AUTOMATIC ":""),
+ ((flags & WHERE_IDX_ONLY)?"COVERING ":""),
+ ((flags & WHERE_TEMP_INDEX)?"":" "),
+ ((flags & WHERE_TEMP_INDEX)?"": pLevel->plan.u.pIdx->zName),
+ zWhere
+ );
+ sqlite3DbFree(db, zWhere);
+ }else if( flags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg);
+
+ if( flags&WHERE_ROWID_EQ ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg);
+ }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid<?)", zMsg);
+ }else if( flags&WHERE_BTM_LIMIT ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>?)", zMsg);
+ }else if( flags&WHERE_TOP_LIMIT ){
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid<?)", zMsg);
+ }
+ }
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ else if( (flags & WHERE_VIRTUALTABLE)!=0 ){
+ sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx;
+ zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg,
+ pVtabIdx->idxNum, pVtabIdx->idxStr);
+ }
+#endif
+ if( wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ){
+ testcase( wctrlFlags & WHERE_ORDERBY_MIN );
+ nRow = 1;
+ }else{
+ nRow = (sqlite3_int64)pLevel->plan.nRow;
+ }
+ zMsg = sqlite3MAppendf(db, zMsg, "%s (~%lld rows)", zMsg, nRow);
+ sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC);
+ }
+}
+#else
+# define explainOneScan(u,v,w,x,y,z)
+#endif /* SQLITE_OMIT_EXPLAIN */
+
+
/*
** Generate code for the start of the iLevel-th loop in the WHERE clause
** implementation described by pWInfo.
if( pRangeStart ){
Expr *pRight = pRangeStart->pExpr->pRight;
sqlite3ExprCode(pParse, pRight, regBase+nEq);
- sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
+ if( (pRangeStart->wtFlags & TERM_VNULL)==0 ){
+ sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
+ }
if( zStartAff ){
if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_NONE){
/* Since the comparison is to be performed with no conversions
Expr *pRight = pRangeEnd->pExpr->pRight;
sqlite3ExprCacheRemove(pParse, regBase+nEq, 1);
sqlite3ExprCode(pParse, pRight, regBase+nEq);
- sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
+ if( (pRangeEnd->wtFlags & TERM_VNULL)==0 ){
+ sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
+ }
if( zEndAff ){
if( sqlite3CompareAffinity(pRight, zEndAff[nEq])==SQLITE_AFF_NONE){
/* Since the comparison is to be performed with no conversions
r1 = sqlite3GetTempReg(pParse);
testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT );
testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT );
- if( pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT) ){
+ if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){
sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1);
sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont);
}
WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE |
WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY);
if( pSubWInfo ){
+ explainOneScan(
+ pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0
+ );
if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
int iSet = ((ii==pOrWc->nTerm-1)?-1:ii);
int r;
memset(&bestPlan, 0, sizeof(bestPlan));
bestPlan.rCost = SQLITE_BIG_DBL;
+ WHERETRACE(("*** Begin search for loop %d ***\n", i));
/* Loop through the remaining entries in the FROM clause to find the
** next nested loop. The loop tests all FROM clause entries
** other FROM clause terms that are notReady. If no notReady terms are
** used then the "optimal" query plan works.
**
+ ** Note that the WhereCost.nRow parameter for an optimal scan might
+ ** not be as small as it would be if the table really were the innermost
+ ** join. The nRow value can be reduced by WHERE clause constraints
+ ** that do not use indices. But this nRow reduction only happens if the
+ ** table really is the innermost join.
+ **
** The second loop iteration is only performed if no optimal scan
- ** strategies were found by the first loop. This 2nd iteration is used to
- ** search for the lowest cost scan overall.
+ ** strategies were found by the first iteration. This second iteration
+ ** is used to search for the lowest cost scan overall.
**
** Previous versions of SQLite performed only the second iteration -
** the next outermost loop was always that with the lowest overall
**
** The best strategy is to iterate through table t1 first. However it
** is not possible to determine this with a simple greedy algorithm.
- ** However, since the cost of a linear scan through table t2 is the same
+ ** Since the cost of a linear scan through table t2 is the same
** as the cost of a linear scan through table t1, a simple greedy
** algorithm may choose to use t2 for the outer loop, which is a much
** costlier approach.
*/
nUnconstrained = 0;
notIndexed = 0;
- for(isOptimal=(iFrom<nTabList-1); isOptimal>=0; isOptimal--){
+ for(isOptimal=(iFrom<nTabList-1); isOptimal>=0 && bestJ<0; isOptimal--){
Bitmask mask; /* Mask of tables not yet ready */
for(j=iFrom, pTabItem=&pTabList->a[j]; j<nTabList; j++, pTabItem++){
int doNotReorder; /* True if this table should not be reordered */
pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0);
if( pTabItem->pIndex==0 ) nUnconstrained++;
+ WHERETRACE(("=== trying table %d with isOptimal=%d ===\n",
+ j, isOptimal));
assert( pTabItem->pTab );
#ifndef SQLITE_OMIT_VIRTUALTABLE
if( IsVirtual(pTabItem->pTab) ){
sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo;
- bestVirtualIndex(pParse, pWC, pTabItem, mask, pOrderBy, &sCost, pp);
+ bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
+ &sCost, pp);
}else
#endif
{
- bestBtreeIndex(pParse, pWC, pTabItem, mask, pOrderBy, &sCost);
+ bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
+ &sCost);
}
assert( isOptimal || (sCost.used¬Ready)==0 );
&& (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */
|| NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0))
&& (bestJ<0 || sCost.rCost<bestPlan.rCost /* (4) */
- || (sCost.rCost<=bestPlan.rCost && sCost.nRow<bestPlan.nRow))
+ || (sCost.rCost<=bestPlan.rCost
+ && sCost.plan.nRow<bestPlan.plan.nRow))
){
- WHERETRACE(("... best so far with cost=%g and nRow=%g\n",
- sCost.rCost, sCost.nRow));
+ WHERETRACE(("=== table %d is best so far"
+ " with cost=%g and nRow=%g\n",
+ j, sCost.rCost, sCost.plan.nRow));
bestPlan = sCost;
bestJ = j;
}
}
assert( bestJ>=0 );
assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) );
- WHERETRACE(("*** Optimizer selects table %d for loop %d\n", bestJ,
- pLevel-pWInfo->a));
+ WHERETRACE(("*** Optimizer selects table %d for loop %d"
+ " with cost=%g and nRow=%g\n",
+ bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow));
if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){
*ppOrderBy = 0;
}
}
notReady &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor);
pLevel->iFrom = (u8)bestJ;
- if( bestPlan.nRow>=(double)1 ) pParse->nQueryLoop *= bestPlan.nRow;
+ if( bestPlan.plan.nRow>=(double)1 ){
+ pParse->nQueryLoop *= bestPlan.plan.nRow;
+ }
/* Check that if the table scanned by this loop iteration had an
** INDEXED BY clause attached to it, that the named index is being
*/
sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */
notReady = ~(Bitmask)0;
+ pWInfo->nRowOut = (double)1;
for(i=0, pLevel=pWInfo->a; i<nTabList; i++, pLevel++){
Table *pTab; /* Table to open */
int iDb; /* Index of database containing table/index */
-#ifndef SQLITE_OMIT_EXPLAIN
- if( pParse->explain==2 ){
- char *zMsg;
- struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom];
- zMsg = sqlite3MPrintf(db, "TABLE %s", pItem->zName);
- if( pItem->zAlias ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias);
- }
- if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s WITH AUTOMATIC INDEX", zMsg);
- }else if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s WITH INDEX %s",
- zMsg, pLevel->plan.u.pIdx->zName);
- }else if( pLevel->plan.wsFlags & WHERE_MULTI_OR ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s VIA MULTI-INDEX UNION", zMsg);
- }else if( pLevel->plan.wsFlags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s USING PRIMARY KEY", zMsg);
- }
-#ifndef SQLITE_OMIT_VIRTUALTABLE
- else if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){
- sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx;
- zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg,
- pVtabIdx->idxNum, pVtabIdx->idxStr);
- }
-#endif
- if( pLevel->plan.wsFlags & WHERE_ORDERBY ){
- zMsg = sqlite3MAppendf(db, zMsg, "%s ORDER BY", zMsg);
- }
- sqlite3VdbeAddOp4(v, OP_Explain, i, pLevel->iFrom, 0, zMsg, P4_DYNAMIC);
- }
-#endif /* SQLITE_OMIT_EXPLAIN */
pTabItem = &pTabList->a[pLevel->iFrom];
pTab = pTabItem->pTab;
pLevel->iTabCur = pTabItem->iCursor;
+ pWInfo->nRowOut *= pLevel->plan.nRow;
iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){
/* Do nothing */
*/
notReady = ~(Bitmask)0;
for(i=0; i<nTabList; i++){
+ pLevel = &pWInfo->a[i];
+ explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags);
notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady);
- pWInfo->iContinue = pWInfo->a[i].addrCont;
+ pWInfo->iContinue = pLevel->addrCont;
}
#ifdef SQLITE_TEST /* For testing and debugging use only */
projects / thirdparty / sqlite.git / commitdiff
