From: drh Date: Wed, 16 Jan 2013 17:08:58 +0000 (+0000) Subject: Enhance the query planner to exploit transitivity of join constraints in X-Git-Tag: version-3.7.16~69^2~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=7a5bcc0f08d1b8b45adb9870c31e4c1b68c644bd;p=thirdparty%2Fsqlite.git Enhance the query planner to exploit transitivity of join constraints in a multi-way join. FossilOrigin-Name: 13171eb5dc19733276fbfd5515d75b70a9f5f5d7 --- diff --git a/manifest b/manifest index a878fe5539..63208c395f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\squery\splanning\sfor\sjoins:\s\sAvoid\sunnecessary\scalls\sto\n"optimal\sscan"\schecks\sin\scases\swhere\stable\sreordering\sis\snot\spossible.\nMake\ssure\soptimal\sscan\schecks\sare\scarried\sout\sfor\sCROSS\sJOINs\sand\sLEFT\nJOINs. -D 2013-01-16T00:46:09.175 +C Enhance\sthe\squery\splanner\sto\sexploit\stransitivity\sof\sjoin\sconstraints\sin\na\smulti-way\sjoin. +D 2013-01-16T17:08:58.150 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a48faa9e7dd7d556d84f5456eabe5825dd8a6282 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/shell.c 11c9611580bb2ffce3a232f31f7f8cc310df0843 F src/sqlite.h.in 39cc33bb08897c748fe3383c29ccf56585704177 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h e998703742455b2241731424c6ec142fd8d0258f +F src/sqliteInt.h a6b3f816df7abd24bbb62b13867b47c2255b11a4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -252,7 +252,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c d48a57d8afd97c51f1b772ebd72431a43a0e48b3 +F src/where.c eb1e1dfc17ff26ba9dc35d6df097e0ecc41d9880 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1033,7 +1033,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ac4e119a87497f2e422ff1cb711112ed8594bfa9 -R 604db48c0336d28f8c2561951db4000b +P d5ebb7877885839e93eee3b322624d4c4215c1c4 +R 3bb8dbee120cecd828508038284032c9 +T *branch * transitive-constraints +T *sym-transitive-constraints * +T -sym-trunk * U drh -Z 14d78037173bf002814299854e4b798b +Z 258dff7809c25174ed1f5fdd919be15b diff --git a/manifest.uuid b/manifest.uuid index 536565f198..7571bed4cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5ebb7877885839e93eee3b322624d4c4215c1c4 \ No newline at end of file +13171eb5dc19733276fbfd5515d75b70a9f5f5d7 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 74fa0b31ca..d5cb69732f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -575,6 +575,11 @@ struct BusyHandler { */ #define ArraySize(X) ((int)(sizeof(X)/sizeof(X[0]))) +/* +** Determine if the argument is a power of two +*/ +#define IsPowerOfTwo(X) (((X)&((X)-1))==0) + /* ** The following value as a destructor means to use sqlite3DbFree(). ** The sqlite3DbFree() routine requires two parameters instead of the diff --git a/src/where.c b/src/where.c index ba39fe9a5d..873aaaf18b 100644 --- a/src/where.c +++ b/src/where.c @@ -98,8 +98,8 @@ struct WhereTerm { int leftCursor; /* Cursor number of X in "X " */ union { int leftColumn; /* Column number of X in "X " */ - WhereOrInfo *pOrInfo; /* Extra information if eOperator==WO_OR */ - WhereAndInfo *pAndInfo; /* Extra information if eOperator==WO_AND */ + WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ + WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ } u; u16 eOperator; /* A WO_xx value describing */ u8 wtFlags; /* TERM_xxx bit flags. See below */ @@ -227,6 +227,7 @@ struct WhereCost { #define WO_ISNULL 0x080 #define WO_OR 0x100 /* Two or more OR-connected terms */ #define WO_AND 0x200 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ #define WO_NOOP 0x800 /* This term does not restrict search space */ #define WO_ALL 0xfff /* Mask of all possible WO_* values */ @@ -639,44 +640,76 @@ static WhereTerm *findTerm( Index *pIdx /* Must be compatible with this index, if not NULL */ ){ WhereTerm *pTerm; - int k; + WhereTerm *pResult = 0; + WhereClause *pWCOrig = pWC; + int j, k; + int aEquiv[8]; + int nEquiv = 2; + int iEquiv = 2; + Expr *pX; + Parse *pParse; + assert( iCur>=0 ); - op &= WO_ALL; - for(; pWC; pWC=pWC->pOuter){ - for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ - if( pTerm->leftCursor==iCur - && (pTerm->prereqRight & notReady)==0 - && pTerm->u.leftColumn==iColumn - && (pTerm->eOperator & op)!=0 - ){ - if( iColumn>=0 && pIdx && pTerm->eOperator!=WO_ISNULL ){ - Expr *pX = pTerm->pExpr; - CollSeq *pColl; - char idxaff; - int j; - Parse *pParse = pWC->pParse; - - idxaff = pIdx->pTable->aCol[iColumn].affinity; - if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue; - - /* Figure out the collation sequence required from an index for - ** it to be useful for optimising expression pX. Store this - ** value in variable pColl. - */ - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - - for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>=pIdx->nColumn) ) return 0; + aEquiv[0] = iCur; + aEquiv[1] = iColumn; + for(;;){ + for(pWC=pWCOrig; pWC; pWC=pWC->pOuter){ + for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ + if( pTerm->leftCursor==iCur + && pTerm->u.leftColumn==iColumn + && (pTerm->eOperator & op & WO_ALL)!=0 + ){ + if( (pTerm->prereqRight & notReady)==0 ){ + if( iColumn>=0 && pIdx && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + char idxaff; + + pX = pTerm->pExpr; + pParse = pWC->pParse; + idxaff = pIdx->pTable->aCol[iColumn].affinity; + if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue; + + /* Figure out the collation sequence required from an index for + ** it to be useful for optimising expression pX. Store this + ** value in variable pColl. + */ + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pParse,pX->pLeft,pX->pRight); + if( pColl==0 ) pColl = pParse->db->pDfltColl; + + for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ + if( NEVER(j>=pIdx->nColumn) ) return 0; + } + if( sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue; + } + pResult = pTerm; + if( pTerm->prereqRight==0 ) goto findTerm_success; + } + if( (op&WO_EQ)!=0 + && (pTerm->eOperator & WO_EQUIV)!=0 + && nEquivpExpr->pRight); + assert( pX->op==TK_COLUMN ); + for(j=0; jiTable && aEquiv[j+1]==pX->iColumn ) break; + } + if( j==nEquiv ){ + aEquiv[j] = pX->iTable; + aEquiv[j+1] = pX->iColumn; + nEquiv += 2; + } } - if( sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue; } - return pTerm; } } + if( iEquiv>=nEquiv ) break; + iCur = aEquiv[iEquiv++]; + iColumn = aEquiv[iEquiv++]; + op &= WO_EQ; } - return 0; +findTerm_success: + return pResult; } /* Forward reference */ @@ -993,7 +1026,7 @@ static void exprAnalyzeOrTerm( b |= getMask(pMaskSet, pOther->leftCursor); } indexable &= b; - if( pOrTerm->eOperator!=WO_EQ ){ + if( (pOrTerm->eOperator & WO_EQ)==0 ){ chngToIN = 0; }else{ chngToIN &= b; @@ -1044,7 +1077,7 @@ static void exprAnalyzeOrTerm( for(j=0; j<2 && !okToChngToIN; j++){ pOrTerm = pOrWc->a; for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); pOrTerm->wtFlags &= ~TERM_OR_OK; if( pOrTerm->leftCursor==iCursor ){ /* This is the 2-bit case and we are on the second iteration and @@ -1070,7 +1103,7 @@ static void exprAnalyzeOrTerm( /* No candidate table+column was found. This can only occur ** on the second iteration */ assert( j==1 ); - assert( (chngToIN&(chngToIN-1))==0 ); + assert( IsPowerOfTwo(chngToIN) ); assert( chngToIN==getMask(pMaskSet, iCursor) ); break; } @@ -1080,7 +1113,7 @@ static void exprAnalyzeOrTerm( ** table and column is common to every term in the OR clause */ okToChngToIN = 1; for(; i>=0 && okToChngToIN; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); if( pOrTerm->leftCursor!=iCursor ){ pOrTerm->wtFlags &= ~TERM_OR_OK; }else if( pOrTerm->u.leftColumn!=iColumn ){ @@ -1116,7 +1149,7 @@ static void exprAnalyzeOrTerm( for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); assert( pOrTerm->leftCursor==iCursor ); assert( pOrTerm->u.leftColumn==iColumn ); pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); @@ -1146,6 +1179,27 @@ static void exprAnalyzeOrTerm( } #endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ +/* +** Check to see if pExpr is an expression of the form A==B where both +** A and B are columns with the same affinity and collating sequence. +** If A and B are equivalent, return true. +*/ +static int isEquivalenceExpr(Parse *pParse, Expr *pExpr){ + const CollSeq *pCLeft, *pCRight; + if( pExpr->op!=TK_EQ ) return 0; + if( ExprHasProperty(pExpr, EP_FromJoin) ) return 0; + assert( sqlite3ExprSkipCollate(pExpr->pRight)->op==TK_COLUMN ); + assert( sqlite3ExprSkipCollate(pExpr->pLeft)->op==TK_COLUMN ); + if( sqlite3ExprAffinity(pExpr->pLeft)!=sqlite3ExprAffinity(pExpr->pRight) ){ + return 0; + } + pCLeft = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + if( pCLeft ){ + pCRight = sqlite3ExprCollSeq(pParse, pExpr->pRight); + if( pCRight && pCRight!=pCLeft ) return 0; + } + return 1; +} /* ** The input to this routine is an WhereTerm structure with only the @@ -1226,6 +1280,7 @@ static void exprAnalyze( if( pRight && pRight->op==TK_COLUMN ){ WhereTerm *pNew; Expr *pDup; + u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ if( pTerm->leftCursor>=0 ){ int idxNew; pDup = sqlite3ExprDup(db, pExpr, 0); @@ -1240,6 +1295,10 @@ static void exprAnalyze( pTerm = &pWC->a[idxTerm]; pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; + if( isEquivalenceExpr(pParse, pExpr) ){ + pTerm->eOperator |= WO_EQUIV; + eExtraOp = WO_EQUIV; + } }else{ pDup = pExpr; pNew = pTerm; @@ -1251,7 +1310,7 @@ static void exprAnalyze( testcase( (prereqLeft | extraRight) != prereqLeft ); pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; - pNew->eOperator = operatorMask(pDup->op); + pNew->eOperator = operatorMask(pDup->op) + eExtraOp; } } @@ -1710,7 +1769,7 @@ static void bestOrClauseIndex(WhereBestIdx *p){ /* Search the WHERE clause terms for a usable WO_OR term. */ for(pTerm=pWC->a; pTermeOperator==WO_OR + if( (pTerm->eOperator & WO_OR)!=0 && ((pTerm->prereqAll & ~maskSrc) & p->notReady)==0 && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 ){ @@ -1731,7 +1790,7 @@ static void bestOrClauseIndex(WhereBestIdx *p){ WHERETRACE(("... Multi-index OR testing for term %d of %d....\n", (pOrTerm - pOrWC->a), (pTerm - pWC->a) )); - if( pOrTerm->eOperator==WO_AND ){ + if( (pOrTerm->eOperator& WO_AND)!=0 ){ sBOI.pWC = &pOrTerm->u.pAndInfo->wc; bestIndex(&sBOI); }else if( pOrTerm->leftCursor==iCur ){ @@ -1792,7 +1851,7 @@ static int termCanDriveIndex( ){ char aff; if( pTerm->leftCursor!=pSrc->iCursor ) return 0; - if( pTerm->eOperator!=WO_EQ ) return 0; + if( (pTerm->eOperator & WO_EQ)==0 ) return 0; if( (pTerm->prereqRight & notReady)!=0 ) return 0; aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; @@ -2054,9 +2113,9 @@ static sqlite3_index_info *allocateIndexInfo(WhereBestIdx *p){ ** to this virtual table */ for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); if( pTerm->eOperator & (WO_ISNULL) ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; nTerm++; @@ -2107,14 +2166,14 @@ static sqlite3_index_info *allocateIndexInfo(WhereBestIdx *p){ for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ u8 op; if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); if( pTerm->eOperator & (WO_ISNULL) ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; - op = (u8)pTerm->eOperator; + op = (u8)pTerm->eOperator & WO_ALL; if( op==WO_IN ) op = WO_EQ; pIdxCons[j].op = op; /* The direct assignment in the previous line is possible only because @@ -2284,7 +2343,7 @@ static void bestVirtualIndex(WhereBestIdx *p){ j = pIdxCons->iTermOffset; pTerm = &pWC->a[j]; if( (pTerm->prereqRight&p->notReady)==0 - && (bAllowIN || pTerm->eOperator!=WO_IN) + && (bAllowIN || (pTerm->eOperator & WO_IN)==0) ){ pIdxCons->usable = 1; }else{ @@ -2316,7 +2375,7 @@ static void bestVirtualIndex(WhereBestIdx *p){ j = pIdxCons->iTermOffset; pTerm = &pWC->a[j]; p->cost.used |= pTerm->prereqRight; - if( pTerm->eOperator==WO_IN && pUsage[i].omit==0 ){ + if( (pTerm->eOperator & WO_IN)!=0 && pUsage[i].omit==0 ){ /* Do not attempt to use an IN constraint if the virtual table ** says that the equivalent EQ constraint cannot be safely omitted. ** If we do attempt to use such a constraint, some rows might be @@ -2622,24 +2681,24 @@ static int whereRangeScanEst( if( pLower ){ Expr *pExpr = pLower->pExpr->pRight; rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); - assert( pLower->eOperator==WO_GT || pLower->eOperator==WO_GE ); + assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); if( rc==SQLITE_OK && whereKeyStats(pParse, p, pRangeVal, 0, a)==SQLITE_OK ){ iLower = a[0]; - if( pLower->eOperator==WO_GT ) iLower += a[1]; + if( (pLower->eOperator & WO_GT)!=0 ) iLower += a[1]; } sqlite3ValueFree(pRangeVal); } if( rc==SQLITE_OK && pUpper ){ Expr *pExpr = pUpper->pExpr->pRight; rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); - assert( pUpper->eOperator==WO_LT || pUpper->eOperator==WO_LE ); + assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); if( rc==SQLITE_OK && whereKeyStats(pParse, p, pRangeVal, 1, a)==SQLITE_OK ){ iUpper = a[0]; - if( pUpper->eOperator==WO_LE ) iUpper += a[1]; + if( (pUpper->eOperator & WO_LE)!=0 ) iUpper += a[1]; } sqlite3ValueFree(pRangeVal); } @@ -2947,12 +3006,12 @@ static int isSortingIndex( WO_EQ|WO_ISNULL|WO_IN, pIdx); if( pConstraint==0 ){ isEq = 0; - }else if( pConstraint->eOperator==WO_IN ){ + }else if( (pConstraint->eOperator & WO_IN)!=0 ){ /* Constraints of the form: "X IN ..." cannot be used with an ORDER BY ** because we do not know in what order the values on the RHS of the IN ** operator will occur. */ break; - }else if( pConstraint->eOperator==WO_ISNULL ){ + }else if( (pConstraint->eOperator & WO_ISNULL)!=0 ){ uniqueNotNull = 0; isEq = 1; /* "X IS NULL" means X has only a single value */ }else if( pConstraint->prereqRight==0 ){ @@ -3365,12 +3424,13 @@ static void bestBtreeIndex(WhereBestIdx *p){ && pFirstTerm!=0 && aiRowEst[1]>1 ){ assert( (pFirstTerm->eOperator & (WO_EQ|WO_ISNULL|WO_IN))!=0 ); if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ - testcase( pFirstTerm->eOperator==WO_EQ ); - testcase( pFirstTerm->eOperator==WO_ISNULL ); + testcase( pFirstTerm->eOperator & WO_EQ ); + testcase( pFirstTerm->eOperator & WO_EQUIV ); + testcase( pFirstTerm->eOperator & WO_ISNULL ); whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &pc.plan.nRow); }else if( bInEst==0 ){ - assert( pFirstTerm->eOperator==WO_IN ); + assert( pFirstTerm->eOperator & WO_IN ); whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &pc.plan.nRow); } @@ -3517,7 +3577,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** selective in practice, on average. */ pc.plan.nRow /= 3; } - }else if( pTerm->eOperator!=WO_NOOP ){ + }else if( (pTerm->eOperator & WO_NOOP)==0 ){ /* Any other expression lowers the output row count by half */ pc.plan.nRow /= 2; } @@ -4153,7 +4213,6 @@ static Bitmask codeOneLoopStart( pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0); assert( pTerm!=0 ); assert( pTerm->pExpr!=0 ); - assert( pTerm->leftCursor==iCur ); assert( omitTable==0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, iReleaseReg); @@ -4544,7 +4603,7 @@ static Bitmask codeOneLoopStart( pTerm = pLevel->plan.u.pTerm; assert( pTerm!=0 ); - assert( pTerm->eOperator==WO_OR ); + assert( pTerm->eOperator & WO_OR ); assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); pOrWc = &pTerm->u.pOrInfo->wc; pLevel->op = OP_Return; @@ -4617,7 +4676,7 @@ static Bitmask codeOneLoopStart( for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; - if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ + if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ Expr *pOrExpr = pOrTerm->pExpr; if( pAndExpr ){