From: drh Date: Tue, 16 Jul 2013 21:31:23 +0000 (+0000) Subject: Enhance the query planner so that it looks at multiple solutions to OR X-Git-Tag: version-3.8.0~86 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=aa32e3c60a86442bda394bdab8d520eec0b10ba5;p=thirdparty%2Fsqlite.git Enhance the query planner so that it looks at multiple solutions to OR expressions in the WHERE clause. FossilOrigin-Name: 5e19d054105fb16ff52d265d48cc87a418603f6f --- diff --git a/manifest b/manifest index c21740737a..aed4f14e23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3_cancel_auto_extension(X)\sinterface\swhich\swill\sundo\sa\sprior\ncall\sto\ssqlite3_auto_extension(X). -D 2013-07-15T17:02:28.816 +C Enhance\sthe\squery\splanner\sso\sthat\sit\slooks\sat\smultiple\ssolutions\sto\sOR\nexpressions\sin\sthe\sWHERE\sclause. +D 2013-07-16T21:31:23.453 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c f5201334501cd23a39315cab479c0dcce0990701 +F src/where.c 927acb798c66af64b5d640e50f0edfe07d6b4085 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1037,13 +1037,13 @@ F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test da54153a4c1571ea1b95659e5bec8119edf786aa -F test/where2.test d712de0ea9a2c3de7b34b0b1e75172556fef5b24 +F test/where2.test b1830f762f837153a4c9743adaab90a5761f73e7 F test/where3.test a0682ba3dc8c8f46ffcc95a3d9f58c4327fc129f F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5a4b0abc207d71da4deecd734ad8579e8dd40aa8 -F test/where8.test f6b9559723564a042927ee0f22003ac9bed71b21 +F test/where8.test 6f95896633cf2d307b5263145b942b7d33e837c6 F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 F test/where9.test 9a7fda4a4512abc26a855e8b2b6572b200f6019b F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a @@ -1103,7 +1103,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 92adaee5bd31c152dbc1592f4aeb5d8da957a1ea -R 3e75a7374bba2c69c55563b837ab8202 +P cdce87eb889a43dafcc560d5f97ab517d0266860 +R 5cd2d361c09a0f2b15feaa159f9c349e U drh -Z a6c699be62fa8981abc8311358615b45 +Z cef10e6dbd00a6e9bd88cff94c534444 diff --git a/manifest.uuid b/manifest.uuid index bad5c44835..89366bbd3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cdce87eb889a43dafcc560d5f97ab517d0266860 \ No newline at end of file +5e19d054105fb16ff52d265d48cc87a418603f6f \ No newline at end of file diff --git a/src/where.c b/src/where.c index e18e88623f..75804782c2 100644 --- a/src/where.c +++ b/src/where.c @@ -45,6 +45,8 @@ typedef struct WherePath WherePath; typedef struct WhereTerm WhereTerm; typedef struct WhereLoopBuilder WhereLoopBuilder; typedef struct WhereScan WhereScan; +typedef struct WhereOrCost WhereOrCost; +typedef struct WhereOrSet WhereOrSet; /* ** Cost X is tracked as 10*log2(X) stored in a 16-bit integer. The @@ -152,6 +154,27 @@ struct WhereLoop { WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ }; +/* This object holds the prerequisites and the cost of running a +** subquery on one operand of an OR operator in the WHERE clause. +** See WhereOrSet for additional information +*/ +struct WhereOrCost { + Bitmask prereq; /* Prerequisites */ + WhereCost rRun; /* Cost of running this subquery */ + WhereCost nOut; /* Number of outputs for this subquery */ +}; + +/* The WhereOrSet object holds a set of possible WhereOrCosts that +** correspond to the subquery(s) of OR-clause processing. At most +** favorable N_OR_COST elements are retained. +*/ +#define N_OR_COST 3 +struct WhereOrSet { + u16 n; /* Number of valid a[] entries */ + WhereOrCost a[N_OR_COST]; /* Set of best costs */ +}; + + /* Forward declaration of methods */ static int whereLoopResize(sqlite3*, WhereLoop*, int); @@ -366,7 +389,7 @@ struct WhereLoopBuilder { WhereClause *pWC; /* WHERE clause terms */ ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ - WhereLoop *pBest; /* If non-NULL, store single best loop here */ + WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ }; /* @@ -509,6 +532,54 @@ int sqlite3WhereOkOnePass(WhereInfo *pWInfo){ return pWInfo->okOnePass; } +/* +** Move the content of pSrc into pDest +*/ +static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ + pDest->n = pSrc->n; + memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); +} + +/* +** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. +** +** The new entry might overwrite an existing entry, or it might be +** appended, or it might be discarded. Do whatever is the right thing +** so that pSet keeps the N_OR_COST best entries seen so far. +*/ +static int whereOrInsert( + WhereOrSet *pSet, /* The WhereOrSet to be updated */ + Bitmask prereq, /* Prerequisites of the new entry */ + WhereCost rRun, /* Run-cost of the new entry */ + WhereCost nOut /* Number of outputs for the new entry */ +){ + u16 i; + WhereOrCost *p; + for(i=pSet->n, p=pSet->a; i>0; i--, p++){ + if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ + goto whereOrInsert_done; + } + if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ + return 0; + } + } + if( pSet->na[pSet->n++]; + p->nOut = nOut; + }else{ + p = pSet->a; + for(i=1; in; i++){ + if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; + } + if( p->rRun<=rRun ) return 0; + } +whereOrInsert_done: + p->prereq = prereq; + p->rRun = rRun; + if( p->nOut>nOut ) p->nOut = nOut; + return 1; +} + /* ** Initialize a preallocated WhereClause structure. */ @@ -3743,8 +3814,9 @@ static Bitmask codeOneLoopStart( int iTerm; for(iTerm=0; iTermnTerm; iTerm++){ Expr *pExpr = pWC->a[iTerm].pExpr; + if( &pWC->a[iTerm] == pTerm ) continue; if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; - if( pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_ORINFO) ) continue; + if( pWC->a[iTerm].wtFlags & (TERM_ORINFO) ) continue; if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); pAndExpr = sqlite3ExprAnd(pParse->db, pAndExpr, pExpr); @@ -4072,12 +4144,12 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** fewer dependencies than the template. Otherwise a new WhereLoop is ** added based on the template. ** -** If pBuilder->pBest is not NULL then we only care about the very -** best template and that template should be stored in pBuilder->pBest. -** If pBuilder->pBest is NULL then a list of the best templates are stored -** in pBuilder->pWInfo->pLoops. +** If pBuilder->pOrSet is not NULL then we only care about only the +** prerequisites and rRun and nOut costs of the N best loops. That +** information is gathered in the pBuilder->pOrSet object. This special +** processing mode is used only for OR clause processing. ** -** When accumulating multiple loops (when pBuilder->pBest is NULL) we +** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we ** still might overwrite similar loops with the new template if the ** template is better. Loops may be overwritten if the following ** conditions are met: @@ -4094,30 +4166,22 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ WhereInfo *pWInfo = pBuilder->pWInfo; sqlite3 *db = pWInfo->pParse->db; - /* If pBuilder->pBest is defined, then only keep track of the single - ** best WhereLoop. pBuilder->pBest->maskSelf==0 indicates that no - ** prior WhereLoops have been evaluated and that the current pTemplate - ** is therefore the first and hence the best and should be retained. + /* If pBuilder->pOrSet is defined, then only keep track of the costs + ** and prereqs. */ - if( (p = pBuilder->pBest)!=0 ){ - if( p->maskSelf!=0 ){ - WhereCost rCost = whereCostAdd(p->rRun,p->rSetup); - WhereCost rTemplate = whereCostAdd(pTemplate->rRun,pTemplate->rSetup); - if( rCost < rTemplate ){ - testcase( rCost==rTemplate-1 ); - goto whereLoopInsert_noop; - } - if( rCost==rTemplate && (p->prereq & pTemplate->prereq)==p->prereq ){ - goto whereLoopInsert_noop; - } - } + if( pBuilder->pOrSet!=0 ){ +#if WHERETRACE_ENABLED + u16 n = pBuilder->pOrSet->n; + int x = +#endif + whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, + pTemplate->nOut); #if WHERETRACE_ENABLED if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf(p->maskSelf==0 ? "ins-init: " : "ins-best: "); + sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); whereLoopPrint(pTemplate, pWInfo->pTabList); } #endif - whereLoopXfer(db, p, pTemplate); return SQLITE_OK; } @@ -4211,7 +4275,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ whereLoopInsert_noop: #if WHERETRACE_ENABLED if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf(pBuilder->pBest ? "ins-skip: " : "ins-noop: "); + sqlite3DebugPrintf("ins-noop: "); whereLoopPrint(pTemplate, pWInfo->pTabList); } #endif @@ -4493,7 +4557,7 @@ static int whereLoopAddBtree( rLogSize = estLog(rSize); /* Automatic indexes */ - if( !pBuilder->pBest + if( !pBuilder->pOrSet && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 && pSrc->pIndex==0 && !pSrc->viaCoroutine @@ -4779,7 +4843,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ int iCur; WhereClause tempWC; WhereLoopBuilder sSubBuild; - WhereLoop sBest; + WhereOrSet sSum, sCur, sPrev; struct SrcList_item *pItem; pWC = pBuilder->pWC; @@ -4794,16 +4858,14 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; WhereTerm *pOrTerm; - WhereCost rTotal = 0; - WhereCost nRow = 0; - Bitmask prereq = mExtra; + int once = 1; + int i, j; - whereLoopInit(&sBest); pItem = pWInfo->pTabList->a + pNew->iTab; iCur = pItem->iCursor; sSubBuild = *pBuilder; sSubBuild.pOrderBy = 0; - sSubBuild.pBest = &sBest; + sSubBuild.pOrSet = &sCur; for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ @@ -4818,39 +4880,48 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ }else{ continue; } - sBest.maskSelf = 0; - sBest.rSetup = 0; - sBest.rRun = 0; + sCur.n = 0; #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ rc = whereLoopAddVirtual(&sSubBuild); + for(i=0; inLSlot>=1 ); - if( sBest.maskSelf ){ - pNew->nLTerm = 1; - pNew->aLTerm[0] = pTerm; - pNew->wsFlags = WHERE_MULTI_OR; - pNew->rSetup = 0; + pNew->nLTerm = 1; + pNew->aLTerm[0] = pTerm; + pNew->wsFlags = WHERE_MULTI_OR; + pNew->rSetup = 0; + pNew->iSortIdx = 0; + memset(&pNew->u, 0, sizeof(pNew->u)); + for(i=0; rc==SQLITE_OK && irRun = rTotal + 18; assert( 18==whereCost(7)-whereCost(2) ); - pNew->nOut = nRow; - pNew->prereq = prereq; - memset(&pNew->u, 0, sizeof(pNew->u)); + pNew->rRun = sSum.a[i].rRun + 18; + pNew->nOut = sSum.a[i].nOut; + pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); } - whereLoopClear(pWInfo->pParse->db, &sBest); } } return rc; diff --git a/test/where2.test b/test/where2.test index 3d4dfc9126..c827ecc7b5 100644 --- a/test/where2.test +++ b/test/where2.test @@ -699,5 +699,16 @@ do_test where2-11.4 { } } {4 8 10} +# Verify that the OR clause is used in an outer loop even when +# the OR clause scores slightly better on an inner loop. +do_execsql_test where2-12.1 { + CREATE TABLE t12(x INTEGER PRIMARY KEY, y); + CREATE INDEX t12y ON t12(y); + EXPLAIN QUERY PLAN + SELECT a.x, b.x + FROM t12 AS a JOIN t12 AS b ON a.y=b.x + WHERE (b.x=$abc OR b.y=$abc); +} {/.*SEARCH TABLE t12 AS b .*SEARCH TABLE t12 AS b .*/} + finish_test diff --git a/test/where8.test b/test/where8.test index 6890e3ac59..9127179292 100644 --- a/test/where8.test +++ b/test/where8.test @@ -212,8 +212,9 @@ do_test where8-3.4 { do_test where8-3.5 { execsql_status { SELECT a, d FROM t1, t2 WHERE (a = 2 OR a = 3) AND (d = a OR e = 'sixteen') + ORDER BY +a, +d; } -} {2 2 2 4 3 3 3 4 0 0} +} {2 2 2 4 3 3 3 4 0 1} do_test where8-3.6 { # The first part of the WHERE clause in this query, (a=2 OR a=3) is @@ -233,7 +234,7 @@ do_test where8-3.7 { WHERE a = 2 AND (d = a OR e = 'sixteen') ORDER BY t1.rowid } -} {2 2 2 4 0 0} +} {/2 2 2 4 0 [01]/} do_test where8-3.8 { execsql_status { SELECT a, d