From: drh Date: Wed, 8 May 2013 03:05:41 +0000 (+0000) Subject: Add the NGQP solver. X-Git-Tag: version-3.8.0~130^2~89 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a18f3d271e52dd964491417cce401dee96449148;p=thirdparty%2Fsqlite.git Add the NGQP solver. FossilOrigin-Name: 5d37587c50d8932b6357bfd03152a851510a4317 --- diff --git a/manifest b/manifest index c79d6f38fd..53780d9f27 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Continued\sprogress\son\sgenerating\sgood\sWhereLoop\sobjects\sfor\sthe\snew\squery\nplanner. -D 2013-05-07T23:06:23.630 +C Add\sthe\sNGQP\ssolver. +D 2013-05-08T03:05:41.388 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 0d76a0aa7c64536c6f55d11a8f9f40df0636af6a +F src/sqliteInt.h aee09cf5ee7d5e899246affdd8c3cf58f086ade5 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e957c2c6fd7b421ea6dd1ff11d254c80d0b11638 +F src/where.c cac7c10d422ece684740285366285ef06529796f F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e8881a8b2f25f38bc8ff77619f96f38fe530d13b -R 90f142cd2d5094cb6ddc049cebfe5b6d +P 15cc8a16482777d8e138c4d0863faf8d54fef33a +R 6dc017dbdcd8ab4e7013a1e96308cbfe U drh -Z d9519f5b83b11415ad77247122aa0d25 +Z 6f2b95a5959d577b5b6c3e6985beb9a5 diff --git a/manifest.uuid b/manifest.uuid index 82bc167f3b..379ce7b07e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15cc8a16482777d8e138c4d0863faf8d54fef33a \ No newline at end of file +5d37587c50d8932b6357bfd03152a851510a4317 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7a405b2fb1..86b123749c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2021,6 +2021,7 @@ struct WhereLevel { Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ } u; double rOptCost; /* "Optimal" cost for this level */ + struct WhereLoop *pWLoop; /* The selected WhereLoop object */ /* The following field is really not part of the current level. But ** we need a place to cache virtual table index information for each diff --git a/src/where.c b/src/where.c index 56568989aa..e87d2ae643 100644 --- a/src/where.c +++ b/src/where.c @@ -53,6 +53,7 @@ typedef struct WhereScan WhereScan; */ struct WhereLoop { Bitmask prereq; /* Bitmask of other loops that must run first */ + Bitmask maskSelf; /* Bitmask identifying table iTab */ int iTab; /* Index of the table coded by this loop */ u16 iOb, nOb; /* ORDER BY terms satisfied by this strategy */ double rSetup; /* One-time setup cost (ex: create transient index) */ @@ -74,9 +75,7 @@ struct WherePath { Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ double nRow; /* Estimated number of rows generated by this path */ double rCost; /* Total cost of this path */ - WhereLoop *aLoop[1]; /* Array of WhereLoop objects implementing this path */ - WherePath *pNextPath; /* Next path in order of increasing cost */ - WherePath *pPrevPath; /* Previous path in cost order */ + WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ }; /* @@ -5050,6 +5049,30 @@ static int nQPlan = 0; /* Next free slow in _query_plan[] */ #endif /* SQLITE_TEST */ +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) +/* +** Print a WhereLoop object for debugging purposes +*/ +static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ + int nb = 2*((pTabList->nSrc+15)/16); + struct SrcList_item *pItem = pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + sqlite3DebugPrintf("%02d.%0*llx", p->iTab, nb, p->prereq); + sqlite3DebugPrintf(" %6s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + if( p->pIndex ){ + sqlite3DebugPrintf(".%-8s %2d", p->pIndex->zName, p->nEq); + }else{ + sqlite3DebugPrintf("%12s",""); + } + sqlite3DebugPrintf(" fg %08x OB %d,%d N %2d", + p->wsFlags, p->iOb, p->nOb, p->nTerm); + sqlite3DebugPrintf(" cost %.4g,%.4g,%.4g\n", + p->prereq, p->rSetup, p->rRun, p->nOut); +} +#endif + /* ** Delete a WhereLoop object */ @@ -5102,6 +5125,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ */ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ WhereLoop **ppPrev, *p, *pNext = 0, *pToFree = 0; + WhereTerm **paTerm = 0; sqlite3 *db = pWInfo->pParse->db; /* Search for an existing WhereLoop to overwrite, or which takes @@ -5127,6 +5151,8 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ ){ /* Overwrite an existing WhereLoop with a better one */ sqlite3DbFree(db, p->aTerm); + p->aTerm = 0; + p->nTerm = 0; pNext = p->pNextLoop; break; } @@ -5140,19 +5166,21 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ p = pToFree = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); if( p==0 ) return SQLITE_NOMEM; } + if( pTemplate->nTerm ){ + paTerm = sqlite3DbMallocRaw(db, pTemplate->nTerm*sizeof(p->aTerm[0])); + if( paTerm==0 ){ + sqlite3DbFree(db, pToFree); + return SQLITE_NOMEM; + } + } *p = *pTemplate; p->pNextLoop = pNext; *ppPrev = p; - p->aTerm = 0; + p->aTerm = paTerm; if( p->pIndex && p->pIndex->tnum==0 ) p->pIndex = 0; - if( pTemplate->nTerm<=0 ) return SQLITE_OK; - p->aTerm = sqlite3DbMallocRaw(db, pTemplate->nTerm*sizeof(p->aTerm[0])); - if( p->aTerm==0 ){ - p->nTerm = 0; - sqlite3DbFree(db, pToFree); - return SQLITE_NOMEM; - } - memcpy(p->aTerm, pTemplate->aTerm, pTemplate->nTerm*sizeof(p->aTerm[0])); + if( pTemplate->nTerm ){ + memcpy(p->aTerm, pTemplate->aTerm, pTemplate->nTerm*sizeof(p->aTerm[0])); + } return SQLITE_OK; } @@ -5165,7 +5193,6 @@ static int whereLoopInsert(WhereInfo *pWInfo, WhereLoop *pTemplate){ */ static void whereLoopAddBtreeIndex( WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ - Bitmask maskSelf, /* Bitmask for table being scanned */ struct SrcList_item *pSrc, /* FROM clause term being analyzed */ Index *pProbe, /* An index on pSrc */ int nInMul /* Number of iterations due to IN */ @@ -5176,6 +5203,7 @@ static void whereLoopAddBtreeIndex( int opMask; /* Valid operators for constraints */ WhereScan scan; /* Iterator for WHERE terms */ WhereLoop savedLoop; /* Saved original content of pNew[] */ + int iCol; /* Index of the column in the table */ db = pBuilder->db; pNew = pBuilder->pNew; @@ -5191,53 +5219,47 @@ static void whereLoopAddBtreeIndex( opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; } - - if( pNew->nEqnColumn ){ - int iCol; /* Index of the column in the table */ - - - iCol = pProbe->aiColumn[pNew->nEq]; - pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, - opMask, iCol>=0 ? pProbe : 0); - savedLoop = *pNew; - pNew->rSetup = (double)0; - for(; pTerm!=0; pTerm = whereScanNext(&scan)){ - int nIn = 1; - pNew->nEq = savedLoop.nEq; - pNew->nTerm = savedLoop.nTerm; - pNew->aTerm[pNew->nTerm++] = pTerm; - pNew->prereq = (savedLoop.prereq | pTerm->prereqRight) & ~maskSelf; - if( pTerm->eOperator & WO_IN ){ - Expr *pExpr = pTerm->pExpr; - pNew->wsFlags |= WHERE_COLUMN_IN; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ - nIn = 25; - }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ - /* "x IN (value, value, ...)" */ - nIn = pExpr->x.pList->nExpr; - } - pNew->nEq++; - pNew->nOut = (double)pProbe->aiRowEst[pNew->nEq] * nInMul * nIn; - }else if( pTerm->eOperator & (WO_EQ|WO_ISNULL) ){ - pNew->wsFlags |= WHERE_COLUMN_EQ; - pNew->nEq++; - pNew->nOut = (double)pProbe->aiRowEst[pNew->nEq] * nInMul; - }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; - pNew->nOut = savedLoop.nOut/3; - }else if( pTerm->eOperator & (WO_LT|WO_LE) ){ - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; - pNew->nOut = savedLoop.nOut/3; - } - pNew->rRun = pNew->nOut + estLog(pProbe->aiRowEst[0])*nIn; - whereLoopInsert(pBuilder->pWInfo, pNew); - if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->nEqnColumn ){ - whereLoopAddBtreeIndex(pBuilder, maskSelf, pSrc, pProbe, nInMul*nIn); + iCol = pProbe->aiColumn[pNew->nEq]; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, + opMask, iCol>=0 ? pProbe : 0); + savedLoop = *pNew; + pNew->rSetup = (double)0; + for(; pTerm!=0; pTerm = whereScanNext(&scan)){ + int nIn = 1; + pNew->nEq = savedLoop.nEq; + pNew->nTerm = savedLoop.nTerm; + pNew->aTerm[pNew->nTerm++] = pTerm; + pNew->prereq = (savedLoop.prereq | pTerm->prereqRight) & ~pNew->maskSelf; + if( pTerm->eOperator & WO_IN ){ + Expr *pExpr = pTerm->pExpr; + pNew->wsFlags |= WHERE_COLUMN_IN; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ + nIn = 25; + }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ + /* "x IN (value, value, ...)" */ + nIn = pExpr->x.pList->nExpr; } - } - *pNew = savedLoop; - } + pNew->nEq++; + pNew->nOut = (double)pProbe->aiRowEst[pNew->nEq] * nInMul * nIn; + }else if( pTerm->eOperator & (WO_EQ|WO_ISNULL) ){ + pNew->wsFlags |= WHERE_COLUMN_EQ; + pNew->nEq++; + pNew->nOut = (double)pProbe->aiRowEst[pNew->nEq] * nInMul; + }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pNew->nOut = savedLoop.nOut/3; + }else if( pTerm->eOperator & (WO_LT|WO_LE) ){ + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pNew->nOut = savedLoop.nOut/3; + } + pNew->rRun = pNew->nOut + estLog(pProbe->aiRowEst[0])*nIn; + whereLoopInsert(pBuilder->pWInfo, pNew); + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->nEqnColumn ){ + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul*nIn); + } + } + *pNew = savedLoop; } /* @@ -5256,12 +5278,11 @@ static void whereLoopAddBtree( struct SrcList_item *pSrc; /* The FROM clause btree term to add */ sqlite3 *db; /* The database connection */ WhereLoop *pNew; /* Template WhereLoop object */ - Bitmask maskSelf; /* Mask for iTab */ pNew = pBuilder->pNew; db = pBuilder->db; pSrc = pBuilder->pTabList->a + iTab; - maskSelf = getMask(pBuilder->pWC->pMaskSet, iTab); + pNew->maskSelf = getMask(pBuilder->pWC->pMaskSet, iTab); if( pSrc->pIndex ){ /* An INDEXED BY clause specifies a particular index to use */ @@ -5330,7 +5351,7 @@ static void whereLoopAddBtree( pNew->aTerm = paTerm; pNew->pIndex = pProbe; - whereLoopAddBtreeIndex(pBuilder, maskSelf, pSrc, pProbe, 1); + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 1); /* If there was an INDEXED BY clause, then only that one index is ** considered. */ @@ -5360,11 +5381,12 @@ static void whereLoopAddAll(WhereLoopBuilder *pBuilder){ struct SrcList_item *pItem; WhereClause *pWC = pBuilder->pWC; sqlite3 *db = pBuilder->db; + int nTabList = pBuilder->pWInfo->nLevel; /* Loop over the tables in the join, from left to right */ pBuilder->pNew = sqlite3DbMallocZero(db, sizeof(WhereLoop)); if( pBuilder->pNew==0 ) return; - for(iTab=0, pItem=pTabList->a; iTabnSrc; iTab++, pItem++){ + for(iTab=0, pItem=pTabList->a; iTabpTab) ){ whereLoopAddVirtual(pBuilder, iTab, mExtra); }else{ @@ -5380,6 +5402,107 @@ static void whereLoopAddAll(WhereLoopBuilder *pBuilder){ pBuilder->pNew = 0; } +/* +** Given the list of WhereLoop objects on pWInfo->pLoops, this routine +** attempts to find the lowest cost path that visits each WhereLoop +** once. This path is then loaded into the pWInfo->a[].pWLoop fields. +** +** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation +** error occurs. +*/ +static int wherePathSolver(WhereInfo *pWInfo){ + const int mxChoice = 10; /* Maximum number of simultaneous paths tracked */ + int nLoop; /* Number of terms in the join */ + sqlite3 *db; /* The database connection */ + int iLoop; /* Loop counter over the terms of the join */ + int ii, jj; /* Loop counters */ + double rCost; /* Cost of a path */ + double mxCost; /* Maximum cost of a set of paths */ + int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ + WherePath *aFrom; /* All nFrom paths at the previous level */ + WherePath *aTo; /* The nTo best paths at the current level */ + WherePath *pFrom; /* An element of aFrom[] that we are working on */ + WherePath *pTo; /* An element of aTo[] that we are working on */ + WhereLoop *pWLoop; /* One of the WhereLoop objects */ + WhereLoop **pX; /* Used to divy up the pSpace memory */ + char *pSpace; /* Temporary memory used by this routine */ + + db = pWInfo->pParse->db; + nLoop = pWInfo->nLevel; + assert( nLoop<=pWInfo->pTabList->nSrc ); + + /* Allocate and initialize space for aTo and aFrom */ + ii = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; + pSpace = sqlite3DbMallocRaw(db, ii); + if( pSpace==0 ) return SQLITE_NOMEM; + aTo = (WherePath*)pSpace; + aFrom = aTo+mxChoice; + memset(aFrom, 0, sizeof(aFrom[0])); + pX = (WhereLoop**)(aFrom+mxChoice); + for(ii=0, pFrom=aTo; iiaLoop = pX; + } + + nFrom = 1; + for(iLoop=0; iLooppLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + Bitmask maskNew; + if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; + if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; + rCost = pWLoop->rSetup + pWLoop->rRun*pFrom->nRow + pFrom->rCost; + maskNew = pFrom->maskLoop | pWLoop->maskSelf; + for(jj=0, pTo=aTo; jjmaskLoop!=maskNew; jj++){} + if( jj>=nTo ){ + if( nTo>=mxChoice && rCost>=mxCost ) continue; + if( nTo=mxCost; jj++){ assert(jj>0); } + } + pTo = &aTo[jj]; + } + pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; + pTo->nRow = pFrom->nRow * pWLoop->nOut; + pTo->rCost = rCost; + memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); + pTo->aLoop[iLoop] = pWLoop; + if( nTo>=mxChoice ){ + mxCost = aTo[0].rCost; + for(jj=1, pTo=&aTo[1]; jjrCost>mxCost ) mxCost = pTo->rCost; + } + } + } + } + + /* Swap the roles of aFrom and aTo in preparation for the next + ** cycle. */ + pFrom = aTo; + aTo = aFrom; + aFrom = pFrom; + nFrom = nTo; + } + + /* TEMPORARY */ + if( nFrom==0 ){ sqlite3DbFree(db, pSpace); return SQLITE_ERROR; } + assert( nFrom>0 ); + + /* Find the lowest cost path and load it into pWInfo->a[].pWLoop */ + pFrom = aFrom; + for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; + } + assert( pWInfo->nLevel==nLoop ); + for(iLoop=0; iLoopa[iLoop].pWLoop = pFrom->aLoop[iLoop]; + } + + /* Free temporary memory and return success */ + sqlite3DbFree(db, pSpace); + return SQLITE_OK; +} /* ** Generate the beginning of the loop used for WHERE clause processing. @@ -5621,28 +5744,28 @@ WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ WHERETRACE(("*** Optimizer Start ***\n")); whereLoopAddAll(&sWLB); + if( db->mallocFailed ) goto whereBeginError; /* Display all of the WhereLoop objects if wheretrace is enabled */ #if defined(SQLITE_DEBUG) \ && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) if( sqlite3WhereTrace ){ WhereLoop *p; - int nb = 2*((nTabList+15)/16); for(p=pWInfo->pLoops; p; p=p->pNextLoop){ - struct SrcList_item *pItem = pTabList->a + p->iTab; - Table *pTab = pItem->pTab; - sqlite3DebugPrintf("%02d.%0*llx", p->iTab, nb, p->prereq); - sqlite3DebugPrintf(" %6s", - pItem->zAlias ? pItem->zAlias : pTab->zName); - if( p->pIndex ){ - sqlite3DebugPrintf(".%-8s %2d", p->pIndex->zName, p->nEq); - }else{ - sqlite3DebugPrintf("%12s",""); - } - sqlite3DebugPrintf(" fg %08x OB %d,%d N %2d", - p->wsFlags, p->iOb, p->nOb, p->nTerm); - sqlite3DebugPrintf(" cost %.4g,%.4g,%.4g\n", - p->prereq, p->rSetup, p->rRun, p->nOut); + whereLoopPrint(p, pTabList); + } + } +#endif + + wherePathSolver(pWInfo); + if( db->mallocFailed ) goto whereBeginError; +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) + if( sqlite3WhereTrace ){ + int ii; + sqlite3DebugPrintf("------------ Solution ----------------\n"); + for(ii=0; iia[ii].pWLoop, pTabList); } } #endif