From: drh Date: Tue, 21 May 2013 15:52:07 +0000 (+0000) Subject: Work toward improving the NGQP's ability to optimize out ORDER BY clauses. X-Git-Tag: version-3.8.0~130^2~75 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=23f98daabb42f2f1def180b9bd2aa81c2f96b608;p=thirdparty%2Fsqlite.git Work toward improving the NGQP's ability to optimize out ORDER BY clauses. FossilOrigin-Name: 67367f1e1f0c3eb6be65eea9873910aa62b49884 --- diff --git a/manifest b/manifest index 325be9d757..ac6a9e67b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sin\sall\strunk\schanges\sup\sthrough\sthe\s3.7.17\srelease. -D 2013-05-20T15:14:42.009 +C Work\stoward\simproving\sthe\sNGQP's\sability\sto\soptimize\sout\sORDER\sBY\sclauses. +D 2013-05-21T15:52:07.219 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f6b58b7bdf6535f0f0620c486dd59aa4662c0b4f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -196,7 +196,7 @@ F src/shell.c 2109d54f67c815a100abd7dc6a6e25eddb3b97eb F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h b9b6a2e97254fb50b6977ec55ad0f4e4ce0510d2 +F src/sqliteInt.h 3b89d4517d984363a96f6f7b348440e01f13ced3 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -265,7 +265,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 51d935c6593748ed004006ee0f2147a860d56e90 +F src/where.c a01d93b37fbce377326849fe810dbcd2334d40c6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1065,7 +1065,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9fe20292558bb9422de91e35648cb834cbf3b306 118a3b35693b134d56ebd780123b7fd6f1497668 -R b3fec2668cd66935dfd81d00e243b60d +P 14ab6675e5eab3761256a06dad23d2b11220788a +R fc1fe5e0f9d1ee7dd6bc32ff7b604dd6 U drh -Z e02385c0316003dbdd17c2ecf0ba1968 +Z 0d777b81f154f68ed82e3cd424f83c9e diff --git a/manifest.uuid b/manifest.uuid index 52fb631158..75cf59ccb0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14ab6675e5eab3761256a06dad23d2b11220788a \ No newline at end of file +67367f1e1f0c3eb6be65eea9873910aa62b49884 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ba0a3597b..176413c946 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1919,8 +1919,8 @@ typedef u64 Bitmask; ** contains more than 63 columns and the 64-th or later column is used. */ struct SrcList { - i16 nSrc; /* Number of tables or subqueries in the FROM clause */ - i16 nAlloc; /* Number of entries allocated in a[] below */ + u8 nSrc; /* Number of tables or subqueries in the FROM clause */ + u8 nAlloc; /* Number of entries allocated in a[] below */ struct SrcList_item { Schema *pSchema; /* Schema to which this item is fixed */ char *zDatabase; /* Name of database holding this table */ diff --git a/src/where.c b/src/where.c index 2eff05ab39..0f61f6ed24 100644 --- a/src/where.c +++ b/src/where.c @@ -56,7 +56,8 @@ typedef struct WhereVtabPlan WhereVtabPlan; struct WhereLoop { Bitmask prereq; /* Bitmask of other loops that must run first */ Bitmask maskSelf; /* Bitmask identifying table iTab */ - u16 iTab; /* Index of the table coded by this loop */ + u8 iTab; /* Position in FROM clause of table coded by this loop */ + u8 iSortIdx; /* Sorting index number. 0==None */ u16 nTerm; /* Number of entries in aTerm[] */ u32 wsFlags; /* WHERE_* flags describing the plan */ double rSetup; /* One-time setup cost (ex: create transient index) */ @@ -1987,6 +1988,7 @@ static int termCanDriveIndex( if( pTerm->leftCursor!=pSrc->iCursor ) return 0; if( (pTerm->eOperator & WO_EQ)==0 ) return 0; if( (pTerm->prereqRight & notReady)!=0 ) return 0; + if( pTerm->u.leftColumn<0 ) return 0; aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; return 1; @@ -5168,6 +5170,21 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** and no insert will occur if an existing WhereLoop is faster and has ** fewer dependencies than the template. Otherwise a new WhereLoop is ** added based no the template. +** +** If pBuilder->pBest is not NULL then we only care about the very +** best template and that template should be stored in pBuilder->pBest. +** If pBuilder->pBest is NULL then a list of the best templates are stored +** in pBuilder->pWInfo->pLoops. +** +** When accumulating multiple loops (when pBuilder->pBest is NULL) we +** still might overwrite similar loops with the new template if the +** template is better. Loops may be overwritten if the following +** conditions are met: +** +** (1) They have the same iTab. +** (2) They have the same iSortIdx. +** (3) The template has same or fewer dependencies than the current loop +** (4) The template has the same or lower cost than the current loop */ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ WhereLoop **ppPrev, *p, *pNext = 0, *pToFree = 0; @@ -5175,6 +5192,11 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ sqlite3 *db = pBuilder->db; WhereInfo *pWInfo = pBuilder->pWInfo; + /* If pBuilder->pBest is defined, then only keep track of the single + ** best WhereLoop. pBuilder->pBest->maskSelf==0 indicates that no + ** prior WhereLoops have been evaluated and that the current pTemplate + ** is therefore the first and hence the best and should be retained. + */ if( (p = pBuilder->pBest)!=0 ){ if( p->maskSelf!=0 ){ if( p->rRun+p->rSetup < pTemplate->rRun+pTemplate->rSetup ){ @@ -5195,7 +5217,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** priority over pTemplate. */ for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){ - if( p->iTab!=pTemplate->iTab ) continue; + if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ) continue; if( (p->prereq & pTemplate->prereq)==p->prereq && p->rSetup<=pTemplate->rSetup && p->rRun<=pTemplate->rRun @@ -5238,7 +5260,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ memcpy(p->aTerm, pTemplate->aTerm, p->nTerm*sizeof(p->aTerm[0])); } if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ - if( p->u.btree.pIndex && p->u.btree.pIndex->tnum==0 ){ + if( p->u.btree.pIndex && p->u.btree.pIndex && p->u.btree.pIndex->tnum==0 ){ p->u.btree.pIndex = 0; } }else{ @@ -5342,6 +5364,36 @@ static int whereLoopAddBtreeIndex( return rc; } +/* +** Return True if it is possible that pIndex might be useful in +** implementing the ORDER BY clause in pBuilder. +** +** Return False if pBuilder does not contain an ORDER BY clause or +** if there is no way for pIndex to be useful in implementing that +** ORDER BY clause. +*/ +static int indexMightHelpWithOrderBy( + WhereLoopBuilder *pBuilder, + Index *pIndex, + int iCursor +){ + ExprList *pOB; + int iCol; + int ii; + + if( (pOB = pBuilder->pOrderBy)==0 ) return 0; + iCol = pIndex->aiColumn[0]; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = pOB->a[ii].pExpr; + if( pExpr->op!=TK_COLUMN ) return 0; + if( pExpr->iTable==iCursor ){ + if( pExpr->iColumn==iCol ) return 1; + return 0; + } + } + return 0; +} + /* ** Add all WhereLoop objects a single table of the join were the table ** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be @@ -5358,9 +5410,11 @@ static int whereLoopAddBtree( struct SrcList_item *pSrc; /* The FROM clause btree term to add */ WhereLoop *pNew; /* Template WhereLoop object */ int rc = SQLITE_OK; /* Return code */ + int iSortIdx = 0; /* Index number */ + int b; /* A boolean value */ double rSize; /* number of rows in the table */ double rLogSize; /* Logarithm of the number of rows in the table */ - + pNew = pBuilder->pNew; pSrc = pBuilder->pTabList->a + pNew->iTab; assert( !IsVirtual(pSrc->pTab) ); @@ -5419,26 +5473,25 @@ static int whereLoopAddBtree( } } - /* Insert a full table scan */ - pNew->u.btree.nEq = 0; - pNew->nTerm = 0; - pNew->rSetup = (double)0; - pNew->prereq = mExtra; - pNew->u.btree.pIndex = 0; - pNew->wsFlags = 0; - pNew->nOut = rSize; - pNew->rRun = rSize + rLogSize; - /* TBD: Reduce nOut using constraints */ - rc = whereLoopInsert(pBuilder, pNew); - /* Loop over all indices */ - for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext){ + for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ pNew->u.btree.nEq = 0; pNew->nTerm = 0; + pNew->iSortIdx = 0; + pNew->rSetup = (double)0; + pNew->prereq = mExtra; + pNew->u.btree.pIndex = pProbe; + b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); if( pProbe->tnum<=0 ){ /* Integer primary key index */ pNew->wsFlags = WHERE_IPK; + + /* Full table scan */ + pNew->nOut = rSize; + pNew->rRun = (rSize + rLogSize)*(3+b); /* 4x penalty for a full-scan */ + rc = whereLoopInsert(pBuilder, pNew); + if( rc ) break; }else{ Bitmask m = pSrc->colUsed; int j; @@ -5448,10 +5501,17 @@ static int whereLoopAddBtree( m &= ~(((Bitmask)1)<wsFlags = m==0 ? WHERE_IDX_ONLY : 0; - } - pNew->u.btree.pIndex = pProbe; + pNew->wsFlags = (m==0) ? WHERE_IDX_ONLY : 0; + /* Full scan via index */ + if( m==0 || b ){ + pNew->iSortIdx = b ? iSortIdx : 0; + pNew->nOut = rSize; + pNew->rRun = (m==0) ? (rSize + rLogSize)*(1+b) : (rSize*rLogSize); + rc = whereLoopInsert(pBuilder, pNew); + if( rc ) break; + } + } rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 1); /* If there was an INDEXED BY clause, then only that one index is @@ -5681,6 +5741,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pNew->rRun = rTotal; pNew->nOut = nRow; pNew->prereq = prereq; + memset(&pNew->u, 0, sizeof(pNew->u)); rc = whereLoopInsert(pBuilder, pNew); } }