From: drh Date: Sat, 29 Sep 2012 19:10:29 +0000 (+0000) Subject: Improved ORDER BY optimization when outer loops of a join return a single row. X-Git-Tag: version-3.7.15~107 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=60441af4f258a4f4d6a2bf6965c145de01b75e22;p=thirdparty%2Fsqlite.git Improved ORDER BY optimization when outer loops of a join return a single row. FossilOrigin-Name: 62225b4a4c4bfe1820ef54cb202edf2cd866429f --- diff --git a/manifest b/manifest index 9706885348..9f30517df5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\sbigfile\stests\son\sMacs. -D 2012-09-29T15:45:12.074 +C Improved\sORDER\sBY\soptimization\swhen\souter\sloops\sof\sa\sjoin\sreturn\sa\ssingle\srow. +D 2012-09-29T19:10:29.355 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f4f26109f9d80829122e0e09f9cda008fa065fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vtab.c d8020c0a0e8ccc490ca449d7e665311b6e9f3ba9 F src/wal.c 5acb3e7bbd31f10ba39acad9ce6b399055337a9d F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c d836df3a2096c41c39e48ab5636f09f94ba02676 +F src/where.c acc2ec5f6879721f332223da393777438ea5a606 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00 @@ -635,6 +635,7 @@ F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/orderby1.test 31c9865626046666e81cd22ecf8e1c24a4ea41b6 +F test/orderby2.test 295d6639e1ce522195354b88ab298d7ede169736 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 2163c6ef119f497a71a84137c957c63763e640ab F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 @@ -1017,7 +1018,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P fd74d3d91721ca404537f195fed04c9edef20bf2 -R acef59cbf66888f83c956993bdbbc818 +P d869eddaf208c4bf03f6bd1848f510392f9dba49 +R 13af348b1bf68a7c36428c6258d6aba5 U drh -Z 60d0d33f11d050cbca81ed7def74feb5 +Z 7bbfe622f493f625120e8c012f0943a5 diff --git a/manifest.uuid b/manifest.uuid index 3a58b39b0f..2aa22ae871 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d869eddaf208c4bf03f6bd1848f510392f9dba49 \ No newline at end of file +62225b4a4c4bfe1820ef54cb202edf2cd866429f \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7f386289c6..708fe7a1d8 100644 --- a/src/where.c +++ b/src/where.c @@ -257,10 +257,11 @@ struct WhereCost { #define WHERE_TOP_LIMIT 0x00100000 /* xEXPR or x>=EXPR constraint */ #define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and xpParse; /* Parser context */ - sqlite3 *db = pParse->db; /* Database connection */ - int nPriorSat; /* ORDER BY terms satisfied by outer loops */ - int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ - int nEqOneRow; /* Idx columns that ref unique values */ - - if( p->i==0 ){ - nPriorSat = 0; - nEqOneRow = nEqCol; - }else{ - if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; - nPriorSat = p->aLevel[p->i-1].plan.nOBSat; - sortOrder = bOuterRev; - nEqOneRow = 0; - } - if( p->i>0 && nEqCol==0 /*&& !allOuterLoopsUnique(p)*/ ) return nPriorSat; - pOrderBy = p->pOrderBy; - if( !pOrderBy ) return nPriorSat; - if( wsFlags & WHERE_COLUMN_IN ) return nPriorSat; - if( pIdx->bUnordered ) return nPriorSat; - nTerm = pOrderBy->nExpr; - assert( nTerm>0 ); - - /* Argument pIdx must either point to a 'real' named index structure, - ** or an index structure allocated on the stack by bestBtreeIndex() to - ** represent the rowid index that is part of every table. */ - assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) ); - - /* Match terms of the ORDER BY clause against columns of - ** the index. - ** - ** Note that indices have pIdx->nColumn regular columns plus - ** one additional column containing the rowid. The rowid column - ** of the index is also allowed to match against the ORDER BY - ** clause. - */ - for(i=0,j=nPriorSat,pTerm=&pOrderBy->a[j]; jnColumn; i++){ - Expr *pExpr; /* The expression of the ORDER BY pTerm */ - CollSeq *pColl; /* The collating sequence of pExpr */ - int termSortOrder; /* Sort order for this term */ - int iColumn; /* The i-th column of the index. -1 for rowid */ - int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ - const char *zColl; /* Name of the collating sequence for i-th index term */ - - pExpr = pTerm->pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ){ - /* Can not use an index sort on anything that is not a column in the - ** left-most table of the FROM clause */ - break; - } - pColl = sqlite3ExprCollSeq(pParse, pExpr); - if( !pColl ){ - pColl = db->pDfltColl; - } - if( pIdx->zName && inColumn ){ - iColumn = pIdx->aiColumn[i]; - if( iColumn==pIdx->pTable->iPKey ){ - iColumn = -1; - } - iSortOrder = pIdx->aSortOrder[i]; - zColl = pIdx->azColl[i]; - }else{ - iColumn = -1; - iSortOrder = 0; - zColl = pColl->zName; - } - if( pExpr->iColumn!=iColumn || sqlite3StrICmp(pColl->zName, zColl) ){ - /* Term j of the ORDER BY clause does not match column i of the index */ - if( inColumn ){ - /* Index column i is the rowid. All other terms match. */ - break; - }else{ - /* If an index column fails to match and is not constrained by == - ** then the index cannot satisfy the ORDER BY constraint. - */ - return nPriorSat; - } - } - assert( pIdx->aSortOrder!=0 || iColumn==-1 ); - assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); - assert( iSortOrder==0 || iSortOrder==1 ); - termSortOrder = iSortOrder ^ pTerm->sortOrder; - if( i>nEqOneRow ){ - if( termSortOrder!=sortOrder ){ - /* Indices can only be used if all ORDER BY terms past the - ** equality constraints are all either DESC or ASC. */ - break; - } - }else{ - sortOrder = termSortOrder; - } - j++; - pTerm++; - if( iColumn<0 ){ - seenRowid = 1; - break; - } - } - *pbRev = sortOrder; - - /* If there was an "ORDER BY rowid" term that matched, or it is only - ** possible for a single row from this table to match, then skip over - ** any additional ORDER BY terms dealing with this table. - */ - if( seenRowid || - ( (wsFlags & WHERE_COLUMN_NULL)==0 - && i>=pIdx->nColumn - && indexIsUniqueNotNull(pIdx, nEqCol) - ) - ){ - /* Advance j over additional ORDER BY terms associated with base */ - WhereMaskSet *pMS = p->pWC->pMaskSet; - Bitmask m = ~getMask(pMS, base); - while( ja[j].pExpr)&m)==0 ){ - j++; - } - } - return j; -} - /* ** Prepare a crude estimate of the logarithm of the input value. ** The results need not be exact. This is only used for estimating @@ -2884,6 +2725,9 @@ static int isOrderedColumn(WhereBestIdx *p, int iTab, int iCol, int *pbRev){ u8 sortOrder; for(i=p->i-1; i>=0; i--, pLevel--){ if( pLevel->iTabCur!=iTab ) continue; + if( (pLevel->plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ + return 1; + } if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ pIdx = pLevel->plan.u.pIdx; if( iCol<0 ){ @@ -2926,9 +2770,6 @@ static int isOrderedTerm(WhereBestIdx *p, WhereTerm *pTerm, int *pbRev){ assert( pExpr->op==TK_EQ ); assert( pExpr->pLeft!=0 && pExpr->pLeft->op==TK_COLUMN ); assert( pExpr->pRight!=0 ); - if( p->i==0 ){ - return 1; /* All == are ordered in the outer loop */ - } if( pTerm->prereqRight==0 ){ return 1; /* RHS of the == is a constant */ } @@ -2942,6 +2783,168 @@ static int isOrderedTerm(WhereBestIdx *p, WhereTerm *pTerm, int *pbRev){ return 0; } +/* +** This routine decides if pIdx can be used to satisfy the ORDER BY +** clause, either in whole or in part. The return value is the +** cumulative number of terms in the ORDER BY clause that are satisfied +** by the index pIdx and other indices in outer loops. +** +** The table being queried has a cursor number of "base". pIdx is the +** index that is postulated for use to access the table. +** +** nEqCol is the number of columns of pIdx that are used as equality +** constraints and where the other side of the == is an ordered column +** or constant. An "order column" in the previous sentence means a column +** in table from an outer loop whose values will always appear in the +** correct order due to othre index, or because the outer loop generates +** a unique result. Any of the first nEqCol columns of pIdx may be missing +** from the ORDER BY clause and the match can still be a success. +** +** The *pbRev value is set to 0 order 1 depending on whether or not +** pIdx should be run in the forward order or in reverse order. +*/ +static int isSortingIndex( + WhereBestIdx *p, /* Best index search context */ + Index *pIdx, /* The index we are testing */ + int base, /* Cursor number for the table to be sorted */ + int nEqCol, /* Number of index columns with ordered == constraints */ + int wsFlags, /* Index usages flags */ + int bOuterRev, /* True if outer loops scan in reverse order */ + int *pbRev /* Set to 1 for reverse-order scan of pIdx */ +){ + int i; /* Number of pIdx terms used */ + int j; /* Number of ORDER BY terms satisfied */ + int sortOrder = 0; /* XOR of index and ORDER BY sort direction */ + int nTerm; /* Number of ORDER BY terms */ + struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Parse *pParse = p->pParse; /* Parser context */ + sqlite3 *db = pParse->db; /* Database connection */ + int nPriorSat; /* ORDER BY terms satisfied by outer loops */ + int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ + int nEqOneRow; /* Idx columns that ref unique values */ + + if( p->i==0 ){ + nPriorSat = 0; + }else{ + nPriorSat = p->aLevel[p->i-1].plan.nOBSat; + if( OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return nPriorSat; + } + if( p->i==0 || (p->aLevel[p->i-1].plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ + nEqOneRow = nEqCol; + }else{ + if( nEqCol==0 ) return nPriorSat; + sortOrder = bOuterRev; + nEqOneRow = 0; + } + pOrderBy = p->pOrderBy; + assert( pOrderBy!=0 ); + if( wsFlags & WHERE_COLUMN_IN ) return nPriorSat; + if( pIdx->bUnordered ) return nPriorSat; + nTerm = pOrderBy->nExpr; + assert( nTerm>0 ); + + /* Argument pIdx must either point to a 'real' named index structure, + ** or an index structure allocated on the stack by bestBtreeIndex() to + ** represent the rowid index that is part of every table. */ + assert( pIdx->zName || (pIdx->nColumn==1 && pIdx->aiColumn[0]==-1) ); + + /* Match terms of the ORDER BY clause against columns of + ** the index. + ** + ** Note that indices have pIdx->nColumn regular columns plus + ** one additional column containing the rowid. The rowid column + ** of the index is also allowed to match against the ORDER BY + ** clause. + */ + for(i=0,j=nPriorSat,pTerm=&pOrderBy->a[j]; jnColumn; i++){ + Expr *pExpr; /* The expression of the ORDER BY pTerm */ + CollSeq *pColl; /* The collating sequence of pExpr */ + int termSortOrder; /* Sort order for this term */ + int iColumn; /* The i-th column of the index. -1 for rowid */ + int iSortOrder; /* 1 for DESC, 0 for ASC on the i-th index term */ + const char *zColl; /* Name of the collating sequence for i-th index term */ + + pExpr = pTerm->pExpr; + if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ){ + /* Can not use an index sort on anything that is not a column in the + ** left-most table of the FROM clause */ + break; + } + pColl = sqlite3ExprCollSeq(pParse, pExpr); + if( !pColl ){ + pColl = db->pDfltColl; + } + if( pIdx->zName && inColumn ){ + iColumn = pIdx->aiColumn[i]; + if( iColumn==pIdx->pTable->iPKey ){ + iColumn = -1; + } + iSortOrder = pIdx->aSortOrder[i]; + zColl = pIdx->azColl[i]; + }else{ + iColumn = -1; + iSortOrder = 0; + zColl = pColl->zName; + } + if( pExpr->iColumn!=iColumn || sqlite3StrICmp(pColl->zName, zColl) ){ + /* Term j of the ORDER BY clause does not match column i of the index */ + if( inColumn ){ + /* Index column i is the rowid. All other terms match. */ + break; + }else{ + /* If an index column fails to match and is not constrained by == + ** then the index cannot satisfy the ORDER BY constraint. + */ + return nPriorSat; + } + } + assert( pIdx->aSortOrder!=0 || iColumn==-1 ); + assert( pTerm->sortOrder==0 || pTerm->sortOrder==1 ); + assert( iSortOrder==0 || iSortOrder==1 ); + termSortOrder = iSortOrder ^ pTerm->sortOrder; + if( i>nEqOneRow ){ + if( termSortOrder!=sortOrder ){ + /* Indices can only be used if all ORDER BY terms past the + ** equality constraints are all either DESC or ASC. */ + break; + } + }else{ + sortOrder = termSortOrder; + } + j++; + pTerm++; + if( iColumn<0 ){ + seenRowid = 1; + break; + } + } + *pbRev = sortOrder; + + /* If there was an "ORDER BY rowid" term that matched, or it is only + ** possible for a single row from this table to match, then skip over + ** any additional ORDER BY terms dealing with this table. + */ + if( seenRowid || + ( (wsFlags & WHERE_COLUMN_NULL)==0 + && i>=pIdx->nColumn + && indexIsUniqueNotNull(pIdx, nEqCol) + ) + ){ + /* Advance j over additional ORDER BY terms associated with base */ + WhereMaskSet *pMS = p->pWC->pMaskSet; + Bitmask m = ~getMask(pMS, base); + while( ja[j].pExpr)&m)==0 ){ + j++; + } + } + return j; +} /* ** Find the best query plan for accessing a particular table. Write the @@ -3177,6 +3180,9 @@ static void bestBtreeIndex(WhereBestIdx *p){ testcase( wsFlags & WHERE_COLUMN_NULL ); if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ wsFlags |= WHERE_UNIQUE; + if( p->i==0 || (p->aLevel[p->i-1].plan.wsFlags & WHERE_ALL_UNIQUE)!=0 ){ + wsFlags |= WHERE_ALL_UNIQUE; + } } }else if( pProbe->bUnordered==0 ){ int j = (nEq==pProbe->nColumn ? -1 : pProbe->aiColumn[nEq]); diff --git a/test/orderby2.test b/test/orderby2.test new file mode 100644 index 0000000000..f07cc57b00 --- /dev/null +++ b/test/orderby2.test @@ -0,0 +1,71 @@ +# 2012 Sept 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing that the optimizations that disable +# ORDER BY clauses when the natural order of a query is correct. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix orderby2 + +# Generate test data for a join. Verify that the join gets the +# correct answer. +# +do_test 1.0 { + db eval { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1,11), (2,22); + CREATE TABLE t2(d, e, UNIQUE(d,e)); + INSERT INTO t2 VALUES(10, 'ten'), (11,'eleven'), (12,'twelve'), + (11, 'oneteen'); + } +} {} + +do_test 1.1a { + db eval { + SELECT e FROM t1, t2 WHERE a=1 AND d=b ORDER BY d, e; + } +} {eleven oneteen} +do_test 1.1b { + db eval { + EXPLAIN QUERY PLAN + SELECT e FROM t1, t2 WHERE a=1 AND d=b ORDER BY d, e; + } +} {~/ORDER BY/} + +do_test 1.2a { + db eval { + SELECT e FROM t1, t2 WHERE a=1 AND d=b ORDER BY e; + } +} {eleven oneteen} +do_test 1.2b { + db eval { + EXPLAIN QUERY PLAN + SELECT e FROM t1, t2 WHERE a=1 AND d=b ORDER BY e; + } +} {~/ORDER BY/} + +do_test 1.3a { + db eval { + SELECT e, b FROM t1, t2 WHERE a=1 ORDER BY d, e; + } +} {ten 11 eleven 11 oneteen 11 twelve 11} +do_test 1.3b { + db eval { + EXPLAIN QUERY PLAN + SELECT e, b FROM t1, t2 WHERE a=1 ORDER BY d, e; + } +} {~/ORDER BY/} + + +finish_test