From: drh Date: Wed, 14 Sep 2016 01:43:24 +0000 (+0000) Subject: Backport the ORDER BY LIMIT optimization to version 3.8.9. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b5c7f2f23523e0cac80c619b0f39946754a7badb;p=thirdparty%2Fsqlite.git Backport the ORDER BY LIMIT optimization to version 3.8.9. FossilOrigin-Name: db3614825fa02ddacc76b85d76a37aad9d2a9dc8 --- diff --git a/manifest b/manifest index e3dc7c37f3..a62fe6bd61 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.9 -D 2015-04-08T12:16:33.323 +C Backport\sthe\sORDER\sBY\sLIMIT\soptimization\sto\sversion\s3.8.9. +D 2016-09-14T01:43:24.978 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 00d12636df7a5b08af09116bcd6c7bfd49b8b3b4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,12 +230,12 @@ F src/printf.c 8ae1fa9d30c1200a9268a390ba9e9cea9197b27a F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 41aa91af56d960e9414ce1d7c17cfb68e0d1c6cb F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c c28c52e353287434fac8473e56ee4be848d12c9d +F src/select.c 4123c2bdabb48a7f5a9da749bf82558117da1707 F src/shell.c 84a1593bd86aaa14f4da8a8f9b16fbc239d262aa F src/sqlite.h.in 278602140d49575e8708e643161f4263e428a02a F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 107b02ed6c64162b653acc2368e982de529e14f6 +F src/sqliteInt.h 81198400a63a72ac2864421a7be78172abe5cecf F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c e7a09215315a978057fb42c640f890160dbcc45e @@ -307,8 +307,8 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 878c8e1a51cb2ec45c395d26b7d5cd9e1a098e4a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 85d832efa5ef57de542db7f430b72fecd3af8b38 -F src/whereInt.h cbe4aa57326998d89e7698ca65bb7c28541d483c +F src/where.c 3028fd1c63c2c9a8f6897f839b01287a024b3ac9 +F src/whereInt.h 1d1fd0b3b9b56e08f5d3583c70a2c785a3c43941 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -698,6 +698,7 @@ F test/like.test 4f2a71d36a536233727f71995fef900756705e56 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/like3.test 7b0525a39e4f25c4fd113de7e2e28eb712dcdedf F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e +F test/limit2.test ac97b8d07060a0280162ba4159e4c0c765861127 F test/loadext.test 648cb95f324d1775c54a55c12271b2d1156b633b F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 F test/lock.test b984ab9034e7389be0d863fe4e64cbbc4d2028f5 @@ -1249,10 +1250,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8e4ac2ce24415926247961b00a62425ae85d6ffb -R 5875861747bb686954783d9ce4259b86 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.9 * +P 8a8ffc862e96f57aa698f93de10dee28e69f6e09 +R cb2965d27bc117ed4d5058b40e772fea +T *branch * branch-3.8.9 +T *sym-branch-3.8.9 * +T -sym-trunk * U drh -Z 05045ab8d5dbbaefca88cc23b8dca09c +Z 9f0b78af32a328047c54447fa4f1ec76 diff --git a/manifest.uuid b/manifest.uuid index a38b33773e..f18499c30d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a8ffc862e96f57aa698f93de10dee28e69f6e09 \ No newline at end of file +db3614825fa02ddacc76b85d76a37aad9d2a9dc8 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 90aaa842a6..f0c1cbc49d 100644 --- a/src/select.c +++ b/src/select.c @@ -54,6 +54,7 @@ struct SortCtx { int labelBkOut; /* Start label for the block-output subroutine */ int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ + u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */ }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -565,14 +566,35 @@ static void pushOntoSorter( if( pSelect->iLimit ){ int addr; int iLimit; + int r1 = 0; if( pSelect->iOffset ){ iLimit = pSelect->iOffset+1; }else{ iLimit = pSelect->iLimit; } + /* Fill the sorter until it contains LIMIT+OFFSET entries. (The iLimit + ** register is initialized with value of LIMIT+OFFSET.) After the sorter + ** fills up, delete the least entry in the sorter after each insert. + ** Thus we never hold more than the LIMIT+OFFSET rows in memory at once */ addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, -1); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); + if( pSort->bOrderedInnerLoop ){ + r1 = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_Column, pSort->iECursor, nExpr, r1); + VdbeComment((v, "seq")); + } sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); + if( pSort->bOrderedInnerLoop ){ + /* If the inner loop is driven by an index such that values from + ** the same iteration of the inner loop are in sorted order, then + ** immediately jump to the next iteration of an inner loop if the + ** entry from the current iteration does not fit into the top + ** LIMIT+OFFSET entries of the sorter. */ + int iBrk = sqlite3VdbeCurrentAddr(v) + 2; + sqlite3VdbeAddOp3(v, OP_Eq, regBase+nExpr, iBrk, r1); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + VdbeCoverage(v); + } sqlite3VdbeJumpHere(v, addr); } } @@ -5001,6 +5023,7 @@ int sqlite3Select( } if( sSort.pOrderBy ){ sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); + sSort.bOrderedInnerLoop = sqlite3WhereOrderedInnerLoop(pWInfo); if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ sSort.pOrderBy = 0; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 1c0ad44e3a..c7d729cf35 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2276,6 +2276,7 @@ struct SrcList { #define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ #define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ #define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ +#define WHERE_ORDERBY_LIMIT 0x2000 /* ORDERBY+LIMIT on the inner loop */ /* Allowed return values from sqlite3WhereIsDistinct() */ @@ -3289,6 +3290,7 @@ void sqlite3WhereEnd(WhereInfo*); u64 sqlite3WhereOutputRowCount(WhereInfo*); int sqlite3WhereIsDistinct(WhereInfo*); int sqlite3WhereIsOrdered(WhereInfo*); +int sqlite3WhereOrderedInnerLoop(WhereInfo*); int sqlite3WhereIsSorted(WhereInfo*); int sqlite3WhereContinueLabel(WhereInfo*); int sqlite3WhereBreakLabel(WhereInfo*); diff --git a/src/where.c b/src/where.c index 921e683d98..6ebe62fcec 100644 --- a/src/where.c +++ b/src/where.c @@ -42,6 +42,18 @@ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ return pWInfo->nOBSat; } +/* +** Return TRUE if the innermost loop of the WHERE clause implementation +** returns rows in ORDER BY order for complete run of the inner loop. +** +** Across multiple iterations of outer loops, the output rows need not be +** sorted. As long as rows are sorted for just the innermost loop, this +** routine can return TRUE. +*/ +int sqlite3WhereOrderedInnerLoop(WhereInfo *pWInfo){ + return pWInfo->bOrderedInnerLoop; +} + /* ** Return the VDBE address or label to jump to in order to continue ** immediately with the next row of a WHERE clause. @@ -5602,7 +5614,7 @@ static i8 wherePathSatisfiesOrderBy( WhereInfo *pWInfo, /* The WHERE clause */ ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ WherePath *pPath, /* The WherePath to check */ - u16 wctrlFlags, /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */ + u16 wctrlFlags, /* WHERE_GROUPBY, _DISTINCTBY or _ORDERBY_LIMIT */ u16 nLoop, /* Number of entries in pPath->aLoop[] */ WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ @@ -5613,6 +5625,7 @@ static i8 wherePathSatisfiesOrderBy( u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */ u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */ u8 isMatch; /* iColumn matches a term of the ORDER BY clause */ + u16 eqOpMask; /* Allowed equality operators */ u16 nKeyCol; /* Number of key columns in pIndex */ u16 nColumn; /* Total number of ordered columns in the index */ u16 nOrderBy; /* Number terms in the ORDER BY clause */ @@ -5663,8 +5676,16 @@ static i8 wherePathSatisfiesOrderBy( obDone = MASKBIT(nOrderBy)-1; orderDistinctMask = 0; ready = 0; + eqOpMask = WO_EQ | WO_ISNULL; + if( wctrlFlags & WHERE_ORDERBY_LIMIT ) eqOpMask |= WO_IN; for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; + if( iLoopaLoop[iLoop]; + if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue; + }else{ + pLoop = pLast; + } pLoop = iLoopaLoop[iLoop] : pLast; if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ if( pLoop->u.vtab.isOrdered ) obSat = obDone; @@ -5683,8 +5704,16 @@ static i8 wherePathSatisfiesOrderBy( if( pOBExpr->op!=TK_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; pTerm = findTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, - ~ready, WO_EQ|WO_ISNULL, 0); + ~ready, eqOpMask, 0); if( pTerm==0 ) continue; + if( pTerm->eOperator==WO_IN ){ + /* IN terms are only valid for sorting in the ORDER BY LIMIT + ** optimization, and then only if they are actually used + ** by the query plan */ + assert( wctrlFlags & WHERE_ORDERBY_LIMIT ); + for(j=0; jnLTerm && pTerm!=pLoop->aLTerm[j]; j++){} + if( j>=pLoop->nLTerm ) continue; + } if( (pTerm->eOperator&WO_EQ)!=0 && pOBExpr->iColumn>=0 ){ const char *z1, *z2; pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); @@ -5721,10 +5750,12 @@ static i8 wherePathSatisfiesOrderBy( for(j=0; ju.btree.nEq && pLoop->nSkip==0 - && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 + && ((i = pLoop->aLTerm[j]->eOperator) & eqOpMask)!=0 ){ if( i & WO_ISNULL ){ testcase( isOrderDistinct ); @@ -6237,7 +6268,19 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } }else{ pWInfo->nOBSat = pFrom->isOrdered; - if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0; + pWInfo->revMask = pFrom->revLoop; + if( pWInfo->nOBSat<=0 ){ + pWInfo->nOBSat = 0; + if( nLoop>0 && (pFrom->aLoop[nLoop-1]->wsFlags & WHERE_ONEROW)==0 ){ + Bitmask m; + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, + WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m); + if( rc==pWInfo->pOrderBy->nExpr ){ + pWInfo->bOrderedInnerLoop = 1; + pWInfo->revMask = m; + } + } + } pWInfo->revMask = pFrom->revLoop; } if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) diff --git a/src/whereInt.h b/src/whereInt.h index 04cc2029d8..860a67f15f 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -412,6 +412,7 @@ struct WhereInfo { u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ u8 nLevel; /* Number of nested loop */ + u8 bOrderedInnerLoop; /* True if only the inner-most loop is ordered */ int iTop; /* The very beginning of the WHERE loop */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ diff --git a/test/limit2.test b/test/limit2.test new file mode 100644 index 0000000000..0f82ab8d55 --- /dev/null +++ b/test/limit2.test @@ -0,0 +1,115 @@ +# 2016-05-20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the LIMIT in combination with ORDER BY +# and in particular, the optimizations in the inner loop that cause an +# early exit of the inner loop when the LIMIT is reached and the inner +# loop is emitting rows in ORDER BY order. + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test limit2-100 { + CREATE TABLE t1(a,b); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) + INSERT INTO t1(a,b) SELECT 1, (x*17)%1000 + 1000 FROM c; + INSERT INTO t1(a,b) VALUES(2,2),(3,1006),(4,4),(5,9999); + CREATE INDEX t1ab ON t1(a,b); +} +set sqlite_search_count 0 +do_execsql_test limit2-100.1 { + SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY b LIMIT 5; +} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |} +set fast_count $sqlite_search_count +set sqlite_search_count 0 +do_execsql_test limit2-100.2 { + SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY +b LIMIT 5; +} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |} +do_test limit2-100.3 { + set slow_count $sqlite_search_count + expr {$fast_count < 0.02*$slow_count} +} {1} + +do_execsql_test limit2-110 { + CREATE TABLE t2(x,y); + INSERT INTO t2(x,y) VALUES('a',1),('a',2),('a',3),('a',4); + INSERT INTO t2(x,y) VALUES('b',1),('c',2),('d',3),('e',4); + CREATE INDEX t2xy ON t2(x,y); +} +set sqlite_search_count 0 +do_execsql_test limit2-110.1 { + SELECT a, b, '|' FROM t2, t1 WHERE t2.x='a' AND t1.a=t2.y ORDER BY t1.b LIMIT 5; +} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |} +set fast_count $sqlite_search_count +set sqlite_search_count 0 +do_execsql_test limit2-110.2 { + SELECT a, b, '|' FROM t2, t1 WHERE t2.x='a' AND t1.a=t2.y ORDER BY +t1.b LIMIT 5; +} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |} +set slow_count $sqlite_search_count +do_test limit2-110.3 { + expr {$fast_count < 0.02*$slow_count} +} {1} + +do_execsql_test limit2-120 { + DROP INDEX t1ab; + CREATE INDEX t1ab ON t1(a,b DESC); +} +set sqlite_search_count 0 +do_execsql_test limit2-120.1 { + SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY b DESC LIMIT 5; +} {5 9999 | 1 1999 | 1 1998 | 1 1997 | 1 1996 |} +set fast_count $sqlite_search_count +set sqlite_search_count 0 +do_execsql_test limit2-120.2 { + SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY +b DESC LIMIT 5; +} {5 9999 | 1 1999 | 1 1998 | 1 1997 | 1 1996 |} +do_test limit2-120.3 { + set slow_count $sqlite_search_count + expr {$fast_count < 0.02*$slow_count} +} {1} + +# Bug report against the new ORDER BY LIMIT optimization just prior to +# release. (Unreleased so there is no ticket). +# +# Make sure the optimization is not applied if the inner loop can only +# provide a single row of output. +# +do_execsql_test limit2-200 { + CREATE TABLE t200(a, b); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) + INSERT INTO t200(a,b) SELECT x, x FROM c; + CREATE TABLE t201(x INTEGER PRIMARY KEY, y); + INSERT INTO t201(x,y) VALUES(2,12345); + + SELECT *, '|' FROM t200, t201 WHERE x=b ORDER BY y LIMIT 3; +} {2 2 2 12345 |} +do_execsql_test limit2-210 { + SELECT *, '|' FROM t200 LEFT JOIN t201 ON x=b ORDER BY y LIMIT 3; +} {1 1 {} {} | 3 3 {} {} | 4 4 {} {} |} + +# Bug in the ORDER BY LIMIT optimization reported on 2016-09-06. +# Ticket https://www.sqlite.org/src/info/559733b09e96 +# +do_execsql_test limit2-300 { + CREATE TABLE t300(a,b,c); + CREATE INDEX t300x ON t300(a,b,c); + INSERT INTO t300 VALUES(0,1,99),(0,1,0),(0,0,0); + SELECT *,'.' FROM t300 WHERE a=0 AND (c=0 OR c=99) ORDER BY c DESC; +} {0 1 99 . 0 0 0 . 0 1 0 .} +do_execsql_test limit2-310 { + SELECT *,'.' FROM t300 WHERE a=0 AND (c=0 OR c=99) ORDER BY c DESC LIMIT 1; +} {0 1 99 .} + + + + +finish_test