From: dan Date: Thu, 30 Jun 2011 20:17:15 +0000 (+0000) Subject: Experimental changes to improve optimization of DISTINCT queries. X-Git-Tag: version-3.7.8~38^2~31^2~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=38cc40c216f080dd309cb47f9e7fc976ba4fc2dd;p=thirdparty%2Fsqlite.git Experimental changes to improve optimization of DISTINCT queries. FossilOrigin-Name: f7ba0219ef2f235543c258be736955d91ca5ecce --- diff --git a/manifest b/manifest index fe85fe4d38..4197d1e43f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Pass\sthe\sBTREE_UNORDERED\shint\sinto\sboth\ssqlite3BtreeOpen()\sand\sinto\nsqlite3BtreeCreateTable(). -D 2011-06-29T17:11:39.745 +C Experimental\schanges\sto\simprove\soptimization\sof\sDISTINCT\squeries. +D 2011-06-30T20:17:15.042 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -132,10 +132,10 @@ F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3 -F src/delete.c 4925f9121525fc871f5d8d13c1f7dcc91abb38bb +F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8 F src/expr.c ab46ab0f0c44979a8164ca31728d7d10ae5e8106 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c 9fabba17a4d4778dc660f0cb9d781fc86d7b9d41 +F src/fkey.c c8492fed772af1ed61251582707266227612b45b F src/func.c 59bb046d7e3df1ab512ac339ccb0a6f996a17cb7 F src/global.c c70a46f28680f8d7c097dbc0430ccf3b932e90b0 F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af @@ -179,11 +179,11 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff +F src/select.c cf259606f91f53f71bafd7a2da47f30cda6efc58 F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 F src/sqlite.h.in 4b7255c10d39c5faf089dbd29cde7c367ff39f1f F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 -F src/sqliteInt.h 7b72f7c6929757c0678d94524dbc7e7e3d6dc398 +F src/sqliteInt.h 58df570a42d605f8e53d6eb920a16ca291d5822e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -234,7 +234,7 @@ F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c c819d9f72168a035d545a5bdafe9b085b20df705 F src/trigger.c c836a6caac16ba96611558922106858f6ca3d6bf -F src/update.c 80d77311d91ebc06b27149e75701f1b3e9356622 +F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e @@ -250,7 +250,7 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c 55403ce19c506be6a321c7f129aff693d6103db5 +F src/where.c 0bdcf6704dd0d8471052712e1d63a02990b7d0bf F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -322,7 +322,7 @@ F test/collate1.test e3eaa48c21e150814be1a7b852d2a8af24458d04 F test/collate2.test 04cebe4a033be319d6ddbb3bbc69464e01700b49 F test/collate3.test d28d2cfab2c3a3d4628ae4b2b7afc9965daa3b4c F test/collate4.test 3d3f123f83fd8ccda6f48d617e44e661b9870c7d -F test/collate5.test fe0f43c4740d7b71b959cac668d19e42f2e06e4d +F test/collate5.test 67f1d3e848e230ff4802815a79acb0a8b5e69bd7 F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907 F test/collate7.test fac8db7aac3978466c04ae892cc74dcf2bc031aa F test/collate8.test df26649cfcbddf109c04122b340301616d3a88f6 @@ -379,7 +379,7 @@ F test/e_fts3.test 75bb0aee26384ef586165e21018a17f7cd843469 F test/e_insert.test 76d4bb5da9b28014d515d91ffe29a79a1e99f2bc F test/e_reindex.test a064f0878b8f848fbca38f1f61f82f15a3000c64 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 -F test/e_select.test 7ac53674e822d4d77bbb4a9a4aaefa5fdc9e493f +F test/e_select.test 8d7fac7a268eaeb80b9a7ba7964505b9d30f5458 F test/e_select2.test 5c3d3da19c7b3e90ae444579db2b70098599ab92 F test/e_update.test b926341a65955d69a6375c9eb4fd82e7089bc83a F test/e_uri.test 6f35b491f80dac005c8144f38b2dfb4d96483596 @@ -491,7 +491,7 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test dd7001ac86d09c154a7dff064f4739c60e2b312c -F test/fuzzer1.test 3105b5a89a6cb0d475f0877debec942fe4143462 +F test/fuzzer1.test ddfb04f3bd5cfdda3b1aa15b78d3ad055c9cc50f F test/hook.test f2277c309e4ee8067d95d6b9b315568e9d5329b2 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 19b642bb134308980a92249750ea4ce3f6c75c2d @@ -514,7 +514,7 @@ F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test aef273dd1cee84cc92407469e6bd1b3cdcb76908 F test/insert2.test 4f3a04d168c728ed5ec2c88842e772606c7ce435 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 -F test/insert4.test b3e02648a5fc3075c29e13c369b5127bf859b5a2 +F test/insert4.test 63ea672b0fc6d3a9a0ccee774a771510b1e684c4 F test/insert5.test 1f93cbe9742110119133d7e8e3ccfe6d7c249766 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test 42e7cf98646fd9cb4a3b131a93ed3c50b9e149f1 @@ -590,7 +590,7 @@ F test/misc1.test e56baf44656dd68d6475a4b44521045a60241e9b F test/misc2.test a628db7b03e18973e5d446c67696b03de718c9fd F test/misc3.test 72c5dc87a78e7865c5ec7a969fc572913dbe96b6 F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 -F test/misc5.test 45b2e3ed5f79af2b4f38ae362eaf4c49674575bd +F test/misc5.test 9f9338f8211c7f5d1cbe16331fa65d019501aa50 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test 29032efcd3d826fbd409e2a7af873e7939f4a4e3 F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33 @@ -655,7 +655,7 @@ F test/select7.test dad6f00f0d49728a879d6eb6451d4752db0b0abe F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d F test/select9.test 74c0fb2c6eecb0219cbed0cbe3df136f8fbf9343 F test/selectA.test 06d1032fa9009314c95394f2ca2e60d9f7ae8532 -F test/selectB.test f305cc6660804cb239aab4e2f26b0e288b59958b +F test/selectB.test 0d072c5846071b569766e6cd7f923f646a8b2bfa F test/selectC.test f9bf1bc4581b5b8158caa6e4e4f682acb379fb25 F test/server1.test f5b790d4c0498179151ca8a7715a65a7802c859c F test/shared.test b9114eaea7e748a3a4c8ff7b9ca806c8f95cef3e @@ -949,7 +949,10 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P b9477eb056d120826ed82b0d65e6f27b5d0c087a -R 04bb3d810ba550780b3b6fda2c65d8d3 -U drh -Z 9a15d45fcfccc0dc2281a762dda887ad +P 591de898f41630156cc0fc6ef17dd3ee5e7c479f +R 5673acf06c190a75a743da9daebc8ac9 +T *branch * experimental +T *sym-experimental * +T -sym-trunk * +U dan +Z ab52fc5a81a2f6dd6b96daa429077988 diff --git a/manifest.uuid b/manifest.uuid index e1c5a804ad..5aa8e13950 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -591de898f41630156cc0fc6ef17dd3ee5e7c479f \ No newline at end of file +f7ba0219ef2f235543c258be736955d91ca5ecce \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index ec0ecec7e2..147a5ca898 100644 --- a/src/delete.c +++ b/src/delete.c @@ -371,7 +371,9 @@ void sqlite3DeleteFrom( /* Collect rowids of every row to be deleted. */ sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,WHERE_DUPLICATES_OK); + pWInfo = sqlite3WhereBegin( + pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK + ); if( pWInfo==0 ) goto delete_from_cleanup; regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid); sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid); diff --git a/src/fkey.c b/src/fkey.c index dda8a50d44..37d4744dda 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -560,7 +560,7 @@ static void fkScanChildren( ** clause. If the constraint is not deferred, throw an exception for ** each row found. Otherwise, for deferred constraints, increment the ** deferred constraint counter by nIncr for each row selected. */ - pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0); + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0); if( nIncr>0 && pFKey->isDeferred==0 ){ sqlite3ParseToplevel(pParse)->mayAbort = 1; } diff --git a/src/select.c b/src/select.c index 8dcfb84b8a..00107e1de6 100644 --- a/src/select.c +++ b/src/select.c @@ -3721,6 +3721,7 @@ int sqlite3Select( int distinct; /* Table to use for the distinct set */ int rc = 1; /* Value to return from this function */ int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ + int addrDistinctIndex; /* Address of an OP_OpenEphemeral instruction */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ @@ -3850,12 +3851,14 @@ int sqlite3Select( /* If possible, rewrite the query to use GROUP BY instead of DISTINCT. ** GROUP BY might use an index, DISTINCT never does. */ +#if 0 assert( p->pGroupBy==0 || (p->selFlags & SF_Aggregate)!=0 ); if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ){ p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; p->selFlags &= ~SF_Distinct; } +#endif /* If there is both a GROUP BY and an ORDER BY clause and they are ** identical, then disable the ORDER BY clause since the GROUP BY @@ -3904,11 +3907,10 @@ int sqlite3Select( */ if( p->selFlags & SF_Distinct ){ KeyInfo *pKeyInfo; - assert( isAgg || pGroupBy ); distinct = pParse->nTab++; pKeyInfo = keyInfoFromExprList(pParse, p->pEList); - sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + addrDistinctIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, + (char*)pKeyInfo, P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); }else{ distinct = -1; @@ -3916,10 +3918,10 @@ int sqlite3Select( /* Aggregate and non-aggregate queries are handled differently */ if( !isAgg && pGroupBy==0 ){ - /* This case is for non-aggregate queries - ** Begin the database scan - */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0); + ExprList *pDist = (isDistinct ? p->pEList : 0); + + /* Begin the database scan. */ + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, pDist, 0); if( pWInfo==0 ) goto select_end; if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; @@ -3932,10 +3934,47 @@ int sqlite3Select( p->addrOpenEphm[2] = -1; } - /* Use the standard inner loop - */ - assert(!isDistinct); - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, -1, pDest, + if( pWInfo->eDistinct ){ + assert( isDistinct ); + assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED + || pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE + ); + distinct = -1; + if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED ){ + int iJump; + int iExpr; + int iFlag = ++pParse->nMem; + int iBase = pParse->nMem+1; + int iBase2 = iBase + pEList->nExpr; + pParse->nMem += (pEList->nExpr*2); + VdbeOp *pOp; + + /* Change the OP_OpenEphemeral coded earlier to an OP_Integer. The + ** OP_Integer initializes the "first row" flag. */ + pOp = sqlite3VdbeGetOp(v, addrDistinctIndex); + pOp->opcode = OP_Integer; + pOp->p1 = 1; + pOp->p2 = iFlag; + + sqlite3ExprCodeExprList(pParse, pEList, iBase, 1); + iJump = sqlite3VdbeCurrentAddr(v) + 1 + pEList->nExpr + 1 + 1; + sqlite3VdbeAddOp2(v, OP_If, iFlag, iJump-1); + for(iExpr=0; iExprnExpr; iExpr++){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[iExpr].pExpr); + sqlite3VdbeAddOp3(v, OP_Ne, iBase+iExpr, iJump, iBase2+iExpr); + sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iContinue); + + sqlite3VdbeAddOp2(v, OP_Integer, 0, iFlag); + assert( sqlite3VdbeCurrentAddr(v)==iJump ); + sqlite3VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr); + } + } + + /* Use the standard inner loop. */ + selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, pDest, pWInfo->iContinue, pWInfo->iBreak); /* End the database scan loop. @@ -4045,7 +4084,7 @@ int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0); if( pWInfo==0 ) goto select_end; if( pGroupBy==0 ){ /* The optimizer is able to deliver rows in group by order so @@ -4307,7 +4346,7 @@ int sqlite3Select( ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, 0, flag); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDel); goto select_end; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 09f64b7939..c1e055b267 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1966,6 +1966,7 @@ struct WhereInfo { u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE or DELETE */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ + u8 eDistinct; SrcList *pTabList; /* List of tables in the join */ int iTop; /* The very beginning of the WHERE loop */ int iContinue; /* Jump here to continue with next record */ @@ -1977,6 +1978,9 @@ struct WhereInfo { WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; +#define WHERE_DISTINCT_UNIQUE 1 +#define WHERE_DISTINCT_ORDERED 2 + /* ** A NameContext defines a context in which to resolve table and column ** names. The context consists of a list of tables (the pSrcList) field and @@ -2738,7 +2742,7 @@ Expr *sqlite3LimitWhere(Parse *, SrcList *, Expr *, ExprList *, Expr *, Expr *, #endif void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); -WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**, u16); +WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16); void sqlite3WhereEnd(WhereInfo*); int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); diff --git a/src/update.c b/src/update.c index d445f01f4e..2b5efff37e 100644 --- a/src/update.c +++ b/src/update.c @@ -311,7 +311,9 @@ void sqlite3Update( /* Begin the database scan */ sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0, WHERE_ONEPASS_DESIRED); + pWInfo = sqlite3WhereBegin( + pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED + ); if( pWInfo==0 ) goto update_cleanup; okOnePass = pWInfo->okOnePass; diff --git a/src/where.c b/src/where.c index cf30d94d67..8aecf9788d 100644 --- a/src/where.c +++ b/src/where.c @@ -253,6 +253,7 @@ struct WhereCost { #define WHERE_VIRTUALTABLE 0x08000000 /* Use virtual-table processing */ #define WHERE_MULTI_OR 0x10000000 /* OR using multiple indices */ #define WHERE_TEMP_INDEX 0x20000000 /* Uses an ephemeral index */ +#define WHERE_DISTINCT 0x40000000 /* Correct order for DISTINCT */ /* ** Initialize a preallocated WhereClause structure. @@ -1433,7 +1434,10 @@ static int isSortingIndex( struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ sqlite3 *db = pParse->db; - assert( pOrderBy!=0 ); + if( !pOrderBy ) return 0; + if( wsFlags & WHERE_COLUMN_IN ) return 0; + if( pIdx->bUnordered ) return 0; + nTerm = pOrderBy->nExpr; assert( nTerm>0 ); @@ -1523,7 +1527,7 @@ static int isSortingIndex( } } - *pbRev = sortOrder!=0; + if( pbRev ) *pbRev = sortOrder!=0; if( j>=nTerm ){ /* All terms of the ORDER BY clause are covered by this index so ** this index can be used for sorting. */ @@ -2689,6 +2693,7 @@ static void bestBtreeIndex( Bitmask notReady, /* Mask of cursors not available for indexing */ Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ + ExprList *pDistinct, /* The select-list if query is DISTINCT */ WhereCost *pCost /* Lowest cost query plan */ ){ int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ @@ -2829,7 +2834,8 @@ static void bestBtreeIndex( int nInMul = 1; /* Number of distinct equalities to lookup */ int estBound = 100; /* Estimated reduction in search space */ int nBound = 0; /* Number of range constraints seen */ - int bSort = 0; /* True if external sort required */ + int bSort = !!pOrderBy; /* True if external sort required */ + int bDist = !!pDistinct; /* True if index cannot help with DISTINCT */ int bLookup = 0; /* True if not a covering index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ #ifdef SQLITE_ENABLE_STAT2 @@ -2893,17 +2899,22 @@ static void bestBtreeIndex( ** naturally scan rows in the required order, set the appropriate flags ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index ** will scan rows in a different order, set the bSort variable. */ - if( pOrderBy ){ - if( (wsFlags & WHERE_COLUMN_IN)==0 - && pProbe->bUnordered==0 - && isSortingIndex(pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, - nEq, wsFlags, &rev) - ){ - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; - wsFlags |= (rev ? WHERE_REVERSE : 0); - }else{ - bSort = 1; - } + if( isSortingIndex( + pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, nEq, wsFlags, &rev) + ){ + bSort = 0; + wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; + wsFlags |= (rev ? WHERE_REVERSE : 0); + } + + /* If there is a DISTINCT qualifier and this index will scan rows in + ** order of the DISTINCT expressions, clear bDist and set the appropriate + ** flags in wsFlags. */ + if( isSortingIndex( + pParse, pWC->pMaskSet, pProbe, iCur, pDistinct, nEq, wsFlags, 0) + ){ + bDist = 0; + wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; } /* If currently calculating the cost of using an index (not the IPK @@ -3020,6 +3031,9 @@ static void bestBtreeIndex( if( bSort ){ cost += nRow*estLog(nRow)*3; } + if( bDist ){ + cost += nRow*estLog(nRow)*3; + } /**** Cost of using this index has now been computed ****/ @@ -3165,7 +3179,7 @@ static void bestIndex( }else #endif { - bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); + bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, 0, pCost); } } @@ -4127,7 +4141,7 @@ static Bitmask codeOneLoopStart( if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ /* Loop through table entries that match term pOrTerm. */ - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, 0, WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY); if( pSubWInfo ){ @@ -4274,6 +4288,79 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } } +/* +** Return true if the DISTINCT expression-list passed as the third argument +** is redundant. A DISTINCT list is redundant if the database contains a +** UNIQUE index that guarantees that the result of the query will be distinct +** anyway. +*/ +static int whereDistinctRedundant( + Parse *pParse, + SrcList *pTabList, + WhereClause *pWC, + ExprList *pDistinct +){ + Table *pTab; + Index *pIdx; + int i; + int iBase; + + /* If there is more than one table or sub-select in the FROM clause of + ** this query, then it will not be possible to show that the DISTINCT + ** clause is redundant. */ + if( pTabList->nSrc!=1 ) return 0; + iBase = pTabList->a[0].iCursor; + pTab = pTabList->a[0].pTab; + + /* Check if all the expressions in the ExprList are of type TK_COLUMN and + ** on the same table. If this is not the case, return early, since it will + ** not be possible to prove that the DISTINCT qualifier is redundant. + ** If any of the expressions is an IPK column, then return true. + */ + for(i=0; inExpr; i++){ + Expr *p = pDistinct->a[i].pExpr; + if( p->op!=TK_COLUMN || p->iTable!=iBase ) return 0; + if( p->iColumn<0 ) return 1; + } + + /* Loop through all indices on the table, checking each to see if it makes + ** the DISTINCT qualifier redundant. It does so if: + ** + ** 1. The index is itself UNIQUE, and + ** + ** 2. All of the columns in the index are either part of the pDistinct + ** list, or else the WHERE clause contains a term of the form "col=X", + ** where X is a constant value. The collation sequences of the + ** comparison and select-list expressions must match those of the index. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->onError==OE_None ) continue; + for(i=0; inColumn; i++){ + int iCol = pIdx->aiColumn[i]; + const char *zColl = pIdx->azColl[i]; + + if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ + int j; + for(j=0; jnExpr; j++){ + Expr *p = pDistinct->a[j].pExpr; + if( p->iColumn==iCol ){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, p); + const char *zEColl = (pColl ? pColl : pParse->db->pDfltColl)->zName; + if( 0==sqlite3StrICmp(zColl, zEColl) ) break; + } + } + if( j==pDistinct->nExpr ) break; + } + } + if( i==pIdx->nColumn ){ + /* This index implies that the DISTINCT qualifier is redundant. */ + return 1; + } + } + + return 0; +} + /* ** Generate the beginning of the loop used for WHERE clause processing. @@ -4368,6 +4455,7 @@ WhereInfo *sqlite3WhereBegin( SrcList *pTabList, /* A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ ExprList **ppOrderBy, /* An ORDER BY clause, or NULL */ + ExprList *pDistinct, /* The select-list for DISTINCT queries - or NULL */ u16 wctrlFlags /* One of the WHERE_* flags defined in sqliteInt.h */ ){ int i; /* Loop counter */ @@ -4495,6 +4583,15 @@ WhereInfo *sqlite3WhereBegin( goto whereBeginError; } + /* Check if the DISTINCT qualifier, if there is one, is redundant. + ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to + ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. + */ + if( pDistinct && whereDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){ + pDistinct = 0; + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } + /* Chose the best index to use for each table in the FROM clause. ** ** This loop fills in the following fields: @@ -4578,6 +4675,7 @@ WhereInfo *sqlite3WhereBegin( int doNotReorder; /* True if this table should not be reordered */ WhereCost sCost; /* Cost information from best[Virtual]Index() */ ExprList *pOrderBy; /* ORDER BY clause for index to optimize */ + ExprList *pDist; /* DISTINCT clause for index to optimize */ doNotReorder = (pTabItem->jointype & (JT_LEFT|JT_CROSS))!=0; if( j!=iFrom && doNotReorder ) break; @@ -4588,6 +4686,7 @@ WhereInfo *sqlite3WhereBegin( } mask = (isOptimal ? m : notReady); pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0); + pDist = (i==0 ? pDistinct : 0); if( pTabItem->pIndex==0 ) nUnconstrained++; WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", @@ -4602,7 +4701,7 @@ WhereInfo *sqlite3WhereBegin( #endif { bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - &sCost); + pDist, &sCost); } assert( isOptimal || (sCost.used¬Ready)==0 ); @@ -4663,6 +4762,10 @@ WhereInfo *sqlite3WhereBegin( if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ *ppOrderBy = 0; } + if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){ + assert( pWInfo->eDistinct==0 ); + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } andFlags &= bestPlan.plan.wsFlags; pLevel->plan = bestPlan.plan; testcase( bestPlan.plan.wsFlags & WHERE_INDEXED ); diff --git a/test/collate5.test b/test/collate5.test index 2e4b89d5ca..e5bea7a5a5 100644 --- a/test/collate5.test +++ b/test/collate5.test @@ -57,17 +57,17 @@ do_test collate5-1.1 { execsql { SELECT DISTINCT a FROM collate5t1; } -} {A B N} +} {a b n} do_test collate5-1.2 { execsql { SELECT DISTINCT b FROM collate5t1; } -} {{} Apple apple banana} +} {apple Apple banana {}} do_test collate5-1.3 { execsql { SELECT DISTINCT a, b FROM collate5t1; } -} {A Apple a apple B banana N {}} +} {a apple A Apple b banana n {}} # Ticket #3376 # diff --git a/test/e_select.test b/test/e_select.test index 6194872fa5..e0f5f0fb91 100644 --- a/test/e_select.test +++ b/test/e_select.test @@ -1238,8 +1238,8 @@ do_select_tests e_select-5 { 3.1 "SELECT x FROM h2" {One Two Three Four one two three four} 3.2 "SELECT x FROM h1, h2 ON (x=b)" {One one Four four} - 4.1 "SELECT DISTINCT x FROM h2" {four one three two} - 4.2 "SELECT DISTINCT x FROM h1, h2 ON (x=b)" {four one} + 4.1 "SELECT DISTINCT x FROM h2" {One Two Three Four} + 4.2 "SELECT DISTINCT x FROM h1, h2 ON (x=b)" {One Four} } # EVIDENCE-OF: R-02054-15343 For the purposes of detecting duplicate @@ -1253,11 +1253,11 @@ do_select_tests e_select-5.5 { # sequence to compare text values with apply. # do_select_tests e_select-5.6 { - 1 "SELECT DISTINCT b FROM h1" {I IV four i iv one} - 2 "SELECT DISTINCT b COLLATE nocase FROM h1" {four i iv one} - 3 "SELECT DISTINCT x FROM h2" {four one three two} + 1 "SELECT DISTINCT b FROM h1" {one I i four IV iv} + 2 "SELECT DISTINCT b COLLATE nocase FROM h1" {one I four IV} + 3 "SELECT DISTINCT x FROM h2" {One Two Three Four} 4 "SELECT DISTINCT x COLLATE binary FROM h2" { - Four One Three Two four one three two + One Two Three Four one two three four } } diff --git a/test/fuzzer1.test b/test/fuzzer1.test index d0575d2d00..6c23211859 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -1376,7 +1376,7 @@ do_test fuzzer1-2.3 { AND f2.distance<=200 AND streetname.n>=f2.word AND streetname.n<=(f2.word || x'F7BFBFBF') } -} {steelewood tallia tallu talwyn taymouth thelema trailer {tyler finley}} +} {{tyler finley} trailer taymouth steelewood tallia tallu talwyn thelema} finish_test diff --git a/test/insert4.test b/test/insert4.test index 44b428a1c7..cb02b9dccb 100644 --- a/test/insert4.test +++ b/test/insert4.test @@ -112,7 +112,7 @@ do_test insert4-2.4.1 { INSERT INTO t3 SELECT DISTINCT * FROM t2; SELECT * FROM t3; } -} {1 9 9 1} +} {9 1 1 9} xferopt_test insert4-2.4.2 0 do_test insert4-2.4.3 { catchsql { diff --git a/test/misc5.test b/test/misc5.test index 34b2284bc1..b3832f18ae 100644 --- a/test/misc5.test +++ b/test/misc5.test @@ -505,7 +505,7 @@ ifcapable subquery { ) ORDER BY LOWER(artist) ASC; } - } {one} + } {two} } # Ticket #1370. Do not overwrite small files (less than 1024 bytes) diff --git a/test/selectB.test b/test/selectB.test index 3fdf85c0f9..b9d979acb7 100644 --- a/test/selectB.test +++ b/test/selectB.test @@ -355,7 +355,7 @@ for {set ii 3} {$ii <= 4} {incr ii} { SELECT DISTINCT (a/10) FROM t1 UNION ALL SELECT DISTINCT(d%2) FROM t2 ) } - } {0 1 0 1} + } {0 1 1 0} do_test selectB-$ii.20 { execsql {