From: dan Date: Fri, 12 Mar 2021 18:24:31 +0000 (+0000) Subject: Attempt to use an index for DISTINCT aggregate queries that have GROUP BY clauses. X-Git-Tag: version-3.36.0~289^2~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5383db714e111b9c72179a3bd9c3c93583108161;p=thirdparty%2Fsqlite.git Attempt to use an index for DISTINCT aggregate queries that have GROUP BY clauses. FossilOrigin-Name: 3bca003cd2b2cb38d4a4e2e5f673ee0ac05bfe31247ec09e7bd379b77a31b44c --- diff --git a/manifest b/manifest index 1e2167b501..ae9cc305fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smissing\scomment\sto\snew\sroutine\sin\sselect.c. -D 2021-03-09T16:47:33.281 +C Attempt\sto\suse\san\sindex\sfor\sDISTINCT\saggregate\squeries\sthat\shave\sGROUP\sBY\sclauses. +D 2021-03-12T18:24:31.455 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -542,7 +542,7 @@ F src/printf.c 2b03a80d7c11bb422115dca175a18bf430e9c9dbaa0eee63b758f0c022f8f34f F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 688070848f0a0c41bcc545a4b4b052921d9abc29ba3102985d3d6f7595d9637c F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 7741bb3b315bd9d36c01275fb7a0b319dc30b70054f46a3521acdd71e8615d07 +F src/select.c be01e5eab0a452687a3c01b810f34b2e64816e91bc7965ecdf5a9ef714ab466f F src/shell.c.in af18a2e980aabe739a8188266464866fe7947b100674e07480e7ba3e37595947 F src/sqlite.h.in 3426a080ea1f222a73e3bd91e7eacbd30570a0117c03d42c6dde606f33e5e318 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -844,7 +844,7 @@ F test/descidx3.test 953c831df7ea219c73826dfbf2f6ee02d95040725aa88ccb4fa43d1a199 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test e7d0cf371944dd0cbedff86420744e2f1ea2b528156451c97eb6ff41a99b9236 F test/distinct2.test cd1d15a4a2abf579298f7161e821ed50c0119136fe0424db85c52cf0adc230d1 -F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 +F test/distinctagg.test bfdd84af7919687d416997bc8f156b11d693c2cc4d002bdb44c5884ec9eb757a F test/e_blobbytes.test 439a945953b35cb6948a552edaec4dc31fd70a05 F test/e_blobclose.test 4b3c8c60c2171164d472059c73e9f3c1844bb66d F test/e_blobopen.test e95e1d40f995056f6f322cd5e1a1b83a27e1a145 @@ -1910,7 +1910,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eb919611fd2f255e4ad1fe7db633363793169f6cf99c650eaefa48c022eb5d22 -R 9199f92495fd5d3674624db01699c96b +P ef2f0cf21ba61bdd29e09cf41b012a2d757683f524a252f0af7dfee7df1a1a0f +R 7b3b84df059d1804d41c3dfed8389b28 U dan -Z a3e310de94fb22aed527b8a42451f5c8 +Z 2285f2ee6ec9cbb99c3f7d9e2d1b3385 diff --git a/manifest.uuid b/manifest.uuid index b05f544f27..49c33aa997 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef2f0cf21ba61bdd29e09cf41b012a2d757683f524a252f0af7dfee7df1a1a0f \ No newline at end of file +3bca003cd2b2cb38d4a4e2e5f673ee0ac05bfe31247ec09e7bd379b77a31b44c \ No newline at end of file diff --git a/src/select.c b/src/select.c index 65345c839b..3b1697db13 100644 --- a/src/select.c +++ b/src/select.c @@ -749,7 +749,7 @@ static void codeOffset( ** on the value of parameter eTnctType: ** ** WHERE_DISTINCT_UNORDERED/WHERE_DISTINCT_NOOP: -** The ephemeral cursor table is queries for a record identical to the +** The ephemeral cursor table is queried for a record identical to the ** record formed by the current array of registers. If one is found, ** jump to VM address addrRepeat. Otherwise, insert a new record into ** the ephemeral cursor and proceed. @@ -771,41 +771,28 @@ static void codeOffset( ** collation sequences that should be used for the comparisons if ** eTnctType is WHERE_DISTINCT_ORDERED. */ -static void codeDistinct( +static int codeDistinct( Parse *pParse, /* Parsing and code generating context */ int eTnctType, /* WHERE_DISTINCT_* value */ int iTab, /* A sorting index used to test for distinctness */ - int iTabAddr, /* Address of OP_OpenEphemeral instruction for iTab */ int addrRepeat, /* Jump to here if not distinct */ ExprList *pEList, /* Expression for each element */ int regElem /* First element */ ){ + int iRet = 0; int nResultCol = pEList->nExpr; Vdbe *v = pParse->pVdbe; switch( eTnctType ){ case WHERE_DISTINCT_ORDERED: { int i; - VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ int iJump; /* Jump destination */ int regPrev; /* Previous row content */ /* Allocate space for the previous row */ - regPrev = pParse->nMem+1; + iRet = regPrev = pParse->nMem+1; pParse->nMem += nResultCol; - /* Change the OP_OpenEphemeral coded earlier to an OP_Null - ** sets the MEM_Cleared bit on the first register of the - ** previous value. This will cause the OP_Ne below to always - ** fail on the first iteration of the loop even if the first - ** row is all NULLs. */ - sqlite3VdbeChangeToNoop(v, iTabAddr); - pOp = sqlite3VdbeGetOp(v, iTabAddr); - pOp->opcode = OP_Null; - pOp->p1 = 1; - pOp->p2 = regPrev; - pOp = 0; /* Ensure pOp is not used after sqlite3VdbeAddOp() */ - iJump = sqlite3VdbeCurrentAddr(v) + nResultCol; for(i=0; ia[i].pExpr); @@ -825,7 +812,7 @@ static void codeDistinct( } case WHERE_DISTINCT_UNIQUE: { - sqlite3VdbeChangeToNoop(v, iTabAddr); + /* nothing to do */ break; } @@ -837,9 +824,34 @@ static void codeDistinct( sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, regElem, nResultCol); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, r1); + iRet = iTab; break; } } + + return iRet; +} + +static void fixDistinctOpenEph( + Parse *pParse, /* Parsing and code generating context */ + int eTnctType, /* WHERE_DISTINCT_* value */ + int iVal, /* Value returned by codeDistinct() */ + int iTabAddr /* Address of OP_OpenEphemeral instruction for iTab */ +){ + if( eTnctType==WHERE_DISTINCT_UNIQUE || eTnctType==WHERE_DISTINCT_ORDERED ){ + Vdbe *v = pParse->pVdbe; + sqlite3VdbeChangeToNoop(v, iTabAddr); + if( eTnctType==WHERE_DISTINCT_ORDERED ){ + /* Change the OP_OpenEphemeral to an OP_Null that sets the MEM_Cleared + ** bit on the first register of the previous value. This will cause the + ** OP_Ne added in codeDistinct() to always fail on the first iteration of + ** the loop even if the first row is all NULLs. */ + VdbeOp *pOp = sqlite3VdbeGetOp(v, iTabAddr); + pOp->opcode = OP_Null; + pOp->p1 = 1; + pOp->p2 = iVal; + } + } } #ifdef SQLITE_ENABLE_SORTER_REFERENCES @@ -1087,9 +1099,11 @@ static void selectInnerLoop( ** part of the result. */ if( hasDistinct ){ + int eType = pDistinct->eTnctType; + int iTab = pDistinct->tabTnct; assert( nResultCol==p->pEList->nExpr ); - codeDistinct(pParse, pDistinct->eTnctType, pDistinct->tabTnct, - pDistinct->addrTnct, iContinue, p->pEList, regResult); + iTab = codeDistinct(pParse, eType, iTab, iContinue, p->pEList, regResult); + fixDistinctOpenEph(pParse, eType, iTab, pDistinct->addrTnct); if( pSort==0 ){ codeOffset(v, p->iOffset, iContinue); } @@ -5749,8 +5763,8 @@ static void updateAccumulator( } testcase( nArg==0 ); /* Error condition */ testcase( nArg>1 ); /* Also an error */ - codeDistinct(pParse, eDistinctType, pF->iDistinct, pF->iDistAddr, - addrNext, pList, regAgg); + pF->iDistinct = codeDistinct(pParse, eDistinctType, + pF->iDistinct, addrNext, pList, regAgg); } if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ CollSeq *pColl = 0; @@ -6778,6 +6792,20 @@ int sqlite3Select( int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */ int addrReset; /* Subroutine for resetting the accumulator */ int regReset; /* Return address register for reset subroutine */ + ExprList *pDistinct = 0; + u16 distFlag = 0; + int eDist = WHERE_DISTINCT_NOOP; + + if( pAggInfo->nFunc==1 + && pAggInfo->aFunc[0].iDistinct>=0 + && pAggInfo->aFunc[0].pFExpr->x.pList + ){ + Expr *pExpr = pAggInfo->aFunc[0].pFExpr->x.pList->a[0].pExpr; + pExpr = sqlite3ExprDup(db, pExpr, 0); + pDistinct = sqlite3ExprListDup(db, pGroupBy, 0); + pDistinct = sqlite3ExprListAppend(pParse, pDistinct, pExpr); + distFlag = pDistinct ? WHERE_WANT_DISTINCT : 0; + } /* If there is a GROUP BY clause we might need a sorting index to ** implement it. Allocate that sorting index now. If it turns out @@ -6814,8 +6842,8 @@ int sqlite3Select( */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); SELECTTRACE(1,pParse,p,("WhereBegin\n")); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, - WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0), 0 + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, + WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 ); if( pWInfo==0 ) goto select_end; SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); @@ -6935,7 +6963,8 @@ int sqlite3Select( ** the current row */ sqlite3VdbeJumpHere(v, addr1); - updateAccumulator(pParse, iUseFlag, pAggInfo, WHERE_DISTINCT_UNORDERED); + eDist = sqlite3WhereIsDistinct(pWInfo); + updateAccumulator(pParse, iUseFlag, pAggInfo, eDist); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); @@ -6991,7 +7020,13 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); VdbeComment((v, "indicate accumulator empty")); sqlite3VdbeAddOp1(v, OP_Return, regReset); - + + if( eDist!=WHERE_DISTINCT_NOOP ){ + struct AggInfo_func *pF = &pAggInfo->aFunc[0]; + fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); + } + + sqlite3ExprListDelete(db, pDistinct); } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ else { Table *pTab; @@ -7057,6 +7092,7 @@ int sqlite3Select( int regAcc = 0; /* "populate accumulators" flag */ ExprList *pDistinct = 0; u16 distFlag = 0; + int eDist; /* If there are accumulator registers but no min() or max() functions ** without FILTER clauses, allocate register regAcc. Register regAcc @@ -7107,9 +7143,13 @@ int sqlite3Select( goto select_end; } SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); - updateAccumulator( - pParse, regAcc, pAggInfo, sqlite3WhereIsDistinct(pWInfo) - ); + eDist = sqlite3WhereIsDistinct(pWInfo); + updateAccumulator(pParse, regAcc, pAggInfo, eDist); + if( eDist!=WHERE_DISTINCT_NOOP ){ + struct AggInfo_func *pF = &pAggInfo->aFunc[0]; + fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); + } + if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); if( minMaxFlag ){ sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); diff --git a/test/distinctagg.test b/test/distinctagg.test index 9b5dc21678..c090bf0275 100644 --- a/test/distinctagg.test +++ b/test/distinctagg.test @@ -16,6 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix distinctagg do_test distinctagg-1.1 { execsql { @@ -31,7 +32,7 @@ do_test distinctagg-1.1 { } {1 2 3 3} do_test distinctagg-1.2 { execsql { - SELECT b, count(distinct c) FROM t1 GROUP BY b ORDER BY b + SELECT b, count(distinct c) FROM t1 GROUP BY b } } {2 1 3 2} do_test distinctagg-1.3 { @@ -59,4 +60,109 @@ do_test distinctagg-2.2 { } } {1 {DISTINCT aggregates must have exactly one argument}} +#-------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b, c); + CREATE TABLE t2(d, e, f); + + INSERT INTO t1 VALUES (1, 1, 1); + INSERT INTO t1 VALUES (2, 2, 2); + INSERT INTO t1 VALUES (3, 3, 3); + INSERT INTO t1 VALUES (4, 1, 4); + INSERT INTO t1 VALUES (5, 2, 1); + INSERT INTO t1 VALUES (5, 3, 2); + INSERT INTO t1 VALUES (4, 1, 3); + INSERT INTO t1 VALUES (3, 2, 4); + INSERT INTO t1 VALUES (2, 3, 1); + INSERT INTO t1 VALUES (1, 1, 2); + + INSERT INTO t2 VALUES('a', 'a', 'a'); + INSERT INTO t2 VALUES('b', 'b', 'b'); + INSERT INTO t2 VALUES('c', 'c', 'c'); + + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1bc ON t1(b, c); +} + +foreach {tn use_eph sql res} { + 1 0 "SELECT count(DISTINCT a) FROM t1" 5 + 2 0 "SELECT count(DISTINCT b) FROM t1" 3 + 3 1 "SELECT count(DISTINCT c) FROM t1" 4 + 4 0 "SELECT count(DISTINCT c) FROM t1 WHERE b=3" 3 + 5 0 "SELECT count(DISTINCT rowid) FROM t1" 10 + 6 0 "SELECT count(DISTINCT a) FROM t1, t2" 5 + 7 0 "SELECT count(DISTINCT a) FROM t2, t1" 5 + 8 1 "SELECT count(DISTINCT a+b) FROM t1, t2, t2, t2" 6 + 9 0 "SELECT count(DISTINCT c) FROM t1 WHERE c=2" 1 + 10 1 "SELECT count(DISTINCT t1.rowid) FROM t1, t2" 10 +} { + do_test 3.$tn.1 { + set prg [db eval "EXPLAIN $sql"] + set idx [lsearch $prg OpenEphemeral] + expr {$idx>=0} + } $use_eph + + do_execsql_test 3.$tn.2 $sql $res +} + +do_execsql_test 3.10 { + SELECT a, count(DISTINCT b) FROM t1 GROUP BY a; +} { + 1 1 2 2 3 2 4 1 5 2 +} + +#-------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1bc ON t1(b, c); + + INSERT INTO t1 VALUES(1, 'A', 1); + INSERT INTO t1 VALUES(1, 'A', 1); + INSERT INTO t1 VALUES(2, 'A', 2); + INSERT INTO t1 VALUES(2, 'A', 2); + INSERT INTO t1 VALUES(1, 'B', 1); + INSERT INTO t1 VALUES(2, 'B', 2); + INSERT INTO t1 VALUES(3, 'B', 3); + INSERT INTO t1 VALUES(NULL, 'B', NULL); + INSERT INTO t1 VALUES(NULL, 'C', NULL); + INSERT INTO t1 VALUES('d', 'D', 'd'); + + CREATE TABLE t2(d, e, f); + CREATE INDEX t2def ON t2(d, e, f); + + INSERT INTO t2 VALUES(1, 1, 'a'); + INSERT INTO t2 VALUES(1, 1, 'a'); + INSERT INTO t2 VALUES(1, 2, 'a'); + INSERT INTO t2 VALUES(1, 2, 'a'); + INSERT INTO t2 VALUES(1, 2, 'b'); + INSERT INTO t2 VALUES(1, 3, 'b'); + INSERT INTO t2 VALUES(1, 3, 'a'); + INSERT INTO t2 VALUES(1, 3, 'b'); + INSERT INTO t2 VALUES(2, 3, 'x'); + INSERT INTO t2 VALUES(2, 3, 'y'); + INSERT INTO t2 VALUES(2, 3, 'z'); +} + +foreach {tn use_eph sql res} { + 1 0 "SELECT count(DISTINCT c) FROM t1 GROUP BY b" {2 3 0 1} + 2 1 "SELECT count(DISTINCT a) FROM t1 GROUP BY b" {2 3 0 1} + 3 1 "SELECT count(DISTINCT a) FROM t1 GROUP BY b+c" {0 1 1 1 1} + + 4 0 "SELECT count(DISTINCT f) FROM t2 GROUP BY d, e" {1 2 2 3} + 5 1 "SELECT count(DISTINCT f) FROM t2 GROUP BY d" {2 3} + 6 0 "SELECT count(DISTINCT f) FROM t2 WHERE d IS 1 GROUP BY e" {1 2 2} +} { + do_test 4.$tn.1 { + set prg [db eval "EXPLAIN $sql"] + set idx [lsearch $prg OpenEphemeral] + expr {$idx>=0} + } $use_eph + + do_execsql_test 4.$tn.2 $sql $res +} + finish_test +