From: drh Date: Mon, 19 Oct 2020 01:23:48 +0000 (+0000) Subject: Allow multiple recursive terms in the compound SELECT of a recursive CTE. X-Git-Tag: version-3.34.0~51^2~1 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=340558540e4799bbcb00ab901c327d5000054920;p=thirdparty%2Fsqlite.git Allow multiple recursive terms in the compound SELECT of a recursive CTE. This facilitates writing a query to find find the connected components of an undirected graph. FossilOrigin-Name: 5481fa8c79c34f434e99ab633ff3d0942a309a74fb0cf38e3d3617b51d5d21dd --- diff --git a/manifest b/manifest index 11d0ac4965..383daa906e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sSQLITE_DESERIALIZE_FREEONCLOSE\sflag\sso\sthat\sit\sworks\sas\sit\sis\ndocumented\sto\swork.\nSee\s[forum:/forumpost/ba1dff667a|forum\spost\sba1dff667a] -D 2020-10-17T22:13:16.902 +C Allow\smultiple\srecursive\sterms\sin\sthe\scompound\sSELECT\sof\sa\srecursive\sCTE.\nThis\sfacilitates\swriting\sa\squery\sto\sfind\sfind\sthe\sconnected\scomponents\sof\nan\sundirected\sgraph. +D 2020-10-19T01:23:48.028 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -481,7 +481,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c c678de5bf6f57933e0ad40578fbdb0fc8113b414bf517e0a2525cb319c379874 F src/btree.h dcdff4037d75b3f032a5de0d922fcfaf35d48589417f634fa8627362709315f9 F src/btreeInt.h ffd66480520d9d70222171b3a026d78b80833b5cea49c89867949f3e023d5f43 -F src/build.c c4bfeaea9ffc91bd1b220a5ae6ff6cac3b966bd3acce370dd2616a4f2c5c6175 +F src/build.c f6449d4e85e998e14d3f537e8ea898dca2fcb83c277db3e60945af9b9177db81 F src/callback.c d0b853dd413255d2e337b34545e54d888ea02f20da5ad0e63585b389624c4a6c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c e98518d2d3d4029a13c805e07313fb60c877be56db76e90dd5f3af73085d0ce6 @@ -537,7 +537,7 @@ F src/printf.c 30e92b638fac71dcd85cdea1d12ecfae354c9adee2c71e8e1ae4727cde7c91ed F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 97b91fb25d86881ff20c9ad2ad98412c6c1bb5f7d6c9bb044db250cbc9cfcd4b F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 0d9ce195bf062f45390c9f9fa0ab7402e26b57bee51b9f67b3034c32f9fc425a +F src/select.c aabd90874b280dd69498c83380fe9a9cd7a703bf578a12fdc68993a91979b17d F src/shell.c.in a04bec163dfafef8b14c36430685e73913b93569e7ae839533fa7e8c23837f6e F src/sqlite.h.in ff32663b457306eb88c3039868280aa39da31162ed69c4e71fa8e028684e7277 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1761,10 +1761,11 @@ F test/windowB.test 7a983ea1cc1cf72be7f378e4b32f6cb2d73014c5cd8b25aaee825164cd42 F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387019e0 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 72375ae71031eabf96bc88d0af128c8628a091ddc99b5a394e848b3df5fc17ad -F test/with1.test 323659eaf309787add657d05e5943e437a7a3494f2b6602a0deb18fdebc1ab4c +F test/with1.test 780be387f01e290e768bdfd1827280f9e37ba37223eb4736aba386864fac5a94 F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1ab -F test/with3.test 13b3336739da648a9e4dfa11bb04e73a920c97620041007c5f75d5d14084c346 +F test/with3.test a261f0ea225c4af0ce6447f1157bb603959b2a665f14a03951c2883d2ef1c0f0 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 +F test/with5.test 9bbf9823dace99ba8fd853ac0961c8f2e4a6df93a2b6846328b7d411a8540699 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 F test/without_rowid1.test e4034c0849ccc2e8bb749c69f15bd69bb9fcf8fe77e8d17ce02369604242fe83 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1882,7 +1883,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 883da4dadc88809192e4ed2f753f8883471fb43aa330bc58b017c66e1acedc6c -R 9776eedd2cd9c8f1b609b4bc8dd4a475 +P d6fac8a1d3efeb2c4f03dae437b5b314765c93770a70603803a8039291dbcabb +R 8cbf46bb87a6f92c4e833b5518759227 +T *branch * cte-enhancement +T *sym-cte-enhancement * +T -sym-trunk * U drh -Z 34fe04425a6d42f6096c739434aec808 +Z c8e202423f476b36d763c0220ea16f8f diff --git a/manifest.uuid b/manifest.uuid index 6e834086a2..d4b492871a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6fac8a1d3efeb2c4f03dae437b5b314765c93770a70603803a8039291dbcabb \ No newline at end of file +5481fa8c79c34f434e99ab633ff3d0942a309a74fb0cf38e3d3617b51d5d21dd \ No newline at end of file diff --git a/src/build.c b/src/build.c index 9cce46dcd6..9779e93732 100644 --- a/src/build.c +++ b/src/build.c @@ -4460,7 +4460,7 @@ void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ assert(pList || pParse->db->mallocFailed ); if( pList ){ for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pItem->iCursor>=0 ) break; + if( pItem->iCursor>=0 ) continue; pItem->iCursor = pParse->nTab++; if( pItem->pSelect ){ sqlite3SrcListAssignCursors(pParse, pItem->pSelect->pSrc); diff --git a/src/select.c b/src/select.c index acbd17ca87..5935964692 100644 --- a/src/select.c +++ b/src/select.c @@ -2340,6 +2340,7 @@ static void generateWithRecursiveQuery( int nCol = p->pEList->nExpr; /* Number of columns in the recursive table */ Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ Select *pSetup = p->pPrior; /* The setup query */ + Select *pFirstRec; /* Left-most recursive term */ int addrTop; /* Top of the loop */ int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ int iCurrent = 0; /* The Current table */ @@ -2415,7 +2416,25 @@ static void generateWithRecursiveQuery( /* Detach the ORDER BY clause from the compound SELECT */ p->pOrderBy = 0; + /* Figure out how many elements of the compound SELECT are part of the + ** recursive query. Make sure no recursive elements use aggregate + ** functions. Mark the recursive elements as UNION ALL even if they + ** are really UNION because the distinctness will be enforced by the + ** iDistinct table. pFirstRec is left pointing to the left-most + ** recursive term of the CTE. + */ + pFirstRec = p; + for(pFirstRec=p; ALWAYS(pFirstRec!=0); pFirstRec=pFirstRec->pPrior){ + if( pFirstRec->selFlags & SF_Aggregate ){ + sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported"); + goto end_of_recursive_query; + } + pFirstRec->op = TK_ALL; + if( (pFirstRec->pPrior->selFlags & SF_Recursive)==0 ) break; + } + /* Store the results of the setup-query in Queue. */ + pSetup = pFirstRec->pPrior; pSetup->pNext = 0; ExplainQueryPlan((pParse, 1, "SETUP")); rc = sqlite3Select(pParse, pSetup, &destQueue); @@ -2448,15 +2467,11 @@ static void generateWithRecursiveQuery( /* Execute the recursive SELECT taking the single row in Current as ** the value for the recursive-table. Store the results in the Queue. */ - if( p->selFlags & SF_Aggregate ){ - sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported"); - }else{ - p->pPrior = 0; - ExplainQueryPlan((pParse, 1, "RECURSIVE STEP")); - sqlite3Select(pParse, p, &destQueue); - assert( p->pPrior==0 ); - p->pPrior = pSetup; - } + pFirstRec->pPrior = 0; + ExplainQueryPlan((pParse, 1, "RECURSIVE STEP")); + sqlite3Select(pParse, p, &destQueue); + assert( pFirstRec->pPrior==0 ); + pFirstRec->pPrior = pSetup; /* Keep running the loop until the Queue is empty */ sqlite3VdbeGoto(v, addrTop); @@ -2525,6 +2540,16 @@ static int multiSelectValues( return rc; } +/* +** Return true if the SELECT statement which is known to be the recursive +** part of a recursive CTE still has its anchor terms attached. If the +** anchor terms have already been removed, then return false. +*/ +static int hasAnchor(Select *p){ + while( p && (p->selFlags & SF_Recursive)!=0 ){ p = p->pPrior; } + return p!=0; +} + /* ** This routine is called to process a compound query form from ** two or more separate queries using UNION, UNION ALL, EXCEPT, or @@ -2610,7 +2635,7 @@ static int multiSelect( assert( p->pEList->nExpr==pPrior->pEList->nExpr ); #ifndef SQLITE_OMIT_CTE - if( p->selFlags & SF_Recursive ){ + if( (p->selFlags & SF_Recursive)!=0 && hasAnchor(p) ){ generateWithRecursiveQuery(pParse, p, &dest); }else #endif @@ -2701,6 +2726,7 @@ static int multiSelect( findRightmost(p)->selFlags |= SF_UsesEphemeral; assert( p->pEList ); } + /* Code the SELECT statements to our left */ @@ -4794,8 +4820,10 @@ static int withExpand( ExprList *pEList; Select *pSel; Select *pLeft; /* Left-most SELECT statement */ + Select *pRecTerm; /* Left-most recursive term */ int bMayRecursive; /* True if compound joined by UNION [ALL] */ With *pSavedWith; /* Initial value of pParse->pWith */ + int iRecTab = -1; /* Cursor for recursive table */ /* If pCte->zCteErr is non-NULL at this point, then this is an illegal ** recursive reference to CTE pCte. Leave an error in pParse and return @@ -4820,44 +4848,48 @@ static int withExpand( assert( pFrom->pSelect ); /* Check if this is a recursive CTE. */ - pSel = pFrom->pSelect; + pRecTerm = pSel = pFrom->pSelect; bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); - if( bMayRecursive ){ + while( bMayRecursive && pRecTerm->op==pSel->op ){ int i; - SrcList *pSrc = pFrom->pSelect->pSrc; + SrcList *pSrc = pRecTerm->pSrc; + assert( pRecTerm->pPrior!=0 ); for(i=0; inSrc; i++){ struct SrcList_item *pItem = &pSrc->a[i]; if( pItem->zDatabase==0 && pItem->zName!=0 && 0==sqlite3StrICmp(pItem->zName, pCte->zName) - ){ + ){ pItem->pTab = pTab; pItem->fg.isRecursive = 1; + if( pRecTerm->selFlags & SF_Recursive ){ + sqlite3ErrorMsg(pParse, + "multiple references to recursive table: %s", pCte->zName + ); + return SQLITE_ERROR; + } pTab->nTabRef++; - pSel->selFlags |= SF_Recursive; + pRecTerm->selFlags |= SF_Recursive; + if( iRecTab<0 ) iRecTab = pParse->nTab++; + pItem->iCursor = iRecTab; } } + if( (pRecTerm->selFlags & SF_Recursive)==0 ) break; + pRecTerm = pRecTerm->pPrior; } - /* Only one recursive reference is permitted. */ - if( pTab->nTabRef>2 ){ - sqlite3ErrorMsg( - pParse, "multiple references to recursive table: %s", pCte->zName - ); - return SQLITE_ERROR; - } - assert( pTab->nTabRef==1 || - ((pSel->selFlags&SF_Recursive) && pTab->nTabRef==2 )); - pCte->zCteErr = "circular reference: %s"; pSavedWith = pParse->pWith; pParse->pWith = pWith; - if( bMayRecursive ){ - Select *pPrior = pSel->pPrior; - assert( pPrior->pWith==0 ); - pPrior->pWith = pSel->pWith; - sqlite3WalkSelect(pWalker, pPrior); - pPrior->pWith = 0; + if( pSel->selFlags & SF_Recursive ){ + assert( pRecTerm!=0 ); + assert( (pRecTerm->selFlags & SF_Recursive)==0 ); + assert( pRecTerm->pNext!=0 ); + assert( (pRecTerm->pNext->selFlags & SF_Recursive)!=0 ); + assert( pRecTerm->pWith==0 ); + pRecTerm->pWith = pSel->pWith; + sqlite3WalkSelect(pWalker, pRecTerm); + pRecTerm->pWith = 0; }else{ sqlite3WalkSelect(pWalker, pSel); } diff --git a/test/with1.test b/test/with1.test index 5386f7ddb0..74997c1db0 100644 --- a/test/with1.test +++ b/test/with1.test @@ -352,7 +352,7 @@ do_catchsql_test 7.4 { SELECT i FROM tree WHERE p IN (SELECT id FROM t) ) SELECT id FROM t; -} {1 {recursive reference in a subquery: t}} +} {1 {circular reference: t}} do_catchsql_test 7.5 { WITH t(id) AS ( diff --git a/test/with3.test b/test/with3.test index 2a67727a38..a574263b89 100644 --- a/test/with3.test +++ b/test/with3.test @@ -30,7 +30,7 @@ do_catchsql_test 1.0 { SELECT 5 FROM t0 UNION SELECT 8 FROM m ) SELECT * FROM i; -} {1 {no such table: t0}} +} {1 {no such table: m}} # 2019-11-09 dbfuzzcheck find do_catchsql_test 1.1 { diff --git a/test/with5.test b/test/with5.test new file mode 100644 index 0000000000..77fc75af8d --- /dev/null +++ b/test/with5.test @@ -0,0 +1,96 @@ +# 2020-10-19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is recursive common table expressions with +# multiple recursive terms in the compound select. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix with5 + +ifcapable {!cte} { + finish_test + return +} + +do_execsql_test 100 { + CREATE TABLE link(aa INT, bb INT); + CREATE INDEX link_f ON link(aa,bb); + CREATE INDEX link_t ON link(bb,aa); + INSERT INTO link(aa,bb) VALUES + (1,3), + (5,3), + (7,1), + (7,9), + (9,9), + (5,11), + (11,7), + (2,4), + (4,6), + (8,6); +} {} +do_execsql_test 110 { + WITH RECURSIVE closure(x) AS ( + VALUES(1) + UNION + SELECT aa FROM closure, link WHERE link.bb=closure.x + UNION + SELECT bb FROM closure, link WHERE link.aa=closure.x + ) + SELECT x FROM closure ORDER BY x; +} {1 3 5 7 9 11} +do_execsql_test 111 { + WITH RECURSIVE closure(x) AS ( + VALUES(1) + UNION + SELECT aa FROM link, closure WHERE link.bb=closure.x + UNION + SELECT bb FROM closure, link WHERE link.aa=closure.x + ) + SELECT x FROM closure ORDER BY x; +} {1 3 5 7 9 11} +do_execsql_test 112 { + WITH RECURSIVE closure(x) AS ( + VALUES(1) + UNION + SELECT bb FROM closure, link WHERE link.aa=closure.x + UNION + SELECT aa FROM link, closure WHERE link.bb=closure.x + ) + SELECT x FROM closure ORDER BY x; +} {1 3 5 7 9 11} +do_execsql_test 113 { + WITH RECURSIVE closure(x) AS ( + VALUES(1),(200),(300),(400) + INTERSECT + VALUES(1) + UNION + SELECT bb FROM closure, link WHERE link.aa=closure.x + UNION + SELECT aa FROM link, closure WHERE link.bb=closure.x + ) + SELECT x FROM closure ORDER BY x; +} {1 3 5 7 9 11} +do_execsql_test 114 { + WITH RECURSIVE closure(x) AS ( + VALUES(1),(200),(300),(400) + UNION ALL + VALUES(2) + UNION + SELECT bb FROM closure, link WHERE link.aa=closure.x + UNION + SELECT aa FROM link, closure WHERE link.bb=closure.x + ) + SELECT x FROM closure ORDER BY x; +} {1 2 3 4 5 6 7 8 9 11 200 300 400} + +finish_test