From: drh <> Date: Tue, 16 Feb 2021 16:32:07 +0000 (+0000) Subject: Trying to get the new AS MATERIALIZE syntax of CTEs to work. There are X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fheads%2Fusing-materialize;p=thirdparty%2Fsqlite.git Trying to get the new AS MATERIALIZE syntax of CTEs to work. There are still performance and memory management issues. This is a WIP check-in. FossilOrigin-Name: bf0fd9b23a77a8fc1be5f8f8f9f7dce62b924d2f4eecaf13f0b21cc58bfd3a61 --- diff --git a/manifest b/manifest index f37fdf04d8..5908541c17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\ssyntax\sfrom\s"GENERATED\sAS"\sto\s"AS\sMATERIALIZED"\sso\sas\sto\smatch\nthe\ssyntax\sof\sPostgreSQL\s12+. -D 2021-02-16T00:48:51.202 +C Trying\sto\sget\sthe\snew\sAS\sMATERIALIZE\ssyntax\sof\sCTEs\sto\swork.\s\sThere\sare\nstill\sperformance\sand\smemory\smanagement\sissues.\s\sThis\sis\sa\sWIP\scheck-in. +D 2021-02-16T16:32:07.810 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -485,15 +485,15 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c 694020ad8a3af3d79b09f74c8f1421272a419cdea42a13401e3b0f7dea6e9c3e F src/btree.h 285f8377aa1353185a32bf455faafa9ff9a0d40d074d60509534d14990c7829e F src/btreeInt.h 7614cae30f95b6aed0c7cac7718276a55cfe2c77058cbfd8bef5b75329757331 -F src/build.c a777f43048a09704b75000c11e6c9fea825deafd676cd8f4e5357764f2d2fb06 +F src/build.c 9d51da4a93227beb389ec6f6f97d4153ec59ffcbdf415f508de8fdf7ac48c0b9 F src/callback.c d0b853dd413255d2e337b34545e54d888ea02f20da5ad0e63585b389624c4a6c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 2a322b9a3d75771fb4d99e0702851f4f68dda982507a0f798eefb0712969a410 F src/date.c dace306a10d9b02ee553d454c8e1cf8d3c9b932e137738a6b15b90253a9bfc10 F src/dbpage.c 8a01e865bf8bc6d7b1844b4314443a6436c07c3efe1d488ed89e81719047833a F src/dbstat.c 3aa79fc3aed7ce906e4ea6c10e85d657299e304f6049861fe300053ac57de36c -F src/delete.c 352ea931218c45a3daf17472d4141b9c7fc026d85da3f1ade404ea5bb6d67f77 -F src/expr.c 47c85263e6d179424e6b09e2c79db5704ab5b8cbc2fae2ee3285faa2566f2e74 +F src/delete.c 8517d4d57dc12e4cb9adec1801249ce18a3be3e6ccd75a266f66499a57e5b062 +F src/expr.c fe76d3b9f606347b1d84cb059264ed0ba0f33ecce57a864f1575bc3d8ae42ba9 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 73adaca988d0dd517d373b432dc9dfa2cd7fa3108b114260132a80832de19037 F src/func.c 479f6929be027eb0210cbdde9d3529c012facf082d64a6b854a9415940761e5e @@ -530,23 +530,23 @@ F src/os_win.c 77d39873836f1831a9b0b91894fec45ab0e9ca8e067dc8c549e1d1eca1566fe9 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c c49952ac5e9cc536778eff528091d79d38b3e45cbeeed4695dc05e207dc6547d F src/pager.h 4bf9b3213a4b2bebbced5eaa8b219cf25d4a82f385d093cd64b7e93e5285f66f -F src/parse.y a6d56266bf453e7fd57f47f39f85f36a8530792cae68f0672f7f03972c112397 +F src/parse.y 7bec75e74a457d1569b093367924c897776375167b66b40c7464e8f63823fa21 F src/pcache.c 385ff064bca69789d199a98e2169445dc16e4291fa807babd61d4890c3b34177 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 6596e10baf3d8f84cc1585d226cf1ab26564a5f5caf85a15757a281ff977d51a F src/pragma.c 6daaaecc26a4b09481d21722525b079ce756751a43a79cc1d8f122d686806193 F src/pragma.h 8dc78ab7e9ec6ce3ded8332810a2066f1ef6267e2e03cd7356ee00276125c6cf -F src/prepare.c f288cbc35f79eb32e162de7e80a63ebe00d80e639dcfac071bee11570cbdb16f +F src/prepare.c e8e056d7e34e5b1092a0f0d4f537278175c806412f8fa605b5f50a40ca0915d3 F src/printf.c 30e92b638fac71dcd85cdea1d12ecfae354c9adee2c71e8e1ae4727cde7c91ed F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 89e4faf6171e179edf279905e8e45c4f9dd108777dc60716396729fbd7cb045e F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 0612a4097f86396f6ec000546dbc16479d6bbb9033a29a0dc07213801e9cd83f +F src/select.c 847bdf40d4b3b04f3316b3ead543ef49596f4f53e48f810c60ce9649748b2774 F src/shell.c.in 9ebc74e4f05cfbd0f4a36060fdaeff1da4e9af4458358722bc08c5a1ab9a0879 F src/sqlite.h.in 8855a19f37ade8dad189a9e48233a2ebe1b46faf469c7eb0906a654e252dcc57 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e -F src/sqliteInt.h e9122436bf0b69ef93f1d922805e891cac614db4b5f5b22efe25395a1a168555 +F src/sqliteInt.h d0f3ca58ec5cf148210b186d9eddbb32e847cc845b887506a759d9c255cf0319 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -628,7 +628,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 69e770e96fd56cc21608992bf2c6f1f3dc5cf2572d0495c6a643b06c3a679f14 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c d9c4e454ebb9499e908aa62d55b8994c375cf5355ac78f60d45af17f7890701c -F src/where.c 6efc4a10bfe0ec908c4f3c9112afb7675c952204b4b0b255672f488860141fec +F src/where.c 228cb57b51ca02853bb932c4548ffaa3949f40f8398a0f393683dd4c08cdbea0 F src/whereInt.h ae03b5e3a4cca9bd9cb1b7d3c63faf8f1f177200fc8cecc87d3d0cab6ca338e6 F src/wherecode.c 43a63441f8662ddf86b15975683a502ec33f08167e9636f4d19e38e265e95fd9 F src/whereexpr.c f7b5469e83db3c3b9eb14e4ba44559a2e125523761d12e5ac8d8fb88301af393 @@ -1900,7 +1900,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8c8618780a2cb80c0c1f244fa8555ce36501ad7e435a2cfee1030bbdc8d644db -R ba922d4317c055cbb896732be33e5b20 +P 78dcddd9697d95629c18131ab0842aa4d08bc3c7451cd0e7a8d83e4dde277bda +R 3e33372448628596b79c8e4959bb7ced +T *branch * using-materialize +T *sym-using-materialize * +T -sym-with-generated-as * U drh -Z c9282fd2eefaea59421fc4e9996c1c87 +Z 4d1724c077b98d0e6366cdc6b7fd4796 diff --git a/manifest.uuid b/manifest.uuid index b1afeda69a..a88a2b68de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -78dcddd9697d95629c18131ab0842aa4d08bc3c7451cd0e7a8d83e4dde277bda \ No newline at end of file +bf0fd9b23a77a8fc1be5f8f8f9f7dce62b924d2f4eecaf13f0b21cc58bfd3a61 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 99f545e5d1..c888894d11 100644 --- a/src/build.c +++ b/src/build.c @@ -5212,13 +5212,14 @@ Cte *sqlite3CteNew( Token *pName, /* Name of the common-table */ ExprList *pArglist, /* Optional column name list for the table */ Select *pQuery, /* Query used to initialize the table */ - int eMaterialized /* Force or prohibit materialization */ + int eMaterialize /* Force or prohibit materialization */ ){ Cte *pNew; sqlite3 *db = pParse->db; - assert( eMaterialized==MAT_NotSpec || eMaterialized==MAT_Yes - || eMaterialized==MAT_No ); + assert( eMaterialize==Materialize_Any + || eMaterialize==Materialize_Yes + || eMaterialize==Materialize_No ); pNew = sqlite3DbMallocZero(db, sizeof(*pNew)); assert( pNew!=0 || db->mallocFailed ); @@ -5230,26 +5231,65 @@ Cte *sqlite3CteNew( pNew->pCols = pArglist; pNew->zName = sqlite3NameFromToken(pParse->db, pName); pNew->zCteErr = 0; - pNew->eMaterialized = (u8)eMaterialized; + pNew->eMaterialize = (u8)eMaterialize; } return pNew; } +#ifdef SQLITE_DEBUG +/* +** Validate the back-reference from a SrcList_item back to its Cte object. +** Raise an assertion fault if anything is wrong. This routine is for +** sanity and design checking and does not appear in release builds. +*/ +void sqlite3AssertValidCteBackRef(struct SrcList_item *pItem){ + Cte *pCte; + if( !pItem->fg.isCte ) return; /* Nothing to check here */ + assert( pItem->pSelect!=0 ); + pCte = pItem->u2.pCteSrc; + if( pCte==0 ) return; + assert( pCte->eCteMagic==CTE_MAGIC ); + assert( pItem->pTab!=0 ); + assert( sqlite3_stricmp(pCte->zName, pItem->pTab->zName)==0 ); + assert( pCte->nRefCte>=1 ); +} +#endif /* SQLITE_DEBUG */ + +/* +** Clear information from a Cte object, but do not deallocate storage. +*/ +static void cteClear(sqlite3 *db, Cte *pCte){ + sqlite3ExprListDelete(db, pCte->pCols); + sqlite3SelectDelete(db, pCte->pSelect); + sqlite3DbFree(db, pCte->zName); +#ifdef SQLITE_DEBUG + if( db==0 || db->pnBytesFreed==0 ){ + pCte->zName = 0; + pCte->eCteMagic = 0; + } +#endif +} + /* ** Free the contents of the CTE object passed as the second argument. */ void sqlite3CteDelete(sqlite3 *db, Cte *pCte){ if( pCte ){ - sqlite3ExprListDelete(db, pCte->pCols); - sqlite3SelectDelete(db, pCte->pSelect); - sqlite3DbFree(db, pCte->zName); + cteClear(db, pCte); sqlite3DbFree(db, pCte); } } /* ** This routine is invoked once per CTE by the parser while parsing a -** WITH clause. +** WITH clause. Construct a new Cte object and append it to the +** input With object, reallocating the With object as needed. If the +** pWith input is NULL, then create a new With object. +** +** The returned With object is unowned. The caller needs to assign +** ownership. Note, however, that ownership should not be assigned +** until the object stops growing. The input pWith parameter must be +** either NULL or a pointer to an unowned With object. */ With *sqlite3WithAdd( Parse *pParse, /* Parsing context */ @@ -5260,6 +5300,8 @@ With *sqlite3WithAdd( With *pNew; char *zName; + /* The input WITH clause is yet unowned */ + assert( pWith==0 || pWith->mOwner==0 ); if( pCte==0 ){ return pWith; @@ -5297,17 +5339,62 @@ With *sqlite3WithAdd( /* ** Free the contents of the With object passed as the second argument. +** +** The sqlite3WithReleaseBySelect() causes the Select-object ownership +** to be released, and sqlite3WithReleaseByParse() causes the Parse-object +** ownership to be released. The With object is only deallocated after +** both owners release it. */ void sqlite3WithDelete(sqlite3 *db, With *pWith){ - if( pWith ){ - int i; - for(i=0; inCte; i++){ - struct Cte *pCte = &pWith->a[i]; - sqlite3ExprListDelete(db, pCte->pCols); - sqlite3SelectDelete(db, pCte->pSelect); - sqlite3DbFree(db, pCte->zName); - } - sqlite3DbFree(db, pWith); + int i; + for(i=0; inCte; i++){ + cteClear(db, &pWith->a[i]); + } + sqlite3DbFree(db, pWith); +} +void sqlite3WithReleaseBySelect(sqlite3 *db, With *pWith){ + if( pWith==0 ) return; + if( db && db->pnBytesFreed ){ + if( pWith->mOwner==WithOwnedBySelect ) sqlite3WithDelete(db, pWith); + }else{ + pWith->mOwner &= ~WithOwnedBySelect; + if( pWith->mOwner==0 ) sqlite3WithDelete(db, pWith); + } +} +void sqlite3WithReleaseByParse(sqlite3 *db, With *pWith){ + if( pWith==0 ) return; + if( db && db->pnBytesFreed ){ + sqlite3WithDelete(db, pWith); + }else{ + pWith->mOwner &= ~WithOwnedByParse; + if( pWith->mOwner==0 ) sqlite3WithDelete(db, pWith); } } + +/* +** Add Select-object or Parse-object ownership to a With object. If +** the With object is already owned appropriately, then these routines +** are no-ops. +** +** The first time that a With becomes owned by a Parse-object, make +** arrangements to automatically release the ownership when the Parse +** object is destroyed using a call to sqlite3ParserAddCleanup(). +** This involves a memory allocation. If that memory allocation fails, +** the With object might be destroyed immediately. In that +** case, return a NULL pointer. But usually, return a copy of the pWith +** pointer. +*/ +void sqlite3WithClaimedBySelect(With *pWith){ + pWith->mOwner |= WithOwnedBySelect; +} +With *sqlite3WithClaimedByParse(Parse *pParse, With *pWith){ + if( (pWith->mOwner & WithOwnedByParse)==0 ){ + pWith->mOwner |= WithOwnedByParse; + pWith = (With*)sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3WithReleaseByParse, + (void*)pWith); + } + return pWith; +} + #endif /* !defined(SQLITE_OMIT_CTE) */ diff --git a/src/delete.c b/src/delete.c index b2edaa9ab9..abbfb6f5a3 100644 --- a/src/delete.c +++ b/src/delete.c @@ -209,7 +209,7 @@ Expr *sqlite3LimitWhere( pSrc->a[0].pTab = 0; pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0); pSrc->a[0].pTab = pTab; - pSrc->a[0].pIBIndex = 0; + pSrc->a[0].u2.pIBIndex = 0; assert( !pSrc->a[0].fg.isCte ); /* generate the SELECT expression tree. */ pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, diff --git a/src/expr.c b/src/expr.c index f225b59bce..2f34c9b4f6 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1401,6 +1401,7 @@ static With *withDup(sqlite3 *db, With *p){ if( pRet ){ int i; pRet->nCte = p->nCte; + pRet->mOwner = WithOwnedBySelect; for(i=0; inCte; i++){ pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0); pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0); @@ -1544,7 +1545,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ if( pNewItem->fg.isIndexedBy ){ pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); } - pNewItem->pIBIndex = pOldItem->pIBIndex; + pNewItem->u2 = pOldItem->u2; if( pNewItem->fg.isTabFunc ){ pNewItem->u1.pFuncArg = sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); diff --git a/src/parse.y b/src/parse.y index 110fc5e0a7..8dafee19d8 100644 --- a/src/parse.y +++ b/src/parse.y @@ -513,29 +513,29 @@ cmd ::= select(X). { } } } -} +#ifndef SQLITE_OMIT_CTE + /* Link a With object to a Select object. + ** Both Select- and Parse-ownership is assigned to the With object + */ + static Select *linkWithToSelect(Parse *pParse, Select *pSelect, With *pWith){ + assert( pWith->mOwner==0 ); + if( pSelect ){ + pSelect->pWith = pWith; + sqlite3WithClaimedBySelect(pWith); + parserDoubleLinkSelect(pParse, pSelect); + }else{ + sqlite3WithClaimedByParse(pParse,pWith); + } + return pSelect; + } +#endif /* SQLITE_OMIT_CTE */ +} // end %include + %ifndef SQLITE_OMIT_CTE -select(A) ::= WITH wqlist(W) selectnowith(X). { - Select *p = X; - if( p ){ - p->pWith = W; - parserDoubleLinkSelect(pParse, p); - }else{ - sqlite3WithDelete(pParse->db, W); - } - A = p; -} -select(A) ::= WITH RECURSIVE wqlist(W) selectnowith(X). { - Select *p = X; - if( p ){ - p->pWith = W; - parserDoubleLinkSelect(pParse, p); - }else{ - sqlite3WithDelete(pParse->db, W); - } - A = p; -} +select(A) ::= WITH wqlist(W) selectnowith(X). { A = linkWithToSelect(pParse,X,W); } +select(A) ::= WITH RECURSIVE wqlist(W) selectnowith(X). + { A = linkWithToSelect(pParse,X,W); } %endif /* SQLITE_OMIT_CTE */ select(A) ::= selectnowith(X). { Select *p = X; @@ -1660,19 +1660,19 @@ anylist ::= anylist ANY. //////////////////////// COMMON TABLE EXPRESSIONS //////////////////////////// %type wqlist {With*} -%destructor wqlist {sqlite3WithDelete(pParse->db, $$);} +%destructor wqlist {if($$)sqlite3WithDelete(pParse->db, $$);} %type wqitem {Cte*} %destructor wqitem {sqlite3CteDelete(pParse->db, $$);} %type wqas {int} with ::= . %ifndef SQLITE_OMIT_CTE -with ::= WITH wqlist(W). { sqlite3WithPush(pParse, W, 1); } -with ::= WITH RECURSIVE wqlist(W). { sqlite3WithPush(pParse, W, 1); } +with ::= WITH wqlist(W). { sqlite3WithPush(pParse, W); } +with ::= WITH RECURSIVE wqlist(W). { sqlite3WithPush(pParse, W); } -wqas(A) ::= AS. {A=MAT_NotSpec;} -wqas(A) ::= AS MATERIALIZED. {A=MAT_Yes;} -//wqas(A) ::= AS NOT MATERIALIZED. {A=MAT_No;} +wqas(A) ::= AS. {A = Materialize_Any;} +wqas(A) ::= AS MATERIALIZED. {A = Materialize_Yes;} +wqas(A) ::= AS NOT MATERIALIZED. {A = Materialize_No;} wqitem(A) ::= nm(X) eidlist_opt(Y) wqas(F) LP select(Z) RP. { A = sqlite3CteNew(pParse, &X, Y, Z, F); /*A-overwrites-X*/ } diff --git a/src/prepare.c b/src/prepare.c index 87c1ab9368..dc3d3ee36a 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -608,7 +608,7 @@ void sqlite3ParserReset(Parse *pParse){ ** ** testcase( pParse->earlyCleanup ); */ -void sqlite3ParserAddCleanup( +void *sqlite3ParserAddCleanup( Parse *pParse, /* Destroy when this Parser finishes */ void (*xCleanup)(sqlite3*,void*), /* The cleanup routine */ void *pPtr /* Pointer to object to be cleaned up */ @@ -621,10 +621,12 @@ void sqlite3ParserAddCleanup( pCleanup->xCleanup = xCleanup; }else{ xCleanup(pParse->db, pPtr); + pPtr = 0; #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) pParse->earlyCleanup = 1; #endif } + return pPtr; } /* diff --git a/src/select.c b/src/select.c index a476644251..8ca39b1790 100644 --- a/src/select.c +++ b/src/select.c @@ -90,7 +90,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ sqlite3WindowListDelete(db, p->pWinDefn); } #endif - if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith); + if( p->pWith ) sqlite3WithReleaseBySelect(db, p->pWith); if( bFree ) sqlite3DbFreeNN(db, p); p = pPrior; bFree = 1; @@ -4723,7 +4723,9 @@ int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ pParse->checkSchema = 1; return SQLITE_ERROR; } - pFrom->pIBIndex = pIdx; + assert( pFrom->fg.isCte==0 ); + assert( pFrom->pSelect==0 ); + pFrom->u2.pIBIndex = pIdx; } return SQLITE_OK; } @@ -4855,25 +4857,13 @@ static struct Cte *searchWith( /* The code generator maintains a stack of active WITH clauses ** with the inner-most WITH clause being at the top of the stack. -** -** This routine pushes the WITH clause passed as the second argument -** onto the top of the stack. If argument bFree is true, then this -** WITH clause will never be popped from the stack. In this case it -** should be freed along with the Parse object. In other cases, when -** bFree==0, the With object will be freed along with the SELECT -** statement with which it is associated. */ -void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ +void sqlite3WithPush(Parse *pParse, With *pWith){ if( pWith ){ assert( pParse->pWith!=pWith ); pWith->pOuter = pParse->pWith; pParse->pWith = pWith; - if( bFree ){ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3WithDelete, - pWith); - testcase( pParse->earlyCleanup ); - } + sqlite3WithClaimedByParse(pParse, pWith); } } @@ -4938,9 +4928,17 @@ static int withExpand( pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; assert( pFrom->pSelect ); - if( pCte->eMaterialized==MAT_Yes ){ + if( pCte->eMaterialize==Materialize_Yes ){ pFrom->pSelect->selFlags |= SF_OptBarrier; } + assert( !pFrom->fg.isIndexedBy ); + pFrom->fg.isCte = 1; + assert( (pCte->eCteMagic==0 && pCte->nRefCte==0) + || (pCte->eCteMagic==CTE_MAGIC && pCte->nRefCte>0) ); + pFrom->u2.pCteSrc = pCte; + sqlite3WithClaimedByParse(pParse, pWith); + pCte->nRefCte++; + VVA_ONLY( pCte->eCteMagic = CTE_MAGIC ); /* Check if this is a recursive CTE. */ pRecTerm = pSel = pFrom->pSelect; @@ -5121,7 +5119,7 @@ static int selectExpander(Walker *pWalker, Select *p){ } pTabList = p->pSrc; pEList = p->pEList; - sqlite3WithPush(pParse, p->pWith, 0); + sqlite3WithPush(pParse, p->pWith); /* Make sure cursor numbers have been assigned to all entries in ** the FROM clause of the SELECT statement. @@ -5760,9 +5758,10 @@ static void havingToWhere(Parse *pParse, Select *p){ } /* -** Check to see if the pThis entry of pTabList is a self-join of a prior view. -** If it is, then return the SrcList_item for the prior view. If it is not, -** then return 0. +** Check to see if the pThis entry of pTabList is a self-join of a prior view, +** or any reuse of a CTE that is not necessarily a self-join. +** If it is, then return the SrcList_item for the prior view or CTE. +** If it is not, then return 0. */ static struct SrcList_item *isSelfJoinView( SrcList *pTabList, /* Search for self-joins in this FROM clause */ @@ -5771,6 +5770,11 @@ static struct SrcList_item *isSelfJoinView( struct SrcList_item *pItem; assert( pThis->pSelect!=0 ); if( pThis->pSelect->selFlags & SF_PushDown ) return 0; + if( pThis->fg.isCte ){ + Cte *pCte = pThis->u2.pCteSrc; + sqlite3AssertValidCteBackRef(pThis); + return pCte->pCteMat; + } for(pItem = pTabList->a; pItempSelect==0 ) continue; @@ -6141,6 +6145,7 @@ int sqlite3Select( */ for(i=0; inSrc; i++){ struct SrcList_item *pItem = &pTabList->a[i]; + struct SrcList_item *pPrior; SelectDest dest; Select *pSub; #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) @@ -6219,12 +6224,15 @@ int sqlite3Select( zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; + /* Verify that the CTE back-reference information is valid */ + sqlite3AssertValidCteBackRef(pItem); + /* Generate code to implement the subquery ** ** The subquery is implemented as a co-routine if (1) the subquery is ** guaranteed to be the outer loop (so that it does not need to be - ** computed more than once) and if (2) the subquery is not optimization - ** barrier. + ** computed more than once) and if (2) the subquery is a CTE that is + ** used only this one time. ** ** TODO: Are there other reasons beside (1) and (2) to use a co-routine ** implementation? @@ -6236,7 +6244,7 @@ int sqlite3Select( if( i==0 && (pTabList->nSrc==1 || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ - && (pSub->selFlags & SF_OptBarrier)==0 /* (2) */ + && (pItem->fg.isCte && pItem->u2.pCteSrc->nRefCte==1) /* (2) */ ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. @@ -6256,16 +6264,20 @@ int sqlite3Select( sqlite3VdbeEndCoroutine(v, pItem->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); + }else if( (pPrior = isSelfJoinView(pTabList,pItem))!=0 ){ + /* This view has been previously materialized. Use the + ** prior materialization */ + if( pPrior->addrFillSub ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->regReturn, pPrior->addrFillSub); + } + sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); + assert( pPrior->pSelect!=0 ); + pSub->nSelectRow = pPrior->pSelect->nSelectRow; }else{ - /* Generate a subroutine that will fill an ephemeral table with - ** the content of this subquery. pItem->addrFillSub will point - ** to the address of the generated subroutine. pItem->regReturn - ** is a register allocated to hold the subroutine return address - */ + /* Materialize the view */ int topAddr; int onceAddr = 0; int retAddr; - struct SrcList_item *pPrior; testcase( pItem->addrFillSub==0 ); /* Ticket c52b09c7f38903b1311 */ pItem->regReturn = ++pParse->nMem; @@ -6280,15 +6292,12 @@ int sqlite3Select( }else{ VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); } - pPrior = isSelfJoinView(pTabList, pItem); - if( pPrior ){ - sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); - assert( pPrior->pSelect!=0 ); - pSub->nSelectRow = pPrior->pSelect->nSelectRow; - }else{ - sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - ExplainQueryPlan((pParse, 1, "MATERIALIZE %u", pSub->selId)); - sqlite3Select(pParse, pSub, &dest); + sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); + ExplainQueryPlan((pParse, 1, "MATERIALIZE %u", pSub->selId)); + sqlite3Select(pParse, pSub, &dest); + if( pItem->fg.isCte ){ + sqlite3AssertValidCteBackRef(pItem); + pItem->u2.pCteSrc->pCteMat = pItem; } pItem->pTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index eca583d1c1..c16cae9713 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2952,6 +2952,7 @@ struct SrcList { unsigned viaCoroutine :1; /* Implemented as a co-routine */ unsigned isRecursive :1; /* True for recursive reference in WITH */ unsigned fromDDL :1; /* Comes from sqlite_schema */ + unsigned isCte :1; /* True if table is a CTE */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ Expr *pOn; /* The ON clause of a join */ @@ -2961,7 +2962,10 @@ struct SrcList { char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ ExprList *pFuncArg; /* Arguments to table-valued-function */ } u1; - Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ + union { + Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ + Cte *pCteSrc; /* The original CTE, if fg.isCte is true */ + } u2; } a[1]; /* One entry for each identifier on the list */ }; @@ -3870,29 +3874,55 @@ void sqlite3SelectWalkAssert2(Walker*, Select*); #define WRC_Abort 2 /* Abandon the tree walk */ /* -** Allowed values for Cte.eMAterialized +** Allowed values for Cte.eMaterialize */ -#define MAT_NotSpec 0 /* Not specified */ -#define MAT_Yes 1 /* AS MATERIALIZED ... */ -#define MAT_No 2 /* AS NOT MATERIALIZED. Not currently used */ +#define Materialize_Any 0 /* Not specified */ +#define Materialize_Yes 1 /* AS MATERIALIZED ... */ +#define Materialize_No 2 /* AS NOT MATERIALIZED. Not currently used */ /* ** An instance of this structure represents a set of one or more CTEs ** (common table expressions) created by a single WITH clause. +** +** A With object describes an entire WITH clause, which is composed of +** one or more Cte objects. During parsing, the Cte objects are created +** separately for each common-table-expression, then copied into the +** With object container. +** +** DUAL OWNERSHIP +** The With object can be owned by a Select object (via the Select.pWith +** field), or it can be owned by the Parse object of the current parse, +** or by both. The With.mOwner bitmask determines who owns each With object. +** The object is not deallocated until all owners have released it. */ struct Cte { char *zName; /* Name of this CTE */ ExprList *pCols; /* List of explicit column names, or NULL */ Select *pSelect; /* The definition of this CTE */ const char *zCteErr; /* Error message for circular references */ - u8 eMaterialized; /* One of the MAT_* values */ + u8 eMaterialize; /* One of the Materialize_* options */ + u16 eCteMagic; /* Magic number for sanity checking */ + u32 nRefCte; /* Number of times used */ + struct SrcList_item *pCteMat; /* The materialization of this CTE */ }; struct With { int nCte; /* Number of CTEs in the WITH clause */ + int mOwner; /* Mask of bits to discribe ownership */ With *pOuter; /* Containing WITH clause, or NULL */ Cte a[1]; /* The CTEs of this WITH clause */ }; +/* +** The correct value for Cte.eCteMagic if the Cte object is valid +*/ +#define CTE_MAGIC 0x3f96 /* Magic number for valid Cte objects */ + +/* +** Ownership flags for With. +*/ +#define WithOwnedBySelect 0x01 /* Owned by a Select object */ +#define WithOwnedByParse 0x10 /* Owned by the Parse object */ + #ifdef SQLITE_DEBUG /* ** An instance of the TreeView object is used for printing the content of @@ -4879,7 +4909,7 @@ sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); void sqlite3ParserReset(Parse*); -void sqlite3ParserAddCleanup(Parse*,void(*)(sqlite3*,void*),void*); +void *sqlite3ParserAddCleanup(Parse*,void(*)(sqlite3*,void*),void*); #ifdef SQLITE_ENABLE_NORMALIZE char *sqlite3Normalize(Vdbe*, const char*); #endif @@ -4898,11 +4928,25 @@ const char *sqlite3JournalModename(int); With *sqlite3WithAdd(Parse*,With*,Cte*); void sqlite3CteDelete(sqlite3*,Cte*); void sqlite3WithDelete(sqlite3*,With*); - void sqlite3WithPush(Parse*, With*, u8); -#else -#define sqlite3WithPush(x,y,z) -#define sqlite3WithDelete(x,y) -#endif + void sqlite3WithReleaseBySelect(sqlite3*,With*); + void sqlite3WithReleaseByParse(sqlite3*,With*); + void sqlite3WithClaimedBySelect(With*); + With *sqlite3WithClaimedByParse(Parse*,With*); + void sqlite3WithPush(Parse*, With*); +# ifdef SQLITE_DEBUG + void sqlite3AssertValidCteBackRef(struct SrcList_item*); +# else +# define sqlite3AssertValidCteBackRef(X) +# endif +#else /* if defined(SQLITE_OMIT_CTE) */ +# define sqlite3WithPush(PARSE,W) +# deifne sqlite3WithDelete(Db,W) +# define sqlite3WithReleaseBySelect(DB,W) +# define sqlite3WithReleaseByParse(DB,W) +# define sqlite3WithClaimedBySelect(W) +# define sqlite3WithClaimedByParse(PARSE,W); +# define sqlite3AssertValidCteBackRef(X) +#endif /* SQLITE_OMIT_CTE */ #ifndef SQLITE_OMIT_UPSERT Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*,Upsert*); void sqlite3UpsertDelete(sqlite3*,Upsert*); diff --git a/src/where.c b/src/where.c index 971c223730..155e2ae66b 100644 --- a/src/where.c +++ b/src/where.c @@ -2954,9 +2954,9 @@ static int whereLoopAddBtree( pWC = pBuilder->pWC; assert( !IsVirtual(pSrc->pTab) ); - if( pSrc->pIBIndex ){ + if( pSrc->fg.isIndexedBy ){ /* An INDEXED BY clause specifies a particular index to use */ - pProbe = pSrc->pIBIndex; + pProbe = pSrc->u2.pIBIndex; }else if( !HasRowid(pTab) ){ pProbe = pTab->pIndex; }else{ @@ -2992,7 +2992,7 @@ static int whereLoopAddBtree( if( !pBuilder->pOrSet /* Not part of an OR optimization */ && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 - && pSrc->pIBIndex==0 /* Has no INDEXED BY clause */ + && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */ && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ && !pSrc->fg.isCorrelated /* Not a correlated subquery */ @@ -3042,7 +3042,7 @@ static int whereLoopAddBtree( /* Loop over all indices. If there was an INDEXED BY clause, then only ** consider index pProbe. */ for(; rc==SQLITE_OK && pProbe; - pProbe=(pSrc->pIBIndex ? 0 : pProbe->pNext), iSortIdx++ + pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++ ){ int isLeft = (pSrc->fg.jointype & JT_OUTER)!=0; if( pProbe->pPartIdxWhere!=0