From: drh <> Date: Sat, 13 Feb 2021 23:46:26 +0000 (+0000) Subject: If the GENERATED keyword occurs before the AS keyword in a common table X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=bd4dfa90713e074bd5d68edef2682174f86a5f58;p=thirdparty%2Fsqlite.git If the GENERATED keyword occurs before the AS keyword in a common table expression (CTE) definition, then that CTE becomes an "optimization barrier". For now, that means the CTE is always materialized. It also means that query flattener or pushdown optimizations that cross the CTE boundary are omitted. FossilOrigin-Name: 186ec18b24d16c5d4d7a5b5e98c7504892950a1018774cb7bb214fc011ec09f4 --- bd4dfa90713e074bd5d68edef2682174f86a5f58 diff --cc manifest index 81d46eaf6d,70814337f7..c87d81f18d --- a/manifest +++ b/manifest @@@ -1,5 -1,5 +1,5 @@@ - C Fix\sincorrect\stest\sname\slabels\sin\sthe\sselect1.test\sscript. - D 2021-02-13T18:14:15.205 -C Parsing\sof\sDML\sstatements\sin\sa\sWITH\sclause.\s\sBut\sat\sthis\spoint,\sit\sjust\ngenerates\san\serror\sabout\s"not\syet\ssupported". -D 2021-02-12T21:07:58.382 ++C If\sthe\sGENERATED\skeyword\soccurs\sbefore\sthe\sAS\skeyword\sin\sa\scommon\stable\nexpression\s(CTE)\sdefinition,\sthen\sthat\sCTE\sbecomes\san\s"optimization\sbarrier".\nFor\snow,\sthat\smeans\sthe\sCTE\sis\salways\smaterialized.\s\sIt\salso\smeans\sthat\nquery\sflattener\sor\spushdown\soptimizations\sthat\scross\sthe\sCTE\sboundary\sare\nomitted. ++D 2021-02-13T23:46:26.876 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@@ -485,7 -485,7 +485,7 @@@ F src/btmutex.c 8acc2f464ee76324bf13310 F src/btree.c 694020ad8a3af3d79b09f74c8f1421272a419cdea42a13401e3b0f7dea6e9c3e F src/btree.h 285f8377aa1353185a32bf455faafa9ff9a0d40d074d60509534d14990c7829e F src/btreeInt.h 7614cae30f95b6aed0c7cac7718276a55cfe2c77058cbfd8bef5b75329757331 - F src/build.c 1bae5588bfdf21bdf41e634f0a053d633fb1ae3a2896117b4eea76412b76c2e0 -F src/build.c 5687c1d14855af7c411de7e4316d3938d56aa6ab2c98c7f7bc31e9e8cf8bac17 ++F src/build.c 9080db2acc487f91b0ac9a49f6faab3be4709fde80707ac94375e141018c6e12 F src/callback.c d0b853dd413255d2e337b34545e54d888ea02f20da5ad0e63585b389624c4a6c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 2a322b9a3d75771fb4d99e0702851f4f68dda982507a0f798eefb0712969a410 @@@ -530,7 -530,7 +530,7 @@@ F src/os_win.c 77d39873836f1831a9b0b918 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c c49952ac5e9cc536778eff528091d79d38b3e45cbeeed4695dc05e207dc6547d F src/pager.h 4bf9b3213a4b2bebbced5eaa8b219cf25d4a82f385d093cd64b7e93e5285f66f - F src/parse.y 67ba503780de64b967ae195b7e14c33531329228e1bc0b83d63324beb733680b -F src/parse.y 75974e46f056c5a6aa18b9b894b784a0c3129fcb77a6d1b42c0356663ea1b179 ++F src/parse.y cb23465fe1f5e9d51a5466a382c0c38386dd43b818c4b703cafe2e3ecf942bfd F src/pcache.c 385ff064bca69789d199a98e2169445dc16e4291fa807babd61d4890c3b34177 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 6596e10baf3d8f84cc1585d226cf1ab26564a5f5caf85a15757a281ff977d51a @@@ -541,12 -541,12 +541,12 @@@ F src/printf.c 30e92b638fac71dcd85cdea1 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 89e4faf6171e179edf279905e8e45c4f9dd108777dc60716396729fbd7cb045e F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 - F src/select.c 9b4c84fd2703ee3c8b5d4b189387482a84c26acf2c38ca4835db5b48c68a09d4 -F src/select.c 738cb746189f721f59972993c13085fa2975c4cbfd04ba26445f3b42c81237dc ++F src/select.c cf08d39bf4ebf53d3bda7a4482b09274fc23a052deb9db69e6ef98fbfa27e9c9 F src/shell.c.in 9ebc74e4f05cfbd0f4a36060fdaeff1da4e9af4458358722bc08c5a1ab9a0879 F src/sqlite.h.in 8855a19f37ade8dad189a9e48233a2ebe1b46faf469c7eb0906a654e252dcc57 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e - F src/sqliteInt.h 4cb469678a0dbf814e4efbde4488a0161a5398e9a63141830d9f676b4e9fb0cc -F src/sqliteInt.h c849d0f05d39733816966f4de2d16e4d86113c2cf2eea764aa603049ccc07179 ++F src/sqliteInt.h 20ee41712b2eeb4293ea6fe6bdbccc1d766a1889bfe6bad9759040b9ee493617 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@@ -1827,7 -1827,7 +1827,7 @@@ F tool/max-limits.c cbb635fbb37ae4d05f2 F tool/mkautoconfamal.sh f62353eb6c06ab264da027fd4507d09914433dbdcab9cb011cdc18016f1ab3b8 F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x F tool/mkctimec.tcl dd183b73ae1c28249669741c250525f0407e579a70482371668fd5f130d9feb3 --F tool/mkkeywordhash.c 750f25aef0e23f8e3367af6d824fbf5ed7d3e285f27cea91aa2dd72c367630eb ++F tool/mkkeywordhash.c 9d60becd70cbf34999654de6d35e3c7f05041c3eff9606f60e93f017f9ce0d92 F tool/mkmsvcmin.tcl 6ecab9fe22c2c8de4d82d4c46797bda3d2deac8e763885f5a38d0c44a895ab33 F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl 352a4319c0ad869eb26442bf7c3b015aa15594c21f1cce5a6420dbe999367c21 @@@ -1899,7 -1899,10 +1899,10 @@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 - P 9692f510803c9b9725abb687d7c10fbc0d5ed784479ec6f3fcc55925a87fe16d - R 0c64c86089d6aa467750f9eba3dbafff -P 66c07a07b21e46529780eec3c82a84c494d586f8b7ed80b78d358e23b80458c7 -R 1961a25c73df65dd61d620956403440e -T *branch * dml-in-cte -T *sym-dml-in-cte * ++P 179c79ea0deb0f5adaa8d369cfcad06d959a9cc18a8a41e01ef013b2d90acd61 964ff68d8fa4d72b1b4d510f1ec62a9674b75d4ebae4ba254bc2f99432857628 ++R 1e58442e28f61184e5599226680b1725 ++T *branch * with-generated-as ++T *sym-with-generated-as * + T -sym-trunk * U drh - Z bef3ac59ff66799b655a50d5e67ecd31 -Z ef9b268336156975e58d505fbf96353f ++Z 34848bab44cd045c96efef9184fac0b9 diff --cc manifest.uuid index 16c0579a8f,fc1bb3a81c..85159612dc --- a/manifest.uuid +++ b/manifest.uuid @@@ -1,1 -1,1 +1,1 @@@ - 179c79ea0deb0f5adaa8d369cfcad06d959a9cc18a8a41e01ef013b2d90acd61 -964ff68d8fa4d72b1b4d510f1ec62a9674b75d4ebae4ba254bc2f99432857628 ++186ec18b24d16c5d4d7a5b5e98c7504892950a1018774cb7bb214fc011ec09f4 diff --cc src/build.c index f5c796fac3,de3c417c8b..015001f252 --- a/src/build.c +++ b/src/build.c @@@ -5204,6 -5204,45 +5204,47 @@@ KeyInfo *sqlite3KeyInfoOfIndex(Parse *p } #ifndef SQLITE_OMIT_CTE + /* + ** Create a new CTE object + */ + Cte *sqlite3CteNew( + Parse *pParse, /* Parsing context */ + Token *pName, /* Name of the common-table */ + ExprList *pArglist, /* Optional column name list for the table */ - Select *pQuery /* Query used to initialize the table */ ++ Select *pQuery, /* Query used to initialize the table */ ++ int bOptBarrier /* This CTE should be an optimization barrier*/ + ){ + Cte *pNew; + sqlite3 *db = pParse->db; + + pNew = sqlite3DbMallocZero(db, sizeof(*pNew)); + assert( pNew!=0 || db->mallocFailed ); + + if( db->mallocFailed ){ + sqlite3ExprListDelete(db, pArglist); + sqlite3SelectDelete(db, pQuery); + }else{ + pNew->pSelect = pQuery; + pNew->pCols = pArglist; + pNew->zName = sqlite3NameFromToken(pParse->db, pName); + pNew->zCteErr = 0; ++ pNew->bOptBarrier = bOptBarrier; + } + return pNew; + } + + /* + ** Free the contents of the CTE object passed as the second argument. + */ + void sqlite3CteDelete(sqlite3 *db, Cte *pCte){ + if( pCte ){ + sqlite3ExprListDelete(db, pCte->pCols); + sqlite3SelectDelete(db, pCte->pSelect); + sqlite3DbFree(db, pCte->zName); + sqlite3DbFree(db, pCte); + } + } + /* ** This routine is invoked once per CTE by the parser while parsing a ** WITH clause. diff --cc src/parse.y index 591cde3b9d,66e6f62493..6d2e2d5519 --- a/src/parse.y +++ b/src/parse.y @@@ -247,8 -247,8 +247,9 @@@ columnname(A) ::= nm(A) typetoken(Y). { CURRENT FOLLOWING PARTITION PRECEDING RANGE UNBOUNDED EXCLUDE GROUPS OTHERS TIES %endif SQLITE_OMIT_WINDOWFUNC ++ GENERATED %ifndef SQLITE_OMIT_GENERATED_COLUMNS -- GENERATED ALWAYS ++ ALWAYS %endif REINDEX RENAME CTIME_KW IF . @@@ -1658,17 -1658,36 +1659,25 @@@ anylist ::= anylist ANY //////////////////////// COMMON TABLE EXPRESSIONS //////////////////////////// %type wqlist {With*} %destructor wqlist {sqlite3WithDelete(pParse->db, $$);} + %type wqitem {Cte*} + %destructor wqitem {sqlite3CteDelete(pParse->db, $$);} ++%type wqgen {int} with ::= . %ifndef SQLITE_OMIT_CTE with ::= WITH wqlist(W). { sqlite3WithPush(pParse, W, 1); } with ::= WITH RECURSIVE wqlist(W). { sqlite3WithPush(pParse, W, 1); } - wqlist(A) ::= nm(X) eidlist_opt(Y) AS LP select(Z) RP. { - A = sqlite3WithAdd(pParse, 0, &X, Y, Z); /*A-overwrites-X*/ -wqitem(A) ::= nm(X) eidlist_opt(Y) AS LP select(Z) RP. { - A = sqlite3CteNew(pParse, &X, Y, Z); /*A-overwrites-X*/ -} -wqitem(A) ::= nm eidlist_opt AS LP DELETE FROM nm dbnm where_opt_ret RP. { - sqlite3ErrorMsg(pParse, "DELETE in WITH clauses not yet implemented"); - A = 0; -} -wqitem(A) ::= nm eidlist_opt AS LP - insert_cmd INTO nm dbnm idlist_opt select upsert RP. { - sqlite3ErrorMsg(pParse, "INSERT in WITH clauses not yet implemented"); - A = 0; -} -wqitem(A) ::= nm eidlist_opt AS LP - UPDATE orconf nm dbnm SET setlist from where_opt_ret RP. { - sqlite3ErrorMsg(pParse, "UPDATE in WITH clauses not yet implemented"); - A = 0; ++wqgen(A) ::= . {A=0;} ++wqgen(A) ::= GENERATED. {A=1;} ++wqitem(A) ::= nm(X) eidlist_opt(Y) wqgen(F) AS LP select(Z) RP. { ++ A = sqlite3CteNew(pParse, &X, Y, Z, F); /*A-overwrites-X*/ } - wqlist(A) ::= wqlist(A) COMMA nm(X) eidlist_opt(Y) AS LP select(Z) RP. { - A = sqlite3WithAdd(pParse, A, &X, Y, Z); + wqlist(A) ::= wqitem(X). { + A = sqlite3WithAdd(pParse, 0, X); /*A-overwrites-X*/ + } + wqlist(A) ::= wqlist(A) COMMA wqitem(X). { + A = sqlite3WithAdd(pParse, A, X); } %endif SQLITE_OMIT_CTE diff --cc src/select.c index 1ab2ec367e,7c3019890c..7866114760 --- a/src/select.c +++ b/src/select.c @@@ -4938,6 -4936,6 +4938,7 @@@ static int withExpand pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; assert( pFrom->pSelect ); ++ if( pCte->bOptBarrier ) pFrom->pSelect->selFlags |= SF_OptBarrier; /* Check if this is a recursive CTE. */ pRecTerm = pSel = pFrom->pSelect; @@@ -6070,6 -6068,6 +6071,9 @@@ int sqlite3Select continue; } ++ /* Do not flatten across an optimization barrier */ ++ if( pSub->selFlags & SF_OptBarrier ) continue; ++ if( flattenSubquery(pParse, p, i, isAgg) ){ if( pParse->nErr ) goto select_end; /* This subquery can be absorbed into its parent. */ @@@ -6194,6 -6192,6 +6198,7 @@@ ** inside the subquery. This can help the subquery to run more efficiently. */ if( OptimizationEnabled(db, SQLITE_PushDown) ++ && (pSub->selFlags & SF_OptBarrier)==0 && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem->iCursor, (pItem->fg.jointype & JT_OUTER)!=0) ){ @@@ -6214,16 -6211,16 +6219,22 @@@ /* Generate code to implement the subquery ** -- ** The subquery is implemented as a co-routine if the subquery is ++ ** The subquery is implemented as a co-routine if (1) the subquery is ** guaranteed to be the outer loop (so that it does not need to be -- ** computed more than once) ++ ** computed more than once) and if (2) the subquery is not optimization ++ ** barrier. ** -- ** TODO: Are there other reasons beside (1) to use a co-routine ++ ** TODO: Are there other reasons beside (1) and (2) to use a co-routine ** implementation? ++ ** ++ ** TODO: We might should allow a subquery that is an optimization barrier ++ ** to be implemented as a co-routine as long as we know that it is the ++ ** only use of the subquery. */ if( i==0 && (pTabList->nSrc==1 || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ ++ && (pSub->selFlags & SF_OptBarrier)==0 /* (2) */ ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. diff --cc src/sqliteInt.h index 17b1cb415d,c0a11ef753..9f423d977b --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@@ -3184,7 -3185,6 +3185,8 @@@ struct Select #define SF_View 0x0200000 /* SELECT statement is a view */ #define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ #define SF_UpdateFrom 0x0800000 /* Statement is an UPDATE...FROM */ +#define SF_PushDown 0x1000000 /* SELECT has be modified by push-down opt */ ++#define SF_OptBarrier 0x2000000 /* SELECT is an optimization barrier */ /* ** The results of a SELECT can be distributed in several ways, as defined @@@ -3871,15 -3871,16 +3873,17 @@@ void sqlite3SelectWalkAssert2(Walker*, ** An instance of this structure represents a set of one or more CTEs ** (common table expressions) created by a single WITH clause. */ + struct Cte { + char *zName; /* Name of this CTE */ + ExprList *pCols; /* List of explicit column names, or NULL */ + Select *pSelect; /* The definition of this CTE */ + const char *zCteErr; /* Error message for circular references */ ++ u8 bOptBarrier; /* Treat this CTE as an optimization barrier */ + }; struct With { - int nCte; /* Number of CTEs in the WITH clause */ - With *pOuter; /* Containing WITH clause, or NULL */ - struct Cte { /* For each CTE in the WITH clause.... */ - char *zName; /* Name of this CTE */ - ExprList *pCols; /* List of explicit column names, or NULL */ - Select *pSelect; /* The definition of this CTE */ - const char *zCteErr; /* Error message for circular references */ - } a[1]; + int nCte; /* Number of CTEs in the WITH clause */ + With *pOuter; /* Containing WITH clause, or NULL */ + Cte a[1]; /* The CTEs of this WITH clause */ }; #ifdef SQLITE_DEBUG @@@ -4883,7 -4884,9 +4887,9 @@@ const char *sqlite3JournalModename(int) int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); #endif #ifndef SQLITE_OMIT_CTE - With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*); - Cte *sqlite3CteNew(Parse*,Token*,ExprList*,Select*); ++ Cte *sqlite3CteNew(Parse*,Token*,ExprList*,Select*,int); + With *sqlite3WithAdd(Parse*,With*,Cte*); + void sqlite3CteDelete(sqlite3*,Cte*); void sqlite3WithDelete(sqlite3*,With*); void sqlite3WithPush(Parse*, With*, u8); #else diff --cc tool/mkkeywordhash.c index ea3763fd19,ea3763fd19..c60999c23b --- a/tool/mkkeywordhash.c +++ b/tool/mkkeywordhash.c @@@ -229,7 -229,7 +229,7 @@@ static Keyword aKeywordTable[] = { "FOREIGN", "TK_FOREIGN", FKEY, 1 }, { "FROM", "TK_FROM", ALWAYS, 10 }, { "FULL", "TK_JOIN_KW", ALWAYS, 3 }, -- { "GENERATED", "TK_GENERATED", GENCOL, 1 }, ++ { "GENERATED", "TK_GENERATED", ALWAYS, 1 }, { "GLOB", "TK_LIKE_KW", ALWAYS, 3 }, { "GROUP", "TK_GROUP", ALWAYS, 5 }, { "GROUPS", "TK_GROUPS", WINDOWFUNC, 2 },