From: drh Date: Mon, 8 Jun 2020 12:49:53 +0000 (+0000) Subject: When an Expr object is changed and that Expr is referenced by an AggInfo, then X-Git-Tag: version-3.32.3~8 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=2aebad17f7741eb80bb01a179ddfad4197055421;p=thirdparty%2Fsqlite.git When an Expr object is changed and that Expr is referenced by an AggInfo, then also update the AggInfo. Also, persist all AggInfo objects until the Parse object is destroyed. This is a new fix for ticket [c8d3b9f0a750a529]. FossilOrigin-Name: 44a58d6cb135a1040807093e27e3d4bdcfd5fd2c2b52e9af1e5f01c26b321d82 --- diff --git a/manifest b/manifest index b09714b0f9..6253541e54 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.32.2 -D 2020-06-04T12:58:43.805 +C When\san\sExpr\sobject\sis\schanged\sand\sthat\sExpr\sis\sreferenced\sby\san\sAggInfo,\sthen\nalso\supdate\sthe\sAggInfo.\s\sAlso,\spersist\sall\sAggInfo\sobjects\suntil\sthe\sParse\nobject\sis\sdestroyed.\s\sThis\sis\sa\snew\sfix\sfor\sticket\s[c8d3b9f0a750a529]. +D 2020-06-08T12:49:53.972 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -477,7 +477,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c f14e415fcfd0b52b4e4ebd193ba5fadac5e8252c30f023389af682813af44025 F src/btree.h 989ef3c33413549e3e148f3dcb46c030f317dac130dc86809ba6b9aa4b16c72a F src/btreeInt.h 5c8b8749805787313ecf49eb5be3ced1e94bbf8ef54bb01470ce6bd0d5185c67 -F src/build.c ca9e7a33b74f1bf2eb3a5f37f9d07dfed335469f2d70c0bd350e0dd42a50183a +F src/build.c 912d6ec338311105821e4439e972a9f5eea78338b181d3e432bfb2306227b00a F src/callback.c d0b853dd413255d2e337b34545e54d888ea02f20da5ad0e63585b389624c4a6c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c e98518d2d3d4029a13c805e07313fb60c877be56db76e90dd5f3af73085d0ce6 @@ -485,7 +485,7 @@ F src/date.c b29b349d277e3d579dcc295b24c0a2caed83fd8f090a9f7cbe6070c0fd662384 F src/dbpage.c 8a01e865bf8bc6d7b1844b4314443a6436c07c3efe1d488ed89e81719047833a F src/dbstat.c 793deaf88a0904f88285d93d6713c636d55ede0ffd9f08d10f4ea825531d367f F src/delete.c 88047c8e59878c920fce14582bc1dde4d81157d1ca5ffdf36c2907e6d41996c4 -F src/expr.c b46669d9fc9e0361dba6cc289901a013789e0b1dc629c4c1bc88ec9403633b38 +F src/expr.c 6ac12b2880d67cacb18a0164abc9ad7bce8143a29161d1a6ae38232eceb38ee1 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 4b575423b0a5d4898b1a7868ce985cf1a8ad91c741c9abbb108ff02536d20f41 F src/func.c 2333eb4277f55a5efdc12ef754e7d7ec9105d257b2fd00301d23ce1e8fa67dc0 @@ -528,17 +528,17 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 6596e10baf3d8f84cc1585d226cf1ab26564a5f5caf85a15757a281ff977d51a F src/pragma.c 1b0db48177e52b256c003b8dc6ac708b1079a82cded944a23820574586a4731f F src/pragma.h 8168e588536bffd95319451f34e9a754dc37d205ebe433031a7813c5b286beae -F src/prepare.c 8d4d6c8aa6afefc48027c54b41cdf134b4d6bc2fc4badbe483ad7fd9e1728a28 +F src/prepare.c 0a263a8840096cb83d952f5b098068a67a93020672bb64b31c744e7b5c9455b9 F src/printf.c 94b5419ad0a17269f76a9e968ca19cf9fa37617abed2e246fc48844e511b6bc6 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c c2008519a0654f1e7490e9281ed0397d0f14bb840d81f0b96946248afcbdb25d F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c e2a59548681bf1c407132863ae87bf2444aca5543867c21d9d09aa07f44aef5d +F src/select.c bafed8a4f28d84e3cae12f0b6393dbcb0e4d7f6a1b2f6cf93e4b99d102829588 F src/shell.c.in cf2d24f54412c06e5fb34af7fabc748651125e1dceac29b740e91f06d23447b6 F src/sqlite.h.in 74342b41e9d68ff9e56b192009046f8dd0aa2bd76ce1a588f330de614ba61de7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 2d1af80082edffd71c6f96f70ad1ce6a4fb46615ad10291fc77fe0dea9ff0197 -F src/sqliteInt.h 37f74544c8e2f59a16d1b13e6847bc02fb3fcbbfae4d6e6342c7a94c3a3984a5 +F src/sqliteInt.h 5483382f76b15c3dba53cae260ac5168ab7e608c0c1b11852b2116590da87c0a F src/sqliteLimit.h 95cb8479ca459496d9c1c6a9f76b38aee12203a56ce1092fe13e50ae2454c032 F src/status.c 9ff2210207c6c3b4d9631a8241a7d45ab1b26a0e9c84cb07a9b5ce2de9a3b278 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -624,7 +624,7 @@ F src/where.c 9546c82056e8cdb27291f98cf1adca5d271240b399bb97b32f77fc2bea6146c9 F src/whereInt.h 6b874aa15f94e43a2cec1080be64d955b04deeafeac90ffb5d6975c0d511be3c F src/wherecode.c 7b939de85d65cc4b4bfa197513136b9e0ae03167e3b82842ca5a0ba1055ba65d F src/whereexpr.c 264d58971eaf8256eb5b0917bcd7fc7a1f1109fdda183a8382308a1b18a2dce7 -F src/window.c 66c5fd1e48af7581cf90b97700268294f4da4037f120f367715f912e1148d3f9 +F src/window.c 7d19c08b62b70d5dc2ef4c098cc5ba611fac246a111fa36ebb39505415124b90 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -1029,7 +1029,7 @@ F test/fuzzdata4.db b502c7d5498261715812dd8b3c2005bad08b3a26e6489414bd13926cd3e4 F test/fuzzdata5.db e35f64af17ec48926481cfaf3b3855e436bd40d1cfe2d59a9474cb4b748a52a5 F test/fuzzdata6.db 92a80e4afc172c24f662a10a612d188fb272de4a9bd19e017927c95f737de6d7 F test/fuzzdata7.db 0166b56fd7a6b9636a1d60ef0a060f86ddaecf99400a666bb6e5bbd7199ad1f2 -F test/fuzzdata8.db 209623791b0ad72ab39110c867af2080a79004e493c4da14ad661e790b5d1ed8 +F test/fuzzdata8.db 6acee588a3311994b57b226c9eac9e9e69f135cb8b000bd48fdb924e32041312 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 @@ -1727,7 +1727,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/window1.test 502a3e72fce895aaaa633bbe709f6de3c24eed3ecb8098ab9fb8b47584b4452c +F test/window1.test 49f81c25c065fd65b64cd2aaa96f20199fd5d8f847dd1c8151cfccae2aeae83f F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03 @@ -1866,10 +1866,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9cfefef591dc555e41fe267e5a5b128f82038d01f8c44480ede16be4a923863f -R 6aa42dcffed4a96d974d4a71a5dcf630 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.32.2 * +P ec02243ea6ce33b090870ae55ab8aa2534b54d216d45c4aa2fdbb00e86861e8c +Q +6e6b3729e0549de028f6c5bf494b2d69d621c81b61a1dc0a329d3950039342fb +R ffc67efd25aa2f46e8c3ffc0407920c8 U drh -Z ba5e4d2ff24d081fa0b14b3b24d4fae5 +Z 629400a4498e0ac085c4a9f3c83c9843 diff --git a/manifest.uuid b/manifest.uuid index 6ca86a7914..aa4b965c6e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec02243ea6ce33b090870ae55ab8aa2534b54d216d45c4aa2fdbb00e86861e8c \ No newline at end of file +44a58d6cb135a1040807093e27e3d4bdcfd5fd2c2b52e9af1e5f01c26b321d82 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 6ff595c623..c2f0abb156 100644 --- a/src/build.c +++ b/src/build.c @@ -207,12 +207,21 @@ void sqlite3FinishCoding(Parse *pParse){ */ sqlite3AutoincrementBegin(pParse); - /* Code constant expressions that where factored out of inner loops */ + /* Code constant expressions that where factored out of inner loops. + ** + ** The pConstExpr list might also contain expressions that we simply + ** want to keep around until the Parse object is deleted. Such + ** expressions have iConstExprReg==0. Do not generate code for + ** those expressions, of course. + */ if( pParse->pConstExpr ){ ExprList *pEL = pParse->pConstExpr; pParse->okConstFactor = 0; for(i=0; inExpr; i++){ - sqlite3ExprCode(pParse, pEL->a[i].pExpr, pEL->a[i].u.iConstExprReg); + int iReg = pEL->a[i].u.iConstExprReg; + if( iReg>0 ){ + sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); + } } } diff --git a/src/expr.c b/src/expr.c index c5b6783871..da22eece70 100644 --- a/src/expr.c +++ b/src/expr.c @@ -52,7 +52,12 @@ char sqlite3ExprAffinity(const Expr *pExpr){ op = pExpr->op; if( op==TK_SELECT ){ assert( pExpr->flags&EP_xIsSelect ); - return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); + if( ALWAYS(pExpr->x.pSelect) + && pExpr->x.pSelect->pEList + && ALWAYS(pExpr->x.pSelect->pEList->a[0].pExpr) + ){ + return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); + } } if( op==TK_REGISTER ) op = pExpr->op2; #ifndef SQLITE_OMIT_CAST @@ -5710,6 +5715,64 @@ int sqlite3FunctionUsesThisSrc(Expr *pExpr, SrcList *pSrcList){ return cnt.nThis>0 || cnt.nOther==0; } +/* +** This is a Walker expression node callback. +** +** For Expr nodes that contain pAggInfo pointers, make sure the AggInfo +** object that is referenced does not refer directly to the Expr. If +** it does, make a copy. This is done because the pExpr argument is +** subject to change. +** +** The copy is stored on pParse->pConstExpr with a register number of 0. +** This will cause the expression to be deleted automatically when the +** Parse object is destroyed, but the zero register number means that it +** will not generate any code in the preamble. +*/ +static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ + if( ALWAYS(!ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced)) + && pExpr->pAggInfo!=0 + ){ + AggInfo *pAggInfo = pExpr->pAggInfo; + int iAgg = pExpr->iAgg; + Parse *pParse = pWalker->pParse; + sqlite3 *db = pParse->db; + assert( pExpr->op==TK_AGG_COLUMN || pExpr->op==TK_AGG_FUNCTION ); + if( pExpr->op==TK_AGG_COLUMN ){ + assert( iAgg>=0 && iAggnColumn ); + if( pAggInfo->aCol[iAgg].pExpr==pExpr ){ + pExpr = sqlite3ExprDup(db, pExpr, 0); + if( pExpr ){ + pAggInfo->aCol[iAgg].pExpr = pExpr; + pParse->pConstExpr = + sqlite3ExprListAppend(pParse, pParse->pConstExpr, pExpr); + } + } + }else{ + assert( iAgg>=0 && iAggnFunc ); + if( pAggInfo->aFunc[iAgg].pExpr==pExpr ){ + pExpr = sqlite3ExprDup(db, pExpr, 0); + if( pExpr ){ + pAggInfo->aFunc[iAgg].pExpr = pExpr; + pParse->pConstExpr = + sqlite3ExprListAppend(pParse, pParse->pConstExpr, pExpr); + } + } + } + } + return WRC_Continue; +} + +/* +** Initialize a Walker object so that will persist AggInfo entries referenced +** by the tree that is walked. +*/ +void sqlite3AggInfoPersistWalkerInit(Walker *pWalker, Parse *pParse){ + memset(pWalker, 0, sizeof(*pWalker)); + pWalker->pParse = pParse; + pWalker->xExprCallback = agginfoPersistExprCb; + pWalker->xSelectCallback = sqlite3SelectWalkNoop; +} + /* ** Add a new element to the pAggInfo->aCol[] array. Return the index of ** the new element. Return a negative number if malloc fails. @@ -5740,7 +5803,7 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ &i ); return i; -} +} /* ** This is the xExprCallback for a tree walker. It is used to diff --git a/src/prepare.c b/src/prepare.c index 228d14876e..5031a0689e 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -529,11 +529,26 @@ int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ return i; } +/* +** Deallocate a single AggInfo object +*/ +static void agginfoFree(sqlite3 *db, AggInfo *p){ + sqlite3DbFree(db, p->aCol); + sqlite3DbFree(db, p->aFunc); + sqlite3DbFree(db, p); +} + /* ** Free all memory allocations in the pParse object */ void sqlite3ParserReset(Parse *pParse){ sqlite3 *db = pParse->db; + AggInfo *pThis = pParse->pAggList; + while( pThis ){ + AggInfo *pNext = pThis->pNext; + agginfoFree(db, pThis); + pThis = pNext; + } sqlite3DbFree(db, pParse->aLabel); sqlite3ExprListDelete(db, pParse->pConstExpr); if( db ){ diff --git a/src/select.c b/src/select.c index 534dfbc01b..9e459cd903 100644 --- a/src/select.c +++ b/src/select.c @@ -3792,6 +3792,7 @@ static int flattenSubquery( Expr *pWhere; /* The WHERE clause */ struct SrcList_item *pSubitem; /* The subquery */ sqlite3 *db = pParse->db; + Walker w; /* Walker to persist agginfo data */ /* Check to see if flattening is permitted. Return 0 if not. */ @@ -4166,6 +4167,8 @@ static int flattenSubquery( /* Finially, delete what is left of the subquery and return ** success. */ + sqlite3AggInfoPersistWalkerInit(&w, pParse); + sqlite3WalkSelect(&w,pSub1); sqlite3SelectDelete(db, pSub1); #if SELECTTRACE_ENABLED @@ -5750,10 +5753,10 @@ int sqlite3Select( Expr *pWhere; /* The WHERE clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ Expr *pHaving; /* The HAVING clause. May be NULL */ + AggInfo *pAggInfo = 0; /* Aggregate information */ int rc = 1; /* Value to return from this function */ DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */ SortCtx sSort; /* Info on how to code the ORDER BY clause */ - AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ ExprList *pMinMaxOrderBy = 0; /* Added ORDER BY for min/max queries */ @@ -5765,7 +5768,6 @@ int sqlite3Select( return 1; } if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; - memset(&sAggInfo, 0, sizeof(sAggInfo)); #if SELECTTRACE_ENABLED SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->addrExplain)); if( sqlite3SelectTrace & 0x100 ){ @@ -6355,14 +6357,20 @@ int sqlite3Select( ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the ** SELECT statement. */ + pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) ); + if( pAggInfo==0 ){ + goto select_end; + } + pAggInfo->pNext = pParse->pAggList; + pParse->pAggList = pAggInfo; memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; sNC.pSrcList = pTabList; - sNC.uNC.pAggInfo = &sAggInfo; + sNC.uNC.pAggInfo = pAggInfo; VVA_ONLY( sNC.ncFlags = NC_UAggInfo; ) - sAggInfo.mnReg = pParse->nMem+1; - sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; - sAggInfo.pGroupBy = pGroupBy; + pAggInfo->mnReg = pParse->nMem+1; + pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; + pAggInfo->pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); if( pHaving ){ @@ -6375,14 +6383,14 @@ int sqlite3Select( } sqlite3ExprAnalyzeAggregates(&sNC, pHaving); } - sAggInfo.nAccumulator = sAggInfo.nColumn; - if( p->pGroupBy==0 && p->pHaving==0 && sAggInfo.nFunc==1 ){ - minMaxFlag = minMaxQuery(db, sAggInfo.aFunc[0].pExpr, &pMinMaxOrderBy); + pAggInfo->nAccumulator = pAggInfo->nColumn; + if( p->pGroupBy==0 && p->pHaving==0 && pAggInfo->nFunc==1 ){ + minMaxFlag = minMaxQuery(db, pAggInfo->aFunc[0].pExpr, &pMinMaxOrderBy); }else{ minMaxFlag = WHERE_ORDERBY_NORMAL; } - for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pExpr; assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); sNC.ncFlags |= NC_InAggFunc; sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList); @@ -6394,22 +6402,22 @@ int sqlite3Select( #endif sNC.ncFlags &= ~NC_InAggFunc; } - sAggInfo.mxReg = pParse->nMem; + pAggInfo->mxReg = pParse->nMem; if( db->mallocFailed ) goto select_end; #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x400 ){ int ii; - SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", &sAggInfo)); + SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); sqlite3TreeViewSelect(0, p, 0); - for(ii=0; iinColumn; ii++){ sqlite3DebugPrintf("agg-column[%d] iMem=%d\n", - ii, sAggInfo.aCol[ii].iMem); - sqlite3TreeViewExpr(0, sAggInfo.aCol[ii].pExpr, 0); + ii, pAggInfo->aCol[ii].iMem); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pExpr, 0); } - for(ii=0; iinFunc; ii++){ sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", - ii, sAggInfo.aFunc[ii].iMem); - sqlite3TreeViewExpr(0, sAggInfo.aFunc[ii].pExpr, 0); + ii, pAggInfo->aFunc[ii].iMem); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pExpr, 0); } } #endif @@ -6434,10 +6442,11 @@ int sqlite3Select( ** that we do not need it after all, the OP_SorterOpen instruction ** will be converted into a Noop. */ - sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pGroupBy,0,sAggInfo.nColumn); + pAggInfo->sortingIdx = pParse->nTab++; + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pGroupBy, + 0, pAggInfo->nColumn); addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, - sAggInfo.sortingIdx, sAggInfo.nSortingColumn, + pAggInfo->sortingIdx, pAggInfo->nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); /* Initialize memory locations used by GROUP BY aggregate processing @@ -6492,8 +6501,8 @@ int sqlite3Select( nGroupBy = pGroupBy->nExpr; nCol = nGroupBy; j = nGroupBy; - for(i=0; i=j ){ + for(i=0; inColumn; i++){ + if( pAggInfo->aCol[i].iSorterColumn>=j ){ nCol++; j++; } @@ -6501,8 +6510,8 @@ int sqlite3Select( regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); j = nGroupBy; - for(i=0; inColumn; i++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[i]; if( pCol->iSorterColumn>=j ){ int r1 = j + regBase; sqlite3ExprCodeGetColumnOfTable(v, @@ -6512,16 +6521,16 @@ int sqlite3Select( } regRecord = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); - sqlite3VdbeAddOp2(v, OP_SorterInsert, sAggInfo.sortingIdx, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); sqlite3WhereEnd(pWInfo); - sAggInfo.sortingIdxPTab = sortPTab = pParse->nTab++; + pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++; sortOut = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); - sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd); + sqlite3VdbeAddOp2(v, OP_SorterSort, pAggInfo->sortingIdx, addrEnd); VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); - sAggInfo.useSortingIdx = 1; + pAggInfo->useSortingIdx = 1; } /* If the index or temporary table used by the GROUP BY sort @@ -6545,14 +6554,14 @@ int sqlite3Select( */ addrTopOfLoop = sqlite3VdbeCurrentAddr(v); if( groupBySort ){ - sqlite3VdbeAddOp3(v, OP_SorterData, sAggInfo.sortingIdx, + sqlite3VdbeAddOp3(v, OP_SorterData, pAggInfo->sortingIdx, sortOut, sortPTab); } for(j=0; jnExpr; j++){ if( groupBySort ){ sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); }else{ - sAggInfo.directMode = 1; + pAggInfo->directMode = 1; sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); } } @@ -6582,14 +6591,14 @@ int sqlite3Select( ** the current row */ sqlite3VdbeJumpHere(v, addr1); - updateAccumulator(pParse, iUseFlag, &sAggInfo); + updateAccumulator(pParse, iUseFlag, pAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); /* End of the loop */ if( groupBySort ){ - sqlite3VdbeAddOp2(v, OP_SorterNext, sAggInfo.sortingIdx, addrTopOfLoop); + sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx, addrTopOfLoop); VdbeCoverage(v); }else{ sqlite3WhereEnd(pWInfo); @@ -6622,7 +6631,7 @@ int sqlite3Select( VdbeCoverage(v); VdbeComment((v, "Groupby result generator entry point")); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); - finalizeAggFunctions(pParse, &sAggInfo); + finalizeAggFunctions(pParse, pAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, @@ -6633,7 +6642,7 @@ int sqlite3Select( /* Generate a subroutine that will reset the group-by accumulator */ sqlite3VdbeResolveLabel(v, addrReset); - resetAccumulator(pParse, &sAggInfo); + resetAccumulator(pParse, pAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); VdbeComment((v, "indicate accumulator empty")); sqlite3VdbeAddOp1(v, OP_Return, regReset); @@ -6641,7 +6650,7 @@ int sqlite3Select( } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ else { Table *pTab; - if( (pTab = isSimpleCount(p, &sAggInfo))!=0 ){ + if( (pTab = isSimpleCount(p, pAggInfo))!=0 ){ /* If isSimpleCount() returns a pointer to a Table structure, then ** the SQL statement is of the form: ** @@ -6696,7 +6705,7 @@ int sqlite3Select( if( pKeyInfo ){ sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } - sqlite3VdbeAddOp2(v, OP_Count, iCsr, sAggInfo.aFunc[0].iMem); + sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); }else{ @@ -6711,12 +6720,16 @@ int sqlite3Select( ** first row visited by the aggregate, so that they are updated at ** least once even if the FILTER clause means the min() or max() ** function visits zero rows. */ - if( sAggInfo.nAccumulator ){ - for(i=0; ifuncFlags&SQLITE_FUNC_NEEDCOLL ) break; + if( pAggInfo->nAccumulator ){ + for(i=0; inFunc; i++){ + if( ExprHasProperty(pAggInfo->aFunc[i].pExpr, EP_WinFunc) ){ + continue; + } + if( pAggInfo->aFunc[i].pFunc->funcFlags&SQLITE_FUNC_NEEDCOLL ){ + break; + } } - if( i==sAggInfo.nFunc ){ + if( i==pAggInfo->nFunc ){ regAcc = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, regAcc); } @@ -6727,7 +6740,7 @@ int sqlite3Select( ** of output. */ assert( p->pGroupBy==0 ); - resetAccumulator(pParse, &sAggInfo); + resetAccumulator(pParse, pAggInfo); /* If this query is a candidate for the min/max optimization, then ** minMaxFlag will have been previously set to either @@ -6743,7 +6756,7 @@ int sqlite3Select( if( pWInfo==0 ){ goto select_end; } - updateAccumulator(pParse, regAcc, &sAggInfo); + updateAccumulator(pParse, regAcc, pAggInfo); if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); if( sqlite3WhereIsOrdered(pWInfo)>0 ){ sqlite3VdbeGoto(v, sqlite3WhereBreakLabel(pWInfo)); @@ -6751,7 +6764,7 @@ int sqlite3Select( (minMaxFlag==WHERE_ORDERBY_MIN?"min":"max"))); } sqlite3WhereEnd(pWInfo); - finalizeAggFunctions(pParse, &sAggInfo); + finalizeAggFunctions(pParse, pAggInfo); } sSort.pOrderBy = 0; @@ -6790,8 +6803,25 @@ int sqlite3Select( */ select_end: sqlite3ExprListDelete(db, pMinMaxOrderBy); - sqlite3DbFree(db, sAggInfo.aCol); - sqlite3DbFree(db, sAggInfo.aFunc); +#ifdef SQLITE_DEBUG + if( pAggInfo ){ + for(i=0; inColumn; i++){ + Expr *pExpr = pAggInfo->aCol[i].pExpr; + assert( pExpr!=0 || db->mallocFailed ); + if( pExpr==0 ) continue; + assert( pExpr->pAggInfo==pAggInfo ); + assert( pExpr->iAgg==i ); + } + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pExpr; + assert( pExpr!=0 || db->mallocFailed ); + if( pExpr==0 ) continue; + assert( pExpr->pAggInfo==pAggInfo ); + assert( pExpr->iAgg==i ); + } + } +#endif + #if SELECTTRACE_ENABLED SELECTTRACE(0x1,pParse,p,("end processing\n")); if( (sqlite3SelectTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 122e26ad73..2c2e7d98a5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2506,11 +2506,11 @@ struct AggInfo { ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ + Expr *pExpr; /* The original expression */ int iTable; /* Cursor number of the source table */ - int iColumn; /* Column number within the source table */ - int iSorterColumn; /* Column number in the sorting index */ int iMem; /* Memory location that acts as accumulator */ - Expr *pExpr; /* The original expression */ + i16 iColumn; /* Column number within the source table */ + i16 iSorterColumn; /* Column number in the sorting index */ } *aCol; int nColumn; /* Number of used entries in aCol[] */ int nAccumulator; /* Number of columns that show through to the output. @@ -2523,8 +2523,17 @@ struct AggInfo { int iDistinct; /* Ephemeral table used to enforce DISTINCT */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ +#ifdef SQLITE_DEBUG + int iAggMagic; /* Magic number when valid */ +#endif + AggInfo *pNext; /* Next in list of them all */ }; +/* +** Value for AggInfo.iAggMagic when the structure is valid +*/ +#define AggInfoMagic 0x2059e99e + /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. ** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater @@ -3321,6 +3330,7 @@ struct Parse { Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ Parse *pParentParse; /* Parent parser if this parser is nested */ + AggInfo *pAggList; /* List of all AggInfo objects */ int addrCrTab; /* Address of OP_CreateBtree opcode on CREATE TABLE */ u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ @@ -4287,6 +4297,7 @@ int sqlite3ExprCompareSkip(Expr*, Expr*, int); int sqlite3ExprListCompare(ExprList*, ExprList*, int); int sqlite3ExprImpliesExpr(Parse*,Expr*, Expr*, int); int sqlite3ExprImpliesNonNullRow(Expr*,int); +void sqlite3AggInfoPersistWalkerInit(Walker*,Parse*); void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx); diff --git a/src/window.c b/src/window.c index 77ec8a02e0..8046e46274 100644 --- a/src/window.c +++ b/src/window.c @@ -956,12 +956,16 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ Window *pMWin = p->pWin; /* Master window object */ Window *pWin; /* Window object iterator */ Table *pTab; + Walker w; + u32 selFlags = p->selFlags; pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ){ return sqlite3ErrorToParser(db, SQLITE_NOMEM); } + sqlite3AggInfoPersistWalkerInit(&w, pParse); + sqlite3WalkSelect(&w, p); p->pSrc = 0; p->pWhere = 0; @@ -1042,7 +1046,6 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ p->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); if( p->pSrc ){ Table *pTab2; - Walker w; p->pSrc->a[0].pSelect = pSub; sqlite3SrcListAssignCursors(pParse, p->pSrc); pSub->selFlags |= SF_Expanded; diff --git a/test/fuzzdata8.db b/test/fuzzdata8.db index 350ffa2631..b183746029 100644 Binary files a/test/fuzzdata8.db and b/test/fuzzdata8.db differ diff --git a/test/window1.test b/test/window1.test index 677297fc89..3336a613ab 100644 --- a/test/window1.test +++ b/test/window1.test @@ -1743,5 +1743,150 @@ do_execsql_test 53.0 { WHERE a.c); } {4 4 4 4} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 54.1 { + CREATE TABLE t1(a VARCHAR(20), b FLOAT); + INSERT INTO t1 VALUES('1',10.0); +} + +do_catchsql_test 54.2 { + SELECT * FROM ( + SELECT sum(b) OVER() AS c FROM t1 + UNION + SELECT b AS c FROM t1 + ) WHERE c>10; +} {1 {misuse of window function sum()}} + +do_execsql_test 54.3 { + INSERT INTO t1 VALUES('2',5.0); + INSERT INTO t1 VALUES('3',15.0); +} + +do_catchsql_test 54.4 { + SELECT * FROM ( + SELECT sum(b) OVER() AS c FROM t1 + UNION + SELECT b AS c FROM t1 + ) WHERE c>10; +} {1 {misuse of window function sum()}} + +# 2020-06-05 ticket c8d3b9f0a750a529 +reset_db +do_execsql_test 55.1 { + CREATE TABLE a(b); + SELECT + (SELECT b FROM a + GROUP BY b + HAVING (SELECT COUNT()OVER() + lead(b)OVER(ORDER BY SUM(DISTINCT b) + b)) + ) + FROM a + UNION + SELECT 99 + ORDER BY 1; +} {99} + +#------------------------------------------------------------------------ +reset_db +do_execsql_test 56.1 { + CREATE TABLE t1(a, b INTEGER); + CREATE TABLE t2(c, d); +} +do_catchsql_test 56.2 { + SELECT avg(b) FROM t1 + UNION ALL + SELECT min(c) OVER () FROM t2 + ORDER BY nosuchcolumn; +} {1 {1st ORDER BY term does not match any column in the result set}} + +reset_db +do_execsql_test 57.1 { + CREATE TABLE t4(a, b, c, d, e); +} + +do_catchsql_test 57.2 { + SELECT b FROM t4 + UNION + SELECT a FROM t4 + ORDER BY ( + SELECT sum(x) OVER() FROM ( + SELECT c AS x FROM t4 + UNION + SELECT d FROM t4 + ORDER BY (SELECT e FROM t4) + ) + ); +} {1 {1st ORDER BY term does not match any column in the result set}} + +# 2020-06-06 various dbsqlfuzz finds and +# ticket 0899cf62f597d7e7 +# +reset_db +do_execsql_test 57.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(NULL,NULL,NULL); + SELECT + sum(a), + min(b) OVER (), + count(c) OVER (ORDER BY b) + FROM t1; +} {{} {} 0} +do_execsql_test 57.2 { + CREATE TABLE v0 ( v1 INTEGER PRIMARY KEY ) ; + INSERT INTO v0 VALUES ( 10 ) ; + SELECT DISTINCT v1, lead(v1) OVER() FROM v0 GROUP BY v1 ORDER BY 2; +} {10 {}} +do_catchsql_test 57.3 { + DROP TABLE t1; + CREATE TABLE t1(a); + INSERT INTO t1(a) VALUES(22); + CREATE TABLE t3(y); + INSERT INTO t3(y) VALUES(5),(11),(-9); + SELECT ( + SELECT max(y) OVER( ORDER BY (SELECT x FROM (SELECT sum(y) AS x FROM t1))) + ) + FROM t3; +} {1 {misuse of aggregate: sum()}} + +# 2020-06-06 ticket 1f6f353b684fc708 +reset_db +do_execsql_test 58.1 { + CREATE TABLE a(a, b, c); + INSERT INTO a VALUES(1, 2, 3); + INSERT INTO a VALUES(4, 5, 6); + SELECT sum(345+b) OVER (ORDER BY b), + sum(avg(678)) OVER (ORDER BY c) FROM a; +} {347 678.0} + +# 2020-06-06 ticket e5504e987e419fb0 +do_catchsql_test 59.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES (123); + SELECT + ntile( (SELECT sum(x)) ) OVER(ORDER BY x), + min(x) OVER(ORDER BY x) + FROM t1; +} {1 {misuse of aggregate: sum()}} + +# 2020-06-07 ticket f7d890858f361402 +do_execsql_test 60.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1 (x INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES (99); + SELECT EXISTS(SELECT count(*) OVER() FROM t1 ORDER BY sum(x) OVER()); +} {1} + +# 2020-06-07 test case generated by dbsqlfuzz showing how an AggInfo +# object might be referenced after the sqlite3Select() call that created +# it returns. This proves the need to persist all AggInfo objects until +# the Parse object is destroyed. +# +reset_db +do_execsql_test 61.1 { +CREATE TABLE t1(a); +INSERT INTO t1 VALUES(5),(NULL),('seventeen'); +SELECT (SELECT max(x)OVER(ORDER BY x) % min(x)OVER(ORDER BY CASE x WHEN 889 THEN x WHEN x THEN x END)) FROM (SELECT (SELECT sum(CAST(a IN(SELECT (SELECT max(x)OVER(ORDER BY CASE x WHEN 889 THEN 299 WHEN 863 THEN 863 END)) FROM (SELECT (SELECT sum(CAST((SELECT (SELECT max(x)OVER(ORDER BY x) / min(x)OVER(ORDER BY CASE x WHEN 889 THEN 299 WHEN -true THEN 863 END)) FROM (SELECT (SELECT sum(CAST(a IN(SELECT (SELECT max(x) & sum ( a )OVER(ORDER BY CASE x WHEN -8 THEN 299 WHEN 863 THEN 863 END)) FROM (SELECT (SELECT sum(CAST(a AS )) FROM t1) AS x FROM t1)) AS t1 )) FROM t1) AS x FROM t1)) AS x )) FROM t1) AS x FROM t1)) AS real)) FROM t1) AS x FROM t1); +} {{} {} {}} finish_test