From b33487b0b6df3b669ba76d386b3a42c75fd904af Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Mar 2019 17:12:32 +0000 Subject: [PATCH] Improvements to the way built-in window functions are handled. FossilOrigin-Name: e8eee566dfca6f4c8af074731dfe91f7fbcd9ca72f0303235b52e4e2e80d5b71 --- manifest | 17 +++---- manifest.uuid | 2 +- src/sqliteInt.h | 1 + src/window.c | 125 +++++++++++++++++++++++++++++++---------------- test/window4.tcl | 2 +- 5 files changed, 95 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index 4ba1418224..1aba9518db 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extend\swindowCodeStep()\sto\shandle\sany\sROWS\sPRECEDING/FOLLOWING\sframe\sspecification. -D 2019-03-05T19:29:36.660 +C Improvements\sto\sthe\sway\sbuilt-in\swindow\sfunctions\sare\shandled. +D 2019-03-06T17:12:32.566 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 1ad7263f38329c0ecea543c80f30af839ee714ea77fc391bf1a3fbb919a5b6b5 @@ -520,7 +520,7 @@ F src/shell.c.in 249c0bf34f7ce272cb17162c297c45ab674a52a5d85193a86191f131196de47 F src/sqlite.h.in f19f7b7646ccd331511b123e2e23d4dc3f3d02f74e1c04d2bb560ea50a323e4c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h 641932dfd799f3dcb3e038d635a6415f62e0b3f2a5668288f3b69062d9afeab3 +F src/sqliteInt.h 24a4818574031facb6ec4fc5fc3b20b8f68c9ec4c85b662a41d1dc4a2ba0524b F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -604,7 +604,7 @@ F src/where.c 8a207cb2ca6b99e1edb1e4bbff9b0504385a759cbf66180d1deb34d80ca4b799 F src/whereInt.h 5f14db426ca46a83eabab1ae9aa6d4b8f27504ad35b64c290916289b1ddb2e88 F src/wherecode.c ce7b21e1be2b981d62683fc59c4ca73a04a7ff2f1ebec23d41baf2da2349afd6 F src/whereexpr.c 36b47f7261d6b6f1a72d774c113b74beddf6745aba1018e64b196e29db233442 -F src/window.c d722f9b0cb3f12ee3fab8cc2d05407078d99346969e9c7e2030da21d0588675f +F src/window.c 607664ea225c3a9c2a7f55795741444bd6a95be37d1681ecf962cd5cb1618dba F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -1681,7 +1681,7 @@ F test/window2.tcl 9bfa842d8a62b0d36dc8c1b5972206393c43847433c6d75940b87fec93ce3 F test/window2.test 8e6d2a1b9f54dfebee1cde961c8590cd87b4db45c50f44947a211e1b63c2a05e F test/window3.tcl 3bf7fc5b86024f47bb5834efc90015c9a64f57d696b82e26a1a4b648b7cd058a F test/window3.test ee56c741e934a0b872fffd7de99f137f9c7788d50885f99f54da4075d951e8ee -F test/window4.tcl 511425f6b0abf9b953df54cc9c7295cc7c25d78f4ed6f7a74b094eec0120eccb +F test/window4.tcl 5fbaab489677914ee5686b2008426e336daf88a2f58be7df92757f780a5ebf91 F test/window4.test c5d6bf3403e4ade2f19df2afe4c16f29fb817c392c6c1c8017edb7165c191a62 F test/window5.test d328dd18221217c49c144181975eea17339eaeaf0e9aa558cee3afb84652821e F test/window6.test 5eae4ae7a590ccf1e605880969ca0bad3955616ac91cad3031baea38748badb3 @@ -1809,8 +1809,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9b4d561f68febdab6624d8c567fb4357bf67dda500664cf708d16a76f37b8ec1 -R f1949de6e35750b066f9dbae4754f91b -T +closed d661659f6934dd7a7ef67d1090e078343fc05e85c3ce3586883921ed32484d13 +P af0ea1363548461b2aad8fd54ee3f2f616111dcae2d6480f5294da44c87a0a5d +R 43bbe25e5bcdc3ad86ad389fa7d11dcc U dan -Z 8f284cc4f98c73f741a81ab913586d8b +Z 3a11921edb0dd283221790b71f6938eb diff --git a/manifest.uuid b/manifest.uuid index 6149dde2d7..d49ce87d1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -af0ea1363548461b2aad8fd54ee3f2f616111dcae2d6480f5294da44c87a0a5d \ No newline at end of file +e8eee566dfca6f4c8af074731dfe91f7fbcd9ca72f0303235b52e4e2e80d5b71 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 52e641ba9f..5c7cac4f25 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3579,6 +3579,7 @@ struct Window { int iArgCol; /* Offset of first argument for this function */ int regFirst; + int regSize; }; #ifndef SQLITE_OMIT_WINDOWFUNC diff --git a/src/window.c b/src/window.c index aebf6f780e..e139bfafcf 100644 --- a/src/window.c +++ b/src/window.c @@ -1092,6 +1092,8 @@ void sqlite3WindowCodeInit(Parse *pParse, Window *pMWin){ pMWin->regFirst = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regFirst); + pMWin->regSize = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, pMWin->regSize); for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ FuncDef *p = pWin->pFunc; @@ -1841,6 +1843,26 @@ static void windowCodeRowExprStep( sqlite3VdbeJumpHere(v, addrGoto); } +/* +** Return true if the entire partition should be cached in the temp +** table before processing. +*/ +static int windowCachePartition(Window *pMWin){ + Window *pWin; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + if( (pFunc->funcFlags & SQLITE_FUNC_WINDOW_SIZE) + || (pFunc->zName==nth_valueName) + || (pFunc->zName==first_valueName) + || (pFunc->zName==leadName) + || (pFunc->zName==lagName) + ){ + return 1; + } + } + return 0; +} + static void windowCodeStep( Parse *pParse, Select *p, @@ -1871,13 +1893,19 @@ static void windowCodeStep( int addrIfStart; int addrGosubFlush; int addrInteger; + int addrCacheRewind; + int addrCacheNext; int addrShortcut = 0; + int bCache = windowCachePartition(pMWin); + int reg = pParse->nMem+1; int regRecord = reg+nSub; int regRowid = regRecord+1; + bCache = 1; + pParse->nMem += 1 + nSub + 1; regFlushPart = ++pParse->nMem; @@ -1903,7 +1931,12 @@ static void windowCodeStep( } sqlite3VdbeAddOp3(v, OP_MakeRecord, reg, nSub, regRecord); - /* Check if the current iteration is the first row of a new partition */ + /* An input row has just been read into an array of registers starting + ** at reg. If the window has a PARTITION clause, this block generates + ** VM code to check if the input row is the start of a new partition. + ** If so, it does an OP_Gosub to an address to be filled in later. The + ** address of the OP_Gosub is stored in local variable addrGosubFlush. + */ if( pMWin->pPartition ){ int addr; ExprList *pPart = pMWin->pPartition; @@ -1914,25 +1947,32 @@ static void windowCodeStep( addrIf = sqlite3VdbeAddOp1(v, OP_If, pMWin->regFirst); addr = sqlite3VdbeAddOp3(v, OP_Compare, regNewPart, pMWin->regPart, nPart); sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); - sqlite3VdbeAddOp3(v, OP_Jump, addr+2, addr+3, addr+2); + sqlite3VdbeAddOp3(v, OP_Jump, addr+2, addr+4, addr+2); VdbeCoverageEqNe(v); addrGosubFlush = sqlite3VdbeAddOp1(v, OP_Gosub, regFlushPart); VdbeComment((v, "call flush_partition")); sqlite3VdbeJumpHere(v, addrIf); + sqlite3VdbeAddOp3(v, OP_Copy, regNewPart, pMWin->regPart, nPart-1); } /* Insert the new row into the ephemeral table */ sqlite3VdbeAddOp2(v, OP_NewRowid, csrWrite, regRowid); sqlite3VdbeAddOp3(v, OP_Insert, csrWrite, regRecord, regRowid); + sqlite3VdbeAddOp2(v, OP_AddImm, pMWin->regSize, 1); - /* This block is run for the first row of each partition */ - addrIf = sqlite3VdbeAddOp1(v, OP_IfNot, pMWin->regFirst); - if( pMWin->pPartition ){ - sqlite3VdbeAddOp3(v, OP_Copy, - reg+pMWin->nBufferCol, pMWin->regPart, pMWin->pPartition->nExpr-1 - ); + if( bCache ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, pMWin->regFirst); + sqlite3WhereEnd(pWInfo); + addrInteger = sqlite3VdbeAddOp2(v, OP_Integer, 0, regFlushPart); + if( pMWin->pPartition ){ + sqlite3VdbeJumpHere(v, addrGosubFlush); + } + addrCacheRewind = sqlite3VdbeAddOp1(v, OP_Rewind, csrWrite); + }else{ + addrIf = sqlite3VdbeAddOp1(v, OP_IfNot, pMWin->regFirst); } + /* This block is run for the first row of each partition */ regArg = windowInitAccum(pParse, pMWin); sqlite3ExprCode(pParse, pMWin->pStart, regStart); @@ -1944,9 +1984,16 @@ static void windowCodeStep( int op = ((pMWin->eStart==TK_FOLLOWING) ? OP_Ge : OP_Le); int addrGe = sqlite3VdbeAddOp3(v, op, regStart, 0, regEnd); windowAggFinal(pParse, pMWin, 0); - sqlite3VdbeAddOp2(v, OP_Rewind, csrCurrent, 1); - windowReturnOneRow(pParse, pMWin, regGosub, addrGosub); - sqlite3VdbeAddOp1(v, OP_ResetSorter, csrCurrent); + if( bCache ){ + sqlite3VdbeAddOp2(v, OP_Rowid, csrWrite, regRowid); + sqlite3VdbeAddOp3(v, OP_NotExists, csrCurrent, 0, regRowid); + windowReturnOneRow(pParse, pMWin, regGosub, addrGosub); + sqlite3VdbeAddOp2(v, OP_Next, csrWrite, addrCacheRewind+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Rewind, csrCurrent, 1); + windowReturnOneRow(pParse, pMWin, regGosub, addrGosub); + sqlite3VdbeAddOp1(v, OP_ResetSorter, csrCurrent); + } addrShortcut = sqlite3VdbeAddOp0(v, OP_Goto); sqlite3VdbeJumpHere(v, addrGe); } @@ -1962,7 +2009,11 @@ static void windowCodeStep( addrGoto = sqlite3VdbeAddOp0(v, OP_Goto); /* This block is run for the second and subsequent rows of each partition */ - sqlite3VdbeJumpHere(v, addrIf); + if( bCache ){ + addrCacheNext = sqlite3VdbeCurrentAddr(v); + }else{ + sqlite3VdbeJumpHere(v, addrIf); + } if( pMWin->eStart==TK_FOLLOWING ){ addrIfEnd = sqlite3VdbeAddOp3(v, OP_IfPos, regEnd, 0, 1); @@ -1973,13 +2024,13 @@ static void windowCodeStep( addrIfStart = sqlite3VdbeAddOp3(v, OP_IfPos, regStart, 0, 1); sqlite3VdbeAddOp2(v, OP_Next, csrStart, sqlite3VdbeCurrentAddr(v)+1); - windowAggStep(pParse, pMWin, csrStart, 1, regArg, 0); + windowAggStep(pParse, pMWin, csrStart, 1, regArg, pMWin->regSize); sqlite3VdbeJumpHere(v, addrIfStart); }else if( pMWin->eEnd==TK_PRECEDING ){ addrIfEnd = sqlite3VdbeAddOp3(v, OP_IfPos, regEnd, 0, 1); sqlite3VdbeAddOp2(v, OP_Next, csrEnd, sqlite3VdbeCurrentAddr(v)+1); - windowAggStep(pParse, pMWin, csrEnd, 0, regArg, 0); + windowAggStep(pParse, pMWin, csrEnd, 0, regArg, pMWin->regSize); sqlite3VdbeJumpHere(v, addrIfEnd); windowAggFinal(pParse, pMWin, 0); @@ -1988,7 +2039,7 @@ static void windowCodeStep( addrIfStart = sqlite3VdbeAddOp3(v, OP_IfPos, regStart, 0, 1); sqlite3VdbeAddOp2(v, OP_Next, csrStart, sqlite3VdbeCurrentAddr(v)+1); - windowAggStep(pParse, pMWin, csrStart, 1, regArg, 0); + windowAggStep(pParse, pMWin, csrStart, 1, regArg, pMWin->regSize); sqlite3VdbeJumpHere(v, addrIfStart); }else{ addrIfEnd = sqlite3VdbeAddOp3(v, OP_IfPos, regEnd, 0, 1); @@ -1997,7 +2048,7 @@ static void windowCodeStep( windowReturnOneRow(pParse, pMWin, regGosub, addrGosub); addrIfStart = sqlite3VdbeAddOp3(v, OP_IfPos, regStart, 0, 1); sqlite3VdbeAddOp2(v, OP_Next, csrStart, sqlite3VdbeCurrentAddr(v)+1); - windowAggStep(pParse, pMWin, csrStart, 1, regArg, 0); + windowAggStep(pParse, pMWin, csrStart, 1, regArg, pMWin->regSize); sqlite3VdbeJumpHere(v, addrIfStart); sqlite3VdbeJumpHere(v, addrIfEnd); } @@ -2005,16 +2056,21 @@ static void windowCodeStep( sqlite3VdbeJumpHere(v, addrGoto); if( pMWin->eEnd!=TK_PRECEDING ){ sqlite3VdbeAddOp2(v, OP_Next, csrEnd, sqlite3VdbeCurrentAddr(v)+1); - windowAggStep(pParse, pMWin, csrEnd, 0, regArg, 0); + windowAggStep(pParse, pMWin, csrEnd, 0, regArg, pMWin->regSize); } /* End of the main input loop */ - if( addrShortcut>0 ) sqlite3VdbeJumpHere(v, addrShortcut); - sqlite3WhereEnd(pWInfo); + if( bCache ){ + sqlite3VdbeAddOp2(v, OP_Next, csrWrite, addrCacheNext); + sqlite3VdbeJumpHere(v, addrCacheRewind); + }else{ + if( addrShortcut>0 ) sqlite3VdbeJumpHere(v, addrShortcut); + sqlite3WhereEnd(pWInfo); + } /* Fall through */ - if( pMWin->pPartition ){ + if( pMWin->pPartition && bCache==0 ){ addrInteger = sqlite3VdbeAddOp2(v, OP_Integer, 0, regFlushPart); sqlite3VdbeJumpHere(v, addrGosubFlush); } @@ -2031,7 +2087,7 @@ static void windowCodeStep( addrIfStart = sqlite3VdbeAddOp3(v, OP_IfPos, regStart, 0, 1); sqlite3VdbeAddOp2(v, OP_Next, csrStart, sqlite3VdbeCurrentAddr(v)+2); sqlite3VdbeAddOp0(v, OP_Goto); - windowAggStep(pParse, pMWin, csrStart, 1, regArg, 0); + windowAggStep(pParse, pMWin, csrStart, 1, regArg, pMWin->regSize); sqlite3VdbeJumpHere(v, addrIfStart); sqlite3VdbeJumpHere(v, addrIfStart+2); @@ -2043,7 +2099,7 @@ static void windowCodeStep( if( pMWin->eEnd==TK_PRECEDING ){ addrIfEnd = sqlite3VdbeAddOp3(v, OP_IfPos, regEnd, 0, 1); sqlite3VdbeAddOp2(v, OP_Next, csrEnd, sqlite3VdbeCurrentAddr(v)+1); - windowAggStep(pParse, pMWin, csrEnd, 0, regArg, 0); + windowAggStep(pParse, pMWin, csrEnd, 0, regArg, pMWin->regSize); sqlite3VdbeJumpHere(v, addrIfEnd); windowAggFinal(pParse, pMWin, 0); windowReturnOneRow(pParse, pMWin, regGosub, addrGosub); @@ -2052,7 +2108,7 @@ static void windowCodeStep( windowReturnOneRow(pParse, pMWin, regGosub, addrGosub); addrIfStart = sqlite3VdbeAddOp3(v, OP_IfPos, regStart, 0, 1); sqlite3VdbeAddOp2(v, OP_Next, csrStart, sqlite3VdbeCurrentAddr(v)+1); - windowAggStep(pParse, pMWin, csrStart, 1, regArg, 0); + windowAggStep(pParse, pMWin, csrStart, 1, regArg, pMWin->regSize); sqlite3VdbeJumpHere(v, addrIfStart); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrGoto-1); } @@ -2060,8 +2116,10 @@ static void windowCodeStep( } + if( bCache && addrShortcut>0 ) sqlite3VdbeJumpHere(v, addrShortcut); sqlite3VdbeAddOp1(v, OP_ResetSorter, csrCurrent); - sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regFirst); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pMWin->regSize); + if( bCache==0 ) sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regFirst); if( pMWin->pPartition ){ sqlite3VdbeChangeP1(v, addrInteger, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp1(v, OP_Return, regFlushPart); @@ -2530,23 +2588,8 @@ void sqlite3WindowCodeStep( if( pMWin->eType==TK_ROWS && (pMWin->eStart!=TK_UNBOUNDED||pMWin->eEnd!=TK_CURRENT||!pMWin->pOrderBy) ){ - Window *pWin; - int bCache = 0; /* True to use CacheStep() */ - for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ - FuncDef *pFunc = pWin->pFunc; - if( (pFunc->funcFlags & SQLITE_FUNC_WINDOW_SIZE) - || (pFunc->zName==nth_valueName) - || (pFunc->zName==first_valueName) - || (pFunc->zName==leadName) - || (pFunc->zName==lagName) - ){ - bCache = 1; - break; - } - } - if( bCache - || (pMWin->eStart!=TK_PRECEDING && pMWin->eStart!=TK_FOLLOWING) - || (pMWin->eEnd!=TK_FOLLOWING && pMWin->eEnd!=TK_PRECEDING) + if( (pMWin->eStart!=TK_PRECEDING && pMWin->eStart!=TK_FOLLOWING) + || (pMWin->eEnd!=TK_FOLLOWING && pMWin->eEnd!=TK_PRECEDING) ){ VdbeModuleComment((pParse->pVdbe, "Begin RowExprStep()")); windowCodeRowExprStep(pParse, p, pWInfo, regGosub, addrGosub); diff --git a/test/window4.tcl b/test/window4.tcl index 33b3a8c0e8..a0229db6b8 100644 --- a/test/window4.tcl +++ b/test/window4.tcl @@ -1,4 +1,4 @@ -# 2018 May 19 +## 2018 May 19 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: -- 2.47.2