From: drh Date: Mon, 30 Apr 2018 19:32:49 +0000 (+0000) Subject: Defer loading result column values into registers on an ORDER BY LIMIT until X-Git-Tag: version-3.24.0~69^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fheads%2Ffaster-order-by-limit;p=thirdparty%2Fsqlite.git Defer loading result column values into registers on an ORDER BY LIMIT until we know that the LIMIT does not exclude the current row. FossilOrigin-Name: ce4ef46058f4aaea6623a41255a2e4b69bb24f16a287391df48f6bacdb4c4989 --- diff --git a/manifest b/manifest index 64af0212bc..8204174977 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scases\sadded\sfor\sSQLITE_DBCONFIG_RESET_DATABASE. -D 2018-04-28T19:08:02.897 +C Defer\sloading\sresult\scolumn\svalues\sinto\sregisters\son\san\sORDER\sBY\sLIMIT\suntil\nwe\sknow\sthat\sthe\sLIMIT\sdoes\snot\sexclude\sthe\scurrent\srow. +D 2018-04-30T19:32:49.040 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 5ce9343cba9c189046f1afe6d2bcc1f68079439febc05267b98aec6ecc752439 @@ -493,12 +493,12 @@ F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 6415381a0e9d22c0e7cba33ca4a53f81474190862f5d4838190f5eb5b0b47bc9 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c daf07d8defce3311f9e69f1280a874d78bc1d16c305f6aa689640f7afa02842f +F src/select.c 170834c5b9710676b192fc9b0fe0cf7970250b1a72b396435deb923f3670a58f F src/shell.c.in 54b902ab7d840f60ddfabc13124c85d4980342c88aff7679f2cc25f67c21ade7 F src/sqlite.h.in d669de545f18f2f01362de02e309cd7f15185958c71bac8f53cd5438b46d2bea F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d -F src/sqliteInt.h 8ac0138eae10337b745b03dad0124fd63ae911c0503e795729503e7fc6234d57 +F src/sqliteInt.h 3e354edb3090e31b51593e21ae54c541330987cfd87d69d86adf6e4d186408ca F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1727,7 +1727,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 564ae8297d417ba4b7978e430d41f125007177673163f6ed9adc3a3974f73d24 -R f62951ddf79b7c70897c7cedc3fc5902 +P 08665a9e2e50a0a1e62529884cf65f8090debe89a306a3904b53268729ab5ad5 +R 4667c6ddbdfa5ae61d508da23236434a +T *branch * faster-order-by-limit +T *sym-faster-order-by-limit * +T -sym-trunk * U drh -Z d48406f4893f856e5db8717a8971e33f +Z 8bfebc9ee79b135d90912539998ba9b3 diff --git a/manifest.uuid b/manifest.uuid index c789d5b903..2981900123 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -08665a9e2e50a0a1e62529884cf65f8090debe89a306a3904b53268729ab5ad5 \ No newline at end of file +ce4ef46058f4aaea6623a41255a2e4b69bb24f16a287391df48f6bacdb4c4989 \ No newline at end of file diff --git a/src/select.c b/src/select.c index ac313d0288..52dc2f94d6 100644 --- a/src/select.c +++ b/src/select.c @@ -78,6 +78,7 @@ struct SortCtx { int nKey; /* Number of PK columns for table pTab (>=1) */ } aDefer[4]; #endif + struct RowLoadInfo *pDeferredRowLoad; /* Deferred row loading info or NULL */ }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -536,6 +537,62 @@ static KeyInfo *keyInfoFromExprList( int nExtra /* Add this many extra columns to the end */ ); +/* +** An instance of this object holds information (beyond pParse and pSelect) +** needed to load the next result row that is to be added to the sorter. +*/ +typedef struct RowLoadInfo RowLoadInfo; +struct RowLoadInfo { + int regResult; /* Store results in array of registers here */ + u8 ecelFlags; /* Flag argument to ExprCodeExprList() */ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + ExprList *pExtra; /* Extra columns needed by sorter refs */ + int regExtraResult; /* Where to load the extra columns */ +#endif +}; + +/* +** This routine does the work of loading query data into an array of +** registers so that it can be added to the sorter. +*/ +static void innerLoopLoadRow( + Parse *pParse, /* Statement under construction */ + Select *pSelect, /* The query being coded */ + RowLoadInfo *pInfo /* Info needed to complete the row load */ +){ + sqlite3ExprCodeExprList(pParse, pSelect->pEList, pInfo->regResult, + 0, pInfo->ecelFlags); +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pInfo->pExtra ){ + sqlite3ExprCodeExprList(pParse, pInfo->pExtra, pInfo->regExtraResult, 0, 0); + sqlite3ExprListDelete(pParse->db, pInfo->pExtra); + } +#endif +} + +/* +** Code the OP_MakeRecord instruction that generates the entry to be +** added into the sorter. +** +** Return the register in which the result is stored. +*/ +static int makeSorterRecord( + Parse *pParse, + SortCtx *pSort, + Select *pSelect, + int regBase, + int nBase +){ + int nOBSat = pSort->nOBSat; + Vdbe *v = pParse->pVdbe; + int regOut = ++pParse->nMem; + if( pSort->pDeferredRowLoad ){ + innerLoopLoadRow(pParse, pSelect, pSort->pDeferredRowLoad); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regOut); + return regOut; +} + /* ** Generate code that will push the record in registers regData ** through regData+nData-1 onto the sorter. @@ -546,7 +603,7 @@ static void pushOntoSorter( Select *pSelect, /* The whole SELECT statement */ int regData, /* First register holding data to be sorted */ int regOrigData, /* First register holding data before packing */ - int nData, /* Number of elements in the data array */ + int nData, /* Number of elements in the regData data array */ int nPrefixReg /* No. of reg prior to regData available for use */ ){ Vdbe *v = pParse->pVdbe; /* Stmt under construction */ @@ -554,16 +611,31 @@ static void pushOntoSorter( int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ int regBase; /* Regs for sorter record */ - int regRecord = ++pParse->nMem; /* Assembled sorter record */ + int regRecord = 0; /* Assembled sorter record */ int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ int iLimit; /* LIMIT counter */ + int iSkip = 0; /* End of the sorter insert loop */ assert( bSeq==0 || bSeq==1 ); + + /* Three cases: + ** (1) The data to be sorted has already been packed into a Record + ** by a prior OP_MakeRecord. In this case nData==1 and regData + ** will be completely unrelated to regOrigData. + ** (2) All output columns are included in the sort record. In that + ** case regData==regOrigData. + ** (3) Some output columns are omitted from the sort record due to + ** the SQLITE_ENABLE_SORTER_REFERENCE optimization, or due to the + ** SQLITE_ECEL_OMITREF optimization. In that case, regOrigData==0 + ** to prevent this routine from trying to copy values that might + ** not exist. + */ assert( nData==1 || regData==regOrigData || regOrigData==0 ); + if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); - regBase = regData - nExpr - bSeq; + regBase = regData - nPrefixReg; }else{ regBase = pParse->nMem + 1; pParse->nMem += nBase; @@ -587,7 +659,7 @@ static void pushOntoSorter( int nKey; /* Number of sorting key columns, including OP_Sequence */ KeyInfo *pKI; /* Original KeyInfo on the sorter table */ - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat,regRecord); + regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase); regPrevKey = pParse->nMem+1; pParse->nMem += pSort->nOBSat; nKey = nExpr - pSort->nOBSat + bSeq; @@ -638,17 +710,16 @@ static void pushOntoSorter( ** of the outer loop. */ int iCsr = pSort->iECursor; - int iJmp = sqlite3VdbeCurrentAddr(v)+5+(nOBSat<=0)+pSort->bOrderedInnerLoop; - assert( pSort->bOrderedInnerLoop==0 || pSort->bOrderedInnerLoop==1 ); sqlite3VdbeAddOp2(v, OP_IfNotZero, iLimit, sqlite3VdbeCurrentAddr(v)+4); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Last, iCsr, 0); - sqlite3VdbeAddOp4Int(v, OP_IdxLE, iCsr, iJmp, regBase+nOBSat, nExpr-nOBSat); + iSkip = sqlite3VdbeAddOp4Int(v, OP_IdxLE, + iCsr, 0, regBase+nOBSat, nExpr-nOBSat); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_Delete, iCsr); } - if( nOBSat<=0 ){ - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat,regRecord); + if( regRecord==0 ){ + regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase); } if( pSort->sortFlags & SORTFLAG_UseSorter ){ op = OP_SorterInsert; @@ -657,6 +728,11 @@ static void pushOntoSorter( } sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord, regBase+nOBSat, nBase-nOBSat); + if( iSkip ){ + assert( pSort->bOrderedInnerLoop==0 || pSort->bOrderedInnerLoop==1 ); + sqlite3VdbeChangeP2(v, iSkip, + sqlite3VdbeCurrentAddr(v) + pSort->bOrderedInnerLoop); + } } /* @@ -742,9 +818,6 @@ static void selectExprDefer( Table *pTab = pExpr->pTab; if( pExpr->op==TK_COLUMN && pTab && !IsVirtual(pTab) && (pTab->aCol[pExpr->iColumn].colFlags & COLFLAG_SORTERREF) -#if 0 - && pTab->pSchema && pTab->pSelect==0 && !IsVirtual(pTab) -#endif ){ int j; for(j=0; jiSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ int nPrefixReg = 0; /* Number of extra registers before regResult */ + RowLoadInfo sRowLoadInfo; /* Info for deferred row loading */ /* Usually, regResult is the first cell in an array of memory cells ** containing the current result row. In this case regOrig is set to the @@ -863,7 +937,8 @@ static void selectInnerLoop( /* If the destination is an EXISTS(...) expression, the actual ** values returned by the SELECT are not required. */ - u8 ecelFlags; + u8 ecelFlags; /* "ecel" is an abbreviation of "ExprCodeExprList" */ + ExprList *pEList; if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){ ecelFlags = SQLITE_ECEL_DUP; }else{ @@ -877,6 +952,7 @@ static void selectInnerLoop( ** This allows the p->pEList field to be omitted from the sorted record, ** saving space and CPU cycles. */ ecelFlags |= (SQLITE_ECEL_OMITREF|SQLITE_ECEL_REF); + for(i=pSort->nOBSat; ipOrderBy->nExpr; i++){ int j; if( (j = pSort->pOrderBy->a[i].u.x.iOrderByCol)>0 ){ @@ -897,20 +973,46 @@ static void selectInnerLoop( pParse->nMem += pExtra->nExpr; } #endif - regOrig = 0; + + /* Adjust nResultCol to account for columns that are omitted + ** from the sorter by the optimizations in this branch */ + pEList = p->pEList; + for(i=0; inExpr; i++){ + if( pEList->a[i].u.x.iOrderByCol>0 +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + || pEList->a[i].bSorterRef +#endif + ){ + nResultCol--; + regOrig = 0; + } + } + + testcase( regOrig ); + testcase( eDest==SRT_Set ); + testcase( eDest==SRT_Mem ); + testcase( eDest==SRT_Coroutine ); + testcase( eDest==SRT_Output ); assert( eDest==SRT_Set || eDest==SRT_Mem || eDest==SRT_Coroutine || eDest==SRT_Output ); } - nResultCol = sqlite3ExprCodeExprList(pParse,p->pEList,regResult, - 0,ecelFlags); + sRowLoadInfo.regResult = regResult; + sRowLoadInfo.ecelFlags = ecelFlags; #ifdef SQLITE_ENABLE_SORTER_REFERENCES - if( pExtra ){ - nResultCol += sqlite3ExprCodeExprList( - pParse, pExtra, regResult + nResultCol, 0, 0 - ); - sqlite3ExprListDelete(pParse->db, pExtra); - } + sRowLoadInfo.pExtra = pExtra; + sRowLoadInfo.regExtraResult = regResult + nResultCol; + if( pExtra ) nResultCol += pExtra->nExpr; #endif + if( p->iLimit + && (ecelFlags & SQLITE_ECEL_OMITREF)!=0 + && nPrefixReg>0 + ){ + assert( pSort!=0 ); + assert( hasDistinct==0 ); + pSort->pDeferredRowLoad = &sRowLoadInfo; + }else{ + innerLoopLoadRow(pParse, p, &sRowLoadInfo); + } } /* If the DISTINCT keyword was present on the SELECT statement @@ -1026,7 +1128,8 @@ static void selectInnerLoop( } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1+nPrefixReg,regResult,1,nPrefixReg); + assert( regResult==regOrig ); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, regOrig, 1, nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c700c261b8..8d59de99e7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2759,11 +2759,8 @@ struct Upsert { ** An instance of the following structure contains all information ** needed to generate code for a single SELECT statement. ** -** nLimit is set to -1 if there is no LIMIT clause. nOffset is set to 0. -** If there is a LIMIT clause, the parser sets nLimit to the value of the -** limit and nOffset to the value of the offset (or 0 if there is not -** offset). But later on, nLimit and nOffset become the memory locations -** in the VDBE that record the limit and offset counters. +** See the header comment on the computeLimitRegisters() routine for a +** detailed description of the meaning of the iLimit and iOffset fields. ** ** addrOpenEphm[] entries contain the address of OP_OpenEphemeral opcodes. ** These addresses must be stored so that we can go back and fill in