From: drh <> Date: Thu, 6 Jun 2024 16:06:21 +0000 (+0000) Subject: Better optimize queries that use parameters in the LIMIT clause. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fheads%2Fbranch-3.28-var-in-limit;p=thirdparty%2Fsqlite.git Better optimize queries that use parameters in the LIMIT clause. FossilOrigin-Name: b44f2676716be37d85eb6f207e187d997fc9d7493883bd7f1541cc7ae52475be --- diff --git a/manifest b/manifest index 0509169ebf..26c7b28ce3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypos\sin\scomments.\s\sProvided\s".wheretrace"\sdebugging\soutput\sfor\sthe\ninterstage\sheuristic\smodule.\s\sDo\somit\sautomatic\sindex\sloops\sin\sthe\ninterstage\sheuristic. -D 2024-04-02T18:48:25.553 +C Better\soptimize\squeries\sthat\suse\sparameters\sin\sthe\sLIMIT\sclause. +D 2024-06-06T16:06:21.901 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -470,7 +470,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 135eb3b5e74f9ef74bde5cec2571192c90c86984fa534c88bf4a055076fa19b7 F src/dbstat.c c12833de69cb655751487d2c5a59607e36be1c58ba1f4bd536609909ad47b319 F src/delete.c fb363f793fc40df36843cc67fb692659164b6109690f4efdd254ab785f8e511d -F src/expr.c 314b08e6502b9bc638fb8a0d686806e4eb4adebd2c728a28617780dcd1bc4c3b +F src/expr.c 4bbaf2ca6d6531bc0f00777f9679ff0b94943707f683f81e226f5125a2b031e6 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c e6408a869edeb222b049f6bb991a862a2c4a161da27794018897f196b1c2f584 F src/func.c 2ccf4ae12430b1ae7096be5f0675887e1bd0732828af0ac0f7496339b7c6edee @@ -516,14 +516,14 @@ F src/pragma.h a776bb9c915207e9d1117b5754743ddf1bf6a39cc092a4a44e74e6cb5fab1177 F src/prepare.c 695a19948348b0202741a650af577c35ef2c8daad271bf818e3d0e4a5881ef3f F src/printf.c 67f79227273a9009d86a017619717c3f554f50b371294526da59faa6014ed2cd F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 0965e1667d16634e24c3c268da4c7e10c6d902c95b40edbb10d4d6c41a80c679 +F src/resolve.c 5153c9fb3fa402cfdef7dc5a50779ce05f40632098b6bfb7d861bff4bfe15dee F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 -F src/select.c 7f5f44a6c6527d9dcf30ee16cc08420aa68e94040635b365ed7128e8ab2e2f1a -F src/shell.c.in c1986496062f9dba4ed5b70db06b5e0f32e1954cdcfab0b30372c6c186796810 +F src/select.c cd4b49d6d8d8898bb26ed1d46537d67978b2e25a40baa928229b382344b62452 +F src/shell.c.in 7081fe81793df713d0c0fc5f82a908b84c7c30c43a8df67d6188bfb373332c91 F src/sqlite.h.in 59f5e145b8d7a915ca29c6bf4a1f00e3112c1605c9ac5c627c45060110332ba2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 -F src/sqliteInt.h 47bb8365d8e78bfb527c151f968a1d09fa410347054dee54a4702019fa2ca2d7 +F src/sqliteInt.h 25a98035df1faf47a9945cfe0a8c0519dd255a1ef2c89d587b006daa727974ce F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -604,11 +604,11 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 9eccc7ebb532a7b0fd3cabc16cff576b9afa763472272db67d84fb8cec96f5c0 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a F src/walker.c 7607f1a68130c028255d8d56094ea602fc402c79e1e35a46e6282849d90d5fe4 -F src/where.c 88042f93c5955c628ee77ff9b1ff675527c92ff5efbc7dbd63c01ae45c2e5069 +F src/where.c 7b276c3110231eebda4d537e32a560030762c2ce7dd44e8e89aa2898af04e263 F src/whereInt.h 064a1508edcc9af400a3b79211ba55e553fab6451b797a31334c0d11cbb7debb F src/wherecode.c cf67460973119c7b2141ad67daf8368dfb4871f225e2489f95effaa139007bfd -F src/whereexpr.c 9813b81c1bb5c9e90c093d3be7b842fec49072ef6b53c454756aa986735d1d36 -F src/window.c d73354234d17bbc8916bf4cf65027854058cbb5d8497e083144522e90bb05c2f +F src/whereexpr.c cb357cf1efd04ae2be4a0d4cf5e0b04c1d666c55f6af72574f593b4a0cf549f6 +F src/window.c bb684e5e2824875dc1c79aa4df313cf7f80fc2a5906d3fbd663a0c93be20cd70 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -1679,7 +1679,8 @@ F test/whereN.test d205fe91148c4bfc50b668ce9f8ebcaafd4ad806447d5bfee8c482d4cd985 F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864 F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f6411db44e3 F test/wherelimit.test 592081800806d297dd7449b1030c863d2883d6d42901837ccd2e5a9bd962edb0 -F test/wherelimit2.test 9bf0aa56cca40ea0e4c5e2915341355a2bbc0859ec4ce1589197fe2a9d94635f +F test/wherelimit2.test a87c0456e9e9927449965313377d116af8acc8193cd5b9d40a913607babf964f +F test/wherelimit3.test 23b24343ceedbe4e722c6055c63d1c65e604781caa4f622a3246619ddc7cb990 F test/widetab1.test ea6e1d8ce3cf3fc9f0b6f5e4264a45d287944ee9becabbe8ad55ba80bb3cc225 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74 @@ -1821,9 +1822,12 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1f97086d628b6f52e6f9a87cd9d2c365f3c51f5b33c5c36a3f83889ec08765be -Q +186dcae19e249db36de15f295999cff25063b54ee3d5d481cd2ba99b6d13148e -R 72348ce53a15ea8deb2d0622361d010b +P 357d9513d2bd13c4e9f227300d43a04d13ae1d34b0b5ec55e8b94eb3204d1921 +Q +e58cb304d1e0ec6e30260a165aaccc2cf096ce6c999efb06683c4ef600ee12ef +R ef49b8a39e4003dd4f26e37870f517f3 +T *branch * branch-3.28-var-in-limit +T *sym-branch-3.28-var-in-limit * +T -sym-branch-3.28 * U drh -Z 298c3dc9319a58678b6aa3cd002b5e96 +Z fe971a71ec90aa07a1a2ecd8a55c9fa5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index aa7ed233c5..116e47f083 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -357d9513d2bd13c4e9f227300d43a04d13ae1d34b0b5ec55e8b94eb3204d1921 \ No newline at end of file +b44f2676716be37d85eb6f207e187d997fc9d7493883bd7f1541cc7ae52475be \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index eb8afb3101..64694ada59 100644 --- a/src/expr.c +++ b/src/expr.c @@ -901,13 +901,13 @@ void sqlite3PExprAddSelect(Parse *pParse, Expr *pExpr, Select *pSelect){ static int exprAlwaysTrue(Expr *p){ int v = 0; if( ExprHasProperty(p, EP_FromJoin) ) return 0; - if( !sqlite3ExprIsInteger(p, &v) ) return 0; + if( !sqlite3ExprIsInteger(p, &v, 0) ) return 0; return v!=0; } static int exprAlwaysFalse(Expr *p){ int v = 0; if( ExprHasProperty(p, EP_FromJoin) ) return 0; - if( !sqlite3ExprIsInteger(p, &v) ) return 0; + if( !sqlite3ExprIsInteger(p, &v, 0) ) return 0; return v==0; } @@ -2085,8 +2085,12 @@ int sqlite3ExprContainsSubquery(Expr *p){ ** to fit in a 32-bit integer, return 1 and put the value of the integer ** in *pValue. If the expression is not an integer or if it is too big ** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged. +** +** If the pParse pointer is provided, then allow the expression p to be +** a parameter (TK_VARIABLE) that is bound to an integer. +** But if pParse is NULL, then p must be a pure integer literal. */ -int sqlite3ExprIsInteger(Expr *p, int *pValue){ +int sqlite3ExprIsInteger(const Expr *p, int *pValue, Parse *pParse){ int rc = 0; if( p==0 ) return 0; /* Can only happen following on OOM */ @@ -2101,18 +2105,38 @@ int sqlite3ExprIsInteger(Expr *p, int *pValue){ } switch( p->op ){ case TK_UPLUS: { - rc = sqlite3ExprIsInteger(p->pLeft, pValue); + rc = sqlite3ExprIsInteger(p->pLeft, pValue, 0); break; } case TK_UMINUS: { int v; - if( sqlite3ExprIsInteger(p->pLeft, &v) ){ + if( sqlite3ExprIsInteger(p->pLeft, &v, 0) ){ assert( v!=(-2147483647-1) ); *pValue = -v; rc = 1; } break; } + case TK_VARIABLE: { + sqlite3_value *pVal; + if( pParse==0 ) break; + if( NEVER(pParse->pVdbe==0) ) break; + if( (pParse->db->flags & SQLITE_EnableQPSG)!=0 ) break; + sqlite3VdbeSetVarmask(pParse->pVdbe, p->iColumn); + pVal = sqlite3VdbeGetBoundValue(pParse->pReprepare, p->iColumn, + SQLITE_AFF_BLOB); + if( pVal ){ + if( sqlite3_value_type(pVal)==SQLITE_INTEGER ){ + sqlite3_int64 vv = sqlite3_value_int64(pVal); + if( vv == (vv & 0x7fffffff) ){ /* non-negative numbers only */ + *pValue = (int)vv; + rc = 1; + } + } + sqlite3ValueFree(pVal); + } + break; + } default: break; } return rc; diff --git a/src/resolve.c b/src/resolve.c index 56240907a2..9217769225 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1049,7 +1049,7 @@ static int resolveOrderByTermToExprList( int rc; /* Return code from subprocedures */ u8 savedSuppErr; /* Saved value of db->suppressErr */ - assert( sqlite3ExprIsInteger(pE, &i)==0 ); + assert( sqlite3ExprIsInteger(pE, &i, 0)==0 ); pEList = pSelect->pEList; /* Resolve all names in the ORDER BY term expression @@ -1145,7 +1145,7 @@ static int resolveCompoundOrderBy( Expr *pE, *pDup; if( pItem->done ) continue; pE = sqlite3ExprSkipCollate(pItem->pExpr); - if( sqlite3ExprIsInteger(pE, &iCol) ){ + if( sqlite3ExprIsInteger(pE, &iCol, 0) ){ if( iCol<=0 || iCol>pEList->nExpr ){ resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); return 1; @@ -1338,7 +1338,7 @@ static int resolveOrderGroupBy( continue; } } - if( sqlite3ExprIsInteger(pE2, &iCol) ){ + if( sqlite3ExprIsInteger(pE2, &iCol, 0) ){ /* The ORDER BY term is an integer constant. Again, set the column ** number so that sqlite3ResolveOrderGroupBy() will convert the ** order-by term to a copy of the result-set expression */ diff --git a/src/select.c b/src/select.c index 5a9000b1b7..867ad66c8e 100644 --- a/src/select.c +++ b/src/select.c @@ -2171,7 +2171,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); assert( v!=0 ); - if( sqlite3ExprIsInteger(pLimit->pLeft, &n) ){ + if( sqlite3ExprIsInteger(pLimit->pLeft, &n, pParse) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ @@ -2624,7 +2624,7 @@ static int multiSelect( p->pPrior = pPrior; p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); if( pPrior->pLimit - && sqlite3ExprIsInteger(pPrior->pLimit->pLeft, &nLimit) + && sqlite3ExprIsInteger(pPrior->pLimit->pLeft, &nLimit, pParse) && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) ){ p->nSelectRow = sqlite3LogEst((u64)nLimit); diff --git a/src/shell.c.in b/src/shell.c.in index d8c57b481e..f53fa9f9fb 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3071,6 +3071,7 @@ static int shell_exec( zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql); rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); if( rc==SQLITE_OK ){ + bind_prepared_stmt(pArg, pExplain); while( sqlite3_step(pExplain)==SQLITE_ROW ){ const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); int iEqpId = sqlite3_column_int(pExplain, 0); @@ -3088,6 +3089,7 @@ static int shell_exec( rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); if( rc==SQLITE_OK ){ pArg->cMode = MODE_Explain; + bind_prepared_stmt(pArg, pExplain); explain_data_prepare(pArg, pExplain); exec_prepared_stmt(pArg, pExplain); explain_data_delete(pArg); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d1305c9373..55071054c5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4093,7 +4093,7 @@ int sqlite3ExprIsTableConstant(Expr*,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS int sqlite3ExprContainsSubquery(Expr*); #endif -int sqlite3ExprIsInteger(Expr*, int*); +int sqlite3ExprIsInteger(const Expr*, int*, Parse*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); int sqlite3IsRowid(const char*); diff --git a/src/where.c b/src/where.c index 179da3d0b4..890fe91761 100644 --- a/src/where.c +++ b/src/where.c @@ -2300,7 +2300,7 @@ static void whereLoopOutputAdjust( if( pTerm->eOperator&(WO_EQ|WO_IS) ){ Expr *pRight = pTerm->pExpr->pRight; testcase( pTerm->pExpr->op==TK_IS ); - if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ + if( sqlite3ExprIsInteger(pRight, &k, 0) && k>=(-1) && k<=1 ){ k = 10; }else{ k = 20; diff --git a/src/whereexpr.c b/src/whereexpr.c index c0250920bd..c8cefa7a63 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1457,6 +1457,7 @@ void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ } } + /* ** Initialize a preallocated WhereClause structure. */ diff --git a/src/window.c b/src/window.c index 5c712b82d7..b1eee39d98 100644 --- a/src/window.c +++ b/src/window.c @@ -875,6 +875,7 @@ static ExprList *exprListAppendList( int nInit = pList ? pList->nExpr : 0; for(i=0; inExpr; i++){ Expr *pDup = sqlite3ExprDup(pParse->db, pAppend->a[i].pExpr, 0); + pList = sqlite3ExprListAppend(pParse, pList, pDup); if( pList ) pList->a[nInit+i].sortOrder = pAppend->a[i].sortOrder; } diff --git a/test/wherelimit2.test b/test/wherelimit2.test index 83c04b14cf..9ac74014d0 100644 --- a/test/wherelimit2.test +++ b/test/wherelimit2.test @@ -296,4 +296,35 @@ do_test 5.5 { } {ax a bx b cx c dx d ex a} +# 2024-06-06 +# Use LIMIT if the argument is a variable. +# +reset_db +do_execsql_test 8.0 { + CREATE TABLE t1(a INT, b INT); + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<1000) + INSERT INTO t1 SELECT n, n FROM c; + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + ANALYZE; +} +do_eqp_test 8.1 { + SELECT * FROM t1 WHERE a>=100 AND b<300 ORDER BY b LIMIT 5; +} { + QUERY PLAN + |--SEARCH t1 USING INDEX t1a (a>? AND a=100 AND b<300 ORDER BY b LIMIT -1; +} { + QUERY PLAN + |--SEARCH t1 USING INDEX t1a (a>? AND a=100 AND a<300 ORDER BY b LIMIT 5; +} { + QUERY PLAN + |--SEARCH TABLE t1 USING INDEX t1a (a>? AND a=100 AND a<300 ORDER BY b LIMIT -1; + } { + QUERY PLAN + `--SCAN TABLE t1 USING INDEX t1b +} +} + +set N [expr 5] +do_eqp_test 1.3 { + SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT $::N; +} { + QUERY PLAN + |--SEARCH TABLE t1 USING INDEX t1a (a>? AND a=100 AND a<300 ORDER BY b LIMIT $::N; + } { + QUERY PLAN + `--SCAN TABLE t1 USING INDEX t1b +} +} + + + + + +finish_test