From: drh <> Date: Wed, 19 Oct 2022 11:22:21 +0000 (+0000) Subject: If a query uses an index where one or more of the columns of the index is X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f44f3290df58badf4788fc10ba36af96c4610090;p=thirdparty%2Fsqlite.git If a query uses an index where one or more of the columns of the index is an expression and if the corresponding expression is used elsewhere in the query, then strive to read the value of the expression out of the index, rather than recomputing it. This is the "Indexed Expression Optimizations". FossilOrigin-Name: 3da1032878bdc93f69b02926fb7243b31fe6b1a0ee93af68df52b203b0603dad --- diff --git a/manifest b/manifest index 7113b8ffd7..720b8aa23c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\squery\splanner,\sadd\sa\sheuristic\sthat\swill\sreduce\sthe\scost\sof\sa\sfull\ntable\sscan\sfor\sa\smaterialized\sview\sor\ssubquery\sif\sthe\sfull\sscan\sis\sthe\nouter-most\sloop.\s\sThis\sis\sshown\sto\sspeed\sup\ssome\squeries. -D 2022-09-01T10:41:24.845 +C If\sa\squery\suses\san\sindex\swhere\sone\sor\smore\sof\sthe\scolumns\sof\sthe\sindex\sis\nan\sexpression\sand\sif\sthe\scorresponding\sexpression\sis\nused\selsewhere\sin\sthe\squery,\sthen\sstrive\sto\sread\sthe\svalue\sof\sthe\sexpression\nout\sof\sthe\sindex,\srather\sthan\srecomputing\sit.\s\sThis\sis\sthe\n"Indexed\sExpression\sOptimizations". +D 2022-10-19T11:22:21.760 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -462,7 +462,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c 47276c5c150c3d99d4b72e3d296a4c16e38a8531a3e8b3ece59fdf48208950d9 F src/btree.h c11446f07ec0e9dc85af8041cb0855c52f5359c8b2a43e47e02a685282504d89 F src/btreeInt.h 6111c15868b90669f79081039d19e7ea8674013f907710baa3c814dc3f8bfd3f -F src/build.c 04bc5a6b6331a30348e59222ab132ecde7cf5dc04c0915a2182b0609d1ab3df0 +F src/build.c 9756d8b0bfd11b5050290ecc4cc34ef6ea45d7b4e8d430c8aa511ba6b7ee31c8 F src/callback.c 25dda5e1c2334a367b94a64077b1d06b2553369f616261ca6783c48bcb6bda73 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b @@ -470,7 +470,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 135eb3b5e74f9ef74bde5cec2571192c90c86984fa534c88bf4a055076fa19b7 F src/dbstat.c c12833de69cb655751487d2c5a59607e36be1c58ba1f4bd536609909ad47b319 F src/delete.c 2bee826a5e1c2b2018895084850d69a7f60269ae6fa5e8c247e2a4e9faf2ccad -F src/expr.c e100212835d20498780e7c6d2bdb16c677ecc04350fb75db3bf192a86ba48c92 +F src/expr.c dc17ca9523293e15abe61f3d0002556d765b0fc3b985e67c07ec3de9dd36e636 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c bd0138acdc008c1845ccf92f8e73787880562de649471804801c06fed814c765 F src/func.c 2ccf4ae12430b1ae7096be5f0675887e1bd0732828af0ac0f7496339b7c6edee @@ -479,7 +479,7 @@ F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 9d56a9079d523b648774c1784b74b89bd93fac7b365210157482e4319a468f38 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c fc3cf5c371f9a400144e8c2f148ab29cd3f67f7da7eaf47e6a6959f8255fd92c +F src/insert.c 1b71223966e6e35ad23a6b69deba4fa502b2ee3c6511ca99933b1e39db523045 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 22afc33c3a61b4fd80a60a54f1882688371e6bc64685df2696b008fce65a999c F src/main.c 49657b103545c36b743d1384dbd03c26d49c69cba954a4abab320d5c1fe060b7 @@ -513,7 +513,7 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c be64b2f3908a7f97c56c963676eb12f0d6254c95b28cdc1d73a186eff213219d F src/pragma.c af67dedaad8bafe9a5f9adcec32a0da6dd118617dd8220ad1d118f5a6bf83a02 F src/pragma.h a776bb9c915207e9d1117b5754743ddf1bf6a39cc092a4a44e74e6cb5fab1177 -F src/prepare.c f739feb4cf0dfe22b5729c3610c80e2261c816731c7884641552e12aa9f7346d +F src/prepare.c 695a19948348b0202741a650af577c35ef2c8daad271bf818e3d0e4a5881ef3f F src/printf.c 67f79227273a9009d86a017619717c3f554f50b371294526da59faa6014ed2cd F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 @@ -523,7 +523,7 @@ F src/shell.c.in c1986496062f9dba4ed5b70db06b5e0f32e1954cdcfab0b30372c6c18679681 F src/sqlite.h.in 59f5e145b8d7a915ca29c6bf4a1f00e3112c1605c9ac5c627c45060110332ba2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 -F src/sqliteInt.h 3d599ca87796deba6ebc5cab518262661c855ae31aa63eed0504223ec2339d13 +F src/sqliteInt.h d748774587502dcf0e77f54ddde94b3fdff32bdea91f2905b72a290d1a891fb9 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -604,10 +604,10 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 9eccc7ebb532a7b0fd3cabc16cff576b9afa763472272db67d84fb8cec96f5c0 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a F src/walker.c 7607f1a68130c028255d8d56094ea602fc402c79e1e35a46e6282849d90d5fe4 -F src/where.c 742ced4550d1df8d683482b11f2432bb60a93f6f50b532ba9c3a32720fc3b78e +F src/where.c 5f54a44d4aa05c592ed8e19bad656ebb1a9d05157488edfa991e12b3098290f1 F src/whereInt.h ffebbbad9359cc602c9cbb24d926f73fc1bf696f0edb4ff896afa32018aad690 F src/wherecode.c 5e0b6dec8591e13f1f0af828d350e4a5dd2e3518b63d328f21bb38e2456dfeb7 -F src/whereexpr.c 90859652920f153d2c03f075488744be2926625ebd36911bcbcb17d0d29c891c +F src/whereexpr.c ca55a11c2443700fe084a1e039660688d7733c594a37697ee4bd99462e2c2f6a F src/window.c 038c248267e74ff70a2bb9b1884d40fd145c5183b017823ecb6cbb14bc781478 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1819,9 +1819,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ba6bf331476d0217f4132b73cb3da559e75bfb21856ec94f82c0f0150a53592b -Q +609fbb94b8f01d6792e5941ab23ce041313d359f6788c4dde6b1ca749ab49137 -R 7b7f582c65e5dee0ef3f194b93a7fa92 +P e3754cc18824aad4113ad6d81e33e5c763beb9705fc6af88d8f8c870a03c731d +Q +2435112867fbd7b6ebb7f2c2b9da57cdf1e23fab6c2869870b66133a9f9faedc +R ffc6c9dee666f2afa446429523ea6435 U drh -Z e7f00c4ae02183e078488febebe7b6ef +Z cbff34b53dc91f70749ae3ac7b08f467 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3f4b0c049a..722622687f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3754cc18824aad4113ad6d81e33e5c763beb9705fc6af88d8f8c870a03c731d \ No newline at end of file +3da1032878bdc93f69b02926fb7243b31fe6b1a0ee93af68df52b203b0603dad \ No newline at end of file diff --git a/src/build.c b/src/build.c index b4434d080f..0abbcf7fe4 100644 --- a/src/build.c +++ b/src/build.c @@ -3406,6 +3406,7 @@ void sqlite3CreateIndex( j = XN_EXPR; pIndex->aiColumn[i] = XN_EXPR; pIndex->uniqNotNull = 0; + pIndex->bHasExpr = 1; }else{ j = pCExpr->iColumn; assert( j<=0x7fff ); diff --git a/src/expr.c b/src/expr.c index 9a5e0345a0..f092294fdf 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3418,6 +3418,53 @@ static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ return iResult; } +/* +** Check to see if pExpr is one of the indexed expressions on pParse->pIdxExpr. +** If it is, then resolve the expression by reading from the index and +** return the register into which the value has been read. If pExpr is +** not an indexed expression, then return negative. +*/ +static SQLITE_NOINLINE int sqlite3IndexedExprLookup( + Parse *pParse, /* The parsing context */ + Expr *pExpr, /* The expression to potentially bypass */ + int target /* Where to store the result of the expression */ +){ + IndexedExpr *p; + Vdbe *v; + for(p=pParse->pIdxExpr; p; p=p->pIENext){ + int iDataCur = p->iDataCur; + if( iDataCur<0 ) continue; + if( pParse->iSelfTab ){ + if( p->iDataCur!=pParse->iSelfTab-1 ) continue; + iDataCur = -1; + } + if( sqlite3ExprCompare(0, pExpr, p->pExpr, iDataCur)!=0 ) continue; + v = pParse->pVdbe; + assert( v!=0 ); + if( p->bMaybeNullRow ){ + /* If the index is on a NULL row due to an outer join, then we + ** cannot extract the value from the index. The value must be + ** computed using the original expression. */ + int addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_IfNullRow, p->iIdxCur, addr+3, target); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + sqlite3VdbeGoto(v, 0); + p = pParse->pIdxExpr; + pParse->pIdxExpr = 0; + sqlite3ExprCode(pParse, pExpr, target); + pParse->pIdxExpr = p; + sqlite3VdbeJumpHere(v, addr+2); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + } + return target; + } + return -1; /* Not found */ +} + /* ** Generate code into the current Vdbe to evaluate the given @@ -3449,6 +3496,11 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ expr_code_doover: if( pExpr==0 ){ op = TK_NULL; + }else if( pParse->pIdxExpr!=0 + && !ExprHasProperty(pExpr, EP_Leaf) + && (r1 = sqlite3IndexedExprLookup(pParse, pExpr, target))>=0 + ){ + return r1; }else{ op = pExpr->op; } @@ -4814,7 +4866,13 @@ int sqlite3ExprCompare(Parse *pParse, Expr *pA, Expr *pB, int iTab){ if( pB->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA,pB->pLeft,iTab)<2 ){ return 1; } - return 2; + if( pA->op==TK_AGG_COLUMN && pB->op==TK_COLUMN + && pB->iTable<0 && pA->iTable==iTab + ){ + /* fall through */ + }else{ + return 2; + } } if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){ if( pA->op==TK_FUNCTION ){ diff --git a/src/insert.c b/src/insert.c index ee63eeda56..46e2849e23 100644 --- a/src/insert.c +++ b/src/insert.c @@ -95,6 +95,7 @@ const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ }else{ char aff; assert( x==XN_EXPR ); + assert( pIdx->bHasExpr ); assert( pIdx->aColExpr!=0 ); aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); if( aff==0 ) aff = SQLITE_AFF_BLOB; diff --git a/src/prepare.c b/src/prepare.c index 62abc9f6c3..2c34de9b1d 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -524,6 +524,12 @@ void sqlite3ParserReset(Parse *pParse){ sqlite3 *db = pParse->db; sqlite3DbFree(db, pParse->aLabel); sqlite3ExprListDelete(db, pParse->pConstExpr); + while( pParse->pIdxExpr!=0 ){ + IndexedExpr *p = pParse->pIdxExpr; + pParse->pIdxExpr = p->pIENext; + sqlite3ExprDelete(db, p->pExpr); + sqlite3DbFreeNN(db, p); + } if( db ){ assert( db->lookaside.bDisable >= pParse->disableLookaside ); db->lookaside.bDisable -= pParse->disableLookaside; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 04814da670..f274866b76 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1078,6 +1078,7 @@ typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; typedef struct Index Index; +typedef struct IndexedExpr IndexedExpr; typedef struct IndexSample IndexSample; typedef struct KeyClass KeyClass; typedef struct KeyInfo KeyInfo; @@ -2255,6 +2256,8 @@ struct Index { unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ + unsigned bHasExpr:1; /* Index contains an expression, either a literal + ** expression, or a reference to a VIRTUAL column */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ @@ -3048,6 +3051,28 @@ struct TriggerPrg { # define DbMaskNonZero(M) (M)!=0 #endif +/* +** For each index X that has as one of its arguments either an expression +** or the name of a virtual generated column, and if X is in scope such that +** the value of the expression can simply be read from the index, then +** there is an instance of this object on the Parse.pIdxExpr list. +** +** During code generation, while generating code to evaluate expressions, +** this list is consulted and if a matching expression is found, the value +** is read from the index rather than being recomputed. +*/ +struct IndexedExpr { + Expr *pExpr; /* The expression contained in the index */ + int iDataCur; /* The data cursor associated with the index */ + int iIdxCur; /* The index cursor */ + int iIdxCol; /* The index column that contains value of pExpr */ + u8 bMaybeNullRow; /* True if we need an OP_IfNullRow check */ + IndexedExpr *pIENext; /* Next in a list of all indexed expressions */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + const char *zIdxName; /* Name of index, used only for bytecode comments */ +#endif +}; + /* ** An SQL parser context. A copy of this structure is passed through ** the parser and down into all the parser action routine in order to @@ -3091,6 +3116,7 @@ struct Parse { int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ + IndexedExpr *pIdxExpr;/* List of expressions used by active indexes */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ diff --git a/src/where.c b/src/where.c index abe29279be..115f4e25aa 100644 --- a/src/where.c +++ b/src/where.c @@ -4519,6 +4519,50 @@ static int exprIsDeterministic(Expr *p){ return w.eCode; } +/* +** The index pIdx is used by a query and contains one or more expressions. +** In other words pIdx is an index on an expression. iIdxCur is the cursor +** number for the index and iDataCur is the cursor number for the corresponding +** table. +** +** This routine adds IndexedExpr entries to the Parse->pIdxExpr field for +** each of the expressions in the index so that the expression code generator +** will know to replace occurrences of the indexed expression with +** references to the corresponding column of the index. +*/ +static SQLITE_NOINLINE void whereAddIndexedExpr( + Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxExpr */ + Index *pIdx, /* The index-on-expression that contains the expressions */ + int iIdxCur, /* Cursor number for pIdx */ + struct SrcList_item *pTabItem /* The FROM clause entry for the table */ +){ + int i; + IndexedExpr *p; + assert( pIdx->bHasExpr ); + for(i=0; inColumn; i++){ + Expr *pExpr; + int j = pIdx->aiColumn[i]; + if( j==XN_EXPR ){ + pExpr = pIdx->aColExpr->a[i].pExpr; + }else{ + continue; + } + if( sqlite3ExprIsConstant(pExpr) ) continue; + p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); + if( p==0 ) break; + p->pIENext = pParse->pIdxExpr; + p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + p->iDataCur = pTabItem->iCursor; + p->iIdxCur = iIdxCur; + p->iIdxCol = i; + p->bMaybeNullRow = (pTabItem->fg.jointype & JT_LEFT)!=0; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + p->zIdxName = pIdx->zName; +#endif + pParse->pIdxExpr = p; + } +} + /* ** Generate the beginning of the loop used for WHERE clause processing. ** The return value is a pointer to an opaque structure that contains @@ -5067,6 +5111,9 @@ WhereInfo *sqlite3WhereBegin( op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; + if( pIx->bHasExpr ){ + whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem); + } } pLevel->iIdxCur = iIndexCur; assert( pIx->pSchema==pTab->pSchema ); @@ -5363,6 +5410,16 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ }else{ last = pWInfo->iEndWhere; } + if( pIdx->bHasExpr ){ + IndexedExpr *p = pParse->pIdxExpr; + while( p ){ + if( p->iIdxCur==pLevel->iIdxCur ){ + p->iDataCur = -1; + p->iIdxCur = -1; + } + p = p->pIENext; + } + } k = pLevel->addrBody + 1; #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeAddopTrace ){ diff --git a/src/whereexpr.c b/src/whereexpr.c index 961495c584..7f1dbf785d 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -972,6 +972,7 @@ static SQLITE_NOINLINE int exprMightBeIndexed2( if( pIdx->aColExpr==0 ) continue; for(i=0; inKeyCol; i++){ if( pIdx->aiColumn[i]!=XN_EXPR ) continue; + assert( pIdx->bHasExpr ); if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ aiCurCol[0] = iCur; aiCurCol[1] = XN_EXPR;