From: drh <> Date: Thu, 13 Oct 2022 21:08:34 +0000 (+0000) Subject: This experimental branch attempts to use columns for an index-on-expression X-Git-Tag: version-3.40.0~134^2~13 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=4bc1cc18476aaa674f8a24f859e1ed754c365505;p=thirdparty%2Fsqlite.git This experimental branch attempts to use columns for an index-on-expression in place of the expression that is being indexed. This particular check-in mostly works, but there are still issues. FossilOrigin-Name: 2e8d4fd4cfd9e82f33c707ba246fe2bb3ca01762cf5ac5905058fbc7adf0abe7 --- diff --git a/manifest b/manifest index 13be78b776..f752561d83 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\sthe\srest\sof\stesting1.js\sinto\stester1.js\sand\seliminate\sthe\sdependency\son\sjaccwabyt_test.c.\sExtend\sthe\slist\sof\sdefault\sconfig-related\s#defines\sin\ssqlite3-wasm.c\sand\sreorganize\sthem\sfor\smaintainability. -D 2022-10-13T16:48:35.302 +C This\sexperimental\sbranch\sattempts\sto\suse\scolumns\sfor\san\sindex-on-expression\nin\splace\sof\sthe\sexpression\sthat\sis\sbeing\sindexed.\s\sThis\sparticular\scheck-in\nmostly\sworks,\sbut\sthere\sare\sstill\sissues. +D 2022-10-13T21:08:34.457 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -550,7 +550,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c 0390ca1d69ec3626cfa9f153114b7ab233e6b2bada6a9eb91361ed385fe90deb -F src/analyze.c aabdf3769c7fd9954a8ec508eb7041ae174b66f88d12c47199fabbea9a646467 +F src/analyze.c d2fce73f6a024897593012c6ca25368629fa4aeb49960d88a52fac664582e483 F src/attach.c 4431f82f0247bf3aaf91589acafdff77d1882235c95407b36da1585c765fbbc8 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 @@ -559,7 +559,7 @@ F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca F src/btree.c 74fc5f6a0577df703d6f98d0c51ee0d8d91d22dbc0ba86e42e056517e2b45576 F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e -F src/build.c 6e3ee380a4f0ff95de4b53bf510f66600cff08e4e04b92e95fae789072563f8f +F src/build.c e9b3adf21cdae3c9ca9450ee8463f2094f133c23734b81fd371f23b3d5d345ca F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 428b47a2548c7c2c2f885189b42fdcc178fb03c276a2d58b3008b10da29373d5 @@ -567,7 +567,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c d199850a925665df7f8ac29fa938909f65a2bd32354aa206aa8c25fc74ff3d3d +F src/expr.c b60036e4bb9080c741a970df3b3cfffc6292aadec3670b7b04bfae6f662ff245 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -576,7 +576,7 @@ F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c aea5361767817f917b0f0f647a1f0b1621bd858938ae6ae545c3b6b9814b798f +F src/insert.c b972b1642cd0f7d5ae4964d713b65158a7bfa6a6e8786a9e8e57bd9c0b4a896d F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b @@ -622,7 +622,7 @@ F src/shell.c.in 2915eaf22bda89ad6533851a051de4773c249185360fe1fc7b4477b8f9063b2 F src/sqlite.h.in d9c8a6243fc0a1c270d69db33758e34b810af3462f9bc5b4af113b347e07c69d F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 5336beea1868d99d2f62e628dbea55e97267dbff8193291ab175e960c5df9141 -F src/sqliteInt.h d15694b228fe5082587e13ac14f3e449621a5fe6d5933f5dee6e3c9491260da3 +F src/sqliteInt.h a856afa8ff20886d1b0c668b9e3c62daf7875989a1db79e60f121778d48015e1 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -685,7 +685,7 @@ F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082 F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298 F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa -F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 +F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 0be191521ff6d2805995f4910f0b6231b42843678b2efdc1abecaf39929a673f F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 @@ -704,10 +704,10 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 63e712bcad47f70e94c2150976cd7da5040933699e3938d4189d064acbe40891 +F src/where.c 5cd2a96f99841e0ba3e5564d53b5ad397d3e8a42ce6e4694cd332da4e2c527f1 F src/whereInt.h 70cd30de9ed784aa33fa6bd1245f060617de7a00d992469b6d8e419eed915743 F src/wherecode.c bb88be457df3a6a01c844074ab79a9ba74279e73185f1362383aa697e3cae5dc -F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 +F src/whereexpr.c bf8c155212c886621d71c951053660de6fcc4ee907b17aa02da0a96a39aa9405 F src/window.c 928e215840e2f2d9a2746e018c9643ef42c66c4ab6630ef0df7fa388fa145e86 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -2031,8 +2031,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cb94350185f555c333b628ee846c47bcc9df5f76bb82de569b8322f30dbbe1bc -R 32dab828066657d904ede486bb592456 -U stephan -Z 3fd1b23c49b42c2df54d60ffd9b89d7f +P 4e2a8aff2dd4b6e148f45184e2523ebe47815257eca97fa3d32bcbf9625f0def +R 6b28e91c7f03fb2da80daeebeb46b73a +T *branch * index-expr-opt +T *sym-index-expr-opt * +T -sym-trunk * +U drh +Z 4c1c25db22a8599a4330b36a4516d3d7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7e72bfdd21..c689c214d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e2a8aff2dd4b6e148f45184e2523ebe47815257eca97fa3d32bcbf9625f0def \ No newline at end of file +2e8d4fd4cfd9e82f33c707ba246fe2bb3ca01762cf5ac5905058fbc7adf0abe7 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 39009899ab..8562b9d7f1 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -953,6 +953,7 @@ static void analyzeVdbeCommentIndexWithColumnName( if( NEVER(i==XN_ROWID) ){ VdbeComment((v,"%s.rowid",pIdx->zName)); }else if( i==XN_EXPR ){ + assert( pIdx->bHasExpr ); VdbeComment((v,"%s.expr(%d)",pIdx->zName, k)); }else{ VdbeComment((v,"%s.%s", pIdx->zName, pIdx->pTable->aCol[i].zCnName)); diff --git a/src/build.c b/src/build.c index be6b01eb08..502826bfc8 100644 --- a/src/build.c +++ b/src/build.c @@ -4192,6 +4192,7 @@ void sqlite3CreateIndex( j = XN_EXPR; pIndex->aiColumn[i] = XN_EXPR; pIndex->uniqNotNull = 0; + pIndex->bHasExpr = 1; }else{ j = pCExpr->iColumn; assert( j<=0x7fff ); diff --git a/src/expr.c b/src/expr.c index bceb5efb08..3630b47aa8 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4043,6 +4043,26 @@ static int exprCodeInlineFunction( return target; } +/* +** Check to see if pExpr is one of the indexed expression on pParse->pIdxExpr. +** If it is, then resolve the expression by reading from the index and +** return the register into which the value has been read. If there is +** no match, return negative. +*/ +static SQLITE_NOINLINE int sqlite3ExprIndexLookup( + Parse *pParse, /* The parsing context */ + Expr *pExpr, /* The expression to potentially bypass */ + int target /* Where to store the result of the expression */ +){ + IndexExpr *p; + for(p=pParse->pIdxExpr; p; p=p->pIENext){ + if( sqlite3ExprCompare(0, pExpr, p->pExpr, p->iDataCur)!=0 ) continue; + sqlite3VdbeAddOp3(pParse->pVdbe, OP_Column, p->iIdxCur, p->iIdxCol, target); + return target; + } + return -1; /* Not found */ +} + /* ** Generate code into the current Vdbe to evaluate the given @@ -4068,6 +4088,14 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( target>0 && target<=pParse->nMem ); assert( v!=0 ); + if( pParse->pIdxExpr!=0 + && pExpr!=0 + && !ExprHasProperty(pExpr, EP_Leaf) + && (r1 = sqlite3ExprIndexLookup(pParse, pExpr, target))>=0 + ){ + return r1; + } + expr_code_doover: if( pExpr==0 ){ op = TK_NULL; diff --git a/src/insert.c b/src/insert.c index 6c71391a1b..3e0d696d74 100644 --- a/src/insert.c +++ b/src/insert.c @@ -96,6 +96,7 @@ const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ aff = SQLITE_AFF_INTEGER; }else{ assert( x==XN_EXPR ); + assert( pIdx->bHasExpr ); assert( pIdx->aColExpr!=0 ); aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6f1109f2f6..caf7610adb 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1189,6 +1189,7 @@ typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; typedef struct Index Index; +typedef struct IndexExpr IndexExpr; typedef struct IndexSample IndexSample; typedef struct KeyClass KeyClass; typedef struct KeyInfo KeyInfo; @@ -2616,6 +2617,7 @@ struct Index { unsigned bNoQuery:1; /* Do not use this index to optimize queries */ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ + unsigned bHasExpr:1; /* This is an index on an expression */ #ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ @@ -3563,6 +3565,23 @@ struct TriggerPrg { # define DbMaskNonZero(M) (M)!=0 #endif +/* +** If there is an index on an expression in scope such that the value +** of the expression pExpr can be read out of the index with cursor iCur +** at column iCol, then an instance of this object records that fact. +** +** A linked list of these objects is attached to Parse and records all +** expressions that can be short-circuited by extracting a valid from +** an index. +*/ +struct IndexExpr { + Expr *pExpr; /* The expression contained in the index */ + int iDataCur; /* The data cursor associated with the index */ + int iIdxCur; /* The index cursor */ + int iIdxCol; /* The column of the index that contains pExpr */ + IndexExpr *pIENext; /* Next in a list of all indexed expressions */ +}; + /* ** An instance of the ParseCleanup object specifies an operation that ** should be performed after parsing to deallocation resources obtained @@ -3621,6 +3640,7 @@ struct Parse { int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ + IndexExpr *pIdxExpr; /* List of all expression in active indexes */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ diff --git a/src/upsert.c b/src/upsert.c index fb6c7c0c07..85994020cf 100644 --- a/src/upsert.c +++ b/src/upsert.c @@ -166,6 +166,7 @@ int sqlite3UpsertAnalyzeTarget( if( pIdx->aiColumn[ii]==XN_EXPR ){ assert( pIdx->aColExpr!=0 ); assert( pIdx->aColExpr->nExpr>ii ); + assert( pIdx->bHasExpr ); pExpr = pIdx->aColExpr->a[ii].pExpr; if( pExpr->op!=TK_COLLATE ){ sCol[0].pLeft = pExpr; diff --git a/src/where.c b/src/where.c index b0c0ea7d4e..220ea5139d 100644 --- a/src/where.c +++ b/src/where.c @@ -5383,6 +5383,55 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( } } +/* +** This is an sqlite3ParserAddCleanup() callback that is invoked to +** free the Parse->pIdxExpr list when the Parse object is destroyed. +*/ +static void whereIndexExprCleanup(sqlite3 *db, void *pObject){ + Parse *pParse = (Parse*)pObject; + while( pParse->pIdxExpr!=0 ){ + IndexExpr *p = pParse->pIdxExpr; + pParse->pIdxExpr = p->pIENext; + sqlite3ExprDelete(db, p->pExpr); + sqlite3DbFreeNN(db, p); + } +} + +/* +** The index pIdx is used by a query and contains one or more expressions. +** In other words pIdx is an index on an expression. iIdxCur is the cursor +** number for the index and iDataCur is the cursor number for the corresponding +** table. +** +** This routine adds IndexExpr entries to the Parse->pIdxExpr field for +** each of the expressions in the index so that the expression code generator +** will know to replace occurrences of the indexed expression with +** references to the corresponding column of the index. +*/ +static SQLITE_NOINLINE void whereAddIndexExpr( + Parse *pParse, /* Add IndexExpr entries to pParse->pIdxExpr */ + Index *pIdx, /* The index-on-expression that contains the expressions */ + int iIdxCur, /* Cursor number for pIdx */ + int iDataCur /* Cursor for the corresponding table */ +){ + int i; + IndexExpr *p; + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]!=XN_EXPR ) continue; + p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexExpr)); + if( p==0 ) break; + p->pIENext = pParse->pIdxExpr; + p->pExpr = sqlite3ExprDup(pParse->db, pIdx->aColExpr->a[i].pExpr, 0); + p->iDataCur = iDataCur; + p->iIdxCur = iIdxCur; + p->iIdxCol = i; + pParse->pIdxExpr = p; + if( p->pIENext==0 ){ + sqlite3ParserAddCleanup(pParse, whereIndexExprCleanup, pParse); + } + } +} + /* ** Generate the beginning of the loop used for WHERE clause processing. ** The return value is a pointer to an opaque structure that contains @@ -5928,6 +5977,9 @@ WhereInfo *sqlite3WhereBegin( op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; + if( pIx->bHasExpr ){ + whereAddIndexExpr(pParse, pIx, iIndexCur, pTabItem->iCursor); + } } pLevel->iIdxCur = iIndexCur; assert( pIx!=0 ); diff --git a/src/whereexpr.c b/src/whereexpr.c index 8c9233ebd3..20e569642a 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -989,6 +989,7 @@ static SQLITE_NOINLINE int exprMightBeIndexed2( if( pIdx->aColExpr==0 ) continue; for(i=0; inKeyCol; i++){ if( pIdx->aiColumn[i]!=XN_EXPR ) continue; + assert( pIdx->bHasExpr ); if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ aiCurCol[0] = iCur; aiCurCol[1] = XN_EXPR;