This experimental branch attempts to use columns for an index-on-expression

in place of the expression that is being indexed.  This particular check-in
mostly works, but there are still issues.

FossilOrigin-Name: 2e8d4fd4cfd9e82f33c707ba246fe2bb3ca01762cf5ac5905058fbc7adf0abe7

diff --git a/manifest b/manifest
index 13be78b77683c9877379d5d5ac87b03b9de2e067..f752561d83c297567573fcc927d07c7e9699ad33 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Move\sthe\srest\sof\stesting1.js\sinto\stester1.js\sand\seliminate\sthe\sdependency\son\sjaccwabyt_test.c.\sExtend\sthe\slist\sof\sdefault\sconfig-related\s#defines\sin\ssqlite3-wasm.c\sand\sreorganize\sthem\sfor\smaintainability.
-D 2022-10-13T16:48:35.302
+C This\sexperimental\sbranch\sattempts\sto\suse\scolumns\sfor\san\sindex-on-expression\nin\splace\sof\sthe\sexpression\sthat\sis\sbeing\sindexed.\s\sThis\sparticular\scheck-in\nmostly\sworks,\sbut\sthere\sare\sstill\sissues.
+D 2022-10-13T21:08:34.457
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -550,7 +550,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2
 F src/alter.c 0390ca1d69ec3626cfa9f153114b7ab233e6b2bada6a9eb91361ed385fe90deb
-F src/analyze.c aabdf3769c7fd9954a8ec508eb7041ae174b66f88d12c47199fabbea9a646467
+F src/analyze.c d2fce73f6a024897593012c6ca25368629fa4aeb49960d88a52fac664582e483
 F src/attach.c 4431f82f0247bf3aaf91589acafdff77d1882235c95407b36da1585c765fbbc8
 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf
 F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7
@@ -559,7 +559,7 @@ F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca
 F src/btree.c 74fc5f6a0577df703d6f98d0c51ee0d8d91d22dbc0ba86e42e056517e2b45576
 F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22
 F src/btreeInt.h 8ce1332edd89dfd2461d561ac10a0ab5601c8e06200cb5230596c3caaf54482e
-F src/build.c 6e3ee380a4f0ff95de4b53bf510f66600cff08e4e04b92e95fae789072563f8f
+F src/build.c e9b3adf21cdae3c9ca9450ee8463f2094f133c23734b81fd371f23b3d5d345ca
 F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 428b47a2548c7c2c2f885189b42fdcc178fb03c276a2d58b3008b10da29373d5
@@ -567,7 +567,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5
 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec
 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d
 F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e
-F src/expr.c d199850a925665df7f8ac29fa938909f65a2bd32354aa206aa8c25fc74ff3d3d
+F src/expr.c b60036e4bb9080c741a970df3b3cfffc6292aadec3670b7b04bfae6f662ff245
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002
 F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1
@@ -576,7 +576,7 @@ F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19
 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
-F src/insert.c aea5361767817f917b0f0f647a1f0b1621bd858938ae6ae545c3b6b9814b798f
+F src/insert.c b972b1642cd0f7d5ae4964d713b65158a7bfa6a6e8786a9e8e57bd9c0b4a896d
 F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 853385cc7a604157e137585097949252d5d0c731768e16b044608e5c95c3614b
@@ -622,7 +622,7 @@ F src/shell.c.in 2915eaf22bda89ad6533851a051de4773c249185360fe1fc7b4477b8f9063b2
 F src/sqlite.h.in d9c8a6243fc0a1c270d69db33758e34b810af3462f9bc5b4af113b347e07c69d
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 5336beea1868d99d2f62e628dbea55e97267dbff8193291ab175e960c5df9141
-F src/sqliteInt.h d15694b228fe5082587e13ac14f3e449621a5fe6d5933f5dee6e3c9491260da3
+F src/sqliteInt.h a856afa8ff20886d1b0c668b9e3c62daf7875989a1db79e60f121778d48015e1
 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -685,7 +685,7 @@ F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082
 F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298
 F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5
 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa
-F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937
+F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
 F src/util.c 0be191521ff6d2805995f4910f0b6231b42843678b2efdc1abecaf39929a673f
 F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8
@@ -704,10 +704,10 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d
 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
 F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 63e712bcad47f70e94c2150976cd7da5040933699e3938d4189d064acbe40891
+F src/where.c 5cd2a96f99841e0ba3e5564d53b5ad397d3e8a42ce6e4694cd332da4e2c527f1
 F src/whereInt.h 70cd30de9ed784aa33fa6bd1245f060617de7a00d992469b6d8e419eed915743
 F src/wherecode.c bb88be457df3a6a01c844074ab79a9ba74279e73185f1362383aa697e3cae5dc
-F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46
+F src/whereexpr.c bf8c155212c886621d71c951053660de6fcc4ee907b17aa02da0a96a39aa9405
 F src/window.c 928e215840e2f2d9a2746e018c9643ef42c66c4ab6630ef0df7fa388fa145e86
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627
@@ -2031,8 +2031,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P cb94350185f555c333b628ee846c47bcc9df5f76bb82de569b8322f30dbbe1bc
-R 32dab828066657d904ede486bb592456
-U stephan
-Z 3fd1b23c49b42c2df54d60ffd9b89d7f
+P 4e2a8aff2dd4b6e148f45184e2523ebe47815257eca97fa3d32bcbf9625f0def
+R 6b28e91c7f03fb2da80daeebeb46b73a
+T *branch * index-expr-opt
+T *sym-index-expr-opt *
+T -sym-trunk *
+U drh
+Z 4c1c25db22a8599a4330b36a4516d3d7
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index 7e72bfdd21dd89cfafbfbf30faefbf481bd7aaf8..c689c214d4231f3a14f6df35d174070ac60cf314 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4e2a8aff2dd4b6e148f45184e2523ebe47815257eca97fa3d32bcbf9625f0def
\ No newline at end of file
+2e8d4fd4cfd9e82f33c707ba246fe2bb3ca01762cf5ac5905058fbc7adf0abe7
\ No newline at end of file

diff --git a/src/analyze.c b/src/analyze.c
index 39009899ab7d757f76c9c572eb3234327490ee81..8562b9d7f1a038dbaaa3919f37c7739a4e9b1463 100644 (file)
--- a/src/analyze.c
+++ b/src/analyze.c
@@ -953,6 +953,7 @@ static void analyzeVdbeCommentIndexWithColumnName(
   if( NEVER(i==XN_ROWID) ){
     VdbeComment((v,"%s.rowid",pIdx->zName));
   }else if( i==XN_EXPR ){
+    assert( pIdx->bHasExpr );
     VdbeComment((v,"%s.expr(%d)",pIdx->zName, k));
   }else{
     VdbeComment((v,"%s.%s", pIdx->zName, pIdx->pTable->aCol[i].zCnName));

diff --git a/src/build.c b/src/build.c
index be6b01eb0884a35661143c4993f4fef7c6dc8309..502826bfc89a267354dfbca00dc64ff3119ade59 100644 (file)
--- a/src/build.c
+++ b/src/build.c
@@ -4192,6 +4192,7 @@ void sqlite3CreateIndex(
       j = XN_EXPR;
       pIndex->aiColumn[i] = XN_EXPR;
       pIndex->uniqNotNull = 0;
+      pIndex->bHasExpr = 1;
     }else{
       j = pCExpr->iColumn;
       assert( j<=0x7fff );

diff --git a/src/expr.c b/src/expr.c
index bceb5efb084d7c67c9a5e9841271a91459fd0cf0..3630b47aa853c458ed2f02cc1e208a79ba173e58 100644 (file)
--- a/src/expr.c
+++ b/src/expr.c
@@ -4043,6 +4043,26 @@ static int exprCodeInlineFunction(
   return target;
 }
 
+/*
+** Check to see if pExpr is one of the indexed expression on pParse->pIdxExpr.
+** If it is, then resolve the expression by reading from the index and
+** return the register into which the value has been read.  If there is
+** no match, return negative.
+*/
+static SQLITE_NOINLINE int sqlite3ExprIndexLookup(
+  Parse *pParse,   /* The parsing context */
+  Expr *pExpr,     /* The expression to potentially bypass */
+  int target       /* Where to store the result of the expression */
+){
+  IndexExpr *p;
+  for(p=pParse->pIdxExpr; p; p=p->pIENext){
+    if( sqlite3ExprCompare(0, pExpr, p->pExpr, p->iDataCur)!=0 ) continue;
+    sqlite3VdbeAddOp3(pParse->pVdbe, OP_Column, p->iIdxCur, p->iIdxCol, target);
+    return target;
+  }
+  return -1;  /* Not found */
+}
+
 
 /*
 ** Generate code into the current Vdbe to evaluate the given
@@ -4068,6 +4088,14 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
   assert( target>0 && target<=pParse->nMem );
   assert( v!=0 );
 
+  if( pParse->pIdxExpr!=0 
+   && pExpr!=0
+   && !ExprHasProperty(pExpr, EP_Leaf)
+   && (r1 = sqlite3ExprIndexLookup(pParse, pExpr, target))>=0
+  ){
+    return r1;
+  }
+
 expr_code_doover:
   if( pExpr==0 ){
     op = TK_NULL;

diff --git a/src/insert.c b/src/insert.c
index 6c71391a1b8646532676a9fccfd2500fc5c19aaa..3e0d696d74af304469a12e2fbc839593a9ce22ee 100644 (file)
--- a/src/insert.c
+++ b/src/insert.c
@@ -96,6 +96,7 @@ const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){
         aff = SQLITE_AFF_INTEGER;
       }else{
         assert( x==XN_EXPR );
+        assert( pIdx->bHasExpr );
         assert( pIdx->aColExpr!=0 );
         aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr);
       }

diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 6f1109f2f600373811c416a42d1674f449b0054d..caf7610adb6287efa07f75cd89b3225a9b88a1e5 100644 (file)
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1189,6 +1189,7 @@ typedef struct FuncDef FuncDef;
 typedef struct FuncDefHash FuncDefHash;
 typedef struct IdList IdList;
 typedef struct Index Index;
+typedef struct IndexExpr IndexExpr;
 typedef struct IndexSample IndexSample;
 typedef struct KeyClass KeyClass;
 typedef struct KeyInfo KeyInfo;
@@ -2616,6 +2617,7 @@ struct Index {
   unsigned bNoQuery:1;     /* Do not use this index to optimize queries */
   unsigned bAscKeyBug:1;   /* True if the bba7b69f9849b5bf bug applies */
   unsigned bHasVCol:1;     /* Index references one or more VIRTUAL columns */
+  unsigned bHasExpr:1;     /* This is an index on an expression */
 #ifdef SQLITE_ENABLE_STAT4
   int nSample;             /* Number of elements in aSample[] */
   int nSampleCol;          /* Size of IndexSample.anEq[] and so on */
@@ -3563,6 +3565,23 @@ struct TriggerPrg {
 # define DbMaskNonZero(M)   (M)!=0
 #endif
 
+/*
+** If there is an index on an expression in scope such that the value
+** of the expression pExpr can be read out of the index with cursor iCur
+** at column iCol, then an instance of this object records that fact.
+**
+** A linked list of these objects is attached to Parse and records all
+** expressions that can be short-circuited by extracting a valid from
+** an index.
+*/
+struct IndexExpr {
+  Expr *pExpr;            /* The expression contained in the index */
+  int iDataCur;           /* The data cursor associated with the index */
+  int iIdxCur;            /* The index cursor */
+  int iIdxCol;            /* The column of the index that contains pExpr */
+  IndexExpr *pIENext;     /* Next in a list of all indexed expressions */
+};
+
 /*
 ** An instance of the ParseCleanup object specifies an operation that
 ** should be performed after parsing to deallocation resources obtained
@@ -3621,6 +3640,7 @@ struct Parse {
   int nLabelAlloc;     /* Number of slots in aLabel */
   int *aLabel;         /* Space to hold the labels */
   ExprList *pConstExpr;/* Constant expressions */
+  IndexExpr *pIdxExpr; /* List of all expression in active indexes */
   Token constraintName;/* Name of the constraint currently being parsed */
   yDbMask writeMask;   /* Start a write transaction on these databases */
   yDbMask cookieMask;  /* Bitmask of schema verified databases */

diff --git a/src/upsert.c b/src/upsert.c
index fb6c7c0c0702f76ca540f76a71d3d2e19f30e45c..85994020cf17a6ea15f21ea59dc8d3cb61bafe13 100644 (file)
--- a/src/upsert.c
+++ b/src/upsert.c
@@ -166,6 +166,7 @@ int sqlite3UpsertAnalyzeTarget(
         if( pIdx->aiColumn[ii]==XN_EXPR ){
           assert( pIdx->aColExpr!=0 );
           assert( pIdx->aColExpr->nExpr>ii );
+          assert( pIdx->bHasExpr );
           pExpr = pIdx->aColExpr->a[ii].pExpr;
           if( pExpr->op!=TK_COLLATE ){
             sCol[0].pLeft = pExpr;

diff --git a/src/where.c b/src/where.c
index b0c0ea7d4e05faf4cc31ab8841a4e6204175d0e5..220ea5139d12d9f604f4e1c8f317ed43b5622b18 100644 (file)
--- a/src/where.c
+++ b/src/where.c
@@ -5383,6 +5383,55 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
   }
 }
 
+/*
+** This is an sqlite3ParserAddCleanup() callback that is invoked to
+** free the Parse->pIdxExpr list when the Parse object is destroyed.
+*/
+static void whereIndexExprCleanup(sqlite3 *db, void *pObject){
+  Parse *pParse = (Parse*)pObject;
+  while( pParse->pIdxExpr!=0 ){
+    IndexExpr *p = pParse->pIdxExpr;
+    pParse->pIdxExpr = p->pIENext;
+    sqlite3ExprDelete(db, p->pExpr);
+    sqlite3DbFreeNN(db, p);
+  }
+}
+
+/*
+** The index pIdx is used by a query and contains one or more expressions.
+** In other words pIdx is an index on an expression.  iIdxCur is the cursor
+** number for the index and iDataCur is the cursor number for the corresponding
+** table.
+**
+** This routine adds IndexExpr entries to the Parse->pIdxExpr field for
+** each of the expressions in the index so that the expression code generator
+** will know to replace occurrences of the indexed expression with
+** references to the corresponding column of the index.
+*/
+static SQLITE_NOINLINE void whereAddIndexExpr(
+  Parse *pParse,    /* Add IndexExpr entries to pParse->pIdxExpr */
+  Index *pIdx,      /* The index-on-expression that contains the expressions */
+  int iIdxCur,      /* Cursor number for pIdx */
+  int iDataCur      /* Cursor for the corresponding table */
+){
+  int i;
+  IndexExpr *p;
+  for(i=0; i<pIdx->nColumn; i++){
+    if( pIdx->aiColumn[i]!=XN_EXPR ) continue;
+    p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexExpr));
+    if( p==0 ) break;
+    p->pIENext = pParse->pIdxExpr;
+    p->pExpr = sqlite3ExprDup(pParse->db, pIdx->aColExpr->a[i].pExpr, 0);
+    p->iDataCur = iDataCur;
+    p->iIdxCur = iIdxCur;
+    p->iIdxCol = i;
+    pParse->pIdxExpr = p;
+    if( p->pIENext==0 ){
+      sqlite3ParserAddCleanup(pParse, whereIndexExprCleanup, pParse);
+    }
+  }
+}
+
 /*
 ** Generate the beginning of the loop used for WHERE clause processing.
 ** The return value is a pointer to an opaque structure that contains
@@ -5928,6 +5977,9 @@ WhereInfo *sqlite3WhereBegin(
         op = OP_ReopenIdx;
       }else{
         iIndexCur = pParse->nTab++;
+        if( pIx->bHasExpr ){
+          whereAddIndexExpr(pParse, pIx, iIndexCur, pTabItem->iCursor);
+        }
       }
       pLevel->iIdxCur = iIndexCur;
       assert( pIx!=0 );

diff --git a/src/whereexpr.c b/src/whereexpr.c
index 8c9233ebd3c98f0e6bf7b714dd5409a41c50f6ac..20e569642ac4d247f5359fbd7b98b6411696582e 100644 (file)
--- a/src/whereexpr.c
+++ b/src/whereexpr.c
@@ -989,6 +989,7 @@ static SQLITE_NOINLINE int exprMightBeIndexed2(
     if( pIdx->aColExpr==0 ) continue;
     for(i=0; i<pIdx->nKeyCol; i++){
       if( pIdx->aiColumn[i]!=XN_EXPR ) continue;
+      assert( pIdx->bHasExpr );
       if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){
         aiCurCol[0] = iCur;
         aiCurCol[1] = XN_EXPR;