Improved reuse of subqueries associated with IN operators, especially when

the IN operator is duplicated due to predicate push-down.

FossilOrigin-Name: 2a07caad4ab1bf5f53049e71b9814cc722bcd8549d6db3e8e1fbe9eb39ed5338

diff --git a/manifest b/manifest
index bdddaf982d039b1e745a65da8fe20367fe71d263..17269a6ece614d57025b4718b37ecb51d7a5dd48 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C When\sconstructing\san\sephermeral\stable\sto\suse\sas\sthe\sright-hand\sside\sof\nan\sIN\soperator,\salso\sconstruct\sa\sBloom\sfilter\sto\sspeed\smembership\stesting.
-D 2024-07-05T10:03:29.147
+C Improved\sreuse\sof\ssubqueries\sassociated\swith\sIN\soperators,\sespecially\swhen\nthe\sIN\soperator\sis\sduplicated\sdue\sto\spredicate\spush-down.
+D 2024-07-05T10:08:34.672
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -712,7 +712,7 @@ F src/date.c 126ba2ab10aeb2e7ba6e089b5f07b747c0625b8287f78b60da346eda8d23c875
 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782
 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43
 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500
-F src/expr.c 229c40aa248ad34b6cea594061ed3b758d3782159b58c4e50b077dc295a633de
+F src/expr.c 2b075932eaa0ba7e106cb7b96911f572503ba20bc74096bdf36043d0f4ddb8bd
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00
 F src/func.c a37134215de0da2744d99df69f4742c1e858a5734161b22d5a309b77cced3c6e
@@ -767,7 +767,7 @@ F src/shell.c.in 885dafabb3f16d68bdb4576683afb0e39a1939f50985b162255bf656c470bab
 F src/sqlite.h.in cf45273cd1fc6aa15c1d675981a0ad9dd6924cdd92a1f918eced83b7fbbd0d7d
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
-F src/sqliteInt.h 33ae0a2c7dc13d610a73418f7e0b86d21f019f1f100985c48786c4d564505ddf
+F src/sqliteInt.h 00c1ce0e6fb92cbfc6a350d541b154d61fb2807eee74b08c158210b137a6e1ae
 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728
 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -833,10 +833,10 @@ F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e
 F src/util.c 4d6d7ebfe6772a1b950c97bbb1d1a72ad4874617ec498ab8aa73b7f5a43e44bb
 F src/vacuum.c b1dd6d73869229b6e08bac910ac011dc9da42e3120ec2b7241accc5a752bd419
 F src/vdbe.c e6bda6d22e819ee96ebef62c8219d16ca10aa438f130a727cbd1234c92ead9ea
-F src/vdbe.h c2d78d15112c3fc5ab87f5e8e0b75d2db1c624409de2e858c3d1aafb1650bb4f
+F src/vdbe.h c2549a215898a390de6669cfa32adba56f0d7e17ba5a7f7b14506d6fd5f0c36a
 F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c
 F src/vdbeapi.c 80235ac380e9467fec1cb0883354d841f2a771976e766995f7e0c77f845406df
-F src/vdbeaux.c beebe9d4aba1008bce40a2350fc3d0913372956eeaff0089e7d2c2975ab8158a
+F src/vdbeaux.c e9343202bdbcecccc29e14136085ab5ad3c887fa76d6d62f6364f46b845845de
 F src/vdbeblob.c 13f9287b55b6356b4b1845410382d6bede203ceb29ef69388a4a3d007ffacbe5
 F src/vdbemem.c 831a244831eaa45335f9ae276b50a7a82ee10d8c46c2c72492d4eb8c98d94d89
 F src/vdbesort.c 237840ca1947511fa59bd4e18b9eeae93f2af2468c34d2427b059f896230a547
@@ -1303,7 +1303,7 @@ F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0
 F test/in4.test bb767ec1cfd1730256f0a83219f0acda36bc251b63f8b8bb7d8c7cff17875a4f
 F test/in5.test 4fd79c70dfa0681313e8cdca07f5ff0400bdc0e20f808a5c59eaef1e4b48082a
 F test/in6.test f5f40d6816a8bb7c784424b58a10ac38efb76ab29127a2c17399e0cbeeda0e4b
-F test/in7.test 742b18c284cd9a9cd1347d3a8affeee44b8de11e875e91a1d40498c18ba16441
+F test/in7.test 9256cdb30dc487f2078bb4bb30f43f2c1ff4d277a9c7c9a14bd1c9510c9c8cae
 F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822
 F test/incrblob2.test a494c9e848560039a23974b9119cfc2cf3ad3bd15cc2694ee6367ae537ef8f1f
 F test/incrblob3.test 67621a04b3084113bf38ce03797d70eca012d9d8f948193b8f655df577b0da6f
@@ -1537,7 +1537,7 @@ F test/printf.test 685fec5a0c5af2490ab0632775a301554361d674211d690f5bee0a97b0533
 F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60
 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb
 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc
-F test/pushdown.test 9e655df51bc6559608dcc7af89a36f727eff520b4860261ed15a5f95a5ebbcd8
+F test/pushdown.test 84b525767442b3695d671f9df59dd91cf0ed8fb24cbbcdc55959f0dadeee8b39
 F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca
 F test/quick.test 1681febc928d686362d50057c642f77a02c62e57
 F test/quickcheck.test a4b7e878cd97e46108291c409b0bf8214f29e18fddd68a42bc5c1375ad1fb80a
@@ -2228,12 +2228,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 19d5fd8a483d23262c98524386d39bb7f4fb9ed79343c574abe51b57eb8296ec
-Q +baa83b460c677c210c7fa3f20314d7e05f305aed8a69026bc5fa106a3de4ea38
-R 6eafca7b654b4b0bf1e4f7d6d1720cee
-T *branch * bedrock-3.46
-T *sym-bedrock-3.46 *
-T -sym-bedrock *
+P 557a14a24a2b4bb94fc0f8a59db19e79611ad007c38162cf6b1a1d1cf17dc427
+Q +c9a3498113074bbcd9a8c8d30286fef6c6a49ad2c84b90ec0f5a148389d6245c
+R d655dc7bc18a89f94d2d7ef28fffaf1d
 U drh
-Z 6264fff789758ce1ec23fd9ae08f41fe
+Z d847535b1d03ee946f785f48673ba6f5
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index e47349032f6b5c986c19eff436a3ed7fdf4bed49..b08354500ca553451a1c7ed0e561318186952d52 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-557a14a24a2b4bb94fc0f8a59db19e79611ad007c38162cf6b1a1d1cf17dc427
+2a07caad4ab1bf5f53049e71b9814cc722bcd8549d6db3e8e1fbe9eb39ed5338

diff --git a/src/expr.c b/src/expr.c
index 233142925181d446df1f6cdfda4cb101b8581a52..692100f86983eb4c0883716b936e723a68ed4cc8 100644 (file)
--- a/src/expr.c
+++ b/src/expr.c
@@ -3394,6 +3394,49 @@ void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){
   }
 }
 
+#ifndef SQLITE_OMIT_SUBQUERY
+/*
+** Scan all previously generated bytecode looking for an OP_BeginSubrtn
+** that is compatible with pExpr.  If found, add the y.sub values
+** to pExpr and return true.  If not found, return false.
+*/
+static int findCompatibleInRhsSubrtn(
+  Parse *pParse,          /* Parsing context */
+  Expr *pExpr,            /* IN operator with RHS that we want to reuse */
+  SubrtnSig *pNewSig      /* Signature for the IN operator */
+){
+  VdbeOp *pOp, *pEnd;
+  SubrtnSig *pSig;
+  Vdbe *v;
+
+  if( pNewSig==0 ) return 0;
+  if( pParse->bHasSubrtn==0 ) return 0;
+  assert( pExpr->op==TK_IN );
+  assert( !ExprUseYSub(pExpr) );
+  assert( ExprUseXSelect(pExpr) );
+  assert( pExpr->x.pSelect!=0 );
+  assert( (pExpr->x.pSelect->selFlags & SF_All)==0 );
+  v = pParse->pVdbe;
+  assert( v!=0 );
+  pOp = sqlite3VdbeGetOp(v, 1);
+  pEnd = sqlite3VdbeGetLastOp(v);
+  for(; pOp<pEnd; pOp++){
+    if( pOp->p4type!=P4_SUBRTNSIG ) continue;
+    assert( pOp->opcode==OP_BeginSubrtn );
+    pSig = pOp->p4.pSubrtnSig;
+    assert( pSig!=0 );
+    if( pNewSig->selId!=pSig->selId ) continue;
+    if( strcmp(pNewSig->zAff,pSig->zAff)!=0 ) continue;
+    pExpr->y.sub.iAddr = pSig->iAddr;
+    pExpr->y.sub.regReturn = pSig->regReturn;
+    pExpr->iTable = pSig->iTable;
+    ExprSetProperty(pExpr, EP_Subrtn);
+    return 1;
+  }
+  return 0;
+}
+#endif /* SQLITE_OMIT_SUBQUERY */
+
 #ifndef SQLITE_OMIT_SUBQUERY
 /*
 ** Generate code that will construct an ephemeral table containing all terms
@@ -3443,11 +3486,28 @@ void sqlite3CodeRhsOfIN(
   ** and reuse it many names.
   */
   if( !ExprHasProperty(pExpr, EP_VarSelect) && pParse->iSelfTab==0 ){
-    /* Reuse of the RHS is allowed */
-    /* If this routine has already been coded, but the previous code
-    ** might not have been invoked yet, so invoke it now as a subroutine.
+    /* Reuse of the RHS is allowed
+    **
+    ** Compute a signature for the RHS of the IN operator to facility
+    ** finding and reusing prior instances of the same IN operator.
     */
-    if( ExprHasProperty(pExpr, EP_Subrtn) ){
+    SubrtnSig *pSig = 0;
+    assert( !ExprUseXSelect(pExpr) || pExpr->x.pSelect!=0 );
+    if( ExprUseXSelect(pExpr) && (pExpr->x.pSelect->selFlags & SF_All)==0 ){
+      pSig = sqlite3DbMallocRawNN(pParse->db, sizeof(pSig[0]));
+      if( pSig ){
+        pSig->selId = pExpr->x.pSelect->selId;
+        pSig->zAff = exprINAffinity(pParse, pExpr);
+      }
+    }
+
+    /* Check to see if there is a prior materialization of the RHS of
+    ** this IN operator.  If there is, then make use of that prior
+    ** materialization rather than recomputing it.
+    */
+    if( ExprHasProperty(pExpr, EP_Subrtn) 
+     || findCompatibleInRhsSubrtn(pParse, pExpr, pSig)
+    ){
       addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
       if( ExprUseXSelect(pExpr) ){
         ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d",
@@ -3459,6 +3519,10 @@ void sqlite3CodeRhsOfIN(
       assert( iTab!=pExpr->iTable );
       sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable);
       sqlite3VdbeJumpHere(v, addrOnce);
+      if( pSig ){
+        sqlite3DbFree(pParse->db, pSig->zAff);
+        sqlite3DbFree(pParse->db, pSig);
+      }
       return;
     }
 
@@ -3469,7 +3533,13 @@ void sqlite3CodeRhsOfIN(
     pExpr->y.sub.regReturn = ++pParse->nMem;
     pExpr->y.sub.iAddr =
       sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1;
-
+    if( pSig ){
+      pSig->iAddr = pExpr->y.sub.iAddr;
+      pSig->regReturn = pExpr->y.sub.regReturn;
+      pSig->iTable = iTab;
+      sqlite3VdbeChangeP4(v, -1, (const char*)pSig, P4_SUBRTNSIG);
+      pParse->bHasSubrtn = 1;
+    }
     addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
   }
 

diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 8941f82976fcfa230925f080aa81684e7078fee7..d20c77aea5ad3fa9905ab3d0c6891d6994c4bfb3 100644 (file)
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3856,6 +3856,7 @@ struct Parse {
   u8 prepFlags;        /* SQLITE_PREPARE_* flags */
   u8 withinRJSubrtn;   /* Nesting level for RIGHT JOIN body subroutines */
   u8 bHasWith;         /* True if statement contains WITH */
+  u8 bHasSubrtn;       /* True if any P4_SUBRTNSIG has been set */
 #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
   u8 earlyCleanup;     /* OOM inside sqlite3ParserAddCleanup() */
 #endif

diff --git a/src/vdbe.h b/src/vdbe.h
index 9001ace2ee0511481eee6673272398f2a9c819a5..f40f68d24b89a90b37a371e55b963e364c9bf76f 100644 (file)
--- a/src/vdbe.h
+++ b/src/vdbe.h
@@ -32,6 +32,19 @@ typedef struct Vdbe Vdbe;
 */
 typedef struct sqlite3_value Mem;
 typedef struct SubProgram SubProgram;
+typedef struct SubrtnSig SubrtnSig;
+
+/*
+** A signature for a reusable subroutine that materializes the RHS of
+** an IN operator.
+*/
+struct SubrtnSig {
+  int selId;          /* SELECT-id for the SELECT statement on the RHS */
+  char *zAff;         /* Affinity of the overall IN expression */
+  int iTable;         /* Ephemeral table generated by the subroutine */
+  int iAddr;          /* Subroutine entry address */
+  int regReturn;      /* Register used to hold return address */
+};
 
 /*
 ** A single instruction of the virtual machine has an opcode
@@ -60,6 +73,7 @@ struct VdbeOp {
     u32 *ai;               /* Used when p4type is P4_INTARRAY */
     SubProgram *pProgram;  /* Used when p4type is P4_SUBPROGRAM */
     Table *pTab;           /* Used when p4type is P4_TABLE */
+    SubrtnSig *pSubrtnSig; /* Used when p4type is P4_SUBRTNSIG */
 #ifdef SQLITE_ENABLE_CURSOR_HINTS
     Expr *pExpr;           /* Used when p4type is P4_EXPR */
 #endif
@@ -127,6 +141,7 @@ typedef struct VdbeOpList VdbeOpList;
 #define P4_INTARRAY   (-14) /* P4 is a vector of 32-bit integers */
 #define P4_FUNCCTX    (-15) /* P4 is a pointer to an sqlite3_context object */
 #define P4_TABLEREF   (-16) /* Like P4_TABLE, but reference counted */
+#define P4_SUBRTNSIG  (-17) /* P4 is a SubrtnSig pointer */
 
 /* Error message codes for OP_Halt */
 #define P5_ConstraintNotNull 1

diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index c1b75651b2cbe4e81e6fc9000171f5ff097ea843..0a946e71840167e8cbd7ffadb83863d894143b46 100644 (file)
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -1413,6 +1413,12 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){
       if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4);
       break;
     }
+    case P4_SUBRTNSIG: {
+      SubrtnSig *pSig = (SubrtnSig*)p4;
+      sqlite3DbFree(db, pSig->zAff);
+      sqlite3DbFree(db, pSig);
+      break;
+    }
   }
 }
 
@@ -1992,6 +1998,11 @@ char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){
       zP4 = pOp->p4.pTab->zName;
       break;
     }
+    case P4_SUBRTNSIG: {
+      SubrtnSig *pSig = pOp->p4.pSubrtnSig;
+      sqlite3_str_appendf(&x, "subrtnsig:%d,%s", pSig->selId, pSig->zAff);
+      break;
+    }
     default: {
       zP4 = pOp->p4.z;
     }

diff --git a/test/in7.test b/test/in7.test
index 099f75c5f70ae3c81e602c816a7cdd4097e5bdbc..29013ff59384a6698a6750f6fad5886352e20f9a 100644 (file)
--- a/test/in7.test
+++ b/test/in7.test
@@ -137,5 +137,61 @@ do_execsql_test 2.1 {
   SELECT b FROM t1 WHERE a IN (1,2,3) ORDER BY b ASC NULLS LAST;
 } {one three {}}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE x1(a);
+  INSERT INTO x1 VALUES(1), (2), (3);
+
+  CREATE TABLE x2(b);
+  INSERT INTO x2 VALUES(4), (5), (6);
+
+  CREATE TABLE t1(u);
+  INSERT INTO t1 VALUES(1), (2), (3), (4), (5), (6);
+
+  CREATE VIEW v1 AS SELECT u FROM t1 WHERE u IN (
+    SELECT a FROM x1
+  );
+  CREATE VIEW v2 AS SELECT u FROM t1 WHERE u IN (
+    SELECT b FROM x2
+  );
+}
+
+do_execsql_test 3.1 {
+  SELECT * FROM v1
+} {
+  1 2 3
+}
+
+do_execsql_test 3.2 {
+  SELECT * FROM v2
+} {
+  4 5 6 
+}
+
+do_execsql_test 3.3 {
+  SELECT * FROM v2
+  UNION ALL
+  SELECT * FROM v1
+} {
+  4 5 6
+  1 2 3 
+}
+
+do_execsql_test 3.4 {
+  WITH w1 AS (
+    SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3
+  ),
+  w2 AS (
+    SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6
+  )
+  SELECT * FROM v1 WHERE u IN w1
+  UNION ALL
+  SELECT * FROM v2 WHERE u IN w2
+} {
+  1 2 3 4 5 6
+}
+
+
 
 finish_test

diff --git a/test/pushdown.test b/test/pushdown.test
index b3663bd6f5aa5a09fdf151a81772603fc5d25cea..271d412e7e9bfb4b6dfcc7e2abcbee17ec921261 100644 (file)
--- a/test/pushdown.test
+++ b/test/pushdown.test
@@ -279,13 +279,9 @@ do_eqp_test 6.1 {
   |     |     `--CREATE BLOOM FILTER
   |     `--UNION ALL
   |        |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
-  |        `--LIST SUBQUERY xxxxxx
-  |           |--SCAN k
-  |           `--CREATE BLOOM FILTER
+  |        `--REUSE LIST SUBQUERY xxxxxx
   |--SEARCH t0
-  `--LIST SUBQUERY xxxxxx
-     |--SCAN k
-     `--CREATE BLOOM FILTER
+  `--REUSE LIST SUBQUERY xxxxxx
 }
 # ^^^^--- The key feature above is that the SEARCH for each subquery
 # uses all three fields of the index w, x, and y.  Prior to the push-down
@@ -307,17 +303,9 @@ do_eqp_test 6.2 {
   |     |     `--CREATE BLOOM FILTER
   |     `--UNION ALL
   |        |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
-  |        `--LIST SUBQUERY xxxxxx
-  |           |--CO-ROUTINE v1
-  |           |  `--SCAN 3 CONSTANT ROWS
-  |           |--SCAN v1
-  |           `--CREATE BLOOM FILTER
+  |        `--REUSE LIST SUBQUERY xxxxxx
   |--SEARCH t0
-  `--LIST SUBQUERY xxxxxx
-     |--CO-ROUTINE v1
-     |  `--SCAN 3 CONSTANT ROWS
-     |--SCAN v1
-     `--CREATE BLOOM FILTER
+  `--REUSE LIST SUBQUERY xxxxxx
 }
 do_eqp_test 6.3 {
   SELECT max(z) FROM t0 WHERE w=123 AND x IN k1 AND y BETWEEN 44 AND 55;
@@ -332,13 +320,9 @@ do_eqp_test 6.3 {
   |     |     `--CREATE BLOOM FILTER
   |     `--UNION ALL
   |        |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
-  |        `--LIST SUBQUERY xxxxxx
-  |           |--SCAN k1
-  |           `--CREATE BLOOM FILTER
+  |        `--REUSE LIST SUBQUERY xxxxxx
   |--SEARCH t0
-  `--LIST SUBQUERY xxxxxx
-     |--SCAN k1
-     `--CREATE BLOOM FILTER
+  `--REUSE LIST SUBQUERY xxxxxx
 }
 
 finish_test