Omit the OP_FilterInit opcode. Use OP_Blob to initialize each Bloom filter

instead.  Size the Bloom filter based on sqlite_stat1 size estimates rather
than a run-time measurement for improved testability.

FossilOrigin-Name: 8a9036ee617a6ad93bfe827b0789773c49d3d45b085cb76fa4b9b20a41b79b97

diff --git a/manifest b/manifest
index 754a38701e59b1deff8a371e4103b440d11cd747..b4f5f5334e9ca120253c5b4eb96ea5f81e4f44d5 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\strunk\sfixes\sinto\sthe\sbloom-filter\sbranch.
-D 2021-12-06T19:11:31.036
+C Omit\sthe\sOP_FilterInit\sopcode.\s\sUse\sOP_Blob\sto\sinitialize\seach\sBloom\sfilter\ninstead.\s\sSize\sthe\sBloom\sfilter\sbased\son\ssqlite_stat1\ssize\sestimates\srather\nthan\sa\srun-time\smeasurement\sfor\simproved\stestability.
+D 2021-12-06T20:16:53.814
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -516,7 +516,7 @@ F src/insert.c e0293a6f686e18cb2c9dd0619a731518e0109d7e1f1db1932974659e7843cfd1
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c e1dcff1c916bf6834e150b492eddda5d9792453182d2ad64294d2266b6e93c4c
 F src/main.c 1ea70751e6005ab6a9f784730fa0919efaa6639440a287deb73cb711e5aae57a
-F src/malloc.c 183c2bf45cee1589254e4047e220f1ffbcc0a3bc8e4fe46fe64ba5db447a79af
+F src/malloc.c d9172a3946f11384f2fd6a799554ee26c6bb407c4bd0874a456ed485a2e362e4
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
 F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75
@@ -556,7 +556,7 @@ F src/shell.c.in 1458b700144c8326fda2514aaddeda49d6f01f1d1ccf7b9b696c53a3535a119
 F src/sqlite.h.in 5cd209ac7dc4180f0e19292846f40440b8488015849ca0110c70b906b57d68f0
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839
-F src/sqliteInt.h ab40ea9c294c656e0d6ab14e67d58f10b015a77e962dd075fdbe3ea3cc1a976b
+F src/sqliteInt.h f4fbb14ea32d57b813aabf82f586d2ac042234dd89df1c03281f557907745b98
 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -621,9 +621,9 @@ F src/trigger.c 2ef56f0b7b75349a5557d0604b475126329c2e1a02432e7d49c4c710613e8254
 F src/update.c d6f5c7b9e072660757ac7d58175aca11c07cb95ebbb297ae7f38853700f52328
 F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
-F src/util.c 30df8356e231dad33be10bb27897655002668343280004ba28c734489414a167
+F src/util.c 6dfbd0bd1954e9531e1c511e5d20390d7dab9ffbf1e20a37c960d1aaf8582b46
 F src/vacuum.c 6c38ddc52f0619865c91dae9c441d4d48bf3040d7dc1bc5b22da1e45547ed0b3
-F src/vdbe.c 6176125ea038f593597b5897898328142b5253201d321369df74e187b2b1abaa
+F src/vdbe.c 9cc221ebae54417b0b47b4ce6aa6eba7e919153f8ae6307f8d18a6749d453f9c
 F src/vdbe.h 25dabb25c7e157b84e59260cfb5b466c3ac103ede9f36f4db371332c47601abe
 F src/vdbeInt.h fd1103c7ecec8c84164038c8eacaa4a633cb3c10a2f725aae7bd865d4a4fcceb
 F src/vdbeapi.c 22c79072ae7d8a01e9bcae8ba16e918d60d202eaa9553b5fda38f99f7464d99a
@@ -638,7 +638,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028
 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
 F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b
-F src/where.c 04ead529a272341a4cae3ef0dcd2f7675d433627acc5fb87fed1407e7b3d8614
+F src/where.c 21bd1078837afb127827243d7ad549a4b47022ffaa43c5baa74dcac7f89809a7
 F src/whereInt.h 5c6601d6d0b7b8936482506d2d835981cc6efcd8e106a829893a27a14cfb10b8
 F src/wherecode.c fa667db48db1077b42731bfd97e9181b39409ffdc7051162ecae6895ca71ad2c
 F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89
@@ -1934,7 +1934,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a7adcf69088cba4b86cc5731a45c9a5263af4355bc0a38f5225cab421c915f7f bb9b1a15f7e80483162049dfd981d059dc69d03348b521f7ac164a8cd3ae3cc4
-R 0f448dc1f18098ffcb6b2fdc27d44d39
+P edacf8034dc6bd892038c220c480ea512dbb4005db2a6b1f8e679e8a4929c6ed
+R 933b551d7a60e6c29e7de8b534d33928
 U drh
-Z a01a7e207ad8c3ff4f1aefb648f1cc10
+Z ded7cd3093221f3f9f8f684eb1a7389d

diff --git a/manifest.uuid b/manifest.uuid
index 63d6d10d76e49af121f93035a3272399a44c4f40..0b0fb52902b73df06191d8a8346285070ebbd170 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-edacf8034dc6bd892038c220c480ea512dbb4005db2a6b1f8e679e8a4929c6ed
\ No newline at end of file
+8a9036ee617a6ad93bfe827b0789773c49d3d45b085cb76fa4b9b20a41b79b97
\ No newline at end of file

diff --git a/src/malloc.c b/src/malloc.c
index ab9b37ddadfc496e079032119d2e964f5f792cbd..9a2eaeec155bd4af32cd53d01accaf701f88061e 100644 (file)
--- a/src/malloc.c
+++ b/src/malloc.c
@@ -211,6 +211,7 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag){
   return mx;
 }
 
+#if 0
 /*
 ** Return an estimate of the amount of unallocated memory.
 **
@@ -229,6 +230,7 @@ sqlite3_int64 sqlite3EstMemoryAvailable(void){
   if( n<0 ) n = 0;
   return n;
 }
+#endif
 
 /*
 ** Trigger the alarm 

diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 9091f02115aca4c5e7122858b3db0c1388bbcde7..90e60d30b0a6365ca7f9e817ca7adc96f2ef77be 100644 (file)
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3927,6 +3927,7 @@ struct Sqlite3Config {
   int iOnceResetThreshold;          /* When to reset OP_Once counters */
   u32 szSorterRef;                  /* Min size in bytes to use sorter-refs */
   unsigned int iPrngSeed;           /* Alternative fixed seed for the PRNG */
+  int iEstCountScale;               /* Multiple RowCountEst() by this amount */
   /* vvvv--- must be last ---vvv */
 #ifdef SQLITE_DEBUG
   sqlite3_int64 aTune[SQLITE_NTUNE]; /* Tuning parameters */
@@ -4293,7 +4294,9 @@ void sqlite3MemSetDefault(void);
 void sqlite3BenignMallocHooks(void (*)(void), void (*)(void));
 #endif
 int sqlite3HeapNearlyFull(void);
+#if 0
 sqlite3_int64 sqlite3EstMemoryAvailable(void);
+#endif
 
 /*
 ** On systems with ample stack space and that support alloca(), make
@@ -4780,11 +4783,7 @@ LogEst sqlite3LogEstAdd(LogEst,LogEst);
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 LogEst sqlite3LogEstFromDouble(double);
 #endif
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
-    defined(SQLITE_ENABLE_STAT4) || \
-    defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
 u64 sqlite3LogEstToInt(LogEst);
-#endif
 VList *sqlite3VListAdd(sqlite3*,VList*,const char*,int,int);
 const char *sqlite3VListNumToName(VList*,int);
 int sqlite3VListNameToNum(VList*,const char*,int);

diff --git a/src/util.c b/src/util.c
index 8452aea665b4a9de49e181b175d6e63b40cb978a..d93c2981165e92d415fd073267fff9a225659504 100644 (file)
--- a/src/util.c
+++ b/src/util.c
@@ -1603,14 +1603,8 @@ LogEst sqlite3LogEstFromDouble(double x){
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
-    defined(SQLITE_ENABLE_STAT4) || \
-    defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
 /*
 ** Convert a LogEst into an integer.
-**
-** Note that this routine is only used when one or more of various
-** non-standard compile-time options is enabled.
 */
 u64 sqlite3LogEstToInt(LogEst x){
   u64 n;
@@ -1618,17 +1612,9 @@ u64 sqlite3LogEstToInt(LogEst x){
   x /= 10;
   if( n>=5 ) n -= 2;
   else if( n>=1 ) n -= 1;
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
-    defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
   if( x>60 ) return (u64)LARGEST_INT64;
-#else
-  /* If only SQLITE_ENABLE_STAT4 is on, then the largest input
-  ** possible to this routine is 310, resulting in a maximum x of 31 */
-  assert( x<=60 );
-#endif
   return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x);
 }
-#endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */
 
 /*
 ** Add a new name/number pair to a VList.  This might require that the

diff --git a/src/vdbe.c b/src/vdbe.c
index 487d731af3cd14e933e035336a747fa989a85d77..c2281e70a12416c06290bb2cabaae201b3d6124d 100644 (file)
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -1367,12 +1367,18 @@ case OP_SoftNull: {
 ** Synopsis: r[P2]=P4 (len=P1)
 **
 ** P4 points to a blob of data P1 bytes long.  Store this
-** blob in register P2.
+** blob in register P2.  If P4 is a NULL pointer, then construct
+** a zero-filled blob that is P1 bytes long in P2.
 */
 case OP_Blob: {                /* out2 */
   assert( pOp->p1 <= SQLITE_MAX_LENGTH );
   pOut = out2Prerelease(p, pOp);
-  sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0);
+  if( pOp->p4.z==0 ){
+    sqlite3VdbeMemSetZeroBlob(pOut, pOp->p1);
+    if( sqlite3VdbeMemExpandBlob(pOut) ) goto no_mem;
+  }else{
+    sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0);
+  }
   pOut->enc = encoding;
   UPDATE_MAX_BLOBSIZE(pOut);
   break;
@@ -3383,7 +3389,7 @@ case OP_MakeRecord: {
   break;
 }
 
-/* Opcode: Count P1 P2 p3 * *
+/* Opcode: Count P1 P2 P3 * *
 ** Synopsis: r[P2]=count()
 **
 ** Store the number of entries (an integer value) in the table or index 
@@ -8169,48 +8175,6 @@ case OP_Function: {            /* group */
   break;
 }
 
-/* Opcode: FilterInit P1 P2 * * *
-**
-** Initialize register P1 so that is an empty bloom filter.
-**
-** If P2 is positive, it is a register that holds an estimate on
-** the number of entries to be added to the Bloom filter.  The
-** Bloom filter is sized accordingly.  If P2 is zero or negative,
-** then a default-size Bloom filter is created.
-**
-** It is ok for P1 and P2 to be the same register.  In that case the
-** integer value originally in that register will be overwritten
-** with the new empty bloom filter.
-*/
-case OP_FilterInit: {
-  i64 n, mx;
-  assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
-  pIn1 = &aMem[pOp->p1];
-  if( pOp->p2>0 ){
-    assert( pOp->p2<=(p->nMem+1 - p->nCursor) );
-    n = sqlite3VdbeIntValue(&aMem[pOp->p2]);
-    if( n<SQLITE_BLOOM_MIN ){
-      n = SQLITE_BLOOM_MIN;
-    }else if( n>SQLITE_BLOOM_MAX ){
-      n = SQLITE_BLOOM_MAX;
-    }
-  }else{
-    n = SQLITE_BLOOM_MIN;
-  }
-  mx = sqlite3EstMemoryAvailable()/2;
-  if( n>mx && mx>SQLITE_BLOOM_MIN ){
-    n = mx;
-  }
-#ifdef SQLITE_DEBUG
-  if( db->flags&SQLITE_VdbeTrace ){
-    printf("Bloom-filter size: %llu bytes\n", n);
-  }
-#endif
-  sqlite3VdbeMemSetZeroBlob(pIn1, n);
-  if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem;
-  break;
-}
-
 /* Opcode: FilterAdd P1 * P3 P4 *
 ** Synopsis: filter(P1) += key(P3@P4)
 **

diff --git a/src/where.c b/src/where.c
index 233e77b6a78adc0431d0b9a00107f7b731973452..e7246b1040912a3595f7858768d5c41b41bb1870 100644 (file)
--- a/src/where.c
+++ b/src/where.c
@@ -911,7 +911,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   VdbeComment((v, "for %s", pTable->zName));
   if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){
     pLevel->regFilter = ++pParse->nMem;
-    sqlite3VdbeAddOp1(v, OP_FilterInit, pLevel->regFilter);
+    sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter);
   }
 
   /* Fill the automatic index with content */
@@ -1006,13 +1006,39 @@ static SQLITE_NOINLINE void constructBloomFilter(
 
   addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
   do{
+    const SrcItem *pItem;
+    const Table *pTab;
+    int sz;
     sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
     addrCont = sqlite3VdbeMakeLabel(pParse);
     iCur = pLevel->iTabCur;
     pLevel->regFilter = ++pParse->nMem;
+
+    /* The Bloom filter is a Blob held in a register.  Initialize it
+    ** to zero-filled blob of at least 80K bits, but maybe more if the
+    ** estimated size of the table is larger.  We could actually
+    ** measure the size of the table at run-time using OP_Count with
+    ** P3==1 and use that value to initialize the blob.  But that makes
+    ** testing complicated.  By basing the blob size on the value in the
+    ** sqlite_stat1 table, testing is much easier.
+    */
+    pItem = &pWInfo->pTabList->a[pLevel->iFrom];
+    assert( pItem!=0 );
+    pTab = pItem->pTab;
+    assert( pTab!=0 );
+    if( pTab->tabFlags & TF_HasStat1 ){
+      sz = sqlite3LogEstToInt(pItem->pTab->nRowLogEst);
+      if( sz<10000 ){
+        sz = 10000;
+      }else if( sz>10000000 ){
+        sz = 10000000;
+      }
+    }else{
+      sz = 10000;
+    }
+    sqlite3VdbeAddOp2(v, OP_Blob, sz, pLevel->regFilter);
+
     addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v);
-    sqlite3VdbeAddOp3(v, OP_Count, iCur, pLevel->regFilter, 1);
-    sqlite3VdbeAddOp2(v, OP_FilterInit, pLevel->regFilter, pLevel->regFilter);
     pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm];
     for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
       Expr *pExpr = pTerm->pExpr;
@@ -1034,13 +1060,14 @@ static SQLITE_NOINLINE void constructBloomFilter(
       int jj;
       for(jj=0; jj<n; jj++){
         int iCol = pIdx->aiColumn[jj];
+        assert( pIdx->pTable==pItem->pTab );
         sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj);
       }
       sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
       sqlite3ReleaseTempRange(pParse, r1, n);
     }
     sqlite3VdbeResolveLabel(v, addrCont);
-    sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+3);
+    sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1);
     VdbeCoverage(v);
     sqlite3VdbeJumpHere(v, addrTop);
     pLoop->wsFlags &= ~WHERE_BLOOMFILTER;