From: drh <> Date: Mon, 6 Dec 2021 20:16:53 +0000 (+0000) Subject: Omit the OP_FilterInit opcode. Use OP_Blob to initialize each Bloom filter X-Git-Tag: version-3.38.0~186^2~9 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=50fb7e09b4247ce34a0b0266614303d00f929def;p=thirdparty%2Fsqlite.git Omit the OP_FilterInit opcode. Use OP_Blob to initialize each Bloom filter instead. Size the Bloom filter based on sqlite_stat1 size estimates rather than a run-time measurement for improved testability. FossilOrigin-Name: 8a9036ee617a6ad93bfe827b0789773c49d3d45b085cb76fa4b9b20a41b79b97 --- diff --git a/manifest b/manifest index 754a38701e..b4f5f5334e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sfixes\sinto\sthe\sbloom-filter\sbranch. -D 2021-12-06T19:11:31.036 +C Omit\sthe\sOP_FilterInit\sopcode.\s\sUse\sOP_Blob\sto\sinitialize\seach\sBloom\sfilter\ninstead.\s\sSize\sthe\sBloom\sfilter\sbased\son\ssqlite_stat1\ssize\sestimates\srather\nthan\sa\srun-time\smeasurement\sfor\simproved\stestability. +D 2021-12-06T20:16:53.814 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -516,7 +516,7 @@ F src/insert.c e0293a6f686e18cb2c9dd0619a731518e0109d7e1f1db1932974659e7843cfd1 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c e1dcff1c916bf6834e150b492eddda5d9792453182d2ad64294d2266b6e93c4c F src/main.c 1ea70751e6005ab6a9f784730fa0919efaa6639440a287deb73cb711e5aae57a -F src/malloc.c 183c2bf45cee1589254e4047e220f1ffbcc0a3bc8e4fe46fe64ba5db447a79af +F src/malloc.c d9172a3946f11384f2fd6a799554ee26c6bb407c4bd0874a456ed485a2e362e4 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 @@ -556,7 +556,7 @@ F src/shell.c.in 1458b700144c8326fda2514aaddeda49d6f01f1d1ccf7b9b696c53a3535a119 F src/sqlite.h.in 5cd209ac7dc4180f0e19292846f40440b8488015849ca0110c70b906b57d68f0 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8ff2fd2c166150b2e48639f5e506fb44e29f1a3f65031710b9e89d1c126ac839 -F src/sqliteInt.h ab40ea9c294c656e0d6ab14e67d58f10b015a77e962dd075fdbe3ea3cc1a976b +F src/sqliteInt.h f4fbb14ea32d57b813aabf82f586d2ac042234dd89df1c03281f557907745b98 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -621,9 +621,9 @@ F src/trigger.c 2ef56f0b7b75349a5557d0604b475126329c2e1a02432e7d49c4c710613e8254 F src/update.c d6f5c7b9e072660757ac7d58175aca11c07cb95ebbb297ae7f38853700f52328 F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 -F src/util.c 30df8356e231dad33be10bb27897655002668343280004ba28c734489414a167 +F src/util.c 6dfbd0bd1954e9531e1c511e5d20390d7dab9ffbf1e20a37c960d1aaf8582b46 F src/vacuum.c 6c38ddc52f0619865c91dae9c441d4d48bf3040d7dc1bc5b22da1e45547ed0b3 -F src/vdbe.c 6176125ea038f593597b5897898328142b5253201d321369df74e187b2b1abaa +F src/vdbe.c 9cc221ebae54417b0b47b4ce6aa6eba7e919153f8ae6307f8d18a6749d453f9c F src/vdbe.h 25dabb25c7e157b84e59260cfb5b466c3ac103ede9f36f4db371332c47601abe F src/vdbeInt.h fd1103c7ecec8c84164038c8eacaa4a633cb3c10a2f725aae7bd865d4a4fcceb F src/vdbeapi.c 22c79072ae7d8a01e9bcae8ba16e918d60d202eaa9553b5fda38f99f7464d99a @@ -638,7 +638,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c ed0398a7adf02c31e34aada42cc86c58f413a7afe5f741a5d373ad087abde028 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 04ead529a272341a4cae3ef0dcd2f7675d433627acc5fb87fed1407e7b3d8614 +F src/where.c 21bd1078837afb127827243d7ad549a4b47022ffaa43c5baa74dcac7f89809a7 F src/whereInt.h 5c6601d6d0b7b8936482506d2d835981cc6efcd8e106a829893a27a14cfb10b8 F src/wherecode.c fa667db48db1077b42731bfd97e9181b39409ffdc7051162ecae6895ca71ad2c F src/whereexpr.c 19394cb463003e9cc9305730b1508b8817a22bb7247170d81234b691a7f05b89 @@ -1934,7 +1934,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a7adcf69088cba4b86cc5731a45c9a5263af4355bc0a38f5225cab421c915f7f bb9b1a15f7e80483162049dfd981d059dc69d03348b521f7ac164a8cd3ae3cc4 -R 0f448dc1f18098ffcb6b2fdc27d44d39 +P edacf8034dc6bd892038c220c480ea512dbb4005db2a6b1f8e679e8a4929c6ed +R 933b551d7a60e6c29e7de8b534d33928 U drh -Z a01a7e207ad8c3ff4f1aefb648f1cc10 +Z ded7cd3093221f3f9f8f684eb1a7389d diff --git a/manifest.uuid b/manifest.uuid index 63d6d10d76..0b0fb52902 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -edacf8034dc6bd892038c220c480ea512dbb4005db2a6b1f8e679e8a4929c6ed \ No newline at end of file +8a9036ee617a6ad93bfe827b0789773c49d3d45b085cb76fa4b9b20a41b79b97 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index ab9b37ddad..9a2eaeec15 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -211,6 +211,7 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag){ return mx; } +#if 0 /* ** Return an estimate of the amount of unallocated memory. ** @@ -229,6 +230,7 @@ sqlite3_int64 sqlite3EstMemoryAvailable(void){ if( n<0 ) n = 0; return n; } +#endif /* ** Trigger the alarm diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9091f02115..90e60d30b0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3927,6 +3927,7 @@ struct Sqlite3Config { int iOnceResetThreshold; /* When to reset OP_Once counters */ u32 szSorterRef; /* Min size in bytes to use sorter-refs */ unsigned int iPrngSeed; /* Alternative fixed seed for the PRNG */ + int iEstCountScale; /* Multiple RowCountEst() by this amount */ /* vvvv--- must be last ---vvv */ #ifdef SQLITE_DEBUG sqlite3_int64 aTune[SQLITE_NTUNE]; /* Tuning parameters */ @@ -4293,7 +4294,9 @@ void sqlite3MemSetDefault(void); void sqlite3BenignMallocHooks(void (*)(void), void (*)(void)); #endif int sqlite3HeapNearlyFull(void); +#if 0 sqlite3_int64 sqlite3EstMemoryAvailable(void); +#endif /* ** On systems with ample stack space and that support alloca(), make @@ -4780,11 +4783,7 @@ LogEst sqlite3LogEstAdd(LogEst,LogEst); #ifndef SQLITE_OMIT_VIRTUALTABLE LogEst sqlite3LogEstFromDouble(double); #endif -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_ENABLE_STAT4) || \ - defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) u64 sqlite3LogEstToInt(LogEst); -#endif VList *sqlite3VListAdd(sqlite3*,VList*,const char*,int,int); const char *sqlite3VListNumToName(VList*,int); int sqlite3VListNameToNum(VList*,const char*,int); diff --git a/src/util.c b/src/util.c index 8452aea665..d93c298116 100644 --- a/src/util.c +++ b/src/util.c @@ -1603,14 +1603,8 @@ LogEst sqlite3LogEstFromDouble(double x){ } #endif /* SQLITE_OMIT_VIRTUALTABLE */ -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_ENABLE_STAT4) || \ - defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) /* ** Convert a LogEst into an integer. -** -** Note that this routine is only used when one or more of various -** non-standard compile-time options is enabled. */ u64 sqlite3LogEstToInt(LogEst x){ u64 n; @@ -1618,17 +1612,9 @@ u64 sqlite3LogEstToInt(LogEst x){ x /= 10; if( n>=5 ) n -= 2; else if( n>=1 ) n -= 1; -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) if( x>60 ) return (u64)LARGEST_INT64; -#else - /* If only SQLITE_ENABLE_STAT4 is on, then the largest input - ** possible to this routine is 310, resulting in a maximum x of 31 */ - assert( x<=60 ); -#endif return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x); } -#endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */ /* ** Add a new name/number pair to a VList. This might require that the diff --git a/src/vdbe.c b/src/vdbe.c index 487d731af3..c2281e70a1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1367,12 +1367,18 @@ case OP_SoftNull: { ** Synopsis: r[P2]=P4 (len=P1) ** ** P4 points to a blob of data P1 bytes long. Store this -** blob in register P2. +** blob in register P2. If P4 is a NULL pointer, then construct +** a zero-filled blob that is P1 bytes long in P2. */ case OP_Blob: { /* out2 */ assert( pOp->p1 <= SQLITE_MAX_LENGTH ); pOut = out2Prerelease(p, pOp); - sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); + if( pOp->p4.z==0 ){ + sqlite3VdbeMemSetZeroBlob(pOut, pOp->p1); + if( sqlite3VdbeMemExpandBlob(pOut) ) goto no_mem; + }else{ + sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); + } pOut->enc = encoding; UPDATE_MAX_BLOBSIZE(pOut); break; @@ -3383,7 +3389,7 @@ case OP_MakeRecord: { break; } -/* Opcode: Count P1 P2 p3 * * +/* Opcode: Count P1 P2 P3 * * ** Synopsis: r[P2]=count() ** ** Store the number of entries (an integer value) in the table or index @@ -8169,48 +8175,6 @@ case OP_Function: { /* group */ break; } -/* Opcode: FilterInit P1 P2 * * * -** -** Initialize register P1 so that is an empty bloom filter. -** -** If P2 is positive, it is a register that holds an estimate on -** the number of entries to be added to the Bloom filter. The -** Bloom filter is sized accordingly. If P2 is zero or negative, -** then a default-size Bloom filter is created. -** -** It is ok for P1 and P2 to be the same register. In that case the -** integer value originally in that register will be overwritten -** with the new empty bloom filter. -*/ -case OP_FilterInit: { - i64 n, mx; - assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); - pIn1 = &aMem[pOp->p1]; - if( pOp->p2>0 ){ - assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); - n = sqlite3VdbeIntValue(&aMem[pOp->p2]); - if( nSQLITE_BLOOM_MAX ){ - n = SQLITE_BLOOM_MAX; - } - }else{ - n = SQLITE_BLOOM_MIN; - } - mx = sqlite3EstMemoryAvailable()/2; - if( n>mx && mx>SQLITE_BLOOM_MIN ){ - n = mx; - } -#ifdef SQLITE_DEBUG - if( db->flags&SQLITE_VdbeTrace ){ - printf("Bloom-filter size: %llu bytes\n", n); - } -#endif - sqlite3VdbeMemSetZeroBlob(pIn1, n); - if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem; - break; -} - /* Opcode: FilterAdd P1 * P3 P4 * ** Synopsis: filter(P1) += key(P3@P4) ** diff --git a/src/where.c b/src/where.c index 233e77b6a7..e7246b1040 100644 --- a/src/where.c +++ b/src/where.c @@ -911,7 +911,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( VdbeComment((v, "for %s", pTable->zName)); if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ pLevel->regFilter = ++pParse->nMem; - sqlite3VdbeAddOp1(v, OP_FilterInit, pLevel->regFilter); + sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter); } /* Fill the automatic index with content */ @@ -1006,13 +1006,39 @@ static SQLITE_NOINLINE void constructBloomFilter( addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); do{ + const SrcItem *pItem; + const Table *pTab; + int sz; sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel); addrCont = sqlite3VdbeMakeLabel(pParse); iCur = pLevel->iTabCur; pLevel->regFilter = ++pParse->nMem; + + /* The Bloom filter is a Blob held in a register. Initialize it + ** to zero-filled blob of at least 80K bits, but maybe more if the + ** estimated size of the table is larger. We could actually + ** measure the size of the table at run-time using OP_Count with + ** P3==1 and use that value to initialize the blob. But that makes + ** testing complicated. By basing the blob size on the value in the + ** sqlite_stat1 table, testing is much easier. + */ + pItem = &pWInfo->pTabList->a[pLevel->iFrom]; + assert( pItem!=0 ); + pTab = pItem->pTab; + assert( pTab!=0 ); + if( pTab->tabFlags & TF_HasStat1 ){ + sz = sqlite3LogEstToInt(pItem->pTab->nRowLogEst); + if( sz<10000 ){ + sz = 10000; + }else if( sz>10000000 ){ + sz = 10000000; + } + }else{ + sz = 10000; + } + sqlite3VdbeAddOp2(v, OP_Blob, sz, pLevel->regFilter); + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Count, iCur, pLevel->regFilter, 1); - sqlite3VdbeAddOp2(v, OP_FilterInit, pLevel->regFilter, pLevel->regFilter); pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm]; for(pTerm=pWInfo->sWC.a; pTermpExpr; @@ -1034,13 +1060,14 @@ static SQLITE_NOINLINE void constructBloomFilter( int jj; for(jj=0; jjaiColumn[jj]; + assert( pIdx->pTable==pItem->pTab ); sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj); } sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n); sqlite3ReleaseTempRange(pParse, r1, n); } sqlite3VdbeResolveLabel(v, addrCont); - sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+3); + sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrTop); pLoop->wsFlags &= ~WHERE_BLOOMFILTER;