From: dan Date: Wed, 3 Jul 2024 20:30:31 +0000 (+0000) Subject: When constructing an ephermeral table to use as the right-hand side of X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fheads%2Fbedrock-3.45-in-bloom;p=thirdparty%2Fsqlite.git When constructing an ephermeral table to use as the right-hand side of an IN operator, also construct a Bloom filter to speed membership testing. FossilOrigin-Name: 0bb306eb70ef1df7734326d30359da7a15397171d3e25ab644633ef3ee1428ec --- diff --git a/manifest b/manifest index 32c2a482e6..17193a98bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\s32-bit\soverflow\swhen\scalculating\sncycle\sfor\s".scanstats\svm". -D 2024-06-27T18:18:53.432 +C When\sconstructing\san\sephermeral\stable\sto\suse\sas\sthe\sright-hand\sside\sof\nan\sIN\soperator,\salso\sconstruct\sa\sBloom\sfilter\sto\sspeed\smembership\stesting. +D 2024-07-03T20:30:31.472 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c 3381ee4c9aa7ccde22a2a7f35ce343925a7a25d96bdc943649131f9decdebad2 +F src/expr.c ca38420de18ba955f8c53dc4aa6f2ec734858c147cd6842878b7d8f43de8adb4 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c bc3538cbdd9d0d7e4bc6869067dbb102affac738765843009e4694dc4eb70c83 @@ -746,12 +746,12 @@ F src/printf.c d3392b2a20ee314ddeef34fb43c904bf4619eb20ff9a9e07e3950a7e4dcd6912 F src/random.c 9bd018738ec450bf35d28050b4b33fa9a6eebf3aaefb1a1cff42dc14a7725673 F src/resolve.c 446bcb8ebf0ea7066c2ca99e5336f0dbc9230ac76f80fafd1bfa82fe7871af2d F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c c581265a87628e1abcff72df3e96b56d89e8137ddb780d2522863cce08cf43cd +F src/select.c 46e4cca5fa61ac0c03a06b395ff84930f36c7d429e45f28e8f436b889ae89ced F src/shell.c.in 627eb35078011f55509c0cbc28889fbb01cfb4370bf52e6195ba97c885408009 F src/sqlite.h.in 882c85b1dbe2988af021a2fe60cd65a0c321f0a3f04ef762e0f3f07ebb53e040 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h ec40714de7bd15c3cb2778158bc7e859205cb453559bab7f8ac78d480df6f62a +F src/sqliteInt.h 8649226c767f98aadfc4c021fdeb772b1c1bfd4e924e6229817a2346eec030bd F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -816,7 +816,7 @@ F src/upsert.c 2e60567a0e9e8520c18671b30712a88dc73534474304af94f32bb5f3ef65ac65 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c c346e2a811285f6bad8a87c862070207855159a783993d03ca20dea2c74ace04 F src/vacuum.c b1dd6d73869229b6e08bac910ac011dc9da42e3120ec2b7241accc5a752bd419 -F src/vdbe.c 6d495a3ba581dbbd90e714563b8bd554f9298bd5157006822a2b574057a7786f +F src/vdbe.c 1306f1a64cf625714dbff522a4d700bfba8dce191d176f5cab1d5cc2bb7e5b7c F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c F src/vdbeapi.c 8f57d60c89da0b60e6d4e272358c511f6bae4e24330bdb11f8b42f986d1bf21b @@ -899,7 +899,7 @@ F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 76d20a7fa136d63bcfcf8bcb65c0b1455ed71078d81f22bcd0550d3eb18594ab F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec F test/autoinc.test 997d6f185f138229dc4251583a1d04816423dddc2fc034871a01aeb1d728cb39 -F test/autoindex1.test d34caffb0384003ee28eae87679214c029e9be4b332d9649a79e0b94ab70502c +F test/autoindex1.test 714cac6e60beeb5a26ed346dd46505ba60b5a5597e9122c9ed3a55f89a922aa4 F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df F test/autoindex3.test dcd6b2f8bed2be67b131e2e671f892e971d934e24fd00988952d0e0a67e24aa7 F test/autoindex4.test 3c2105e9172920e26f950ba3c5823e4972190e022c1e6f260ba476b0af24c593 @@ -1108,7 +1108,7 @@ F test/enc.test 9a7be5479da985381d740b15f432800f65e2c87029ee57a318f42cb2eb43763a F test/enc2.test 848bf05f15b011719f478dddb7b5e9aea35e39e457493cba4c4eef75d849a5ec F test/enc3.test 55ef64416d72975c66167310a51dc9fc544ba3ae4858b8d5ab22f4cb6500b087 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test f3f7548d2f3df03e2f23ecaf35c7c2cc7b89848bd7c3606d94a010521b7ea4f6 +F test/eqp.test 9219ca234b8fe8da049c04baca18e495aa25d8491779688b5deecb54c2b927f0 F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9 F test/eval.test 73969a2d43a511bf44080c44485a8c4d796b6a4f038d19e491867081155692c0 F test/exclusive.test 7ff63be7503990921838d5c9f77f6e33e68e48ed1a9d48cd28745bf650bf0747 @@ -1511,7 +1511,7 @@ F test/printf.test 685fec5a0c5af2490ab0632775a301554361d674211d690f5bee0a97b0533 F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/pushdown.test 1495a09837a1cedfc0adf07ba42dc6b83be05a2c15de331b67c39a0e22078238 +F test/pushdown.test eab987a373887855d614fda539f4fc2350ea9bb93bf37c4c94608a4692193c44 F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quickcheck.test a4b7e878cd97e46108291c409b0bf8214f29e18fddd68a42bc5c1375ad1fb80a @@ -1541,7 +1541,7 @@ F test/rowid.test e29025be95baf6b32f0d5edef59a7633028325896a98f1caa8019559ca9103 F test/rowvalue.test baf4fa3ec1a8c1c920c3faa5fd25959cb454bbd99ac8960397c34549d9fc4abe F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 1347e25ca11c547c5a6ff0cc5626f95aa9740e9275bfaec096029f57cb2130ce -F test/rowvalue4.test 441e7e366ac6d939a3a95a574031c56ec2a854077a91d66eee5ff1d86cb5be58 +F test/rowvalue4.test bac9326d1e886656650f67c0ec484eb5f452244a8209c6af508e9a862ace08ed F test/rowvalue5.test 00740304ea6a53a8704640c7405690f0045d5d2a6b4b04dde7bccc14c3068ea7 F test/rowvalue6.test d19b54feb604d5601f8614b15e214e0774c01087 F test/rowvalue7.test c1cbdbf407029db01f87764097c6ac02a1c5a37efd2776eff32a9cdfdf6f2dba @@ -2197,9 +2197,12 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8362edb45b993f8467b464ca0491ef347c8215ea2e65504f7b3d1f7232eb3c63 -Q +2858efa06d4fc7b412b892f35f5e9a766b467b4a5b74d602a030d25443f9efb4 -R 9f2f34b51784ba4176cf6dda819b8df2 +P 78022f90d5e3922b6568a01c8723116eb5c79d74d16005c3d8d6525d57de5e78 +Q +baa83b460c677c210c7fa3f20314d7e05f305aed8a69026bc5fa106a3de4ea38 +R d8d852688978a92b3656edbd57f94415 +T *branch * bedrock-3.45-in-bloom +T *sym-bedrock-3.45-in-bloom * +T -sym-bedrock-3.45 * U dan -Z 849e01900f8ab047a01241d3e121cd9d +Z 1cea51c643b53b8129fa361e45baac12 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2caa1ff451..4ea6d40977 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -78022f90d5e3922b6568a01c8723116eb5c79d74d16005c3d8d6525d57de5e78 +0bb306eb70ef1df7734326d30359da7a15397171d3e25ab644633ef3ee1428ec diff --git a/src/expr.c b/src/expr.c index f9b280bbc5..47ab7abc7b 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3393,19 +3393,34 @@ void sqlite3CodeRhsOfIN( SelectDest dest; int i; int rc; + int addrBloom = 0; sqlite3SelectDestInit(&dest, SRT_Set, iTab); dest.zAffSdst = exprINAffinity(pParse, pExpr); pSelect->iLimit = 0; + if( addrOnce && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + int regBloom = ++pParse->nMem; + addrBloom = sqlite3VdbeAddOp2(v, OP_Blob, 10000, regBloom); + VdbeComment((v, "Bloom filter")); + dest.iSDParm2 = regBloom; + } testcase( pSelect->selFlags & SF_Distinct ); testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ pCopy = sqlite3SelectDup(pParse->db, pSelect, 0); rc = pParse->db->mallocFailed ? 1 :sqlite3Select(pParse, pCopy, &dest); sqlite3SelectDelete(pParse->db, pCopy); sqlite3DbFree(pParse->db, dest.zAffSdst); + if( addrBloom ){ + sqlite3VdbeGetOp(v, addrOnce)->p3 = dest.iSDParm2; + if( dest.iSDParm2==0 ){ + sqlite3VdbeChangeToNoop(v, addrBloom); + }else{ + sqlite3VdbeGetOp(v, addrOnce)->p3 = dest.iSDParm2; + } + } if( rc ){ sqlite3KeyInfoUnref(pKeyInfo); return; - } + } assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ assert( pEList!=0 ); assert( pEList->nExpr>0 ); @@ -3844,6 +3859,15 @@ static void sqlite3ExprCodeIN( sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); if( destIfFalse==destIfNull ){ /* Combine Step 3 and Step 5 into a single opcode */ + if( ExprHasProperty(pExpr, EP_Subrtn) ){ + const VdbeOp *pOp = sqlite3VdbeGetOp(v, pExpr->y.sub.iAddr); + assert( pOp->opcode==OP_Once || pParse->nErr ); + if( pOp->opcode==OP_Once && pOp->p3>0 ){ + assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ); + sqlite3VdbeAddOp4Int(v, OP_Filter, pOp->p3, destIfFalse, + rLhs, nVector); VdbeCoverage(v); + } + } sqlite3VdbeAddOp4Int(v, OP_NotFound, iTab, destIfFalse, rLhs, nVector); VdbeCoverage(v); goto sqlite3ExprCodeIN_finished; diff --git a/src/select.c b/src/select.c index bed81495c3..4dd9cc51b4 100644 --- a/src/select.c +++ b/src/select.c @@ -1377,12 +1377,18 @@ static void selectInnerLoop( ** case the order does matter */ pushOntoSorter( pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + pDest->iSDParm2 = 0; /* Signal that any Bloom filter is unpopulated */ }else{ int r1 = sqlite3GetTempReg(pParse); assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, r1, pDest->zAffSdst, nResultCol); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); + if( pDest->iSDParm2 ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pDest->iSDParm2, 0, + regResult, nResultCol); + ExplainQueryPlan((pParse, 0, "CREATE BLOOM FILTER")); + } sqlite3ReleaseTempReg(pParse, r1); } break; @@ -3310,6 +3316,11 @@ static int generateOutputSubroutine( r1, pDest->zAffSdst, pIn->nSdst); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pDest->iSDParm, r1, pIn->iSdst, pIn->nSdst); + if( pDest->iSDParm2>0 ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pDest->iSDParm2, 0, + pIn->iSdst, pIn->nSdst); + ExplainQueryPlan((pParse, 0, "CREATE BLOOM FILTER")); + } sqlite3ReleaseTempReg(pParse, r1); break; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 301999666a..a94ac3039b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3625,7 +3625,11 @@ struct Select { ** SRT_Set The result must be a single column. Store each ** row of result as the key in table pDest->iSDParm. ** Apply the affinity pDest->affSdst before storing -** results. Used to implement "IN (SELECT ...)". +** results. if pDest->iSDParm2 is positive, then it is +** a regsiter holding a Bloom filter for the IN operator +** that should be populated in addition to the +** pDest->iSDParm table. This SRT is used to +** implement "IN (SELECT ...)". ** ** SRT_EphemTab Create an temporary table pDest->iSDParm and store ** the result there. The cursor is left open after diff --git a/src/vdbe.c b/src/vdbe.c index 0efbb6f7ef..8ce0d27e9f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5328,6 +5328,7 @@ case OP_Found: { /* jump, in3, ncycle */ r.pKeyInfo = pC->pKeyInfo; r.default_rc = 0; #ifdef SQLITE_DEBUG + (void)sqlite3FaultSim(50); /* For use by --counter in TH3 */ for(ii=0; ii0; } {- - 3 4 5} + finish_test diff --git a/test/rowvalue4.test b/test/rowvalue4.test index 784859cb19..1ef5fc2920 100644 --- a/test/rowvalue4.test +++ b/test/rowvalue4.test @@ -236,9 +236,11 @@ do_eqp_test 5.1 { QUERY PLAN |--SEARCH d2 USING INDEX d2ab (a=? AND b=?) |--LIST SUBQUERY xxxxxx - | `--SCAN d1 + | |--SCAN d1 + | `--CREATE BLOOM FILTER `--LIST SUBQUERY xxxxxx - `--SCAN d1 + |--SCAN d1 + `--CREATE BLOOM FILTER } do_execsql_test 6.0 {