From: drh Date: Fri, 8 Jun 2018 18:22:10 +0000 (+0000) Subject: Consider doing a partial table scan to fulfill an IN operator rather X-Git-Tag: version-3.25.0~205^2~2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=da4c409aea3ccd9cc30219a67ddb3677d0d4521f;p=thirdparty%2Fsqlite.git Consider doing a partial table scan to fulfill an IN operator rather than using an index. Try to pick the plan with the lowest cost. FossilOrigin-Name: 1fa40a78fef4516c39b217bff67efe7e7d2077cca00aae0ef5c2c9cff94f008b --- diff --git a/manifest b/manifest index f31ac5a47b..d068879da9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sIN-early-out\soptimization:\s\sWhen\sdoing\sa\slook-up\son\sa\smulti-column\sindex\nand\san\sIN\soperator\sis\sused\son\sa\scolumn\sother\sthan\sthe\sleft-most\scolumn,\sthen\nif\sno\srows\smatch\sagainst\sthe\sfirst\sIN\svalue,\scheck\sto\smake\ssure\sthere\sexist\nrows\sthat\smatch\sthe\scolumns\sto\sthe\sright\sbefore\scontinuing\swith\sthe\snext\sIN\nvalue. -D 2018-06-07T18:13:49.091 +C Consider\sdoing\sa\spartial\stable\sscan\sto\sfulfill\san\sIN\soperator\srather\nthan\susing\san\sindex.\s\sTry\sto\spick\sthe\splan\swith\sthe\slowest\scost. +D 2018-06-08T18:22:10.089 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in bfc40f350586923e0419d2ea4b559c37ec10ee4b6e210e08c14401f8e340f0da @@ -579,7 +579,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c 7dcb13bbcfd8c926546946556014c8f5aa0829eb8b65a6c18f8d187d265200a5 +F src/where.c 8e95858b398f7451c30ed05dd372f39767f04f423fa1d15a19d568268ae9a557 F src/whereInt.h b09753e74bf92a8b17cf0e41ca94c44432c454544be6699b5311dcc57bf229c6 F src/wherecode.c 3317f2b083a66d3e65a03edf316ade4ccb0a99c9956273282ebb579b95d4ba96 F src/whereexpr.c e90b2e76dcabc81edff56633bf281bc01d93b71e0c81482dc06925ce39f5844a @@ -1731,8 +1731,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb87c054b1b76959e46258ac66b24027f468b390a4148ac67f208a1fbeda4060 4b0b4e14039469b656662312a5f80f086ede293e9ad04c7bc99a202b683a1e55 -R fe7d026f8be3134e2723fc39eeec5ed5 -T +closed 4b0b4e14039469b656662312a5f80f086ede293e9ad04c7bc99a202b683a1e55 +P 09fffbdf9f2f6ce31a22d5a6df7a45f19a16628da622f12d6e33171cce09fb21 +R e63ef1b65ea2f09e13127dc0f6a98d71 +T *branch * in-scan-vs-index +T *sym-in-scan-vs-index * +T -sym-trunk * U drh -Z 8a623af2a2288d1935dd9b3507daa5d8 +Z f877a3ca3a67b98847cd7a3f31dd6a75 diff --git a/manifest.uuid b/manifest.uuid index 9bc93f1fe0..6f59a009e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09fffbdf9f2f6ce31a22d5a6df7a45f19a16628da622f12d6e33171cce09fb21 \ No newline at end of file +1fa40a78fef4516c39b217bff67efe7e7d2077cca00aae0ef5c2c9cff94f008b \ No newline at end of file diff --git a/src/where.c b/src/where.c index 10cb138143..9acfbcf568 100644 --- a/src/where.c +++ b/src/where.c @@ -2451,7 +2451,7 @@ static int whereLoopAddBtreeIndex( if( eOp & WO_IN ){ Expr *pExpr = pTerm->pExpr; - pNew->wsFlags |= WHERE_COLUMN_IN; + LogEst M, logK; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ int i; @@ -2471,6 +2471,36 @@ static int whereLoopAddBtreeIndex( assert( nIn>0 ); /* RHS always has 2 or more terms... The parser ** changes "x IN (?)" into "x=?". */ } + /* Let: + ** N = the total number of rows in the table + ** K = the number of entries on the right-hand side of the IN operator + ** M = the number of rows in the table that match terms to the + ** to the left in the same index. If the IN operator is on + ** the left-most index column, M==N. + ** + ** Given the definitions above, it is better to omit the IN operator + ** from the index lookup and instead do a scan of the M elements, + ** testing each scanned row against the IN operator separately, if: + ** + ** M*log(K) < K*log(N) + ** + ** Our estimates for M, K, and N might be inaccurate, so we build in + ** a safety margin of 2 (LogEst: 10) that favors using the IN operator + ** with the index, as using an index has better worst-case behavior. + */ + M = pProbe->aiRowLogEst[saved_nEq+1]; + logK = sqlite3LogEst(nIn); + if( M + logK + 10 < nIn + rLogSize ){ + WHERETRACE(0x40, + ("IN operator costs more than scan on column %d of \"%s\" (%d<%d)\n", + saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + continue; + }else{ + WHERETRACE(0x40, + ("IN operator cheaper than scan on column %d of \"%s\" (%d>=%d)\n", + saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + } + pNew->wsFlags |= WHERE_COLUMN_IN; }else if( eOp & (WO_EQ|WO_IS) ){ int iCol = pProbe->aiColumn[saved_nEq]; pNew->wsFlags |= WHERE_COLUMN_EQ;