From: drh Date: Fri, 8 Jun 2018 21:21:01 +0000 (+0000) Subject: Only choose to scan an IN operator rather than use an index if we have X-Git-Tag: version-3.25.0~205^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=6d6decb8d92ea6422f301ffa6dfbb1015aec8110;p=thirdparty%2Fsqlite.git Only choose to scan an IN operator rather than use an index if we have real STAT1 data to suggest it is advantageous. FossilOrigin-Name: 30e874661dcc1a2ecb40df2ef74582151d85bb36c754a38548829a3b6285f18d --- diff --git a/manifest b/manifest index 363f32cb2e..bb077b2153 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sbtreeNext()\sassertion\sbug\sfix\sfrom\strunk. -D 2018-06-08T19:54:07.050 +C Only\schoose\sto\sscan\san\sIN\soperator\srather\sthan\suse\san\sindex\sif\swe\shave\nreal\sSTAT1\sdata\sto\ssuggest\sit\sis\sadvantageous. +D 2018-06-08T21:21:01.417 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in bfc40f350586923e0419d2ea4b559c37ec10ee4b6e210e08c14401f8e340f0da @@ -579,7 +579,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c 8e95858b398f7451c30ed05dd372f39767f04f423fa1d15a19d568268ae9a557 +F src/where.c e4d48338ca4718c8034f313cca202cc5ca75639a24d27b959b65578198f63c81 F src/whereInt.h b09753e74bf92a8b17cf0e41ca94c44432c454544be6699b5311dcc57bf229c6 F src/wherecode.c 3317f2b083a66d3e65a03edf316ade4ccb0a99c9956273282ebb579b95d4ba96 F src/whereexpr.c e90b2e76dcabc81edff56633bf281bc01d93b71e0c81482dc06925ce39f5844a @@ -967,7 +967,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068 F test/in5.test 7ae37fcd4a5e198291c6ab5f31a5bb3d15397efe8b75a6736d7a95a7b8dd9e08 -F test/in6.test 77c3e1d356d8aeb0864051f0677d3c0a032cf97b7f33a0ba8fa2b04a663f6b7b +F test/in6.test 62d943a02f722948f4410ee0b53c3cb39acd7c41afb083df8d7004238fe90a20 F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822 F test/incrblob2.test a494c9e848560039a23974b9119cfc2cf3ad3bd15cc2694ee6367ae537ef8f1f F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 @@ -1185,7 +1185,7 @@ F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d F test/rowvalue.test ef851a80f7e6de93b51caca9e4b6b7d2dcd540bbcca7d51860e80435b8b4c0de F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256 -F test/rowvalue4.test cbd1cead27a797d11ec93301fd801c89e97eb1809b3d984b7f16a3876e362eac +F test/rowvalue4.test 2b20468da3775aba971caf3158e9696a4d99c69a7623fb495f332a596daebbee F test/rowvalue5.test c81c7d8cf36711ab37675ad7376084ae2a359cb6 F test/rowvalue6.test d19b54feb604d5601f8614b15e214e0774c01087 F test/rowvalue7.test 5d06ff19d9e6969e574a2e662a531dd0c67801a8 @@ -1731,7 +1731,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1fa40a78fef4516c39b217bff67efe7e7d2077cca00aae0ef5c2c9cff94f008b 99057383acc8f92093530e216c621d40386a06fe98131ff0af6df524d80a6410 -R 56ae5d735c58aa67ec9684ecac6c800d +P 11bd66e09035039fbfc4c5631196962dd3f2e6634f5123a7b2fec17aeaaab07b +R 54c7353f86aece043f7640340784a722 U drh -Z b50b0532bda90928b75cfc1466c6bb97 +Z b0c1273ce7490f506712c4b76e5917fd diff --git a/manifest.uuid b/manifest.uuid index 481057e482..764612de10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11bd66e09035039fbfc4c5631196962dd3f2e6634f5123a7b2fec17aeaaab07b \ No newline at end of file +30e874661dcc1a2ecb40df2ef74582151d85bb36c754a38548829a3b6285f18d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9acfbcf568..7491027f68 100644 --- a/src/where.c +++ b/src/where.c @@ -2471,34 +2471,38 @@ static int whereLoopAddBtreeIndex( assert( nIn>0 ); /* RHS always has 2 or more terms... The parser ** changes "x IN (?)" into "x=?". */ } - /* Let: - ** N = the total number of rows in the table - ** K = the number of entries on the right-hand side of the IN operator - ** M = the number of rows in the table that match terms to the - ** to the left in the same index. If the IN operator is on - ** the left-most index column, M==N. - ** - ** Given the definitions above, it is better to omit the IN operator - ** from the index lookup and instead do a scan of the M elements, - ** testing each scanned row against the IN operator separately, if: - ** - ** M*log(K) < K*log(N) - ** - ** Our estimates for M, K, and N might be inaccurate, so we build in - ** a safety margin of 2 (LogEst: 10) that favors using the IN operator - ** with the index, as using an index has better worst-case behavior. - */ - M = pProbe->aiRowLogEst[saved_nEq+1]; - logK = sqlite3LogEst(nIn); - if( M + logK + 10 < nIn + rLogSize ){ - WHERETRACE(0x40, - ("IN operator costs more than scan on column %d of \"%s\" (%d<%d)\n", - saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); - continue; - }else{ - WHERETRACE(0x40, - ("IN operator cheaper than scan on column %d of \"%s\" (%d>=%d)\n", - saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + if( pProbe->hasStat1 ){ + /* Let: + ** N = the total number of rows in the table + ** K = the number of entries on the RHS of the IN operator + ** M = the number of rows in the table that match terms to the + ** to the left in the same index. If the IN operator is on + ** the left-most index column, M==N. + ** + ** Given the definitions above, it is better to omit the IN operator + ** from the index lookup and instead do a scan of the M elements, + ** testing each scanned row against the IN operator separately, if: + ** + ** M*log(K) < K*log(N) + ** + ** Our estimates for M, K, and N might be inaccurate, so we build in + ** a safety margin of 2 (LogEst: 10) that favors using the IN operator + ** with the index, as using an index has better worst-case behavior. + ** If we do not have real sqlite_stat1 data, always prefer to use + ** the index. + */ + M = pProbe->aiRowLogEst[saved_nEq]; + logK = estLog(nIn); + if( M + logK + 10 < nIn + rLogSize ){ + WHERETRACE(0x40, + ("Scan preferred over IN operator on column %d of \"%s\" (%d<%d)\n", + saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + continue; + }else{ + WHERETRACE(0x40, + ("IN operator preferred on column %d of \"%s\" (%d>=%d)\n", + saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + } } pNew->wsFlags |= WHERE_COLUMN_IN; }else if( eOp & (WO_EQ|WO_IS) ){ diff --git a/test/in6.test b/test/in6.test index ad16c4e8a8..773ee589d4 100644 --- a/test/in6.test +++ b/test/in6.test @@ -28,6 +28,9 @@ do_test in6-1.1 { INSERT INTO t1(a,b,c,d) SELECT 100, 200+x/2, 300+x/5, x FROM c; CREATE INDEX t1abc ON t1(a,b,c); + ANALYZE; + UPDATE sqlite_stat1 SET stat='1000000 500000 500 50'; + ANALYZE sqlite_master; } set ::sqlite_search_count 0 db eval { diff --git a/test/rowvalue4.test b/test/rowvalue4.test index ce025fc923..5c0d170f59 100644 --- a/test/rowvalue4.test +++ b/test/rowvalue4.test @@ -224,7 +224,7 @@ do_execsql_test 5.0 { WITH i(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM i WHERE i<1000 ) - INSERT INTO d2 SELECT i/3, i%3, i/3 FROM i; + INSERT INTO d2 SELECT i/100, i%100, i/100 FROM i; ANALYZE; }