From: drh Date: Fri, 4 Oct 2013 02:36:19 +0000 (+0000) Subject: Progress toward using the iScanRatio information on indices. Many tests X-Git-Tag: version-3.8.1~39^2~18 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=d9e3cad2f8d1f8cad8d4925daff5248b9964ba18;p=thirdparty%2Fsqlite.git Progress toward using the iScanRatio information on indices. Many tests are still failing. FossilOrigin-Name: 6c352edbba85a15ca356b5e131f4b3b2723d1774 --- diff --git a/manifest b/manifest index c1cf71301a..31dd5afb71 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\sbranch\sallowing\sdifferent\spostulated\sscan\srates\sfor\seach\sindex. -D 2013-10-03T19:21:41.923 +C Progress\stoward\susing\sthe\siScanRatio\sinformation\son\sindices.\s\sMany\stests\nare\sstill\sfailing. +D 2013-10-04T02:36:19.375 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 41caf5d01c2aefe442a52dbceff81095260e4c78 +F src/analyze.c 6aa46f043f495c337238630f219d6fcc74f87d9f F src/attach.c 0a17c9364895316ca4f52d06a97a72c0af1ae8b3 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 2f1987981139bd2f6d8c728d64bf09fb387443c3 @@ -216,7 +216,7 @@ F src/printf.c da9119eb31a187a4b99f60aa4a225141c0ebb74b F src/random.c 0b2dbc37fdfbfa6bd455b091dfcef5bdb32dba68 F src/resolve.c 7459801d02997b07e8b8da85ef255392ba1d022b F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c fc60e8e539cb9a895cac197de95048759b0f3ab0 +F src/select.c 340aa50982b61c4e41bc2416a0caddda5d4499f0 F src/shell.c 5ee50ca3e35453bbd6ccdf1bdd0f6bbe9738e9fb F src/sqlite.h.in ec40aa958a270416fb04b4f72210357bf163d2c5 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -290,7 +290,7 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c d8c2c5ff3af348870cb56d4a577a51625640563d +F src/where.c 903e295f949590760aabef7203956f86588859f2 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -306,7 +306,7 @@ F test/analyze.test 1772936d66471c65221e437b6d1999c3a03166c4 F test/analyze3.test 412f690dfe95b337475e3e78a84a85d25f6f125d F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 -F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab +F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 F test/analyze9.test 1b9b7e9a096d1536f03d9ad7b72f638ef5669347 @@ -446,7 +446,7 @@ F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test d9e7ad4e7bd36d976f1b631f76e6e6ea090c41a0 +F test/eqp.test d4386348af3088a01fd21fb27badaa553c3ba7bd F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 @@ -1119,10 +1119,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 500c5932fe3f5fcd0940522f7839d581c555e0eb -R c43f0933dc867b5f8554e92f9664b3d0 -T *branch * index-scan-rate -T *sym-index-scan-rate * -T -sym-trunk * +P d59d97b0a8d70dc31d45db61bbc11ebb5375a224 +R 753a9b9a56a7c0c63483772726286f21 U drh -Z ac91788b1f42a4c56b173a94be0ebf5a +Z b513b854bdc830eeffd07ca885795f7d diff --git a/manifest.uuid b/manifest.uuid index 7697e0083f..fe8c52efa1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d59d97b0a8d70dc31d45db61bbc11ebb5375a224 \ No newline at end of file +6c352edbba85a15ca356b5e131f4b3b2723d1774 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 83c0950249..cb3c394938 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1282,8 +1282,13 @@ static void decodeIntArray( pIndex->bUnordered = 1; }else if( i>2 && memcmp(z, "r=", 2)==0 && sqlite3GetInt32(z+2, &v32) ){ - if( v32>255 ) v32 = 255; - if( v32<0 ) v32 = 0; + if( v32>=200 ){ + v32 = 255; + }else if( v32<0 ){ + v32 = 0; + }else{ + v32 = (128*v32)/100; + } pIndex->iScanRatio = (u8)v32; } z += i; diff --git a/src/select.c b/src/select.c index 2ea8e1fc88..caa069e034 100644 --- a/src/select.c +++ b/src/select.c @@ -4597,13 +4597,7 @@ int sqlite3Select( sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - /* Search for the index that has the least amount of columns. If - ** there is such an index, and it has less columns than the table - ** does, then we can assume that it consumes less space on disk and - ** will therefore be cheaper to scan to determine the query result. - ** In this case set iRoot to the root page number of the index b-tree - ** and pKeyInfo to the KeyInfo structure required to navigate the - ** index. + /* Search for the index that has the lowest scan cost. ** ** (2011-04-15) Do not do a full scan of an unordered index. ** @@ -4611,11 +4605,14 @@ int sqlite3Select( ** passed to keep OP_OpenRead happy. */ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->bUnordered==0 && (!pBest || pIdx->nColumnnColumn) ){ + if( pIdx->bUnordered==0 + && pIdx->iScanRatio<128 + && (!pBest || pIdx->iScanRatioiScanRatio) + ){ pBest = pIdx; } } - if( pBest && pBest->nColumnnCol ){ + if( pBest ){ iRoot = pBest->tnum; pKeyInfo = sqlite3IndexKeyinfo(pParse, pBest); } diff --git a/src/where.c b/src/where.c index 8a15a8642b..1e0f6e2de6 100644 --- a/src/where.c +++ b/src/where.c @@ -4669,10 +4669,8 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; /* TUNING: Cost of full table scan is 3*(N + log2(N)). - ** + The extra 3 factor is to encourage the use of indexed lookups - ** over full scans. A smaller constant 2 is used for covering - ** index scans so that a covering index scan will be favored over - ** a table scan. */ + ** + The extra 4 factor is to encourage the use of indexed lookups + ** over full scans. */ pNew->rRun = whereCostAdd(rSize,rLogSize) + 16; whereLoopOutputAdjust(pWC, pNew, pSrc->iCursor); rc = whereLoopInsert(pBuilder, pNew); @@ -4686,6 +4684,7 @@ static int whereLoopAddBtree( if( b || ( m==0 && pProbe->bUnordered==0 + && pProbe->iScanRatio<128 && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 && sqlite3GlobalConfig.bUseCis && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan) @@ -4693,15 +4692,12 @@ static int whereLoopAddBtree( ){ pNew->iSortIdx = b ? iSortIdx : 0; if( m==0 ){ - /* TUNING: Cost of a covering index scan is 2*(N + log2(N)). - ** + The extra 2 factor is to encourage the use of indexed lookups - ** over index scans. A table scan uses a factor of 3 so that - ** index scans are favored over table scans. - ** + If this covering index might also help satisfy the ORDER BY - ** clause, then the cost is fudged down slightly so that this - ** index is favored above other indices that have no hope of - ** helping with the ORDER BY. */ - pNew->rRun = 10 + whereCostAdd(rSize,rLogSize) - b; + /* TUNING: Cost of a covering index scan is K*(N + log2(N)). + ** + The extra factor K of between 1.0 and 3.0 is added to + ** encourage the use of indexed lookups. The value of K + ** depends on the iScanRatio value for the index. + */ + pNew->rRun = whereCostAdd(rSize,rLogSize) + pProbe->iScanRatio/9 + 1; }else{ assert( b!=0 ); /* TUNING: Cost of scanning a non-covering index is (N+1)*log2(N) diff --git a/test/analyze6.test b/test/analyze6.test index d2c96518be..f77f05324a 100644 --- a/test/analyze6.test +++ b/test/analyze6.test @@ -30,14 +30,14 @@ proc eqp {sql {db db}} { do_test analyze6-1.0 { db eval { - CREATE TABLE cat(x INT); + CREATE TABLE cat(x INT, yz TEXT); CREATE UNIQUE INDEX catx ON cat(x); /* Give cat 16 unique integers */ - INSERT INTO cat VALUES(1); - INSERT INTO cat VALUES(2); - INSERT INTO cat SELECT x+2 FROM cat; - INSERT INTO cat SELECT x+4 FROM cat; - INSERT INTO cat SELECT x+8 FROM cat; + INSERT INTO cat(x) VALUES(1); + INSERT INTO cat(x) VALUES(2); + INSERT INTO cat(x) SELECT x+2 FROM cat; + INSERT INTO cat(x) SELECT x+4 FROM cat; + INSERT INTO cat(x) SELECT x+8 FROM cat; CREATE TABLE ev(y INT); CREATE INDEX evy ON ev(y); diff --git a/test/eqp.test b/test/eqp.test index 480bcd1501..89bda719e3 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -33,11 +33,11 @@ set testprefix eqp proc det {args} { uplevel do_eqp_test $args } do_execsql_test 1.1 { - CREATE TABLE t1(a, b); + CREATE TABLE t1(a INT, b INT, ex TEXT); CREATE INDEX i1 ON t1(a); CREATE INDEX i2 ON t1(b); - CREATE TABLE t2(a, b); - CREATE TABLE t3(a, b); + CREATE TABLE t2(a INT, b INT, ex TEXT); + CREATE TABLE t3(a INT, b INT, ex TEXT); } do_eqp_test 1.2 { @@ -122,9 +122,9 @@ do_eqp_test 1.11 { # drop_all_tables do_execsql_test 2.1 { - CREATE TABLE t1(x, y); + CREATE TABLE t1(x INT, y INT, ex TEXT); - CREATE TABLE t2(x, y); + CREATE TABLE t2(x INT, y INT, ex TEXT); CREATE INDEX t2i1 ON t2(x); } @@ -374,7 +374,7 @@ drop_all_tables # FROM t1 WHERE a=1; # 0|0|0|SCAN TABLE t1 # -do_execsql_test 5.1.0 { CREATE TABLE t1(a, b) } +do_execsql_test 5.1.0 { CREATE TABLE t1(a INT, b INT, ex TEXT) } det 5.1.1 "SELECT a, b FROM t1 WHERE a=1" { 0 0 0 {SCAN TABLE t1} } @@ -402,8 +402,8 @@ det 5.3.1 "SELECT a, b FROM t1 WHERE a=1" { # 0|0|0|SEARCH TABLE t1 USING COVERING INDEX i2 (a=? AND b>?) # 0|1|1|SCAN TABLE t2 # -do_execsql_test 5.4.0 {CREATE TABLE t2(c, d)} -det 5.4.1 "SELECT t1.*, t2.* FROM t1, t2 WHERE t1.a=1 AND t1.b>2" { +do_execsql_test 5.4.0 {CREATE TABLE t2(c INT, d INT, ex TEXT)} +det 5.4.1 "SELECT t1.a, t2.c FROM t1, t2 WHERE t1.a=1 AND t1.b>2" { 0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=? AND b>?)} 0 1 1 {SCAN TABLE t2} } @@ -413,7 +413,7 @@ det 5.4.1 "SELECT t1.*, t2.* FROM t1, t2 WHERE t1.a=1 AND t1.b>2" { # 0|0|1|SEARCH TABLE t1 USING COVERING INDEX i2 (a=? AND b>?) # 0|1|0|SCAN TABLE t2 # -det 5.5 "SELECT t1.*, t2.* FROM t2, t1 WHERE t1.a=1 AND t1.b>2" { +det 5.5 "SELECT t1.a, t2.c FROM t2, t1 WHERE t1.a=1 AND t1.b>2" { 0 0 1 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=? AND b>?)} 0 1 0 {SCAN TABLE t2} } @@ -424,7 +424,7 @@ det 5.5 "SELECT t1.*, t2.* FROM t2, t1 WHERE t1.a=1 AND t1.b>2" { # 0|0|0|SEARCH TABLE t1 USING INDEX i3 (b=?) # do_execsql_test 5.5.0 {CREATE INDEX i3 ON t1(b)} -det 5.6.1 "SELECT * FROM t1 WHERE a=1 OR b=2" { +det 5.6.1 "SELECT a, b FROM t1 WHERE a=1 OR b=2" { 0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i2 (a=?)} 0 0 0 {SEARCH TABLE t1 USING INDEX i3 (b=?)} } @@ -485,7 +485,7 @@ det 5.10 { # 0|0|0|SEARCH TABLE t2 USING INDEX i4 (c=?) # 0|1|1|SCAN TABLE t1 # -det 5.11 "SELECT * FROM (SELECT * FROM t2 WHERE c=1), t1" { +det 5.11 "SELECT a, b FROM (SELECT * FROM t2 WHERE c=1), t1" { 0 0 0 {SEARCH TABLE t2 USING INDEX i4 (c=?)} 0 1 1 {SCAN TABLE t1 USING COVERING INDEX i2} } @@ -563,8 +563,8 @@ do_peqp_test 6.1 { drop_all_tables do_execsql_test 7.0 { - CREATE TABLE t1(a, b); - CREATE TABLE t2(a, b); + CREATE TABLE t1(a INT, b INT, ex TEXT); + CREATE TABLE t2(a INT, b INT, ex TEXT); CREATE INDEX i1 ON t2(a); }