From d8b77e20fc9bde3a55c1355705287cc23b203b95 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Sep 2014 01:35:57 +0000 Subject: [PATCH] Query planner heuristic update: When doing a full table scan on a table that has an equality constraint on an unindexed column, do not allow the estimated number of output rows to be greater than half the total number of rows in the table. FossilOrigin-Name: 73954f93c4c6f880c6e01d0d130e3fed40fd4106 --- manifest | 18 +-- manifest.uuid | 2 +- src/sqliteInt.h | 1 - src/where.c | 34 ++++-- test/whereJ.test | 303 ++++++++++++++++++++++++++++++++++++++++------- 5 files changed, 297 insertions(+), 61 deletions(-) diff --git a/manifest b/manifest index 8ae6336224..cf1f0a6433 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning. -D 2014-09-05T05:58:37.378 +C Query\splanner\sheuristic\supdate:\nWhen\sdoing\sa\sfull\stable\sscan\son\sa\stable\sthat\shas\san\sequality\sconstraint\son\nan\sunindexed\scolumn,\sdo\snot\sallow\sthe\sestimated\snumber\sof\soutput\srows\sto\nbe\sgreater\sthan\shalf\sthe\stotal\snumber\sof\srows\sin\sthe\stable. +D 2014-09-06T01:35:57.122 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50 F src/sqlite.h.in 43852c8b68b4c579948cb37182918078836c5c06 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 6244ee9052752e26d1275ab20c9b774385aa57d2 +F src/sqliteInt.h dbc9a4984a7e40f4334f8aefd3505f5aef4650a8 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -298,7 +298,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c d9eae96b2cbbe4842eac3ee156ccd1b933d802c4 +F src/where.c 738213f25bc74c0a7cc5427712c1ac7537270a3d F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1125,7 +1125,7 @@ F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test 69f5ec4b15760a8c860f80e2d55525669390aab3 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test 1d89199697919d4930be05a71e7fe620f114e622 -F test/whereJ.test 8880784c211c459595f734a35bcc5f2061fce987 +F test/whereJ.test 1c35169106be343b7aa1a2c1338322462f27901a F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9779c7a9eb1e2bd36e9286331a9314f064014d80 -R 05490459da8c4172c4abbda0933bfe68 -U mistachkin -Z b93a43f7e285c77b049b454cce5380d9 +P 733119067757814609a9cea6b975818607bee4e3 +R bfd7d8b21e261616a30a072bf666f263 +U drh +Z 6641f77f6592cd795a1a1b9c2011a8f5 diff --git a/manifest.uuid b/manifest.uuid index fb28827b7e..a84ea01b2d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -733119067757814609a9cea6b975818607bee4e3 \ No newline at end of file +73954f93c4c6f880c6e01d0d130e3fed40fd4106 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7fd999d9ee..e56da4edd0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1159,7 +1159,6 @@ struct sqlite3 { #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ #define SQLITE_Stat3 0x0800 /* Use the SQLITE_STAT3 table */ -#define SQLITE_AdjustOutEst 0x1000 /* Adjust output estimates using WHERE */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* diff --git a/src/where.c b/src/where.c index e1e1e1d528..49b16b345d 100644 --- a/src/where.c +++ b/src/where.c @@ -4221,14 +4221,16 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** the number of output rows by a factor of 10 and each additional term ** reduces the number of output rows by sqrt(2). */ -static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ +static void whereLoopOutputAdjust( + WhereClause *pWC, /* The WHERE clause */ + WhereLoop *pLoop, /* The loop to adjust downward */ + LogEst nRow /* Number of rows in the entire table */ +){ WhereTerm *pTerm, *pX; Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); int i, j; + int nEq = 0; /* Number of = constraints not within likely()/unlike() */ - if( !OptimizationEnabled(pWC->pWInfo->pParse->db, SQLITE_AdjustOutEst) ){ - return; - } for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; @@ -4240,9 +4242,21 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } if( j<0 ){ - pLoop->nOut += (pTerm->truthProb<=0 ? pTerm->truthProb : -1); + if( pTerm->truthProb<=0 ){ + pLoop->nOut += pTerm->truthProb; + }else{ + pLoop->nOut--; + if( pTerm->eOperator&WO_EQ ) nEq++; + } } } + /* TUNING: If there is at least one equality constraint in the WHERE + ** clause that does not have a likelihood() explicitly assigned to it + ** then do not let the estimated number of output rows exceed half + ** the number of rows in the table. */ + if( nEq && pLoop->nOut>nRow-10 ){ + pLoop->nOut = nRow - 10; + } } /* @@ -4288,6 +4302,7 @@ static int whereLoopAddBtreeIndex( LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ + LogEst rSize; /* Number of rows in the table */ LogEst rLogSize; /* Logarithm of table size */ WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ @@ -4317,7 +4332,8 @@ static int whereLoopAddBtreeIndex( saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; - rLogSize = estLog(pProbe->aiRowLogEst[0]); + rSize = pProbe->aiRowLogEst[0]; + rLogSize = estLog(rSize); /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average @@ -4494,7 +4510,7 @@ static int whereLoopAddBtreeIndex( nOutUnadjusted = pNew->nOut; pNew->rRun += nInMul + nIn; pNew->nOut += nInMul + nIn; - whereLoopOutputAdjust(pBuilder->pWC, pNew); + whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ @@ -4748,7 +4764,7 @@ static int whereLoopAddBtree( /* TUNING: Cost of full table scan is (N*3.0). */ pNew->rRun = rSize + 16; ApplyCostMultiplier(pNew->rRun, pTab->costMult); - whereLoopOutputAdjust(pWC, pNew); + whereLoopOutputAdjust(pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; if( rc ) break; @@ -4784,7 +4800,7 @@ static int whereLoopAddBtree( pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); } ApplyCostMultiplier(pNew->rRun, pTab->costMult); - whereLoopOutputAdjust(pWC, pNew); + whereLoopOutputAdjust(pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; if( rc ) break; diff --git a/test/whereJ.test b/test/whereJ.test index 7c37321cbf..fbc2e57a3f 100644 --- a/test/whereJ.test +++ b/test/whereJ.test @@ -373,50 +373,271 @@ do_execsql_test whereJ-2.2 { ############################################################################ -ifcapable stat4 { - # Create and populate table. - do_execsql_test 3.1 { CREATE TABLE t1(a, b, c) } - for {set i 0} {$i < 32} {incr i 2} { - for {set x 0} {$x < 100} {incr x} { - execsql { INSERT INTO t1 VALUES($i, $x, $c) } - incr c - } - execsql { INSERT INTO t1 VALUES($i+1, 5, $c) } +# Create and populate table. +do_execsql_test 3.1 { CREATE TABLE t1(a, b, c) } +for {set i 0} {$i < 32} {incr i 2} { + for {set x 0} {$x < 100} {incr x} { + execsql { INSERT INTO t1 VALUES($i, $x, $c) } incr c } - - do_execsql_test 3.2 { - SELECT a, count(*) FROM t1 GROUP BY a HAVING a < 8; - } { - 0 100 1 1 2 100 3 1 4 100 5 1 6 100 7 1 - } - - do_execsql_test 3.3 { - CREATE INDEX idx_ab ON t1(a, b); - CREATE INDEX idx_c ON t1(c); - ANALYZE; - } {} - - # This one should use index "idx_c". - do_eqp_test 3.4 { - SELECT * FROM t1 WHERE - a = 4 AND b BETWEEN 20 AND 80 -- Matches 80 rows - AND - c BETWEEN 150 AND 160 -- Matches 10 rows - } { - 0 0 0 {SEARCH TABLE t1 USING INDEX idx_c (c>? AND c? AND b? AND c? AND b