From: drh Date: Tue, 8 Oct 2013 18:40:37 +0000 (+0000) Subject: Further refinement of the idea of multiplying run-time cost estimates by X-Git-Tag: version-3.8.1~39^2~5 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=186ad8cc3294c42f6b52144ec1d51b93dbfb2105;p=thirdparty%2Fsqlite.git Further refinement of the idea of multiplying run-time cost estimates by the estimated row size. FossilOrigin-Name: 18bd6ba96d19de6047baebfa15b1f739577c9ec4 --- diff --git a/manifest b/manifest index 623db16001..3f72457301 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Multiply\sall\scursor\sstep\scost\sestimates\sby\sthe\sestimated\ssize\sof\sthe\srow\sin\nbytes,\sin\sorder\sto\sget\sthe\squery\splanner\sot\smake\suse\sof\sestimated\srow\ssizes.\nThis\scheck-in\suses\smagic\snumbers\sin\sa\sfew\splaces\s(for\sexample,\sestimates\sof\nthe\ssize\sof\soutput\srows)\sand\sneeds\slots\sof\srefinement.\s\sConsider\sthis\sa\nproof-of-concept\sonly. -D 2013-10-07T17:32:15.338 +C Further\srefinement\sof\sthe\sidea\sof\smultiplying\srun-time\scost\sestimates\sby\nthe\sestimated\srow\ssize. +D 2013-10-08T18:40:37.532 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c d5720cbb21ae56e7e5b07847e05e5b203818acac F src/btree.h bfe0e8c5759b4ec77b0d18390064a6ef3cdffaaf F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 -F src/build.c ea07ec35354cdcd017b128679391320220030336 +F src/build.c 3da02c07b0f198a11ce3766cd34eac311656f1e8 F src/callback.c f99a8957ba2adf369645fac0db09ad8adcf1caa2 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c ea4b7f3623a0fcb1146e7f245d7410033e86859c @@ -216,7 +216,7 @@ F src/printf.c da9119eb31a187a4b99f60aa4a225141c0ebb74b F src/random.c 0b2dbc37fdfbfa6bd455b091dfcef5bdb32dba68 F src/resolve.c 7459801d02997b07e8b8da85ef255392ba1d022b F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 13be733297f415b388444a2e296d23569cd83b8a +F src/select.c 9d111a1adef56151b4bbddbcaea6d117d374e17d F src/shell.c 5ee50ca3e35453bbd6ccdf1bdd0f6bbe9738e9fb F src/sqlite.h.in ec40aa958a270416fb04b4f72210357bf163d2c5 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -290,7 +290,7 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c ad5e680c0b95014bb9e227990d6b0efe634e7749 +F src/where.c 2a04ab5856f8c964369d181f3580a8d14e4a325c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -437,7 +437,7 @@ F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 F test/e_insert.test 291e056e1a442a5e5166a989a8a03a46e38225ca F test/e_reindex.test e175794fc41f8e8aef34772e87a7d7b7a9251dd3 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 -F test/e_select.test f2358d074bd82240bc79a8348f284a2a8909dc1f +F test/e_select.test d3226cb94fae4af3f198e68e71f655e106d0be47 F test/e_select2.test 22c660a7becf0712c95e1ca1b2d9e716a1261460 F test/e_update.test bea00499e43ee1da77b03cdb0b20c7c864c1ec5a F test/e_uri.test a2c92d80093a7efdcfbb11093651cbea87097b6b @@ -1059,7 +1059,7 @@ F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5a4b0abc207d71da4deecd734ad8579e8dd40aa8 F test/where8.test 6f95896633cf2d307b5263145b942b7d33e837c6 F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 -F test/where9.test 4f3eab951353a3ae164befc521c777dfa903e46c +F test/where9.test 06c5f1e13168239951d2426078334d97fdf2d26f F test/whereA.test 4d253178d135ec46d1671e440cd8f2b916aa6e6b F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a @@ -1121,7 +1121,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1d7b2dc0eae70c0c0e523b715acf758bb4cfa9ac -R b3d2afdb618d61f33659270b34382478 +P cb34cfe57c2a664fbfae8106e95114400ea222d5 +R 2b7915a92bd0315e781ed0d1390f4f69 U drh -Z 2fed1ec905830e828c488397bcac95d0 +Z 27bbb4f4b3fe1d2932c4d2e467493b71 diff --git a/manifest.uuid b/manifest.uuid index b45f4b6ece..d8fa3b1ba2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb34cfe57c2a664fbfae8106e95114400ea222d5 \ No newline at end of file +18bd6ba96d19de6047baebfa15b1f739577c9ec4 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 3c8d22d3cd..e14988ae1f 100644 --- a/src/build.c +++ b/src/build.c @@ -879,7 +879,7 @@ void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; - pTable->nRowEst = 1000000; + pTable->nRowEst = 1048576; assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; diff --git a/src/select.c b/src/select.c index f9c7835fd2..cbf8e14cf3 100644 --- a/src/select.c +++ b/src/select.c @@ -1424,8 +1424,7 @@ static int selectColumnsFromExprList( */ static void selectAddColumnTypeAndCollation( Parse *pParse, /* Parsing contexts */ - int nCol, /* Number of columns */ - Column *aCol, /* List of columns */ + Table *pTab, /* Add column type information to this table */ Select *pSelect /* SELECT used to determine types and collations */ ){ sqlite3 *db = pParse->db; @@ -1435,17 +1434,20 @@ static void selectAddColumnTypeAndCollation( int i; Expr *p; struct ExprList_item *a; + u64 szAll = 0; assert( pSelect!=0 ); assert( (pSelect->selFlags & SF_Resolved)!=0 ); - assert( nCol==pSelect->pEList->nExpr || db->mallocFailed ); + assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed ); if( db->mallocFailed ) return; memset(&sNC, 0, sizeof(sNC)); sNC.pSrcList = pSelect->pSrc; a = pSelect->pEList->a; - for(i=0, pCol=aCol; iaCol; inCol; i++, pCol++){ p = a[i].pExpr; pCol->zType = sqlite3DbStrDup(db, columnType(&sNC, p, 0, 0, 0)); + sqlite3AffinityType(pCol->zType, &pCol->szEst); + szAll += pCol->szEst; pCol->affinity = sqlite3ExprAffinity(p); if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_NONE; pColl = sqlite3ExprCollSeq(pParse, p); @@ -1453,6 +1455,7 @@ static void selectAddColumnTypeAndCollation( pCol->zColl = sqlite3DbStrDup(db, pColl->zName); } } + pTab->szTabRow = sqlite3LogEst(szAll*4); } /* @@ -1480,9 +1483,9 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ assert( db->lookaside.bEnabled==0 ); pTab->nRef = 1; pTab->zName = 0; - pTab->nRowEst = 1000000; + pTab->nRowEst = 1048576; selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSelect); + selectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); @@ -3394,11 +3397,11 @@ static int selectExpander(Walker *pWalker, Select *p){ pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; pTab->nRef = 1; - pTab->zName = sqlite3MPrintf(db, "sqlite_subquery_%p_", (void*)pTab); + pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; - pTab->nRowEst = 1000000; + pTab->nRowEst = 1048576; pTab->tabFlags |= TF_Ephemeral; #endif }else{ @@ -3682,7 +3685,7 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Select *pSel = pFrom->pSelect; assert( pSel ); while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSel); + selectAddColumnTypeAndCollation(pParse, pTab, pSel); } } } diff --git a/src/where.c b/src/where.c index 3cf09a8179..b576d4f2b7 100644 --- a/src/where.c +++ b/src/where.c @@ -2450,15 +2450,15 @@ static int whereRangeScanEst( WhereLoopBuilder *pBuilder, WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - LogEst *pnOut /* IN/OUT: Number of rows visited */ + WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */ ){ int rc = SQLITE_OK; - int nOut = (int)*pnOut; + int nOut = pLoop->nOut; + int nEq = pLoop->u.btree.nEq; LogEst nNew; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - Index *p = pBuilder->pNew->u.btree.pIndex; - int nEq = pBuilder->pNew->u.btree.nEq; + Index *p = pLoop->u.btree.pIndex; if( p->nSample>0 && nEq==pBuilder->nRecValid @@ -2546,7 +2546,7 @@ static int whereRangeScanEst( if( nNewnOut = (LogEst)nOut; WHERETRACE(0x100, ("range scan regions: %u..%u est=%d\n", (u32)iLower, (u32)iUpper, nOut)); return SQLITE_OK; @@ -2570,7 +2570,7 @@ static int whereRangeScanEst( } if( nNew<10 ) nNew = 10; if( nNewnOut = (LogEst)nOut; return rc; } @@ -4337,7 +4337,13 @@ static int whereLoopAddBtreeIndex( if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ /* Adjust nOut and rRun for STAT3 range values */ assert( pNew->nOut==saved_nOut ); - whereRangeScanEst(pParse, pBuilder, pBtm, pTop, &pNew->nOut); + whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); + + /* If the range constraint is the only constraint on the index and + ** if the range constraint does not reduce the search space, + ** then this is really just an index scan which has already + ** been analyzed. */ + if( pNew->nOut>=saved_nOut && pNew->u.btree.nEq==0 ) continue; } #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( nInMul==0 @@ -4816,6 +4822,8 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pWCEnd = pWC->a + pWC->nTerm; pNew = pBuilder->pNew; memset(&sSum, 0, sizeof(sSum)); + pItem = pWInfo->pTabList->a + pNew->iTab; + iCur = pItem->iCursor; for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 @@ -4827,8 +4835,6 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ int once = 1; int i, j; - pItem = pWInfo->pTabList->a + pNew->iTab; - iCur = pItem->iCursor; sSubBuild = *pBuilder; sSubBuild.pOrderBy = 0; sSubBuild.pOrSet = &sCur; @@ -5266,8 +5272,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( pWInfo->pOrderBy==0 || nRowEst==0 ){ aFrom[0].isOrderedValid = 1; }else{ - /* TUNING: Estimated cost of sorting is N*log2(N) where N is the - ** number of output rows. */ + /* TUNING: Estimated cost of sorting is 48*N*log2(N) where N is the + ** number of output rows. The 48 is the expected size of a row to sort. + ** FIXME: compute a better estimate of the 48 multiplier based on the + ** result set expressions. */ rSortCost = nRowEst + estLog(nRowEst) + 55; WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost)); } diff --git a/test/e_select.test b/test/e_select.test index 108b6b1540..61646a2a31 100644 --- a/test/e_select.test +++ b/test/e_select.test @@ -450,17 +450,17 @@ do_join_test e_select-1.4.1.4 { # left-hand and right-hand datasets. # do_join_test e_select-1.4.2.1 { - SELECT * FROM x2 %JOIN% x3 + SELECT * FROM x2 %JOIN% x3 ORDER BY +c, +f } [list -60.06 {} {} -39.24 {} encompass -1 \ - -60.06 {} {} presenting 51 reformation dignified \ - -60.06 {} {} conducting -87.24 37.56 {} \ - -60.06 {} {} coldest -96 dramatists 82.3 \ -60.06 {} {} alerting {} -93.79 {} \ + -60.06 {} {} coldest -96 dramatists 82.3 \ + -60.06 {} {} conducting -87.24 37.56 {} \ + -60.06 {} {} presenting 51 reformation dignified \ -58 {} 1.21 -39.24 {} encompass -1 \ - -58 {} 1.21 presenting 51 reformation dignified \ - -58 {} 1.21 conducting -87.24 37.56 {} \ - -58 {} 1.21 coldest -96 dramatists 82.3 \ -58 {} 1.21 alerting {} -93.79 {} \ + -58 {} 1.21 coldest -96 dramatists 82.3 \ + -58 {} 1.21 conducting -87.24 37.56 {} \ + -58 {} 1.21 presenting 51 reformation dignified \ ] # TODO: Come back and add a few more like the above. diff --git a/test/where9.test b/test/where9.test index 1c32ff82f7..963d3bc1b7 100644 --- a/test/where9.test +++ b/test/where9.test @@ -781,27 +781,7 @@ do_test where9-6.8.2 { OR (b NOT NULL AND c NOT NULL AND d IS NULL) } } {1 {no query solution}} -ifcapable stat4||stat3 { - # When STAT3 is enabled, the "b NOT NULL" terms get translated - # into b>NULL, which can be satified by the index t1b. It is a very - # expensive way to do the query, but it works, and so a solution is possible. - do_test where9-6.8.3-stat4 { - catchsql { - UPDATE t1 INDEXED BY t1b SET a=a+100 - WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) - OR (b NOT NULL AND c IS NULL AND d NOT NULL) - OR (b NOT NULL AND c NOT NULL AND d IS NULL) - } - } {0 {}} - do_test where9-6.8.4-stat4 { - catchsql { - DELETE FROM t1 INDEXED BY t1b - WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) - OR (b NOT NULL AND c IS NULL AND d NOT NULL) - OR (b NOT NULL AND c NOT NULL AND d IS NULL) - } - } {0 {}} -} else { +if {1} { do_test where9-6.8.3 { catchsql { UPDATE t1 INDEXED BY t1b SET a=a+100