From: drh <> Date: Fri, 15 Sep 2023 20:04:31 +0000 (+0000) Subject: Drop support for the view-scan optimization as X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ad856bbc336c614229ec6a2c9ccdc6ed07ebe375;p=thirdparty%2Fsqlite.git Drop support for the view-scan optimization as it was causing multiple performance regressions. In its place, reduce the estimated row count for DISTINCT subsqueries by a factor of 8. FossilOrigin-Name: 796a65fa61373b5def9bf94e068e2dae9381a084c7e2c1e5df26929e9b07ca07 --- diff --git a/manifest b/manifest index 9756b068e1..705d25038e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\s[/info/7c2d3406000dc8ac|omit-unused-subquery-columns\soptimization],\sbe\nsure\sto\sremove\sthe\sEP_Skip\sand\sEP_Unlikely\sflags\sfrom\sthe\sresult\sset\sexpressions\nthat\sget\snulled-out.\s\sdbsqlfuzz\sbf1d3ed6e0e0dd8766027797d43db40c776d2b15.\nAlso\sfix\san\sincorrect\s".selecttrace"\scode\sblock. -D 2023-02-26T11:52:37.661 +C Drop\ssupport\sfor\sthe\sview-scan\soptimization\sas\nit\swas\scausing\smultiple\sperformance\sregressions.\s\sIn\sits\splace,\sreduce\sthe\nestimated\srow\scount\sfor\sDISTINCT\ssubsqueries\sby\sa\sfactor\sof\s8. +D 2023-09-15T20:04:31.903 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -604,8 +604,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 9eccc7ebb532a7b0fd3cabc16cff576b9afa763472272db67d84fb8cec96f5c0 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a F src/walker.c 7607f1a68130c028255d8d56094ea602fc402c79e1e35a46e6282849d90d5fe4 -F src/where.c 6a8b2a825d3d2af6c9c290e4b93804a54551b75bef1b8c40407e298d5fae54a5 -F src/whereInt.h 752a870900422b8a6d77c51d74b9e9ce935ff7d5e7b6caf1e786823c51a18a14 +F src/where.c 9adf9d9e55a17d6e89d7286c85836738efcb081c8630d214156542b4f6ea6ae6 +F src/whereInt.h 064a1508edcc9af400a3b79211ba55e553fab6451b797a31334c0d11cbb7debb F src/wherecode.c cf67460973119c7b2141ad67daf8368dfb4871f225e2489f95effaa139007bfd F src/whereexpr.c ca55a11c2443700fe084a1e039660688d7733c594a37697ee4bd99462e2c2f6a F src/window.c 038c248267e74ff70a2bb9b1884d40fd145c5183b017823ecb6cbb14bc781478 @@ -1278,7 +1278,7 @@ F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95 F test/selectA.test 68de52409e45a3313d00b8461b48bef4fb729faf36ade9067a994eae55cc86f4 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 -F test/selectC.test ef9eddcd89277fdfa8d97b35324110fcc8d9fa94337fe0c26b3321301b2d1c7c +F test/selectC.test 5bbdc8459aa3890f15e60f86f4505d46aab9e2643b0efa8b75a42ca3de110608 F test/selectD.test fc20452847a01775710090383cfb4423275d2f745fed61f34fbf37573ac0d214 F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3 @@ -1353,7 +1353,7 @@ F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1 F test/stmt.test 54ed2cc0764bf3e48a058331813c3dbd19fc1d0827c3d8369914a5d8f564ec75 F test/stmtvtab1.test 6873dfb24f8e79cbb5b799b95c2e4349060eb7a3b811982749a84b359468e2d5 F test/subjournal.test 8d4e2572c0ee9a15549f0d8e40863161295107e52f07a3e8012a2e1fdd093c49 -F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f +F test/subquery.test 3f46cc25b0284ddb7aeba0c69681f13a0c1f3fc56d1e5b658aa76dc378908752 F test/subquery2.test 8250dfd6a773b04c7a5c37ac63276f62b329157ce171244d0cbe1acc365e3303 F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a @@ -1703,7 +1703,7 @@ F test/windowerr.test 675b5e6debfc9370bfacb0b91e2a93a8923512f92600b16f4ea70a1cd9 F test/windowfault.test 16e906a2c4110c88372ff4bd5de59ac7397ec2f025912eff8e5677eedd126898 F test/with1.test a07b5aad7f77acdf13e52e8814ea94606fcc72e9ea4c99baf293e9d7c63940be F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1ab -F test/with3.test e76a354523a17233830f0d98cc63379cf7ebeb07354a0aff027d892451f44ba3 +F test/with3.test 8d26920c88283e0a473ceebd3451554922108ce7b2a6a1157c47eb0a7011212c F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 F test/without_rowid1.test b5ec93f7df2c1d684e0923247dac6aca8888e088bf50a9f244c3933e0e813a72 @@ -1820,9 +1820,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ad6ac5d3e84cf7f32378a45f276dd331d6fbac8a102d700c44c2167080fb6a1c -Q +21aec65e5e2a01e58dd0bb8c8b9b29b8414373b53353fc7ca80a152fdd27566b -R c46d642a7787c728b73bfedc777d9613 +P 83a7f13edb74c95f0bccafb931e612ecca568211c3e720db168bd1ff32b3181a +Q +f911f1c4977fbcae041243955cf2b98d8cc8baa337885a69be0f2b9bd2efa6f3 +R 0e56265885eb0cd9b75aadb6c1b32d5f U drh -Z f42e3df278e59d85c7e3fa953d3cae51 +Z 1555b53e0c6a7deb6142f68b0bfb4eb6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c266f638be..6d1c4ad451 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83a7f13edb74c95f0bccafb931e612ecca568211c3e720db168bd1ff32b3181a \ No newline at end of file +796a65fa61373b5def9bf94e068e2dae9381a084c7e2c1e5df26929e9b07ca07 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6bbf4b8cc1..c944dda06e 100644 --- a/src/where.c +++ b/src/where.c @@ -3102,9 +3102,6 @@ static int whereLoopAddBtree( #else pNew->rRun = rSize + 16; #endif - if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ - pNew->wsFlags |= WHERE_VIEWSCAN; - } ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); @@ -4256,13 +4253,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */ } - /* TUNING: A full-scan of a VIEW or subquery in the outer loop - ** is not so bad. */ - if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 ){ - rCost += -10; - nOut += -30; - } - /* Check to see if pWLoop should be added to the set of ** mxChoice best-so-far paths. ** @@ -4966,6 +4956,16 @@ WhereInfo *sqlite3WhereBegin( wherePathSolver(pWInfo, pWInfo->nRowOut+1); if( db->mallocFailed ) goto whereBeginError; } + + /* TUNING: Assume that a DISTINCT clause on a subquery reduces + ** the output size by a factor of 8 (LogEst -30). + */ + if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){ + WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n", + pWInfo->nRowOut, pWInfo->nRowOut-30)); + pWInfo->nRowOut -= 30; + } + } if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ pWInfo->revMask = ALLBITS; diff --git a/src/whereInt.h b/src/whereInt.h index 1fea3b9ffb..789850e799 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -588,4 +588,4 @@ void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*); #define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ #define WHERE_IN_EARLYOUT 0x00040000 /* Perhaps quit IN loops early */ #define WHERE_IN_SEEKSCAN 0x00100000 /* Seek-scan optimization for IN */ -#define WHERE_VIEWSCAN 0x02000000 /* A full-scan of a VIEW or subquery */ + /* 0x02000000 -- available for reuse */ diff --git a/test/selectC.test b/test/selectC.test index de3bb6498c..34ee207deb 100644 --- a/test/selectC.test +++ b/test/selectC.test @@ -267,9 +267,10 @@ do_execsql_test 5.2 { do_execsql_test 5.3 { SELECT * FROM x1, (SELECT b FROM vvv UNION ALL SELECT c from x3) + ORDER BY +2 } { a 21 b 21 a 22 b 22 a 23 b 23 a 24 b 24 a 25 b 25 - a 302 b 302 a 303 b 303 a 301 b 301 + a 301 b 301 a 302 b 302 a 303 b 303 } finish_test diff --git a/test/subquery.test b/test/subquery.test index 06facbbae0..12dfd372e4 100644 --- a/test/subquery.test +++ b/test/subquery.test @@ -11,8 +11,6 @@ # This file implements regression tests for SQLite library. The # focus of this script is testing correlated subqueries # -# $Id: subquery.test,v 1.17 2009/01/09 01:12:28 drh Exp $ -# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -594,4 +592,45 @@ do_execsql_test subquery-8.1 { SELECT (SELECT 0 FROM (SELECT * FROM (SELECT 0))) AS x WHERE x; } {} + +# 2023-09-15 +# Query planner performance regression reported by private email +# on 2023-09-14, caused by VIEWSCAN optimization of check-in 609fbb94b8f01d67 +# from 2022-09-01. +# +reset_db +do_execsql_test subquery-10.1 { + CREATE TABLE t1(aa TEXT, bb INT, cc TEXT); + CREATE INDEX x11 on t1(bb); + CREATE INDEX x12 on t1(aa); + CREATE TABLE t2(aa TEXT, xx INT); + ANALYZE sqlite_master; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES('t1', 'x11', '156789 28'); + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES('t1', 'x12', '156789 1'); + ANALYZE sqlite_master; +} +do_eqp_test subquery-10.2 { + WITH v1(aa,cc,bb) AS (SELECT aa, cc, bb FROM t1 WHERE bb=12345), + v2(aa,mx) AS (SELECT aa, max(xx) FROM t2 GROUP BY aa) + SELECT * FROM v1 JOIN v2 ON v1.aa=v2.aa; +} { + QUERY PLAN + |--MATERIALIZE xxxxxx + | |--SCAN TABLE t2 + | `--USE TEMP B-TREE FOR GROUP BY + |--SEARCH TABLE t1 USING INDEX x11 (bb=?) + `--SEARCH SUBQUERY xxxxxx USING AUTOMATIC COVERING INDEX (aa=?) +} +# ^^^^^^^^^^^^^ +# Prior to the fix the incorrect (slow) plan caused by the +# VIEWSCAN optimization was: +# +# QUERY PLAN +# |--CO-ROUTINE v2 +# | |--SCAN t2 +# | `--USE TEMP B-TREE FOR GROUP BY +# |--SCAN v2 +# `--SEARCH t1 USING INDEX x12 (aa=?) +# + finish_test diff --git a/test/with3.test b/test/with3.test index 91bb8f016c..0f49f06859 100644 --- a/test/with3.test +++ b/test/with3.test @@ -123,8 +123,8 @@ do_eqp_test 3.2.2 { | | `--SCALAR SUBQUERY xxxxxx | | `--SCAN TABLE w2 | `--RECURSIVE STEP - | |--SCAN TABLE c - | `--SCAN TABLE w1 + | |--SCAN TABLE w1 + | `--SCAN TABLE c |--SCAN SUBQUERY xxxxxx |--SEARCH TABLE w2 USING INTEGER PRIMARY KEY (rowid=?) `--SEARCH TABLE w1 USING INTEGER PRIMARY KEY (rowid=?)