Drop support for the view-scan optimization as

it was causing multiple performance regressions.  In its place, reduce the
estimated row count for DISTINCT subsqueries by a factor of 8.

FossilOrigin-Name: 796a65fa61373b5def9bf94e068e2dae9381a084c7e2c1e5df26929e9b07ca07

diff --git a/manifest b/manifest
index 9756b068e167eb20fcdcec5f1e5b9963a174585c..705d25038e88d0b5c140d721d7074743b1507457 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sthe\s[/info/7c2d3406000dc8ac|omit-unused-subquery-columns\soptimization],\sbe\nsure\sto\sremove\sthe\sEP_Skip\sand\sEP_Unlikely\sflags\sfrom\sthe\sresult\sset\sexpressions\nthat\sget\snulled-out.\s\sdbsqlfuzz\sbf1d3ed6e0e0dd8766027797d43db40c776d2b15.\nAlso\sfix\san\sincorrect\s".selecttrace"\scode\sblock.
-D 2023-02-26T11:52:37.661
+C Drop\ssupport\sfor\sthe\sview-scan\soptimization\sas\nit\swas\scausing\smultiple\sperformance\sregressions.\s\sIn\sits\splace,\sreduce\sthe\nestimated\srow\scount\sfor\sDISTINCT\ssubsqueries\sby\sa\sfactor\sof\s8.
+D 2023-09-15T20:04:31.903
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -604,8 +604,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 9eccc7ebb532a7b0fd3cabc16cff576b9afa763472272db67d84fb8cec96f5c0
 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a
 F src/walker.c 7607f1a68130c028255d8d56094ea602fc402c79e1e35a46e6282849d90d5fe4
-F src/where.c 6a8b2a825d3d2af6c9c290e4b93804a54551b75bef1b8c40407e298d5fae54a5
-F src/whereInt.h 752a870900422b8a6d77c51d74b9e9ce935ff7d5e7b6caf1e786823c51a18a14
+F src/where.c 9adf9d9e55a17d6e89d7286c85836738efcb081c8630d214156542b4f6ea6ae6
+F src/whereInt.h 064a1508edcc9af400a3b79211ba55e553fab6451b797a31334c0d11cbb7debb
 F src/wherecode.c cf67460973119c7b2141ad67daf8368dfb4871f225e2489f95effaa139007bfd
 F src/whereexpr.c ca55a11c2443700fe084a1e039660688d7733c594a37697ee4bd99462e2c2f6a
 F src/window.c 038c248267e74ff70a2bb9b1884d40fd145c5183b017823ecb6cbb14bc781478
@@ -1278,7 +1278,7 @@ F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d
 F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95
 F test/selectA.test 68de52409e45a3313d00b8461b48bef4fb729faf36ade9067a994eae55cc86f4
 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25
-F test/selectC.test ef9eddcd89277fdfa8d97b35324110fcc8d9fa94337fe0c26b3321301b2d1c7c
+F test/selectC.test 5bbdc8459aa3890f15e60f86f4505d46aab9e2643b0efa8b75a42ca3de110608
 F test/selectD.test fc20452847a01775710090383cfb4423275d2f745fed61f34fbf37573ac0d214
 F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf
 F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3
@@ -1353,7 +1353,7 @@ F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1
 F test/stmt.test 54ed2cc0764bf3e48a058331813c3dbd19fc1d0827c3d8369914a5d8f564ec75
 F test/stmtvtab1.test 6873dfb24f8e79cbb5b799b95c2e4349060eb7a3b811982749a84b359468e2d5
 F test/subjournal.test 8d4e2572c0ee9a15549f0d8e40863161295107e52f07a3e8012a2e1fdd093c49
-F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f
+F test/subquery.test 3f46cc25b0284ddb7aeba0c69681f13a0c1f3fc56d1e5b658aa76dc378908752
 F test/subquery2.test 8250dfd6a773b04c7a5c37ac63276f62b329157ce171244d0cbe1acc365e3303
 F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303
 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a
@@ -1703,7 +1703,7 @@ F test/windowerr.test 675b5e6debfc9370bfacb0b91e2a93a8923512f92600b16f4ea70a1cd9
 F test/windowfault.test 16e906a2c4110c88372ff4bd5de59ac7397ec2f025912eff8e5677eedd126898
 F test/with1.test a07b5aad7f77acdf13e52e8814ea94606fcc72e9ea4c99baf293e9d7c63940be
 F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1ab
-F test/with3.test e76a354523a17233830f0d98cc63379cf7ebeb07354a0aff027d892451f44ba3
+F test/with3.test 8d26920c88283e0a473ceebd3451554922108ce7b2a6a1157c47eb0a7011212c
 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205
 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64
 F test/without_rowid1.test b5ec93f7df2c1d684e0923247dac6aca8888e088bf50a9f244c3933e0e813a72
@@ -1820,9 +1820,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ad6ac5d3e84cf7f32378a45f276dd331d6fbac8a102d700c44c2167080fb6a1c
-Q +21aec65e5e2a01e58dd0bb8c8b9b29b8414373b53353fc7ca80a152fdd27566b
-R c46d642a7787c728b73bfedc777d9613
+P 83a7f13edb74c95f0bccafb931e612ecca568211c3e720db168bd1ff32b3181a
+Q +f911f1c4977fbcae041243955cf2b98d8cc8baa337885a69be0f2b9bd2efa6f3
+R 0e56265885eb0cd9b75aadb6c1b32d5f
 U drh
-Z f42e3df278e59d85c7e3fa953d3cae51
+Z 1555b53e0c6a7deb6142f68b0bfb4eb6
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index c266f638be3befab49e77ba8eca16b8c1707fe36..6d1c4ad45100371041a61e9ea43ce54ae5eb52e8 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-83a7f13edb74c95f0bccafb931e612ecca568211c3e720db168bd1ff32b3181a
\ No newline at end of file
+796a65fa61373b5def9bf94e068e2dae9381a084c7e2c1e5df26929e9b07ca07
\ No newline at end of file

diff --git a/src/where.c b/src/where.c
index 6bbf4b8cc13af7fea1bc3afd130c28057281242c..c944dda06e06b5ce1b476032ce3c3373a66b3b30 100644 (file)
--- a/src/where.c
+++ b/src/where.c
@@ -3102,9 +3102,6 @@ static int whereLoopAddBtree(
 #else
       pNew->rRun = rSize + 16;
 #endif
-      if( (pTab->tabFlags & TF_Ephemeral)!=0 ){
-        pNew->wsFlags |= WHERE_VIEWSCAN;
-      }
       ApplyCostMultiplier(pNew->rRun, pTab->costMult);
       whereLoopOutputAdjust(pWC, pNew, rSize);
       rc = whereLoopInsert(pBuilder, pNew);
@@ -4256,13 +4253,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
           rUnsorted -= 2;  /* TUNING:  Slight bias in favor of no-sort plans */
         }
 
-        /* TUNING:  A full-scan of a VIEW or subquery in the outer loop
-        ** is not so bad. */
-        if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 ){
-          rCost += -10;
-          nOut += -30;
-        }
-
         /* Check to see if pWLoop should be added to the set of
         ** mxChoice best-so-far paths.
         **
@@ -4966,6 +4956,16 @@ WhereInfo *sqlite3WhereBegin(
        wherePathSolver(pWInfo, pWInfo->nRowOut+1);
        if( db->mallocFailed ) goto whereBeginError;
     }
+
+    /* TUNING:  Assume that a DISTINCT clause on a subquery reduces
+    ** the output size by a factor of 8 (LogEst -30).
+    */
+    if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){
+      WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n",
+                         pWInfo->nRowOut, pWInfo->nRowOut-30));
+      pWInfo->nRowOut -= 30;
+    }
+
   }
   if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
      pWInfo->revMask = ALLBITS;

diff --git a/src/whereInt.h b/src/whereInt.h
index 1fea3b9ffbb44507b7f4ce7c80daec1a441e6836..789850e79933bc91964fe81adbd220deb4da0bd1 100644 (file)
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -588,4 +588,4 @@ void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*);
 #define WHERE_PARTIALIDX   0x00020000  /* The automatic index is partial */
 #define WHERE_IN_EARLYOUT  0x00040000  /* Perhaps quit IN loops early */
 #define WHERE_IN_SEEKSCAN  0x00100000  /* Seek-scan optimization for IN */
-#define WHERE_VIEWSCAN     0x02000000  /* A full-scan of a VIEW or subquery */
+                      /*   0x02000000  -- available for reuse */

diff --git a/test/selectC.test b/test/selectC.test
index de3bb6498cdd972fe1338c2f170ba76214e1544f..34ee207deb7e75375f6c8e2649638844ccc2a506 100644 (file)
--- a/test/selectC.test
+++ b/test/selectC.test
@@ -267,9 +267,10 @@ do_execsql_test 5.2 {
 
 do_execsql_test 5.3 {
   SELECT * FROM x1, (SELECT b FROM vvv UNION ALL SELECT c from x3)
+  ORDER BY +2
 } {
   a 21 b 21 a 22 b 22 a 23 b 23 a 24 b 24 a 25 b 25
-  a 302 b 302 a 303 b 303 a 301 b 301
+  a 301 b 301 a 302 b 302 a 303 b 303
 }
 
 finish_test

diff --git a/test/subquery.test b/test/subquery.test
index 06facbbae05206e72039bf31516ff56cd555592a..12dfd372e4c92321b8be5ab3f5bc2234f203e8de 100644 (file)
--- a/test/subquery.test
+++ b/test/subquery.test
@@ -11,8 +11,6 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is testing correlated subqueries
 #
-# $Id: subquery.test,v 1.17 2009/01/09 01:12:28 drh Exp $
-#
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -594,4 +592,45 @@ do_execsql_test subquery-8.1 {
   SELECT (SELECT 0 FROM (SELECT * FROM (SELECT 0))) AS x WHERE x;
 } {}
 
+
+# 2023-09-15
+# Query planner performance regression reported by private email 
+# on 2023-09-14, caused by VIEWSCAN optimization of check-in 609fbb94b8f01d67
+# from 2022-09-01.
+#
+reset_db
+do_execsql_test subquery-10.1 {
+  CREATE TABLE t1(aa TEXT, bb INT, cc TEXT);
+  CREATE INDEX x11 on t1(bb);
+  CREATE INDEX x12 on t1(aa);
+  CREATE TABLE t2(aa TEXT, xx INT);
+  ANALYZE sqlite_master;
+  INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES('t1', 'x11', '156789 28');
+  INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES('t1', 'x12', '156789 1');
+  ANALYZE sqlite_master;
+}
+do_eqp_test subquery-10.2 {
+  WITH v1(aa,cc,bb) AS (SELECT aa, cc, bb FROM t1 WHERE bb=12345),
+       v2(aa,mx)    AS (SELECT aa, max(xx) FROM t2 GROUP BY aa)
+  SELECT * FROM v1 JOIN v2 ON v1.aa=v2.aa;
+} {
+  QUERY PLAN
+  |--MATERIALIZE xxxxxx
+  |  |--SCAN TABLE t2
+  |  `--USE TEMP B-TREE FOR GROUP BY
+  |--SEARCH TABLE t1 USING INDEX x11 (bb=?)
+  `--SEARCH SUBQUERY xxxxxx USING AUTOMATIC COVERING INDEX (aa=?)
+}
+# ^^^^^^^^^^^^^
+# Prior to the fix the incorrect (slow) plan caused by the
+# VIEWSCAN optimization was:
+#
+# QUERY PLAN
+# |--CO-ROUTINE v2
+# |  |--SCAN t2
+# |  `--USE TEMP B-TREE FOR GROUP BY
+# |--SCAN v2
+# `--SEARCH t1 USING INDEX x12 (aa=?)
+#
+
 finish_test

diff --git a/test/with3.test b/test/with3.test
index 91bb8f016cbfa189827e49eb1d6b640118f72c9e..0f49f0685903b49854a1ba44c5c21d530f0d0e9a 100644 (file)
--- a/test/with3.test
+++ b/test/with3.test
@@ -123,8 +123,8 @@ do_eqp_test 3.2.2 {
   |  |  `--SCALAR SUBQUERY xxxxxx
   |  |     `--SCAN TABLE w2
   |  `--RECURSIVE STEP
-  |     |--SCAN TABLE c
-  |     `--SCAN TABLE w1
+  |     |--SCAN TABLE w1
+  |     `--SCAN TABLE c
   |--SCAN SUBQUERY xxxxxx
   |--SEARCH TABLE w2 USING INTEGER PRIMARY KEY (rowid=?)
   `--SEARCH TABLE w1 USING INTEGER PRIMARY KEY (rowid=?)