Omit the "noquery" argument in sqlite_stat1. Instead, add the "uneven" argument

for indexes that have an uneven distribution of values.  Modify the query
planner to avoid doing equality look-ups on uneven indexes.

FossilOrigin-Name: 0ec4b880bbf276f905d5c09fcb09cee4748c92c83baf241910223e59b723b86e

diff --git a/manifest b/manifest
index 88f99ed4d8809426e2f82c50ce27766da0d0b3d5..0b247dc2d92b6952c89d0b607ad9c2b25a4aad81 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Revise\sthe\s"noquery"\sdecision\salgorithm\sagain.\s\sThe\sindex\snow\smust\sselect\nmore\sthan\s150\srows\son\saverage,\sand\sthe\snumber\sof\srows\sselect\smust\sbe\senough\nthat\sit\sseems\sfaster\sto\sdo\sa\sfull\sscan\sof\sthe\sassociated\stable.
-D 2023-12-29T21:07:00.789
+C Omit\sthe\s"noquery"\sargument\sin\ssqlite_stat1.\s\sInstead,\sadd\sthe\s"uneven"\sargument\nfor\sindexes\sthat\shave\san\suneven\sdistribution\sof\svalues.\s\sModify\sthe\squery\nplanner\sto\savoid\sdoing\sequality\slook-ups\son\suneven\sindexes.
+D 2023-12-30T15:07:31.646
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -670,7 +670,7 @@ F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2
 F src/alter.c 30c2333b8bb3af71e4eb9adeadee8aa20edb15917ed44b8422e5cd15f3dfcddc
-F src/analyze.c 0c4b43ba2e4e7db574c8af8e6cfdb09aff6a91a11caed821867c5b51708534d5
+F src/analyze.c 1841ee237180db76d8732f6041b21fc09318268090f80c332a6b3524009ebc58
 F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39
 F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4
 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523
@@ -742,7 +742,7 @@ F src/shell.c.in 85f8d52fa4f7773823736dd39d0a268fd739207fcae95883c9ec8ce4af59f7d
 F src/sqlite.h.in 61a60b4ea04db8ead15e1579b20b64cb56e9f55d52c5f9f9694de630110593a3
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
-F src/sqliteInt.h 4552ac3dbe5be857817cff0855cdabbd20db0252739f463b5a193117c8880a02
+F src/sqliteInt.h 1a06a2255168e6ca09ab8c4a158ef21454fd75d4916e5c2aa0d48253fe205639
 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -822,7 +822,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c e5247a3406531b705b44630e9ccf9ca0e5c74955ef19c06fbb146d765c500c20
 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452
 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2
-F src/where.c 0bfab37c7f787e320a8010e51ae97c2e51964d3b3a24fbc246b8e8fee50de4e9
+F src/where.c 927a1c803e4c7c20be17d292975581681c78b3182bf3d763fedf664f6ae48fc2
 F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8
 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1
 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00
@@ -2156,8 +2156,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6aaa65adfc3b0c8e1ee7066de38d4385904c80da704b97440b0699fc78176ffa
-R 25c81b1dd9b3d0d849705539d9441d5c
+P f516ef80a0ba87fa3b74da0fc899a7d14ad3de61fac6b467e36e7fa9fe76dcdf
+R c297d3ac74063268f6b81923ab4cb3f5
 U drh
-Z eef2167dc56c437be295ceda1d33c4be
+Z f5c75810e499792105e5109c0faf69d2
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index c4928fe2c8fefe0a26aa94a5fc6d92f27acb320d..d691bd95dcb88dcfe747ec89f73c3769aeca0e20 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f516ef80a0ba87fa3b74da0fc899a7d14ad3de61fac6b467e36e7fa9fe76dcdf
\ No newline at end of file
+0ec4b880bbf276f905d5c09fcb09cee4748c92c83baf241910223e59b723b86e
\ No newline at end of file

diff --git a/src/analyze.c b/src/analyze.c
index 1223a9f0679ecdd8ef4e3fcba26d5e31e361c6b5..7bcf917a9a3d07211c8a6710dc77488a2cc1dedf 100644 (file)
--- a/src/analyze.c
+++ b/src/analyze.c
@@ -865,16 +865,15 @@ static void statGet(
     */
     sqlite3_str sStat;   /* Text of the constructed "stat" line */
     int i;               /* Loop counter */
-    u64 nRow, iVal = 0;
+    int bUneven = 0;     /* True if there is an uneven distribution of values */
 
     sqlite3StrAccumInit(&sStat, 0, 0, 0, (p->nKeyCol+1)*100);
-    nRow = p->nSkipAhead ? p->nEst : p->nRow;
-    sqlite3_str_appendf(&sStat, "%llu", nRow);
+    sqlite3_str_appendf(&sStat, "%llu", 
+        p->nSkipAhead ? (u64)p->nEst : (u64)p->nRow);
     for(i=0; i<p->nKeyCol; i++){
-      u64 nDistinct, mx;
-      nDistinct = p->current.anDLt[i] + 1;
-      iVal = (p->nRow + nDistinct - 1) / nDistinct;
-      mx = p->current.amxEq[i];
+      u64 nDistinct = p->current.anDLt[i] + 1;
+      u64 iVal = (p->nRow + nDistinct - 1) / nDistinct;
+      u64 mx = p->current.amxEq[i];
       if( nDistinct==1 && p->nLimit>0 ){
         /* If we never saw more than a single value in a PRAGMA analysis_limit
         ** search, then set the estimated number of matching rows to the
@@ -884,6 +883,7 @@ static void statGet(
         /* Never let the estimated number of matching rows be less than
         ** 1/8th the greatest number of identical rows */
         iVal = mx/8;
+        bUneven = 1;
       }else if( iVal==2 && p->nRow*10 <= nDistinct*11 ){
         /* If the value is less than or equal to 1.1, round it down to 1.0 */
         iVal = 1;
@@ -891,14 +891,8 @@ static void statGet(
       sqlite3_str_appendf(&sStat, " %llu", iVal);
       assert( p->current.anEq[i] );
     }
-    if( iVal>=150 && iVal>=(nRow*10)/sqlite3LogEst(nRow) ){
-      /* If this index is likely to match 150 or more rows even if all columns
-      ** match, and if the the number of rows matched is such a large fraction
-      ** of the index that a full scan of the table would be faster than
-      ** doing a binary search for each row identified by the index,
-      ** then this is a very low selectivity index.  Mark it as "noquery"
-      ** so that the query planner won't waste any time trying to use it. */
-      sqlite3_str_appendf(&sStat, " noquery");
+    if( bUneven ){
+      sqlite3_str_appendf(&sStat, " uneven");
     }
     sqlite3ResultStrAccum(context, &sStat);
   }
@@ -1578,8 +1572,8 @@ static void decodeIntArray(
         pIndex->szIdxRow = sqlite3LogEst(sz);
       }else if( sqlite3_strglob("noskipscan*", z)==0 ){
         pIndex->noSkipScan = 1;
-      }else if( sqlite3_strglob("noquery*", z)==0 ){
-        pIndex->bNoQuery = 1;
+      }else if( sqlite3_strglob("uneven*", z)==0 ){
+        pIndex->bUneven = 1;
       }
 #ifdef SQLITE_ENABLE_COSTMULT
       else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){

diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 75ac749c0d7d88e0772a8fcd0298258de91f9558..cf5de5008d2804de93ba0d8c9b2b43688090f8e2 100644 (file)
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2767,6 +2767,7 @@ struct Index {
   unsigned noSkipScan:1;   /* Do not try to use skip-scan if true */
   unsigned hasStat1:1;     /* aiRowLogEst values come from sqlite_stat1 */
   unsigned bNoQuery:1;     /* Do not use this index to optimize queries */
+  unsigned bUneven:1;      /* sqlite_stat1 has the "uneven" flag */
   unsigned bAscKeyBug:1;   /* True if the bba7b69f9849b5bf bug applies */
   unsigned bHasVCol:1;     /* Index references one or more VIRTUAL columns */
   unsigned bHasExpr:1;     /* Index contains an expression, either a literal

diff --git a/src/where.c b/src/where.c
index bffca3413daf269682256fc8cd27b9771a89fc36..d2feaa712c185ff021bc2f55a59bb30e387a56b8 100644 (file)
--- a/src/where.c
+++ b/src/where.c
@@ -2973,7 +2973,10 @@ static int whereLoopAddBtreeIndex(
     assert( pNew->u.btree.nBtm==0 );
     opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
   }
-  if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
+  if( pProbe->bUnordered || pProbe->bUneven ){
+    if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
+    if( pProbe->bUneven ) opMask &= ~(WO_EQ|WO_IN|WO_IS);
+  }
 
   assert( pNew->u.btree.nEq<pProbe->nColumn );
   assert( pNew->u.btree.nEq<pProbe->nKeyCol