Deploy heuristics (well-commented) to better estimate how much unindexed terms

in the WHERE clause filter the number of output rows from a single table.

FossilOrigin-Name: 221659945c3f78d3b6789bfe8fdeb8d3ee1fa038

diff --git a/manifest b/manifest
index 1536469c1185e973b394493d605594dde6473e36..ab39af5891763636d13ebfd5cf86f90e47eb78af 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sa\sredundant\stest\scase\s(probably\sa\scopy/paste\serror).\s\sAdd\san\sassert()\nto\swhere.c\sto\sensure\sthat\sautomatic\sindexes\sdo\snot\shave\sthere\soutput\srow\s\ncounts\sadjusted\sdownward\sby\ssupplementary\sconstraints.
-D 2014-11-22T12:22:13.640
+C Deploy\sheuristics\s(well-commented)\sto\sbetter\sestimate\show\smuch\sunindexed\sterms\nin\sthe\sWHERE\sclause\sfilter\sthe\snumber\sof\soutput\srows\sfrom\sa\ssingle\stable.
+D 2014-11-22T18:50:44.269
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -304,7 +304,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793
 F src/wal.c 486e644b3b8aa5ad066f625bc428aa8ff7001405
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804
-F src/where.c 13a9920431358fc4885b31fe13a893d98f813a74
+F src/where.c 33285024c5d96a17901e1beb04798cd8c15d00ae
 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@@ -811,7 +811,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0
 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd
 F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7
 F test/savepoint7.test db3db281486c925095f305aad09fe806e5188ff3
-F test/scanstatus.test a6dd739bc4d9638e8f5c2493b518057f2b681655
+F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285
 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481
 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38
@@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b1e6c02f8b9a2afaa12ac15a33e3f698c3be27d6
-R e5e0eb3b1af9c334c014cb61507da7b4
+P eea47933493c85a85247ad383bd148b06f627d04
+R c99efa7eb98e96584d3723ffcac72bfb
 U drh
-Z 8955433275197ef8c0924c06e567755e
+Z ec1ff7ab563a777168df83432a271a84

diff --git a/manifest.uuid b/manifest.uuid
index 070039fabe5f3a4ea3a4179e4c8cc5451910a230..e47defa2d3748c0cf53562c9e0ec7bdf14f77daa 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-eea47933493c85a85247ad383bd148b06f627d04
\ No newline at end of file
+221659945c3f78d3b6789bfe8fdeb8d3ee1fa038
\ No newline at end of file

diff --git a/src/where.c b/src/where.c
index 6dd47816c3e70d69528e3258eca8488e33d52c5a..05663ce4726f9ae1f5da9e135316b601118f29c0 100644 (file)
--- a/src/where.c
+++ b/src/where.c
@@ -4291,10 +4291,30 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
 ** Adjust the WhereLoop.nOut value downward to account for terms of the
 ** WHERE clause that reference the loop but which are not used by an
 ** index.
-**
-** In the current implementation, the first extra WHERE clause term reduces
-** the number of output rows by a factor of 10 and each additional term
-** reduces the number of output rows by sqrt(2).
+*
+** For every WHERE clause term that is not used by the index
+** and which has a truth probability assigned by one of the likelihood(),
+** likely(), or unlikely() SQL functions, reduce the estimated number
+** of output rows by the probability specified.
+**
+** TUNING:  For every WHERE clause term that is not used by the index
+** and which does not have an assigned truth probability, heuristics
+** described below are used to try to estimate the truth probability.
+** TODO --> Perhaps this is something that could be improved by better
+** table statistics.
+**
+** Heuristic 1:  Estimate the truth probability as 6.25%.  The 6.25%
+** value corresponds to 1 in LogEst notation, so this means decrement
+** the WhereLoop.nOut field for every such WHERE clause term.
+**
+** Heuristic 2:  If there exists one or more WHERE clause terms of the
+** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the
+** final output row estimate is no greater than 1/4 of the total number
+** of rows in the table.  In other words, assume that x==EXPR will filter
+** out at least 3 out of 4 rows.  If EXPR is -1 or 0 or 1, then maybe the
+** "x" column is boolean or else -1 or 0 or 1 is a common default value
+** on the "x" column and so in that case only cap the output row estimate
+** at 1/2 instead of 1/4.
 */
 static void whereLoopOutputAdjust(
   WhereClause *pWC,      /* The WHERE clause */
@@ -4303,8 +4323,8 @@ static void whereLoopOutputAdjust(
 ){
   WhereTerm *pTerm, *pX;
   Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf);
-  int i, j;
-  int nEq = 0;    /* Number of = constraints not within likely()/unlikely() */
+  int i, j, k;
+  LogEst iReduce = 0;    /* pLoop->nOut should not exceed nRow-iReduce */
 
   assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 );
   for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){
@@ -4319,20 +4339,26 @@ static void whereLoopOutputAdjust(
     }
     if( j<0 ){
       if( pTerm->truthProb<=0 ){
+        /* If a truth probability is specified using the likelihood() hints,
+        ** then use the probability provided by the application. */
         pLoop->nOut += pTerm->truthProb;
       }else{
+        /* In the absence of explicit truth probabilities, use heuristics to
+        ** guess a reasonable truth probability. */
         pLoop->nOut--;
-        if( pTerm->eOperator&WO_EQ ) nEq++;
+        if( pTerm->eOperator&WO_EQ ){
+          Expr *pRight = pTerm->pExpr->pRight;
+          if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){
+            k = 10;
+          }else{
+            k = 20;
+          }
+          if( iReduce<k ) iReduce = k;
+        }
       }
     }
   }
-  /* TUNING:  If there is at least one equality constraint in the WHERE
-  ** clause that does not have a likelihood() explicitly assigned to it
-  ** then do not let the estimated number of output rows exceed half 
-  ** the number of rows in the table. */
-  if( nEq && pLoop->nOut>nRow-10 ){
-    pLoop->nOut = nRow - 10;
-  }
+  if( pLoop->nOut > nRow-iReduce )  pLoop->nOut = nRow - iReduce;
 }
 
 /*

diff --git a/test/scanstatus.test b/test/scanstatus.test
index 7713bae5fcecf1ac1dd281a2e6bfd0cc807237bd..ed24d9743710cff66b630a3f796e9ff9bd3504a5 100644 (file)
--- a/test/scanstatus.test
+++ b/test/scanstatus.test
@@ -268,7 +268,7 @@ do_scanstatus_test 4.2.2 {
   nLoop 1 nVisit 1 nEst 1.0 zName sqlite_autoindex_p1_1 
   zExplain {SEARCH TABLE p1 USING INDEX sqlite_autoindex_p1_1 (x=?)}
 
-  nLoop 1 nVisit 3 nEst 524288.0 zName c1 zExplain {SCAN TABLE c1}
+  nLoop 1 nVisit 3 nEst 262144.0 zName c1 zExplain {SCAN TABLE c1}
 }
 
 #-------------------------------------------------------------------------