Rework this changes so that instead of setting the WhereTerm.truthProb when

a term is seen to be of low selectivity, it merely sets a new flag
(the TERM_HIGHTRUTH flag) which causes whereLoopOutputAdjust() to ignore
that term.

FossilOrigin-Name: 4558163b6a525990f0f1b6629dbb76daf49bcaf1ddbaf0c50fe05ce9ee480ff8

diff --git a/manifest b/manifest
index 7ba7392d6789acbec32cecd6580ca1be755b2599..ac8c74c984df733eb1ef70741ac30a4d16d37428 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Disable\sthe\snew\sanalyzeG.test\smodule\sif\snot\sbuilding\swith\sSTAT4.
-D 2020-02-24T13:35:34.777
+C Rework\sthis\schanges\sso\sthat\sinstead\sof\ssetting\sthe\sWhereTerm.truthProb\swhen\na\sterm\sis\sseen\sto\sbe\sof\slow\sselectivity,\sit\smerely\ssets\sa\snew\sflag\n(the\sTERM_HIGHTRUTH\sflag)\swhich\scauses\swhereLoopOutputAdjust()\sto\signore\nthat\sterm.
+D 2020-02-24T16:46:08.182
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -617,8 +617,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 697424314e40d99f93f548c7bfa526c10e87f4bdf64d5a76a96b999dd7133ebc
 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a
 F src/walker.c a137468bf36c92e64d2275caa80c83902e3a0fc59273591b96c6416d3253d05d
-F src/where.c 44695e878a287d8c1d4976e2e85bea29994facec3972beb7ca22437d62cda6a5
-F src/whereInt.h 94e3aadcf43b4d16279182d147c9e4f8ef6ed5a5bd1ecc021639c29336b0a3eb
+F src/where.c 3b8c9bd013eb0736e16f60bdc109e83337ef99513a3aff5f16ddac036e6c277e
+F src/whereInt.h 6b874aa15f94e43a2cec1080be64d955b04deeafeac90ffb5d6975c0d511be3c
 F src/wherecode.c f5df56e395ade2240cabb2d39500c681bd29f8cc0636c3301c4996ad160df94d
 F src/whereexpr.c 264d58971eaf8256eb5b0917bcd7fc7a1f1109fdda183a8382308a1b18a2dce7
 F src/window.c f8ba2ee12a19b51d3ba42c16277c74185ee9215306bc0d5a03974ade8b5bc98f
@@ -1859,7 +1859,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b542dee9de843c19664c19df7435c6034d23d0d213804d588ec0ff599082d576
-R f29b5aa99c284afcd3f25a8fb29ee973
+P 4a9d3005769e0398183b03a3e132e3946b9d1c48073af2e0559d7beeac3245c0
+R 2f59a601b560c09f0f628a3935395fd4
 U drh
-Z 9cae8ff3aee64bed711ead2769a79514
+Z 144378d74bf58f80a9e1ca77433de7e1

diff --git a/manifest.uuid b/manifest.uuid
index 46fc370397f4ec73e5979321c03b42f871b70f58..0f1166a39fe10b9285e11c459972bfabb10e4991 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-4a9d3005769e0398183b03a3e132e3946b9d1c48073af2e0559d7beeac3245c0
\ No newline at end of file
+4558163b6a525990f0f1b6629dbb76daf49bcaf1ddbaf0c50fe05ce9ee480ff8
\ No newline at end of file

diff --git a/src/where.c b/src/where.c
index 03bfad3b7d29a964de78f92355f6566932add495..d9f11296fa5318e68a4c776ffc2b785a96404c4f 100644 (file)
--- a/src/where.c
+++ b/src/where.c
@@ -2298,14 +2298,16 @@ static void whereLoopOutputAdjust(
         /* In the absence of explicit truth probabilities, use heuristics to
         ** guess a reasonable truth probability. */
         pLoop->nOut--;
-        if( pTerm->eOperator&(WO_EQ|WO_IS) ){
+        if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0
+         && (pTerm->wtFlags & TERM_HIGHTRUTH)==0  /* tag-20200224-1 */
+        ){
           Expr *pRight = pTerm->pExpr->pRight;
           int k = 0;
           testcase( pTerm->pExpr->op==TK_IS );
           if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){
             k = 10;
           }else{
-            k = 20;  /* Keep the "20" value in sync. See tag-20200222-1 */
+            k = 20;
           }
           if( iReduce<k ){
             pTerm->wtFlags |= TERM_HEURTRUTH;
@@ -2660,23 +2662,23 @@ static int whereLoopAddBtreeIndex(
           if( nOut ){
             pNew->nOut = sqlite3LogEst(nOut);
             if( nEq==1
-             && pTerm->truthProb>0
-             /* TUNING: Adjust truthProb from the default heuristic only if the
-             ** probability is close to 1.0. The "20" constant is copied from
-             ** the heuristic at tag-20200222-1. Keep values in sync */
-             && pNew->nOut+20 > pProbe->aiRowLogEst[0]
+             /* TUNING: Mark terms as "low selectivity" if they seem likely
+             ** to be true for half or more of the rows in the table.
+             ** See tag-202002240-1 */
+             && pNew->nOut+10 > pProbe->aiRowLogEst[0]
             ){
 #if WHERETRACE_ENABLED /* 0x01 */
               if( sqlite3WhereTrace & 0x01 ){
-                sqlite3DebugPrintf("Update truthProb from %d to %d:\n",
-                       pTerm->truthProb, pNew->nOut - pProbe->aiRowLogEst[0]);
+                sqlite3DebugPrintf(
+                   "STAT4 determines term has low selectivity:\n");
                 sqlite3WhereTermPrint(pTerm, 999);
               }
 #endif
-              pTerm->truthProb = pNew->nOut - pProbe->aiRowLogEst[0];
+              pTerm->wtFlags |= TERM_HIGHTRUTH;
               if( pTerm->wtFlags & TERM_HEURTRUTH ){
-                /* If the old heuristic truthProb was previously used, signal
-                ** that all loops will need to be recomputed */
+                /* If the term has previously been used with an assumption of
+                ** higher selectivity, then set the flag to rerun the
+                ** loop computations. */
                 pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS;
               }
             }
@@ -4894,7 +4896,8 @@ WhereInfo *sqlite3WhereBegin(
     if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){
       WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
       WHERETRACE(0xffff, 
-         ("**** Redo all loop computations due to truthProb changes ****\n"));
+           ("**** Redo all loop computations due to"
+            " TERM_HIGHTRUTH changes ****\n"));
       while( pWInfo->pLoops ){
         WhereLoop *p = pWInfo->pLoops;
         pWInfo->pLoops = p->pNextLoop;

diff --git a/src/whereInt.h b/src/whereInt.h
index 9d4559c74250e5f8607528dbdda581265f6afa8f..e33dde55e293786a7cd16f3e615055d98df54aad 100644 (file)
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -292,6 +292,11 @@ struct WhereTerm {
 #define TERM_IS         0x0800 /* Term.pExpr is an IS operator */
 #define TERM_VARSELECT  0x1000 /* Term.pExpr contains a correlated sub-query */
 #define TERM_HEURTRUTH  0x2000 /* Heuristic truthProb used */
+#ifdef SQLITE_ENABLE_STAT4
+#  define TERM_HIGHTRUTH  0x4000 /* Term excludes few rows */
+#else
+#  define TERM_HIGHTRUTH  0      /* Only used with STAT4 */
+#endif
 
 /*
 ** An instance of the WhereScan object is used as an iterator for locating