From: drh Date: Sat, 22 Feb 2020 16:58:49 +0000 (+0000) Subject: When stat4 information is available, try to use it to improve the truth X-Git-Tag: version-3.32.0~148^2~5 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=89efac94fdfac50fac7f971edbde4850b008ecd8;p=thirdparty%2Fsqlite.git When stat4 information is available, try to use it to improve the truth probability of WHERE clause terms that do not participate in the index. FossilOrigin-Name: 1babd6ec5d60e2c34aa1c0285ead768a88004218468e97262411973fe3487022 --- diff --git a/manifest b/manifest index 8bec1d1c7e..d09662988a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sOP_Column\sopcode,\sif\sthe\scursor\sis\smarked\sNullRow\s(due\sto\sbeing\sthe\nright\stable\sof\sa\sLEFT\sJOIN\sthat\sdoes\snot\smatch)\sand\sthe\scursor\sis\sthe\stable\ncursor\sfor\san\sOR-optimization\swith\sa\scovering\sindex,\sthen\sdo\snot\ssubstitute\nthe\scovering\sindex\scursor,\ssince\sthe\scovering\sindex\scursor\sdoes\snot\shave\nthe\sNullRow\sflag\sset.\s\sTicket\s[aa4378693018aa99] -D 2020-02-22T13:01:19.240 +C When\sstat4\sinformation\sis\savailable,\stry\sto\suse\sit\sto\simprove\sthe\struth\nprobability\sof\sWHERE\sclause\sterms\sthat\sdo\snot\sparticipate\sin\sthe\sindex. +D 2020-02-22T16:58:49.287 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -617,8 +617,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 697424314e40d99f93f548c7bfa526c10e87f4bdf64d5a76a96b999dd7133ebc F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a F src/walker.c a137468bf36c92e64d2275caa80c83902e3a0fc59273591b96c6416d3253d05d -F src/where.c cbad14f1d8e11b9f052e937274315c7c17266a89eda408c86084ee894debb7d5 -F src/whereInt.h 9157228db086f436a574589f8cc5749bd971e94017c552305ad9ec472ed2e098 +F src/where.c 74a2fc5a900eab9a2fdda2017a290f0eeaa9c5597fdb86322ea2ccbc3758c71d +F src/whereInt.h 94e3aadcf43b4d16279182d147c9e4f8ef6ed5a5bd1ecc021639c29336b0a3eb F src/wherecode.c f5df56e395ade2240cabb2d39500c681bd29f8cc0636c3301c4996ad160df94d F src/whereexpr.c 264d58971eaf8256eb5b0917bcd7fc7a1f1109fdda183a8382308a1b18a2dce7 F src/window.c f8ba2ee12a19b51d3ba42c16277c74185ee9215306bc0d5a03974ade8b5bc98f @@ -1858,7 +1858,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 14d14eb537075c6ac77513b1e7305bed8bc01a9034dfb763fd96f76400f2b705 -R 1f345124b6aafeeb9e600617cf39817f +P f02030b3403d67734bba471a91ad5bfdb03ddf6fdc3ef14808a04495e43b0470 +R 0b7e29ed07d82aa79f09869cc21d3459 +T *branch * stat4-truthprob +T *sym-stat4-truthprob * +T -sym-trunk * U drh -Z 38c81c56b51d8e5ae42e4e25b041736c +Z b67db2665ec4d9de443f342c9e446178 diff --git a/manifest.uuid b/manifest.uuid index 83685737a8..0338a5dfbe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f02030b3403d67734bba471a91ad5bfdb03ddf6fdc3ef14808a04495e43b0470 \ No newline at end of file +1babd6ec5d60e2c34aa1c0285ead768a88004218468e97262411973fe3487022 \ No newline at end of file diff --git a/src/where.c b/src/where.c index da9c5a7233..58cf11ab1b 100644 --- a/src/where.c +++ b/src/where.c @@ -2307,7 +2307,10 @@ static void whereLoopOutputAdjust( }else{ k = 20; } - if( iReducewtFlags |= TERM_HEURTRUTH; + iReduce = k; + } } } } @@ -2489,9 +2492,9 @@ static int whereLoopAddBtreeIndex( } if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ - pBuilder->bldFlags |= SQLITE_BLDF_UNIQUE; + pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE; }else{ - pBuilder->bldFlags |= SQLITE_BLDF_INDEXED; + pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED; } pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; @@ -2656,6 +2659,21 @@ static int whereLoopAddBtreeIndex( if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ if( nOut ){ pNew->nOut = sqlite3LogEst(nOut); + if( nEq==1 && pTerm->truthProb>0 ){ +#if WHERETRACE_ENABLED /* 0x01 */ + if( sqlite3WhereTrace & 0x01 ){ + sqlite3DebugPrintf("Update truthProb from %d to %d:\n", + pTerm->truthProb, pNew->nOut - pProbe->aiRowLogEst[0]); + sqlite3WhereTermPrint(pTerm, 999); + } +#endif + pTerm->truthProb = pNew->nOut - pProbe->aiRowLogEst[0]; + if( pTerm->wtFlags & TERM_HEURTRUTH ){ + /* If the old heuristic truthProb was previously used, signal + ** that all loops will need to be recomputed */ + pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS; + } + } if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; pNew->nOut -= nIn; } @@ -3080,9 +3098,9 @@ static int whereLoopAddBtree( } } - pBuilder->bldFlags = 0; + pBuilder->bldFlags1 = 0; rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); - if( pBuilder->bldFlags==SQLITE_BLDF_INDEXED ){ + if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){ /* If a non-unique index is used, or if a prefix of the key for ** unique index is used (making the index functionally non-unique) ** then the sqlite_stat1 data becomes important for scoring the @@ -4838,6 +4856,25 @@ WhereInfo *sqlite3WhereBegin( if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ rc = whereLoopAddAll(&sWLB); if( rc ) goto whereBeginError; + +#ifdef SQLITE_ENABLE_STAT4 + /* If one or more WhereTerm.truthProb values were used in estimating + ** loop parameters, but then those truthProb values were subsequently + ** changed based on STAT4 information while computing subsequent loops, + ** then we need to rerun the whole loop building process so that all + ** loops will be built using the revised truthProb values. */ + if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){ + WHERETRACE(0xffff, + ("**** Redo all loop computations due to truthProb changes ****\n")); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); + } + rc = whereLoopAddAll(&sWLB); + if( rc ) goto whereBeginError; + } +#endif #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */ diff --git a/src/whereInt.h b/src/whereInt.h index 74101624d5..9d4559c742 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -291,6 +291,7 @@ struct WhereTerm { #define TERM_LIKE 0x0400 /* The original LIKE operator */ #define TERM_IS 0x0800 /* Term.pExpr is an IS operator */ #define TERM_VARSELECT 0x1000 /* Term.pExpr contains a correlated sub-query */ +#define TERM_HEURTRUTH 0x2000 /* Heuristic truthProb used */ /* ** An instance of the WhereScan object is used as an iterator for locating @@ -405,13 +406,16 @@ struct WhereLoopBuilder { UnpackedRecord *pRec; /* Probe for stat4 (if required) */ int nRecValid; /* Number of valid fields currently in pRec */ #endif - unsigned int bldFlags; /* SQLITE_BLDF_* flags */ + unsigned char bldFlags1; /* First set of SQLITE_BLDF_* flags */ + unsigned char bldFlags2; /* Second set of SQLITE_BLDF_* flags */ unsigned int iPlanLimit; /* Search limiter */ }; /* Allowed values for WhereLoopBuider.bldFlags */ -#define SQLITE_BLDF_INDEXED 0x0001 /* An index is used */ -#define SQLITE_BLDF_UNIQUE 0x0002 /* All keys of a UNIQUE index used */ +#define SQLITE_BLDF1_INDEXED 0x0001 /* An index is used */ +#define SQLITE_BLDF1_UNIQUE 0x0002 /* All keys of a UNIQUE index used */ + +#define SQLITE_BLDF2_2NDPASS 0x0004 /* Second builder pass needed */ /* The WhereLoopBuilder.iPlanLimit is used to limit the number of ** index+constraint combinations the query planner will consider for a