From: drh Date: Fri, 17 Feb 2017 15:26:36 +0000 (+0000) Subject: Set the TF_StatsUsed flag on tables when the query planner outcome is X-Git-Tag: version-3.18.0~81^2~14 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a3928dd7bee435f2cf4288b336968f6334376d39;p=thirdparty%2Fsqlite.git Set the TF_StatsUsed flag on tables when the query planner outcome is affected by the sqlite_stat1 data. Also, change the column names of the "PRAGMA stats" command so that they are not keywords. FossilOrigin-Name: fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7 --- diff --git a/manifest b/manifest index 04051afae1..360ecf7903 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sIndex\sand\sTable\sobjects\sso\sthat\sthey\sremember\sif\stheir\sstats\scome\nfrom\sthe\ssqlite_stat1\stable.\s\sMake\sthe\s"PRAGMA\sstats"\san\sSQLITE_DEBUG\sonly\npragma.\s\sAdd\sthe\sflags\scolumn\sto\s"PRAGMA\sstats".\s\sThese\sare\sall\spreliminary\nsteps\stoward\sa\s"PRAGMA\sanalyze_ifneeded;"\sfeature. -D 2017-02-17T13:38:15.256 +C Set\sthe\sTF_StatsUsed\sflag\son\stables\swhen\sthe\squery\splanner\soutcome\sis\naffected\sby\sthe\ssqlite_stat1\sdata.\s\sAlso,\schange\sthe\scolumn\snames\sof\sthe\n"PRAGMA\sstats"\scommand\sso\sthat\sthey\sare\snot\skeywords. +D 2017-02-17T15:26:36.765 F Makefile.in edb6bcdd37748d2b1c3422ff727c748df7ffe918 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a89ea37ab5928026001569f056973b9059492fe2 @@ -388,7 +388,7 @@ F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 F src/pragma.c d4918a737f0bf1ad825654ebf07c4d009ff0ca63 -F src/pragma.h cea24a631982fd1a26fcddd46f596d22303b4247 +F src/pragma.h 9e65a9033cce7387c27cd89aecb2b2f205d1edf7 F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a F src/printf.c 67427bbee66d891fc6f6f5aada857e9cdb368c1c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 @@ -399,7 +399,7 @@ F src/shell.c bb8e20789499aec921a01d8744c616b81b8214f1 F src/sqlite.h.in 751ff125eb159c8f92c182b8df980a5e4f50e966 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae -F src/sqliteInt.h 54bc20855f2a1a99a1b726f1384b5ce9ab67a0ff +F src/sqliteInt.h 5d50606deed2b38b35fb1b5e4ab658f8faa37d4a F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -475,8 +475,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 -F src/where.c 01baf58b72f3ddb7793cdee2871f751e3e09b35e -F src/whereInt.h c0b092180f04608d80c258174b0a14a1f9c8d02f +F src/where.c 1a3a8adb717a20f17c186f3baa22b0b5f3a5ab13 +F src/whereInt.h 2d50c2b74a33be44cb68fdecee30b4d93552f1f4 F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04 F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -521,6 +521,7 @@ F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 F test/auth.test c6ede04bee65637ff354b43fc1235aa560c0863e F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 0d48b901cf111c14b4b1b5205c7d28f1a278190f +F test/autoanalyze1.test b31c18e73cf4e97b896288d8c817fce743f23e76 F test/autoinc.test 6ae8fb69c9f656962464ae4e6667045d0dfc3b46 F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df @@ -1494,7 +1495,7 @@ F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 9c0a855e0daf83e54ffd9fd438260cd0c92f25d0 +F tool/mkpragmatab.tcl c955db934f7b1d800081447042cef692f26b2516 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e @@ -1556,10 +1557,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 218b2bbb0de07288889f6762d4461ea8acd78969 -R af8aeffdf276070e8e8912906300a1a4 -T *branch * auto-analyze -T *sym-auto-analyze * -T -sym-trunk * +P 85026c8ee143bbd46565660fff8346ef81421546 +R 7cc0cc258beb93b198b99dc8e7b58520 U drh -Z f8720207819c88fb362c87de069d287a +Z 76eea0dfcfb8fa07347b6cdca98e2297 diff --git a/manifest.uuid b/manifest.uuid index fd9de00f55..1142241825 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -85026c8ee143bbd46565660fff8346ef81421546 \ No newline at end of file +fb2b8ae8310e4ea4b42354bbf36c3084a9d5c6d7 \ No newline at end of file diff --git a/src/pragma.h b/src/pragma.h index 2a96ecaa43..1592bbd2d5 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -71,11 +71,11 @@ static const char *const pragCName[] = { /* 4 */ "notnull", /* 5 */ "dflt_value", /* 6 */ "pk", - /* 7 */ "table", /* Used by: stats */ - /* 8 */ "index", - /* 9 */ "width", - /* 10 */ "height", - /* 11 */ "flags", + /* 7 */ "tbl", /* Used by: stats */ + /* 8 */ "idx", + /* 9 */ "wdth", + /* 10 */ "hght", + /* 11 */ "flgs", /* 12 */ "seqno", /* Used by: index_info */ /* 13 */ "cid", /* 14 */ "name", diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6d58157b1b..3c75f2ff6e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1883,10 +1883,13 @@ struct Table { #define TF_Ephemeral 0x0002 /* An ephemeral table */ #define TF_HasPrimaryKey 0x0004 /* Table has a primary key */ #define TF_Autoincrement 0x0008 /* Integer primary key is autoincrement */ -#define TF_HasStat1 0x0010 /* Has STAT1 data */ +#define TF_HasStat1 0x0010 /* nRowLogEst set from sqlite_stat1 */ #define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ #define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ +#define TF_SizeChng 0x0100 /* nRowLogEst might be inaccurate */ +#define TF_StatsUsed 0x0200 /* Query planner decisions affected by + ** Index.aiRowLogEst[] values */ /* ** Test to see whether or not a table is a virtual table. This is @@ -2129,7 +2132,7 @@ struct Index { unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ - unsigned hasStat1:1; /* True if stat1 data has been loaded */ + unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ diff --git a/src/where.c b/src/where.c index f2bf400a2d..8d0dbb0979 100644 --- a/src/where.c +++ b/src/where.c @@ -2387,6 +2387,11 @@ static int whereLoopAddBtreeIndex( continue; } + if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ + pBuilder->bldFlags |= SQLITE_BLDF_UNIQUE; + }else{ + pBuilder->bldFlags |= SQLITE_BLDF_INDEXED; + } pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->u.btree.nBtm = saved_nBtm; @@ -2934,7 +2939,15 @@ static int whereLoopAddBtree( } } + pBuilder->bldFlags = 0; rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); + if( pBuilder->bldFlags==SQLITE_BLDF_INDEXED ){ + /* If a non-unique index is used, or if a prefix of the key for + ** unique index is used (making the index functionally non-unique) + ** then the sqlite_stat1 data becomes important for scoring the + ** plan */ + pTab->tabFlags |= TF_StatsUsed; + } #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); pBuilder->nRecValid = 0; diff --git a/src/whereInt.h b/src/whereInt.h index c6195f53ed..f065fae6ba 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -395,8 +395,13 @@ struct WhereLoopBuilder { UnpackedRecord *pRec; /* Probe for stat4 (if required) */ int nRecValid; /* Number of valid fields currently in pRec */ #endif + unsigned int bldFlags; /* SQLITE_BLDF_* flags */ }; +/* Allowed values for WhereLoopBuider.bldFlags */ +#define SQLITE_BLDF_INDEXED 0x0001 /* An index is used */ +#define SQLITE_BLDF_UNIQUE 0x0002 /* All keys of a UNIQUE index used */ + /* ** The WHERE clause processing routine has two halves. The ** first part does the start of the WHERE loop and the second diff --git a/test/autoanalyze1.test b/test/autoanalyze1.test new file mode 100644 index 0000000000..c36e0fcbc7 --- /dev/null +++ b/test/autoanalyze1.test @@ -0,0 +1,124 @@ +# 2017-02-17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests for the logic used to estimate when +# running ANALYZE would be beneficial. +# +# Note that this test uses some hard-coded bitmask values from sqliteInt.h. +# If any of the following constants changes: +# +# define TF_HasStat1 0x0010 +# define TF_SizeChng 0x0100 +# define TF_StatsUsed 0x0200 +# +# then some of the magic numbers in test results below might need to be +# adjusted. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# There is nothing to test if ANALYZE is disable for this build. +# These tests also use "PRAGMA stats" which are only enabled for +# debugging builds. +# +ifcapable {!debug || !analyze} { + finish_test + return +} + +do_execsql_test autoanalyze1-100 { + -- Build up a test table with some indexes + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + CREATE UNIQUE INDEX t1bc ON t1(b,c); + CREATE INDEX t1d ON t1(d); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a,b,c,d) SELECT x, x, x, x FROM c; + -- Verify that the hasStat1 flag is clear on on indexes + SELECT idx, flgs FROM pragma_stats + WHERE idx IS NOT NULL + ORDER BY idx; + -- Verify that the TF_HasStat1 flag is clear on the table + SELECT tbl, (flgs & 0x10)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {t1bc 0 t1d 0 t1 0} + +# No use of stat1 recorded so far +do_execsql_test autoanalyze1-110 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +# Access using a unique index does not set the TF_StatsUsed flag. +# +do_execsql_test autoanalyze1-200 { + SELECT * FROM t1 WHERE a=55; +} {55 55 55 55} +do_execsql_test autoanalyze1-201 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +do_execsql_test autoanalyze1-210 { + SELECT * FROM t1 WHERE a IN (55,199,299); +} {55 55 55 55} +do_execsql_test autoanalyze1-211 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +do_execsql_test autoanalyze1-220 { + SELECT * FROM t1 WHERE (b,c)=(45,45); +} {45 45 45 45} +do_execsql_test autoanalyze1-221 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +# Any use of the non-unique t1d index triggers the use of stats. +# +sqlite3 db test.db +do_execsql_test autoanalyze1-300 { + SELECT * FROM t1 WHERE d=45; +} {45 45 45 45} +do_execsql_test autoanalyze1-301 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {1} + +sqlite3 db test.db +do_execsql_test autoanalyze1-310 { + SELECT * FROM t1 WHERE d=45 AND a=45; +} {45 45 45 45} +do_execsql_test autoanalyze1-311 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} ;# The ROWID lookup short-circuits the d=45 constraint + +sqlite3 db test.db +do_execsql_test autoanalyze1-320 { + SELECT * FROM t1 WHERE d=45 AND a IN (45,46); +} {45 45 45 45} +do_execsql_test autoanalyze1-321 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {1} + +# Any use of prefix of a unique index triggers the use of stats +# +sqlite3 db test.db +do_execsql_test autoanalyze1-400 { + SELECT * FROM t1 WHERE b=45; +} {45 45 45 45} +do_execsql_test autoanalyze1-401 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {1} + +# The TF_StatsUsed flag is reset when the database is reopened +# +sqlite3 db test.db +do_execsql_test autoanalyze1-500 { + SELECT (flgs & 0x0200)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL; +} {0} + +finish_test diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 1c8bad54d3..005c4bc48e 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -225,7 +225,7 @@ set pragma_def { NAME: stats FLAG: NeedSchema Result0 SchemaReq - COLS: table index width height flags + COLS: tbl idx wdth hght flgs IF: !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) NAME: index_info