From: drh Date: Fri, 4 Oct 2013 15:30:21 +0000 (+0000) Subject: Improved estimates of the relative speed of index scans based on declared X-Git-Tag: version-3.8.1~39^2~17 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=fdaac671b8004ee478961ec6a22db4ffa4eaaab2;p=thirdparty%2Fsqlite.git Improved estimates of the relative speed of index scans based on declared datatypes of columns in the table. Add "r" column to PRAGMA index_info, showing the estimated relative scan rate. FossilOrigin-Name: 07462bb6059f023c22a6c84a4a02afbd84e69255 --- diff --git a/manifest b/manifest index 31dd5afb71..2f950ae931 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Progress\stoward\susing\sthe\siScanRatio\sinformation\son\sindices.\s\sMany\stests\nare\sstill\sfailing. -D 2013-10-04T02:36:19.375 +C Improved\sestimates\sof\sthe\srelative\sspeed\sof\sindex\sscans\sbased\son\sdeclared\ndatatypes\sof\scolumns\sin\sthe\stable.\s\sAdd\s"r"\scolumn\sto\sPRAGMA\sindex_info,\nshowing\sthe\sestimated\srelative\sscan\srate. +D 2013-10-04T15:30:21.142 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,13 +166,13 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 5ccbbaed7a32ba774306f610da4ab4f3e5348294 F src/btree.h bfe0e8c5759b4ec77b0d18390064a6ef3cdffaaf F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 -F src/build.c 7926e17a5dc3f698dd8b15511a4902f48c73ac50 +F src/build.c 7a91b0db709404c8b606a6d8abcaf29b1bf75bbf F src/callback.c f99a8957ba2adf369645fac0db09ad8adcf1caa2 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c ea4b7f3623a0fcb1146e7f245d7410033e86859c F src/date.c 65196e95e69f36993659bd7781abe7c2f1994739 F src/delete.c 45788c5e48734f2af4acd75a876466e5b9838e34 -F src/expr.c 1017f482217e093ecdaca91e8666681e83733252 +F src/expr.c e7338ccffdc391c53ba2d51c5eb6a2f5299e040e F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c be866cd8c4fa6cae98ba33109578fd1a3311ee5b F src/func.c 2c47b65e6e00e3e9374942f28254faf8adafe398 @@ -210,7 +210,7 @@ F src/parse.y a97566d6da75075589a7c716d1bda14b586cf8da F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c a467393909a4ed7ca9de066d85ba5c5b04a5be63 -F src/pragma.c 5a4d8d9c3c7d22ef920487a260ea3d4b59d3b3a6 +F src/pragma.c 3bcdc7a98ad9429218179d2d81fc4aa9e39d8592 F src/prepare.c fa6988589f39af8504a61731614cd4f6ae71554f F src/printf.c da9119eb31a187a4b99f60aa4a225141c0ebb74b F src/random.c 0b2dbc37fdfbfa6bd455b091dfcef5bdb32dba68 @@ -221,7 +221,7 @@ F src/shell.c 5ee50ca3e35453bbd6ccdf1bdd0f6bbe9738e9fb F src/sqlite.h.in ec40aa958a270416fb04b4f72210357bf163d2c5 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h c32d524facfb39084cc1a830aef0c31aeb93c04e +F src/sqliteInt.h da0e92df9550e57107aa43eab63b29d040fe9c80 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -722,7 +722,7 @@ F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 F test/permutations.test 72f4f8881d388163ddbbeec0a6ed812e863ea2e6 -F test/pragma.test 5e7de6c32a5d764f09437d2025f07e4917b9e178 +F test/pragma.test de128f0253ee42a1edf6c80f21813babfb19bd3b F test/pragma2.test 224f0381f9411a78ae685cac24c13656a62021b7 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d @@ -1119,7 +1119,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d59d97b0a8d70dc31d45db61bbc11ebb5375a224 -R 753a9b9a56a7c0c63483772726286f21 +P 6c352edbba85a15ca356b5e131f4b3b2723d1774 +R bacabbd9616babddd027da683b20e83a U drh -Z b513b854bdc830eeffd07ca885795f7d +Z 469f9776554648d8f1435b44fd89545f diff --git a/manifest.uuid b/manifest.uuid index fe8c52efa1..4822e4715e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c352edbba85a15ca356b5e131f4b3b2723d1774 \ No newline at end of file +07462bb6059f023c22a6c84a4a02afbd84e69255 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 83ce948dc1..245961808f 100644 --- a/src/build.c +++ b/src/build.c @@ -1026,6 +1026,7 @@ void sqlite3AddColumn(Parse *pParse, Token *pName){ ** be called next to set pCol->affinity correctly. */ pCol->affinity = SQLITE_AFF_NONE; + pCol->szEst = 1; p->nCol++; } @@ -1067,15 +1068,18 @@ void sqlite3AddNotNull(Parse *pParse, int onError){ ** If none of the substrings in the above table are found, ** SQLITE_AFF_NUMERIC is returned. */ -char sqlite3AffinityType(const char *zIn){ +char sqlite3AffinityType(const char *zIn, u8 *pszEst){ u32 h = 0; char aff = SQLITE_AFF_NUMERIC; + const char *zChar; - if( zIn ) while( zIn[0] ){ + if( zIn==0 ) return aff; + while( zIn[0] ){ h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; zIn++; if( h==(('c'<<24)+('h'<<16)+('a'<<8)+'r') ){ /* CHAR */ - aff = SQLITE_AFF_TEXT; + aff = SQLITE_AFF_TEXT; + zChar = zIn; }else if( h==(('c'<<24)+('l'<<16)+('o'<<8)+'b') ){ /* CLOB */ aff = SQLITE_AFF_TEXT; }else if( h==(('t'<<24)+('e'<<16)+('x'<<8)+'t') ){ /* TEXT */ @@ -1099,7 +1103,25 @@ char sqlite3AffinityType(const char *zIn){ break; } } - + if( pszEst ){ + if( aff>=SQLITE_AFF_NUMERIC ){ + *pszEst = 1; + }else if( zChar ){ + *pszEst = 1; + while( zChar[0] ){ + int v; + if( sqlite3Isdigit(zChar[0]) && sqlite3GetInt32(zChar, &v) ){ + v = v/4 + 1; + if( v>255 ) v = 255; + *pszEst = v; + break; + } + zChar++; + } + }else{ + *pszEst = 3; + } + } return aff; } @@ -1121,7 +1143,7 @@ void sqlite3AddColumnType(Parse *pParse, Token *pType){ pCol = &p->aCol[p->nCol-1]; assert( pCol->zType==0 ); pCol->zType = sqlite3NameFromToken(pParse->db, pType); - pCol->affinity = sqlite3AffinityType(pCol->zType); + pCol->affinity = sqlite3AffinityType(pCol->zType, &pCol->szEst); } /* @@ -1469,7 +1491,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){ zType = azType[pCol->affinity - SQLITE_AFF_TEXT]; len = sqlite3Strlen30(zType); assert( pCol->affinity==SQLITE_AFF_NONE - || pCol->affinity==sqlite3AffinityType(zType) ); + || pCol->affinity==sqlite3AffinityType(zType, 0) ); memcpy(&zStmt[k], zType, len); k += len; assert( k<=n ); @@ -1478,6 +1500,39 @@ static char *createTableStmt(sqlite3 *db, Table *p){ return zStmt; } +/* +** Estimate the total row width for a table. +*/ +static unsigned estimatedTableWidth(const Table *pTab){ + unsigned wTable = 0; + const Column *pTabCol; + int i; + for(i=pTab->nCol, pTabCol=pTab->aCol; i>0; i--, pTabCol++){ + wTable += pTabCol->szEst; + } + if( pTab->iPKey<0 ) wTable++; + return wTable; +} + +/* +** Set the iScanRatio for an index based on estimates of the average +** table row width and average index row width. Estimates are derived +** from the declared datatypes of the various columns. +*/ +static void setIndexScanRatio(Index *pIdx, unsigned wTable){ + unsigned wIndex = 1; + int i; + const Column *aCol = pIdx->pTable->aCol; + for(i=0; inColumn; i++){ + assert( pIdx->aiColumn[i]>=0 && pIdx->aiColumn[i]pTable->nCol ); + wIndex += aCol[pIdx->aiColumn[i]].szEst; + } + assert( 100*wIndex/wTable <= 255 ); + pIdx->iScanRatio = (u8)(128*wIndex/wTable); + /* printf("%s: wIndex=%d wTable=%d ratio=%d\n", + ** pIdx->zName, wIndex, wTable, (100*pIdx->iScanRatio)/128); */ +} + /* ** This routine is called to report the final ")" that terminates ** a CREATE TABLE statement. @@ -1504,9 +1559,11 @@ void sqlite3EndTable( Token *pEnd, /* The final ')' token in the CREATE TABLE */ Select *pSelect /* Select from a "CREATE ... AS SELECT" */ ){ - Table *p; - sqlite3 *db = pParse->db; - int iDb; + Table *p; /* The new table */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb; /* Database in which the table lives */ + Index *pIdx; /* An implied index of the table */ + unsigned wTable; /* Estimated average width of a row in the table */ if( (pEnd==0 && pSelect==0) || db->mallocFailed ){ return; @@ -1526,6 +1583,12 @@ void sqlite3EndTable( } #endif /* !defined(SQLITE_OMIT_CHECK) */ + /* Compute the iScanRatio of implied indices */ + wTable = estimatedTableWidth(p); + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + setIndexScanRatio(pIdx, wTable); + } + /* If the db->init.busy is 1 it means we are reading the SQL off the ** "sqlite_master" or "sqlite_temp_master" table on the disk. ** So do not write to the disk again. Extract the root page number @@ -2442,15 +2505,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp1(v, OP_Close, iSorter); } -/* -** Estimate the average width of a table column based on its affinity type. -*/ -static unsigned estimatedColumnWidth(const Column *p){ - if( p->affinity>=SQLITE_AFF_NUMERIC ) return 1; - if( p->affinity==SQLITE_AFF_TEXT ) return 3; - return 2; -} - /* ** Create a new index for an SQL table. pName1.pName2 is the name of the index ** and pTblList is the name of the table that is to be indexed. Both will @@ -2493,8 +2547,6 @@ Index *sqlite3CreateIndex( int iDb; /* Index of the database that is being written */ Token *pName = 0; /* Unqualified name of the index to create */ struct ExprList_item *pListItem; /* For looping over pList */ - unsigned wTable = 0; /* Approximate "width" of the table */ - unsigned wIndex = 0; /* Approximate "width" of this index */ const Column *pTabCol; /* A column in the table */ int nCol; /* Number of columns */ int nExtra = 0; /* Space allocated for zExtra[] */ @@ -2738,7 +2790,6 @@ Index *sqlite3CreateIndex( goto exit_create_index; } pIndex->aiColumn[i] = j; - wIndex += estimatedColumnWidth(pTabCol); if( pListItem->pExpr ){ int nColl; assert( pListItem->pExpr->op==TK_COLLATE ); @@ -2762,11 +2813,9 @@ Index *sqlite3CreateIndex( if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0; } sqlite3DefaultRowEst(pIndex); - for(j=pTab->nCol, pTabCol=pTab->aCol; j>0; j--, pTabCol++){ - wTable += estimatedColumnWidth(pTabCol); + if( pParse->pNewTable==0 ){ + setIndexScanRatio(pIndex, estimatedTableWidth(pTab)); } - assert( 100*wIndex/wTable <= 255 ); - pIndex->iScanRatio = (u8)(128*wIndex/wTable); if( pTab==pParse->pNewTable ){ /* This routine has been called to create an automatic index as a diff --git a/src/expr.c b/src/expr.c index b8b0eb7313..eb2f545639 100644 --- a/src/expr.c +++ b/src/expr.c @@ -41,7 +41,7 @@ char sqlite3ExprAffinity(Expr *pExpr){ #ifndef SQLITE_OMIT_CAST if( op==TK_CAST ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); - return sqlite3AffinityType(pExpr->u.zToken); + return sqlite3AffinityType(pExpr->u.zToken, 0); } #endif if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER) @@ -2461,7 +2461,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ int aff, to_op; inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); assert( !ExprHasProperty(pExpr, EP_IntValue) ); - aff = sqlite3AffinityType(pExpr->u.zToken); + aff = sqlite3AffinityType(pExpr->u.zToken, 0); to_op = aff - SQLITE_AFF_TEXT + OP_ToText; assert( to_op==OP_ToText || aff!=SQLITE_AFF_TEXT ); assert( to_op==OP_ToBlob || aff!=SQLITE_AFF_NONE ); @@ -3128,7 +3128,7 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ const char *zAff = "unk"; - switch( sqlite3AffinityType(pExpr->u.zToken) ){ + switch( sqlite3AffinityType(pExpr->u.zToken, 0) ){ case SQLITE_AFF_TEXT: zAff = "TEXT"; break; case SQLITE_AFF_NONE: zAff = "NONE"; break; case SQLITE_AFF_NUMERIC: zAff = "NUMERIC"; break; diff --git a/src/pragma.c b/src/pragma.c index f533b71c21..a0f49b9080 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1455,17 +1455,19 @@ void sqlite3Pragma( pIdx = pTab->pIndex; if( pIdx ){ int i = 0; - sqlite3VdbeSetNumCols(v, 3); - pParse->nMem = 3; + sqlite3VdbeSetNumCols(v, 4); + pParse->nMem = 4; sqlite3CodeVerifySchema(pParse, iDb); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "r", SQLITE_STATIC); while(pIdx){ sqlite3VdbeAddOp2(v, OP_Integer, i, 1); sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->onError!=OE_None, 3); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->iScanRatio*100/128, 4); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); ++i; pIdx = pIdx->pNext; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 2fb5c40639..5acfe38c5b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1197,7 +1197,8 @@ struct Column { char *zColl; /* Collating sequence. If NULL, use the default */ u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ char affinity; /* One of the SQLITE_AFF_... values */ - u16 colFlags; /* Boolean properties. See COLFLAG_ defines below */ + u8 szEst; /* Estimated size of this column. INT==1 */ + u8 colFlags; /* Boolean properties. See COLFLAG_ defines below */ }; /* Allowed values for Column.colFlags: @@ -3082,7 +3083,7 @@ void sqlite3ColumnDefault(Vdbe *, Table *, int, int); void sqlite3AlterFinishAddColumn(Parse *, Token *); void sqlite3AlterBeginAddColumn(Parse *, SrcList *); CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); -char sqlite3AffinityType(const char*); +char sqlite3AffinityType(const char*, u8*); void sqlite3Analyze(Parse*, Token*, Token*); int sqlite3InvokeBusyHandler(BusyHandler*); int sqlite3FindDb(sqlite3*, Token*); diff --git a/test/pragma.test b/test/pragma.test index a6d198eb69..dce4cfda40 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -574,7 +574,7 @@ ifcapable {foreignkey} { execsql { pragma index_list(t3); } - } {0 sqlite_autoindex_t3_1 1} + } {/0 sqlite_autoindex_t3_1 1 \d+/} } ifcapable {!foreignkey} { execsql {CREATE TABLE t3(a,b UNIQUE)} @@ -647,7 +647,7 @@ do_test pragma-7.1.1 { execsql { pragma index_list(t3); } -} {0 t3i1 0 1 sqlite_autoindex_t3_1 1} +} {/0 t3i1 0 \d+ 1 sqlite_autoindex_t3_1 1 \d+/} do_test pragma-7.1.2 { execsql { pragma index_list(t3_bogus); @@ -1661,7 +1661,7 @@ do_test 23.3 { CREATE INDEX i3 ON t1(d,b,c); } db2 eval {PRAGMA index_list(t1)} -} {0 i3 0 1 i2 0 2 i1 0} +} {/0 i3 0 \d+ 1 i2 0 \d+ 2 i1 0 \d+/} do_test 23.4 { db eval { ALTER TABLE t1 ADD COLUMN e;