From: dan Date: Fri, 7 Apr 2017 20:14:22 +0000 (+0000) Subject: Refactor code to suggest indexes from the shell tool into an extension in X-Git-Tag: version-3.22.0~147^2~41 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=8e0b8e06884c56eb0e275d3f3cfb18510de84db8;p=thirdparty%2Fsqlite.git Refactor code to suggest indexes from the shell tool into an extension in ext/expert. Unfinished. FossilOrigin-Name: 305e19f976ca064638a294e0817bf547ea745e1eb74746c5855514e6ced9c5fa --- diff --git a/ext/expert/expert.c b/ext/expert/expert.c new file mode 100644 index 0000000000..5a5cda185f --- /dev/null +++ b/ext/expert/expert.c @@ -0,0 +1,104 @@ +/* +** 2017 April 07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + + +#include +#include +#include +#include +#include "sqlite3expert.h" + + +static void option_requires_argument(const char *zOpt){ + fprintf(stderr, "Option requires an argument: %s\n", zOpt); + exit(-3); +} + +static void usage(char **argv){ + fprintf(stderr, "\n"); + fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]); + fprintf(stderr, "\n"); + fprintf(stderr, "Options are:\n"); + fprintf(stderr, " -sql SQL (analyze SQL statements passed as argument)\n"); + fprintf(stderr, " -file FILE (read SQL statements from file FILE)\n"); + exit(-1); +} + +static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){ + return SQLITE_OK; +} + +int main(int argc, char **argv){ + const char *zDb; + int rc = 0; + char *zErr = 0; + int i; + + sqlite3 *db = 0; + sqlite3expert *p = 0; + + if( argc<2 ) usage(argv); + zDb = argv[argc-1]; + rc = sqlite3_open(zDb, &db); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db)); + exit(-2); + } + + p = sqlite3_expert_new(db, &zErr); + if( p==0 ){ + fprintf(stderr, "Cannot run analysis: %s\n", zErr); + rc = 1; + }else{ + for(i=1; i<(argc-1); i++){ + char *zArg = argv[i]; + int nArg = strlen(zArg); + if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){ + if( ++i==(argc-1) ) option_requires_argument("-file"); + rc = readSqlFromFile(p, argv[i], &zErr); + } + + else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){ + if( ++i==(argc-1) ) option_requires_argument("-sql"); + rc = sqlite3_expert_sql(p, argv[i], &zErr); + } + + else{ + usage(argv); + } + } + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_expert_analyze(p, &zErr); + } + + if( rc==SQLITE_OK ){ + int nQuery = sqlite3_expert_count(p); + for(i=0; i +#include +#include + typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; typedef struct IdxConstraint IdxConstraint; typedef struct IdxContext IdxContext; typedef struct IdxScan IdxScan; +typedef struct IdxStatement IdxStatement; typedef struct IdxWhere IdxWhere; typedef struct IdxColumn IdxColumn; @@ -70,6 +76,19 @@ struct IdxScan { IdxScan *pNextScan; /* Next IdxScan object for same query */ }; +/* +** Data regarding a database table. Extracted from "PRAGMA table_info" +*/ +struct IdxColumn { + char *zName; + char *zColl; + int iPk; +}; +struct IdxTable { + int nCol; + IdxColumn *aCol; +}; + /* ** Context object passed to idxWhereInfo() and other functions. */ @@ -84,19 +103,31 @@ struct IdxContext { i64 iIdxRowid; /* Rowid of first index created */ }; +struct IdxStatement { + int iId; /* Statement number */ + char *zSql; /* SQL statement */ + char *zIdx; /* Indexes */ + char *zEQP; /* Plan */ + IdxStatement *pNext; +}; + /* -** Data regarding a database table. Extracted from "PRAGMA table_info" +** sqlite3expert object. */ -struct IdxColumn { - char *zName; - char *zColl; - int iPk; -}; -struct IdxTable { - int nCol; - IdxColumn *aCol; +struct sqlite3expert { + sqlite3 *db; /* Users database */ + sqlite3 *dbm; /* In-memory db for this analysis */ + + int bRun; /* True once analysis has run */ + char **pzErrmsg; + + IdxScan *pScan; /* List of scan objects */ + IdxStatement *pStatement; /* List of IdxStatement objects */ + int rc; /* Error code from whereinfo hook */ + i64 iIdxRowid; /* Rowid of first index created */ }; + /* ** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). ** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL. @@ -132,7 +163,7 @@ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){ } /* -** SQLITE_DBCONFIG_WHEREINFO callback. +** sqlite3_whereinfo_hook() callback. */ static void idxWhereInfo( void *pCtx, /* Pointer to IdxContext structure */ @@ -141,7 +172,7 @@ static void idxWhereInfo( int iVal, u64 mask ){ - IdxContext *p = (IdxContext*)pCtx; + sqlite3expert *p = (sqlite3expert*)pCtx; #if 0 const char *zOp = @@ -165,7 +196,6 @@ static void idxWhereInfo( pNew->pNextScan = p->pScan; pNew->covering = mask; p->pScan = pNew; - p->pCurrent = &pNew->where; break; } @@ -193,16 +223,17 @@ static void idxWhereInfo( pNew->depmask = mask; if( eOp==SQLITE_WHEREINFO_RANGE ){ - pNew->pNext = p->pCurrent->pRange; - p->pCurrent->pRange = pNew; + pNew->pNext = p->pScan->where.pRange; + p->pScan->where.pRange = pNew; }else{ - pNew->pNext = p->pCurrent->pEq; - p->pCurrent->pEq = pNew; + pNew->pNext = p->pScan->where.pEq; + p->pScan->where.pEq = pNew; } - +#if 0 sqlite3_bind_int64(p->pInsertMask, 1, mask); sqlite3_step(p->pInsertMask); p->rc = sqlite3_reset(p->pInsertMask); +#endif break; } } @@ -325,35 +356,6 @@ static int idxGetTableInfo( return rc; } -static int idxCreateTables( - sqlite3 *db, /* User database */ - sqlite3 *dbm, /* In-memory database to create tables in */ - IdxScan *pScan, /* List of scans */ - char **pzErrmsg /* OUT: Error message */ -){ - int rc = SQLITE_OK; - int rc2; - IdxScan *pIter; - sqlite3_stmt *pSql = 0; - - /* Copy the entire schema of database [db] into [dbm]. */ - rc = idxPrintfPrepareStmt(db, &pSql, pzErrmsg, - "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'" - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ - const char *zSql = (const char*)sqlite3_column_text(pSql, 0); - rc = sqlite3_exec(dbm, zSql, 0, 0, pzErrmsg); - } - rc2 = sqlite3_finalize(pSql); - if( rc==SQLITE_OK ) rc = rc2; - - /* Load IdxTable objects */ - for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){ - rc = idxGetTableInfo(db, pIter, pzErrmsg); - } - return rc; -} - /* ** This function is a no-op if *pRc is set to anything other than ** SQLITE_OK when it is called. @@ -456,7 +458,6 @@ static int idxFindCompatible( int nEq = 0; /* Number of elements in pEq */ int rc, rc2; - /* Count the elements in list pEq */ for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++; @@ -514,12 +515,12 @@ static int idxFindCompatible( } static int idxCreateFromCons( - IdxContext *pCtx, + sqlite3expert *p, IdxScan *pScan, IdxConstraint *pEq, IdxConstraint *pTail ){ - sqlite3 *dbm = pCtx->dbm; + sqlite3 *dbm = p->dbm; int rc = SQLITE_OK; if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){ IdxTable *pTab = pScan->pTable; @@ -552,20 +553,20 @@ static int idxCreateFromCons( if( !zIdx ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_exec(dbm, zIdx, 0, 0, pCtx->pzErrmsg); -#if 0 + rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg); +#if 1 printf("CANDIDATE: %s\n", zIdx); #endif } } - if( rc==SQLITE_OK && pCtx->iIdxRowid==0 ){ + if( rc==SQLITE_OK && p->iIdxRowid==0 ){ int rc2; sqlite3_stmt *pLast = 0; - rc = idxPrepareStmt(dbm, &pLast, pCtx->pzErrmsg, + rc = idxPrepareStmt(dbm, &pLast, p->pzErrmsg, "SELECT max(rowid) FROM sqlite_master" ); if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLast) ){ - pCtx->iIdxRowid = sqlite3_column_int64(pLast, 0); + p->iIdxRowid = sqlite3_column_int64(pLast, 0); } rc2 = sqlite3_finalize(pLast); if( rc==SQLITE_OK ) rc = rc2; @@ -578,7 +579,7 @@ static int idxCreateFromCons( } static int idxCreateFromWhere( - IdxContext*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint* + sqlite3expert*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint* ); /* @@ -594,7 +595,7 @@ static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){ } static int idxCreateFromWhere( - IdxContext *pCtx, + sqlite3expert *p, i64 mask, /* Consider only these constraints */ IdxScan *pScan, /* Create indexes for this scan */ IdxWhere *pWhere, /* Read constraints from here */ @@ -618,7 +619,7 @@ static int idxCreateFromWhere( /* Create an index using the == constraints collected above. And the ** range constraint/ORDER BY terms passed in by the caller, if any. */ - rc = idxCreateFromCons(pCtx, pScan, p1, pTail); + rc = idxCreateFromCons(p, pScan, p1, pTail); /* If no range/ORDER BY passed by the caller, create a version of the ** index for each range constraint that matches the mask. */ @@ -629,7 +630,7 @@ static int idxCreateFromWhere( && idxFindConstraint(pEq, pCon)==0 && idxFindConstraint(pTail, pCon)==0 ){ - rc = idxCreateFromCons(pCtx, pScan, p1, pCon); + rc = idxCreateFromCons(p, pScan, p1, pCon); } } } @@ -641,56 +642,90 @@ static int idxCreateFromWhere( ** Create candidate indexes in database [dbm] based on the data in ** linked-list pScan. */ -static int idxCreateCandidates(IdxContext *pCtx){ - sqlite3 *dbm = pCtx->dbm; +static int idxCreateCandidates(sqlite3expert *p, char **pzErr){ + sqlite3 *dbm = p->dbm; int rc2; int rc = SQLITE_OK; - sqlite3_stmt *pDepmask; /* Foreach depmask */ + sqlite3_stmt *pDepmask = 0; /* Foreach depmask */ + sqlite3_stmt *pInsert = 0; /* insert */ IdxScan *pIter; - rc = idxPrepareStmt(dbm, &pDepmask, pCtx->pzErrmsg, - "SELECT mask FROM depmask" + rc = idxPrepareStmt(dbm, &pInsert, pzErr, + "INSERT OR IGNORE INTO aux.depmask SELECT mask | ?1 FROM aux.depmask;" ); + if( rc==SQLITE_OK ){ + rc = idxPrepareStmt(dbm, &pDepmask, pzErr, "SELECT mask FROM depmask"); + } - for(pIter=pCtx->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){ + for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){ IdxWhere *pWhere = &pIter->where; + IdxConstraint *pCons; + rc = sqlite3_exec(dbm, + "DELETE FROM aux.depmask;" + "INSERT INTO aux.depmask VALUES(0);" + , 0, 0, pzErr + ); + for(pCons=pIter->where.pEq; pCons; pCons=pCons->pNext){ + sqlite3_bind_int64(pInsert, 1, pCons->depmask); + sqlite3_step(pInsert); + rc = sqlite3_reset(pInsert); + } + while( SQLITE_ROW==sqlite3_step(pDepmask) && rc==SQLITE_OK ){ i64 mask = sqlite3_column_int64(pDepmask, 0); - rc = idxCreateFromWhere(pCtx, mask, pIter, pWhere, 0, 0); + rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, 0); if( rc==SQLITE_OK && pIter->pOrder ){ - rc = idxCreateFromWhere(pCtx, mask, pIter, pWhere, 0, pIter->pOrder); + rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, pIter->pOrder); } } + rc2 = sqlite3_reset(pDepmask); + if( rc==SQLITE_OK ) rc = rc2; } rc2 = sqlite3_finalize(pDepmask); if( rc==SQLITE_OK ) rc = rc2; + rc2 = sqlite3_finalize(pInsert); + if( rc==SQLITE_OK ) rc = rc2; return rc; } -static void idxScanFree(IdxScan *pScan){ - IdxScan *pIter; - IdxScan *pNext; - for(pIter=pScan; pIter; pIter=pNext){ - pNext = pIter->pNextScan; - } +/* +** Free all elements of the linked list starting from pScan up until pLast +** (pLast is not freed). +*/ +static void idxScanFree(IdxScan *pScan, IdxScan *pLast){ + /* TODO! */ +} + +/* +** Free all elements of the linked list starting from pStatement up +** until pLast (pLast is not freed). +*/ +static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){ + /* TODO! */ +} + +static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( *pRc==SQLITE_OK ) *pRc = rc; } +static void idxReset(int *pRc, sqlite3_stmt *pStmt){ + int rc = sqlite3_reset(pStmt); + if( *pRc==SQLITE_OK ) *pRc = rc; +} + int idxFindIndexes( - IdxContext *pCtx, - const char *zSql, /* SQL to find indexes for */ - void (*xOut)(void*, const char*), /* Output callback */ - void *pOutCtx, /* Context for xOut() */ + sqlite3expert *p, char **pzErr /* OUT: Error message (sqlite3_malloc) */ ){ - sqlite3 *dbm = pCtx->dbm; - sqlite3_stmt *pExplain = 0; + IdxStatement *pStmt; + sqlite3 *dbm = p->dbm; sqlite3_stmt *pSelect = 0; sqlite3_stmt *pInsert = 0; int rc, rc2; int bFound = 0; - rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,"EXPLAIN QUERY PLAN %s",zSql); if( rc==SQLITE_OK ){ rc = idxPrepareStmt(dbm, &pSelect, pzErr, "SELECT rowid, sql FROM sqlite_master WHERE name = ?" @@ -702,76 +737,81 @@ int idxFindIndexes( ); } - while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){ - int i; - const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3); - int nDetail = strlen(zDetail); - - for(i=0; ipStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){ + sqlite3_stmt *pExplain = 0; + rc = sqlite3_exec(dbm, "DELETE FROM aux.indexes", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr, + "EXPLAIN QUERY PLAN %s", pStmt->zSql + ); + } + while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){ + int i; + const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3); + int nDetail = strlen(zDetail); + + for(i=0; i=pCtx->iIdxRowid ){ - sqlite3_bind_text(pInsert, 1, zSql, -1, SQLITE_STATIC); - sqlite3_step(pInsert); - rc = sqlite3_reset(pInsert); - if( rc ) goto find_indexes_out; + if( zIdx ){ + int nIdx = 0; + while( zIdx[nIdx]!='\0' && (zIdx[nIdx]!=' ' || zIdx[nIdx+1]!='(') ){ + nIdx++; + } + sqlite3_bind_text(pSelect, 1, zIdx, nIdx, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + i64 iRowid = sqlite3_column_int64(pSelect, 0); + const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); + if( iRowid>=p->iIdxRowid ){ + sqlite3_bind_text(pInsert, 1, zSql, -1, SQLITE_STATIC); + sqlite3_step(pInsert); + rc = sqlite3_reset(pInsert); + if( rc ) goto find_indexes_out; + } } + rc = sqlite3_reset(pSelect); + break; } - rc = sqlite3_reset(pSelect); - break; } } - } - rc2 = sqlite3_reset(pExplain); - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK ){ - sqlite3_stmt *pLoop = 0; - rc = idxPrepareStmt(dbm, &pLoop, pzErr,"SELECT name||';' FROM aux.indexes"); + idxReset(&rc, pExplain); if( rc==SQLITE_OK ){ - while( SQLITE_ROW==sqlite3_step(pLoop) ){ - bFound = 1; - xOut(pOutCtx, (const char*)sqlite3_column_text(pLoop, 0)); + sqlite3_stmt *pLoop = 0; + rc = idxPrepareStmt(dbm,&pLoop,pzErr,"SELECT name||';' FROM aux.indexes"); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==sqlite3_step(pLoop) ){ + bFound = 1; + pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s\n", + (const char*)sqlite3_column_text(pLoop, 0) + ); + } + idxFinalize(&rc, pLoop); + } + if( bFound==0 ){ + pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "(no new indexes)\n"); } - rc = sqlite3_finalize(pLoop); - } - if( rc==SQLITE_OK ){ - if( bFound==0 ) xOut(pOutCtx, "(no new indexes)"); - xOut(pOutCtx, ""); } - } - while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){ - int iSelectid = sqlite3_column_int(pExplain, 0); - int iOrder = sqlite3_column_int(pExplain, 1); - int iFrom = sqlite3_column_int(pExplain, 2); - const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3); - char *zOut; + while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){ + int iSelectid = sqlite3_column_int(pExplain, 0); + int iOrder = sqlite3_column_int(pExplain, 1); + int iFrom = sqlite3_column_int(pExplain, 2); + const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3); - zOut = sqlite3_mprintf("%d|%d|%d|%s", iSelectid, iOrder, iFrom, zDetail); - if( zOut==0 ){ - rc = SQLITE_NOMEM; - }else{ - xOut(pOutCtx, zOut); - sqlite3_free(zOut); + pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%d|%d|%d|%s\n", + iSelectid, iOrder, iFrom, zDetail + ); } + + rc2 = sqlite3_finalize(pExplain); + if( rc==SQLITE_OK ) rc = rc2; } find_indexes_out: - rc2 = sqlite3_finalize(pExplain); - if( rc==SQLITE_OK ) rc = rc2; rc2 = sqlite3_finalize(pSelect); if( rc==SQLITE_OK ) rc = rc2; rc2 = sqlite3_finalize(pInsert); @@ -794,6 +834,7 @@ int shellIndexesCommand( char **pzErrmsg /* OUT: Error message (sqlite3_malloc) */ ){ int rc = SQLITE_OK; +#if 0 sqlite3 *dbm = 0; IdxContext ctx; sqlite3_stmt *pStmt = 0; /* Statement compiled from zSql */ @@ -851,10 +892,175 @@ int shellIndexesCommand( rc = idxFindIndexes(&ctx, zSql, xOut, pOutCtx, pzErrmsg); } - idxScanFree(ctx.pScan); + idxScanFree(ctx.pScan, 0); sqlite3_finalize(ctx.pInsertMask); sqlite3_close(dbm); +#endif return rc; } +/*************************************************************************/ + +/* +** Allocate a new sqlite3expert object. +*/ +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ + int rc = SQLITE_OK; + sqlite3expert *pNew; + + pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert)); + pNew->db = db; + + /* Open an in-memory database to work with. The main in-memory + ** database schema contains tables similar to those in the users + ** database (handle db). The attached in-memory db (aux) contains + ** application tables used by the code in this file. */ + rc = sqlite3_open(":memory:", &pNew->dbm); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(pNew->dbm, + "ATTACH ':memory:' AS aux;" + "CREATE TABLE aux.depmask(mask PRIMARY KEY) WITHOUT ROWID;" + "CREATE TABLE aux.indexes(name PRIMARY KEY) WITHOUT ROWID;" + , 0, 0, pzErrmsg + ); + } + + /* Copy the entire schema of database [db] into [dbm]. */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSql; + int rc2; + rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, + "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'" + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + const char *zSql = (const char*)sqlite3_column_text(pSql, 0); + rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg); + } + rc2 = sqlite3_finalize(pSql); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* If an error has occurred, free the new object and reutrn NULL. Otherwise, + ** return the new sqlite3expert handle. */ + if( rc!=SQLITE_OK ){ + sqlite3_expert_destroy(pNew); + pNew = 0; + } + return pNew; +} + +/* +** Add an SQL statement to the analysis. +*/ +int sqlite3_expert_sql( + sqlite3expert *p, /* From sqlite3_expert_new() */ + const char *zSql, /* SQL statement to add */ + char **pzErr /* OUT: Error message (if any) */ +){ + IdxScan *pScanOrig = p->pScan; + IdxStatement *pStmtOrig = p->pStatement; + int rc = SQLITE_OK; + const char *zStmt = zSql; + + if( p->bRun ) return SQLITE_MISUSE; + + sqlite3_whereinfo_hook(p->db, idxWhereInfo, p); + while( rc==SQLITE_OK && zStmt && zStmt[0] ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(p->db, zStmt, -1, &pStmt, &zStmt); + if( rc==SQLITE_OK ){ + if( pStmt ){ + IdxStatement *pNew; + const char *z = sqlite3_sql(pStmt); + int n = strlen(z); + pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1); + if( rc==SQLITE_OK ){ + pNew->zSql = (char*)&pNew[1]; + memcpy(pNew->zSql, z, n+1); + pNew->pNext = p->pStatement; + if( p->pStatement ) pNew->iId = p->pStatement->iId+1; + p->pStatement = pNew; + } + sqlite3_finalize(pStmt); + } + }else{ + idxDatabaseError(p->db, pzErr); + } + } + sqlite3_whereinfo_hook(p->db, 0, 0); + + if( rc!=SQLITE_OK ){ + idxScanFree(p->pScan, pScanOrig); + idxStatementFree(p->pStatement, pStmtOrig); + p->pScan = pScanOrig; + p->pStatement = pStmtOrig; + } + + return rc; +} + +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){ + int rc = SQLITE_OK; + IdxScan *pIter; + + /* Load IdxTable objects */ + for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){ + rc = idxGetTableInfo(p->dbm, pIter, pzErr); + } + + + /* Create candidate indexes within the in-memory database file */ + if( rc==SQLITE_OK ){ + rc = idxCreateCandidates(p, pzErr); + } + + /* Figure out which of the candidate indexes are preferred by the query + ** planner and report the results to the user. */ + if( rc==SQLITE_OK ){ + rc = idxFindIndexes(p, pzErr); + } + + if( rc==SQLITE_OK ){ + p->bRun = 1; + } + return rc; +} + +int sqlite3_expert_count(sqlite3expert *p){ + int nRet = 0; + if( p->pStatement ) nRet = p->pStatement->iId+1; + return nRet; +} + +const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){ + const char *zRet = 0; + IdxStatement *pStmt; + + if( p->bRun==0 ) return 0; + for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext); + if( pStmt ){ + switch( eReport ){ + case EXPERT_REPORT_SQL: + zRet = pStmt->zSql; + break; + case EXPERT_REPORT_INDEXES: + zRet = pStmt->zIdx; + break; + case EXPERT_REPORT_PLAN: + zRet = pStmt->zEQP; + break; + } + } + return zRet; +} + +/* +** Free an sqlite3expert object. +*/ +void sqlite3_expert_destroy(sqlite3expert *p){ + sqlite3_close(p->dbm); + idxScanFree(p->pScan, 0); + idxStatementFree(p->pStatement, 0); + sqlite3_free(p); +} diff --git a/ext/expert/sqlite3expert.h b/ext/expert/sqlite3expert.h new file mode 100644 index 0000000000..6384f8e556 --- /dev/null +++ b/ext/expert/sqlite3expert.h @@ -0,0 +1,57 @@ +/* +** 2017 April 07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + + +#include "sqlite3.h" + +typedef struct sqlite3expert sqlite3expert; + +/* +** Create a new sqlite3expert object. +*/ +sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr); + +/* +** Add an SQL statement to the analysis. +*/ +int sqlite3_expert_sql( + sqlite3expert *p, /* From sqlite3_expert_new() */ + const char *zSql, /* SQL statement to add */ + char **pzErr /* OUT: Error message (if any) */ +); + +int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr); + +/* +** Return the total number of SQL queries loaded via sqlite3_expert_sql(). +*/ +int sqlite3_expert_count(sqlite3expert*); + +/* +** Return a component of the report. +*/ +const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport); + +/* +** Values for the third argument passed to sqlite3_expert_report(). +*/ +#define EXPERT_REPORT_SQL 1 +#define EXPERT_REPORT_INDEXES 2 +#define EXPERT_REPORT_PLAN 3 + +/* +** Free an (sqlite3expert*) handle allocated by sqlite3-expert_new(). +*/ +void sqlite3_expert_destroy(sqlite3expert*); + + diff --git a/main.mk b/main.mk index b22d07d5a2..cb41cb7e00 100644 --- a/main.mk +++ b/main.mk @@ -491,7 +491,7 @@ libsqlite3.a: $(LIBOBJ) $(AR) libsqlite3.a $(LIBOBJ) $(RANLIB) libsqlite3.a -sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h $(TOP)/src/shell_indexes.c +sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h $(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \ $(TOP)/src/shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB) @@ -761,6 +761,9 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl sqlite3_analyzer$(EXE): sqlite3_analyzer.c $(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) +sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c + $(TCCX) -DSQLITE_ENABLE_WHEREINFO_HOOK $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(THREADLIB) + sqlite3_schemalint.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/schemalint.tcl echo "#define TCLSH 2" > $@ echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@ @@ -979,6 +982,7 @@ clean: rm -f sqlite3rc.h rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c + rm -f sqlite3_expert sqlite3_expert.exe rm -f sqlite-*-output.vsix rm -f mptester mptester.exe rm -f fuzzershell fuzzershell.exe diff --git a/manifest b/manifest index ce8fc15893..8a7b97520c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sallow\sindexes\sto\sbe\srecommended\sfor\squeries\son\sSQL\sviews. -D 2017-04-06T18:44:18.391 +C Refactor\scode\sto\ssuggest\sindexes\sfrom\sthe\sshell\stool\sinto\san\sextension\sin\next/expert.\sUnfinished. +D 2017-04-07T20:14:22.417 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a @@ -40,6 +40,9 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef +F ext/expert/expert.c c73a0da702a2e9f5fd48046ab182683a73ee0318cefa3f45826536d015f39021 +F ext/expert/sqlite3expert.c 75ee320cf38b50df331232cbb60ec3b7703dd2770cc8df4ebbb445f664f6827d +F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea @@ -325,7 +328,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk e6f8f6895b75e4615a284b546933ca88fd56f3b85b560f4fb0da731abaa14f80 +F main.mk 6d4f1f1f78a6ac453d35f18c4e696fdefbe65dfec9530a41c5579ef8ec076072 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -402,8 +405,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa -F src/shell.c e524688c2544167f835ba43e24309f8707ca60c8ab6eb5c263a12c8618a233b8 -F src/shell_indexes.c 1f5ab036ec189411aeea27e6e74ab0009d831764d5d8517455dcb6b6a734beb7 +F src/shell.c ceb2b2f1f958ea2c47a7f37972d0f715fbf9dcf6a34a5e98c886b85e3ce6a238 F src/sqlite.h.in ae5c9cbf2e77492c319fca08769575d9695e64718a16d32324944d24e291bcf7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28 @@ -1571,7 +1573,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5cd070000da1d9e399090677b4db75dc5639c33211385d6eb84f14a4d0b617cd -R aa13547c61fd31f43629472eaf9b4008 +P 0884ff1da2e27b146c764b73cf299a1f2cfe213c4a79bde34dec02d1fc946e70 +R 2f1dd4cb0b48a4b54a46f30181a94d12 U dan -Z 2f3b5e2a515dd18e4123d763066c9a33 +Z bc46ca2174a3c856c4daa1344a9d88d3 diff --git a/manifest.uuid b/manifest.uuid index 3b9f56b1a1..9b195887ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0884ff1da2e27b146c764b73cf299a1f2cfe213c4a79bde34dec02d1fc946e70 \ No newline at end of file +305e19f976ca064638a294e0817bf547ea745e1eb74746c5855514e6ced9c5fa \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index e51694bc76..1236b12a0e 100644 --- a/src/shell.c +++ b/src/shell.c @@ -166,7 +166,6 @@ static void setTextMode(FILE *file, int isOutput){ # define setTextMode(X,Y) #endif -#include "shell_indexes.c" /* True if the timer is enabled */ static int enableTimer = 0; @@ -1364,7 +1363,6 @@ struct ShellState { sqlite3 *db; /* The database */ int autoExplain; /* Automatically turn on .explain mode */ int autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ - int bRecommend; /* Instead of sqlite3_exec(), recommend indexes */ int statsOn; /* True to display memory stats before each finalize */ int scanstatsOn; /* True to display scan stats before each finalize */ int outCount; /* Revert to stdout when reaching zero */ @@ -2586,19 +2584,6 @@ static void explain_data_delete(ShellState *p){ p->iIndent = 0; } -typedef struct RecCommandCtx RecCommandCtx; -struct RecCommandCtx { - int (*xCallback)(void*,int,char**,char**,int*); - ShellState *pArg; -}; - -static void recCommandOut(void *pCtx, const char *zLine){ - const char *zCol = "output"; - RecCommandCtx *p = (RecCommandCtx*)pCtx; - int t = SQLITE_TEXT; - p->xCallback(p->pArg, 1, (char**)&zLine, (char**)&zCol, &t); -} - /* ** Disable and restore .wheretrace and .selecttrace settings. */ @@ -2723,13 +2708,6 @@ static int shell_exec( *pzErrMsg = NULL; } - if( pArg->bRecommend ){ - RecCommandCtx ctx; - ctx.xCallback = xCallback; - ctx.pArg = pArg; - rc = shellIndexesCommand(db, zSql, recCommandOut, &ctx, pzErrMsg); - }else - while( zSql[0] && (SQLITE_OK == rc) ){ static const char *zStmtSql; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); @@ -5522,15 +5500,6 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_close(pSrc); }else - if( c=='r' && n>=2 && strncmp(azArg[0], "recommend", n)==0 ){ - if( nArg==2 ){ - p->bRecommend = booleanValue(azArg[1]); - }else{ - raw_printf(stderr, "Usage: .recommend on|off\n"); - rc = 1; - } - }else - if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ @@ -7337,9 +7306,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( bail_on_error ) return rc; } } - - }else if( strcmp(z, "-recommend") ){ - data.bRecommend = 1; }else{ utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); raw_printf(stderr,"Use -help for a list of options.\n");