From: dan Date: Mon, 15 Feb 2016 20:12:16 +0000 (+0000) Subject: Progress towards integrating schemalint into the shell tool. Some cases work now. X-Git-Tag: version-3.22.0~147^2~55 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=47dd4867bc22c57668cc9fd11e49dbd1d70ccf55;p=thirdparty%2Fsqlite.git Progress towards integrating schemalint into the shell tool. Some cases work now. FossilOrigin-Name: 58d4cf26e15f90463148bec63d6ab514ffbbae60 --- diff --git a/manifest b/manifest index 7da4cdeb1d..c7c4b5f60c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\sintegration\sof\sschemalint\sfunctionality\swith\sthe\sshell\stool.\sDoes\snot\swork\syet. -D 2016-02-11T21:01:16.253 +C Progress\stowards\sintegrating\sschemalint\sinto\sthe\sshell\stool.\sSome\scases\swork\snow. +D 2016-02-15T20:12:16.368 F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b0493f10caddb8adf992a4e6f1943141fc7c6816 @@ -350,7 +350,7 @@ F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d F src/shell.c 2cde87e03712204231167c4a6c61b0eb5129e105 -F src/shell_indexes.c 3cff393ee86d15fbfbe31f30cd752b46d7779b52 +F src/shell_indexes.c eefa3e9b7d992d10d1f71e27a2e12024a838b365 F src/sqlite.h.in c7db059d3b810b70b83d9ed1436fa813eba22462 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d @@ -1430,7 +1430,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1a4182eedd0143c3f71b3d97f1d1bb25adeba617 -R 6253a80a5a7097c3b24b24ce68900613 +P ed49f297bcee86674ed673e195610b8cc1d35647 +R 6a53f87d207675e944487abae6392564 U dan -Z bddf3d3a7cd183a6a2362ed54e10358b +Z dd362fd8eca18359c7fdd88fbcc31963 diff --git a/manifest.uuid b/manifest.uuid index 38f9cc7372..9d51efbd2d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed49f297bcee86674ed673e195610b8cc1d35647 \ No newline at end of file +58d4cf26e15f90463148bec63d6ab514ffbbae60 \ No newline at end of file diff --git a/src/shell_indexes.c b/src/shell_indexes.c index 8e105fec1b..b7b44e47e0 100644 --- a/src/shell_indexes.c +++ b/src/shell_indexes.c @@ -18,6 +18,9 @@ typedef struct IdxContext IdxContext; typedef struct IdxScan IdxScan; typedef struct IdxWhere IdxWhere; +typedef struct IdxColumn IdxColumn; +typedef struct IdxTable IdxTable; + /* ** A single constraint. Equivalent to either "col = ?" or "col < ?". ** @@ -34,16 +37,22 @@ struct IdxConstraint { }; /* -** A WHERE clause. Made up of IdxConstraint objects. +** A WHERE clause. Made up of IdxConstraint objects. Example WHERE clause: ** ** a=? AND b=? AND (c=? OR d=?) AND (e=? OR f=?) ** +** The above +** +** +** */ struct IdxWhere { IdxConstraint *pEq; /* List of == constraints */ IdxConstraint *pRange; /* List of < constraints */ - IdxWhere **apOr; /* Array of OR branches (joined by pNextOr) */ - IdxWhere *pNextOr; /* Next in OR'd terms */ + IdxWhere *pOr; /* List of OR constraints */ + + IdxWhere *pNextOr; /* Next in OR constraints of same IdxWhere */ + IdxWhere *pSibling; /* Next branch in single OR constraint */ IdxWhere *pParent; /* Parent object (or NULL) */ }; @@ -51,6 +60,7 @@ struct IdxWhere { ** A single scan of a single table. */ struct IdxScan { + IdxTable *pTable; /* Table-info */ char *zTable; /* Name of table to scan */ int iDb; /* Database containing table zTable */ i64 covering; /* Mask of columns required for cov. index */ @@ -64,10 +74,26 @@ struct IdxScan { */ struct IdxContext { IdxWhere *pCurrent; /* Current where clause */ + int rc; /* Error code (if error has occurred) */ IdxScan *pScan; /* List of scan objects */ sqlite3 *dbm; /* In-memory db for this analysis */ - int rc; /* Error code (if error has occurred) */ + sqlite3 *db; /* User database under analysis */ + sqlite3_stmt *pInsertMask; /* To write to aux.depmask */ +}; + +/* +** Data regarding a database table. Extracted from "PRAGMA table_info" +*/ +struct IdxColumn { + char *zName; + char *zColl; + int iPk; }; +struct IdxTable { + int nCol; + IdxColumn *aCol; +}; + typedef struct PragmaTable PragmaTable; typedef struct PragmaCursor PragmaCursor; @@ -94,7 +120,7 @@ static int pragmaConnect( char **pzErr ){ const char *zSchema = - "CREATE TABLE a(tbl HIDDEN, cid, name, type, notnull, dflt_value, pk)"; + "CREATE TABLE a(tbl HIDDEN, cid, name, type, isnotnull, dflt_value, pk)"; PragmaTable *pTab = 0; int rc = SQLITE_OK; @@ -102,6 +128,8 @@ static int pragmaConnect( if( rc==SQLITE_OK ){ pTab = (PragmaTable *)sqlite3_malloc64(sizeof(PragmaTable)); if( pTab==0 ) rc = SQLITE_NOMEM; + }else{ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } assert( rc==SQLITE_OK || pTab==0 ); @@ -152,7 +180,6 @@ static int pragmaBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ** Open a new pragma cursor. */ static int pragmaOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - PragmaTable *pTab = (PragmaTable *)pVTab; PragmaCursor *pCsr; pCsr = (PragmaCursor*)sqlite3_malloc64(sizeof(PragmaCursor)); @@ -167,6 +194,13 @@ static int pragmaOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ return SQLITE_OK; } +static int pragmaClose(sqlite3_vtab_cursor *pCursor){ + PragmaCursor *pCsr = (PragmaCursor*)pCursor; + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr); + return SQLITE_OK; +} + /* ** Move a statvfs cursor to the next entry in the file. */ @@ -212,6 +246,7 @@ static int pragmaFilter( }else{ rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); } + if( rc ) return rc; return pragmaNext(pCursor);; } @@ -308,7 +343,7 @@ static void idxWhereInfo( ){ IdxContext *p = (IdxContext*)pCtx; -#if 1 +#if 0 const char *zOp = eOp==SQLITE_WHEREINFO_TABLE ? "TABLE" : eOp==SQLITE_WHEREINFO_EQUALS ? "EQUALS" : @@ -361,19 +396,36 @@ static void idxWhereInfo( pNew->pNext = p->pCurrent->pEq; p->pCurrent->pEq = pNew; } + + sqlite3_bind_int64(p->pInsertMask, 1, mask); + sqlite3_step(p->pInsertMask); + p->rc = sqlite3_reset(p->pInsertMask); break; } case SQLITE_WHEREINFO_BEGINOR: { - assert( 0 ); + IdxWhere *pNew = (IdxWhere*)idxMalloc(&p->rc, sizeof(IdxWhere)); + if( pNew==0 ) return; + pNew->pParent = p->pCurrent; + pNew->pNextOr = p->pCurrent->pOr; + p->pCurrent->pOr = pNew; + p->pCurrent = pNew; break; } - case SQLITE_WHEREINFO_ENDOR: { - assert( 0 ); + + case SQLITE_WHEREINFO_NEXTOR: { + IdxWhere *pNew = (IdxWhere*)idxMalloc(&p->rc, sizeof(IdxWhere)); + if( pNew==0 ) return; + pNew->pParent = p->pCurrent->pParent; + assert( p->pCurrent->pSibling==0 ); + p->pCurrent->pSibling = pNew; + p->pCurrent = pNew; break; } - case SQLITE_WHEREINFO_NEXTOR: { - assert( 0 ); + + case SQLITE_WHEREINFO_ENDOR: { + assert( p->pCurrent->pParent ); + p->pCurrent = p->pCurrent->pParent; break; } } @@ -391,16 +443,418 @@ static void idxDatabaseError( *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } -static int idxCreateTables(sqlite3 *db, sqlite3 *dbm, IdxScan *pScan){ +static char *idxQueryToList( + sqlite3 *db, + const char *zBind, + int *pRc, + char **pzErrmsg, + const char *zSql +){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + int rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pStmt, 1, zBind, -1, SQLITE_TRANSIENT); + while( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *z = (const char*)sqlite3_column_text(pStmt, 0); + zRet = sqlite3_mprintf("%z%s%Q", zRet, zRet?", ":"", z); + if( zRet==0 ){ + rc = SQLITE_NOMEM; + } + } + rc = sqlite3_finalize(pStmt); + } + + if( rc ){ + idxDatabaseError(db, pzErrmsg); + sqlite3_free(zRet); + zRet = 0; + } + *pRc = rc; + } + + return zRet; +} + +static int idxGetTableInfo( + sqlite3 *db, + IdxScan *pScan, + char **pzErrmsg +){ + const char *zSql = "SELECT name, pk FROM pragma_table_info(?)"; + sqlite3_stmt *p1 = 0; + int nCol = 0; + int nByte = sizeof(IdxTable); + IdxTable *pNew = 0; + int rc, rc2; + char *pCsr; + + rc = sqlite3_prepare_v2(db, zSql, -1, &p1, 0); + if( rc!=SQLITE_OK ){ + idxDatabaseError(db, pzErrmsg); + return rc; + } + sqlite3_bind_text(p1, 1, pScan->zTable, -1, SQLITE_TRANSIENT); + while( SQLITE_ROW==sqlite3_step(p1) ){ + const char *zCol = sqlite3_column_text(p1, 0); + nByte += 1 + strlen(zCol); + rc = sqlite3_table_column_metadata( + db, "main", pScan->zTable, zCol, 0, &zCol, 0, 0, 0 + ); + nByte += 1 + strlen(zCol); + nCol++; + } + rc2 = sqlite3_reset(p1); + if( rc==SQLITE_OK ) rc = rc2; + + nByte += sizeof(IdxColumn) * nCol; + if( rc==SQLITE_OK ){ + pNew = idxMalloc(&rc, nByte); + } + if( rc==SQLITE_OK ){ + pNew->aCol = (IdxColumn*)&pNew[1]; + pNew->nCol = nCol; + pCsr = (char*)&pNew->aCol[nCol]; + } + + nCol = 0; + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){ + const char *zCol = sqlite3_column_text(p1, 0); + int nCopy = strlen(zCol) + 1; + pNew->aCol[nCol].zName = pCsr; + pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 1); + memcpy(pCsr, zCol, nCopy); + pCsr += nCopy; + + rc = sqlite3_table_column_metadata( + db, "main", pScan->zTable, zCol, 0, &zCol, 0, 0, 0 + ); + if( rc==SQLITE_OK ){ + nCopy = strlen(zCol) + 1; + pNew->aCol[nCol].zColl = pCsr; + memcpy(pCsr, zCol, nCopy); + pCsr += nCopy; + } + + nCol++; + } + rc2 = sqlite3_finalize(p1); + if( rc==SQLITE_OK ) rc = rc2; + + if( rc==SQLITE_OK ){ + pScan->pTable = pNew; + }else{ + sqlite3_free(pNew); + } + + return rc; +} + + +static int idxCreateTables( + sqlite3 *db, /* User database */ + sqlite3 *dbm, /* In-memory database to create tables in */ + IdxScan *pScan, /* List of scans */ + char **pzErrmsg /* OUT: Error message */ +){ + int rc = SQLITE_OK; + IdxScan *pIter; + for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){ + int nPk = 0; + char *zCols = 0; + char *zPk = 0; + char *zCreate = 0; + int iCol; + + rc = idxGetTableInfo(db, pIter, pzErrmsg); + + for(iCol=0; rc==SQLITE_OK && iColpTable->nCol; iCol++){ + IdxColumn *pCol = &pIter->pTable->aCol[iCol]; + if( pCol->iPk>nPk ) nPk = pCol->iPk; + zCols = sqlite3_mprintf("%z%s%Q", zCols, (zCols?", ":""), pCol->zName); + if( zCols==0 ) rc = SQLITE_NOMEM; + } + + for(iCol=1; rc==SQLITE_OK && iCol<=nPk; iCol++){ + int j; + for(j=0; jpTable->nCol; j++){ + IdxColumn *pCol = &pIter->pTable->aCol[j]; + if( pCol->iPk==iCol ){ + zPk = sqlite3_mprintf("%z%s%Q", zPk, (zPk?", ":""), pCol->zName); + if( zPk==0 ) rc = SQLITE_NOMEM; + break; + } + } + } + + if( rc==SQLITE_OK ){ + if( zPk ){ + zCreate = sqlite3_mprintf("CREATE TABLE %Q(%s, PRIMARY KEY(%s))", + pIter->zTable, zCols, zPk + ); + }else{ + zCreate = sqlite3_mprintf("CREATE TABLE %Q(%s)", pIter->zTable, zCols); + } + if( zCreate==0 ) rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ +#if 1 + printf("/* %s */\n", zCreate); +#endif + rc = sqlite3_exec(dbm, zCreate, 0, 0, pzErrmsg); + } + sqlite3_free(zCols); + sqlite3_free(zPk); + sqlite3_free(zCreate); + } + return rc; +} + +/* +** This function is a no-op if *pRc is set to anything other than +** SQLITE_OK when it is called. +** +** If *pRc is initially set to SQLITE_OK, then the text specified by +** the printf() style arguments is appended to zIn and the result returned +** in a buffer allocated by sqlite3_malloc(). sqlite3_free() is called on +** zIn before returning. +*/ +static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){ + va_list ap; + char *zAppend = 0; + char *zRet = 0; + int nIn = zIn ? strlen(zIn) : 0; + int nAppend = 0; + va_start(ap, zFmt); + if( *pRc==SQLITE_OK ){ + zAppend = sqlite3_vmprintf(zFmt, ap); + if( zAppend ){ + nAppend = strlen(zAppend); + zRet = (char*)sqlite3_malloc(nIn + nAppend); + } + if( zAppend && zRet ){ + memcpy(zRet, zIn, nIn); + memcpy(&zRet[nIn], zAppend, nAppend+1); + }else{ + sqlite3_free(zRet); + zRet = 0; + *pRc = SQLITE_NOMEM; + } + sqlite3_free(zAppend); + sqlite3_free(zIn); + } + va_end(ap); + return zRet; +} + +static char *idxAppendColDefn( + int *pRc, + char *zIn, + IdxTable *pTab, + IdxConstraint *pCons +){ + char *zRet = zIn; + IdxColumn *p = &pTab->aCol[pCons->iCol]; + if( zRet ) zRet = idxAppendText(pRc, zRet, ", "); + zRet = idxAppendText(pRc, zRet, "%Q", p->zName); + if( sqlite3_stricmp(p->zColl, pCons->zColl) ){ + zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl); + } + return zRet; +} + +static int idxCreateFromCons( + sqlite3 *dbm, + IdxScan *pScan, + IdxConstraint *pEq, + IdxConstraint *pTail +){ int rc = SQLITE_OK; + if( pEq || pTail ){ + IdxTable *pTab = pScan->pTable; + char *zCols = 0; + char *zIdx = 0; + IdxConstraint *pCons; + int h = 0; + + for(pCons=pEq; pCons; pCons=pCons->pLink){ + zCols = idxAppendColDefn(&rc, zCols, pTab, pCons); + } + for(pCons=pTail; pCons; pCons=pCons->pLink){ + zCols = idxAppendColDefn(&rc, zCols, pTab, pCons); + } + + /* Hash the list of columns to come up with a name for the index */ + if( rc==SQLITE_OK ){ + int i; + for(i=0; zCols[i]; i++){ + h += ((h<<3) + zCols[i]); + } + + zIdx = sqlite3_mprintf("CREATE INDEX IF NOT EXISTS " + "'%q_idx_%08x' ON %Q(%s)", pScan->zTable, h, pScan->zTable, zCols + ); + if( !zIdx ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(dbm, zIdx, 0, 0, 0); + printf("/* %s */\n", zIdx); + } + } + + sqlite3_free(zIdx); + sqlite3_free(zCols); + } + return rc; +} + +static int idxCreateFromWhere( + sqlite3*, i64, IdxScan*, IdxWhere*, IdxConstraint*, IdxConstraint* +); + +static int idxCreateForeachOr( + sqlite3 *dbm, + i64 mask, /* Consider only these constraints */ + IdxScan *pScan, /* Create indexes for this scan */ + IdxWhere *pWhere, /* Read constraints from here */ + IdxConstraint *pEq, /* == constraints for inclusion */ + IdxConstraint *pTail /* range/ORDER BY constraints for inclusion */ +){ + int rc = SQLITE_OK; + IdxWhere *p1; + IdxWhere *p2; + for(p1=pWhere->pOr; p1 && rc==SQLITE_OK; p1=p1->pNextOr){ + rc = idxCreateFromWhere(dbm, mask, pScan, p1, pEq, pTail); + for(p2=p1->pSibling; p2 && rc==SQLITE_OK; p2=p2->pSibling){ + rc = idxCreateFromWhere(dbm, mask, pScan, p2, pEq, pTail); + } + } + return rc; +} + +static int idxCreateFromWhere( + sqlite3 *dbm, + i64 mask, /* Consider only these constraints */ + IdxScan *pScan, /* Create indexes for this scan */ + IdxWhere *pWhere, /* Read constraints from here */ + IdxConstraint *pEq, /* == constraints for inclusion */ + IdxConstraint *pTail /* range/ORDER BY constraints for inclusion */ +){ + IdxConstraint *p1 = pEq; + IdxConstraint *pCon; + int rc; + + /* Gather up all the == constraints that match the mask. */ + for(pCon=pWhere->pEq; pCon; pCon=pCon->pNext){ + if( (mask & pCon->depmask)==pCon->depmask ){ + pCon->pLink = p1; + p1 = pCon; + } + } + + /* Create an index using the == constraints collected above. And the + ** range constraint/ORDER BY terms passed in by the caller, if any. */ + rc = idxCreateFromCons(dbm, pScan, p1, pTail); + if( rc==SQLITE_OK ){ + rc = idxCreateForeachOr(dbm, mask, pScan, pWhere, p1, pTail); + } + + /* If no range/ORDER BY passed by the caller, create a version of the + ** index for each range constraint that matches the mask. */ + if( pTail==0 ){ + for(pCon=pWhere->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){ + assert( pCon->pLink==0 ); + if( (mask & pCon->depmask)==pCon->depmask ){ + rc = idxCreateFromCons(dbm, pScan, p1, pCon); + if( rc==SQLITE_OK ){ + rc = idxCreateForeachOr(dbm, mask, pScan, pWhere, p1, pCon); + } + } + } + } + + return rc; +} + +static int idxPrepareStmt( + sqlite3 *db, /* Database handle to compile against */ + const char *zSql, /* SQL statement to compile */ + sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ + char **pzErrmsg /* OUT: sqlite3_malloc()ed error message */ +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + *ppStmt = 0; + idxDatabaseError(db, pzErrmsg); + } + return rc; +} + +/* +** Create candidate indexes in database [dbm] based on the data in +** linked-list pScan. +*/ +static int idxCreateCandidates( + sqlite3 *dbm, + IdxScan *pScan, + char **pzErrmsg +){ + int rc2; + int rc = SQLITE_OK; + sqlite3_stmt *pDepmask; /* Foreach depmask */ IdxScan *pIter; - for(pIter=pScan; pIter; pIter=pIter->pNextScan){ + + rc = idxPrepareStmt(dbm, "SELECT mask FROM depmask", &pDepmask, pzErrmsg); + + for(pIter=pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){ + IdxWhere *pWhere = &pIter->where; + while( SQLITE_ROW==sqlite3_step(pDepmask) && rc==SQLITE_OK ){ + i64 mask = sqlite3_column_int64(pDepmask, 0); + rc = idxCreateFromWhere(dbm, mask, pIter, pWhere, 0, 0); + if( rc==SQLITE_OK && pIter->pOrder ){ + rc = idxCreateFromWhere(dbm, mask, pIter, pWhere, 0, pIter->pOrder); + } + } } + + rc2 = sqlite3_finalize(pDepmask); + if( rc==SQLITE_OK ) rc = rc2; + return rc; } static void idxScanFree(IdxScan *pScan){ } +int idxFindIndexes( + sqlite3 *dbm, /* Database handle */ + const char *zSql, /* SQL to find indexes for */ + void (*xOut)(void*, const char*), /* Output callback */ + void *pOutCtx, /* Context for xOut() */ + char **pzErrmsg /* OUT: Error message (sqlite3_malloc) */ +){ + char *zExplain; + sqlite3_stmt *pExplain; + int rc; + + zExplain = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zSql); + if( zExplain==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = idxPrepareStmt(dbm, zExplain, &pExplain, pzErrmsg); + sqlite3_free(zExplain); + } + if( rc!=SQLITE_OK ) return rc; + + while( sqlite3_step(pExplain)==SQLITE_ROW ){ + int iCol; + // for(iCol=0; iCol