From: dan Date: Mon, 10 Apr 2017 20:00:26 +0000 (+0000) Subject: Add ext/expert/README.md. X-Git-Tag: version-3.22.0~147^2~35 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=96d43a05ecbc611a27751446c808d1710b2d8e6e;p=thirdparty%2Fsqlite.git Add ext/expert/README.md. FossilOrigin-Name: 9318f1b9ed2d8da3a82ea69179e2d56a99d326c7721642665f87f6a4534e7bf0 --- diff --git a/ext/expert/README.md b/ext/expert/README.md new file mode 100644 index 0000000000..3cede6c09d --- /dev/null +++ b/ext/expert/README.md @@ -0,0 +1,68 @@ +## SQLite Expert Extension + +This folder contains code for a simple system to propose useful indexes +given a database and a set of SQL queries. It works as follows: + + 1. The user database schema is copied to a temporary database. + + 1. All SQL queries are prepared against the temporary database. The + **sqlite3\_whereinfo\_hook()** API is used to record information regarding + the WHERE and ORDER BY clauses attached to each query. + + 1. The information gathered in step 2 is used to create (possibly a large + number of) candidate indexes. + + 1. The SQL queries are prepared a second time. If the planner uses any + of the indexes created in step 3, they are recommended to the user. + +No ANALYZE data is available to the planner in step 4 above. This can lead to sub-optimal results. + +This extension requires that SQLite be built with the +SQLITE\_ENABLE\_WHEREINFO\_HOOK pre-processor symbol defined. + +# C API + +The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed +as follows: + + 1. An sqlite3expert object is created by calling **sqlite3\_expert\_new()**. + A database handle opened by the user is passed as an argument. + + 1. The sqlite3expert object is configured with one or more SQL statements + by making one or more calls to **sqlite3\_expert\_sql()**. Each call may + specify a single SQL statement, or multiple statements separated by + semi-colons. + + 1. **sqlite3\_expert\_analyze()** is called to run the analysis. + + 1. One or more calls are made to **sqlite3\_expert\_report()** to extract + components of the results of the analysis. + + 1. **sqlite3\_expert\_destroy()** is called to free all resources. + +Refer to comments in sqlite3expert.h for further details. + +# sqlite3_expert application + +The file "expert.c" contains the code for a command line application that +uses the API described above. It can be compiled with (for example): + +
+  gcc -O2 -DSQLITE_ENABLE_WHEREINFO_HOOK sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
+
+ +Assuming the database is "test.db", it can then be run to analyze a single query: + +
+  ./sqlite3_expert -sql <sql-query> test.db
+
+ +Or an entire text file worth of queries with: + +
+  ./sqlite3_expert -file <text-file> test.db
+
+ + + + diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test index 684c618780..2e8c1c7e76 100644 --- a/ext/expert/expert1.test +++ b/ext/expert/expert1.test @@ -17,7 +17,9 @@ # Test plan: # # -set testdir [file dirname $argv0] +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} source $testdir/tester.tcl set testprefix expert1 diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c index 6bcc697fcf..faa6966932 100644 --- a/ext/expert/sqlite3expert.c +++ b/ext/expert/sqlite3expert.c @@ -1,5 +1,5 @@ /* -** 2016 February 10 +** 2017 April 09 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -21,16 +21,15 @@ typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; +typedef struct IdxColumn IdxColumn; typedef struct IdxConstraint IdxConstraint; typedef struct IdxScan IdxScan; typedef struct IdxStatement IdxStatement; -typedef struct IdxWhere IdxWhere; - -typedef struct IdxColumn IdxColumn; typedef struct IdxTable IdxTable; /* -** A single constraint. Equivalent to either "col = ?" or "col < ?". +** A single constraint. Equivalent to either "col = ?" or "col < ?" (or +** any other type of single-ended range constraint on a column). ** ** pLink: ** Used to temporarily link IdxConstraint objects into lists while @@ -47,38 +46,22 @@ struct IdxConstraint { IdxConstraint *pLink; /* See above */ }; -/* -** A WHERE clause. Made up of IdxConstraint objects. Example WHERE clause: -** -** a=? AND b=? AND c=? AND d=? AND e>? AND fdepmask = mask; if( eOp==SQLITE_WHEREINFO_RANGE ){ - pNew->pNext = p->pScan->where.pRange; - p->pScan->where.pRange = pNew; + pNew->pNext = p->pScan->pRange; + p->pScan->pRange = pNew; }else{ - pNew->pNext = p->pScan->where.pEq; - p->pScan->where.pEq = pNew; + pNew->pNext = p->pScan->pEq; + p->pScan->pEq = pNew; } break; } @@ -369,6 +397,9 @@ static void idxDatabaseError( *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } +/* +** Prepare an SQL statement. +*/ static int idxPrepareStmt( sqlite3 *db, /* Database handle to compile against */ sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ @@ -383,6 +414,9 @@ static int idxPrepareStmt( return rc; } +/* +** Prepare an SQL statement using the results of a printf() formatting. +*/ static int idxPrintfPrepareStmt( sqlite3 *db, /* Database handle to compile against */ sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ @@ -405,17 +439,32 @@ static int idxPrintfPrepareStmt( return rc; } +/* +** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function +** is called, set it to the return value of sqlite3_finalize() before +** returning. Otherwise, discard the sqlite3_finalize() return value. +*/ static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){ int rc = sqlite3_finalize(pStmt); if( *pRc==SQLITE_OK ) *pRc = rc; } +/* +** Attempt to allocate an IdxTable structure corresponding to table zTab +** in the main database of connection db. If successful, set (*ppOut) to +** point to the new object and return SQLITE_OK. Otherwise, return an +** SQLite error code and set (*ppOut) to NULL. In this case *pzErrmsg may be +** set to point to an error string. +** +** It is the responsibility of the caller to eventually free either the +** IdxTable object or error message using sqlite3_free(). +*/ static int idxGetTableInfo( - sqlite3 *db, - IdxScan *pScan, - char **pzErrmsg + sqlite3 *db, /* Database connection to read details from */ + const char *zTab, /* Table name */ + IdxTable **ppOut, /* OUT: New object (if successful) */ + char **pzErrmsg /* OUT: Error message (if not) */ ){ - const char *zTbl = pScan->zTable; sqlite3_stmt *p1 = 0; int nCol = 0; int nByte = sizeof(IdxTable); @@ -423,12 +472,12 @@ static int idxGetTableInfo( int rc, rc2; char *pCsr; - rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTbl); + rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){ const char *zCol = (const char*)sqlite3_column_text(p1, 1); nByte += 1 + strlen(zCol); rc = sqlite3_table_column_metadata( - db, "main", zTbl, zCol, 0, &zCol, 0, 0, 0 + db, "main", zTab, zCol, 0, &zCol, 0, 0, 0 ); nByte += 1 + strlen(zCol); nCol++; @@ -456,7 +505,7 @@ static int idxGetTableInfo( pCsr += nCopy; rc = sqlite3_table_column_metadata( - db, "main", zTbl, zCol, 0, &zCol, 0, 0, 0 + db, "main", zTab, zCol, 0, &zCol, 0, 0, 0 ); if( rc==SQLITE_OK ){ nCopy = strlen(zCol) + 1; @@ -469,12 +518,12 @@ static int idxGetTableInfo( } idxFinalize(&rc, p1); - if( rc==SQLITE_OK ){ - pScan->pTable = pNew; - }else{ + if( rc!=SQLITE_OK ){ sqlite3_free(pNew); + pNew = 0; } + *ppOut = pNew; return rc; } @@ -515,6 +564,10 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){ return zRet; } +/* +** Return true if zId must be quoted in order to use it as an SQL +** identifier, or false otherwise. +*/ static int idxIdentifierRequiresQuotes(const char *zId){ int i; for(i=0; zId[i]; i++){ @@ -529,10 +582,14 @@ static int idxIdentifierRequiresQuotes(const char *zId){ return 0; } +/* +** This function appends an index column definition suitable for constraint +** pCons to the string passed as zIn and returns the result. +*/ static char *idxAppendColDefn( - int *pRc, - char *zIn, - IdxTable *pTab, + int *pRc, /* IN/OUT: Error code */ + char *zIn, /* Column defn accumulated so far */ + IdxTable *pTab, /* Table index will be created on */ IdxConstraint *pCons ){ char *zRet = zIn; @@ -636,6 +693,7 @@ static int idxFindCompatible( static int idxCreateFromCons( sqlite3expert *p, + IdxTable *pTab, IdxScan *pScan, IdxConstraint *pEq, IdxConstraint *pTail @@ -643,7 +701,6 @@ static int idxCreateFromCons( sqlite3 *dbm = p->dbm; int rc = SQLITE_OK; if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){ - IdxTable *pTab = pScan->pTable; char *zCols = 0; char *zIdx = 0; IdxConstraint *pCons; @@ -704,9 +761,9 @@ static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){ static int idxCreateFromWhere( sqlite3expert *p, + IdxTable *pTab, i64 mask, /* Consider only these constraints */ IdxScan *pScan, /* Create indexes for this scan */ - IdxWhere *pWhere, /* Read constraints from here */ IdxConstraint *pEq, /* == constraints for inclusion */ IdxConstraint *pTail /* range/ORDER BY constraints for inclusion */ ){ @@ -715,7 +772,7 @@ static int idxCreateFromWhere( int rc; /* Gather up all the == constraints that match the mask. */ - for(pCon=pWhere->pEq; pCon; pCon=pCon->pNext){ + for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){ if( (mask & pCon->depmask)==pCon->depmask && idxFindConstraint(p1, pCon)==0 && idxFindConstraint(pTail, pCon)==0 @@ -727,18 +784,18 @@ static int idxCreateFromWhere( /* Create an index using the == constraints collected above. And the ** range constraint/ORDER BY terms passed in by the caller, if any. */ - rc = idxCreateFromCons(p, pScan, p1, pTail); + rc = idxCreateFromCons(p, pTab, pScan, p1, pTail); /* If no range/ORDER BY passed by the caller, create a version of the ** index for each range constraint that matches the mask. */ if( pTail==0 ){ - for(pCon=pWhere->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){ + for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){ assert( pCon->pLink==0 ); if( (mask & pCon->depmask)==pCon->depmask && idxFindConstraint(pEq, pCon)==0 && idxFindConstraint(pTail, pCon)==0 ){ - rc = idxCreateFromCons(p, pScan, p1, pCon); + rc = idxCreateFromCons(p, pTab, pScan, p1, pCon); } } } @@ -758,30 +815,36 @@ static int idxCreateCandidates(sqlite3expert *p, char **pzErr){ for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){ IdxHash64Entry *pEntry; - IdxWhere *pWhere = &pIter->where; IdxConstraint *pCons; + IdxTable *pTab = 0; + + rc = idxGetTableInfo(p->dbm, pIter->zTable, &pTab, pzErr); idxHash64Add(&rc, &hMask, 0); - for(pCons=pIter->where.pEq; pCons; pCons=pCons->pNext){ + for(pCons=pIter->pEq; pCons; pCons=pCons->pNext){ for(pEntry=hMask.pFirst; pEntry; pEntry=pEntry->pNext){ idxHash64Add(&rc, &hMask, pEntry->iVal | (u64)pCons->depmask); } } - for(pEntry=hMask.pFirst; pEntry; pEntry=pEntry->pNext){ + for(pEntry=hMask.pFirst; rc==SQLITE_OK && pEntry; pEntry=pEntry->pNext){ i64 mask = (i64)pEntry->iVal; - rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, 0); + rc = idxCreateFromWhere(p, pTab, mask, pIter, 0, 0); if( rc==SQLITE_OK && pIter->pOrder ){ - rc = idxCreateFromWhere(p, mask, pIter, pWhere, 0, pIter->pOrder); + rc = idxCreateFromWhere(p, pTab, mask, pIter, 0, pIter->pOrder); } } + sqlite3_free(pTab); idxHash64Clear(&hMask); } return rc; } +/* +** Free all elements of the linked list starting at pConstraint. +*/ static void idxConstraintFree(IdxConstraint *pConstraint){ IdxConstraint *pNext; IdxConstraint *p; @@ -802,9 +865,8 @@ static void idxScanFree(IdxScan *pScan, IdxScan *pLast){ for(p=pScan; p!=pLast; p=pNext){ pNext = p->pNextScan; idxConstraintFree(p->pOrder); - idxConstraintFree(p->where.pEq); - idxConstraintFree(p->where.pRange); - sqlite3_free(p->pTable); + idxConstraintFree(p->pEq); + idxConstraintFree(p->pRange); sqlite3_free(p); } } @@ -825,6 +887,11 @@ static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){ } +/* +** This function is called after candidate indexes have been created. It +** runs all the queries to see which indexes they prefer, and populates +** IdxStatement.zIdx and IdxStatement.zEQP with the results. +*/ int idxFindIndexes( sqlite3expert *p, char **pzErr /* OUT: Error message (sqlite3_malloc) */ @@ -981,18 +1048,10 @@ int sqlite3_expert_sql( } int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){ - int rc = SQLITE_OK; - IdxScan *pIter; - - /* Load IdxTable objects */ - for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){ - rc = idxGetTableInfo(p->dbm, pIter, pzErr); - } + int rc; /* Create candidate indexes within the in-memory database file */ - if( rc==SQLITE_OK ){ - rc = idxCreateCandidates(p, pzErr); - } + rc = idxCreateCandidates(p, pzErr); /* Figure out which of the candidate indexes are preferred by the query ** planner and report the results to the user. */ diff --git a/manifest b/manifest index 7660557a22..ab763c7f7c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sallow\sthe\scode\sin\ssqlite3expert.c\sto\sbe\stested\sdirectly\s(via\sthe\nAPI\sin\ssqlite3expert.h)\sinstead\sof\sby\sinvoking\sthe\ssqlite3_expert\sapplication.\nFix\smemory\sleaks\sand\sother\sproblems. -D 2017-04-10T16:13:20.707 +C Add\sext/expert/README.md. +D 2017-04-10T20:00:26.414 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a @@ -40,9 +40,10 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef +F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01 F ext/expert/expert.c bf0fd71921cb7b807cda9a76fb380e3d6e6b980d6167093b2952b41ec9ad8f46 -F ext/expert/expert1.test c1b1405f3ac20e9f71dacdf7bd68ff22e273b249a419260b123ebe385daf2db5 w test/expert1.test -F ext/expert/sqlite3expert.c b87f13e90b999b5b10c4ec004b6a935150c00d3af1a16944e262172b9b831b8c +F ext/expert/expert1.test d4451ccdca910ec3652a7a1994210fe2b7bc8f56431ed0cf2ff3dee798c89084 +F ext/expert/sqlite3expert.c 8bcb83b3723239dc6a2a199e7a030741b7ecf47814828a1da7ea559aa42f094a F ext/expert/sqlite3expert.h feeaee4ab73ba52426329781bbb28032ce18cf5abd2bf6221bac2df4c32b3013 F ext/expert/test_expert.c bad0611732d07180d586bd589cbb7713dc3ab0338c52bff29680eb2007678c05 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e @@ -1576,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0857c48e02a76490fc623364f77363165dea94ec254f93d8f0fd0bac2968c572 -R edcb3d0fdfbb918bffbc8d72b5429335 +P 5dd9831721b70a89a26728adcd49c7f6103ef8266891a79c2db34d913702709e +R 119e273740189bcdef018d8609f3a756 U dan -Z 0d7d47356215948792ad81449f48b82b +Z 664c10ad907cc8fb129a4f4feaf498a1 diff --git a/manifest.uuid b/manifest.uuid index a059662f4d..935458cfd1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5dd9831721b70a89a26728adcd49c7f6103ef8266891a79c2db34d913702709e \ No newline at end of file +9318f1b9ed2d8da3a82ea69179e2d56a99d326c7721642665f87f6a4534e7bf0 \ No newline at end of file