From: dan Date: Tue, 24 Jun 2014 16:59:06 +0000 (+0000) Subject: Add simple full-table-scan and rowid lookup support to fts5. X-Git-Tag: version-3.8.11~114^2~176 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=d0bfb36a08ad41304cc65ddd4b4893b545f138c6;p=thirdparty%2Fsqlite.git Add simple full-table-scan and rowid lookup support to fts5. FossilOrigin-Name: 3515da85d09220c464979467b476c611da4a6a7a --- diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c index 7a6c361068..56a74d6486 100644 --- a/ext/fts5/fts5.c +++ b/ext/fts5/fts5.c @@ -16,6 +16,7 @@ #include "fts5Int.h" typedef struct Fts5Table Fts5Table; +typedef struct Fts5Cursor Fts5Cursor; struct Fts5Table { sqlite3_vtab base; /* Base class used by SQLite core */ @@ -24,6 +25,13 @@ struct Fts5Table { Fts5Storage *pStorage; /* Document store */ }; +struct Fts5Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + int idxNum; /* idxNum passed to xFilter() */ + sqlite3_stmt *pStmt; /* Statement used to read %_content */ + int bEof; /* True at EOF */ +}; + /* ** Close a virtual table handle opened by fts5InitVtab(). If the bDestroy ** argument is non-zero, attempt delete the shadow tables from teh database @@ -145,15 +153,69 @@ static int fts5CreateMethod( return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); } +/* +** The three query plans xBestIndex may choose between. +*/ +#define FTS5_PLAN_SCAN 1 /* No usable constraint */ +#define FTS5_PLAN_MATCH 2 /* ( MATCH ?) */ +#define FTS5_PLAN_ROWID 3 /* (rowid = ?) */ + +#define FTS5_PLAN(idxNum) ((idxNum) & 0x7) + +#define FTS5_ORDER_DESC 8 /* ORDER BY rowid DESC */ +#define FTS5_ORDER_ASC 16 /* ORDER BY rowid ASC */ + + +static int fts5FindConstraint(sqlite3_index_info *pInfo, int eOp, int iCol){ + int i; + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + if( p->usable && p->iColumn==iCol && p->op==eOp ) return i; + } + + return -1; +} + /* -** Implementation of the xBestIndex method for FTS3 tables. There +** Implementation of the xBestIndex method for FTS5 tables. There ** are three possible strategies, in order of preference: ** -** 1. Direct lookup by rowid or docid. -** 2. Full-text search using a MATCH operator on a non-docid column. -** 3. Linear scan of %_content table. +** 1. Full-text search using a MATCH operator. +** 2. A by-rowid lookup. +** 3. A full-table scan. */ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts5Table *pTab = (Fts5Table*)pVTab; + Fts5Config *pConfig = pTab->pConfig; + int iCons; + int ePlan = FTS5_PLAN_SCAN; + + iCons = fts5FindConstraint(pInfo,SQLITE_INDEX_CONSTRAINT_MATCH,pConfig->nCol); + if( iCons>=0 ){ + ePlan = FTS5_PLAN_MATCH; + pInfo->estimatedCost = 1.0; + }else{ + iCons = fts5FindConstraint(pInfo, SQLITE_INDEX_CONSTRAINT_EQ, -1); + if( iCons>=0 ){ + ePlan = FTS5_PLAN_ROWID; + pInfo->estimatedCost = 2.0; + } + } + + if( iCons>=0 ){ + pInfo->aConstraintUsage[iCons].argvIndex = 1; + pInfo->aConstraintUsage[iCons].omit = 1; + }else{ + pInfo->estimatedCost = 10000000.0; + } + + if( pInfo->nOrderBy==1 && pInfo->aOrderBy[0].iColumn<0 ){ + pInfo->orderByConsumed = 1; + ePlan |= pInfo->aOrderBy[0].desc ? FTS5_ORDER_DESC : FTS5_ORDER_ASC; + } + + pInfo->idxNum = ePlan; return SQLITE_OK; } @@ -161,7 +223,23 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ ** Implementation of xOpen method. */ static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - return SQLITE_OK; + Fts5Cursor *pCsr; + int rc = SQLITE_OK; + pCsr = (Fts5Cursor*)sqlite3_malloc(sizeof(Fts5Cursor)); + if( pCsr ){ + memset(pCsr, 0, sizeof(Fts5Cursor)); + }else{ + rc = SQLITE_NOMEM; + } + *ppCsr = (sqlite3_vtab_cursor*)pCsr; + return rc; +} + +static int fts5StmtType(int idxNum){ + if( FTS5_PLAN(idxNum)==FTS5_PLAN_SCAN ){ + return (idxNum&FTS5_ORDER_ASC) ? FTS5_STMT_SCAN_ASC : FTS5_STMT_SCAN_DESC; + } + return FTS5_STMT_LOOKUP; } /* @@ -169,6 +247,13 @@ static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ ** on the xClose method of the virtual table interface. */ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + if( pCsr->pStmt ){ + int eStmt = fts5StmtType(pCsr->idxNum); + sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); + } + sqlite3_free(pCsr); return SQLITE_OK; } @@ -182,7 +267,22 @@ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ ** subsequently to determine whether or not an EOF was hit. */ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ - return SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int ePlan = FTS5_PLAN(pCsr->idxNum); + int rc = SQLITE_OK; + + assert( ePlan!=FTS5_PLAN_MATCH ); + if( ePlan!=FTS5_PLAN_MATCH ){ + rc = sqlite3_step(pCsr->pStmt); + if( rc!=SQLITE_ROW ){ + pCsr->bEof = 1; + rc = sqlite3_reset(pCsr->pStmt); + }else{ + rc = SQLITE_OK; + } + } + + return rc; } /* @@ -197,7 +297,25 @@ static int fts5FilterMethod( int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ - return SQLITE_OK; + Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int rc = SQLITE_OK; + int ePlan = FTS5_PLAN(idxNum); + int eStmt = fts5StmtType(idxNum); + + assert( ePlan!=FTS5_PLAN_MATCH ); + memset(&pCursor[1], 0, sizeof(Fts5Cursor) - sizeof(sqlite3_vtab_cursor)); + pCsr->idxNum = idxNum; + + rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt); + if( ePlan==FTS5_PLAN_ROWID ){ + sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + } + + if( rc==SQLITE_OK ){ + rc = fts5NextMethod(pCursor); + } + return rc; } /* @@ -205,7 +323,8 @@ static int fts5FilterMethod( ** routine to find out if it has reached the end of a result set. */ static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){ - return 1; + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + return pCsr->bEof; } /* @@ -215,6 +334,16 @@ static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){ ** rowid should be written to *pRowid. */ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int ePlan = FTS5_PLAN(pCsr->idxNum); + + assert( pCsr->bEof==0 ); + assert( ePlan!=FTS5_PLAN_MATCH ); + + if( ePlan!=FTS5_PLAN_MATCH ){ + *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); + } + return SQLITE_OK; } @@ -227,6 +356,14 @@ static int fts5ColumnMethod( sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + int ePlan = FTS5_PLAN(pCsr->idxNum); + + assert( pCsr->bEof==0 ); + assert( ePlan!=FTS5_PLAN_MATCH ); + if( ePlan!=FTS5_PLAN_MATCH ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } return SQLITE_OK; } @@ -241,7 +378,7 @@ static int fts5ColumnMethod( ** error code if an error occurs. */ static int fts5SpecialCommand(Fts5Table *pTab, sqlite3_value *pVal){ - const char *z = sqlite3_value_text(pVal); + const char *z = (const char*)sqlite3_value_text(pVal); int n = sqlite3_value_bytes(pVal); int rc = SQLITE_ERROR; diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 5329c207c8..eb6d447cac 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -226,6 +226,15 @@ int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*); int sqlite3Fts5StorageIntegrity(Fts5Storage *p); +#define FTS5_STMT_SCAN_ASC 0 /* SELECT rowid, * FROM ... ORDER BY 1 ASC */ +#define FTS5_STMT_SCAN_DESC 1 /* SELECT rowid, * FROM ... ORDER BY 1 DESC */ +#define FTS5_STMT_LOOKUP 2 /* SELECT rowid, * FROM ... WHERE rowid=? */ + +int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **); +void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); + + + /* ** End of interface to code in fts5_storage.c. **************************************************************************/ diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 76cd2e1da7..6b86218e42 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -18,18 +18,26 @@ struct Fts5Storage { Fts5Config *pConfig; Fts5Index *pIndex; - sqlite3_stmt *aStmt[7]; + sqlite3_stmt *aStmt[8]; }; -#define FTS5_STMT_INSERT_CONTENT 0 -#define FTS5_STMT_REPLACE_CONTENT 1 -#define FTS5_STMT_DELETE_CONTENT 2 -#define FTS5_STMT_INSERT_DOCSIZE 3 -#define FTS5_STMT_DELETE_DOCSIZE 4 +#if FTS5_STMT_SCAN_ASC!=0 +# error "FTS5_STMT_SCAN_ASC mismatch" +#endif +#if FTS5_STMT_SCAN_DESC!=1 +# error "FTS5_STMT_SCAN_DESC mismatch" +#endif +#if FTS5_STMT_LOOKUP!=2 +# error "FTS5_STMT_LOOKUP mismatch" +#endif -#define FTS5_STMT_SCAN_CONTENT 5 -#define FTS5_STMT_SEEK_CONTENT 6 +#define FTS5_STMT_INSERT_CONTENT 3 +#define FTS5_STMT_REPLACE_CONTENT 4 + +#define FTS5_STMT_DELETE_CONTENT 5 +#define FTS5_STMT_INSERT_DOCSIZE 6 +#define FTS5_STMT_DELETE_DOCSIZE 7 /* ** Prepare the two insert statements - Fts5Storage.pInsertContent and @@ -47,13 +55,15 @@ static int fts5StorageGetStmt( assert( eStmt>=0 && eStmtaStmt) ); if( p->aStmt[eStmt]==0 ){ const char *azStmt[] = { - "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ - "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ - "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */ - "INSERT INTO %Q.'%q_docsize' VALUES(?,?)", /* INSERT_DOCSIZE */ - "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */ - "SELECT * FROM %Q.'%q_content'", /* SCAN_CONTENT */ - "SELECT * FROM %Q.'%q_content' WHERE rowid=?", /* SEEK_CONTENT */ + "SELECT * FROM %Q.'%q_content' ORDER BY id ASC", /* SCAN_ASC */ + "SELECT * FROM %Q.'%q_content' ORDER BY id DESC", /* SCAN_DESC */ + "SELECT * FROM %Q.'%q_content' WHERE rowid=?", /* LOOKUP */ + + "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ + "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ + "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */ + "INSERT INTO %Q.'%q_docsize' VALUES(?,?)", /* INSERT_DOCSIZE */ + "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */ }; Fts5Config *pConfig = p->pConfig; char *zSql = 0; @@ -253,7 +263,7 @@ static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){ sqlite3_stmt *pSeek; /* SELECT to read row iDel from %_data */ int rc; /* Return code */ - rc = fts5StorageGetStmt(p, FTS5_STMT_SEEK_CONTENT, &pSeek); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek); if( rc==SQLITE_OK ){ int rc2; sqlite3_bind_int64(pSeek, 1, iDel); @@ -377,7 +387,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ /* Generate the expected index checksum based on the contents of the ** %_content table. This block stores the checksum in ctx.cksum. */ - rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_CONTENT, &pScan); + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN_ASC, &pScan); if( rc==SQLITE_OK ){ int rc2; while( SQLITE_ROW==sqlite3_step(pScan) ){ @@ -408,4 +418,44 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ return rc; } +/* +** Obtain an SQLite statement handle that may be used to read data from the +** %_content table. +*/ +int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **pp){ + int rc; + assert( eStmt==FTS5_STMT_SCAN_ASC + || eStmt==FTS5_STMT_SCAN_DESC + || eStmt==FTS5_STMT_LOOKUP + ); + rc = fts5StorageGetStmt(p, eStmt, pp); + if( rc==SQLITE_OK ){ + assert( p->aStmt[eStmt]==*pp ); + p->aStmt[eStmt] = 0; + } + return rc; +} + +/* +** Release an SQLite statement handle obtained via an earlier call to +** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function +** must match that passed to the sqlite3Fts5StorageStmt() call. +*/ +void sqlite3Fts5StorageStmtRelease( + Fts5Storage *p, + int eStmt, + sqlite3_stmt *pStmt +){ + assert( eStmt==FTS5_STMT_SCAN_ASC + || eStmt==FTS5_STMT_SCAN_DESC + || eStmt==FTS5_STMT_LOOKUP + ); + if( p->aStmt[eStmt]==0 ){ + sqlite3_reset(pStmt); + p->aStmt[eStmt] = pStmt; + }else{ + sqlite3_finalize(pStmt); + } +} + diff --git a/manifest b/manifest index fe4d032557..099fc5ea9c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssome\scode\sfor\san\sexperimental\sfts5\smodule.\sDoes\snot\swork\syet. -D 2014-06-23T11:33:22.754 +C Add\ssimple\sfull-table-scan\sand\srowid\slookup\ssupport\sto\sfts5. +D 2014-06-24T16:59:06.519 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -103,12 +103,12 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368 -F ext/fts5/fts5.c 2cb2cc3c1acefa36d9e8ce8e68bceaac8515059a -F ext/fts5/fts5Int.h cc41cf776a3e612aa3a461e96463647fd3957bed +F ext/fts5/fts5.c 3efba544818662a02e8e5ebd73d57cff6182b2dd +F ext/fts5/fts5Int.h 6f11697324ebaafe92872ee5b19f3661b2b621f1 F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef F ext/fts5/fts5_expr.c bdfb98dab7729cf967022d7a4a815828bbad8c23 F ext/fts5/fts5_index.c 0548e8925a0664cfa00b2477ebe9afa18bc7848f -F ext/fts5/fts5_storage.c aa1ff4b6b283303ffd8c5dc57a45ebe55e62a7b2 +F ext/fts5/fts5_storage.c 7848d8f8528d798bba159900ea310a6d4a279da8 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 @@ -591,7 +591,8 @@ F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36 -F test/fts5aa.test bbea71fed733b1d433bf83dbc8d86077936d1efc +F test/fts5aa.test c8d3b9694f6b2864161c7437408464a535d19343 +F test/fts5ab.test 0c44271259bfba089e9e2ab3c18c2760d8a5392c F test/fts5ea.test 814287a2cb25ac3e59abbe4ccbcabf6bda821868 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef @@ -1187,10 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 07dda49c1bf8997a18c3368acb81b6d863ea38d6 -R 66e5d0ccaa728e4d98b92edeb331ffb3 -T *branch * fts5 -T *sym-fts5 * -T -sym-trunk * +P 1e0648dcf283d4f1f6159db4d2433b6cc635992e +R 21f33a2cea70ea3a9d3ce73abf49bcfc U dan -Z e3b7f827041011d2f1d78b39cdee11d7 +Z 81c134ce13df1e24ae1f1936c8a52cf8 diff --git a/manifest.uuid b/manifest.uuid index 6f7226ff29..28b2a683d7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e0648dcf283d4f1f6159db4d2433b6cc635992e \ No newline at end of file +3515da85d09220c464979467b476c611da4a6a7a \ No newline at end of file diff --git a/test/fts5aa.test b/test/fts5aa.test index 699d01d033..f8b8b54d08 100644 --- a/test/fts5aa.test +++ b/test/fts5aa.test @@ -198,9 +198,6 @@ do_execsql_test 8.1 { } -#finish_test - - #------------------------------------------------------------------------- # reset_db diff --git a/test/fts5ab.test b/test/fts5ab.test new file mode 100644 index 0000000000..d075eb01e7 --- /dev/null +++ b/test/fts5ab.test @@ -0,0 +1,57 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts5ab + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + INSERT INTO t1 VALUES('hello', 'world'); + INSERT INTO t1 VALUES('one two', 'three four'); + INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five'); +} + +do_execsql_test 1.1 { + SELECT * FROM t1; +} { forty five {one two} {three four} hello world } + +do_execsql_test 1.2 { + SELECT rowid FROM t1; +} {45 2 1} + +do_execsql_test 1.3 { + SELECT rowid FROM t1 ORDER BY rowid ASC; +} {1 2 45} + +do_execsql_test 1.4 { + SELECT * FROM t1 WHERE rowid=2; +} {{one two} {three four}} + +do_execsql_test 1.5 { + SELECT * FROM t1 WHERE rowid=2.01; +} {} + +do_execsql_test 1.6 { + SELECT * FROM t1 WHERE rowid=1.99; +} {} + +finish_test