From 5e38f1c9bf726872fa55edadf849c9936fd1d93e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 8 May 2015 20:21:24 +0000 Subject: [PATCH] Add the fts5vocab module, for direct access to the fts5 index. FossilOrigin-Name: 6bf93e3b56e6705b7d12bab5024fc615f373b36c --- ext/fts5/fts5.c | 41 ++-- ext/fts5/fts5Int.h | 53 +++-- ext/fts5/fts5_index.c | 48 ++++- ext/fts5/fts5_vocab.c | 370 +++++++++++++++++++++++++++++++++++ ext/fts5/test/fts5vocab.test | 55 ++++++ main.mk | 7 +- manifest | 20 +- manifest.uuid | 2 +- 8 files changed, 553 insertions(+), 43 deletions(-) create mode 100644 ext/fts5/fts5_vocab.c create mode 100644 ext/fts5/test/fts5vocab.test diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c index cd4eff325d..73bcd88953 100644 --- a/ext/fts5/fts5.c +++ b/ext/fts5/fts5.c @@ -145,11 +145,9 @@ struct Fts5Sorter { /* ** Virtual-table cursor object. ** -** zSpecial: +** iSpecial: ** If this is a 'special' query (refer to function fts5SpecialMatch()), -** then this variable points to a nul-terminated buffer containing the -** result to return through the table-name column. It is nul-terminated -** and should eventually be freed using sqlite3_free(). +** then this variable contains the result of the query. */ struct Fts5Cursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ @@ -159,7 +157,7 @@ struct Fts5Cursor { Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */ int csrflags; /* Mask of cursor flags (see below) */ Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */ - char *zSpecial; /* Result of special query */ + i64 iSpecial; /* Result of special query */ /* "rank" function. Populated on demand from vtab.xColumn(). */ char *zRank; /* Custom rank function */ @@ -564,7 +562,6 @@ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ sqlite3_finalize(pCsr->pRankArgStmt); sqlite3_free(pCsr->apRankArg); - sqlite3_free(pCsr->zSpecial); if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){ sqlite3_free(pCsr->zRank); sqlite3_free(pCsr->zRankArgs); @@ -799,12 +796,13 @@ static int fts5SpecialMatch( for(n=0; z[n] && z[n]!=' '; n++); assert( pTab->base.zErrMsg==0 ); - assert( pCsr->zSpecial==0 ); + pCsr->idxNum = FTS5_PLAN_SPECIAL; if( 0==sqlite3_strnicmp("reads", z, n) ){ - pCsr->zSpecial = sqlite3_mprintf("%d", sqlite3Fts5IndexReads(pTab->pIndex)); - pCsr->idxNum = FTS5_PLAN_SPECIAL; - if( pCsr->zSpecial==0 ) rc = SQLITE_NOMEM; + pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->pIndex); + } + else if( 0==sqlite3_strnicmp("id", z, n) ){ + pCsr->iSpecial = pCsr->iCsrId; } else{ /* An unrecognized directive. Return an error message. */ @@ -1668,6 +1666,26 @@ static void fts5ApiCallback( } } + +/* +** Given cursor id iId, return a pointer to the corresponding Fts5Index +** object. Or NULL If the cursor id does not exist. +*/ +Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ + Fts5Cursor *pCsr; + Fts5Index *pIndex = 0; + + for(pCsr=pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ + if( pCsr->iCsrId==iCsrId ) break; + } + if( pCsr ){ + Fts5Table *pTab = (Fts5Table*)pCsr->base.pVtab; + pIndex = pTab->pIndex; + } + + return pIndex; +} + /* ** Return a "position-list blob" corresponding to the current position of ** cursor pCsr via sqlite3_result_blob(). A position-list blob contains @@ -1728,7 +1746,7 @@ static int fts5ColumnMethod( if( pCsr->idxNum==FTS5_PLAN_SPECIAL ){ if( iCol==pConfig->nCol ){ - sqlite3_result_text(pCtx, pCsr->zSpecial, -1, SQLITE_TRANSIENT); + sqlite3_result_int64(pCtx, pCsr->iSpecial); } }else @@ -2059,6 +2077,7 @@ int sqlite3Fts5Init(sqlite3 *db){ if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api); if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api); + if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db); if( rc==SQLITE_OK ){ rc = sqlite3_create_function( db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0 diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 696a8ea5a7..018f26c00a 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -51,24 +51,8 @@ extern int sqlite3_fts5_may_be_corrupt; # define assert_nc(x) assert(x) #endif -/************************************************************************** -** Interface to code in fts5.c. -*/ typedef struct Fts5Global Fts5Global; -int sqlite3Fts5GetTokenizer( - Fts5Global*, - const char **azArg, - int nArg, - Fts5Tokenizer**, - fts5_tokenizer**, - char **pzErr -); - -/* -** End of interface to code in fts5.c. -**************************************************************************/ - /************************************************************************** ** Interface to code in fts5_config.c. fts5_config.c contains contains code ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement. @@ -260,6 +244,7 @@ typedef struct Fts5IndexIter Fts5IndexIter; #define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ #define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ +#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ /* ** Create/destroy an Fts5Index object. @@ -303,6 +288,13 @@ int sqlite3Fts5IterPoslist(Fts5IndexIter*, const u8 **pp, int *pn); */ void sqlite3Fts5IterClose(Fts5IndexIter*); +/* +** This interface is used by the fts5vocab module. +*/ +const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*); +int sqlite3Fts5IterNextScan(Fts5IndexIter*); + + /* ** Insert or remove data to or from the index. Each time a document is ** added to or removed from the index, this function is called one or more @@ -390,6 +382,25 @@ int sqlite3Fts5GetVarintLen(u32 iVal); ** End of interface to code in fts5_index.c. **************************************************************************/ +/************************************************************************** +** Interface to code in fts5.c. +*/ + +int sqlite3Fts5GetTokenizer( + Fts5Global*, + const char **azArg, + int nArg, + Fts5Tokenizer**, + fts5_tokenizer**, + char **pzErr +); + +Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64); + +/* +** End of interface to code in fts5.c. +**************************************************************************/ + /************************************************************************** ** Interface to code in fts5_hash.c. */ @@ -607,4 +618,14 @@ int sqlite3Fts5SorterNew(Fts5Expr *pExpr, Fts5Sorter **pp); ** End of interface to code in fts5_sorter.c. **************************************************************************/ +/************************************************************************** +** Interface to code in fts5_vocab.c. +*/ + +int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*); + +/* +** End of interface to code in fts5_vocab.c. +**************************************************************************/ + #endif diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index a680c20c28..cd15c71b56 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -2075,7 +2075,7 @@ static void fts5SegIterSeekInit( ){ int iPg = 1; int h; - int bGe = (flags & FTS5INDEX_QUERY_PREFIX); + int bGe = (flags & FTS5INDEX_QUERY_SCAN); int bDlidx = 0; /* True if there is a doclist-index */ assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); @@ -2171,7 +2171,7 @@ static void fts5SegIterHashInit( assert( p->pHash ); assert( p->rc==SQLITE_OK ); - if( pTerm==0 || (flags & FTS5INDEX_QUERY_PREFIX) ){ + if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN) ){ p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); n = (z ? strlen((const char*)z) : 0); @@ -4004,6 +4004,7 @@ static void fts5SetupPrefixIter( pStruct = fts5StructureRead(p); if( aBuf && pStruct ){ + const int flags = FTS5INDEX_QUERY_SCAN; Fts5DoclistIter *pDoclist; int i; i64 iLastRowid = 0; @@ -4011,7 +4012,7 @@ static void fts5SetupPrefixIter( Fts5Buffer doclist; memset(&doclist, 0, sizeof(doclist)); - for(fts5MultiIterNew(p, pStruct, 1, 1, pToken, nToken, -1, 0, &p1); + for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1); fts5MultiIterEof(p, p1)==0; fts5MultiIterNext(p, p1, 0, 0) ){ @@ -4272,6 +4273,11 @@ int sqlite3Fts5IndexQuery( int iIdx = 0; Fts5Buffer buf = {0, 0, 0}; + /* If the QUERY_SCAN flag is set, all other flags must be clear. */ + assert( (flags & FTS5INDEX_QUERY_SCAN)==0 + || (flags & FTS5INDEX_QUERY_SCAN)==FTS5INDEX_QUERY_SCAN + ); + if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){ memcpy(&buf.p[1], pToken, nToken); } @@ -4296,9 +4302,8 @@ int sqlite3Fts5IndexQuery( buf.p[0] = FTS5_MAIN_PREFIX + iIdx; pRet->pStruct = fts5StructureRead(p); if( pRet->pStruct ){ - int f = (flags & ~FTS5INDEX_QUERY_PREFIX); fts5MultiIterNew( - p, pRet->pStruct, 1, f, buf.p, nToken+1, -1, 0, &pRet->pMulti + p, pRet->pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet->pMulti ); } }else{ @@ -4343,6 +4348,29 @@ int sqlite3Fts5IterNext(Fts5IndexIter *pIter){ return fts5IndexReturn(pIter->pIndex); } +/* +** Move to the next matching term/rowid. Used by the fts5vocab module. +*/ +int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){ + Fts5Index *p = pIter->pIndex; + Fts5MultiSegIter *pMulti = pIter->pMulti; + + assert( pIter->pIndex->rc==SQLITE_OK ); + assert( pMulti ); + + fts5BufferZero(&pIter->poslist); + fts5MultiIterNext(p, pMulti, 0, 0); + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; + if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){ + fts5DataRelease(pSeg->pLeaf); + pSeg->pLeaf = 0; + } + } + + return fts5IndexReturn(pIter->pIndex); +} + /* ** Move the doclist-iter passed as the first argument to the next ** matching rowid that occurs at or after iMatch. The definition of "at @@ -4383,6 +4411,16 @@ i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){ } } +/* +** Return the current term. +*/ +const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIter, int *pn){ + int n; + const char *z = fts5MultiIterTerm(pIter->pMulti, &n); + *pn = n-1; + return &z[1]; +} + /* ** Return a pointer to a buffer containing a copy of the position list for diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c new file mode 100644 index 0000000000..8d4d6c1762 --- /dev/null +++ b/ext/fts5/fts5_vocab.c @@ -0,0 +1,370 @@ +/* +** 2015 May 08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + +#if defined(SQLITE_ENABLE_FTS5) + +#include "fts5Int.h" + + +typedef struct Fts5VocabTable Fts5VocabTable; +typedef struct Fts5VocabCursor Fts5VocabCursor; + +struct Fts5VocabTable { + sqlite3_vtab base; + char *zFts5Tbl; /* Name of fts5 table */ + char *zFts5Db; /* Db containing fts5 table */ + sqlite3 *db; /* Database handle */ + Fts5Global *pGlobal; /* FTS5 global object for this database */ +}; + +struct Fts5VocabCursor { + sqlite3_vtab_cursor base; + sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */ + Fts5Index *pIndex; /* Associated FTS5 index */ + + Fts5IndexIter *pIter; /* Iterator object */ + int bEof; /* True if this cursor is at EOF */ + Fts5Buffer term; /* Current value of 'term' column */ + i64 nRow; /* Current value of 'row' column */ + i64 nInst; /* Current value of 'inst' column */ + i64 rowid; /* Current value of rowid column */ +}; + + +/* +** The xDisconnect() virtual table method. +*/ +static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts5VocabDestroyMethod(sqlite3_vtab *pVtab){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the FTS3 virtual table. +** +** The argv[] array contains the following: +** +** argv[0] -> module name ("fts5vocab") +** argv[1] -> database name +** argv[2] -> table name +** argv[3] -> name of fts5 table +*/ +static int fts5VocabInitVtab( + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Pointer to Fts5Global object */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + const char *zSchema = "CREATE TABLE vvv(term, row, inst)"; + Fts5VocabTable *pRet = 0; + int rc = SQLITE_OK; /* Return code */ + + if( argc!=4 ){ + *pzErr = sqlite3_mprintf("wrong number of vtable arguments"); + rc = SQLITE_ERROR; + }else{ + int nByte; /* Bytes of space to allocate */ + const char *zDb = argv[1]; + const char *zTab = argv[3]; + int nDb = strlen(zDb) + 1; + int nTab = strlen(zTab) + 1; + + rc = sqlite3_declare_vtab(db, zSchema); + + nByte = sizeof(Fts5VocabTable) + nDb + nTab; + pRet = sqlite3Fts5MallocZero(&rc, nByte); + if( pRet ){ + pRet->pGlobal = (Fts5Global*)pAux; + pRet->db = db; + pRet->zFts5Tbl = (char*)&pRet[1]; + pRet->zFts5Db = &pRet->zFts5Tbl[nTab]; + memcpy(pRet->zFts5Tbl, zTab, nTab); + memcpy(pRet->zFts5Db, zDb, nDb); + } + } + + *ppVTab = (sqlite3_vtab*)pRet; + return rc; +} + + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts5VocabInitVtab(). +*/ +static int fts5VocabConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts5VocabCreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Implementation of the xBestIndex method. +*/ +static int fts5VocabBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + return SQLITE_OK; +} + +/* +** Implementation of xOpen method. +*/ +static int fts5VocabOpenMethod( + sqlite3_vtab *pVTab, + sqlite3_vtab_cursor **ppCsr +){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab; + Fts5VocabCursor *pCsr; + int rc = SQLITE_OK; + + pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5VocabCursor)); + if( pCsr ){ + char *zSql = sqlite3_mprintf( + "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'", + pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl + ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + } + sqlite3_free(zSql); + if( rc==SQLITE_OK && sqlite3_step(pCsr->pStmt)==SQLITE_ROW ){ + i64 iId = sqlite3_column_int64(pCsr->pStmt, 0); + pCsr->pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId); + } + + if( rc==SQLITE_OK && pCsr->pIndex==0 ){ + rc = sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + if( rc==SQLITE_OK ){ + pVTab->zErrMsg = sqlite3_mprintf( + "no such fts5 table: %Q.%Q", pTab->zFts5Db, pTab->zFts5Tbl + ); + rc = SQLITE_ERROR; + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(pCsr); + pCsr = 0; + } + } + + + *ppCsr = (sqlite3_vtab_cursor*)pCsr; + return rc; +} + +static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ + pCsr->rowid = 0; + sqlite3Fts5IterClose(pCsr->pIter); + pCsr->pIter = 0; +} + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){ + if( pCursor ){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + fts5VocabResetCursor(pCsr); + sqlite3Fts5BufferFree(&pCsr->term); + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr); + } + return SQLITE_OK; +} + + +/* +** Advance the cursor to the next row in the table. +*/ +static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int rc = SQLITE_OK; + + if( sqlite3Fts5IterEof(pCsr->pIter) ){ + pCsr->bEof = 1; + }else{ + const char *zTerm; + int nTerm; + + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm); + pCsr->nInst = 0; + pCsr->nRow = 0; + pCsr->rowid++; + + while( 1 ){ + const u8 *pPos; int nPos; /* Position list */ + i64 dummy = 0; + int iOff = 0; + + rc = sqlite3Fts5IterPoslist(pCsr->pIter, &pPos, &nPos); + if( rc!=SQLITE_OK ) break; + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &dummy) ){ + pCsr->nInst++; + } + pCsr->nRow++; + + rc = sqlite3Fts5IterNextScan(pCsr->pIter); + if( rc!=SQLITE_OK ) break; + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break; + if( sqlite3Fts5IterEof(pCsr->pIter) ) break; + } + } + return rc; +} + +/* +** This is the xFilter implementation for the virtual table. +*/ +static int fts5VocabFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int rc; + const int flags = FTS5INDEX_QUERY_SCAN; + + fts5VocabResetCursor(pCsr); + rc = sqlite3Fts5IndexQuery(pCsr->pIndex, 0, 0, flags, &pCsr->pIter); + if( rc==SQLITE_OK ){ + rc = fts5VocabNextMethod(pCursor); + } + + return rc; +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + return pCsr->bEof; +} + +static int fts5VocabColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + switch( iCol ){ + case 0: /* term */ + sqlite3_result_text( + pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT + ); + break; + + case 1: /* row */ + sqlite3_result_int64(pCtx, pCsr->nRow); + break; + + case 2: /* inst */ + sqlite3_result_int64(pCtx, pCsr->nInst); + break; + + default: + assert( 0 ); + } + return SQLITE_OK; +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts5 +** exposes %_content.docid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts5VocabRowidMethod( + sqlite3_vtab_cursor *pCursor, + sqlite_int64 *pRowid +){ + Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + *pRowid = pCsr->rowid; + return SQLITE_OK; +} + +int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){ + static const sqlite3_module fts5Vocab = { + /* iVersion */ 2, + /* xCreate */ fts5VocabCreateMethod, + /* xConnect */ fts5VocabConnectMethod, + /* xBestIndex */ fts5VocabBestIndexMethod, + /* xDisconnect */ fts5VocabDisconnectMethod, + /* xDestroy */ fts5VocabDestroyMethod, + /* xOpen */ fts5VocabOpenMethod, + /* xClose */ fts5VocabCloseMethod, + /* xFilter */ fts5VocabFilterMethod, + /* xNext */ fts5VocabNextMethod, + /* xEof */ fts5VocabEofMethod, + /* xColumn */ fts5VocabColumnMethod, + /* xRowid */ fts5VocabRowidMethod, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindFunction */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + }; + void *p = (void*)pGlobal; + + return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0); +} +#endif /* defined(SQLITE_ENABLE_FTS5) */ + + diff --git a/ext/fts5/test/fts5vocab.test b/ext/fts5/test/fts5vocab.test new file mode 100644 index 0000000000..fb7c24e1ff --- /dev/null +++ b/ext/fts5/test/fts5vocab.test @@ -0,0 +1,55 @@ +# 2015 Apr 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file focus on testing the fts5vocab module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5vocab + + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1); + PRAGMA table_info = v1; +} { + 0 term {} 0 {} 0 + 1 row {} 0 {} 0 + 2 inst {} 0 {} 0 +} + +do_execsql_test 1.2 { SELECT * FROM v1 } { } + +do_execsql_test 1.3 { + INSERT INTO t1 VALUES('x y z'); + INSERT INTO t1 VALUES('x x x'); +} + +do_execsql_test 1.4 { + SELECT * FROM v1; +} {x 2 4 y 1 1 z 1 1} + +do_execsql_test 1.5 { + BEGIN; + INSERT INTO t1 VALUES('a b c'); + SELECT * FROM v1 WHERE term<'d'; + COMMIT; +} {a 1 1 b 1 1 c 1 1} + +do_execsql_test 1.6 { + DELETE FROM t1 WHERE one = 'a b c'; + SELECT * FROM v1; +} {x 2 4 y 1 1 z 1 1} + + + +finish_test + diff --git a/main.mk b/main.mk index 11dc94ea59..bc63a3b569 100644 --- a/main.mk +++ b/main.mk @@ -82,6 +82,7 @@ LIBOBJ += fts5_index.o LIBOBJ += fts5_storage.o LIBOBJ += fts5_tokenize.o LIBOBJ += fts5_unicode2.o +LIBOBJ += fts5_vocab.o LIBOBJ += fts5parse.o @@ -246,7 +247,8 @@ SRC += \ fts5parse.c fts5parse.h \ $(TOP)/ext/fts5/fts5_storage.c \ $(TOP)/ext/fts5/fts5_tokenize.c \ - $(TOP)/ext/fts5/fts5_unicode2.c + $(TOP)/ext/fts5/fts5_unicode2.c \ + $(TOP)/ext/fts5/fts5_vocab.c # Generated source code files @@ -656,6 +658,9 @@ fts5_tokenize.o: $(TOP)/ext/fts5/fts5_tokenize.c $(HDR) $(EXTHDR) fts5_unicode2.o: $(TOP)/ext/fts5/fts5_unicode2.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_unicode2.c +fts5_vocab.o: $(TOP)/ext/fts5/fts5_vocab.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_vocab.c + fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h diff --git a/manifest b/manifest index 76b5dac082..ddaea8872b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\serror\smessage\sreturned\sby\sFTS5\sif\sit\sencounters\san\sunknown\sfile\sformat. -D 2015-05-08T09:21:05.416 +C Add\sthe\sfts5vocab\smodule,\sfor\sdirect\saccess\sto\sthe\sfts5\sindex. +D 2015-05-08T20:21:24.206 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 31b38b9da2e4b36f54a013bd71a5c3f6e45ca78f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -104,19 +104,20 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 159c1194da0bc72f51b3c2eb71022568006dc5ad F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a -F ext/fts5/fts5.c 7f58ea9ba1e72038137963719c5b5335f499cecd +F ext/fts5/fts5.c 9e521f3556b9929996909402ddf337f2e771e87c F ext/fts5/fts5.h 24a2cc35b5e76eec57b37ba48c12d9d2cb522b3a -F ext/fts5/fts5Int.h be8ac04ce40705aa088c3d2509cadad0f98085fa +F ext/fts5/fts5Int.h fc3edf2538551c5bdb02885c517483d604394d3c F ext/fts5/fts5_aux.c d53f00f31ad615ca4f139dd8751f9041afa00971 F ext/fts5/fts5_buffer.c 70b971e13503566f1e257941c60817ba0920a16b F ext/fts5/fts5_config.c 05811f0bd80c396afcf3ceea68da16149a9a3258 F ext/fts5/fts5_expr.c 3fe1170453d6a322d2de8a3fd0aed3edff7b8b09 F ext/fts5/fts5_hash.c 54dd25348a46ea62ea96322c572e08cd1fb37304 -F ext/fts5/fts5_index.c aa8d73d043417740c07861beb78c86103a6a9d90 +F ext/fts5/fts5_index.c 6a4fed2d64d7dbb0416c4278b23201f77daf94ea F ext/fts5/fts5_storage.c cb8b585bfb7870a36101f1a8fa0b0777f4d1b68d F ext/fts5/fts5_tcl.c aa3b102bb01f366174718be7ce8e9311b9abb482 F ext/fts5/fts5_tokenize.c 830eae0d35a5a5a90af34df65da3427f46d942fc F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d +F ext/fts5/fts5_vocab.c 9e021b7f95890f1403e84dc4be4c94559c07ee54 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32 @@ -164,6 +165,7 @@ F ext/fts5/test/fts5unicode.test 79b3e34eb29ce4929628aa514a40cb467fdabe4d F ext/fts5/test/fts5unicode2.test 64a5267fd6082fcb46439892ebd0cbaa5c38acee F ext/fts5/test/fts5unindexed.test f388605341a476b6ab622b4c267cd168f59a5944 F ext/fts5/test/fts5version.test 1c902eaa7359336293ac45c7a34616527513e9fb +F ext/fts5/test/fts5vocab.test d0cb4286a0d900f46498587366efacfc75741f0f F ext/fts5/tool/loadfts5.tcl 8a8f10d7d2d0d77f622e0a84cc0824c158c34a52 F ext/fts5/tool/showfts5.tcl 921f33b30c3189deefd2b2cc81f951638544aaf1 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 @@ -215,7 +217,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 76306018b967871262c6c8290e3914685f279ded +F main.mk 063cdc009247a9b543875ea12f4e27b8f3bcca54 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk d5e22023b5238985bb54a72d33e0ac71fe4f8a32 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1317,7 +1319,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a684b5e2d9d52cf4700e7e5f9dd547a2ba54e8e9 -R ee87a64da50ec14a005ebb86ec227c20 +P f369caec145f311bb136cf7af144e2695badcb9b +R c4f9cd11ccfbc0fa9f03125deb45e448 U dan -Z 8aaf7ae5929104d0a1ed12733f883e2b +Z 31686ee8f2f28db91dc188b739012da2 diff --git a/manifest.uuid b/manifest.uuid index 9406775af2..1a592dd121 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f369caec145f311bb136cf7af144e2695badcb9b \ No newline at end of file +6bf93e3b56e6705b7d12bab5024fc615f373b36c \ No newline at end of file -- 2.47.3