**
******************************************************************************
**
-** This is an SQLite module implementing full-text search.
+** This is an SQLite virtual table module implementing direct access to an
+** existing FTS5 index. The module may create several different types of
+** tables:
+**
+** col:
+** CREATE TABLE vocab(term, col, doc, cnt, PRIMARY KEY(term, col));
+**
+** One row for each term/column combination. The value of $doc is set to
+** the number of fts5 rows that contain at least one instance of term
+** $term within column $col. Field $cnt is set to the total number of
+** instances of term $term in column $col (in any row of the fts5 table).
+**
+** row:
+** CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term));
+**
+** One row for each term in the database. The value of $doc is set to
+** the number of fts5 rows that contain at least one instance of term
+** $term. Field $cnt is set to the total number of instances of term
+** $term in the database.
*/
#if defined(SQLITE_ENABLE_FTS5)
char *zFts5Db; /* Db containing fts5 table */
sqlite3 *db; /* Database handle */
Fts5Global *pGlobal; /* FTS5 global object for this database */
+ int eType; /* FTS5_VOCAB_COL or ROW */
};
struct Fts5VocabCursor {
sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */
Fts5Index *pIndex; /* Associated FTS5 index */
- Fts5IndexIter *pIter; /* Iterator object */
int bEof; /* True if this cursor is at EOF */
+ Fts5IndexIter *pIter; /* Term/rowid iterator object */
+
+ /* These are used by 'col' tables only */
+ int nCol;
+ int iCol;
+ i64 *aCnt;
+ i64 *aDoc;
+
+ /* Output values */
+ i64 rowid; /* This table's current rowid value */
Fts5Buffer term; /* Current value of 'term' column */
- i64 nRow; /* Current value of 'row' column */
- i64 nInst; /* Current value of 'inst' column */
- i64 rowid; /* Current value of rowid column */
+ i64 aVal[3]; /* Up to three columns left of 'term' */
};
+#define FTS5_VOCAB_COL 0
+#define FTS5_VOCAB_ROW 1
+
+#define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt"
+#define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt"
+
+/*
+** Translate a string containing an fts5vocab table type to an
+** FTS5_VOCAB_XXX constant. If successful, set *peType to the output
+** value and return SQLITE_OK. Otherwise, set *pzErr to an error message
+** and return SQLITE_ERROR.
+*/
+static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){
+ int rc = SQLITE_OK;
+ char *zCopy = sqlite3Fts5Strndup(&rc, zType, -1);
+ if( rc==SQLITE_OK ){
+ sqlite3Fts5Dequote(zCopy);
+ if( sqlite3_stricmp(zCopy, "col")==0 ){
+ *peType = FTS5_VOCAB_COL;
+ }else
+
+ if( sqlite3_stricmp(zCopy, "row")==0 ){
+ *peType = FTS5_VOCAB_ROW;
+ }else
+ {
+ *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy);
+ rc = SQLITE_ERROR;
+ }
+ sqlite3_free(zCopy);
+ }
+
+ return rc;
+}
+
/*
** The xDisconnect() virtual table method.
** argv[0] -> module name ("fts5vocab")
** argv[1] -> database name
** argv[2] -> table name
+**
+** then:
+**
** argv[3] -> name of fts5 table
+** argv[4] -> type of fts5vocab table
+**
+** or, for tables in the TEMP schema only.
+**
+** argv[3] -> name of fts5 tables database
+** argv[4] -> name of fts5 table
+** argv[5] -> type of fts5vocab table
*/
static int fts5VocabInitVtab(
sqlite3 *db, /* The SQLite database connection */
sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
char **pzErr /* Write any error message here */
){
- const char *zSchema = "CREATE TABLE vvv(term, row, inst)";
+ const char *azSchema[] = {
+ "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")",
+ "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")"
+ };
+
Fts5VocabTable *pRet = 0;
int rc = SQLITE_OK; /* Return code */
+ int bDb;
+
+ bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0);
- if( argc!=4 ){
+ if( argc!=5 && bDb==0 ){
*pzErr = sqlite3_mprintf("wrong number of vtable arguments");
rc = SQLITE_ERROR;
}else{
int nByte; /* Bytes of space to allocate */
- const char *zDb = argv[1];
- const char *zTab = argv[3];
- int nDb = strlen(zDb) + 1;
- int nTab = strlen(zTab) + 1;
-
- rc = sqlite3_declare_vtab(db, zSchema);
+ const char *zDb = bDb ? argv[3] : argv[1];
+ const char *zTab = bDb ? argv[4] : argv[3];
+ const char *zType = bDb ? argv[5] : argv[4];
+ int nDb = strlen(zDb)+1;
+ int nTab = strlen(zTab)+1;
+ int eType;
+
+ rc = fts5VocabTableType(zType, pzErr, &eType);
+ if( rc==SQLITE_OK ){
+ assert( eType>=0 && eType<sizeof(azSchema)/sizeof(azSchema[0]) );
+ rc = sqlite3_declare_vtab(db, azSchema[eType]);
+ }
nByte = sizeof(Fts5VocabTable) + nDb + nTab;
pRet = sqlite3Fts5MallocZero(&rc, nByte);
if( pRet ){
pRet->pGlobal = (Fts5Global*)pAux;
+ pRet->eType = eType;
pRet->db = db;
pRet->zFts5Tbl = (char*)&pRet[1];
pRet->zFts5Db = &pRet->zFts5Tbl[nTab];
sqlite3_vtab_cursor **ppCsr
){
Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab;
- Fts5VocabCursor *pCsr;
+ Fts5Index *pIndex = 0;
+ int nCol = 0;
+ Fts5VocabCursor *pCsr = 0;
int rc = SQLITE_OK;
+ sqlite3_stmt *pStmt = 0;
+ char *zSql = 0;
+ int nByte;
+
+ zSql = sqlite3_mprintf(
+ "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
+ pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl
+ );
+ if( zSql==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
+ }
+ sqlite3_free(zSql);
- pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5VocabCursor));
- if( pCsr ){
- char *zSql = sqlite3_mprintf(
- "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
- pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl
- );
- if( zSql==0 ){
- rc = SQLITE_NOMEM;
- }else{
- rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
- }
- sqlite3_free(zSql);
- if( rc==SQLITE_OK && sqlite3_step(pCsr->pStmt)==SQLITE_ROW ){
- i64 iId = sqlite3_column_int64(pCsr->pStmt, 0);
- pCsr->pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId);
- }
-
- if( rc==SQLITE_OK && pCsr->pIndex==0 ){
- rc = sqlite3_finalize(pCsr->pStmt);
- pCsr->pStmt = 0;
- if( rc==SQLITE_OK ){
- pVTab->zErrMsg = sqlite3_mprintf(
- "no such fts5 table: %Q.%Q", pTab->zFts5Db, pTab->zFts5Tbl
- );
- rc = SQLITE_ERROR;
- }
- }
+ if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+ i64 iId = sqlite3_column_int64(pStmt, 0);
+ pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &nCol);
+ }
- if( rc!=SQLITE_OK ){
- sqlite3_free(pCsr);
- pCsr = 0;
+ if( rc==SQLITE_OK && pIndex==0 ){
+ rc = sqlite3_finalize(pStmt);
+ pStmt = 0;
+ if( rc==SQLITE_OK ){
+ pVTab->zErrMsg = sqlite3_mprintf(
+ "no such fts5 table: %Q.%Q", pTab->zFts5Db, pTab->zFts5Tbl
+ );
+ rc = SQLITE_ERROR;
}
}
+ nByte = nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor);
+ pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
+ if( pCsr ){
+ pCsr->pIndex = pIndex;
+ pCsr->pStmt = pStmt;
+ pCsr->nCol = nCol;
+ pCsr->aCnt = (i64*)&pCsr[1];
+ pCsr->aDoc = &pCsr->aCnt[nCol];
+ }else{
+ sqlite3_finalize(pStmt);
+ }
*ppCsr = (sqlite3_vtab_cursor*)pCsr;
return rc;
*/
static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
+ Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
int rc = SQLITE_OK;
- if( sqlite3Fts5IterEof(pCsr->pIter) ){
- pCsr->bEof = 1;
- }else{
- const char *zTerm;
- int nTerm;
-
- zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
- sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
- pCsr->nInst = 0;
- pCsr->nRow = 0;
- pCsr->rowid++;
-
- while( 1 ){
- const u8 *pPos; int nPos; /* Position list */
- i64 dummy = 0;
- int iOff = 0;
-
- rc = sqlite3Fts5IterPoslist(pCsr->pIter, &pPos, &nPos);
- if( rc!=SQLITE_OK ) break;
- while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &dummy) ){
- pCsr->nInst++;
- }
- pCsr->nRow++;
+ pCsr->rowid++;
+
+ if( pTab->eType==FTS5_VOCAB_COL ){
+ for(pCsr->iCol++; pCsr->iCol<pCsr->nCol; pCsr->iCol++){
+ if( pCsr->aCnt[pCsr->iCol] ) break;
+ }
+ }
+
+ if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=pCsr->nCol ){
+ if( sqlite3Fts5IterEof(pCsr->pIter) ){
+ pCsr->bEof = 1;
+ }else{
+ const char *zTerm;
+ int nTerm;
- rc = sqlite3Fts5IterNextScan(pCsr->pIter);
- if( rc!=SQLITE_OK ) break;
zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
- if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break;
- if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
+ sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
+ memset(pCsr->aVal, 0, sizeof(pCsr->aVal));
+ memset(pCsr->aCnt, 0, pCsr->nCol * sizeof(i64));
+ memset(pCsr->aDoc, 0, pCsr->nCol * sizeof(i64));
+ pCsr->iCol = 0;
+
+ assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
+ while( 1 ){
+ const u8 *pPos; int nPos; /* Position list */
+ i64 iPos = 0; /* 64-bit position read from poslist */
+ int iOff = 0; /* Current offset within position list */
+
+ rc = sqlite3Fts5IterPoslist(pCsr->pIter, &pPos, &nPos);
+ if( rc!=SQLITE_OK ) break;
+
+ if( pTab->eType==FTS5_VOCAB_ROW ){
+ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
+ pCsr->aVal[1]++;
+ }
+ pCsr->aVal[0]++;
+ }else{
+ int iCol = -1;
+ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
+ int ii = FTS5_POS2COLUMN(iPos);
+ pCsr->aCnt[ii]++;
+ if( iCol!=ii ){
+ pCsr->aDoc[ii]++;
+ iCol = ii;
+ }
+ }
+ }
+
+ rc = sqlite3Fts5IterNextScan(pCsr->pIter);
+ if( rc!=SQLITE_OK ) break;
+ zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
+ if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break;
+ if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
+ }
}
}
+
+ if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
+ while( pCsr->aCnt[pCsr->iCol]==0 ) pCsr->iCol++;
+ pCsr->aVal[0] = pCsr->iCol;
+ pCsr->aVal[1] = pCsr->aDoc[pCsr->iCol];
+ pCsr->aVal[2] = pCsr->aCnt[pCsr->iCol];
+ }
return rc;
}
);
break;
- case 1: /* row */
- sqlite3_result_int64(pCtx, pCsr->nRow);
- break;
-
- case 2: /* inst */
- sqlite3_result_int64(pCtx, pCsr->nInst);
- break;
-
default:
- assert( 0 );
+ assert( iCol<4 && iCol>0 );
+ sqlite3_result_int64(pCtx, pCsr->aVal[iCol-1]);
+ break;
}
return SQLITE_OK;
}
set testprefix fts5vocab
-do_execsql_test 1.1 {
+do_execsql_test 1.1.1 {
CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1);
- CREATE VIRTUAL TABLE v1 USING fts5vocab(t1);
+ CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row');
PRAGMA table_info = v1;
} {
0 term {} 0 {} 0
- 1 row {} 0 {} 0
- 2 inst {} 0 {} 0
+ 1 doc {} 0 {} 0
+ 2 cnt {} 0 {} 0
}
-do_execsql_test 1.2 { SELECT * FROM v1 } { }
+do_execsql_test 1.1.2 {
+ CREATE VIRTUAL TABLE v2 USING fts5vocab(t1, 'col');
+ PRAGMA table_info = v2;
+} {
+ 0 term {} 0 {} 0
+ 1 col {} 0 {} 0
+ 2 doc {} 0 {} 0
+ 3 cnt {} 0 {} 0
+}
+
+do_execsql_test 1.2.1 { SELECT * FROM v1 } { }
+do_execsql_test 1.2.2 { SELECT * FROM v2 } { }
do_execsql_test 1.3 {
INSERT INTO t1 VALUES('x y z');
INSERT INTO t1 VALUES('x x x');
}
-do_execsql_test 1.4 {
+do_execsql_test 1.4.1 {
SELECT * FROM v1;
} {x 2 4 y 1 1 z 1 1}
-do_execsql_test 1.5 {
+do_execsql_test 1.4.2 {
+ SELECT * FROM v2;
+} {x 0 2 4 y 0 1 1 z 0 1 1}
+
+do_execsql_test 1.5.1 {
BEGIN;
INSERT INTO t1 VALUES('a b c');
SELECT * FROM v1 WHERE term<'d';
- COMMIT;
} {a 1 1 b 1 1 c 1 1}
+do_execsql_test 1.5.2 {
+ SELECT * FROM v2 WHERE term<'d';
+ COMMIT;
+} {a 0 1 1 b 0 1 1 c 0 1 1}
+
do_execsql_test 1.6 {
DELETE FROM t1 WHERE one = 'a b c';
SELECT * FROM v1;
} {x 2 4 y 1 1 z 1 1}
+#-------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+ CREATE VIRTUAL TABLE tt USING fts5(a, b);
+ INSERT INTO tt VALUES('d g b f d f', 'f c e c d a');
+ INSERT INTO tt VALUES('f a e a a b', 'e d c f d d');
+ INSERT INTO tt VALUES('b c a a a b', 'f f c c b c');
+ INSERT INTO tt VALUES('f d c a c e', 'd g d e g d');
+ INSERT INTO tt VALUES('g d e f a g x', 'f f d a a b');
+ INSERT INTO tt VALUES('g c f b c g', 'a g f d c b');
+ INSERT INTO tt VALUES('c e c f g b', 'f e d b g a');
+ INSERT INTO tt VALUES('g d e f d e', 'a c d b a g');
+ INSERT INTO tt VALUES('e f a c c b', 'b f e a f d y');
+ INSERT INTO tt VALUES('c c a a c f', 'd g a e b g');
+ CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'col');
+ SELECT * FROM tv;
+} {
+ a 0 6 11 a 1 7 9
+ b 0 6 7 b 1 7 7
+ c 0 6 12 c 1 5 8
+ d 0 4 6 d 1 9 13
+ e 0 6 7 e 1 6 6
+ f 0 9 10 f 1 7 10
+ g 0 5 7 g 1 5 7
+ x 0 1 1 y 1 1 1
+}
+do_execsql_test 2.1 {
+ CREATE VIRTUAL TABLE temp.tv2 USING fts5vocab(main, tt, 'row');
+ SELECT * FROM tv2;
+} {
+ a 10 20 b 9 14 c 9 20 d 9 19
+ e 8 13 f 10 20 g 7 14 x 1 1
+ y 1 1
+}
+#-------------------------------------------------------------------------
+#
+foreach {tn sql} {
+ 1 { CREATE VIRTUAL TABLE aa USING fts5vocab() }
+ 2 { CREATE VIRTUAL TABLE aa USING fts5vocab(x) }
+ 3 { CREATE VIRTUAL TABLE aa USING fts5vocab(x,y,z) }
+ 4 { CREATE VIRTUAL TABLE temp.aa USING fts5vocab(x,y,z,y) }
+} {
+ do_catchsql_test 3.$tn $sql {1 {wrong number of vtable arguments}}
+}
finish_test
-C Add\sthe\sfts5vocab\smodule,\sfor\sdirect\saccess\sto\sthe\sfts5\sindex.
-D 2015-05-08T20:21:24.206
+C Allow\sthe\sfts5vocab\stable\sto\soptionally\sprovide\sdata\son\sa\sper-column\sbasis.
+D 2015-05-09T18:28:27.134
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 31b38b9da2e4b36f54a013bd71a5c3f6e45ca78f
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl 159c1194da0bc72f51b3c2eb71022568006dc5ad
F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
-F ext/fts5/fts5.c 9e521f3556b9929996909402ddf337f2e771e87c
+F ext/fts5/fts5.c a5a908a68c79c352a0dfa77d16712de43896bd07
F ext/fts5/fts5.h 24a2cc35b5e76eec57b37ba48c12d9d2cb522b3a
-F ext/fts5/fts5Int.h fc3edf2538551c5bdb02885c517483d604394d3c
+F ext/fts5/fts5Int.h 5b9e4afe80d18648bc236b9b5bc2f873634326f6
F ext/fts5/fts5_aux.c d53f00f31ad615ca4f139dd8751f9041afa00971
F ext/fts5/fts5_buffer.c 70b971e13503566f1e257941c60817ba0920a16b
F ext/fts5/fts5_config.c 05811f0bd80c396afcf3ceea68da16149a9a3258
F ext/fts5/fts5_tcl.c aa3b102bb01f366174718be7ce8e9311b9abb482
F ext/fts5/fts5_tokenize.c 830eae0d35a5a5a90af34df65da3427f46d942fc
F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d
-F ext/fts5/fts5_vocab.c 9e021b7f95890f1403e84dc4be4c94559c07ee54
+F ext/fts5/fts5_vocab.c 2e37ea9b4d4d5460bc778f2adb872c6a869601e7
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32
F ext/fts5/test/fts5unicode2.test 64a5267fd6082fcb46439892ebd0cbaa5c38acee
F ext/fts5/test/fts5unindexed.test f388605341a476b6ab622b4c267cd168f59a5944
F ext/fts5/test/fts5version.test 1c902eaa7359336293ac45c7a34616527513e9fb
-F ext/fts5/test/fts5vocab.test d0cb4286a0d900f46498587366efacfc75741f0f
+F ext/fts5/test/fts5vocab.test 2d1bddfb6e1effd9e1d2f5d1d25bf05e9ab33e64
F ext/fts5/tool/loadfts5.tcl 8a8f10d7d2d0d77f622e0a84cc0824c158c34a52
F ext/fts5/tool/showfts5.tcl 921f33b30c3189deefd2b2cc81f951638544aaf1
F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P f369caec145f311bb136cf7af144e2695badcb9b
-R c4f9cd11ccfbc0fa9f03125deb45e448
+P 6bf93e3b56e6705b7d12bab5024fc615f373b36c
+R f31ac5d295b9e0df865bd081bc32aa0b
U dan
-Z 31686ee8f2f28db91dc188b739012da2
+Z 9e8a79e0ffff336d7475aff60e841c57
projects / thirdparty / sqlite.git / commitdiff
