sqlite3_vtab_cursor base; /* Base class used by SQLite core */
int idxNum; /* idxNum passed to xFilter() */
sqlite3_stmt *pStmt; /* Statement used to read %_content */
- int bEof; /* True at EOF */
Fts5Expr *pExpr; /* Expression for MATCH queries */
- int bSeekRequired; /* True if seek is required */
+ int csrflags; /* Mask of cursor flags (see below) */
Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */
/* Variables used by auxiliary functions */
i64 iCsrId; /* Cursor id */
Fts5Auxiliary *pAux; /* Currently executing function */
+ int *aColumnSize; /* Values for xColumnSize() */
};
+/*
+** Values for Fts5Cursor.csrflags
+*/
+#define FTS5CSR_REQUIRE_CONTENT 0x01
+#define FTS5CSR_REQUIRE_DOCSIZE 0x02
+#define FTS5CSR_EOF 0x04
+
+/*
+** Macros to Set(), Clear() and Test() cursor flags.
+*/
+#define CsrFlagSet(pCsr, flag) ((pCsr)->csrflags |= (flag))
+#define CsrFlagClear(pCsr, flag) ((pCsr)->csrflags &= ~(flag))
+#define CsrFlagTest(pCsr, flag) ((pCsr)->csrflags & (flag))
+
/*
** Close a virtual table handle opened by fts5InitVtab(). If the bDestroy
** argument is non-zero, attempt delete the shadow tables from teh database
*/
static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
Fts5Table *pTab = (Fts5Table*)pVTab;
+ Fts5Config *pConfig = pTab->pConfig;
Fts5Cursor *pCsr; /* New cursor object */
+ int nByte; /* Bytes of space to allocate */
int rc = SQLITE_OK; /* Return code */
- pCsr = (Fts5Cursor*)sqlite3_malloc(sizeof(Fts5Cursor));
+ nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
+ pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
if( pCsr ){
Fts5Global *pGlobal = pTab->pGlobal;
- memset(pCsr, 0, sizeof(Fts5Cursor));
+ memset(pCsr, 0, nByte);
+ pCsr->aColumnSize = (int*)&pCsr[1];
pCsr->pNext = pGlobal->pCsr;
pGlobal->pCsr = pCsr;
pCsr->iCsrId = ++pGlobal->iNextId;
if( ePlan!=FTS5_PLAN_MATCH ){
rc = sqlite3_step(pCsr->pStmt);
if( rc!=SQLITE_ROW ){
- pCsr->bEof = 1;
+ CsrFlagSet(pCsr, FTS5CSR_EOF);
rc = sqlite3_reset(pCsr->pStmt);
}else{
rc = SQLITE_OK;
}
}else{
rc = sqlite3Fts5ExprNext(pCsr->pExpr);
- pCsr->bEof = sqlite3Fts5ExprEof(pCsr->pExpr);
- pCsr->bSeekRequired = 1;
+ if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
+ CsrFlagSet(pCsr, FTS5CSR_EOF);
+ }
+ CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
}
return rc;
int eStmt = fts5StmtType(idxNum);
int bAsc = ((idxNum & FTS5_ORDER_ASC) ? 1 : 0);
- memset(&pCursor[1], 0, sizeof(Fts5Cursor) - sizeof(sqlite3_vtab_cursor));
pCsr->idxNum = idxNum;
+ assert( pCsr->pStmt==0 );
+ assert( pCsr->pExpr==0 );
+ assert( pCsr->csrflags==0 );
rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bAsc);
- pCsr->bEof = sqlite3Fts5ExprEof(pCsr->pExpr);
- pCsr->bSeekRequired = 1;
+ if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
+ CsrFlagSet(pCsr, FTS5CSR_EOF);
+ }
+ CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
}
}else{
if( ePlan==FTS5_PLAN_ROWID ){
*/
static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
- return pCsr->bEof;
+ return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0);
}
/*
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
int ePlan = FTS5_PLAN(pCsr->idxNum);
- assert( pCsr->bEof==0 );
+ assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
if( ePlan!=FTS5_PLAN_MATCH ){
*pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
}else{
*/
static int fts5SeekCursor(Fts5Cursor *pCsr){
int rc = SQLITE_OK;
- if( pCsr->bSeekRequired ){
+ if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){
assert( pCsr->pExpr );
sqlite3_reset(pCsr->pStmt);
sqlite3_bind_int64(pCsr->pStmt, 1, sqlite3Fts5ExprRowid(pCsr->pExpr));
rc = sqlite3_step(pCsr->pStmt);
if( rc==SQLITE_ROW ){
rc = SQLITE_OK;
+ CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT);
}else{
rc = sqlite3_reset(pCsr->pStmt);
if( rc==SQLITE_OK ){
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
int rc = SQLITE_OK;
- assert( pCsr->bEof==0 );
+ assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
if( iCol==pConfig->nCol ){
/* User is requesting the value of the special column with the same name
}
static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
- assert( 0 );
- return 0;
+ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
+ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
+ int rc = SQLITE_OK;
+
+ if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
+ i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
+ rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
+ }
+ if( iCol>=0 && iCol<pTab->pConfig->nCol ){
+ *pnToken = pCsr->aColumnSize[iCol];
+ }else{
+ *pnToken = 0;
+ }
+ return rc;
}
static int fts5ApiPoslist(
);
/*
+**
+** xUserData:
+** Return a copy of the context pointer the extension function was
+** registered with.
+**
** xColumnCount:
** Returns the number of columns in the FTS5 table.
**
+** xColumnSize:
+** Reports the size in tokens of a column value from the current row.
+**
** xPhraseCount:
** Returns the number of phrases in the current query expression.
**
int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
+int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
+int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol);
+
/*
** End of interface to code in fts5_storage.c.
if( zReq==0 ){
sqlite3Fts5BufferAppendPrintf(&rc, &s, "columncount ");
}
+ nCol = pApi->xColumnCount(pFts);
if( 0==zReq || 0==sqlite3_stricmp(zReq, "columncount") ){
- nCol = pApi->xColumnCount(pFts);
sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nCol);
}
+ if( zReq==0 ){
+ sqlite3Fts5BufferAppendPrintf(&rc, &s, "columnsize ");
+ }
+ if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnsize") ){
+ if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
+ for(i=0; rc==SQLITE_OK && i<nCol; i++){
+ int colsz = 0;
+ rc = pApi->xColumnSize(pFts, i, &colsz);
+ sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
+ }
+ if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
+ }
+
if( zReq==0 ){
sqlite3Fts5BufferAppendPrintf(&rc, &s, " phrasecount ");
}
struct Fts5Storage {
Fts5Config *pConfig;
Fts5Index *pIndex;
-
- sqlite3_stmt *aStmt[8];
+ sqlite3_stmt *aStmt[9];
};
#define FTS5_STMT_REPLACE_CONTENT 4
#define FTS5_STMT_DELETE_CONTENT 5
-#define FTS5_STMT_INSERT_DOCSIZE 6
+#define FTS5_STMT_REPLACE_DOCSIZE 6
#define FTS5_STMT_DELETE_DOCSIZE 7
+#define FTS5_STMT_LOOKUP_DOCSIZE 8
+
/*
** Prepare the two insert statements - Fts5Storage.pInsertContent and
** Fts5Storage.pInsertDocsize - if they have not already been prepared.
"INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */
"REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */
"DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */
- "INSERT INTO %Q.'%q_docsize' VALUES(?,?)", /* INSERT_DOCSIZE */
+ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */
"DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */
+
+ "SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */
};
Fts5Config *pConfig = p->pConfig;
char *zSql = 0;
struct Fts5InsertCtx {
Fts5Storage *pStorage;
int iCol;
+ int szCol; /* Size of column value in tokens */
};
/*
){
Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
Fts5Index *pIdx = pCtx->pStorage->pIndex;
+ pCtx->szCol = iPos+1;
sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken);
return SQLITE_OK;
}
return rc;
}
+/*
+** Insert a record into the %_docsize table. Specifically, do:
+**
+** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
+*/
+static int fts5StorageInsertDocsize(
+ Fts5Storage *p, /* Storage module to write to */
+ i64 iRowid, /* id value */
+ Fts5Buffer *pBuf /* sz value */
+){
+ sqlite3_stmt *pReplace = 0;
+ int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pReplace, 1, iRowid);
+ sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
+ sqlite3_step(pReplace);
+ rc = sqlite3_reset(pReplace);
+ }
+ return rc;
+}
+
/*
** Insert a new row into the FTS table.
*/
int eStmt; /* Type of statement used on %_content */
int i; /* Counter variable */
Fts5InsertCtx ctx; /* Tokenization callback context object */
+ Fts5Buffer buf; /* Buffer used to build up %_docsize blob */
+
+ memset(&buf, 0, sizeof(Fts5Buffer));
/* Insert the new row into the %_content table. */
if( eConflict==SQLITE_REPLACE ){
sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
ctx.pStorage = p;
for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
+ ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig,
(const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
sqlite3_value_bytes(apVal[ctx.iCol+2]),
(void*)&ctx,
fts5StorageInsertCallback
);
+ sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
+ }
+
+ /* Write the %_docsize record */
+ if( rc==SQLITE_OK ){
+ rc = fts5StorageInsertDocsize(p, *piRowid, &buf);
}
+ sqlite3_free(buf.p);
return rc;
}
}
}
+static int fts5StorageDecodeSizeArray(
+ int *aCol, int nCol, /* Array to populate */
+ const u8 *aBlob, int nBlob /* Record to read varints from */
+){
+ int i;
+ int iOff = 0;
+ for(i=0; i<nCol; i++){
+ if( iOff>=nBlob ) return 1;
+ iOff += getVarint32(&aBlob[iOff], aCol[i]);
+ }
+ return (iOff!=nBlob);
+}
+
+/*
+** Argument aCol points to an array of integers containing one entry for
+** each table column. This function reads the %_docsize record for the
+** specified rowid and populates aCol[] with the results.
+**
+** An SQLite error code is returned if an error occurs, or SQLITE_OK
+** otherwise.
+*/
+int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
+ int nCol = p->pConfig->nCol;
+ sqlite3_stmt *pLookup = 0;
+ int rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup);
+ if( rc==SQLITE_OK ){
+ int bCorrupt = 1;
+ sqlite3_bind_int64(pLookup, 1, iRowid);
+ if( SQLITE_ROW==sqlite3_step(pLookup) ){
+ const u8 *aBlob = sqlite3_column_blob(pLookup, 0);
+ int nBlob = sqlite3_column_bytes(pLookup, 0);
+ if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
+ bCorrupt = 0;
+ }
+ }
+ rc = sqlite3_reset(pLookup);
+ if( bCorrupt && rc==SQLITE_OK ){
+ rc = SQLITE_CORRUPT_VTAB;
+ }
+ }
+ return rc;
+}
+
+int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol){
+ return 0;
+}
-C Fix\sissues\swith\sposition\slists\sand\sNEAR\sconstraints.
-D 2014-07-18T19:59:00.547
+C Fixes\sfor\sthe\sxColumnSize()\sfts5\sextension\sAPI.
+D 2014-07-19T15:35:09.453
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
-F ext/fts5/fts5.c 20bcb1e10756c72b550947236960edf96929ca2f
-F ext/fts5/fts5.h cda3b9d73e6ffa6d0cd35b7da6b808bf3a1ada32
-F ext/fts5/fts5Int.h 6cf315d3999c14572012d676fa1baf4f4323587b
-F ext/fts5/fts5_aux.c 27b082732fd76277fd7e9277f52903723d97f99b
+F ext/fts5/fts5.c 86655d1e2ba35c719d3cc480cb9fdd7f3887b74e
+F ext/fts5/fts5.h 844898034fa3e0458d93a6c34dd6ba6bd3c7e03a
+F ext/fts5/fts5Int.h cca221a5cf7234f92faf3b4b5f2e4cf43bce83ee
+F ext/fts5/fts5_aux.c 978a90fe90a6d34d9bd260948b5678caf5489894
F ext/fts5/fts5_buffer.c 71cf2016b2881e7aea39f952995eafa510d96cbd
F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
F ext/fts5/fts5_expr.c 288b3e016253eab69ea8cefbff346a4697b44291
F ext/fts5/fts5_index.c 9ff3008e903aa9077b0a7a7aa76ab6080eb07a36
-F ext/fts5/fts5_storage.c 7848d8f8528d798bba159900ea310a6d4a279da8
+F ext/fts5/fts5_storage.c fcf66173e55927cee0675ecfb1038d0000e4fa10
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb
F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397
F test/fts5ac.test 9be418d037763f4cc5d86f4239db41fc86bb4f85
F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07
-F test/fts5ae.test 5d5ffba68e850d9ade99cdd3f5c6431c82dad81d
+F test/fts5ae.test 7da37ac01debf2e238552d0ef2f61669fd232936
F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4
F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d
F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54
-F test/permutations.test 0b5333e5dcdeffba0ecbe5ee8dc7577029ffab6c
+F test/permutations.test c3eb62a88337d9a5046c509dd90ba6d43debc76d
F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0
F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13
F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 5808f30fae0d844c52a785bf18872be371d4af68
-R c645036fa73431553c03d7990bbe09ec
+P 16352d3654d5672cd0251db51dbe19f779373feb
+R a0466df485f1c616be1f64f0989b7925
U dan
-Z 973aba4d2c5c8ae6a3e94f9739fdfe1b
+Z affe87335b53c4c8634348527de72153
-16352d3654d5672cd0251db51dbe19f779373feb
\ No newline at end of file
+43fcb844726cfeeb1c8a0dbfaa0d2ca22e6ac16c
\ No newline at end of file
VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
}
-breakpoint
do_execsql_test 4.1 {
SELECT rowid, fts5_test(t4, 'poslist') FROM t4 WHERE t4 MATCH 'a OR b AND c';
} {
1 {0.5 {} {}}
}
-#93 {0.5 1.6 {}}
+#-------------------------------------------------------------------------
+# Test that the xColumnSize() API works.
+#
+reset_db
+do_execsql_test 5.1 {
+ CREATE VIRTUAL TABLE t5 USING fts5(x, y);
+ INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
+ INSERT INTO t5 VALUES('', 'a');
+ INSERT INTO t5 VALUES('a', '');
+}
+do_execsql_test 5.2 {
+ SELECT rowid, fts5_test(t5, 'columnsize') FROM t5 WHERE t5 MATCH 'a'
+ ORDER BY rowid DESC;
+} {
+ 3 {1 0}
+ 2 {0 1}
+ 1 {4 6}
+}
-finish_test
+finish_test
test_suite "fts5" -prefix "" -description {
All FTS5 tests.
} -files {
- fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ea.test
+ fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ae.test fts5ea.test
}
test_suite "nofaultsim" -prefix "" -description {