From: dan Date: Sat, 2 Aug 2014 20:49:36 +0000 (+0000) Subject: Start changing things to use doclist indexes as required. code is not activated yet. X-Git-Tag: version-3.8.11~114^2~146 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=9af0705e844274e721abb82de42fb2574e613a77;p=thirdparty%2Fsqlite.git Start changing things to use doclist indexes as required. code is not activated yet. FossilOrigin-Name: b8864da95db2c0e611116304d607e35a86c9247d --- diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c index ec9529c444..d2a5f0ce6e 100644 --- a/ext/fts5/fts5.c +++ b/ext/fts5/fts5.c @@ -84,6 +84,12 @@ struct Fts5Sorter { /* ** Virtual-table cursor object. +** +** zSpecial: +** If this is a 'special' query (refer to function fts5SpecialMatch()), +** then this variable points to a nul-terminated buffer containing the +** result to return through the table-name column. It is nul-terminated +** and should eventually be freed using sqlite3_free(). */ struct Fts5Cursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ @@ -94,6 +100,7 @@ struct Fts5Cursor { int csrflags; /* Mask of cursor flags (see below) */ Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */ Fts5Auxiliary *pRank; /* Rank callback (or NULL) */ + char *zSpecial; /* Result of special query */ /* Variables used by auxiliary functions */ i64 iCsrId; /* Cursor id */ @@ -253,6 +260,7 @@ static int fts5CreateMethod( #define FTS5_PLAN_SORTED_MATCH 3 /* ( MATCH ? ORDER BY rank) */ #define FTS5_PLAN_ROWID 4 /* (rowid = ?) */ #define FTS5_PLAN_SOURCE 5 /* A source cursor for SORTED_MATCH */ +#define FTS5_PLAN_SPECIAL 6 /* An internal query */ #define FTS5_PLAN(idxNum) ((idxNum) & 0x7) @@ -395,6 +403,7 @@ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); *pp = pCsr->pNext; + sqlite3_free(pCsr->zSpecial); sqlite3_free(pCsr); return SQLITE_OK; } @@ -457,6 +466,11 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE ); break; + case FTS5_PLAN_SPECIAL: { + CsrFlagSet(pCsr, FTS5CSR_EOF); + break; + } + case FTS5_PLAN_SORTED_MATCH: { rc = fts5SorterNext(pCsr); break; @@ -536,6 +550,42 @@ static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bAsc){ return rc; } +/* +** Process a "special" query. A special query is identified as one with a +** MATCH expression that begins with a '*' character. The remainder of +** the text passed to the MATCH operator are used as the special query +** parameters. +*/ +static int fts5SpecialMatch( + Fts5Table *pTab, + Fts5Cursor *pCsr, + const char *zQuery +){ + int rc = SQLITE_OK; /* Return code */ + const char *z = zQuery; /* Special query text */ + int n; /* Number of bytes in text at z */ + + while( z[0]==' ' ) z++; + for(n=0; z[n] && z[n]!=' '; n++); + + assert( pTab->base.zErrMsg==0 ); + assert( pCsr->zSpecial==0 ); + + if( 0==sqlite3_strnicmp("reads", z, n) ){ + pCsr->zSpecial = sqlite3_mprintf("%d", sqlite3Fts5IndexReads(pTab->pIndex)); + pCsr->idxNum = FTS5_PLAN_SPECIAL; + if( pCsr->zSpecial==0 ) rc = SQLITE_NOMEM; + } + else{ + /* An unrecognized directive. Return an error message. */ + pTab->base.zErrMsg = sqlite3_mprintf("unknown special query: %.*s", n, z); + rc = SQLITE_ERROR; + } + + return rc; +} + + /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional @@ -559,19 +609,30 @@ static int fts5FilterMethod( assert( pCsr->pRank==0 ); if( pTab->pSortCsr ){ + /* If pSortCsr is non-NULL, then this call is being made as part of + ** processing for a "... MATCH ORDER BY rank" query (ePlan is + ** set to FTS5_PLAN_SORTED_MATCH). pSortCsr is the cursor that will + ** return results to the user for this query. The current cursor + ** (pCursor) is used to execute the query issued by function + ** fts5CursorFirstSorted() above. */ + assert( FTS5_PLAN(idxNum)==FTS5_PLAN_SCAN ); pCsr->idxNum = FTS5_PLAN_SOURCE; pCsr->pRank = pTab->pSortCsr->pRank; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bAsc); }else{ int ePlan = FTS5_PLAN(idxNum); - int eStmt = fts5StmtType(idxNum); pCsr->idxNum = idxNum; - rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt); - if( rc==SQLITE_OK ){ - if( ePlan==FTS5_PLAN_MATCH || ePlan==FTS5_PLAN_SORTED_MATCH ){ + if( ePlan==FTS5_PLAN_MATCH || ePlan==FTS5_PLAN_SORTED_MATCH ){ + const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); + + if( zExpr[0]=='*' ){ + /* The user has issued a query of the form "MATCH '*...'". This + ** indicates that the MATCH expression is not a full text query, + ** but a request for an internal parameter. */ + rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]); + }else{ char **pzErr = &pTab->base.zErrMsg; - const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); pCsr->pRank = pTab->pGlobal->pAux; rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr); if( rc==SQLITE_OK ){ @@ -581,7 +642,13 @@ static int fts5FilterMethod( rc = fts5CursorFirstSorted(pTab, pCsr, bAsc); } } - }else{ + } + }else{ + /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup + ** by rowid (ePlan==FTS5_PLAN_ROWID). */ + int eStmt = fts5StmtType(idxNum); + rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt); + if( rc==SQLITE_OK ){ if( ePlan==FTS5_PLAN_ROWID ){ sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); } @@ -629,6 +696,10 @@ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); switch( ePlan ){ + case FTS5_PLAN_SPECIAL: + *pRowid = 0; + break; + case FTS5_PLAN_SOURCE: case FTS5_PLAN_MATCH: case FTS5_PLAN_SORTED_MATCH: @@ -649,7 +720,16 @@ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ */ static int fts5SeekCursor(Fts5Cursor *pCsr){ int rc = SQLITE_OK; - if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){ + + /* If the cursor does not yet have a statement handle, obtain one now. */ + if( pCsr->pStmt==0 ){ + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + int eStmt = fts5StmtType(pCsr->idxNum); + rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt); + assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ); + } + + if( rc==SQLITE_OK && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){ assert( pCsr->pExpr ); sqlite3_reset(pCsr->pStmt); sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr)); @@ -1089,6 +1169,12 @@ static int fts5ColumnMethod( assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); + if( pCsr->idxNum==FTS5_PLAN_SPECIAL ){ + if( iCol==pConfig->nCol ){ + sqlite3_result_text(pCtx, pCsr->zSpecial, -1, SQLITE_TRANSIENT); + } + }else + if( iCol==pConfig->nCol ){ if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){ fts5PoslistBlob(pCtx, pCsr); diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index bfbc8d7aca..07903abf83 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -188,7 +188,8 @@ Fts5IndexIter *sqlite3Fts5IndexQuery( ** Docid list iteration. */ int sqlite3Fts5IterEof(Fts5IndexIter*); -void sqlite3Fts5IterNext(Fts5IndexIter*, i64 iMatch); +void sqlite3Fts5IterNext(Fts5IndexIter*); +void sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); i64 sqlite3Fts5IterRowid(Fts5IndexIter*); /* @@ -273,6 +274,8 @@ void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz); int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf); int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); +int sqlite3Fts5IndexReads(Fts5Index *p); + /* ** End of interface to code in fts5_index.c. **************************************************************************/ diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 68d8a75b9a..82645a2619 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -583,7 +583,7 @@ static int fts5ExprNearAdvanceAll( Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; for(j=0; jnTerm; j++){ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; - sqlite3Fts5IterNext(pIter, 0); + sqlite3Fts5IterNext(pIter); if( sqlite3Fts5IterEof(pIter) ){ *pbEof = 1; return rc; @@ -612,19 +612,18 @@ static int fts5ExprAdvanceto( ){ i64 iLast = *piLast; i64 iRowid; - while( 1 ){ - iRowid = sqlite3Fts5IterRowid(pIter); - if( (bAsc==0 && iRowid<=iLast) || (bAsc==1 && iRowid>=iLast) ) break; - sqlite3Fts5IterNext(pIter, 0); + + iRowid = sqlite3Fts5IterRowid(pIter); + if( (bAsc==0 && iRowid>iLast) || (bAsc && iRowid=iLast) ); } - if( iRowid!=iLast ){ - assert( (bAsc==0 && iRowidiLast) ); - *piLast = iRowid; - } + *piLast = iRowid; return 0; } @@ -769,7 +768,7 @@ static int fts5ExprNearInitAll( return SQLITE_OK; } -/* fts3ExprNodeNext() calls fts5ExprNodeNextMatch(). And vice-versa. */ +/* fts5ExprNodeNext() calls fts5ExprNodeNextMatch(). And vice-versa. */ static int fts5ExprNodeNextMatch(Fts5Expr*, Fts5ExprNode*); /* diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 8c4734e265..271caa8a0d 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -306,6 +306,7 @@ struct Fts5Index { sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ + int nRead; /* Total number of blocks read */ }; struct Fts5DoclistIter { @@ -702,6 +703,7 @@ static Fts5Data *fts5DataReadOrBuffer( } } p->rc = rc; + p->nRead++; } return pRet; @@ -1666,9 +1668,28 @@ static int fts5MultiIterEof(Fts5Index *p, Fts5MultiSegIter *pIter){ ** results are undefined. */ static i64 fts5MultiIterRowid(Fts5MultiSegIter *pIter){ + assert( pIter->aSeg[ pIter->aFirst[1] ].pLeaf ); return pIter->aSeg[ pIter->aFirst[1] ].iRowid; } +/* +** Move the iterator to the next entry at or following iMatch. +*/ +static void fts5MultiIterNextFrom( + Fts5Index *p, + Fts5MultiSegIter *pIter, + i64 iMatch +){ + while( 1 ){ + i64 iRowid; + fts5MultiIterNext(p, pIter); + if( fts5MultiIterEof(p, pIter) ) break; + iRowid = fts5MultiIterRowid(pIter); + if( pIter->bRev==0 && iRowid<=iMatch ) break; + if( pIter->bRev!=0 && iRowid>=iMatch ) break; + } +} + /* ** Return a pointer to a buffer containing the term associated with the ** entry that the iterator currently points to. @@ -3759,7 +3780,7 @@ int sqlite3Fts5IterEof(Fts5IndexIter *pIter){ /* ** Move to the next matching rowid. */ -void sqlite3Fts5IterNext(Fts5IndexIter *pIter, i64 iMatch){ +void sqlite3Fts5IterNext(Fts5IndexIter *pIter){ if( pIter->pDoclist ){ fts5DoclistIterNext(pIter->pDoclist); }else{ @@ -3768,6 +3789,20 @@ void sqlite3Fts5IterNext(Fts5IndexIter *pIter, i64 iMatch){ } } +/* +** Move to the next matching rowid that occurs at or after iMatch. The +** definition of "at or after" depends on whether this iterator iterates +** in ascending or descending rowid order. +*/ +void sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){ + if( pIter->pDoclist ){ + assert( 0 ); + /* fts5DoclistIterNextFrom(pIter->pDoclist, iMatch); */ + }else{ + fts5MultiIterNextFrom(pIter->pIndex, pIter->pMulti, iMatch); + } +} + /* ** Return the current rowid. */ @@ -3840,3 +3875,11 @@ int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){ return p->rc; } +/* +** Return the total number of blocks this module has read from the %_data +** table since it was created. +*/ +int sqlite3Fts5IndexReads(Fts5Index *p){ + return p->nRead; +} + diff --git a/manifest b/manifest index 089486568e..87e8bca504 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sspecial\scase\sto\sthe\sintegrity-check\scode\sto\scheck\sthat\sthe\sfinal\sinteger\sin\sa\sdoclist\sindex\sis\sas\sexpected. -D 2014-08-01T20:13:49.462 +C Start\schanging\sthings\sto\suse\sdoclist\sindexes\sas\srequired.\scode\sis\snot\sactivated\syet. +D 2014-08-02T20:49:36.405 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -103,14 +103,14 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368 -F ext/fts5/fts5.c aa269bbecf78cdb7aaa9c6dba26f6ee906ceedaf +F ext/fts5/fts5.c 23f875e24ffa722107690d14b449141a25a2d697 F ext/fts5/fts5.h 8ace10d5b249a3baa983c79e7a1306d2a79cfd6a -F ext/fts5/fts5Int.h 9a195c1706876c538902f007149b39e982e9da53 +F ext/fts5/fts5Int.h aef50f3078e60707aeb2e4b2787d8c5eecdd02dc F ext/fts5/fts5_aux.c 366057c7186bc3615deb5ecc0ff61de50b6d2dbc F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00 F ext/fts5/fts5_config.c f4ebf143e141b8c77355e3b15aba81b7be51d710 -F ext/fts5/fts5_expr.c e764d75c58a3accda795f1da1b45960ac87dc77a -F ext/fts5/fts5_index.c 13f9dd9788f90c419ea33db0fcb2214c2f1290ef +F ext/fts5/fts5_expr.c 9402474456732ddb5019f83a77907852f108a96a +F ext/fts5/fts5_index.c 20c905c323d866251e15d7ed2486c309914ceeb9 F ext/fts5/fts5_storage.c 2866e7e1de9dc851756c3a9c76b6e1d75e0facb7 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 @@ -595,14 +595,14 @@ F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36 -F test/fts5aa.test f90836c002804a82386d66c79b380128c5f3005e +F test/fts5aa.test ec150ac2778f871550bcdbea34598fba08717a4e F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397 F test/fts5ac.test 9be418d037763f4cc5d86f4239db41fc86bb4f85 F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07 F test/fts5ae.test cb37b3135a00d3afd5492ec534ecf654be5ff69e F test/fts5af.test 9ebe23aa3875896076952c7bc6e8308813a63c74 F test/fts5ag.test 0747bf3bade16d5165810cf891f875933b28b420 -F test/fts5ah.test dfb54897c470e2dcf88912fc4f5b1ca4ac8307f7 +F test/fts5ah.test 2b01e7d2b3a31b668cba2afad5cb1c651895a255 F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef @@ -1199,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 37a7d3035eb4bbad7e32fe550321ac9fae611a57 -R 21856f96ed42128f5622f6977d9547a6 +P c98934155cb48adfda57bd0fd1b950226d45f67a +R 0addd1f8d0c92beb67e8a764402a7ad4 U dan -Z 83c769d88a67f25fcdc4af671199630b +Z f1895ff018d8274d19451cd024daaa99 diff --git a/manifest.uuid b/manifest.uuid index 312136f109..8a1f54c530 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c98934155cb48adfda57bd0fd1b950226d45f67a \ No newline at end of file +b8864da95db2c0e611116304d607e35a86c9247d \ No newline at end of file diff --git a/test/fts5aa.test b/test/fts5aa.test index c59183512a..87e1494d55 100644 --- a/test/fts5aa.test +++ b/test/fts5aa.test @@ -285,5 +285,20 @@ do_catchsql_test 11.2 { CREATE VIRTUAL TABLE rank USING fts5(a, b, c); } {1 {reserved fts5 table name: rank}} +#------------------------------------------------------------------------- +# +do_execsql_test 12.1 { + CREATE VIRTUAL TABLE t2 USING fts5(x,y); +} {} + +do_catchsql_test 12.2 { + SELECT t2 FROM t2 WHERE t2 MATCH '*stuff' +} {1 {unknown special query: stuff}} + +do_test 12.3 { + set res [db one { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }] + string is integer $res +} {1} + finish_test diff --git a/test/fts5ah.test b/test/fts5ah.test index f5d1eee1b9..e108ec2ddd 100644 --- a/test/fts5ah.test +++ b/test/fts5ah.test @@ -49,6 +49,21 @@ do_execsql_test 1.3 { INSERT INTO t1(t1) VALUES('integrity-check'); } +proc reads {} { + db one {SELECT t1 FROM t1 WHERE t1 MATCH '*reads'} +} + +do_test 1.4 { + set nRead [reads] + db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x' } + set a [expr [reads] - $nRead] +} {} + +do_test 1.5 { + set nRead [reads] + db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w' } + set a [expr [reads] - $nRead] +} {} finish_test