typedef struct Fts5PoslistWriter Fts5PoslistWriter;
struct Fts5PoslistWriter {
- int iCol;
- int iOff;
+ i64 iPrev;
};
int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
int *piOff /* IN/OUT: Current token offset */
);
+int sqlite3Fts5PoslistNext64(
+ const u8 *a, int n, /* Buffer containing poslist */
+ int *pi, /* IN/OUT: Offset within a[] */
+ i64 *piOff /* IN/OUT: Current offset */
+);
+
/*
** End of interface to code in fts5_buffer.c.
**************************************************************************/
sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData);
}
+int sqlite3Fts5PoslistNext64(
+ const u8 *a, int n, /* Buffer containing poslist */
+ int *pi, /* IN/OUT: Offset within a[] */
+ i64 *piOff /* IN/OUT: Current offset */
+){
+ int i = *pi;
+ if( i>=n ){
+ /* EOF */
+ return 1;
+ }else{
+ i64 iOff = *piOff;
+ int iVal;
+ i += getVarint32(&a[i], iVal);
+ if( iVal==1 ){
+ i += getVarint32(&a[i], iVal);
+ iOff = ((i64)iVal) << 32;
+ i += getVarint32(&a[i], iVal);
+ }
+ *piOff = iOff + (iVal-2);
+ *pi = i;
+ return 0;
+ }
+}
+
/*
** Advance the iterator object passed as the only argument. Return true
** if the iterator reaches EOF, or false otherwise.
*/
int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){
- if( pIter->i>=pIter->n ){
+ if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos)
+ || (pIter->iCol>=0 && (pIter->iPos >> 32) > pIter->iCol)
+ ){
pIter->bEof = 1;
- }else{
- int iVal;
- pIter->i += getVarint32(&pIter->a[pIter->i], iVal);
- if( iVal==1 ){
- pIter->i += getVarint32(&pIter->a[pIter->i], iVal);
- if( pIter->iCol>=0 && iVal>pIter->iCol ){
- pIter->bEof = 1;
- }else{
- pIter->iPos = ((u64)iVal << 32);
- pIter->i += getVarint32(&pIter->a[pIter->i], iVal);
- }
- }
- pIter->iPos += (iVal-2);
}
return pIter->bEof;
}
Fts5PoslistWriter *pWriter,
i64 iPos
){
+ static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
int rc = SQLITE_OK;
- int iCol = (int)(iPos >> 32);
- int iOff = (iPos & 0x7FFFFFFF);
-
- if( iCol!=pWriter->iCol ){
+ if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
fts5BufferAppendVarint(&rc, pBuf, 1);
- fts5BufferAppendVarint(&rc, pBuf, iCol);
- pWriter->iCol = iCol;
- pWriter->iOff = 0;
+ fts5BufferAppendVarint(&rc, pBuf, (iPos >> 32));
+ pWriter->iPrev = (iPos & colmask);
}
- fts5BufferAppendVarint(&rc, pBuf, (iOff - pWriter->iOff) + 2);
-
+ fts5BufferAppendVarint(&rc, pBuf, (iPos - pWriter->iPrev) + 2);
+ pWriter->iPrev = iPos;
return rc;
}
Fts5ExprNode *pRight; /* Right hand child node */
Fts5ExprNearset *pNear; /* For FTS5_STRING - cluster of phrases */
int bEof; /* True at EOF */
- i64 iRowid;
+ i64 iRowid; /* Current rowid */
};
/*
** within a document for it to match.
*/
struct Fts5ExprPhrase {
+ Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */
Fts5Buffer poslist; /* Current position list */
int nTerm; /* Number of entries in aTerm[] */
Fts5ExprTerm aTerm[0]; /* Terms that make up this phrase */
Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */
int *pbMatch /* OUT: Set to true if really a match */
){
- Fts5PoslistWriter writer = {0, 0};
+ Fts5PoslistWriter writer = {0};
Fts5PoslistReader aStatic[4];
Fts5PoslistReader *aIter = aStatic;
int i;
return rc;
}
+typedef struct Fts5LookaheadReader Fts5LookaheadReader;
+struct Fts5LookaheadReader {
+ const u8 *a; /* Buffer containing position list */
+ int n; /* Size of buffer a[] in bytes */
+ int i; /* Current offset in position list */
+ i64 iPos; /* Current position */
+ i64 iLookahead; /* Next position */
+};
+
+#define FTS5_LOOKAHEAD_EOF (((i64)1) << 62)
+
+static int fts5LookaheadReaderNext(Fts5LookaheadReader *p){
+ p->iPos = p->iLookahead;
+ if( sqlite3Fts5PoslistNext64(p->a, p->n, &p->i, &p->iLookahead) ){
+ p->iLookahead = FTS5_LOOKAHEAD_EOF;
+ }
+ return (p->iPos==FTS5_LOOKAHEAD_EOF);
+}
+
+static int fts5LookaheadReaderInit(
+ const u8 *a, int n, /* Buffer to read position list from */
+ Fts5LookaheadReader *p /* Iterator object to initialize */
+){
+ memset(p, 0, sizeof(Fts5LookaheadReader));
+ p->a = a;
+ p->n = n;
+ fts5LookaheadReaderNext(p);
+ return fts5LookaheadReaderNext(p);
+}
+
+static int fts5LookaheadReaderEof(Fts5LookaheadReader *p){
+ return (p->iPos==FTS5_LOOKAHEAD_EOF);
+}
+
+typedef struct Fts5NearTrimmer Fts5NearTrimmer;
+struct Fts5NearTrimmer {
+ Fts5LookaheadReader reader; /* Input iterator */
+ Fts5PoslistWriter writer; /* Writer context */
+ Fts5Buffer *pOut; /* Output poslist */
+};
/*
** The near-set object passed as the first argument contains more than
** a set of intances that collectively matches the NEAR constraint.
*/
static int fts5ExprNearIsMatch(Fts5ExprNearset *pNear, int *pbMatch){
- Fts5PoslistReader aStatic[4];
- Fts5PoslistReader *aIter = aStatic;
+ Fts5NearTrimmer aStatic[4];
+ Fts5NearTrimmer *a = aStatic;
+
+ Fts5ExprPhrase **apPhrase = pNear->apPhrase;
+
int i;
int rc = SQLITE_OK;
int bMatch;
/* If the aStatic[] array is not large enough, allocate a large array
** using sqlite3_malloc(). This approach could be improved upon. */
if( pNear->nPhrase>(sizeof(aStatic) / sizeof(aStatic[0])) ){
- int nByte = sizeof(Fts5PoslistReader) * pNear->nPhrase;
- aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte);
- if( !aIter ) return SQLITE_NOMEM;
+ int nByte = sizeof(Fts5LookaheadReader) * pNear->nPhrase;
+ a = (Fts5NearTrimmer*)sqlite3_malloc(nByte);
+ if( !a ) return SQLITE_NOMEM;
+ memset(a, 0, nByte);
+ }else{
+ memset(aStatic, 0, sizeof(aStatic));
}
- /* Initialize a term iterator for each phrase */
+ /* Initialize a lookahead iterator for each phrase. After passing the
+ ** buffer and buffer size to the lookaside-reader init function, zero
+ ** the phrase poslist buffer. The new poslist for the phrase (containing
+ ** the same entries as the original with some entries removed on account
+ ** of the NEAR constraint) is written over the original even as it is
+ ** being read. This is safe as the entries for the new poslist are a
+ ** subset of the old, so it is not possible for data yet to be read to
+ ** be overwritten. */
for(i=0; i<pNear->nPhrase; i++){
- Fts5Buffer *pPoslist = &pNear->apPhrase[i]->poslist;
- sqlite3Fts5PoslistReaderInit(-1, pPoslist->p, pPoslist->n, &aIter[i]);
+ Fts5Buffer *pPoslist = &apPhrase[i]->poslist;
+ fts5LookaheadReaderInit(pPoslist->p, pPoslist->n, &a[i].reader);
+ pPoslist->n = 0;
+ a[i].pOut = pPoslist;
}
- iMax = aIter[0].iPos;
- do {
- bMatch = 1;
- for(i=0; i<pNear->nPhrase; i++){
- Fts5PoslistReader *pPos = &aIter[i];
- i64 iMin = iMax - pNear->apPhrase[i]->nTerm - pNear->nNear;
- if( pPos->iPos<iMin || pPos->iPos>iMax ){
- bMatch = 0;
- while( pPos->iPos<iMin ){
- if( sqlite3Fts5PoslistReaderNext(pPos) ) goto ismatch_out;
+ while( 1 ){
+ int iAdv;
+ i64 iMin;
+ i64 iMax;
+
+ /* This block advances the phrase iterators until they point to a set of
+ ** entries that together comprise a match. */
+ iMax = a[0].reader.iPos;
+ do {
+ bMatch = 1;
+ for(i=0; i<pNear->nPhrase; i++){
+ Fts5LookaheadReader *pPos = &a[i].reader;
+ iMin = iMax - pNear->apPhrase[i]->nTerm - pNear->nNear;
+ if( pPos->iPos<iMin || pPos->iPos>iMax ){
+ bMatch = 0;
+ while( pPos->iPos<iMin ){
+ if( fts5LookaheadReaderNext(pPos) ) goto ismatch_out;
+ }
+ if( pPos->iPos>iMax ) iMax = pPos->iPos;
}
- if( pPos->iPos>iMax ) iMax = pPos->iPos;
+ }
+ }while( bMatch==0 );
+
+ /* Add an entry to each output position list */
+ for(i=0; i<pNear->nPhrase; i++){
+ i64 iPos = a[i].reader.iPos;
+ Fts5PoslistWriter *pWriter = &a[i].writer;
+ if( a[i].pOut->n==0 || iPos!=pWriter->iPrev ){
+ sqlite3Fts5PoslistWriterAppend(a[i].pOut, pWriter, iPos);
}
}
- }while( bMatch==0 );
+
+ iAdv = 0;
+ iMin = a[0].reader.iLookahead;
+ for(i=0; i<pNear->nPhrase; i++){
+ if( a[i].reader.iLookahead < iMin ){
+ iMin = a[i].reader.iLookahead;
+ iAdv = i;
+ }
+ }
+ if( fts5LookaheadReaderNext(&a[iAdv].reader) ) goto ismatch_out;
+ }
ismatch_out:
- *pbMatch = bMatch;
- if( aIter!=aStatic ) sqlite3_free(aIter);
+ *pbMatch = (a[0].pOut->n>0);
+ if( a!=aStatic ) sqlite3_free(a);
return rc;
}
while( 1 ){
int i;
- /* Advance the iterators until they are a match */
+ /* Advance the iterators until they all point to the same rowid */
rc = fts5ExprNearNextRowidMatch(pExpr, pNode);
if( pNode->bEof || rc!=SQLITE_OK ) break;
if( rc!=SQLITE_OK || bMatch ) break;
}
+ /* If control flows to here, then the current rowid is not a match.
+ ** Advance all term iterators in all phrases to the next rowid. */
rc = fts5ExprNearAdvanceAll(pExpr, pNear, &pNode->bEof);
if( pNode->bEof || rc!=SQLITE_OK ) break;
}
return rc;
}
+static void fts5ExprSetEof(Fts5ExprNode *pNode){
+ if( pNode ){
+ pNode->bEof = 1;
+ fts5ExprSetEof(pNode->pLeft);
+ fts5ExprSetEof(pNode->pRight);
+ }
+}
+
/*
**
*/
rc = fts5ExprNodeNext(pExpr, pAdv);
if( rc!=SQLITE_OK ) break;
}
- pNode->bEof = p1->bEof || p2->bEof;
+ if( p1->bEof || p2->bEof ){
+ fts5ExprSetEof(pNode);
+ }
pNode->iRowid = p1->iRowid;
break;
}
pRet->pLeft = pLeft;
pRet->pRight = pRight;
pRet->pNear = pNear;
+ if( eType==FTS5_STRING ){
+ int iPhrase;
+ for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
+ pNear->apPhrase[iPhrase]->pNode = pRet;
+ }
+ }
}
}
int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){
if( iPhrase>=0 && iPhrase<pExpr->nPhrase ){
Fts5ExprPhrase *pPhrase = pExpr->apPhrase[iPhrase];
- if( sqlite3Fts5IterRowid(pPhrase->aTerm[0].pIter)==pExpr->pRoot->iRowid ){
+ Fts5ExprNode *pNode = pPhrase->pNode;
+ if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){
*pa = pPhrase->poslist.p;
return pPhrase->poslist.n;
}
-C Fix\sa\sproblem\swith\sposition\slist\sprocessing\sfor\sOR\squeries.
-D 2014-07-17T15:14:07.541
+C Fix\sissues\swith\sposition\slists\sand\sNEAR\sconstraints.
+D 2014-07-18T19:59:00.547
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
F ext/fts5/fts5.c 20bcb1e10756c72b550947236960edf96929ca2f
F ext/fts5/fts5.h cda3b9d73e6ffa6d0cd35b7da6b808bf3a1ada32
-F ext/fts5/fts5Int.h 2d4c1e1ebdf18278776fcd8a64233ff3c04ea51f
+F ext/fts5/fts5Int.h 6cf315d3999c14572012d676fa1baf4f4323587b
F ext/fts5/fts5_aux.c 27b082732fd76277fd7e9277f52903723d97f99b
-F ext/fts5/fts5_buffer.c b7aa6cdf4a63642fcc12359cedc4be748ca400cc
+F ext/fts5/fts5_buffer.c 71cf2016b2881e7aea39f952995eafa510d96cbd
F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
-F ext/fts5/fts5_expr.c 52a1b47cfd30feb09c522392b1ba246eda7023f4
+F ext/fts5/fts5_expr.c 288b3e016253eab69ea8cefbff346a4697b44291
F ext/fts5/fts5_index.c 9ff3008e903aa9077b0a7a7aa76ab6080eb07a36
F ext/fts5/fts5_storage.c 7848d8f8528d798bba159900ea310a6d4a279da8
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36
F test/fts5aa.test c8d3b9694f6b2864161c7437408464a535d19343
F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397
-F test/fts5ac.test 84599f8253abc7e10b929b8ee0b47c5edd4eafbd
+F test/fts5ac.test 9be418d037763f4cc5d86f4239db41fc86bb4f85
F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07
+F test/fts5ae.test 5d5ffba68e850d9ade99cdd3f5c6431c82dad81d
F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4
F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P c1f9a4b76c0bbc1ef9f6cdb5d62aa5d536fdf38e
-R c3e4d3fb829636894b73dbf001062a3f
+P 5808f30fae0d844c52a785bf18872be371d4af68
+R c645036fa73431553c03d7990bbe09ec
U dan
-Z 4c19ad3988f765a5de2b9174d276eba9
+Z 973aba4d2c5c8ae6a3e94f9739fdfe1b
-5808f30fae0d844c52a785bf18872be371d4af68
\ No newline at end of file
+16352d3654d5672cd0251db51dbe19f779373feb
\ No newline at end of file
2.3 "d+c OR u"
2.4 "d+c OR u+d"
+ 3.1 { a AND b AND c }
} {
set res [matchdata 1 $expr]
- do_execsql_test 2.1.$tn.[llength $res] {
+ do_execsql_test 2.$tn.[llength $res] {
SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
} $res
}
+#-------------------------------------------------------------------------
+# Queries on a specific column.
+#
+foreach {tn expr} {
+ 1 "x:a"
+ 2 "y:a"
+ 3 "x:b"
+ 4 "y:b"
+} {
+ set res [matchdata 1 $expr]
+ do_execsql_test 3.$tn.[llength $res] {
+ SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
+ } $res
+}
+
+#-------------------------------------------------------------------------
+# Some NEAR queries.
+#
+foreach {tn expr} {
+ 1 "NEAR(a b)"
+ 2 "NEAR(r c)"
+ 2 { NEAR(r c, 5) }
+ 3 { NEAR(r c, 3) }
+ 4 { NEAR(r c, 2) }
+ 5 { NEAR(r c, 0) }
+ 6 { NEAR(a b c) }
+ 7 { NEAR(a b c, 8) }
+ 8 { x : NEAR(r c) }
+ 9 { y : NEAR(r c) }
+} {
+ set res [matchdata 1 $expr]
+ do_execsql_test 4.1.$tn.[llength $res] {
+ SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
+ } $res
+}
-do_test 2.1 { poslist {{a b c}} -- a } {0.0}
-do_test 2.2 { poslist {{a b c}} -- c } {0.2}
+do_test 4.1 { poslist {{a b c}} -- a } {0.0}
+do_test 4.2 { poslist {{a b c}} -- c } {0.2}
foreach {tn expr tclexpr} {
1 {a b} {[N $x -- {a}] && [N $x -- {b}]}
} {
- do_execsql_test 3.$tn {SELECT fts5_expr_tcl($expr, 'N $x')} [list $tclexpr]
+ do_execsql_test 5.$tn {SELECT fts5_expr_tcl($expr, 'N $x')} [list $tclexpr]
}
#-------------------------------------------------------------------------
19 { c NOT b OR a AND d }
} {
set res [matchdata 0 $expr $bAsc]
- do_execsql_test 4.$bAsc.$tn.[llength $res] $sql $res
+ do_execsql_test 6.$bAsc.$tn.[llength $res] $sql $res
}
}
--- /dev/null
+# 2014 June 17
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library. The
+# focus of this script is testing the FTS5 module.
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts5ae
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+ finish_test
+ return
+}
+
+do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+ INSERT INTO t1(t1) VALUES('pgsz=32');
+}
+
+do_execsql_test 1.1 {
+ INSERT INTO t1 VALUES('hello', 'world');
+ SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
+} {1}
+
+do_execsql_test 1.2 {
+ INSERT INTO t1 VALUES('world', 'hello');
+ SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
+} {1 2}
+
+do_execsql_test 1.3 {
+ INSERT INTO t1 VALUES('world', 'world');
+ SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
+} {1 2}
+
+do_execsql_test 1.4.1 {
+ INSERT INTO t1 VALUES('hello', 'hello');
+}
+
+do_execsql_test 1.4.2 {
+ SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
+} {1 2 4}
+
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+ CREATE VIRTUAL TABLE t2 USING fts5(x, y);
+ INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e');
+ INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p');
+}
+
+do_execsql_test 2.1 {
+ SELECT rowid, fts5_test(t2, 'poslist') FROM t2
+ WHERE t2 MATCH 'm' ORDER BY rowid;
+} {
+ 1 {{0.5 1.0 1.2}}
+ 2 {{0.7 1.5}}
+}
+
+do_execsql_test 2.2 {
+ SELECT rowid, fts5_test(t2, 'poslist') FROM t2
+ WHERE t2 MATCH 'u OR q' ORDER BY rowid;
+} {
+ 1 {0.0 {}}
+ 2 {{} {0.2 0.10}}
+}
+
+do_execsql_test 2.3 {
+ SELECT rowid, fts5_test(t2, 'poslist') FROM t2
+ WHERE t2 MATCH 'y:o' ORDER BY rowid;
+} {
+ 1 {{1.3 1.7}}
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 3.0 {
+ CREATE VIRTUAL TABLE t3 USING fts5(x, y);
+ INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t');
+ INSERT INTO t3 VALUES( 'r c', '');
+}
+
+do_execsql_test 3.1 {
+ SELECT rowid, fts5_test(t3, 'poslist') FROM t3 WHERE t3 MATCH 'NEAR(a b)';
+} {
+ 1 {{0.6 0.10 0.12} {0.9 0.15}}
+}
+
+do_execsql_test 3.2 {
+ SELECT rowid, fts5_test(t3, 'poslist') FROM t3 WHERE t3 MATCH 'NEAR(r c)';
+} {
+ 2 {0.0 0.1}
+}
+
+do_execsql_test 3.3 {
+ INSERT INTO t3
+ VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
+ SELECT rowid, fts5_test(t3, 'poslist')
+ FROM t3 WHERE t3 MATCH 'a OR b AND c';
+} {
+ 3 {0.5 {} {}}
+ 1 {{0.6 0.10 0.12} {0.9 0.15} 1.2}
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 4.0 {
+ CREATE VIRTUAL TABLE t4 USING fts5(x, y);
+ INSERT INTO t4
+ VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
+}
+
+breakpoint
+do_execsql_test 4.1 {
+ SELECT rowid, fts5_test(t4, 'poslist') FROM t4 WHERE t4 MATCH 'a OR b AND c';
+} {
+ 1 {0.5 {} {}}
+}
+
+#93 {0.5 1.6 {}}
+
+
+finish_test
+
+
projects / thirdparty / sqlite.git / commitdiff
