From: dan Date: Thu, 16 Jun 2011 16:06:05 +0000 (+0000) Subject: Fix a problem with NEAR queries executed inside a transaction that writes the FTS... X-Git-Tag: version-3.7.7~54 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=d4d21fea8e6f9e8ad262a42886a581257f5e2402;p=thirdparty%2Fsqlite.git Fix a problem with NEAR queries executed inside a transaction that writes the FTS table. FossilOrigin-Name: 051c756c367837908f6691c0a36108e088c94f99 --- diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 024b1150a2..f2c907c8c0 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3493,7 +3493,7 @@ static int fts3EvalPhraseNext( while( pIterpNextDocid = pIter; - assert( *pIter || pIter>=&pDL->aAll[pDL->nAll] ); + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); *pbEof = 0; } } @@ -4131,13 +4131,14 @@ static void fts3EvalRestart( if( pPhrase ){ fts3EvalZeroPoslist(pPhrase); if( pPhrase->bIncr ){ - sqlite3Fts3EvalPhraseCleanup(pPhrase); - memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); - *pRc = sqlite3Fts3EvalStart(pCsr, pExpr, 0); - }else{ - pPhrase->doclist.pNextDocid = 0; - pPhrase->doclist.iDocid = 0; + assert( pPhrase->nToken==1 ); + assert( pPhrase->aToken[0].pSegcsr ); + sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr); + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); } + + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; } pExpr->iDocid = 0; diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 06f2c09bb8..5d3d22a57a 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -439,6 +439,7 @@ struct Fts3MultiSegReader { int nBuffer; /* Allocated size of aBuffer[] in bytes */ int iColFilter; /* If >=0, filter for this column */ + int bRestart; /* Used by fts3.c only. */ int nCost; /* Cost of running iterator */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index ea59e6eb62..da84d6ca61 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1247,6 +1247,7 @@ static int fts3SegReaderNextDocid( pReader->pOffsetList = p; } }else{ + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; /* Pointer p currently points at the first byte of an offset list. The ** following block advances it to point one byte past the end of @@ -1275,13 +1276,15 @@ static int fts3SegReaderNextDocid( *ppOffsetList = pReader->pOffsetList; *pnOffsetList = (int)(p - pReader->pOffsetList - 1); } + + while( p=&pReader->aDoclist[pReader->nDoclist] ){ + if( p>=pEnd ){ pReader->pOffsetList = 0; }else{ rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); @@ -2265,51 +2268,27 @@ static void fts3ColumnFilter( *pnList = nList; } -int sqlite3Fts3MsrIncrStart( - Fts3Table *p, /* Virtual table handle */ - Fts3MultiSegReader *pCsr, /* Cursor object */ - int iCol, /* Column to match on. */ - const char *zTerm, /* Term to iterate through a doclist for */ - int nTerm /* Number of bytes in zTerm */ +/* +** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any +** existing data). Grow the buffer if required. +** +** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered +** trying to resize the buffer, return SQLITE_NOMEM. +*/ +static int fts3MsrBufferData( + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + char *pList, + int nList ){ - int i; - int nSegment = pCsr->nSegment; - int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( - p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp - ); - - assert( pCsr->pFilter==0 ); - assert( zTerm && nTerm>0 ); - - /* Advance each segment iterator until it points to the term zTerm/nTerm. */ - for(i=0; iapSegment[i]; - do { - int rc = fts3SegReaderNext(p, pSeg, 1); - if( rc!=SQLITE_OK ) return rc; - }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); - } - fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp); - - /* Determine how many of the segments actually point to zTerm/nTerm. */ - for(i=0; iapSegment[i]; - if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ - break; - } + if( nList>pMsr->nBuffer ){ + char *pNew; + pMsr->nBuffer = nList*2; + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; } - pCsr->nAdvance = i; - - /* Advance each of the segments to point to the first docid. */ - for(i=0; inAdvance; i++){ - int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); - if( rc!=SQLITE_OK ) return rc; - } - fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); - - assert( iCol<0 || iColnColumn ); - pCsr->iColFilter = iCol; + memcpy(pMsr->aBuffer, pList, nList); return SQLITE_OK; } @@ -2363,27 +2342,32 @@ int sqlite3Fts3MsrIncrNext( } if( nList>0 ){ + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + *paPoslist = pMsr->aBuffer; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + }else{ + *paPoslist = pList; + } *piDocid = iDocid; - *paPoslist = pList; *pnPoslist = nList; break; } } - } return SQLITE_OK; } -int sqlite3Fts3SegReaderStart( +static int fts3SegReaderStart( Fts3Table *p, /* Virtual table handle */ Fts3MultiSegReader *pCsr, /* Cursor object */ - Fts3SegFilter *pFilter /* Restrictions on range of iteration */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ ){ int i; - - /* Initialize the cursor object */ - pCsr->pFilter = pFilter; + int nSeg = pCsr->nSegment; /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of @@ -2391,21 +2375,102 @@ int sqlite3Fts3SegReaderStart( ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ - for(i=0; inSegment; i++){ - int nTerm = pFilter->nTerm; - const char *zTerm = pFilter->zTerm; + for(i=0; pCsr->bRestart==0 && inSegment; i++){ Fts3SegReader *pSeg = pCsr->apSegment[i]; do { int rc = fts3SegReaderNext(p, pSeg, 0); if( rc!=SQLITE_OK ) return rc; }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } - fts3SegReaderSort( - pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp); + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); + + return SQLITE_OK; +} + +int sqlite3Fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ +){ + pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); +} + +int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; return SQLITE_OK; } +/* +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: +** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() +** +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. +*/ +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ + + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); + + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } + + return SQLITE_OK; +} + + int sqlite3Fts3SegReaderStep( Fts3Table *p, /* Virtual table handle */ Fts3MultiSegReader *pCsr /* Cursor object */ @@ -2478,9 +2543,14 @@ int sqlite3Fts3SegReaderStep( && !isIgnoreEmpty && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) ){ - pCsr->aDoclist = apSegment[0]->aDoclist; pCsr->nDoclist = apSegment[0]->nDoclist; - rc = SQLITE_ROW; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; }else{ int nDoclist = 0; /* Size of doclist */ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ diff --git a/manifest b/manifest index d847584939..2efad53b59 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\s#ifdefs\sso\sthat\sthe\sbuild\sgoes\scorrectly\sif\sthe\sonly\sFTS\smacro\ndefined\sis\sSQLITE_ENABLE_FTS4. -D 2011-06-16T00:54:45.816 +C Fix\sa\sproblem\swith\sNEAR\squeries\sexecuted\sinside\sa\stransaction\sthat\swrites\sthe\sFTS\stable. +D 2011-06-16T16:06:05.320 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,9 +61,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 014e06f585f7ff0e34229a8eb7955ae03b8086fe +F ext/fts3/fts3.c 78b02b5f0195e397c4239ef9213e5506b7d3fa97 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 27b0b75c4d50f4022eeacbd62801d8e204ae2584 +F ext/fts3/fts3Int.h df761492ae2308d3d56123907ca29cdf9bdd3748 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b F ext/fts3/fts3_hash.c aad95afa01cf2a5ffaa448e4b0ab043880cd1efb @@ -76,7 +76,7 @@ F ext/fts3/fts3_test.c 4e833729c13cea9a6bb98d3b353f6e3b8f756004 F ext/fts3/fts3_tokenizer.c 90ba6cdd8bb1b3686ab7a3d72333131e13c8fdb2 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c b0df39416b69e735c1209a2c1cd61bc3cd21c2c9 +F ext/fts3/fts3_write.c 5774a7ee9632355ebf1ec4b7a5071fc9ab9eb956 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test a98cc895bd92df14ce4a6e94f5c68d33edcc1372 +F test/fts3auto.test b0d360b331ff68bd9fb497a6192d23dc0783637c F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 03d9480fc4a8310bc8da81a64f9206c8f4b501d9 -R d0766e6b55a873753b193b31dc04f4d0 -U drh -Z d9decbc0f2b53d6559c9602b3b81dc73 +P a0b43a320e6491de7070966ed7c3ec55fd660a85 +R a91f82245adaecbc51456523e27b5133 +U dan +Z 67955b067165a6636fb02e1c55fe6931 diff --git a/manifest.uuid b/manifest.uuid index 19f7afc0ac..999de7fa63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0b43a320e6491de7070966ed7c3ec55fd660a85 \ No newline at end of file +051c756c367837908f6691c0a36108e088c94f99 \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index 1fd02c537f..ed96129a70 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -521,6 +521,7 @@ foreach {tn create} { do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five} do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five} do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five} + do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five} do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*} @@ -534,18 +535,23 @@ foreach {tn create} { # The following test cases - fts3auto-5.* - focus on using prefix indexes. # set chunkconfig [fts3_configure_incr_load 1 1] -foreach {tn create} { - 1 "fts4(a, b)" - 2 "fts4(a, b, order=DESC, prefix=1)" - 3 "fts4(a, b, order=ASC, prefix=1,3)" - 4 "fts4(a, b, order=DESC, prefix=2,4)" +foreach {tn create pending} { + 2 "fts4(a, b, order=ASC, prefix=1)" 1 + + 1 "fts4(a, b)" 1 + 3 "fts4(a, b, order=ASC, prefix=1,3)" 0 + 4 "fts4(a, b, order=DESC, prefix=2,4)" 0 + 5 "fts4(a, b, order=DESC, prefix=1)" 0 + 6 "fts4(a, b, order=ASC, prefix=1,3)" 0 } { execsql [subst { - DROP TABLE t1; + DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING $create; }] + if {$pending} {execsql BEGIN} + foreach {a b} { "the song of songs which is solomons" "let him kiss me with the kisses of his mouth for thy love is better than wine" @@ -568,12 +574,16 @@ foreach {tn create} { execsql {INSERT INTO t1(a, b) VALUES($a, $b)} } + do_fts3query_test 5.$tn.1.1 t1 {s*} do_fts3query_test 5.$tn.1.2 t1 {so*} do_fts3query_test 5.$tn.1.3 t1 {"s* o*"} do_fts3query_test 5.$tn.1.4 t1 {b* NEAR/3 a*} - do_fts3query_test 5.$tn.1.5 t1 {th* NEAR/5 a* NEAR/5 w*} - do_fts3query_test 5.$tn.1.6 t1 {"b* th* art* fair*"} + do_fts3query_test 5.$tn.1.5 t1 {a*} + do_fts3query_test 5.$tn.1.6 t1 {th* NEAR/5 a* NEAR/5 w*} + do_fts3query_test 5.$tn.1.7 t1 {"b* th* art* fair*"} + + if {$pending} {execsql COMMIT} } eval fts3_configure_incr_load $chunkconfig