From: dan Date: Wed, 3 Feb 2016 20:04:59 +0000 (+0000) Subject: Improve performance of fts5 prefix queries on detail=col tables. X-Git-Tag: version-3.11.0~60 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=eb715c62f9367794b11cc9210a9e61a4a1943b04;p=thirdparty%2Fsqlite.git Improve performance of fts5 prefix queries on detail=col tables. FossilOrigin-Name: ca11f46db047e7f131cef3893f73824758a2076b --- diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 9e5e52fa41..fc71584909 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -332,6 +332,12 @@ struct Fts5IndexIter { #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ #define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ +/* The following are used internally by the fts5_index.c module. They are +** defined here only to make it easier to avoid clashes with the flags +** above. */ +#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 +#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 + /* ** Create/destroy an Fts5Index object. */ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 3b627dabad..1a8cc63dad 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -2264,9 +2264,6 @@ static void fts5SegIterSeekInit( int bGe = (flags & FTS5INDEX_QUERY_SCAN); int bDlidx = 0; /* True if there is a doclist-index */ - static int nCall = 0; - nCall++; - assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); assert( pTerm && nTerm ); memset(pIter, 0, sizeof(*pIter)); @@ -2832,196 +2829,116 @@ static Fts5Iter *fts5MultiIterAlloc( return pNew; } -/* -** Allocate a new Fts5Iter object. -** -** The new object will be used to iterate through data in structure pStruct. -** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel -** is zero or greater, data from the first nSegment segments on level iLevel -** is merged. -** -** The iterator initially points to the first term/rowid entry in the -** iterated data. -*/ -static void fts5MultiIterNew( - Fts5Index *p, /* FTS5 backend to iterate within */ - Fts5Structure *pStruct, /* Structure of specific index */ - int bSkipEmpty, /* True to ignore delete-keys */ - int flags, /* FTS5INDEX_QUERY_XXX flags */ - const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ - int iLevel, /* Level to iterate (-1 for all) */ - int nSegment, /* Number of segments to merge (iLevel>=0) */ - Fts5Iter **ppOut /* New object */ +static void fts5PoslistCallback( + Fts5Index *p, + void *pContext, + const u8 *pChunk, int nChunk ){ - int nSeg = 0; /* Number of segment-iters in use */ - int iIter = 0; /* */ - int iSeg; /* Used to iterate through segments */ - Fts5Buffer buf = {0,0,0}; /* Buffer used by fts5SegIterSeekInit() */ - Fts5StructureLevel *pLvl; - Fts5Iter *pNew; + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk); + } +} - assert( (pTerm==0 && nTerm==0) || iLevel<0 ); +typedef struct PoslistCallbackCtx PoslistCallbackCtx; +struct PoslistCallbackCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int eState; /* See above */ +}; - /* Allocate space for the new multi-seg-iterator. */ - if( p->rc==SQLITE_OK ){ - if( iLevel<0 ){ - assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); - nSeg = pStruct->nSegment; - nSeg += (p->pHash ? 1 : 0); - }else{ - nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); - } - } - *ppOut = pNew = fts5MultiIterAlloc(p, nSeg); - if( pNew==0 ) return; - pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); - pNew->bSkipEmpty = (u8)bSkipEmpty; - pNew->pStruct = pStruct; - fts5StructureRef(pStruct); +typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; +struct PoslistOffsetsCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int iRead; + int iWrite; +}; - /* Initialize each of the component segment iterators. */ - if( iLevel<0 ){ - Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; - if( p->pHash ){ - /* Add a segment iterator for the current contents of the hash table. */ - Fts5SegIter *pIter = &pNew->aSeg[iIter++]; - fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); - } - for(pLvl=&pStruct->aLevel[0]; pLvlnSeg-1; iSeg>=0; iSeg--){ - Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; - Fts5SegIter *pIter = &pNew->aSeg[iIter++]; - if( pTerm==0 ){ - fts5SegIterInit(p, pSeg, pIter); - }else{ - fts5SegIterSeekInit(p, &buf, pTerm, nTerm, flags, pSeg, pIter); - } - } - } - }else{ - pLvl = &pStruct->aLevel[iLevel]; - for(iSeg=nSeg-1; iSeg>=0; iSeg--){ - fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); - } +/* +** TODO: Make this more efficient! +*/ +static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ + int i; + for(i=0; inCol; i++){ + if( pColset->aiCol[i]==iCol ) return 1; } - assert( iIter==nSeg ); + return 0; +} - /* If the above was successful, each component iterators now points - ** to the first entry in its segment. In this case initialize the - ** aFirst[] array. Or, if an error has occurred, free the iterator - ** object and set the output variable to NULL. */ - if( p->rc==SQLITE_OK ){ - for(iIter=pNew->nSeg-1; iIter>0; iIter--){ - int iEq; - if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ - Fts5SegIter *pSeg = &pNew->aSeg[iEq]; - if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); - fts5MultiIterAdvanced(p, pNew, iEq, iIter); +static void fts5PoslistOffsetsCallback( + Fts5Index *p, + void *pContext, + const u8 *pChunk, int nChunk +){ + PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + int i = 0; + while( iiRead - 2; + pCtx->iRead = iVal; + if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ + fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); + pCtx->iWrite = iVal; } } - fts5MultiIterSetEof(pNew); - fts5AssertMultiIterSetup(p, pNew); - - if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ - fts5MultiIterNext(p, pNew, 0, 0); - } - }else{ - fts5MultiIterFree(p, pNew); - *ppOut = 0; } - fts5BufferFree(&buf); } -/* -** Create an Fts5Iter that iterates through the doclist provided -** as the second argument. -*/ -static void fts5MultiIterNew2( - Fts5Index *p, /* FTS5 backend to iterate within */ - Fts5Data *pData, /* Doclist to iterate through */ - int bDesc, /* True for descending rowid order */ - Fts5Iter **ppOut /* New object */ +static void fts5PoslistFilterCallback( + Fts5Index *p, + void *pContext, + const u8 *pChunk, int nChunk ){ - Fts5Iter *pNew; - pNew = fts5MultiIterAlloc(p, 2); - if( pNew ){ - Fts5SegIter *pIter = &pNew->aSeg[1]; + PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + /* Search through to find the first varint with value 1. This is the + ** start of the next columns hits. */ + int i = 0; + int iStart = 0; - pNew->bFiltered = 1; - pIter->flags = FTS5_SEGITER_ONETERM; - if( pData->szLeaf>0 ){ - pIter->pLeaf = pData; - pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); - pIter->iEndofDoclist = pData->nn; - pNew->aFirst[1].iFirst = 1; - if( bDesc ){ - pNew->bRev = 1; - pIter->flags |= FTS5_SEGITER_REVERSE; - fts5SegIterReverseInitPage(p, pIter); + if( pCtx->eState==2 ){ + int iCol; + fts5FastGetVarint32(pChunk, i, iCol); + if( fts5IndexColsetTest(pCtx->pColset, iCol) ){ + pCtx->eState = 1; + fts5BufferSafeAppendVarint(pCtx->pBuf, 1); }else{ - fts5SegIterLoadNPos(p, pIter); + pCtx->eState = 0; } - pData = 0; - }else{ - pNew->base.bEof = 1; } - fts5SegIterSetNext(p, pIter); - - *ppOut = pNew; - } - - fts5DataRelease(pData); -} - -/* -** Return true if the iterator is at EOF or if an error has occurred. -** False otherwise. -*/ -static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ - assert( p->rc - || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof - ); - return (p->rc || pIter->base.bEof); -} - -/* -** Return the rowid of the entry that the iterator currently points -** to. If the iterator points to EOF when this function is called the -** results are undefined. -*/ -static i64 fts5MultiIterRowid(Fts5Iter *pIter){ - assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf ); - return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid; -} -/* -** Move the iterator to the next entry at or following iMatch. -*/ -static void fts5MultiIterNextFrom( - Fts5Index *p, - Fts5Iter *pIter, - i64 iMatch -){ - while( 1 ){ - i64 iRowid; - fts5MultiIterNext(p, pIter, 1, iMatch); - if( fts5MultiIterEof(p, pIter) ) break; - iRowid = fts5MultiIterRowid(pIter); - if( pIter->bRev==0 && iRowid>=iMatch ) break; - if( pIter->bRev!=0 && iRowid<=iMatch ) break; + do { + while( ieState ){ + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); + } + if( i=nChunk ){ + pCtx->eState = 2; + }else{ + fts5FastGetVarint32(pChunk, i, iCol); + pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol); + if( pCtx->eState ){ + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); + iStart = i; + } + } + } + }while( iaSeg[ pIter->aFirst[1].iFirst ]; - *pn = p->term.n; - return p->term.p; -} - static void fts5ChunkIterate( Fts5Index *p, /* Index object */ Fts5SegIter *pSeg, /* Poslist of this iterator */ @@ -3063,18 +2980,470 @@ static void fts5ChunkIterate( } } - - /* -** Allocate a new segment-id for the structure pStruct. The new segment -** id must be between 1 and 65335 inclusive, and must not be used by -** any currently existing segment. If a free segment id cannot be found, -** SQLITE_FULL is returned. -** -** If an error has already occurred, this function is a no-op. 0 is -** returned in this case. +** Iterator pIter currently points to a valid entry (not EOF). This +** function appends the position list data for the current entry to +** buffer pBuf. It does not make a copy of the position-list size +** field. */ -static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ +static void fts5SegiterPoslist( + Fts5Index *p, + Fts5SegIter *pSeg, + Fts5Colset *pColset, + Fts5Buffer *pBuf +){ + if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){ + if( pColset==0 ){ + fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); + }else{ + if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ + PoslistCallbackCtx sCtx; + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + sCtx.eState = fts5IndexColsetTest(pColset, 0); + assert( sCtx.eState==0 || sCtx.eState==1 ); + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + }else{ + PoslistOffsetsCtx sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); + } + } + } +} + +/* +** IN/OUT parameter (*pa) points to a position list n bytes in size. If +** the position list contains entries for column iCol, then (*pa) is set +** to point to the sub-position-list for that column and the number of +** bytes in it returned. Or, if the argument position list does not +** contain any entries for column iCol, return 0. +*/ +static int fts5IndexExtractCol( + const u8 **pa, /* IN/OUT: Pointer to poslist */ + int n, /* IN: Size of poslist in bytes */ + int iCol /* Column to extract from poslist */ +){ + int iCurrent = 0; /* Anything before the first 0x01 is col 0 */ + const u8 *p = *pa; + const u8 *pEnd = &p[n]; /* One byte past end of position list */ + + while( iCol>iCurrent ){ + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint. Note that it is not possible for a negative + ** or extremely large varint to occur within an uncorrupted position + ** list. So the last byte of each varint may be assumed to have a clear + ** 0x80 bit. */ + while( *p!=0x01 ){ + while( *p++ & 0x80 ); + if( p>=pEnd ) return 0; + } + *pa = p++; + iCurrent = *p++; + if( iCurrent & 0x80 ){ + p--; + p += fts5GetVarint32(p, iCurrent); + } + } + if( iCol!=iCurrent ) return 0; + + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint */ + while( pnCol; i++){ + const u8 *pSub = pPos; + int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); + if( nSub ){ + fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); + } + } + return rc; +} + +/* +** xSetOutputs callback used by detail=none tables. +*/ +static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){ + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE ); + pIter->base.iRowid = pSeg->iRowid; + pIter->base.nData = pSeg->nPos; +} + +/* +** xSetOutputs callback used by detail=full and detail=col tables when no +** column filters are specified. +*/ +static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ + pIter->base.iRowid = pSeg->iRowid; + pIter->base.nData = pSeg->nPos; + + assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE ); + assert( pIter->pColset==0 || pIter->bFiltered ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + } +} + +/* +** xSetOutputs callback used by detail=col when there is a column filter +** and there are 100 or more columns. Also called as a fallback from +** fts5IterSetOutputs_Col100 if the column-list spans more than one page. +*/ +static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist); + pIter->base.iRowid = pSeg->iRowid; + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; +} + +/* +** xSetOutputs callback used when: +** +** * detail=col, +** * there is a column filter, and +** * the table contains 100 or fewer columns. +** +** The last point is to ensure all column numbers are stored as +** single-byte varints. +*/ +static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + assert( pIter->pColset ); + + if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){ + fts5IterSetOutputs_Col(pIter, pSeg); + }else{ + u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; + u8 *pEnd = (u8*)&a[pSeg->nPos]; + int iPrev = 0; + int *aiCol = pIter->pColset->aiCol; + int *aiColEnd = &aiCol[pIter->pColset->nCol]; + + u8 *aOut = pIter->poslist.p; + int iPrevOut = 0; + + pIter->base.iRowid = pSeg->iRowid; + + while( abase.pData = pIter->poslist.p; + pIter->base.nData = aOut - pIter->poslist.p; + } +} + +/* +** xSetOutputs callback used by detail=full when there is a column filter. +*/ +static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ + Fts5Colset *pColset = pIter->pColset; + pIter->base.iRowid = pSeg->iRowid; + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL ); + assert( pColset ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + if( pColset->nCol==1 ){ + pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]); + pIter->base.pData = a; + }else{ + fts5BufferZero(&pIter->poslist); + fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } +} + +static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ + if( *pRc==SQLITE_OK ){ + Fts5Config *pConfig = pIter->pIndex->pConfig; + if( pConfig->eDetail==FTS5_DETAIL_NONE ){ + pIter->xSetOutputs = fts5IterSetOutputs_None; + } + + else if( pIter->pColset==0 || pIter->bFiltered ){ + pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; + } + + else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ + pIter->xSetOutputs = fts5IterSetOutputs_Full; + } + + else{ + assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + if( pConfig->nCol<=100 ){ + pIter->xSetOutputs = fts5IterSetOutputs_Col100; + sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol); + }else{ + pIter->xSetOutputs = fts5IterSetOutputs_Col; + } + } + } +} + + +/* +** Allocate a new Fts5Iter object. +** +** The new object will be used to iterate through data in structure pStruct. +** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel +** is zero or greater, data from the first nSegment segments on level iLevel +** is merged. +** +** The iterator initially points to the first term/rowid entry in the +** iterated data. +*/ +static void fts5MultiIterNew( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Structure *pStruct, /* Structure of specific index */ + int flags, /* FTS5INDEX_QUERY_XXX flags */ + Fts5Colset *pColset, /* Colset to filter on (or NULL) */ + const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ + int iLevel, /* Level to iterate (-1 for all) */ + int nSegment, /* Number of segments to merge (iLevel>=0) */ + Fts5Iter **ppOut /* New object */ +){ + int nSeg = 0; /* Number of segment-iters in use */ + int iIter = 0; /* */ + int iSeg; /* Used to iterate through segments */ + Fts5Buffer buf = {0,0,0}; /* Buffer used by fts5SegIterSeekInit() */ + Fts5StructureLevel *pLvl; + Fts5Iter *pNew; + + assert( (pTerm==0 && nTerm==0) || iLevel<0 ); + + /* Allocate space for the new multi-seg-iterator. */ + if( p->rc==SQLITE_OK ){ + if( iLevel<0 ){ + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); + nSeg = pStruct->nSegment; + nSeg += (p->pHash ? 1 : 0); + }else{ + nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); + } + } + *ppOut = pNew = fts5MultiIterAlloc(p, nSeg); + if( pNew==0 ) return; + pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); + pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY)); + pNew->pStruct = pStruct; + pNew->pColset = pColset; + fts5StructureRef(pStruct); + if( (flags & FTS5INDEX_QUERY_NOOUTPUT)==0 ){ + fts5IterSetOutputCb(&p->rc, pNew); + } + + /* Initialize each of the component segment iterators. */ + if( p->rc==SQLITE_OK ){ + if( iLevel<0 ){ + Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; + if( p->pHash ){ + /* Add a segment iterator for the current contents of the hash table. */ + Fts5SegIter *pIter = &pNew->aSeg[iIter++]; + fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); + } + for(pLvl=&pStruct->aLevel[0]; pLvlnSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + Fts5SegIter *pIter = &pNew->aSeg[iIter++]; + if( pTerm==0 ){ + fts5SegIterInit(p, pSeg, pIter); + }else{ + fts5SegIterSeekInit(p, &buf, pTerm, nTerm, flags, pSeg, pIter); + } + } + } + }else{ + pLvl = &pStruct->aLevel[iLevel]; + for(iSeg=nSeg-1; iSeg>=0; iSeg--){ + fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); + } + } + assert( iIter==nSeg ); + } + + /* If the above was successful, each component iterators now points + ** to the first entry in its segment. In this case initialize the + ** aFirst[] array. Or, if an error has occurred, free the iterator + ** object and set the output variable to NULL. */ + if( p->rc==SQLITE_OK ){ + for(iIter=pNew->nSeg-1; iIter>0; iIter--){ + int iEq; + if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ + Fts5SegIter *pSeg = &pNew->aSeg[iEq]; + if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); + fts5MultiIterAdvanced(p, pNew, iEq, iIter); + } + } + fts5MultiIterSetEof(pNew); + fts5AssertMultiIterSetup(p, pNew); + + if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ + fts5MultiIterNext(p, pNew, 0, 0); + }else if( pNew->base.bEof==0 ){ + Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; + pNew->xSetOutputs(pNew, pSeg); + } + + }else{ + fts5MultiIterFree(p, pNew); + *ppOut = 0; + } + fts5BufferFree(&buf); + +} + +/* +** Create an Fts5Iter that iterates through the doclist provided +** as the second argument. +*/ +static void fts5MultiIterNew2( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Data *pData, /* Doclist to iterate through */ + int bDesc, /* True for descending rowid order */ + Fts5Iter **ppOut /* New object */ +){ + Fts5Iter *pNew; + pNew = fts5MultiIterAlloc(p, 2); + if( pNew ){ + Fts5SegIter *pIter = &pNew->aSeg[1]; + + pNew->bFiltered = 1; + pIter->flags = FTS5_SEGITER_ONETERM; + if( pData->szLeaf>0 ){ + pIter->pLeaf = pData; + pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); + pIter->iEndofDoclist = pData->nn; + pNew->aFirst[1].iFirst = 1; + if( bDesc ){ + pNew->bRev = 1; + pIter->flags |= FTS5_SEGITER_REVERSE; + fts5SegIterReverseInitPage(p, pIter); + }else{ + fts5SegIterLoadNPos(p, pIter); + } + pData = 0; + }else{ + pNew->base.bEof = 1; + } + fts5SegIterSetNext(p, pIter); + + *ppOut = pNew; + } + + fts5DataRelease(pData); +} + +/* +** Return true if the iterator is at EOF or if an error has occurred. +** False otherwise. +*/ +static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ + assert( p->rc + || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof + ); + return (p->rc || pIter->base.bEof); +} + +/* +** Return the rowid of the entry that the iterator currently points +** to. If the iterator points to EOF when this function is called the +** results are undefined. +*/ +static i64 fts5MultiIterRowid(Fts5Iter *pIter){ + assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf ); + return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid; +} + +/* +** Move the iterator to the next entry at or following iMatch. +*/ +static void fts5MultiIterNextFrom( + Fts5Index *p, + Fts5Iter *pIter, + i64 iMatch +){ + while( 1 ){ + i64 iRowid; + fts5MultiIterNext(p, pIter, 1, iMatch); + if( fts5MultiIterEof(p, pIter) ) break; + iRowid = fts5MultiIterRowid(pIter); + if( pIter->bRev==0 && iRowid>=iMatch ) break; + if( pIter->bRev!=0 && iRowid<=iMatch ) break; + } +} + +/* +** Return a pointer to a buffer containing the term associated with the +** entry that the iterator currently points to. +*/ +static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){ + Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + *pn = p->term.n; + return p->term.p; +} + +/* +** Allocate a new segment-id for the structure pStruct. The new segment +** id must be between 1 and 65335 inclusive, and must not be used by +** any currently existing segment. If a free segment id cannot be found, +** SQLITE_FULL is returned. +** +** If an error has already occurred, this function is a no-op. 0 is +** returned in this case. +*/ +static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ int iSegid = 0; if( p->rc==SQLITE_OK ){ @@ -3684,6 +4053,7 @@ static void fts5IndexMergeLevel( Fts5Buffer term; int bOldest; /* True if the output segment is the oldest */ int eDetail = p->pConfig->eDetail; + const int flags = FTS5INDEX_QUERY_NOOUTPUT; assert( iLvlnLevel ); assert( pLvl->nMerge<=pLvl->nSeg ); @@ -3728,7 +4098,7 @@ static void fts5IndexMergeLevel( bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); assert( iLvl>=0 ); - for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, iLvl, nInput, &pIter); + for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter); fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) ){ @@ -4140,314 +4510,61 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){ } pNew->nSegment = pLvl->nSeg = nSeg; }else{ - sqlite3_free(pNew); - pNew = 0; - } - } - - if( pNew ){ - int iLvl = pNew->nLevel-1; - while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ - int nRem = FTS5_OPT_WORK_UNIT; - fts5IndexMergeLevel(p, &pNew, iLvl, &nRem); - } - - fts5StructureWrite(p, pNew); - fts5StructureRelease(pNew); - } - - fts5StructureRelease(pStruct); - return fts5IndexReturn(p); -} - -int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ - Fts5Structure *pStruct; - - pStruct = fts5StructureRead(p); - if( pStruct && pStruct->nLevel ){ - fts5IndexMerge(p, &pStruct, nMerge); - fts5StructureWrite(p, pStruct); - } - fts5StructureRelease(pStruct); - - return fts5IndexReturn(p); -} - -static void fts5PoslistCallback( - Fts5Index *p, - void *pContext, - const u8 *pChunk, int nChunk -){ - assert_nc( nChunk>=0 ); - if( nChunk>0 ){ - fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk); - } -} - -typedef struct PoslistCallbackCtx PoslistCallbackCtx; -struct PoslistCallbackCtx { - Fts5Buffer *pBuf; /* Append to this buffer */ - Fts5Colset *pColset; /* Restrict matches to this column */ - int eState; /* See above */ -}; - -typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; -struct PoslistOffsetsCtx { - Fts5Buffer *pBuf; /* Append to this buffer */ - Fts5Colset *pColset; /* Restrict matches to this column */ - int iRead; - int iWrite; -}; - -/* -** TODO: Make this more efficient! -*/ -static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ - int i; - for(i=0; inCol; i++){ - if( pColset->aiCol[i]==iCol ) return 1; - } - return 0; -} - -static void fts5PoslistOffsetsCallback( - Fts5Index *p, - void *pContext, - const u8 *pChunk, int nChunk -){ - PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; - assert_nc( nChunk>=0 ); - if( nChunk>0 ){ - int i = 0; - while( iiRead - 2; - pCtx->iRead = iVal; - if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ - fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); - pCtx->iWrite = iVal; - } - } - } -} - -static void fts5PoslistFilterCallback( - Fts5Index *p, - void *pContext, - const u8 *pChunk, int nChunk -){ - PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; - assert_nc( nChunk>=0 ); - if( nChunk>0 ){ - /* Search through to find the first varint with value 1. This is the - ** start of the next columns hits. */ - int i = 0; - int iStart = 0; - - if( pCtx->eState==2 ){ - int iCol; - fts5FastGetVarint32(pChunk, i, iCol); - if( fts5IndexColsetTest(pCtx->pColset, iCol) ){ - pCtx->eState = 1; - fts5BufferSafeAppendVarint(pCtx->pBuf, 1); - }else{ - pCtx->eState = 0; - } - } - - do { - while( ieState ){ - fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); - } - if( i=nChunk ){ - pCtx->eState = 2; - }else{ - fts5FastGetVarint32(pChunk, i, iCol); - pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol); - if( pCtx->eState ){ - fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); - iStart = i; - } - } - } - }while( irc, pBuf, pSeg->nPos) ){ - if( pColset==0 ){ - fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); - }else{ - if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ - PoslistCallbackCtx sCtx; - sCtx.pBuf = pBuf; - sCtx.pColset = pColset; - sCtx.eState = fts5IndexColsetTest(pColset, 0); - assert( sCtx.eState==0 || sCtx.eState==1 ); - fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); - }else{ - PoslistOffsetsCtx sCtx; - memset(&sCtx, 0, sizeof(sCtx)); - sCtx.pBuf = pBuf; - sCtx.pColset = pColset; - fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); - } + sqlite3_free(pNew); + pNew = 0; } } -} - -/* -** IN/OUT parameter (*pa) points to a position list n bytes in size. If -** the position list contains entries for column iCol, then (*pa) is set -** to point to the sub-position-list for that column and the number of -** bytes in it returned. Or, if the argument position list does not -** contain any entries for column iCol, return 0. -*/ -static int fts5IndexExtractCol( - const u8 **pa, /* IN/OUT: Pointer to poslist */ - int n, /* IN: Size of poslist in bytes */ - int iCol /* Column to extract from poslist */ -){ - int iCurrent = 0; /* Anything before the first 0x01 is col 0 */ - const u8 *p = *pa; - const u8 *pEnd = &p[n]; /* One byte past end of position list */ - while( iCol>iCurrent ){ - /* Advance pointer p until it points to pEnd or an 0x01 byte that is - ** not part of a varint. Note that it is not possible for a negative - ** or extremely large varint to occur within an uncorrupted position - ** list. So the last byte of each varint may be assumed to have a clear - ** 0x80 bit. */ - while( *p!=0x01 ){ - while( *p++ & 0x80 ); - if( p>=pEnd ) return 0; - } - *pa = p++; - iCurrent = *p++; - if( iCurrent & 0x80 ){ - p--; - p += fts5GetVarint32(p, iCurrent); + if( pNew ){ + int iLvl = pNew->nLevel-1; + while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ + int nRem = FTS5_OPT_WORK_UNIT; + fts5IndexMergeLevel(p, &pNew, iLvl, &nRem); } + + fts5StructureWrite(p, pNew); + fts5StructureRelease(pNew); } - if( iCol!=iCurrent ) return 0; - /* Advance pointer p until it points to pEnd or an 0x01 byte that is - ** not part of a varint */ - while( pnLevel ){ + fts5IndexMerge(p, &pStruct, nMerge); + fts5StructureWrite(p, pStruct); } + fts5StructureRelease(pStruct); - return p - (*pa); + return fts5IndexReturn(p); } -static int fts5AppendRowid( +static void fts5AppendRowid( Fts5Index *p, i64 iDelta, Fts5Iter *pMulti, - Fts5Colset *pColset, Fts5Buffer *pBuf ){ fts5BufferAppendVarint(&p->rc, pBuf, iDelta); - return 0; } -/* -** Iterator pMulti currently points to a valid entry (not EOF). This -** function appends the following to buffer pBuf: -** -** * The varint iDelta, and -** * the position list that currently points to, including the size field. -** -** If argument pColset is NULL, then the position list is filtered according -** to pColset before being appended to the buffer. If this means there are -** no entries in the position list, nothing is appended to the buffer (not -** even iDelta). -** -** If an error occurs, an error code is left in p->rc. -*/ -static int fts5AppendPoslist( +static void fts5AppendPoslist( Fts5Index *p, i64 iDelta, Fts5Iter *pMulti, - Fts5Colset *pColset, Fts5Buffer *pBuf ){ - if( p->rc==SQLITE_OK ){ - Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; - assert( fts5MultiIterEof(p, pMulti)==0 ); - assert( pSeg->nPos>0 ); - if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ - if( p->pConfig->eDetail==FTS5_DETAIL_FULL - && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf - && (pColset==0 || pColset->nCol==1) - ){ - const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; - int nPos; - if( pColset ){ - nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]); - if( nPos==0 ) return 1; - }else{ - nPos = pSeg->nPos; - } - assert( nPos>0 ); - fts5BufferSafeAppendVarint(pBuf, iDelta); - fts5BufferSafeAppendVarint(pBuf, nPos*2); - fts5BufferSafeAppendBlob(pBuf, pPos, nPos); - }else{ - int iSv1; - int iSv2; - int iData; - - /* Append iDelta */ - iSv1 = pBuf->n; - fts5BufferSafeAppendVarint(pBuf, iDelta); - - /* WRITEPOSLISTSIZE */ - iSv2 = pBuf->n; - fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2); - iData = pBuf->n; - - fts5SegiterPoslist(p, pSeg, pColset, pBuf); - - if( pColset ){ - int nActual = pBuf->n - iData; - if( nActual!=pSeg->nPos ){ - if( nActual==0 ){ - pBuf->n = iSv1; - return 1; - }else{ - int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2)); - while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; } - sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2); - } - } - } - } - } + int nData = pMulti->base.nData; + assert( nData>0 ); + if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nData+9+9) ){ + fts5BufferSafeAppendVarint(pBuf, iDelta); + fts5BufferSafeAppendVarint(pBuf, nData*2); + fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData); } - - return 0; } @@ -4674,7 +4791,7 @@ static void fts5SetupPrefixIter( const int nBuf = 32; void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*); - int (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Colset*, Fts5Buffer*); + void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ xMerge = fts5MergeRowidLists; xAppend = fts5AppendRowid; @@ -4687,7 +4804,9 @@ static void fts5SetupPrefixIter( pStruct = fts5StructureRead(p); if( aBuf && pStruct ){ - const int flags = FTS5INDEX_QUERY_SCAN; + const int flags = FTS5INDEX_QUERY_SCAN + | FTS5INDEX_QUERY_SKIPEMPTY + | FTS5INDEX_QUERY_NOOUTPUT; int i; i64 iLastRowid = 0; Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ @@ -4696,19 +4815,25 @@ static void fts5SetupPrefixIter( int bNewTerm = 1; memset(&doclist, 0, sizeof(doclist)); - for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1); + fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); + fts5IterSetOutputCb(&p->rc, p1); + for( /* no-op */ ; fts5MultiIterEof(p, p1)==0; fts5MultiIterNext2(p, p1, &bNewTerm) ){ - i64 iRowid = fts5MultiIterRowid(p1); - int nTerm; - const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm); + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + int nTerm = pSeg->term.n; + const u8 *pTerm = pSeg->term.p; + p1->xSetOutputs(p1, pSeg); + assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); if( bNewTerm ){ if( nTerm0 && iRowid<=iLastRowid ){ + if( p1->base.nData==0 ) continue; + + if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ assert( ibase.iRowid-iLastRowid, p1, &doclist); + iLastRowid = p1->base.iRowid; } for(i=0; inCol; i++){ - const u8 *pSub = pPos; - int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); - if( nSub ){ - fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); - } - } - return rc; -} - -/* -** xSetOutputs callback used by detail=none tables. -*/ -static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){ - assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE ); - pIter->base.iRowid = pSeg->iRowid; - pIter->base.nData = pSeg->nPos; -} - -/* -** xSetOutputs callback used by detail=full and detail=col tables when no -** column filters are specified. -*/ -static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ - pIter->base.iRowid = pSeg->iRowid; - pIter->base.nData = pSeg->nPos; - - assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE ); - assert( pIter->pColset==0 || pIter->bFiltered ); - - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ - /* All data is stored on the current page. Populate the output - ** variables to point into the body of the page object. */ - pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset]; - }else{ - /* The data is distributed over two or more pages. Copy it into the - ** Fts5Iter.poslist buffer and then set the output pointer to point - ** to this buffer. */ - fts5BufferZero(&pIter->poslist); - fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist); - pIter->base.pData = pIter->poslist.p; - } -} - -/* -** xSetOutputs callback used by detail=col when there is a column filter -** and there are 100 or more columns. Also called as a fallback from -** fts5IterSetOutputs_Col100 if the column-list spans more than one page. -*/ -static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ - fts5BufferZero(&pIter->poslist); - fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist); - pIter->base.iRowid = pSeg->iRowid; - pIter->base.pData = pIter->poslist.p; - pIter->base.nData = pIter->poslist.n; -} - -/* -** xSetOutputs callback used when: -** -** * detail=col, -** * there is a column filter, and -** * the table contains 100 or fewer columns. -** -** The last point is to ensure all column numbers are stored as -** single-byte varints. -*/ -static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ - - assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); - assert( pIter->pColset ); - - if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){ - fts5IterSetOutputs_Col(pIter, pSeg); - }else{ - u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; - u8 *pEnd = (u8*)&a[pSeg->nPos]; - int iPrev = 0; - int *aiCol = pIter->pColset->aiCol; - int *aiColEnd = &aiCol[pIter->pColset->nCol]; - - u8 *aOut = pIter->poslist.p; - int iPrevOut = 0; - - pIter->base.iRowid = pSeg->iRowid; - - while( abase.pData = pIter->poslist.p; - pIter->base.nData = aOut - pIter->poslist.p; - } -} - -/* -** xSetOutputs callback used by detail=full when there is a column filter. -*/ -static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ - Fts5Colset *pColset = pIter->pColset; - pIter->base.iRowid = pSeg->iRowid; - - assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL ); - assert( pColset ); - - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ - /* All data is stored on the current page. Populate the output - ** variables to point into the body of the page object. */ - const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; - if( pColset->nCol==1 ){ - pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]); - pIter->base.pData = a; - }else{ - fts5BufferZero(&pIter->poslist); - fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist); - pIter->base.pData = pIter->poslist.p; - pIter->base.nData = pIter->poslist.n; - } - }else{ - /* The data is distributed over two or more pages. Copy it into the - ** Fts5Iter.poslist buffer and then set the output pointer to point - ** to this buffer. */ - fts5BufferZero(&pIter->poslist); - fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); - pIter->base.pData = pIter->poslist.p; - pIter->base.nData = pIter->poslist.n; - } -} - -static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ - Fts5Config *pConfig = pIter->pIndex->pConfig; - if( pConfig->eDetail==FTS5_DETAIL_NONE ){ - pIter->xSetOutputs = fts5IterSetOutputs_None; - } - - else if( pIter->pColset==0 || pIter->bFiltered ){ - pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; - } - - else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ - pIter->xSetOutputs = fts5IterSetOutputs_Full; - } - - else{ - assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS ); - if( pConfig->nCol<=100 ){ - pIter->xSetOutputs = fts5IterSetOutputs_Col100; - sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol); - }else{ - pIter->xSetOutputs = fts5IterSetOutputs_Col; - } - } -} - /* ** Open a new iterator to iterate though all rowid that match the ** specified token or token prefix. @@ -5177,7 +5128,9 @@ int sqlite3Fts5IndexQuery( Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); if( pStruct ){ - fts5MultiIterNew(p, pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet); + fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY, + pColset, buf.p, nToken+1, -1, 0, &pRet + ); fts5StructureRelease(pStruct); } }else{ @@ -5185,14 +5138,13 @@ int sqlite3Fts5IndexQuery( int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; buf.p[0] = FTS5_MAIN_PREFIX; fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet); - } - - if( p->rc==SQLITE_OK ){ - Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; - pRet->pColset = pColset; fts5IterSetOutputCb(&p->rc, pRet); - if( p->rc==SQLITE_OK && pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; + if( p->rc==SQLITE_OK && pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); + } } + if( p->rc ){ sqlite3Fts5IterClose(&pRet->base); pRet = 0; @@ -5769,6 +5721,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ #endif + const int flags = FTS5INDEX_QUERY_NOOUTPUT; /* Load the FTS index structure */ pStruct = fts5StructureRead(p); @@ -5797,7 +5750,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ ** same term is performed. cksum3 is calculated based on the entries ** extracted by these queries. */ - for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, -1, 0, &pIter); + for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, -1, 0, &pIter); fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) ){ diff --git a/manifest b/manifest index b7ee931a14..0efc8787a5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smarkup\serrors\sin\scomments\sused\sto\sgenerate\sthe\sdocumentation\s-\sspecifically\nin\sthe\sdocumentation\son\sthe\sOP_Seek\sopcode. -D 2016-02-03T19:52:06.161 +C Improve\sperformance\sof\sfts5\sprefix\squeries\son\sdetail=col\stables. +D 2016-02-03T20:04:59.805 F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 72b7858f02017611c3ac1ddc965251017fed0845 @@ -98,13 +98,13 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7 -F ext/fts5/fts5Int.h 9505f3bc8d0b2ca4cd2b112e7e042e3c6a3222a7 +F ext/fts5/fts5Int.h 2095cc38e776f19cc083ca90e00772ea0b204ab3 F ext/fts5/fts5_aux.c b9bcce753ef5b451267b2232f0ca153ddeb3951d F ext/fts5/fts5_buffer.c f6e0c6018ffc8e39fc0b333b5daa8b8d528ae6e4 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238 F ext/fts5/fts5_expr.c ff5c451a6d025909639ac0f0d0af0cc595b50feb F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c 471ff6935068a4579830474249e1046b57137103 +F ext/fts5/fts5_index.c e634a4a05b066f7122db93558c871148bd9893f2 F ext/fts5/fts5_main.c 7e8a5f27d504bc04e3de7f1cba8867f0332aee9d F ext/fts5/fts5_storage.c 2a1f44deae090cd711f02cec0c2af8e660360d24 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966 @@ -1423,7 +1423,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P af92401826f5cf49e62c278f344ab75252a18da2 -R f36c5f43b820b4830c00d3f3959d5c82 -U drh -Z cdc17003a87221b5f2cfde36f98aed07 +P ef252bc4b59d272460aaebdc0d4b8e347b0d25a8 +R 671ea22942ad2d2bcb9568b9e59d452e +U dan +Z 34c6688fd340fcbe0aee4e43e23acf72 diff --git a/manifest.uuid b/manifest.uuid index 6563093eae..06d0c2d828 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef252bc4b59d272460aaebdc0d4b8e347b0d25a8 \ No newline at end of file +ca11f46db047e7f131cef3893f73824758a2076b \ No newline at end of file