}
for(i=0; i<pWriter->nWriter; i++){
Fts5PageWriter *pPg = &pWriter->aWriter[i];
- assert( pPg || p->rc!=SQLITE_OK );
- if( pPg ){
- fts5BufferFree(&pPg->term);
- fts5BufferFree(&pPg->buf);
- }
+ fts5BufferFree(&pPg->term);
+ fts5BufferFree(&pPg->buf);
}
sqlite3_free(pWriter->aWriter);
sqlite3Fts5BufferFree(&pWriter->cdlidx);
/*
-** Return a simple checksum value based on the arguments.
+** Iterator pMulti currently points to a valid entry (not EOF). This
+** function appends a copy of the position-list of the entry pMulti
+** currently points to to buffer pBuf.
+**
+** If an error occurs, an error code is left in p->rc. It is assumed
+** no error has already occurred when this function is called.
*/
-static u64 fts5IndexEntryCksum(
- i64 iRowid,
- int iCol,
- int iPos,
- const char *pTerm,
- int nTerm
-){
- int i;
- u64 ret = iRowid;
- ret += (ret<<3) + iCol;
- ret += (ret<<3) + iPos;
- for(i=0; i<nTerm; i++) ret += (ret<<3) + pTerm[i];
- return ret;
-}
-
-static void fts5BtreeIterInit(
- Fts5Index *p,
- int iIdx,
- Fts5StructureSegment *pSeg,
- Fts5BtreeIter *pIter
+static void fts5MultiIterPoslist(
+ Fts5Index *p,
+ Fts5MultiSegIter *pMulti,
+ int bSz,
+ Fts5Buffer *pBuf
){
- int nByte;
- int i;
- nByte = sizeof(pIter->aLvl[0]) * (pSeg->nHeight-1);
- memset(pIter, 0, sizeof(*pIter));
- if( nByte ){
- pIter->aLvl = (Fts5BtreeIterLevel*)fts5IdxMalloc(p, nByte);
- }
if( p->rc==SQLITE_OK ){
- pIter->nLvl = pSeg->nHeight-1;
- pIter->iIdx = iIdx;
- pIter->p = p;
- pIter->pSeg = pSeg;
- }
- for(i=0; p->rc==SQLITE_OK && i<pIter->nLvl; i++){
- i64 iRowid = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, i+1, 1);
- Fts5Data *pData;
- pIter->aLvl[i].pData = pData = fts5DataRead(p, iRowid);
- if( pData ){
- fts5NodeIterInit(pData->p, pData->n, &pIter->aLvl[i].s);
- }
- }
+ Fts5ChunkIter iter;
+ Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
+ assert( fts5MultiIterEof(p, pMulti)==0 );
+ static int nCall = 0;
+ nCall++;
- if( pIter->nLvl==0 || p->rc ){
- pIter->bEof = 1;
- pIter->iLeaf = pSeg->pgnoLast;
- }else{
- pIter->nEmpty = pIter->aLvl[0].s.nEmpty;
- pIter->iLeaf = pIter->aLvl[0].s.iChild;
- pIter->bDlidx = pIter->aLvl[0].s.bDlidx;
+ fts5ChunkIterInit(p, pSeg, &iter);
+
+ if( fts5ChunkIterEof(p, &iter)==0 ){
+ if( bSz ){
+ /* WRITEPOSLISTSIZE */
+ fts5BufferAppendVarint(&p->rc, pBuf, iter.nRem * 2);
+ }
+ while( fts5ChunkIterEof(p, &iter)==0 ){
+ fts5BufferAppendBlob(&p->rc, pBuf, iter.n, iter.p);
+ fts5ChunkIterNext(p, &iter);
+ }
+ }
+ fts5ChunkIterRelease(&iter);
}
}
-static void fts5BtreeIterNext(Fts5BtreeIter *pIter){
- Fts5Index *p = pIter->p;
- int i;
-
- assert( pIter->bEof==0 && pIter->aLvl[0].s.aData );
- for(i=0; i<pIter->nLvl && p->rc==SQLITE_OK; i++){
- Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
- fts5NodeIterNext(&p->rc, &pLvl->s);
- if( pLvl->s.aData ){
- fts5BufferSet(&p->rc, &pIter->term, pLvl->s.term.n, pLvl->s.term.p);
- break;
+static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
+ if( pIter->i<pIter->n ){
+ int bDummy;
+ if( pIter->i ){
+ i64 iDelta;
+ pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&iDelta);
+ if( pIter->bDesc ){
+ pIter->iRowid -= iDelta;
+ }else{
+ pIter->iRowid += iDelta;
+ }
}else{
- fts5NodeIterFree(&pLvl->s);
- fts5DataRelease(pLvl->pData);
- pLvl->pData = 0;
+ pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
}
- }
- if( i==pIter->nLvl || p->rc ){
- pIter->bEof = 1;
+ pIter->i += fts5GetPoslistSize(
+ &pIter->a[pIter->i], &pIter->nPoslist, &bDummy
+ );
+ pIter->aPoslist = &pIter->a[pIter->i];
+ pIter->i += pIter->nPoslist;
}else{
- int iSegid = pIter->pSeg->iSegid;
- for(i--; i>=0; i--){
- Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
- i64 iRowid = FTS5_SEGMENT_ROWID(pIter->iIdx,iSegid,i+1,pLvl[1].s.iChild);
- pLvl->pData = fts5DataRead(p, iRowid);
- if( pLvl->pData ){
- fts5NodeIterInit(pLvl->pData->p, pLvl->pData->n, &pLvl->s);
- }
- }
+ pIter->aPoslist = 0;
}
-
- pIter->nEmpty = pIter->aLvl[0].s.nEmpty;
- pIter->bDlidx = pIter->aLvl[0].s.bDlidx;
- pIter->iLeaf = pIter->aLvl[0].s.iChild;
}
-static void fts5BtreeIterFree(Fts5BtreeIter *pIter){
- int i;
- for(i=0; i<pIter->nLvl; i++){
- Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
- fts5NodeIterFree(&pLvl->s);
- if( pLvl->pData ){
- fts5DataRelease(pLvl->pData);
- pLvl->pData = 0;
- }
- }
- sqlite3_free(pIter->aLvl);
- fts5BufferFree(&pIter->term);
+static void fts5DoclistIterInit(
+ Fts5Buffer *pBuf,
+ int bDesc,
+ Fts5DoclistIter *pIter
+){
+ memset(pIter, 0, sizeof(*pIter));
+ pIter->a = pBuf->p;
+ pIter->n = pBuf->n;
+ pIter->bDesc = bDesc;
+ fts5DoclistIterNext(pIter);
}
/*
-** This function is purely an internal test. It does not contribute to
-** FTS functionality, or even the integrity-check, in any way.
-**
-** Instead, it tests that the same set of pgno/rowid combinations are
-** visited regardless of whether the doclist-index identified by parameters
-** iIdx/iSegid/iLeaf is iterated in forwards or reverse order.
+** Append a doclist to buffer pBuf.
*/
-#ifdef SQLITE_DEBUG
-static void fts5DlidxIterTestReverse(
- Fts5Index *p,
- int iIdx, /* Index to load doclist-index from */
- int iSegid, /* Segment id to load from */
- int iLeaf /* Load doclist-index for this leaf */
+static void fts5MergeAppendDocid(
+ int *pRc, /* IN/OUT: Error code */
+ int bDesc,
+ Fts5Buffer *pBuf, /* Buffer to write to */
+ i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */
+ i64 iRowid /* Rowid to append */
){
- Fts5DlidxIter *pDlidx = 0;
- i64 cksum1 = 13;
- i64 cksum2 = 13;
-
- for(pDlidx=fts5DlidxIterInit(p, 0, iIdx, iSegid, iLeaf);
- fts5DlidxIterEof(p, pDlidx)==0;
- fts5DlidxIterNext(pDlidx)
- ){
- assert( pDlidx->iLeafPgno>iLeaf );
- cksum1 = (cksum1 ^ ( (i64)(pDlidx->iLeafPgno) << 32 ));
- cksum1 = (cksum1 ^ pDlidx->iRowid);
- }
- fts5DlidxIterFree(pDlidx);
- pDlidx = 0;
-
- for(pDlidx=fts5DlidxIterInit(p, 1, iIdx, iSegid, iLeaf);
- fts5DlidxIterEof(p, pDlidx)==0;
- fts5DlidxIterPrev(pDlidx)
- ){
- assert( pDlidx->iLeafPgno>iLeaf );
- cksum2 = (cksum2 ^ ( (i64)(pDlidx->iLeafPgno) << 32 ));
- cksum2 = (cksum2 ^ pDlidx->iRowid);
+ if( pBuf->n==0 ){
+ fts5BufferAppendVarint(pRc, pBuf, iRowid);
+ }else if( bDesc ){
+ fts5BufferAppendVarint(pRc, pBuf, *piLastRowid - iRowid);
+ }else{
+ fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid);
}
- fts5DlidxIterFree(pDlidx);
- pDlidx = 0;
-
- if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT;
+ *piLastRowid = iRowid;
}
-#else
-# define fts5DlidxIterTestReverse(w,x,y,z)
-#endif
-static void fts5IndexIntegrityCheckSegment(
+/*
+** Buffers p1 and p2 contain doclists. This function merges the content
+** of the two doclists together and sets buffer p1 to the result before
+** returning.
+**
+** If an error occurs, an error code is left in p->rc. If an error has
+** already occurred, this function is a no-op.
+*/
+static void fts5MergePrefixLists(
Fts5Index *p, /* FTS5 backend object */
- int iIdx, /* Index that pSeg is a part of */
- Fts5StructureSegment *pSeg /* Segment to check internal consistency */
+ int bDesc,
+ Fts5Buffer *p1, /* First list to merge */
+ Fts5Buffer *p2 /* Second list to merge */
){
- Fts5BtreeIter iter; /* Used to iterate through b-tree hierarchy */
-
- /* Iterate through the b-tree hierarchy. */
- for(fts5BtreeIterInit(p, iIdx, pSeg, &iter);
- p->rc==SQLITE_OK && iter.bEof==0;
- fts5BtreeIterNext(&iter)
- ){
- i64 iRow; /* Rowid for this leaf */
- Fts5Data *pLeaf; /* Data for this leaf */
- int iOff; /* Offset of first term on leaf */
- int i; /* Used to iterate through empty leaves */
-
- /* If the leaf in question has already been trimmed from the segment,
- ** ignore this b-tree entry. Otherwise, load it into memory. */
- if( iter.iLeaf<pSeg->pgnoFirst ) continue;
- iRow = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, iter.iLeaf);
- pLeaf = fts5DataRead(p, iRow);
- if( pLeaf==0 ) break;
-
- /* Check that the leaf contains at least one term, and that it is equal
- ** to or larger than the split-key in iter.term. */
- iOff = fts5GetU16(&pLeaf->p[2]);
- if( iOff==0 ){
- p->rc = FTS5_CORRUPT;
- }else{
- int nTerm; /* Size of term on leaf in bytes */
- int res; /* Comparison of term and split-key */
- iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
- res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n));
- if( res==0 ) res = nTerm - iter.term.n;
- if( res<0 ){
- p->rc = FTS5_CORRUPT;
- }
- }
- fts5DataRelease(pLeaf);
- if( p->rc ) break;
-
- /* Now check that the iter.nEmpty leaves following the current leaf
- ** (a) exist and (b) contain no terms. */
- for(i=1; p->rc==SQLITE_OK && i<=iter.nEmpty; i++){
- pLeaf = fts5DataRead(p, iRow+i);
- if( pLeaf && 0!=fts5GetU16(&pLeaf->p[2]) ){
- p->rc = FTS5_CORRUPT;
- }
- fts5DataRelease(pLeaf);
- }
-
- /* If there is a doclist-index, check that it looks right. */
- if( iter.bDlidx ){
- Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */
- int iPrevLeaf = iter.iLeaf;
- int iSegid = pSeg->iSegid;
- int iPg;
- i64 iKey;
-
- for(pDlidx=fts5DlidxIterInit(p, 0, iIdx, iSegid, iter.iLeaf);
- fts5DlidxIterEof(p, pDlidx)==0;
- fts5DlidxIterNext(pDlidx)
- ){
-
- /* Check any rowid-less pages that occur before the current leaf. */
- for(iPg=iPrevLeaf+1; iPg<pDlidx->iLeafPgno; iPg++){
- iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, iPg);
- pLeaf = fts5DataRead(p, iKey);
- if( pLeaf ){
- if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT;
- fts5DataRelease(pLeaf);
- }
- }
- iPrevLeaf = pDlidx->iLeafPgno;
-
- /* Check that the leaf page indicated by the iterator really does
- ** contain the rowid suggested by the same. */
- iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, pDlidx->iLeafPgno);
- pLeaf = fts5DataRead(p, iKey);
- if( pLeaf ){
- i64 iRowid;
- int iRowidOff = fts5GetU16(&pLeaf->p[0]);
- getVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
- if( iRowid!=pDlidx->iRowid ) p->rc = FTS5_CORRUPT;
- fts5DataRelease(pLeaf);
- }
-
- }
-
- for(iPg=iPrevLeaf+1; iPg<=(iter.iLeaf + iter.nEmpty); iPg++){
- iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, iPg);
- pLeaf = fts5DataRead(p, iKey);
- if( pLeaf ){
- if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT;
- fts5DataRelease(pLeaf);
- }
- }
-
- fts5DlidxIterFree(pDlidx);
- fts5DlidxIterTestReverse(p, iIdx, iSegid, iter.iLeaf);
- }
- }
-
- /* Either iter.iLeaf must be the rightmost leaf-page in the segment, or
- ** else the segment has been completely emptied by an ongoing merge
- ** operation. */
- if( p->rc==SQLITE_OK
- && iter.iLeaf!=pSeg->pgnoLast
- && (pSeg->pgnoFirst || pSeg->pgnoLast)
- ){
- p->rc = FTS5_CORRUPT;
- }
-
- fts5BtreeIterFree(&iter);
-}
-
-/*
-** Iterator pMulti currently points to a valid entry (not EOF). This
-** function appends a copy of the position-list of the entry pMulti
-** currently points to to buffer pBuf.
-**
-** If an error occurs, an error code is left in p->rc. It is assumed
-** no error has already occurred when this function is called.
-*/
-static void fts5MultiIterPoslist(
- Fts5Index *p,
- Fts5MultiSegIter *pMulti,
- int bSz,
- Fts5Buffer *pBuf
-){
- if( p->rc==SQLITE_OK ){
- Fts5ChunkIter iter;
- Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
- assert( fts5MultiIterEof(p, pMulti)==0 );
- static int nCall = 0;
- nCall++;
-
- fts5ChunkIterInit(p, pSeg, &iter);
-
- if( fts5ChunkIterEof(p, &iter)==0 ){
- if( bSz ){
- /* WRITEPOSLISTSIZE */
- fts5BufferAppendVarint(&p->rc, pBuf, iter.nRem * 2);
- }
- while( fts5ChunkIterEof(p, &iter)==0 ){
- fts5BufferAppendBlob(&p->rc, pBuf, iter.n, iter.p);
- fts5ChunkIterNext(p, &iter);
- }
- }
- fts5ChunkIterRelease(&iter);
- }
-}
-
-static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
- if( pIter->i<pIter->n ){
- int bDummy;
- if( pIter->i ){
- i64 iDelta;
- pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&iDelta);
- if( pIter->bDesc ){
- pIter->iRowid -= iDelta;
- }else{
- pIter->iRowid += iDelta;
- }
- }else{
- pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
- }
- pIter->i += fts5GetPoslistSize(
- &pIter->a[pIter->i], &pIter->nPoslist, &bDummy
- );
- pIter->aPoslist = &pIter->a[pIter->i];
- pIter->i += pIter->nPoslist;
- }else{
- pIter->aPoslist = 0;
- }
-}
-
-static void fts5DoclistIterInit(
- Fts5Buffer *pBuf,
- int bDesc,
- Fts5DoclistIter *pIter
-){
- memset(pIter, 0, sizeof(*pIter));
- pIter->a = pBuf->p;
- pIter->n = pBuf->n;
- pIter->bDesc = bDesc;
- fts5DoclistIterNext(pIter);
-}
-
-/*
-** Append a doclist to buffer pBuf.
-*/
-static void fts5MergeAppendDocid(
- int *pRc, /* IN/OUT: Error code */
- int bDesc,
- Fts5Buffer *pBuf, /* Buffer to write to */
- i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */
- i64 iRowid /* Rowid to append */
-){
- if( pBuf->n==0 ){
- fts5BufferAppendVarint(pRc, pBuf, iRowid);
- }else if( bDesc ){
- fts5BufferAppendVarint(pRc, pBuf, *piLastRowid - iRowid);
- }else{
- fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid);
- }
- *piLastRowid = iRowid;
-}
-
-/*
-** Buffers p1 and p2 contain doclists. This function merges the content
-** of the two doclists together and sets buffer p1 to the result before
-** returning.
-**
-** If an error occurs, an error code is left in p->rc. If an error has
-** already occurred, this function is a no-op.
-*/
-static void fts5MergePrefixLists(
- Fts5Index *p, /* FTS5 backend object */
- int bDesc,
- Fts5Buffer *p1, /* First list to merge */
- Fts5Buffer *p2 /* Second list to merge */
-){
- if( p2->n ){
- i64 iLastRowid = 0;
- Fts5DoclistIter i1;
- Fts5DoclistIter i2;
- Fts5Buffer out;
- Fts5Buffer tmp;
- memset(&out, 0, sizeof(out));
- memset(&tmp, 0, sizeof(tmp));
+ if( p2->n ){
+ i64 iLastRowid = 0;
+ Fts5DoclistIter i1;
+ Fts5DoclistIter i2;
+ Fts5Buffer out;
+ Fts5Buffer tmp;
+ memset(&out, 0, sizeof(out));
+ memset(&tmp, 0, sizeof(tmp));
fts5DoclistIterInit(p1, bDesc, &i1);
fts5DoclistIterInit(p2, bDesc, &i2);
sqlite3_free(aBuf);
}
-static int fts5QueryCksum(
- Fts5Index *p,
- const char *z,
- int n,
- int flags,
- u64 *pCksum
-){
- u64 cksum = *pCksum;
- Fts5IndexIter *pIdxIter = 0;
- int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter);
- while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
- const u8 *pPos;
- int nPos;
- i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
- rc = sqlite3Fts5IterPoslist(pIdxIter, &pPos, &nPos);
+/*
+** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
+** to the document with rowid iRowid.
+*/
+int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){
+ assert( p->rc==SQLITE_OK );
+
+ /* Allocate hash tables if they have not already been allocated */
+ if( p->apHash==0 ){
+ int i;
+ int rc = SQLITE_OK;
+ int nHash = p->pConfig->nPrefix + 1;
+ Fts5Hash **apNew;
+
+ apNew = (Fts5Hash**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Hash*)*nHash);
+ for(i=0; rc==SQLITE_OK && i<nHash; i++){
+ rc = sqlite3Fts5HashNew(&apNew[i], &p->nPendingData);
+ }
if( rc==SQLITE_OK ){
- Fts5PoslistReader sReader;
- for(sqlite3Fts5PoslistReaderInit(-1, pPos, nPos, &sReader);
- sReader.bEof==0;
- sqlite3Fts5PoslistReaderNext(&sReader)
- ){
- int iCol = FTS5_POS2COLUMN(sReader.iPos);
- int iOff = FTS5_POS2OFFSET(sReader.iPos);
- cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, z, n);
+ p->apHash = apNew;
+ }else{
+ if( apNew ){
+ for(i=0; i<nHash; i++){
+ sqlite3Fts5HashFree(apNew[i]);
+ }
+ sqlite3_free(apNew);
}
- rc = sqlite3Fts5IterNext(pIdxIter);
+ return rc;
}
}
- sqlite3Fts5IterClose(pIdxIter);
- *pCksum = cksum;
- return rc;
-}
-
-/*
-** Run internal checks to ensure that the FTS index (a) is internally
-** consistent and (b) contains entries for which the XOR of the checksums
-** as calculated by fts5IndexEntryCksum() is cksum.
-**
-** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
-** checksum does not match. Return SQLITE_OK if all checks pass without
-** error, or some other SQLite error code if another error (e.g. OOM)
-** occurs.
-*/
-int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
- Fts5Config *pConfig = p->pConfig;
- int iIdx; /* Used to iterate through indexes */
- u64 cksum2 = 0; /* Checksum based on contents of indexes */
- u64 cksum3 = 0; /* Checksum based on contents of indexes */
- Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */
- Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */
-
- /* Check that the internal nodes of each segment match the leaves */
- for(iIdx=0; p->rc==SQLITE_OK && iIdx<=pConfig->nPrefix; iIdx++){
- Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
- if( pStruct ){
- int iLvl, iSeg;
- for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
- for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
- Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
- fts5IndexIntegrityCheckSegment(p, iIdx, pSeg);
- }
- }
- }
- fts5StructureRelease(pStruct);
- }
-
- /* The cksum argument passed to this function is a checksum calculated
- ** based on all expected entries in the FTS index (including prefix index
- ** entries). This block checks that a checksum calculated based on the
- ** actual contents of FTS index is identical.
- **
- ** Two versions of the same checksum are calculated. The first (stack
- ** variable cksum2) based on entries extracted from the full-text index
- ** while doing a linear scan of each individual index in turn.
- **
- ** As each term visited by the linear scans, a separate query for the
- ** same term is performed. cksum3 is calculated based on the entries
- ** extracted by these queries.
- */
- for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
- Fts5MultiSegIter *pIter;
- Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
- for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, -1, 0, &pIter);
- fts5MultiIterEof(p, pIter)==0;
- fts5MultiIterNext(p, pIter, 0, 0)
- ){
- int n; /* Size of term in bytes */
- i64 iPos = 0; /* Position read from poslist */
- int iOff = 0; /* Offset within poslist */
- i64 iRowid = fts5MultiIterRowid(pIter);
- char *z = (char*)fts5MultiIterTerm(pIter, &n);
-
- poslist.n = 0;
- fts5MultiIterPoslist(p, pIter, 0, &poslist);
- while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
- int iCol = FTS5_POS2COLUMN(iPos);
- int iTokOff = FTS5_POS2OFFSET(iPos);
- cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, z, n);
- }
-
- /* If this is a new term, query for it. Update cksum3 with the results. */
- if( p->rc==SQLITE_OK && (term.n!=n || memcmp(term.p, z, n)) ){
- int rc;
- int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX);
- u64 ck1 = 0;
- u64 ck2 = 0;
-
- /* Check that the results returned for ASC and DESC queries are
- ** the same. If not, call this corruption. */
- rc = fts5QueryCksum(p, z, n, flags, &ck1);
- if( rc==SQLITE_OK ){
- rc = fts5QueryCksum(p, z, n, flags|FTS5INDEX_QUERY_DESC, &ck2);
- }
- if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
-
- /* If this is a prefix query, check that the results returned if the
- ** the index is disabled are the same. In both ASC and DESC order. */
- if( iIdx>0 && rc==SQLITE_OK ){
- int f = flags|FTS5INDEX_QUERY_TEST_NOIDX;
- ck2 = 0;
- rc = fts5QueryCksum(p, z, n, f, &ck2);
- if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
- }
- if( iIdx>0 && rc==SQLITE_OK ){
- int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC;
- ck2 = 0;
- rc = fts5QueryCksum(p, z, n, f, &ck2);
- if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
- }
-
- cksum3 ^= ck1;
- fts5BufferSet(&rc, &term, n, (const u8*)z);
- p->rc = rc;
- }
- }
- fts5MultiIterFree(p, pIter);
- fts5StructureRelease(pStruct);
- }
- if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;
- if( p->rc==SQLITE_OK && cksum!=cksum3 ) p->rc = FTS5_CORRUPT;
-
- fts5BufferFree(&term);
- fts5BufferFree(&poslist);
- return fts5IndexReturn(p);
-}
-
-
-/*
-** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
-** to the document with rowid iRowid.
-*/
-int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){
- assert( p->rc==SQLITE_OK );
-
- /* Allocate hash tables if they have not already been allocated */
- if( p->apHash==0 ){
- int i;
- int rc = SQLITE_OK;
- int nHash = p->pConfig->nPrefix + 1;
- Fts5Hash **apNew;
-
- apNew = (Fts5Hash**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Hash*)*nHash);
- for(i=0; rc==SQLITE_OK && i<nHash; i++){
- rc = sqlite3Fts5HashNew(&apNew[i], &p->nPendingData);
- }
- if( rc==SQLITE_OK ){
- p->apHash = apNew;
- }else{
- if( apNew ){
- for(i=0; i<nHash; i++){
- sqlite3Fts5HashFree(apNew[i]);
- }
- sqlite3_free(apNew);
- }
- return rc;
- }
- }
-
- if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){
- fts5IndexFlush(p);
- }
- p->iWriteRowid = iRowid;
- return fts5IndexReturn(p);
+ if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){
+ fts5IndexFlush(p);
+ }
+ p->iWriteRowid = iRowid;
+ return fts5IndexReturn(p);
}
/*
return nChar;
}
-/*
-** Calculate and return a checksum that is the XOR of the index entry
-** checksum of all entries that would be generated by the token specified
-** by the final 5 arguments.
-*/
-u64 sqlite3Fts5IndexCksum(
- Fts5Config *pConfig, /* Configuration object */
- i64 iRowid, /* Document term appears in */
- int iCol, /* Column term appears in */
- int iPos, /* Position term appears in */
- const char *pTerm, int nTerm /* Term at iPos */
-){
- u64 ret = 0; /* Return value */
- int iIdx; /* For iterating through indexes */
-
- ret = fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, nTerm);
-
- for(iIdx=0; iIdx<pConfig->nPrefix; iIdx++){
- int nByte = fts5IndexCharlenToBytelen(pTerm, nTerm, pConfig->aPrefix[iIdx]);
- if( nByte ){
- ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, nByte);
- }
- }
-
- return ret;
-}
-
/*
** Insert or remove data to or from the index. Each time a document is
** added to or removed from the index, this function is called one or more
return fts5IndexReturn(p);
}
+
+/*************************************************************************
+**************************************************************************
+** Below this point is the implementation of the integrity-check
+** functionality.
+*/
+
+/*
+** Return a simple checksum value based on the arguments.
+*/
+static u64 fts5IndexEntryCksum(
+ i64 iRowid,
+ int iCol,
+ int iPos,
+ const char *pTerm,
+ int nTerm
+){
+ int i;
+ u64 ret = iRowid;
+ ret += (ret<<3) + iCol;
+ ret += (ret<<3) + iPos;
+ for(i=0; i<nTerm; i++) ret += (ret<<3) + pTerm[i];
+ return ret;
+}
+
+static void fts5BtreeIterInit(
+ Fts5Index *p,
+ int iIdx,
+ Fts5StructureSegment *pSeg,
+ Fts5BtreeIter *pIter
+){
+ int nByte;
+ int i;
+ nByte = sizeof(pIter->aLvl[0]) * (pSeg->nHeight-1);
+ memset(pIter, 0, sizeof(*pIter));
+ if( nByte ){
+ pIter->aLvl = (Fts5BtreeIterLevel*)fts5IdxMalloc(p, nByte);
+ }
+ if( p->rc==SQLITE_OK ){
+ pIter->nLvl = pSeg->nHeight-1;
+ pIter->iIdx = iIdx;
+ pIter->p = p;
+ pIter->pSeg = pSeg;
+ }
+ for(i=0; p->rc==SQLITE_OK && i<pIter->nLvl; i++){
+ i64 iRowid = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, i+1, 1);
+ Fts5Data *pData;
+ pIter->aLvl[i].pData = pData = fts5DataRead(p, iRowid);
+ if( pData ){
+ fts5NodeIterInit(pData->p, pData->n, &pIter->aLvl[i].s);
+ }
+ }
+
+ if( pIter->nLvl==0 || p->rc ){
+ pIter->bEof = 1;
+ pIter->iLeaf = pSeg->pgnoLast;
+ }else{
+ pIter->nEmpty = pIter->aLvl[0].s.nEmpty;
+ pIter->iLeaf = pIter->aLvl[0].s.iChild;
+ pIter->bDlidx = pIter->aLvl[0].s.bDlidx;
+ }
+}
+
+static void fts5BtreeIterNext(Fts5BtreeIter *pIter){
+ Fts5Index *p = pIter->p;
+ int i;
+
+ assert( pIter->bEof==0 && pIter->aLvl[0].s.aData );
+ for(i=0; i<pIter->nLvl && p->rc==SQLITE_OK; i++){
+ Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
+ fts5NodeIterNext(&p->rc, &pLvl->s);
+ if( pLvl->s.aData ){
+ fts5BufferSet(&p->rc, &pIter->term, pLvl->s.term.n, pLvl->s.term.p);
+ break;
+ }else{
+ fts5NodeIterFree(&pLvl->s);
+ fts5DataRelease(pLvl->pData);
+ pLvl->pData = 0;
+ }
+ }
+ if( i==pIter->nLvl || p->rc ){
+ pIter->bEof = 1;
+ }else{
+ int iSegid = pIter->pSeg->iSegid;
+ for(i--; i>=0; i--){
+ Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
+ i64 iRowid = FTS5_SEGMENT_ROWID(pIter->iIdx,iSegid,i+1,pLvl[1].s.iChild);
+ pLvl->pData = fts5DataRead(p, iRowid);
+ if( pLvl->pData ){
+ fts5NodeIterInit(pLvl->pData->p, pLvl->pData->n, &pLvl->s);
+ }
+ }
+ }
+
+ pIter->nEmpty = pIter->aLvl[0].s.nEmpty;
+ pIter->bDlidx = pIter->aLvl[0].s.bDlidx;
+ pIter->iLeaf = pIter->aLvl[0].s.iChild;
+}
+
+static void fts5BtreeIterFree(Fts5BtreeIter *pIter){
+ int i;
+ for(i=0; i<pIter->nLvl; i++){
+ Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
+ fts5NodeIterFree(&pLvl->s);
+ if( pLvl->pData ){
+ fts5DataRelease(pLvl->pData);
+ pLvl->pData = 0;
+ }
+ }
+ sqlite3_free(pIter->aLvl);
+ fts5BufferFree(&pIter->term);
+}
+
+/*
+** This function is purely an internal test. It does not contribute to
+** FTS functionality, or even the integrity-check, in any way.
+**
+** Instead, it tests that the same set of pgno/rowid combinations are
+** visited regardless of whether the doclist-index identified by parameters
+** iIdx/iSegid/iLeaf is iterated in forwards or reverse order.
+*/
+#ifdef SQLITE_DEBUG
+static void fts5DlidxIterTestReverse(
+ Fts5Index *p,
+ int iIdx, /* Index to load doclist-index from */
+ int iSegid, /* Segment id to load from */
+ int iLeaf /* Load doclist-index for this leaf */
+){
+ Fts5DlidxIter *pDlidx = 0;
+ i64 cksum1 = 13;
+ i64 cksum2 = 13;
+
+ for(pDlidx=fts5DlidxIterInit(p, 0, iIdx, iSegid, iLeaf);
+ fts5DlidxIterEof(p, pDlidx)==0;
+ fts5DlidxIterNext(pDlidx)
+ ){
+ assert( pDlidx->iLeafPgno>iLeaf );
+ cksum1 = (cksum1 ^ ( (i64)(pDlidx->iLeafPgno) << 32 ));
+ cksum1 = (cksum1 ^ pDlidx->iRowid);
+ }
+ fts5DlidxIterFree(pDlidx);
+ pDlidx = 0;
+
+ for(pDlidx=fts5DlidxIterInit(p, 1, iIdx, iSegid, iLeaf);
+ fts5DlidxIterEof(p, pDlidx)==0;
+ fts5DlidxIterPrev(pDlidx)
+ ){
+ assert( pDlidx->iLeafPgno>iLeaf );
+ cksum2 = (cksum2 ^ ( (i64)(pDlidx->iLeafPgno) << 32 ));
+ cksum2 = (cksum2 ^ pDlidx->iRowid);
+ }
+ fts5DlidxIterFree(pDlidx);
+ pDlidx = 0;
+
+ if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT;
+}
+#else
+# define fts5DlidxIterTestReverse(w,x,y,z)
+#endif
+
+static void fts5IndexIntegrityCheckSegment(
+ Fts5Index *p, /* FTS5 backend object */
+ int iIdx, /* Index that pSeg is a part of */
+ Fts5StructureSegment *pSeg /* Segment to check internal consistency */
+){
+ Fts5BtreeIter iter; /* Used to iterate through b-tree hierarchy */
+
+ /* Iterate through the b-tree hierarchy. */
+ for(fts5BtreeIterInit(p, iIdx, pSeg, &iter);
+ p->rc==SQLITE_OK && iter.bEof==0;
+ fts5BtreeIterNext(&iter)
+ ){
+ i64 iRow; /* Rowid for this leaf */
+ Fts5Data *pLeaf; /* Data for this leaf */
+ int iOff; /* Offset of first term on leaf */
+ int i; /* Used to iterate through empty leaves */
+
+ /* If the leaf in question has already been trimmed from the segment,
+ ** ignore this b-tree entry. Otherwise, load it into memory. */
+ if( iter.iLeaf<pSeg->pgnoFirst ) continue;
+ iRow = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, iter.iLeaf);
+ pLeaf = fts5DataRead(p, iRow);
+ if( pLeaf==0 ) break;
+
+ /* Check that the leaf contains at least one term, and that it is equal
+ ** to or larger than the split-key in iter.term. */
+ iOff = fts5GetU16(&pLeaf->p[2]);
+ if( iOff==0 ){
+ p->rc = FTS5_CORRUPT;
+ }else{
+ int nTerm; /* Size of term on leaf in bytes */
+ int res; /* Comparison of term and split-key */
+ iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
+ res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n));
+ if( res==0 ) res = nTerm - iter.term.n;
+ if( res<0 ){
+ p->rc = FTS5_CORRUPT;
+ }
+ }
+ fts5DataRelease(pLeaf);
+ if( p->rc ) break;
+
+ /* Now check that the iter.nEmpty leaves following the current leaf
+ ** (a) exist and (b) contain no terms. */
+ for(i=1; p->rc==SQLITE_OK && i<=iter.nEmpty; i++){
+ pLeaf = fts5DataRead(p, iRow+i);
+ if( pLeaf && 0!=fts5GetU16(&pLeaf->p[2]) ){
+ p->rc = FTS5_CORRUPT;
+ }
+ fts5DataRelease(pLeaf);
+ }
+
+ /* If there is a doclist-index, check that it looks right. */
+ if( iter.bDlidx ){
+ Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */
+ int iPrevLeaf = iter.iLeaf;
+ int iSegid = pSeg->iSegid;
+ int iPg;
+ i64 iKey;
+
+ for(pDlidx=fts5DlidxIterInit(p, 0, iIdx, iSegid, iter.iLeaf);
+ fts5DlidxIterEof(p, pDlidx)==0;
+ fts5DlidxIterNext(pDlidx)
+ ){
+
+ /* Check any rowid-less pages that occur before the current leaf. */
+ for(iPg=iPrevLeaf+1; iPg<pDlidx->iLeafPgno; iPg++){
+ iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, iPg);
+ pLeaf = fts5DataRead(p, iKey);
+ if( pLeaf ){
+ if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT;
+ fts5DataRelease(pLeaf);
+ }
+ }
+ iPrevLeaf = pDlidx->iLeafPgno;
+
+ /* Check that the leaf page indicated by the iterator really does
+ ** contain the rowid suggested by the same. */
+ iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, pDlidx->iLeafPgno);
+ pLeaf = fts5DataRead(p, iKey);
+ if( pLeaf ){
+ i64 iRowid;
+ int iRowidOff = fts5GetU16(&pLeaf->p[0]);
+ getVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
+ if( iRowid!=pDlidx->iRowid ) p->rc = FTS5_CORRUPT;
+ fts5DataRelease(pLeaf);
+ }
+
+ }
+
+ for(iPg=iPrevLeaf+1; iPg<=(iter.iLeaf + iter.nEmpty); iPg++){
+ iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, iPg);
+ pLeaf = fts5DataRead(p, iKey);
+ if( pLeaf ){
+ if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT;
+ fts5DataRelease(pLeaf);
+ }
+ }
+
+ fts5DlidxIterFree(pDlidx);
+ fts5DlidxIterTestReverse(p, iIdx, iSegid, iter.iLeaf);
+ }
+ }
+
+ /* Either iter.iLeaf must be the rightmost leaf-page in the segment, or
+ ** else the segment has been completely emptied by an ongoing merge
+ ** operation. */
+ if( p->rc==SQLITE_OK
+ && iter.iLeaf!=pSeg->pgnoLast
+ && (pSeg->pgnoFirst || pSeg->pgnoLast)
+ ){
+ p->rc = FTS5_CORRUPT;
+ }
+
+ fts5BtreeIterFree(&iter);
+}
+
+
+static int fts5QueryCksum(
+ Fts5Index *p,
+ const char *z,
+ int n,
+ int flags,
+ u64 *pCksum
+){
+ u64 cksum = *pCksum;
+ Fts5IndexIter *pIdxIter = 0;
+ int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter);
+
+ while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
+ const u8 *pPos;
+ int nPos;
+ i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
+ rc = sqlite3Fts5IterPoslist(pIdxIter, &pPos, &nPos);
+ if( rc==SQLITE_OK ){
+ Fts5PoslistReader sReader;
+ for(sqlite3Fts5PoslistReaderInit(-1, pPos, nPos, &sReader);
+ sReader.bEof==0;
+ sqlite3Fts5PoslistReaderNext(&sReader)
+ ){
+ int iCol = FTS5_POS2COLUMN(sReader.iPos);
+ int iOff = FTS5_POS2OFFSET(sReader.iPos);
+ cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, z, n);
+ }
+ rc = sqlite3Fts5IterNext(pIdxIter);
+ }
+ }
+ sqlite3Fts5IterClose(pIdxIter);
+
+ *pCksum = cksum;
+ return rc;
+}
+
+/*
+** Run internal checks to ensure that the FTS index (a) is internally
+** consistent and (b) contains entries for which the XOR of the checksums
+** as calculated by fts5IndexEntryCksum() is cksum.
+**
+** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
+** checksum does not match. Return SQLITE_OK if all checks pass without
+** error, or some other SQLite error code if another error (e.g. OOM)
+** occurs.
+*/
+int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
+ Fts5Config *pConfig = p->pConfig;
+ int iIdx; /* Used to iterate through indexes */
+ u64 cksum2 = 0; /* Checksum based on contents of indexes */
+ u64 cksum3 = 0; /* Checksum based on contents of indexes */
+ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */
+ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */
+
+ /* Check that the internal nodes of each segment match the leaves */
+ for(iIdx=0; p->rc==SQLITE_OK && iIdx<=pConfig->nPrefix; iIdx++){
+ Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
+ if( pStruct ){
+ int iLvl, iSeg;
+ for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
+ for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
+ Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
+ fts5IndexIntegrityCheckSegment(p, iIdx, pSeg);
+ }
+ }
+ }
+ fts5StructureRelease(pStruct);
+ }
+
+ /* The cksum argument passed to this function is a checksum calculated
+ ** based on all expected entries in the FTS index (including prefix index
+ ** entries). This block checks that a checksum calculated based on the
+ ** actual contents of FTS index is identical.
+ **
+ ** Two versions of the same checksum are calculated. The first (stack
+ ** variable cksum2) based on entries extracted from the full-text index
+ ** while doing a linear scan of each individual index in turn.
+ **
+ ** As each term visited by the linear scans, a separate query for the
+ ** same term is performed. cksum3 is calculated based on the entries
+ ** extracted by these queries.
+ */
+ for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
+ Fts5MultiSegIter *pIter;
+ Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
+ for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, -1, 0, &pIter);
+ fts5MultiIterEof(p, pIter)==0;
+ fts5MultiIterNext(p, pIter, 0, 0)
+ ){
+ int n; /* Size of term in bytes */
+ i64 iPos = 0; /* Position read from poslist */
+ int iOff = 0; /* Offset within poslist */
+ i64 iRowid = fts5MultiIterRowid(pIter);
+ char *z = (char*)fts5MultiIterTerm(pIter, &n);
+
+ poslist.n = 0;
+ fts5MultiIterPoslist(p, pIter, 0, &poslist);
+ while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
+ int iCol = FTS5_POS2COLUMN(iPos);
+ int iTokOff = FTS5_POS2OFFSET(iPos);
+ cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, z, n);
+ }
+
+ /* If this is a new term, query for it. Update cksum3 with the results. */
+ if( p->rc==SQLITE_OK && (term.n!=n || memcmp(term.p, z, n)) ){
+ int rc;
+ int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX);
+ u64 ck1 = 0;
+ u64 ck2 = 0;
+
+ /* Check that the results returned for ASC and DESC queries are
+ ** the same. If not, call this corruption. */
+ rc = fts5QueryCksum(p, z, n, flags, &ck1);
+ if( rc==SQLITE_OK ){
+ rc = fts5QueryCksum(p, z, n, flags|FTS5INDEX_QUERY_DESC, &ck2);
+ }
+ if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
+
+ /* If this is a prefix query, check that the results returned if the
+ ** the index is disabled are the same. In both ASC and DESC order. */
+ if( iIdx>0 && rc==SQLITE_OK ){
+ int f = flags|FTS5INDEX_QUERY_TEST_NOIDX;
+ ck2 = 0;
+ rc = fts5QueryCksum(p, z, n, f, &ck2);
+ if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
+ }
+ if( iIdx>0 && rc==SQLITE_OK ){
+ int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC;
+ ck2 = 0;
+ rc = fts5QueryCksum(p, z, n, f, &ck2);
+ if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
+ }
+
+ cksum3 ^= ck1;
+ fts5BufferSet(&rc, &term, n, (const u8*)z);
+ p->rc = rc;
+ }
+ }
+ fts5MultiIterFree(p, pIter);
+ fts5StructureRelease(pStruct);
+ }
+ if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;
+ if( p->rc==SQLITE_OK && cksum!=cksum3 ) p->rc = FTS5_CORRUPT;
+
+ fts5BufferFree(&term);
+ fts5BufferFree(&poslist);
+ return fts5IndexReturn(p);
+}
+
+
+/*
+** Calculate and return a checksum that is the XOR of the index entry
+** checksum of all entries that would be generated by the token specified
+** by the final 5 arguments.
+*/
+u64 sqlite3Fts5IndexCksum(
+ Fts5Config *pConfig, /* Configuration object */
+ i64 iRowid, /* Document term appears in */
+ int iCol, /* Column term appears in */
+ int iPos, /* Position term appears in */
+ const char *pTerm, int nTerm /* Term at iPos */
+){
+ u64 ret = 0; /* Return value */
+ int iIdx; /* For iterating through indexes */
+
+ ret = fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, nTerm);
+
+ for(iIdx=0; iIdx<pConfig->nPrefix; iIdx++){
+ int nByte = fts5IndexCharlenToBytelen(pTerm, nTerm, pConfig->aPrefix[iIdx]);
+ if( nByte ){
+ ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, nByte);
+ }
+ }
+
+ return ret;
+}
+
/*************************************************************************
**************************************************************************
** Below this point is the implementation of the fts5_decode() scalar
projects / thirdparty / sqlite.git / commitdiff
