sqlite3_stmt *pIdxSelect;
int nRead; /* Total number of blocks read */
+ sqlite3_stmt *pDeleteFromIdx;
+
sqlite3_stmt *pDataVersion;
i64 iStructVersion; /* data_version when pStruct read */
Fts5Structure *pStruct; /* Current db structure (or NULL) */
** iLeafOffset:
** Byte offset within the current leaf that is the first byte of the
** position list data (one byte passed the position-list size field).
-** rowid field of the current entry. Usually this is the size field of the
-** position list data. The exception is if the rowid for the current entry
-** is the last thing on the leaf page.
**
** pLeaf:
** Buffer containing current leaf page data. Set to NULL at EOF.
pLvl->bEof = 1;
}else{
u8 *a = pLvl->pData->p;
- i64 iVal;
- int iLimit;
- int ii;
- int nZero = 0;
-
- /* Currently iOff points to the first byte of a varint. This block
- ** decrements iOff until it points to the first byte of the previous
- ** varint. Taking care not to read any memory locations that occur
- ** before the buffer in memory. */
- iLimit = (iOff>9 ? iOff-9 : 0);
- for(iOff--; iOff>iLimit; iOff--){
- if( (a[iOff-1] & 0x80)==0 ) break;
- }
-
- fts5GetVarint(&a[iOff], (u64*)&iVal);
- pLvl->iRowid -= iVal;
- pLvl->iLeafPgno--;
-
- /* Skip backwards past any 0x00 varints. */
- for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){
- nZero++;
- }
- if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){
- /* The byte immediately before the last 0x00 byte has the 0x80 bit
- ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80
- ** bytes before a[ii]. */
- int bZero = 0; /* True if last 0x00 counts */
- if( (ii-8)>=pLvl->iFirstOff ){
- int j;
- for(j=1; j<=8 && (a[ii-j] & 0x80); j++);
- bZero = (j>8);
+
+ pLvl->iOff = 0;
+ fts5DlidxLvlNext(pLvl);
+ while( 1 ){
+ int nZero = 0;
+ int ii = pLvl->iOff;
+ u64 delta = 0;
+
+ while( a[ii]==0 ){
+ nZero++;
+ ii++;
}
- if( bZero==0 ) nZero--;
+ ii += sqlite3Fts5GetVarint(&a[ii], &delta);
+
+ if( ii>=iOff ) break;
+ pLvl->iLeafPgno += nZero+1;
+ pLvl->iRowid += delta;
+ pLvl->iOff = ii;
}
- pLvl->iLeafPgno -= nZero;
- pLvl->iOff = iOff - nZero;
}
return pLvl->bEof;
i64 iOff = pIter->iLeafOffset;
ASSERT_SZLEAF_OK(pIter->pLeaf);
- if( iOff>=pIter->pLeaf->szLeaf ){
+ while( iOff>=pIter->pLeaf->szLeaf ){
fts5SegIterNextPage(p, pIter);
if( pIter->pLeaf==0 ){
if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT;
fts5SegIterSetNext(p, pIter);
pIter->pSeg = pSeg;
pIter->iLeafPgno = pSeg->pgnoFirst-1;
- fts5SegIterNextPage(p, pIter);
+ do {
+ fts5SegIterNextPage(p, pIter);
+ }while( p->rc==SQLITE_OK && pIter->pLeaf && pIter->pLeaf->nn==4 );
}
- if( p->rc==SQLITE_OK ){
+ if( p->rc==SQLITE_OK && pIter->pLeaf ){
pIter->iLeafOffset = 4;
assert( pIter->pLeaf!=0 );
assert_nc( pIter->pLeaf->nn>4 );
Fts5Data *pLast = 0;
int pgnoLast = 0;
- if( pDlidx ){
+ if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION ){
int iSegid = pIter->pSeg->iSegid;
pgnoLast = fts5DlidxIterPgno(pDlidx);
pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast));
/*
** Move the seg-iter so that it points to the first rowid on page iLeafPgno.
-** It is an error if leaf iLeafPgno does not exist or contains no rowids.
+** It is an error if leaf iLeafPgno does not exist. Unless the db is
+** a 'secure-delete' db, if it contains no rowids then this is also an error.
*/
static void fts5SegIterGotoPage(
Fts5Index *p, /* FTS5 backend object */
fts5DataRelease(pIter->pNextLeaf);
pIter->pNextLeaf = 0;
pIter->iLeafPgno = iLeafPgno-1;
- fts5SegIterNextPage(p, pIter);
- assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno );
- if( p->rc==SQLITE_OK && ALWAYS(pIter->pLeaf!=0) ){
+ while( p->rc==SQLITE_OK ){
int iOff;
- u8 *a = pIter->pLeaf->p;
- int n = pIter->pLeaf->szLeaf;
-
+ fts5SegIterNextPage(p, pIter);
+ if( pIter->pLeaf==0 ) break;
iOff = fts5LeafFirstRowidOff(pIter->pLeaf);
- if( iOff<4 || iOff>=n ){
- p->rc = FTS5_CORRUPT;
- }else{
- iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
- pIter->iLeafOffset = iOff;
- fts5SegIterLoadNPos(p, pIter);
+ if( iOff>0 ){
+ u8 *a = pIter->pLeaf->p;
+ int n = pIter->pLeaf->szLeaf;
+ if( iOff<4 || iOff>=n ){
+ p->rc = FTS5_CORRUPT;
+ }else{
+ iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
+ pIter->iLeafOffset = iOff;
+ fts5SegIterLoadNPos(p, pIter);
+ }
+ break;
}
}
}
if( iLevel<0 ){
assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
nSeg = pStruct->nSegment;
- nSeg += (p->pHash ? 1 : 0);
+ nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH));
}else{
nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
}
if( p->rc==SQLITE_OK ){
if( iLevel<0 ){
Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
- if( p->pHash ){
+ if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH) ){
/* Add a segment iterator for the current contents of the hash table. */
Fts5SegIter *pIter = &pNew->aSeg[iIter++];
fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter);
fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr);
fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n);
fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p);
- fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff,&pData->p[iOff]);
+ fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff]);
if( p->rc==SQLITE_OK ){
/* Set the szLeaf field */
fts5PutU16(&buf.p[2], (u16)buf.n);
return ret;
}
+/*
+** Execute the SQL statement:
+**
+** DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno);
+**
+** This is used when a secure-delete operation removes the last term
+** from a segment leaf page. In that case the %_idx entry is removed
+** too. This is done to ensure that if all instances of a token are
+** removed from an fts5 database in secure-delete mode, no trace of
+** the token itself remains in the database.
+*/
+static void fts5SecureDeleteIdxEntry(
+ Fts5Index *p, /* FTS5 backend object */
+ int iSegid, /* Id of segment to delete entry for */
+ int iPgno /* Page number within segment */
+){
+ if( iPgno!=1 ){
+ assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE );
+ if( p->pDeleteFromIdx==0 ){
+ fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintf(
+ "DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)",
+ p->pConfig->zDb, p->pConfig->zName
+ ));
+ }
+ if( p->rc==SQLITE_OK ){
+ sqlite3_bind_int(p->pDeleteFromIdx, 1, iSegid);
+ sqlite3_bind_int(p->pDeleteFromIdx, 2, iPgno);
+ sqlite3_step(p->pDeleteFromIdx);
+ p->rc = sqlite3_reset(p->pDeleteFromIdx);
+ }
+ }
+}
+
+/*
+** This is called when a secure-delete operation removes a position-list
+** that overflows onto segment page iPgno of segment pSeg. This function
+** rewrites node iPgno, and possibly one or more of its right-hand peers,
+** to remove this portion of the position list.
+**
+** Output variable (*pbLastInDoclist) is set to true if the position-list
+** removed is followed by a new term or the end-of-segment, or false if
+** it is followed by another rowid/position list.
+*/
+static void fts5SecureDeleteOverflow(
+ Fts5Index *p,
+ Fts5StructureSegment *pSeg,
+ int iPgno,
+ int *pbLastInDoclist
+){
+ int pgno;
+ Fts5Data *pLeaf = 0;
+ assert( iPgno!=1 );
+
+ *pbLastInDoclist = 1;
+ for(pgno=iPgno; p->rc==SQLITE_OK && pgno<=pSeg->pgnoLast; pgno++){
+ i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno);
+ int iNext = 0;
+ u8 *aPg = 0;
+
+ pLeaf = fts5DataRead(p, iRowid);
+ if( pLeaf==0 ) break;
+ aPg = pLeaf->p;
+
+ iNext = fts5GetU16(&aPg[0]);
+ if( iNext!=0 ){
+ *pbLastInDoclist = 0;
+ }
+ if( iNext==0 && pLeaf->szLeaf!=pLeaf->nn ){
+ fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext);
+ }
+
+ if( iNext==0 ){
+ /* The page contains no terms or rowids. Replace it with an empty
+ ** page and move on to the right-hand peer. */
+ const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04};
+ fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty));
+ fts5DataRelease(pLeaf);
+ pLeaf = 0;
+ }else{
+ int nShift = iNext - 4;
+ int nPg;
+
+ int nIdx = 0;
+ u8 *aIdx = 0;
+
+ /* Unless the current page footer is 0 bytes in size (in which case
+ ** the new page footer will be as well), allocate and populate a
+ ** buffer containing the new page footer. Set stack variables aIdx
+ ** and nIdx accordingly. */
+ if( pLeaf->nn>pLeaf->szLeaf ){
+ int iFirst = 0;
+ int i1 = pLeaf->szLeaf;
+ int i2 = 0;
+
+ aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2);
+ if( aIdx==0 ) break;
+ i1 += fts5GetVarint32(&aPg[i1], iFirst);
+ i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift);
+ if( i1<pLeaf->nn ){
+ memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1);
+ i2 += (pLeaf->nn-i1);
+ }
+ nIdx = i2;
+ }
+
+ /* Modify the contents of buffer aPg[]. Set nPg to the new size
+ ** in bytes. The new page is always smaller than the old. */
+ nPg = pLeaf->szLeaf - nShift;
+ memmove(&aPg[4], &aPg[4+nShift], nPg-4);
+ fts5PutU16(&aPg[2], nPg);
+ if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4);
+ if( nIdx>0 ){
+ memcpy(&aPg[nPg], aIdx, nIdx);
+ nPg += nIdx;
+ }
+ sqlite3_free(aIdx);
+
+ /* Write the new page to disk and exit the loop */
+ assert( nPg>4 || fts5GetU16(aPg)==0 );
+ fts5DataWrite(p, iRowid, aPg, nPg);
+ break;
+ }
+ }
+ fts5DataRelease(pLeaf);
+}
+
+
+/*
+** This is called as part of flushing a delete to disk in 'secure-delete'
+** mode. It edits the segments within the database described by argument
+** pStruct to remove the entries for term zTerm, rowid iRowid.
+*/
+static void fts5FlushSecureDelete(
+ Fts5Index *p,
+ Fts5Structure *pStruct,
+ const char *zTerm,
+ i64 iRowid
+){
+ const int f = FTS5INDEX_QUERY_SKIPHASH;
+ int nTerm = strlen(zTerm);
+ Fts5Iter *pIter = 0; /* Used to find term instance */
+
+ fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter);
+ if( fts5MultiIterEof(p, pIter)==0 ){
+ i64 iThis = fts5MultiIterRowid(pIter);
+ if( iThis<iRowid ){
+ fts5MultiIterNextFrom(p, pIter, iRowid);
+ }
+
+ if( p->rc==SQLITE_OK
+ && fts5MultiIterEof(p, pIter)==0
+ && iRowid==fts5MultiIterRowid(pIter)
+ ){
+ Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
+ int iSegid = pSeg->pSeg->iSegid;
+ u8 *aPg = pSeg->pLeaf->p;
+ int nPg = pSeg->pLeaf->nn;
+ int iPgIdx = pSeg->pLeaf->szLeaf;
+
+ u64 iDelta = 0;
+ u64 iNextDelta = 0;
+ int iNextOff = 0;
+ int iOff = 0;
+ int nMove = 0;
+ int nIdx = 0;
+
+ u8 *aIdx = 0;
+
+ nIdx = nPg-iPgIdx;
+ aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16);
+ if( aIdx ){
+ int bLastInDoclist = 0;
+ int iIdx = 0;
+ int iStart = 0;
+ int iKeyOff = 0;
+ int iPrevKeyOff = 0;
+ int nShift = 0;
+ int iDelKeyOff = 0; /* Offset of deleted key, if any */
+ memcpy(aIdx, &aPg[iPgIdx], nIdx);
+
+ /* At this point segment iterator pSeg points to the entry
+ ** this function should remove from the b-tree segment.
+ **
+ ** More specifically, pSeg->iLeafOffset is the offset of the
+ ** first byte in the position-list for the entry to remove.
+ ** Immediately before this comes two varints that will also
+ ** need to be removed:
+ **
+ ** + the rowid or delta rowid value for the entry, and
+ ** + the size of the position list in bytes.
+ */
+ {
+ int iSOP;
+ int nPos = 0;
+ if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
+ iStart = pSeg->iTermLeafOffset;
+ }else{
+ iStart = fts5GetU16(&aPg[0]);
+ }
+
+ iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
+ iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
+ assert_nc( iSOP<=pSeg->iLeafOffset );
+ while( iSOP<pSeg->iLeafOffset ){
+ iStart = iSOP + (nPos/2);
+ iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
+ iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
+ }
+ assert_nc( iSOP==pSeg->iLeafOffset );
+ }
+
+ iOff = iStart;
+ iNextOff = pSeg->iLeafOffset + pSeg->nPos;
+ if( iNextOff>=iPgIdx ){
+ int pgno = pSeg->iLeafPgno+1;
+ fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
+ iNextOff = iPgIdx;
+ }else{
+ /* Set bLastInDoclist to true if the entry being removed is the last
+ ** in its doclist. */
+ for(iIdx=0, iKeyOff=0; iIdx<nIdx; /* no-op */){
+ u32 iVal = 0;
+ iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+ iKeyOff += iVal;
+ if( iKeyOff==iNextOff ){
+ bLastInDoclist = 1;
+ }
+ }
+ }
+
+ if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist||iNextOff==iPgIdx) ){
+ fts5PutU16(&aPg[0], 0);
+ }
+
+ if( bLastInDoclist==0 ){
+ if( iNextOff!=iPgIdx ){
+ iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta);
+ iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
+ }
+ }else if(
+ iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno
+ ){
+ /* The entry being removed was the only position list in its
+ ** doclist. Therefore the term needs to be removed as well. */
+ int iKey = 0;
+ for(iIdx=0, iKeyOff=0; iIdx<nIdx; iKey++){
+ u32 iVal = 0;
+ iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+ if( (iKeyOff+iVal)>iStart ) break;
+ iKeyOff += iVal;
+ }
+
+ iDelKeyOff = iOff = iKeyOff;
+ if( iNextOff!=iPgIdx ){
+ int nPrefix = 0;
+ int nSuffix = 0;
+ int nPrefix2 = 0;
+ int nSuffix2 = 0;
+
+ iDelKeyOff = iNextOff;
+ iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2);
+ iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2);
+
+ if( iKey!=1 ){
+ iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix);
+ }
+ iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix);
+
+ nPrefix = MIN(nPrefix, nPrefix2);
+ nSuffix = (nPrefix2 + nSuffix2) - nPrefix;
+
+ if( iKey!=1 ){
+ iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix);
+ }
+ iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix);
+ if( nPrefix2>nPrefix ){
+ memcpy(&aPg[iOff], &zTerm[nPrefix], nPrefix2-nPrefix);
+ iOff += (nPrefix2-nPrefix);
+ }
+ memcpy(&aPg[iOff], &aPg[iNextOff], nSuffix2);
+ iOff += nSuffix2;
+ iNextOff += nSuffix2;
+ }
+ }else if( iStart==4 ){
+ assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno );
+ /* The entry being removed may be the only position list in
+ ** its doclist. */
+ int iPgno = pSeg->iLeafPgno-1;
+
+ for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
+ Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno));
+ int bEmpty = (pPg && pPg->nn==4);
+ fts5DataRelease(pPg);
+ if( bEmpty==0 ) break;
+ }
+
+ if( iPgno==pSeg->iTermLeafPgno ){
+ i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno);
+ Fts5Data *pTerm = fts5DataRead(p, iId);
+ if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
+ u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
+ int nTermIdx = pTerm->nn - pTerm->szLeaf;
+ int iTermIdx = 0;
+ int iTermOff = 0;
+
+ while( 1 ){
+ u32 iVal = 0;
+ int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal);
+ iTermOff += iVal;
+ if( (iTermIdx+nByte)>=nTermIdx ) break;
+ iTermIdx += nByte;
+ }
+ nTermIdx = iTermIdx;
+
+ memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
+ fts5PutU16(&pTerm->p[2], iTermOff);
+
+ fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
+ if( nTermIdx==0 ){
+ fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
+ }
+ }
+ fts5DataRelease(pTerm);
+ }
+ }
+
+ nMove = nPg - iNextOff;
+ memmove(&aPg[iOff], &aPg[iNextOff], nMove);
+ iPgIdx -= (iNextOff - iOff);
+ nPg = iPgIdx;
+ fts5PutU16(&aPg[2], iPgIdx);
+
+ nShift = iNextOff - iOff;
+ for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){
+ u32 iVal = 0;
+ iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+ iKeyOff += iVal;
+ if( iKeyOff!=iDelKeyOff ){
+ if( iKeyOff>iOff ){
+ iKeyOff -= nShift;
+ nShift = 0;
+ }
+ nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
+ iPrevKeyOff = iKeyOff;
+ }
+ }
+
+ if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
+ fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
+ }
+
+ assert( nPg>4 || fts5GetU16(aPg)==0 );
+ fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid, pSeg->iLeafPgno), aPg, nPg);
+ sqlite3_free(aIdx);
+ }
+ }
+ }
+
+ fts5MultiIterFree(pIter);
+}
+
+
/*
** Flush the contents of in-memory hash table iHash to a new level-0
** segment on disk. Also update the corresponding structure record.
if( iSegid ){
const int pgsz = p->pConfig->pgsz;
int eDetail = p->pConfig->eDetail;
+ int bSecureDelete = p->pConfig->bSecureDelete;
Fts5StructureSegment *pSeg; /* New segment within pStruct */
Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */
Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */
}
while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){
const char *zTerm; /* Buffer containing term */
+ int nTerm; /* Size of zTerm in bytes */
const u8 *pDoclist; /* Pointer to doclist for this term */
int nDoclist; /* Size of doclist in bytes */
- /* Write the term for this entry to disk. */
+ /* Get the term and doclist for this entry. */
sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
- fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm);
- if( p->rc!=SQLITE_OK ) break;
+ nTerm = (int)strlen(zTerm);
+ if( bSecureDelete==0 ){
+ fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
+ if( p->rc!=SQLITE_OK ) break;
+ assert( writer.bFirstRowidInPage==0 );
+ }
- assert( writer.bFirstRowidInPage==0 );
- if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
+ if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
/* The entire doclist will fit on the current leaf. */
fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
}else{
+ int bTermWritten = !bSecureDelete;
i64 iRowid = 0;
- u64 iDelta = 0;
+ i64 iPrev = 0;
int iOff = 0;
/* The entire doclist will not fit on this leaf. The following
** loop iterates through the poslists that make up the current
** doclist. */
while( p->rc==SQLITE_OK && iOff<nDoclist ){
+ u64 iDelta = 0;
iOff += fts5GetVarint(&pDoclist[iOff], &iDelta);
iRowid += iDelta;
+
+ /* If in secure delete mode, and if this entry in the poslist is
+ ** in fact a delete, then edit the existing segments directly
+ ** using fts5FlushSecureDelete(). */
+ if( bSecureDelete && (pDoclist[iOff] & 0x01) ){
+ fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
+ if( pDoclist[iOff]==0x01 ){
+ iOff++;
+ continue;
+ }
+ }
+
+ if( bTermWritten==0 ){
+ fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
+ if( p->rc!=SQLITE_OK ) break;
+ bTermWritten = 1;
+ assert( writer.bFirstRowidInPage==0 );
+ }
if( writer.bFirstRowidInPage ){
fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */
fts5WriteDlidxAppend(p, &writer, iRowid);
if( p->rc!=SQLITE_OK ) break;
}else{
- pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta);
+ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev);
}
assert( pBuf->n<=pBuf->nSpace );
+ iPrev = iRowid;
if( eDetail==FTS5_DETAIL_NONE ){
if( iOff<nDoclist && pDoclist[iOff]==0 ){
sqlite3Fts5HashClear(pHash);
fts5WriteFinish(p, &writer, &pgnoLast);
- /* Update the Fts5Structure. It is written back to the database by the
- ** fts5StructureRelease() call below. */
- if( pStruct->nLevel==0 ){
- fts5StructureAddLevel(&p->rc, &pStruct);
- }
- fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
- if( p->rc==SQLITE_OK ){
- pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
- pSeg->iSegid = iSegid;
- pSeg->pgnoFirst = 1;
- pSeg->pgnoLast = pgnoLast;
- pStruct->nSegment++;
+ assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 );
+ if( pgnoLast>0 ){
+ /* Update the Fts5Structure. It is written back to the database by the
+ ** fts5StructureRelease() call below. */
+ if( pStruct->nLevel==0 ){
+ fts5StructureAddLevel(&p->rc, &pStruct);
+ }
+ fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
+ if( p->rc==SQLITE_OK ){
+ pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
+ pSeg->iSegid = iSegid;
+ pSeg->pgnoFirst = 1;
+ pSeg->pgnoLast = pgnoLast;
+ pStruct->nSegment++;
+ }
+ fts5StructurePromote(p, 0, pStruct);
}
- fts5StructurePromote(p, 0, pStruct);
}
fts5IndexAutomerge(p, &pStruct, pgnoLast);
sqlite3_finalize(p->pIdxDeleter);
sqlite3_finalize(p->pIdxSelect);
sqlite3_finalize(p->pDataVersion);
+ sqlite3_finalize(p->pDeleteFromIdx);
sqlite3Fts5HashFree(p->pHash);
sqlite3_free(p->zDataTbl);
sqlite3_free(p);
int pgno = fts5DlidxIterPgno(pDlidx);
assert( pgno>iLeaf );
cksum1 += iRowid + ((i64)pgno<<32);
+ // printf("1: rowid=%lld pgno=%d\n", iRowid, pgno);
}
fts5DlidxIterFree(pDlidx);
pDlidx = 0;
int pgno = fts5DlidxIterPgno(pDlidx);
assert( fts5DlidxIterPgno(pDlidx)>iLeaf );
cksum2 += iRowid + ((i64)pgno<<32);
+ // printf("2: rowid=%lld pgno=%d\n", iRowid, pgno);
}
fts5DlidxIterFree(pDlidx);
pDlidx = 0;
Fts5StructureSegment *pSeg /* Segment to check internal consistency */
){
Fts5Config *pConfig = p->pConfig;
+ int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE);
sqlite3_stmt *pStmt = 0;
int rc2;
int iIdxPrevLeaf = pSeg->pgnoFirst-1;
** is also a rowid pointer within the leaf page header, it points to a
** location before the term. */
if( pLeaf->nn<=pLeaf->szLeaf ){
- p->rc = FTS5_CORRUPT;
+
+ if( nIdxTerm==0
+ && pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE
+ && pLeaf->nn==pLeaf->szLeaf
+ && pLeaf->nn==4
+ ){
+ /* special case - the very first page in a segment keeps its %_idx
+ ** entry even if all the terms are removed from it by secure-delete
+ ** operations. */
+ }else{
+ p->rc = FTS5_CORRUPT;
+ }
+
}else{
int iOff; /* Offset of first term on leaf */
int iRowidOff; /* Offset of first rowid on leaf */
ASSERT_SZLEAF_OK(pLeaf);
if( iRowidOff>=pLeaf->szLeaf ){
p->rc = FTS5_CORRUPT;
- }else{
+ }else if( bSecureDelete==0 || iRowidOff>0 ){
+ i64 iDlRowid = fts5DlidxIterRowid(pDlidx);
fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
- if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT;
+ if( iRowid<iDlRowid || (bSecureDelete==0 && iRowid!=iDlRowid) ){
+ p->rc = FTS5_CORRUPT;
+ }
}
fts5DataRelease(pLeaf);
}
--- /dev/null
+# 2023 Feb 17
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5secure
+
+proc dump {tname} {
+ execsql_pp "SELECT * FROM ${tname}_idx"
+ execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data"
+}
+
+
+do_execsql_test 0.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(ab);
+ CREATE VIRTUAL TABLE v1 USING fts5vocab('t1', 'instance');
+ INSERT INTO t1(rowid, ab) VALUES
+ (0,'abc'), (1,'abc'), (2,'abc'), (3,'abc'), (4,'def');
+}
+
+do_execsql_test 0.1 {
+ INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 0.2 {
+ DELETE FROM t1 WHERE rowid=2;
+}
+
+do_execsql_test 0.3 {
+ SELECT count(*) FROM t1_data
+} 3
+
+do_execsql_test 0.4 {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+do_execsql_test 0.5 {
+ DELETE FROM t1 WHERE rowid=3;
+}
+
+do_execsql_test 0.6 {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+do_execsql_test 0.7 {
+ DELETE FROM t1 WHERE rowid=0;
+}
+
+do_execsql_test 0.8 {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+#----------------------------------
+
+do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE t2 USING fts5(ab);
+ INSERT INTO t2(rowid, ab) VALUES (5, 'key'), (6, 'value');
+ INSERT INTO t2(t2, rank) VALUES('secure-delete', 1);
+}
+
+#execsql_pp { SELECT id, quote(block) FROM t1_data }
+#execsql_pp { SELECT segid, quote(term), pgno FROM t1_idx }
+
+do_execsql_test 1.1 {
+ DELETE FROM t2 WHERE rowid = 5;
+}
+
+do_execsql_test 1.2 {
+ INSERT INTO t2(t2) VALUES('integrity-check');
+}
+
+do_execsql_test 1.3 {
+ DELETE FROM t2 WHERE rowid = 6;
+}
+
+do_execsql_test 1.4 {
+ INSERT INTO t2(t2) VALUES('integrity-check');
+}
+
+do_execsql_test 1.5 {
+ SELECT * FROM t2('value');
+ SELECT * FROM t2('v*');
+}
+
+do_execsql_test 1.6 {
+ SELECT * FROM t2('value') ORDER BY rowid DESC;
+ SELECT * FROM t2('v*') ORDER BY rowid DESC;
+}
+execsql_pp {
+ SELECT id, quote(block) FROM t2_data;
+}
+
+#----------------------------------
+
+do_execsql_test 2.0 {
+ CREATE VIRTUAL TABLE ft USING fts5(ab);
+ CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', 'instance');
+ INSERT INTO ft(rowid, ab) VALUES
+ (1, 'one'),
+ (2, 'two'),
+ (3, 'three'),
+ (4, 'four'),
+ (5, 'one one'),
+ (6, 'one two'),
+ (7, 'one three'),
+ (8, 'one four'),
+ (9, 'two one'),
+ (10, 'two two'),
+ (11, 'two three'),
+ (12, 'two four'),
+ (13, 'three one'),
+ (14, 'three two'),
+ (15, 'three three'),
+ (16, 'three four');
+}
+
+do_execsql_test 2.1 {
+ SELECT count(*) FROM ft_data;
+} {3}
+
+do_execsql_test 2.2 {
+ INSERT INTO ft(ft, rank) VALUES('secure-delete', 1);
+}
+
+do_execsql_test 2.3 {
+ DELETE FROM ft WHERE rowid=9;
+}
+
+do_execsql_test 2.4 {
+ INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.5 {
+ DELETE FROM ft WHERE ab LIKE '%two%'
+}
+
+do_execsql_test 2.6 {
+ INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 2.7 {
+ SELECT count(*) FROM ft_data;
+} {3}
+
+#----------------------------------
+reset_db
+
+set ::vocab {
+ one two three four five six seven eight nine ten
+ eleven twelve thirteen fourteen fifteen sixteen
+ seventeen eighteen nineteen twenty
+}
+proc rnddoc {} {
+ set nVocab [llength $::vocab]
+ set ret [list]
+ for {set ii 0} {$ii < 8} {incr ii} {
+ lappend ret [lindex $::vocab [expr int(abs(rand()) * $nVocab)]]
+ }
+ set ret
+}
+
+proc contains {list val} {
+ expr {[lsearch $list $val]>=0}
+}
+
+foreach {tn pgsz} {
+ 2 64
+ 1 1000
+} {
+ reset_db
+ db function rnddoc rnddoc
+ db function contains contains
+
+ expr srand(1)
+
+ do_execsql_test 3.$tn.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(x);
+ INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
+ WITH s(i) AS (
+ VALUES(1) UNION SELECT i+1 FROM s WHERE i<20
+ )
+ INSERT INTO t1 SELECT rnddoc() FROM s;
+ }
+
+ do_execsql_test 3.$tn.1 {
+ INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+ }
+
+ foreach {rowid} {
+ 6 16 3 4 9 14 13 7 20 15 19 10 11 2 5 18 17 1 12 8
+ } {
+
+ do_execsql_test 3.$tn.2.$rowid {
+ DELETE FROM t1 WHERE rowid=$rowid;
+ }
+ do_execsql_test 3.$tn.2.$rowid.ic {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+ }
+
+ foreach v $::vocab {
+ do_execsql_test 3.$tn.2.$rowid.q.$v {
+ SELECT rowid FROM t1($v)
+ } [db eval {SELECT rowid FROM t1 WHERE contains(x, $v)}]
+
+ do_execsql_test 3.$tn.2.$rowid.q.$v.DESC {
+ SELECT rowid FROM t1($v) ORDER BY 1 DESC
+ } [db eval {SELECT rowid FROM t1 WHERE contains(x, $v) ORDER BY 1 DESC}]
+ }
+ }
+}
+
+do_execsql_test 3.3 {
+ INSERT INTO t1(x) VALUES('optimize');
+ INSERT INTO t1(t1) VALUES('optimize');
+ SELECT count(*) FROM t1_data;
+} {3}
+
+#----------------------------------
+reset_db
+do_execsql_test 4.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(x);
+ INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+ INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+
+set L1 [string repeat abcdefghij 10]
+set L2 [string repeat 1234567890 10]
+
+do_execsql_test 4.1 {
+ INSERT INTO t1 VALUES('aa' || $L1 || ' ' || $L2);
+}
+do_execsql_test 4.2 {
+ DELETE FROM t1 WHERE rowid=1
+}
+do_execsql_test 4.3 {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+#----------------------------------
+reset_db
+do_execsql_test 5.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(x);
+ INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+ INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+
+set doc "aa [string repeat {abc } 60]"
+
+do_execsql_test 5.1 {
+ BEGIN;
+ INSERT INTO t1 VALUES($doc);
+ INSERT INTO t1 VALUES('aa abc');
+ COMMIT;
+}
+
+do_execsql_test 5.2 {
+ DELETE FROM t1 WHERE rowid = 1;
+}
+
+do_execsql_test 5.3 {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+do_execsql_test 5.4 { SELECT rowid FROM t1('abc'); } 2
+do_execsql_test 5.5 { SELECT rowid FROM t1('aa'); } 2
+
+
+finish_test
+
projects / thirdparty / sqlite.git / commitdiff
