#define FTS5_MAX_LEVEL 64
+#define FTS5_STRUCTURE_V2 "\xFF\x00\x00\x01"
+
/*
** Details:
**
** + first leaf page number (often 1, always greater than 0)
** + final leaf page number
**
+** Then, for V2 structures only:
+**
+** + lower location counter value,
+** + upper location counter value
+**
** 2. The Averages Record:
**
** A single record within the %_data table. The data is a list of varints.
#define FTS5_SEGMENT_ROWID(segid, pgno) fts5_dri(segid, 0, 0, pgno)
#define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno)
+#define FTS5_TOMBSTONE_ROWID(segid) fts5_dri(segid + (1<<16), 0, 0, 0)
+
#ifdef SQLITE_DEBUG
int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
#endif
/* State used by the fts5DataXXX() functions. */
sqlite3_blob *pReader; /* RO incr-blob open on %_data table */
+ sqlite3_stmt *pReaderOpt;
sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */
sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */
sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */
** The contents of the "structure" record for each index are represented
** using an Fts5Structure record in memory. Which uses instances of the
** other Fts5StructureXXX types as components.
+**
+** nLocCounter:
+** This value is set to non-zero for structure records created for
+** contentlessdelete=1 tables only. In that case it represents the
+** location value to apply to the next top-level segment created.
*/
struct Fts5StructureSegment {
int iSegid; /* Segment id */
int pgnoFirst; /* First leaf page number in segment */
int pgnoLast; /* Last leaf page number in segment */
+
+ /* contentlessdelete=1 tables only: */
+ u64 iLoc1;
+ u64 iLoc2;
};
struct Fts5StructureLevel {
int nMerge; /* Number of segments in incr-merge */
struct Fts5Structure {
int nRef; /* Object reference count */
u64 nWriteCounter; /* Total leaves written to level 0 */
+ u64 nLocCounter;
int nSegment; /* Total segments in this structure */
int nLevel; /* Number of levels in this index */
Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */
Fts5Data *pLeaf; /* Current leaf data */
Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */
i64 iLeafOffset; /* Byte offset within current leaf */
+ Fts5Data *pTombstone;
/* Next method */
void (*xNext)(Fts5Index*, Fts5SegIter*, int*);
}
}
+
/*
** Retrieve a record from the %_data table.
**
return p->rc;
}
+static Fts5Data *fts5DataReadOpt(Fts5Index *p, i64 iRowid){
+ Fts5Data *pRet = 0;
+ int rc2 = SQLITE_OK;
+
+ if( p->pReaderOpt==0 ){
+ Fts5Config *pConfig = p->pConfig;
+ fts5IndexPrepareStmt(p, &p->pReaderOpt, sqlite3_mprintf(
+ "SELECT block FROM '%q'.'%q_data' WHERE id=?",
+ pConfig->zDb, pConfig->zName
+ ));
+ }
+
+ if( p->rc==SQLITE_OK ){
+ sqlite3_bind_int64(p->pReaderOpt, 1, iRowid);
+ if( SQLITE_ROW==sqlite3_step(p->pReaderOpt) ){
+ int nByte = sqlite3_column_bytes(p->pReaderOpt, 0);
+ const u8 *aByte = (const u8*)sqlite3_column_blob(p->pReaderOpt, 0);
+ i64 nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING;
+
+ pRet = (Fts5Data*)sqlite3_malloc64(nAlloc);
+ if( pRet==0 ){
+ p->rc = SQLITE_NOMEM;
+ }else{
+ pRet->nn = nByte;
+ pRet->p = (u8*)&pRet[1];
+ memcpy(pRet->p, aByte, nByte);
+ }
+ }
+ rc2 = sqlite3_reset(p->pReaderOpt);
+ if( p->rc==SQLITE_OK ) p->rc = rc2;
+ }
+
+ return pRet;
+}
/*
** INSERT OR REPLACE a record into the %_data table.
i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0);
i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1;
fts5DataDelete(p, iFirst, iLast);
+ if( p->pConfig->bContentlessDelete ){
+ i64 iTomb = FTS5_TOMBSTONE_ROWID(iSegid);
+ fts5DataDelete(p, iTomb, iTomb);
+ }
if( p->pIdxDeleter==0 ){
Fts5Config *pConfig = p->pConfig;
fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf(
int nSegment = 0;
sqlite3_int64 nByte; /* Bytes of space to allocate at pRet */
Fts5Structure *pRet = 0; /* Structure object to return */
+ int bStructureV2 = 0; /* True for FTS5_STRUCTURE_V2 */
+ i64 nLocCounter = 0;
/* Grab the cookie value */
if( piCookie ) *piCookie = sqlite3Fts5Get32(pData);
i = 4;
+ if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2, 4) ){
+ i += 4;
+ bStructureV2 = 1;
+ }
+
/* Read the total number of levels and segments from the start of the
** structure record. */
i += fts5GetVarint32(&pData[i], nLevel);
i += fts5GetVarint32(&pData[i], pSeg->iSegid);
i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst);
i += fts5GetVarint32(&pData[i], pSeg->pgnoLast);
+ if( bStructureV2 ){
+ i += fts5GetVarint(&pData[i], &pSeg->iLoc1);
+ i += fts5GetVarint(&pData[i], &pSeg->iLoc2);
+ nLocCounter = MAX(nLocCounter, pSeg->iLoc2);
+ }
if( pSeg->pgnoLast<pSeg->pgnoFirst ){
rc = FTS5_CORRUPT;
break;
}
}
if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT;
+ if( bStructureV2 ){
+ pRet->nLocCounter = nLocCounter+1;
+ }
if( rc!=SQLITE_OK ){
fts5StructureRelease(pRet);
Fts5Buffer buf; /* Buffer to serialize record into */
int iLvl; /* Used to iterate through levels */
int iCookie; /* Cookie value to store */
+ int nHdr = (pStruct->nLocCounter>0 ? (4+4+9+9+9) : (4+9+9));
assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
memset(&buf, 0, sizeof(Fts5Buffer));
iCookie = p->pConfig->iCookie;
if( iCookie<0 ) iCookie = 0;
- if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){
+ if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){
sqlite3Fts5Put32(buf.p, iCookie);
buf.n = 4;
+ if( pStruct->nLocCounter>0 ){
+ fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4);
+ }
fts5BufferSafeAppendVarint(&buf, pStruct->nLevel);
fts5BufferSafeAppendVarint(&buf, pStruct->nSegment);
fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter);
fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid);
fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst);
fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast);
+ if( pStruct->nLocCounter>0 ){
+ fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iLoc1);
+ fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iLoc2);
+ }
}
}
}
}
+static void fts5SegIterLoadTombstone(Fts5Index *p, Fts5SegIter *pIter){
+ i64 iRowid = FTS5_TOMBSTONE_ROWID(pIter->pSeg->iSegid);
+ pIter->pTombstone = fts5DataReadOpt(p, iRowid);
+}
+
/*
** Initialize the iterator object pIter to iterate through the entries in
** segment pSeg. The iterator is left pointing to the first entry when
fts5SegIterLoadTerm(p, pIter, 0);
fts5SegIterLoadNPos(p, pIter);
}
+
+ fts5SegIterLoadTombstone(p, pIter);
}
/*
}
fts5SegIterSetNext(p, pIter);
+ fts5SegIterLoadTombstone(p, pIter);
/* Either:
**
fts5BufferFree(&pIter->term);
fts5DataRelease(pIter->pLeaf);
fts5DataRelease(pIter->pNextLeaf);
+ fts5DataRelease(pIter->pTombstone);
fts5DlidxIterFree(pIter->pDlidx);
sqlite3_free(pIter->aRowidOffset);
memset(pIter, 0, sizeof(Fts5SegIter));
pIter->iSwitchRowid = pSeg->iRowid;
}
+static u64 fts5GetU64(u8 *a){
+ return ((u64)a[0] << 56)
+ + ((u64)a[1] << 48)
+ + ((u64)a[2] << 40)
+ + ((u64)a[3] << 32)
+ + ((u64)a[4] << 24)
+ + ((u64)a[5] << 16)
+ + ((u64)a[6] << 8)
+ + ((u64)a[7] << 0);
+}
+
+static void fts5PutU64(u8 *a, u64 iVal){
+ a[0] = ((iVal >> 56) & 0xFF);
+ a[1] = ((iVal >> 48) & 0xFF);
+ a[2] = ((iVal >> 40) & 0xFF);
+ a[3] = ((iVal >> 32) & 0xFF);
+ a[4] = ((iVal >> 24) & 0xFF);
+ a[5] = ((iVal >> 16) & 0xFF);
+ a[6] = ((iVal >> 8) & 0xFF);
+ a[7] = ((iVal >> 0) & 0xFF);
+}
+
+static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
+ int iFirst = pIter->aFirst[1].iFirst;
+ Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
+
+ if( pSeg->pTombstone ){
+ int ii;
+ for(ii=0; ii<pSeg->pTombstone->nn; ii+=8){
+ i64 iVal = (i64)fts5GetU64(&pSeg->pTombstone->p[ii]);
+ if( iVal==pSeg->iRowid ) return 1;
+ }
+ }
+
+ return 0;
+}
+
/*
** Move the iterator to the next entry.
**
fts5AssertMultiIterSetup(p, pIter);
assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf );
- if( pIter->bSkipEmpty==0 || pSeg->nPos ){
+ if( (pIter->bSkipEmpty==0 || pSeg->nPos)
+ && 0==fts5MultiIterIsDeleted(pIter)
+ ){
pIter->xSetOutputs(pIter, pSeg);
return;
}
}
fts5AssertMultiIterSetup(p, pIter);
- }while( fts5MultiIterIsEmpty(p, pIter) );
+ }while( fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter) );
}
}
fts5MultiIterSetEof(pNew);
fts5AssertMultiIterSetup(p, pNew);
- if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
+ if( (pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew))
+ || fts5MultiIterIsDeleted(pNew)
+ ){
fts5MultiIterNext(p, pNew, 0, 0);
}else if( pNew->base.bEof==0 ){
Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst];
/* Read input from all segments in the input level */
nInput = pLvl->nSeg;
+
+ /* Set the range of locations that will go into the output segment */
+ if( pStruct->nLocCounter>0 ){
+ pSeg->iLoc1 = pLvl->aSeg[0].iLoc1;
+ pSeg->iLoc2 = pLvl->aSeg[pLvl->nSeg-1].iLoc2;
+ }
}
bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2);
pSeg->iSegid = iSegid;
pSeg->pgnoFirst = 1;
pSeg->pgnoLast = pgnoLast;
+ if( pStruct->nLocCounter>0 ){
+ pSeg->iLoc1 = pStruct->nLocCounter;
+ pSeg->iLoc2 = pStruct->nLocCounter;
+ pStruct->nLocCounter++;
+ }
pStruct->nSegment++;
}
fts5StructurePromote(p, 0, pStruct);
pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL);
pNew->nRef = 1;
pNew->nWriteCounter = pStruct->nWriteCounter;
+ pNew->nLocCounter = pStruct->nLocCounter;
pLvl = &pNew->aLevel[pNew->nLevel-1];
pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
if( pLvl->aSeg ){
fts5StructureInvalidate(p);
fts5IndexDiscardData(p);
memset(&s, 0, sizeof(Fts5Structure));
+ if( p->pConfig->bContentlessDelete ){
+ s.nLocCounter = 1;
+ }
fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
fts5StructureWrite(p, &s);
return fts5IndexReturn(p);
assert( p->pReader==0 );
fts5StructureInvalidate(p);
sqlite3_finalize(p->pWriter);
+ sqlite3_finalize(p->pReaderOpt);
sqlite3_finalize(p->pDeleter);
sqlite3_finalize(p->pIdxWriter);
sqlite3_finalize(p->pIdxDeleter);
return fts5IndexReturn(p);
}
+int sqlite3Fts5IndexGetLocation(Fts5Index *p, i64 *piLoc){
+ Fts5Structure *pStruct;
+ pStruct = fts5StructureRead(p);
+ if( pStruct ){
+ *piLoc = pStruct->nLocCounter;
+ fts5StructureRelease(pStruct);
+ }
+ return fts5IndexReturn(p);
+}
+
+static void fts5IndexTombstoneAdd(
+ Fts5Index *p,
+ Fts5StructureSegment *pSeg,
+ i64 iRowid
+){
+ Fts5Data *pHash = 0;
+ u8 *aNew = 0;
+ int nNew = 0;
+
+ pHash = fts5DataReadOpt(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid));
+ if( p->rc ) return;
+
+ if( pHash ){
+ nNew = 8 + pHash->nn;
+ }else{
+ nNew = 8;
+ }
+ aNew = sqlite3_malloc(nNew);
+ if( aNew==0 ){
+ p->rc = SQLITE_NOMEM;
+ }else{
+ if( pHash ){
+ memcpy(aNew, pHash->p, pHash->nn);
+ }
+ fts5PutU64(&aNew[nNew-8], iRowid);
+ fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid), aNew, nNew);
+ }
+
+ sqlite3_free(aNew);
+ fts5DataRelease(pHash);
+}
+
+/*
+** Add iRowid to the tombstone list of the segment or segments that contain
+** rows from location iLoc.
+*/
+int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iLoc, i64 iRowid){
+ Fts5Structure *pStruct;
+ pStruct = fts5StructureRead(p);
+ if( pStruct ){
+ int iLvl;
+ for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
+ int iSeg;
+ for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
+ Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
+ if( pSeg->iLoc1<=iLoc && pSeg->iLoc2>=iLoc ){
+ fts5IndexTombstoneAdd(p, pSeg, iRowid);
+ }
+ }
+ }
+ fts5StructureRelease(pStruct);
+ }
+ return fts5IndexReturn(p);
+}
/*************************************************************************
**************************************************************************
);
for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
- sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}",
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d",
pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast
);
+ if( pSeg->iLoc1>0 ){
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " loc=%lld..%lld",
+ pSeg->iLoc1, pSeg->iLoc2
+ );
+ }
+ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
}
sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
}
}
#endif /* SQLITE_TEST */
+#ifdef SQLITE_TEST
+
+typedef struct Fts5StructVtab Fts5StructVtab;
+struct Fts5StructVtab {
+ sqlite3_vtab base;
+};
+
+typedef struct Fts5StructVcsr Fts5StructVcsr;
+struct Fts5StructVcsr {
+ sqlite3_vtab_cursor base;
+ Fts5Structure *pStruct;
+ int iLevel;
+ int iSeg;
+ int iRowid;
+};
+
+/*
+** Create a new fts5_structure() table-valued function.
+*/
+static int fts5structConnectMethod(
+ sqlite3 *db,
+ void *pAux,
+ int argc, const char *const*argv,
+ sqlite3_vtab **ppVtab,
+ char **pzErr
+){
+ Fts5StructVtab *pNew = 0;
+ int rc = SQLITE_OK;
+
+ rc = sqlite3_declare_vtab(db,
+ "CREATE TABLE xyz("
+ "level, segment, merge, segid, leaf1, leaf2, loc1, loc2,"
+ "struct HIDDEN);"
+ );
+ if( rc==SQLITE_OK ){
+ pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
+ }
+
+ *ppVtab = (sqlite3_vtab*)pNew;
+ return rc;
+}
+
+/*
+** We must have a single struct=? constraint that will be passed through
+** into the xFilter method. If there is no valid stmt=? constraint,
+** then return an SQLITE_CONSTRAINT error.
+*/
+static int fts5structBestIndexMethod(
+ sqlite3_vtab *tab,
+ sqlite3_index_info *pIdxInfo
+){
+ int i;
+ int rc = SQLITE_CONSTRAINT;
+ struct sqlite3_index_constraint *p;
+ pIdxInfo->estimatedCost = (double)100;
+ pIdxInfo->estimatedRows = 100;
+ pIdxInfo->idxNum = 0;
+ for(i=0, p=pIdxInfo->aConstraint; i<pIdxInfo->nConstraint; i++, p++){
+ if( p->usable==0 ) continue;
+ if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==8 ){
+ rc = SQLITE_OK;
+ pIdxInfo->aConstraintUsage[i].omit = 1;
+ pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+ break;
+ }
+ }
+ return rc;
+}
+
+/*
+** This method is the destructor for bytecodevtab objects.
+*/
+static int fts5structDisconnectMethod(sqlite3_vtab *pVtab){
+ Fts5StructVtab *p = (Fts5StructVtab*)pVtab;
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+/*
+** Constructor for a new bytecodevtab_cursor object.
+*/
+static int fts5structOpenMethod(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
+ int rc = SQLITE_OK;
+ Fts5StructVcsr *pNew = 0;
+
+ pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
+ *ppCsr = (sqlite3_vtab_cursor*)pNew;
+
+ return SQLITE_OK;
+}
+
+/*
+** Destructor for a bytecodevtab_cursor.
+*/
+static int fts5structCloseMethod(sqlite3_vtab_cursor *cur){
+ Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+ fts5StructureRelease(pCsr->pStruct);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+
+/*
+** Advance a bytecodevtab_cursor to its next row of output.
+*/
+static int fts5structNextMethod(sqlite3_vtab_cursor *cur){
+ Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+
+ assert( pCsr->pStruct );
+ pCsr->iSeg++;
+ pCsr->iRowid++;
+ while( pCsr->iSeg>=pCsr->pStruct->aLevel[pCsr->iLevel].nSeg ){
+ pCsr->iLevel++;
+ pCsr->iSeg = 0;
+ }
+ if( pCsr->iLevel>=pCsr->pStruct->nLevel ){
+ fts5StructureRelease(pCsr->pStruct);
+ pCsr->pStruct = 0;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int fts5structEofMethod(sqlite3_vtab_cursor *cur){
+ Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+ return pCsr->pStruct==0;
+}
+
+static int fts5structRowidMethod(
+ sqlite3_vtab_cursor *cur,
+ sqlite_int64 *piRowid
+){
+ Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+ *piRowid = pCsr->iRowid;
+ return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the bytecodevtab_cursor
+** is currently pointing.
+*/
+static int fts5structColumnMethod(
+ sqlite3_vtab_cursor *cur, /* The cursor */
+ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
+ int i /* Which column to return */
+){
+ Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
+ Fts5Structure *p = pCsr->pStruct;
+ Fts5StructureSegment *pSeg = &p->aLevel[pCsr->iLevel].aSeg[pCsr->iSeg];
+
+ switch( i ){
+ case 0: /* level */
+ sqlite3_result_int(ctx, pCsr->iLevel);
+ break;
+ case 1: /* segment */
+ sqlite3_result_int(ctx, pCsr->iSeg);
+ break;
+ case 2: /* merge */
+ sqlite3_result_int(ctx, pCsr->iSeg < p->aLevel[pCsr->iLevel].nMerge);
+ break;
+ case 3: /* segid */
+ sqlite3_result_int(ctx, pSeg->iSegid);
+ break;
+ case 4: /* leaf1 */
+ sqlite3_result_int(ctx, pSeg->pgnoFirst);
+ break;
+ case 5: /* leaf2 */
+ sqlite3_result_int(ctx, pSeg->pgnoLast);
+ break;
+ case 6: /* loc1 */
+ sqlite3_result_int(ctx, pSeg->iLoc1);
+ break;
+ case 7: /* loc2 */
+ sqlite3_result_int(ctx, pSeg->iLoc2);
+ break;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Initialize a cursor.
+**
+** idxNum==0 means show all subprograms
+** idxNum==1 means show only the main bytecode and omit subprograms.
+*/
+static int fts5structFilterMethod(
+ sqlite3_vtab_cursor *pVtabCursor,
+ int idxNum, const char *idxStr,
+ int argc, sqlite3_value **argv
+){
+ Fts5StructVcsr *pCsr = (Fts5StructVcsr *)pVtabCursor;
+ int rc = SQLITE_OK;
+
+ const u8 *aBlob = 0;
+ int nBlob = 0;
+
+ assert( argc==1 );
+ fts5StructureRelease(pCsr->pStruct);
+ pCsr->pStruct = 0;
+
+ nBlob = sqlite3_value_bytes(argv[0]);
+ aBlob = (const u8*)sqlite3_value_blob(argv[0]);
+ rc = fts5StructureDecode(aBlob, nBlob, 0, &pCsr->pStruct);
+ if( rc==SQLITE_OK ){
+ pCsr->iLevel = 0;
+ pCsr->iRowid = 0;
+ pCsr->iSeg = -1;
+ rc = fts5structNextMethod(pVtabCursor);
+ }
+
+ return rc;
+}
+
+#endif /* SQLITE_TEST */
+
/*
** This is called as part of registering the FTS5 module with database
** connection db. It registers several user-defined scalar functions useful
db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
);
}
+
+ if( rc==SQLITE_OK ){
+ static const sqlite3_module fts5structure_module = {
+ 0, /* iVersion */
+ 0, /* xCreate */
+ fts5structConnectMethod, /* xConnect */
+ fts5structBestIndexMethod, /* xBestIndex */
+ fts5structDisconnectMethod, /* xDisconnect */
+ 0, /* xDestroy */
+ fts5structOpenMethod, /* xOpen */
+ fts5structCloseMethod, /* xClose */
+ fts5structFilterMethod, /* xFilter */
+ fts5structNextMethod, /* xNext */
+ fts5structEofMethod, /* xEof */
+ fts5structColumnMethod, /* xColumn */
+ fts5structRowidMethod, /* xRowid */
+ 0, /* xUpdate */
+ 0, /* xBegin */
+ 0, /* xSync */
+ 0, /* xCommit */
+ 0, /* xRollback */
+ 0, /* xFindFunction */
+ 0, /* xRename */
+ 0, /* xSavepoint */
+ 0, /* xRelease */
+ 0, /* xRollbackTo */
+ 0 /* xShadowName */
+ };
+ rc = sqlite3_create_module(db, "fts5_structure", &fts5structure_module, 0);
+ }
return rc;
#else
return SQLITE_OK;
"INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */
"REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */
"DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */
- "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */
+ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)", /* REPLACE_DOCSIZE */
"DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */
- "SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */
+ "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */
"REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */
"SELECT %s FROM %s AS T", /* SCAN */
break;
}
+ case FTS5_STMT_REPLACE_DOCSIZE:
+ zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName,
+ (pC->bContentlessDelete ? ",?" : "")
+ );
+ break;
+
+ case FTS5_STMT_LOOKUP_DOCSIZE:
+ zSql = sqlite3_mprintf(azStmt[eStmt],
+ (pC->bContentlessDelete ? ",location" : ""),
+ pC->zDb, pC->zName
+ );
+ break;
+
default:
zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
break;
}
if( rc==SQLITE_OK && pConfig->bColumnsize ){
- rc = sqlite3Fts5CreateTable(
- pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
- );
+ const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB";
+ if( pConfig->bContentlessDelete ){
+ zCols = "id INTEGER PRIMARY KEY, sz BLOB, location INTEGER";
+ }
+ rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr);
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5CreateTable(
return rc;
}
+static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
+ i64 iLoc = 0;
+ sqlite3_stmt *pLookup = 0;
+ int rc = SQLITE_OK;
+
+ assert( p->pConfig->bContentlessDelete );
+ assert( p->pConfig->eContent==FTS5_CONTENT_NONE );
+
+ /* Look up the location of the document in the %_docsize table. Store
+ ** this in stack variable iLoc. */
+ rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pLookup, 1, iDel);
+ if( SQLITE_ROW==sqlite3_step(pLookup) ){
+ iLoc = sqlite3_column_int64(pLookup, 1);
+ }
+ rc = sqlite3_reset(pLookup);
+ if( rc==SQLITE_OK && iLoc==0 ){
+ rc = FTS5_CORRUPT;
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iLoc, iDel);
+ }
+
+ return rc;
+}
/*
** Insert a record into the %_docsize table. Specifically, do:
rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pReplace, 1, iRowid);
- sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
- sqlite3_step(pReplace);
- rc = sqlite3_reset(pReplace);
- sqlite3_bind_null(pReplace, 2);
+ if( p->pConfig->bContentlessDelete ){
+ i64 iLoc = 0;
+ rc = sqlite3Fts5IndexGetLocation(p->pIndex, &iLoc);
+ sqlite3_bind_int64(pReplace, 3, iLoc);
+ }
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
+ sqlite3_step(pReplace);
+ rc = sqlite3_reset(pReplace);
+ sqlite3_bind_null(pReplace, 2);
+ }
}
}
return rc;
/* Delete the index records */
if( rc==SQLITE_OK ){
- rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
+ if( p->pConfig->bContentlessDelete ){
+ rc = fts5StorageContentlessDelete(p, iDel);
+ }else{
+ rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
+ }
}
/* Delete the %_docsize record */
--- /dev/null
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the content= and content_rowid= options.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5contentless
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+ finish_test
+ return
+}
+
+# Check that it is not possible to specify "contentless_delete=1" for
+# anything other than a contentless table.
+#
+set res(0) {0 {}}
+set res(1) {1 {contentless_delete=1 requires a contentless table}}
+foreach {tn sql bError} {
+ 1 "(a, b, contentless_delete=1)" 1
+ 2 "(a, b, contentless_delete=1, content=abc)" 1
+ 3 "(a, b, contentless_delete=1, content=)" 0
+ 4 "(content=, contentless_delete=1, a)" 0
+ 5 "(content='', contentless_delete=1, hello)" 0
+} {
+ execsql { BEGIN }
+ do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError)
+ execsql { ROLLBACK }
+}
+
+# Check that it is not possible to specify "contentless_delete=1"
+# along with columnsize=1.
+#
+set res(0) {0 {}}
+set res(1) {1 {contentless_delete=1 is incompatible with columnsize=0}}
+foreach {tn sql bError} {
+ 2 "(a, b, content='', contentless_delete=1, columnsize=0)" 1
+} {
+ execsql { BEGIN }
+ do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError)
+ execsql { ROLLBACK }
+}
+
+# Check that if contentless_delete=1 is specified, then the "location"
+# column is added to the %_docsize table.
+reset_db
+do_execsql_test 3.0 {
+ CREATE VIRTUAL TABLE x1 USING fts5(c, content='');
+ CREATE VIRTUAL TABLE x2 USING fts5(c, content='', contentless_delete=1);
+}
+do_execsql_test 3.1 {
+ SELECT sql FROM sqlite_schema WHERE name IN ('x1_docsize', 'x2_docsize');
+} {
+ {CREATE TABLE 'x1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)}
+ {CREATE TABLE 'x2_docsize'(id INTEGER PRIMARY KEY, sz BLOB, location INTEGER)}
+}
+
+do_execsql_test 3.2.1 {
+ SELECT hex(block) FROM x1_data WHERE id=10
+} {00000000000000}
+do_execsql_test 3.2.2 {
+ SELECT hex(block) FROM x2_data WHERE id=10
+} {00000000FF000001000000}
+
+do_execsql_test 3.3 {
+ INSERT INTO x2 VALUES('first text');
+ INSERT INTO x2 VALUES('second text');
+}
+do_execsql_test 3.4 {
+ SELECT id, location FROM x2_docsize
+} {1 1 2 2}
+do_execsql_test 3.5 {
+ SELECT level, segment, loc1, loc2 FROM fts5_structure(
+ (SELECT block FROM x2_data WHERE id=10)
+ )
+} {
+ 0 0 1 1
+ 0 1 2 2
+}
+do_execsql_test 3.6 {
+ INSERT INTO x2(x2) VALUES('optimize');
+}
+do_execsql_test 3.7 {
+ SELECT level, segment, loc1, loc2 FROM fts5_structure(
+ (SELECT block FROM x2_data WHERE id=10)
+ )
+} {
+ 1 0 1 2
+}
+
+do_execsql_test 3.8 {
+ INSERT INTO x2(x2, rowid) VALUES('delete', 2);
+}
+
+do_execsql_test 3.9 {
+ SELECT rowid FROM x2('text')
+} {1}
+
+#--------------------------------------------------------------------------
+reset_db
+proc document {n} {
+ set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
+ set ret [list]
+ for {set ii 0} {$ii < $n} {incr ii} {
+ lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]]
+ }
+ set ret
+}
+
+set nRow 1000
+
+do_execsql_test 4.0 {
+ CREATE TABLE t1(x);
+ CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
+}
+do_test 4.1 {
+ for {set ii 0} {$ii < $nRow} {incr ii} {
+ set doc [document 6]
+ execsql {
+ INSERT INTO t1 VALUES($doc);
+ INSERT INTO ft VALUES($doc);
+ }
+ }
+} {}
+
+foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
+ set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
+ set L2 [execsql {SELECT rowid FROM ft($v)}]
+ do_test 4.2.$v { set L1 } $L2
+}
+
+do_test 4.3 {
+ for {set ii 1} {$ii < $nRow} {incr ii 2} {
+ execsql {
+ INSERT INTO ft(ft, rowid) VALUES('delete', $ii);
+ DELETE FROM t1 WHERE rowid=$ii;
+ }
+ }
+} {}
+
+foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
+ set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
+ set L2 [execsql {SELECT rowid FROM ft($v)}]
+ do_test 4.4.$v { set L1 } $L2
+}
+
+do_execsql_test 4.5 {
+ INSERT INTO ft(ft) VALUES('optimize');
+} {}
+
+foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
+ set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
+ set L2 [execsql {SELECT rowid FROM ft($v)}]
+ do_test 4.6.$v { set L1 } $L2
+}
+
+execsql_pp {
+ SELECT fts5_decode(id, block) FROM ft_data
+}
+
+
+
+
+finish_test
+
projects / thirdparty / sqlite.git / commitdiff
