** without overreading if the records are corrupt.
*/
#define FTS5_DATA_ZERO_PADDING 8
+#define FTS5_DATA_PADDING 20
-typedef struct Fts5BtreeIter Fts5BtreeIter;
-typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel;
typedef struct Fts5Data Fts5Data;
typedef struct Fts5DlidxIter Fts5DlidxIter;
typedef struct Fts5DlidxLvl Fts5DlidxLvl;
sqlite3_blob *pReader; /* RO incr-blob open on %_data table */
sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */
sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */
+ sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */
+ sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */
+ sqlite3_stmt *pIdxSelect;
int nRead; /* Total number of blocks read */
};
};
struct Fts5SegWriter {
int iSegid; /* Segid to write to */
- int nWriter; /* Number of entries in aWriter */
- Fts5PageWriter *aWriter; /* Array of PageWriter objects */
+ Fts5PageWriter writer; /* PageWriter object */
i64 iPrevRowid; /* Previous docid written to current leaf */
u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */
u8 bFirstRowidInPage; /* True if next rowid is first in page */
int nDlidx; /* Allocated size of aDlidx[] array */
Fts5DlidxWriter *aDlidx; /* Array of Fts5DlidxWriter objects */
+
+ /* Values to insert into the %_idx table */
+ Fts5Buffer btterm; /* TODO: Docs */
+ int iBtPage; /* TODO: This */
};
/*
-/*
-** An Fts5BtreeIter object is used to iterate through all entries in the
-** b-tree hierarchy belonging to a single fts5 segment. In this case the
-** "b-tree hierarchy" is all b-tree nodes except leaves. Each entry in the
-** b-tree hierarchy consists of the following:
-**
-** iLeaf: The page number of the leaf page the entry points to.
-**
-** term: A split-key that all terms on leaf page $iLeaf must be greater
-** than or equal to. The "term" associated with the first b-tree
-** hierarchy entry (the one that points to leaf page 1) is always
-** an empty string.
-**
-** nEmpty: The number of empty (termless) leaf pages that immediately
-** following iLeaf.
-**
-** The Fts5BtreeIter object is only used as part of the integrity-check code.
-*/
-struct Fts5BtreeIterLevel {
- Fts5NodeIter s; /* Iterator for the current node */
- Fts5Data *pData; /* Data for the current node */
-};
-struct Fts5BtreeIter {
- Fts5Index *p; /* FTS5 backend object */
- Fts5StructureSegment *pSeg; /* Iterate through this segment's b-tree */
- int nLvl; /* Size of aLvl[] array */
- Fts5BtreeIterLevel *aLvl; /* Level for each tier of b-tree */
-
- /* Output variables */
- Fts5Buffer term; /* Current term */
- int iLeaf; /* Leaf containing terms >= current term */
- int nEmpty; /* Number of "empty" leaves following iLeaf */
- int bEof; /* Set to true at EOF */
- int bDlidx; /* True if there exists a dlidx */
-};
-
-
/*
** The first argument passed to this macro is a pointer to an Fts5Buffer
** object.
rc = SQLITE_NOMEM;
}
}else{
- int nSpace = nByte + FTS5_DATA_ZERO_PADDING;
+ int nSpace = nByte + FTS5_DATA_PADDING;
pRet = (Fts5Data*)sqlite3_malloc(nSpace+sizeof(Fts5Data));
if( pRet ){
pRet->n = nByte;
sqlite3_free(pData);
}
+static int fts5IndexPrepareStmt(
+ Fts5Index *p,
+ sqlite3_stmt **ppStmt,
+ char *zSql
+){
+ if( p->rc==SQLITE_OK ){
+ if( zSql ){
+ p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0);
+ }else{
+ p->rc = SQLITE_NOMEM;
+ }
+ }
+ sqlite3_free(zSql);
+ return p->rc;
+}
+
+
/*
** INSERT OR REPLACE a record into the %_data table.
*/
if( p->pWriter==0 ){
int rc = SQLITE_OK;
Fts5Config *pConfig = p->pConfig;
- char *zSql = sqlite3Fts5Mprintf(&rc,
- "REPLACE INTO '%q'.%Q(id, block) VALUES(?,?)", pConfig->zDb, p->zDataTbl
- );
- if( zSql ){
- rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pWriter, 0);
- sqlite3_free(zSql);
- }
- if( rc!=SQLITE_OK ){
- p->rc = rc;
- return;
- }
+ fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf(
+ "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)",
+ pConfig->zDb, pConfig->zName
+ ));
+ if( p->rc ) return;
}
sqlite3_bind_int64(p->pWriter, 1, iRowid);
int rc;
Fts5Config *pConfig = p->pConfig;
char *zSql = sqlite3_mprintf(
- "DELETE FROM '%q'.%Q WHERE id>=? AND id<=?", pConfig->zDb, p->zDataTbl
+ "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?",
+ pConfig->zDb, pConfig->zName
);
if( zSql==0 ){
rc = SQLITE_NOMEM;
i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0, 0);
i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0, 0)-1;
fts5DataDelete(p, iFirst, iLast);
+ if( p->pIdxDeleter==0 ){
+ Fts5Config *pConfig = p->pConfig;
+ fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf(
+ "DELETE FROM '%q'.'%q_idx' WHERE segid=?",
+ pConfig->zDb, pConfig->zName
+ ));
+ }
+ if( p->rc==SQLITE_OK ){
+ sqlite3_bind_int(p->pIdxDeleter, 1, iSegid);
+ sqlite3_step(p->pIdxDeleter);
+ p->rc = sqlite3_reset(p->pIdxDeleter);
+ }
}
/*
/* This block sets stack variable iPg to the leaf page number that may
** contain term (pTerm/nTerm), if it is present in the segment. */
- for(h=pSeg->nHeight-1; h>0; h--){
- i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, h, iPg);
- fts5DataBuffer(p, pBuf, iRowid);
- if( p->rc ) break;
- iPg = fts5NodeSeek(pBuf, pTerm, nTerm, &bDlidx);
+ if( p->pIdxSelect==0 ){
+ Fts5Config *pConfig = p->pConfig;
+ fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf(
+ "SELECT pgno, dlidx FROM '%q'.'%q_idx' WHERE "
+ "segid=? AND term<=? ORDER BY term DESC LIMIT 1",
+ pConfig->zDb, pConfig->zName
+ ));
+ }
+ if( p->rc ) return;
+ sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid);
+ sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC);
+ if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){
+ iPg = sqlite3_column_int(p->pIdxSelect, 0);
+ bDlidx = sqlite3_column_int(p->pIdxSelect, 1);
}
+ p->rc = sqlite3_reset(p->pIdxSelect);
if( iPg<pSeg->pgnoFirst ){
iPg = pSeg->pgnoFirst;
}
/*
-** If an "nEmpty" record must be written to the b-tree before the next
-** term, write it now.
+** If the current doclist-index accumulating in pWriter->aDlidx[] is large
+** enough, flush it to disk and return 1. Otherwise discard it and return
+** zero.
*/
-static void fts5WriteBtreeNEmpty(Fts5Index *p, Fts5SegWriter *pWriter){
- if( pWriter->nEmpty ){
- int bFlag = 0;
- Fts5PageWriter *pPg;
- pPg = &pWriter->aWriter[1];
+static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){
+ int bFlag = 0;
- /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written
- ** to the database, also write the doclist-index to disk. */
- if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){
- bFlag = 1;
- }
- fts5WriteDlidxClear(p, pWriter, bFlag);
- fts5BufferAppendVarint(&p->rc, &pPg->buf, bFlag);
- fts5BufferAppendVarint(&p->rc, &pPg->buf, pWriter->nEmpty);
- pWriter->nEmpty = 0;
- }else{
- fts5WriteDlidxClear(p, pWriter, 0);
+ /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written
+ ** to the database, also write the doclist-index to disk. */
+ if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){
+ bFlag = 1;
}
-
- assert( pWriter->nDlidx==0 || pWriter->aDlidx[0].buf.n==0 );
- assert( pWriter->nDlidx==0 || pWriter->aDlidx[0].bPrevValid==0 );
+ fts5WriteDlidxClear(p, pWriter, bFlag);
+ pWriter->nEmpty = 0;
+ return bFlag;
}
-static void fts5WriteBtreeGrow(Fts5Index *p, Fts5SegWriter *pWriter){
- if( p->rc==SQLITE_OK ){
- Fts5PageWriter *aNew;
- Fts5PageWriter *pNew;
- int nNew = sizeof(Fts5PageWriter) * (pWriter->nWriter+1);
-
- aNew = (Fts5PageWriter*)sqlite3_realloc(pWriter->aWriter, nNew);
- if( aNew==0 ){
- p->rc = SQLITE_NOMEM;
- return;
- }
+/*
+** This function is called whenever processing of the doclist for the
+** last term on leaf page (pWriter->iBtPage) is completed.
+**
+** The doclist-index for that term is currently stored in-memory within the
+** Fts5SegWriter.aDlidx[] array. If it is large enough, this function
+** writes it out to disk. Or, if it is too small to bother with, discards
+** it.
+**
+** Fts5SegWriter.btterm currently contains the first term on page iBtPage.
+*/
+static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){
+ int bFlag;
- pNew = &aNew[pWriter->nWriter];
- memset(pNew, 0, sizeof(Fts5PageWriter));
- pNew->pgno = 1;
- fts5BufferAppendVarint(&p->rc, &pNew->buf, 1);
+ assert( pWriter->iBtPage || pWriter->nEmpty==0 );
+ if( pWriter->iBtPage==0 ) return;
+ bFlag = fts5WriteFlushDlidx(p, pWriter);
- pWriter->nWriter++;
- pWriter->aWriter = aNew;
+ if( p->rc==SQLITE_OK ){
+ const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:"");
+ /* The following was already done in fts5WriteInit(): */
+ /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */
+ sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC);
+ sqlite3_bind_int(p->pIdxWriter, 3, pWriter->iBtPage);
+ sqlite3_bind_int(p->pIdxWriter, 4, bFlag);
+ sqlite3_step(p->pIdxWriter);
+ p->rc = sqlite3_reset(p->pIdxWriter);
}
+ pWriter->iBtPage = 0;
}
/*
Fts5SegWriter *pWriter, /* Writer object */
int nTerm, const u8 *pTerm /* First term on new page */
){
- int iHeight;
- for(iHeight=1; 1; iHeight++){
- Fts5PageWriter *pPage;
-
- if( iHeight>=pWriter->nWriter ){
- fts5WriteBtreeGrow(p, pWriter);
- if( p->rc ) return;
- }
- pPage = &pWriter->aWriter[iHeight];
-
- fts5WriteBtreeNEmpty(p, pWriter);
-
- if( pPage->buf.n>=p->pConfig->pgsz ){
- /* pPage will be written to disk. The term will be written into the
- ** parent of pPage. */
- i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, iHeight, pPage->pgno);
- fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n);
- fts5BufferZero(&pPage->buf);
- fts5BufferZero(&pPage->term);
- fts5BufferAppendVarint(&p->rc, &pPage->buf, pPage[-1].pgno);
- pPage->pgno++;
- }else{
- int nPre = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
- fts5BufferAppendVarint(&p->rc, &pPage->buf, nPre+2);
- fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm-nPre);
- fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm-nPre, pTerm+nPre);
- fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm);
- break;
- }
- }
+ fts5WriteFlushBtree(p, pWriter);
+ fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm);
+ pWriter->iBtPage = pWriter->writer.pgno;
}
/*
if( pDlidx->bPrevValid ){
iVal = iRowid - pDlidx->iPrev;
}else{
- i64 iPgno = (i==0 ? pWriter->aWriter[0].pgno : pDlidx[-1].pgno);
+ i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno);
assert( pDlidx->buf.n==0 );
sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone);
sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno);
static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
- Fts5PageWriter *pPage = &pWriter->aWriter[0];
+ Fts5PageWriter *pPage = &pWriter->writer;
i64 iRowid;
if( pWriter->bFirstTermInPage ){
int nTerm, const u8 *pTerm
){
int nPrefix; /* Bytes of prefix compression for term */
- Fts5PageWriter *pPage = &pWriter->aWriter[0];
+ Fts5PageWriter *pPage = &pWriter->writer;
assert( pPage->buf.n==0 || pPage->buf.n>4 );
if( pPage->buf.n==0 ){
n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
}
fts5WriteBtreeTerm(p, pWriter, n, pTerm);
- pPage = &pWriter->aWriter[0];
+ pPage = &pWriter->writer;
}
}else{
nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
int nPos
){
if( p->rc==SQLITE_OK ){
- Fts5PageWriter *pPage = &pWriter->aWriter[0];
+ Fts5PageWriter *pPage = &pWriter->writer;
/* If this is to be the first docid written to the page, set the
** docid-pointer in the page-header. Also append a value to the dlidx
const u8 *aData,
int nData
){
- Fts5PageWriter *pPage = &pWriter->aWriter[0];
+ Fts5PageWriter *pPage = &pWriter->writer;
const u8 *a = aData;
int n = nData;
}
static void fts5WriteAppendZerobyte(Fts5Index *p, Fts5SegWriter *pWriter){
- fts5BufferAppendVarint(&p->rc, &pWriter->aWriter[0].buf, 0);
+ fts5BufferAppendVarint(&p->rc, &pWriter->writer.buf, 0);
}
/*
int *pnLeaf /* OUT: Number of leaf pages in b-tree */
){
int i;
+ Fts5PageWriter *pLeaf = &pWriter->writer;
if( p->rc==SQLITE_OK ){
- Fts5PageWriter *pLeaf = &pWriter->aWriter[0];
if( pLeaf->pgno==1 && pLeaf->buf.n==0 ){
*pnLeaf = 0;
*pnHeight = 0;
fts5WriteFlushLeaf(p, pWriter);
}
*pnLeaf = pLeaf->pgno-1;
- if( pWriter->nWriter==1 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){
- fts5WriteBtreeGrow(p, pWriter);
- }
- if( pWriter->nWriter>1 ){
- fts5WriteBtreeNEmpty(p, pWriter);
- }
- *pnHeight = pWriter->nWriter;
- for(i=1; i<pWriter->nWriter; i++){
- Fts5PageWriter *pPg = &pWriter->aWriter[i];
- fts5DataWrite(p,
- FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pPg->pgno),
- pPg->buf.p, pPg->buf.n
- );
- }
+ fts5WriteFlushBtree(p, pWriter);
+ *pnHeight = 0;
}
}
- for(i=0; i<pWriter->nWriter; i++){
- Fts5PageWriter *pPg = &pWriter->aWriter[i];
- fts5BufferFree(&pPg->term);
- fts5BufferFree(&pPg->buf);
- }
- sqlite3_free(pWriter->aWriter);
+ fts5BufferFree(&pLeaf->term);
+ fts5BufferFree(&pLeaf->buf);
+ fts5BufferFree(&pWriter->btterm);
for(i=0; i<pWriter->nDlidx; i++){
sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf);
memset(pWriter, 0, sizeof(Fts5SegWriter));
pWriter->iSegid = iSegid;
- pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, sizeof(Fts5PageWriter));
- if( fts5WriteDlidxGrow(p, pWriter, 1) ) return;
- pWriter->nWriter = 1;
- pWriter->nDlidx = 1;
- pWriter->aWriter[0].pgno = 1;
+ fts5WriteDlidxGrow(p, pWriter, 1);
+ pWriter->writer.pgno = 1;
pWriter->bFirstTermInPage = 1;
-}
+ pWriter->iBtPage = 1;
-static void fts5WriteInitForAppend(
- Fts5Index *p, /* FTS5 backend object */
- Fts5SegWriter *pWriter, /* Writer to initialize */
- Fts5StructureSegment *pSeg /* Segment object to append to */
-){
- int nByte = pSeg->nHeight * sizeof(Fts5PageWriter);
- memset(pWriter, 0, sizeof(Fts5SegWriter));
- pWriter->iSegid = pSeg->iSegid;
- pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, nByte);
- pWriter->aDlidx = (Fts5DlidxWriter*)fts5IdxMalloc(p, sizeof(Fts5DlidxWriter));
+ if( p->pIdxWriter==0 ){
+ Fts5Config *pConfig = p->pConfig;
+ fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf(
+ "INSERT INTO '%q'.'%q_idx'(segid,term,pgno,dlidx) VALUES(?,?,?,?)",
+ pConfig->zDb, pConfig->zName
+ ));
+ }
if( p->rc==SQLITE_OK ){
- int pgno = 1;
- int i;
- pWriter->nDlidx = 1;
- pWriter->nWriter = pSeg->nHeight;
- pWriter->aWriter[0].pgno = pSeg->pgnoLast+1;
- for(i=pSeg->nHeight-1; i>0; i--){
- i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pgno);
- Fts5PageWriter *pPg = &pWriter->aWriter[i];
- pPg->pgno = pgno;
- fts5DataBuffer(p, &pPg->buf, iRowid);
- if( p->rc==SQLITE_OK ){
- Fts5NodeIter ss;
- fts5NodeIterInit(pPg->buf.p, pPg->buf.n, &ss);
- while( ss.aData ) fts5NodeIterNext(&p->rc, &ss);
- fts5BufferSet(&p->rc, &pPg->term, ss.term.n, ss.term.p);
- pgno = ss.iChild;
- fts5NodeIterFree(&ss);
- }
- }
- assert( p->rc!=SQLITE_OK || (pgno+pWriter->nEmpty)==pSeg->pgnoLast );
- pWriter->bFirstTermInPage = 1;
- assert( pWriter->aWriter[0].term.n==0 );
+ sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid);
}
}
fts5BufferAppendBlob(&p->rc, &buf, pData->n - iOff, &pData->p[iOff]);
fts5DataRelease(pData);
pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno;
- fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1),iLeafRowid);
+ fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1), iLeafRowid);
fts5DataWrite(p, iLeafRowid, buf.p, buf.n);
}
}
pLvlOut = &pStruct->aLevel[iLvl+1];
assert( pLvlOut->nSeg>0 );
nInput = pLvl->nMerge;
- fts5WriteInitForAppend(p, &writer, &pLvlOut->aSeg[pLvlOut->nSeg-1]);
pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1];
+
+ fts5WriteInit(p, &writer, pSeg->iSegid);
+ writer.writer.pgno = pSeg->pgnoLast+1;
+ writer.iBtPage = 0;
}else{
int iSegid = fts5AllocateSegid(p, pStruct);
pStruct->nSegment--;
}
}else{
- assert( pSeg->nHeight>0 && pSeg->pgnoLast>0 );
+ assert( pSeg->pgnoLast>0 );
fts5TrimSegments(p, pIter);
pLvl->nMerge = nInput;
}
/* Pre-allocate the buffer used to assemble leaf pages to the target
** page size. */
assert( pgsz>0 );
- pBuf = &writer.aWriter[0].buf;
+ pBuf = &writer.writer.buf;
fts5BufferGrow(&p->rc, pBuf, pgsz + 20);
/* Begin scanning through hash table entries. This loop runs once for each
** flush the leaf to disk here. */
if( (pBuf->n + nTerm + 2) > pgsz ){
fts5WriteFlushLeaf(p, &writer);
- pBuf = &writer.aWriter[0].buf;
+ pBuf = &writer.writer.buf;
if( (nTerm + 32) > pBuf->nSpace ){
fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n);
if( p->rc ) break;
}else{
fts5PutU16(&pBuf->p[2], pBuf->n);
writer.bFirstTermInPage = 0;
- if( writer.aWriter[0].pgno!=1 ){
+ if( writer.writer.pgno!=1 ){
int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm);
fts5WriteBtreeTerm(p, &writer, nPre+1, (const u8*)zTerm);
- pBuf = &writer.aWriter[0].buf;
+ pBuf = &writer.writer.buf;
assert( nPre<nTerm );
}
nSuffix = nTerm;
** doclist-index is to be generated for this doclist, it will be
** associated with this page. */
assert( writer.nDlidx>0 && writer.aDlidx[0].buf.n==0 );
- writer.aDlidx[0].pgno = writer.aWriter[0].pgno;
+ writer.aDlidx[0].pgno = writer.writer.pgno;
if( pgsz>=(pBuf->n + nDoclist + 1) ){
/* The entire doclist will fit on the current leaf. */
iPos += n;
if( pBuf->n>=pgsz ){
fts5WriteFlushLeaf(p, &writer);
- pBuf = &writer.aWriter[0].buf;
+ pBuf = &writer.writer.buf;
}
if( iPos>=nCopy ) break;
}
fts5StructurePromote(p, 0, pStruct);
}
-
fts5IndexAutomerge(p, &pStruct, pgnoLast);
fts5IndexCrisismerge(p, &pStruct);
fts5StructureWrite(p, pStruct);
rc = sqlite3Fts5CreateTable(
pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
);
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts5CreateTable(pConfig, "idx",
+ "segid, term, pgno, dlidx, PRIMARY KEY(segid, term)",
+ 1, pzErr
+ );
+ }
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexReinit(p);
}
assert( p->pReader==0 );
sqlite3_finalize(p->pWriter);
sqlite3_finalize(p->pDeleter);
+ sqlite3_finalize(p->pIdxWriter);
+ sqlite3_finalize(p->pIdxDeleter);
+ sqlite3_finalize(p->pIdxSelect);
sqlite3Fts5HashFree(p->pHash);
sqlite3Fts5BufferFree(&p->scratch);
sqlite3_free(p->zDataTbl);
return ret;
}
-static void fts5BtreeIterInit(
- Fts5Index *p,
- Fts5StructureSegment *pSeg,
- Fts5BtreeIter *pIter
-){
- int nByte;
- int i;
- nByte = sizeof(pIter->aLvl[0]) * (pSeg->nHeight-1);
- memset(pIter, 0, sizeof(*pIter));
- if( nByte ){
- pIter->aLvl = (Fts5BtreeIterLevel*)fts5IdxMalloc(p, nByte);
- }
- if( p->rc==SQLITE_OK ){
- pIter->nLvl = pSeg->nHeight-1;
- pIter->p = p;
- pIter->pSeg = pSeg;
- }
- for(i=0; p->rc==SQLITE_OK && i<pIter->nLvl; i++){
- i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, i+1, 1);
- Fts5Data *pData;
- pIter->aLvl[i].pData = pData = fts5DataRead(p, iRowid);
- if( pData ){
- fts5NodeIterInit(pData->p, pData->n, &pIter->aLvl[i].s);
- }
- }
-
- if( pIter->nLvl==0 || p->rc ){
- pIter->bEof = 1;
- pIter->iLeaf = pSeg->pgnoLast;
- }else{
- pIter->nEmpty = pIter->aLvl[0].s.nEmpty;
- pIter->iLeaf = pIter->aLvl[0].s.iChild;
- pIter->bDlidx = pIter->aLvl[0].s.bDlidx;
- }
-}
-
-static void fts5BtreeIterNext(Fts5BtreeIter *pIter){
- Fts5Index *p = pIter->p;
- int i;
-
- assert( pIter->bEof==0 && pIter->aLvl[0].s.aData );
- for(i=0; i<pIter->nLvl && p->rc==SQLITE_OK; i++){
- Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
- fts5NodeIterNext(&p->rc, &pLvl->s);
- if( pLvl->s.aData ){
- fts5BufferSet(&p->rc, &pIter->term, pLvl->s.term.n, pLvl->s.term.p);
- break;
- }else{
- fts5NodeIterFree(&pLvl->s);
- fts5DataRelease(pLvl->pData);
- pLvl->pData = 0;
- }
- }
- if( i==pIter->nLvl || p->rc ){
- pIter->bEof = 1;
- }else{
- int iSegid = pIter->pSeg->iSegid;
- for(i--; i>=0; i--){
- Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
- i64 iRowid = FTS5_SEGMENT_ROWID(iSegid, i+1, pLvl[1].s.iChild);
- pLvl->pData = fts5DataRead(p, iRowid);
- if( pLvl->pData ){
- fts5NodeIterInit(pLvl->pData->p, pLvl->pData->n, &pLvl->s);
- }
- }
- }
-
- pIter->nEmpty = pIter->aLvl[0].s.nEmpty;
- pIter->bDlidx = pIter->aLvl[0].s.bDlidx;
- pIter->iLeaf = pIter->aLvl[0].s.iChild;
-}
-
-static void fts5BtreeIterFree(Fts5BtreeIter *pIter){
- int i;
- for(i=0; i<pIter->nLvl; i++){
- Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
- fts5NodeIterFree(&pLvl->s);
- if( pLvl->pData ){
- fts5DataRelease(pLvl->pData);
- pLvl->pData = 0;
- }
- }
- sqlite3_free(pIter->aLvl);
- fts5BufferFree(&pIter->term);
-}
-
#ifdef SQLITE_DEBUG
/*
** This function is purely an internal test. It does not contribute to
# define fts5TestTerm(u,v,w,x,y,z)
#endif
+/*
+** Check that:
+**
+** 1) All leaves of pSeg between iFirst and iLast (inclusive) exist and
+** contain zero terms.
+** 2) All leaves of pSeg between iNoRowid and iLast (inclusive) exist and
+** contain zero rowids.
+*/
+static void fts5IndexIntegrityCheckEmpty(
+ Fts5Index *p,
+ Fts5StructureSegment *pSeg, /* Segment to check internal consistency */
+ int iFirst,
+ int iNoRowid,
+ int iLast
+){
+ int i;
+
+ /* Now check that the iter.nEmpty leaves following the current leaf
+ ** (a) exist and (b) contain no terms. */
+ for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){
+ Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, i));
+ if( pLeaf ){
+ if( 0!=fts5GetU16(&pLeaf->p[2]) ) p->rc = FTS5_CORRUPT;
+ if( i>=iNoRowid && 0!=fts5GetU16(&pLeaf->p[0]) ) p->rc = FTS5_CORRUPT;
+ }
+ fts5DataRelease(pLeaf);
+ if( p->rc ) break;
+ }
+}
+
static void fts5IndexIntegrityCheckSegment(
Fts5Index *p, /* FTS5 backend object */
Fts5StructureSegment *pSeg /* Segment to check internal consistency */
){
- Fts5BtreeIter iter; /* Used to iterate through b-tree hierarchy */
+ Fts5Config *pConfig = p->pConfig;
+ sqlite3_stmt *pStmt = 0;
+ int rc2;
+ int iIdxPrevLeaf = pSeg->pgnoFirst-1;
+ int iDlidxPrevLeaf = pSeg->pgnoLast;
if( pSeg->pgnoFirst==0 ) return;
+ fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf(
+ "SELECT segid, term, pgno, dlidx FROM '%q'.'%q_idx' WHERE segid=%d",
+ pConfig->zDb, pConfig->zName, pSeg->iSegid
+ ));
+
/* Iterate through the b-tree hierarchy. */
- for(fts5BtreeIterInit(p, pSeg, &iter);
- p->rc==SQLITE_OK && iter.bEof==0;
- fts5BtreeIterNext(&iter)
- ){
+ while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
i64 iRow; /* Rowid for this leaf */
Fts5Data *pLeaf; /* Data for this leaf */
int iOff; /* Offset of first term on leaf */
int i; /* Used to iterate through empty leaves */
+ int nIdxTerm = sqlite3_column_bytes(pStmt, 1);
+ const char *zIdxTerm = (const char*)sqlite3_column_text(pStmt, 1);
+ int iIdxLeaf = sqlite3_column_int(pStmt, 2);
+ int bIdxDlidx = sqlite3_column_int(pStmt, 3);
+
/* If the leaf in question has already been trimmed from the segment,
** ignore this b-tree entry. Otherwise, load it into memory. */
- if( iter.iLeaf<pSeg->pgnoFirst ) continue;
- iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iter.iLeaf);
+ if( iIdxLeaf<pSeg->pgnoFirst ) continue;
+ iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iIdxLeaf);
pLeaf = fts5DataRead(p, iRow);
if( pLeaf==0 ) break;
/* Check that the leaf contains at least one term, and that it is equal
- ** to or larger than the split-key in iter.term. Also check that if there
+ ** to or larger than the split-key in zIdxTerm. Also check that if there
** is also a rowid pointer within the leaf page header, it points to a
** location before the term. */
iOff = fts5GetU16(&pLeaf->p[2]);
p->rc = FTS5_CORRUPT;
}else{
iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
- res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n));
- if( res==0 ) res = nTerm - iter.term.n;
+ res = memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm));
+ if( res==0 ) res = nTerm - nIdxTerm;
if( res<0 ) p->rc = FTS5_CORRUPT;
}
}
/* Now check that the iter.nEmpty leaves following the current leaf
** (a) exist and (b) contain no terms. */
- for(i=1; p->rc==SQLITE_OK && i<=iter.nEmpty; i++){
- pLeaf = fts5DataRead(p, iRow+i);
- if( pLeaf && 0!=fts5GetU16(&pLeaf->p[2]) ){
- p->rc = FTS5_CORRUPT;
- }
- fts5DataRelease(pLeaf);
- }
+ fts5IndexIntegrityCheckEmpty(
+ p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1
+ );
+ if( p->rc ) break;
/* If there is a doclist-index, check that it looks right. */
- if( iter.bDlidx ){
+ if( bIdxDlidx ){
Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */
- int iPrevLeaf = iter.iLeaf;
+ int iPrevLeaf = iIdxLeaf;
int iSegid = pSeg->iSegid;
int iPg;
i64 iKey;
- for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iter.iLeaf);
+ for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iIdxLeaf);
fts5DlidxIterEof(p, pDlidx)==0;
fts5DlidxIterNext(p, pDlidx)
){
}
}
- for(iPg=iPrevLeaf+1; iPg<=(iter.iLeaf + iter.nEmpty); iPg++){
- iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPg);
- pLeaf = fts5DataRead(p, iKey);
- if( pLeaf ){
- if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT;
- fts5DataRelease(pLeaf);
- }
- }
-
+ iDlidxPrevLeaf = iPg;
fts5DlidxIterFree(pDlidx);
- fts5TestDlidxReverse(p, iSegid, iter.iLeaf);
+ fts5TestDlidxReverse(p, iSegid, iIdxLeaf);
+ }else{
+ iDlidxPrevLeaf = pSeg->pgnoLast;
+ /* TODO: Check there is no doclist index */
}
+
+ iIdxPrevLeaf = iIdxLeaf;
}
+ rc2 = sqlite3_finalize(pStmt);
+ if( p->rc==SQLITE_OK ) p->rc = rc2;
+
/* Page iter.iLeaf must now be the rightmost leaf-page in the segment */
+#if 0
if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){
p->rc = FTS5_CORRUPT;
}
-
- fts5BtreeIterFree(&iter);
+#endif
}