*/
struct Fts5PageWriter {
int pgno; /* Page number for this page */
+ int iPrevPgidx; /* Previous value written into pgidx */
Fts5Buffer buf; /* Buffer containing leaf data */
Fts5Buffer pgidx; /* Buffer containing page-index */
Fts5Buffer term; /* Buffer containing previous term on page */
int iTermLeafPgno;
int iTermLeafOffset;
- int iTermIdx;
+ int iPgidxOff; /* Next offset in pgidx */
int iEndofDoclist;
/* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */
#define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p))
-#define fts5LeafFirstTermOff(x) fts5LeafTermOff(x, 0)
-
/*
** poslist:
** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
}
#endif
+static int fts5LeafFirstTermOff(Fts5Data *pLeaf){
+ int ret;
+ fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret);
+ return ret;
+}
/*
** Close the read-only blob handle, if it is open.
}
}
-static void fts5SegIterLoadEod(Fts5Index *p, Fts5SegIter *pIter){
- Fts5Data *pLeaf = pIter->pLeaf;
- int nPg = (pLeaf->nn - pLeaf->szLeaf) / 2;
-
- assert( pIter->iLeafPgno==pIter->iTermLeafPgno );
- if( (pIter->iTermIdx+1)<nPg ){
- int iRead = pLeaf->szLeaf + (pIter->iTermIdx + 1) * 2;
- pIter->iEndofDoclist = fts5GetU16(&pLeaf->p[iRead]);
- }else{
- pIter->iEndofDoclist = pLeaf->nn+1;
- }
-
-}
-
static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){
u8 *a = pIter->pLeaf->p; /* Buffer to read data from */
int iOff = pIter->iLeafOffset;
pIter->iTermLeafPgno = pIter->iLeafPgno;
pIter->iLeafOffset = iOff;
- fts5SegIterLoadEod(p, pIter);
+ if( pIter->iPgidxOff>=pIter->pLeaf->nn ){
+ pIter->iEndofDoclist = pIter->pLeaf->nn+1;
+ }else{
+ int nExtra;
+ pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra);
+ pIter->iEndofDoclist += nExtra;
+ }
+
fts5SegIterLoadRowid(p, pIter);
}
pIter->iLeafOffset = 4;
assert_nc( pIter->pLeaf->nn>4 );
assert( fts5LeafFirstTermOff(pIter->pLeaf)==4 );
- pIter->iTermIdx = 0;
+ pIter->iPgidxOff = pIter->pLeaf->szLeaf+1;
fts5SegIterLoadTerm(p, pIter, 0);
fts5SegIterLoadNPos(p, pIter);
}
assert_nc( iOff<=pIter->iEndofDoclist );
if( iOff>=pIter->iEndofDoclist ){
bNewTerm = 1;
- if( pIter->iTermLeafPgno==pIter->iLeafPgno ){
- pIter->iTermIdx++;
- }else{
- pIter->iTermIdx = 0;
- }
if( iOff!=fts5LeafFirstTermOff(pLeaf) ){
iOff += fts5GetVarint32(&a[iOff], nKeep);
}
if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){
iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
pIter->iLeafOffset = iOff;
+
+ if( pLeaf->nn>pLeaf->szLeaf ){
+ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
+ &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist
+ );
+ }
+
}
else if( pLeaf->nn>pLeaf->szLeaf ){
- iOff = fts5LeafFirstTermOff(pLeaf);
+ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
+ &pLeaf->p[pLeaf->szLeaf], iOff
+ );
pIter->iLeafOffset = iOff;
+ pIter->iEndofDoclist = iOff;
bNewTerm = 1;
- pIter->iTermIdx = 0;
}
if( iOff>=pLeaf->szLeaf ){
p->rc = FTS5_CORRUPT;
fts5DataRelease(pIter->pLeaf);
pIter->pLeaf = 0;
}else{
+ int nExtra;
fts5SegIterLoadTerm(p, pIter, nKeep);
fts5SegIterLoadNPos(p, pIter);
if( pbNewTerm ) *pbNewTerm = 1;
if( fts5LeafIsTermless(pLast) ){
pIter->iEndofDoclist = pLast->nn+1;
}else{
- pIter->iEndofDoclist = fts5LeafTermOff(pLast, 0);
+ pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast);
}
}
){
int iOff;
const u8 *a = pIter->pLeaf->p;
- int n = pIter->pLeaf->szLeaf;
+ int szLeaf = pIter->pLeaf->szLeaf;
+ int n = pIter->pLeaf->nn;
int nMatch = 0;
int nKeep = 0;
int nNew = 0;
int iTerm = 0;
- int nPgTerm = (pIter->pLeaf->nn - pIter->pLeaf->szLeaf) >> 1;
+ int iTermOff;
+ int iPgidx; /* Current offset in pgidx */
+ int bEndOfPage = 0;
assert( p->rc==SQLITE_OK );
- assert( pIter->pLeaf );
- iOff = fts5LeafFirstTermOff(pIter->pLeaf);
- if( iOff<4 || iOff>=n ){
- p->rc = FTS5_CORRUPT;
- return;
- }
+ iPgidx = szLeaf;
+ iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff);
+ iOff = iTermOff;
while( 1 ){
- int i;
- int nCmp;
/* Figure out how many new bytes are in this term */
fts5IndexGetVarint32(a, iOff, nNew);
-
if( nKeep<nMatch ){
goto search_failed;
}
assert( nKeep>=nMatch );
if( nKeep==nMatch ){
+ int nCmp;
+ int i;
nCmp = MIN(nNew, nTerm-nMatch);
for(i=0; i<nCmp; i++){
if( a[iOff+i]!=pTerm[nMatch+i] ) break;
goto search_failed;
}
}
- iOff += nNew;
- iTerm++;
- if( iTerm>=nPgTerm ){
- iOff = n;
+ if( iPgidx>=n ){
+ bEndOfPage = 1;
break;
}
- iOff = fts5GetU16(&a[n + iTerm*2]);
+
+ iPgidx += fts5GetVarint32(&a[iPgidx], nKeep);
+ iTermOff += nKeep;
+ iOff = iTermOff;
/* Read the nKeep field of the next term. */
fts5IndexGetVarint32(a, iOff, nKeep);
fts5DataRelease(pIter->pLeaf);
pIter->pLeaf = 0;
return;
- }else if( iOff>=n ){
+ }else if( bEndOfPage ){
do {
iTerm = 0;
fts5SegIterNextPage(p, pIter);
if( pIter->pLeaf==0 ) return;
a = pIter->pLeaf->p;
if( fts5LeafIsTermless(pIter->pLeaf)==0 ){
- iOff = fts5LeafFirstTermOff(pIter->pLeaf);
+ fts5GetVarint32(&pIter->pLeaf->p[pIter->pLeaf->szLeaf], iOff);
if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){
p->rc = FTS5_CORRUPT;
}else{
}
search_success:
- pIter->iTermIdx = iTerm;
pIter->iLeafOffset = iOff + nNew;
pIter->iTermLeafOffset = pIter->iLeafOffset;
fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm);
fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]);
- fts5SegIterLoadEod(p, pIter);
+ if( iPgidx>=n ){
+ pIter->iEndofDoclist = pIter->pLeaf->nn+1;
+ }else{
+ int nExtra;
+ iPgidx += fts5GetVarint32(&a[iPgidx], nExtra);
+ pIter->iEndofDoclist = iTermOff + nExtra;
+ }
+ pIter->iPgidxOff = iPgidx;
+
fts5SegIterLoadRowid(p, pIter);
fts5SegIterLoadNPos(p, pIter);
}
fts5BufferZero(&pPage->buf);
fts5BufferZero(&pPage->pgidx);
fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero);
+ pPage->iPrevPgidx = 0;
pPage->pgno++;
/* Increase the leaves written counter */
assert( pPage->buf.n>4 || pWriter->bFirstTermInPage );
/* If the current leaf page is full, flush it to disk. */
- if( (pPage->buf.n + pPage->pgidx.n + nTerm + 2)>=p->pConfig->pgsz ){
+ if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){
if( pPage->buf.n>4 ){
fts5WriteFlushLeaf(p, pWriter);
}
}
/* TODO1: Updating pgidx here. */
+ pPgidx->n += sqlite3Fts5PutVarint(
+ &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx
+ );
+ pPage->iPrevPgidx = pPage->buf.n;
+#if 0
fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n);
pPgidx->n += 2;
+#endif
if( pWriter->bFirstTermInPage ){
nPrefix = 0;
}
}
-/*
-** The buffer passed as the second argument contains a leaf page that is
-** missing its page-idx array. The first term is guaranteed to start at
-** byte offset 4 of the buffer. The szLeaf field of the leaf page header
-** is already populated.
-**
-** This function appends a page-index to the buffer. The buffer is
-** guaranteed to be large enough to fit the page-index.
-*/
-static void fts5MakePageidx(
- Fts5Index *p, /* Fts index object to store any error in */
- Fts5Data *pOld, /* Original page data */
- int iOldidx, /* Index of term in pOld pgidx */
- Fts5Buffer *pBuf /* Buffer containing new page (no pgidx) */
-){
- if( p->rc==SQLITE_OK ){
- int iOff;
- int iEnd;
- int nByte;
- int iEndOld; /* Byte after term iOldIdx on old page */
- int iEndNew; /* Byte after term iOldIdx on new page */
- int ii;
- int nPgIdx = (pOld->nn - pOld->szLeaf) / 2;
-
- /* Determine end of term on old page */
- iEndOld = fts5LeafTermOff(pOld, iOldidx);
- if( iOldidx>0 ){
- iEndOld += fts5GetVarint32(&pOld->p[iEndOld], nByte);
- }
- iEndOld += fts5GetVarint32(&pOld->p[iEndOld], nByte);
- iEndOld += nByte;
-
- /* Determine end of term on new page */
- iEndNew = 4 + fts5GetVarint32(&pBuf->p[4], nByte);
- iEndNew += nByte;
-
- fts5PutU16(&pBuf->p[pBuf->n], 4);
- pBuf->n += 2;
- for(ii=iOldidx+1; ii<nPgIdx; ii++){
- int iVal = fts5LeafTermOff(pOld, ii);
- fts5PutU16(&pBuf->p[pBuf->n], iVal + (iEndNew - iEndOld));
- pBuf->n += 2;
- }
- }
-}
-
/*
** Iterator pIter was used to iterate through the input segments of on an
** incremental merge operation. This function is called if the incremental
}
/* Set up the new page-index array */
- fts5MakePageidx(p, pData, pSeg->iTermIdx, &buf);
+ fts5BufferAppendVarint(&p->rc, &buf, 4);
+ if( pSeg->iLeafPgno==pSeg->iTermLeafPgno
+ && pSeg->iEndofDoclist<pData->szLeaf
+ ){
+ int nDiff = pData->szLeaf - pSeg->iEndofDoclist;
+ fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4);
+ fts5BufferAppendBlob(&p->rc, &buf,
+ pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff]
+ );
+ }
fts5DataRelease(pData);
pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno;
static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){
int nPg = (pLeaf->nn - pLeaf->szLeaf) / 2;
+ int iTermOff = 0;
int ii;
+
Fts5Buffer buf1 = {0,0,0};
Fts5Buffer buf2 = {0,0,0};
- for(ii=0; p->rc==SQLITE_OK && ii<nPg; ii++){
+ ii = pLeaf->szLeaf;
+ while( ii<pLeaf->nn && p->rc==SQLITE_OK ){
int res;
- int iOff = fts5LeafTermOff(pLeaf, ii);
+ int iOff;
+ int nIncr;
+
+ ii += fts5GetVarint32(&pLeaf->p[ii], nIncr);
+ iTermOff += nIncr;
+ iOff = iTermOff;
+
if( iOff>=pLeaf->szLeaf ){
p->rc = FTS5_CORRUPT;
- }else if( ii==0 ){
+ }else if( iTermOff==nIncr ){
int nByte;
iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte);
if( (iOff+nByte)>pLeaf->szLeaf ){
fts5DecodeStructure(&rc, &s, a, n);
}
}else{
- Fts5Buffer term;
+ Fts5Buffer term; /* Current term read from page */
+ int szLeaf; /* Offset of pgidx in a[] */
+ int iPgidxOff;
+ int iPgidxPrev = 0; /* Previous value read from pgidx */
int iTermOff = 0;
- int szLeaf = 0;
int iRowidOff = 0;
int iOff;
- int nPgTerm = 0;
int nDoclist;
- int i;
memset(&term, 0, sizeof(Fts5Buffer));
goto decode_out;
}else{
iRowidOff = fts5GetU16(&a[0]);
- szLeaf = fts5GetU16(&a[2]);
- nPgTerm = (n - szLeaf) / 2;
- if( nPgTerm ){
- iTermOff = fts5GetU16(&a[szLeaf]);
+ iPgidxOff = szLeaf = fts5GetU16(&a[2]);
+ if( iPgidxOff<n ){
+ fts5GetVarint32(&a[iPgidxOff], iTermOff);
}
}
nDoclist = (iTermOff ? iTermOff : szLeaf) - iOff;
fts5DecodeDoclist(&rc, &s, &a[iOff], nDoclist);
- for(i=0; i<nPgTerm; i++){
- int nByte;
- int iEnd = (i+1)<nPgTerm ? fts5GetU16(&a[szLeaf+i*2+2]) : szLeaf;
- iOff = fts5GetU16(&a[szLeaf + i*2]);
- if( i>0 ){
+ while( iPgidxOff<n ){
+ int bFirst = (iPgidxOff==szLeaf); /* True for first term on page */
+ int nByte; /* Bytes of data */
+ int iEnd;
+
+ iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nByte);
+ iPgidxPrev += nByte;
+ iOff = iPgidxPrev;
+
+ if( iPgidxOff<n ){
+ fts5GetVarint32(&a[iPgidxOff], nByte);
+ iEnd = iPgidxPrev + nByte;
+ }else{
+ iEnd = szLeaf;
+ }
+
+ if( bFirst==0 ){
iOff += fts5GetVarint32(&a[iOff], nByte);
term.n = nByte;
}
iOff += fts5GetVarint32(&a[iOff], nByte);
fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]);
iOff += nByte;
+
sqlite3Fts5BufferAppendPrintf(
&rc, &s, " term=%.*s", term.n, (const char*)term.p
);