From: dan <dan@noemail.net>
Date: Tue, 29 Mar 2016 21:19:04 +0000 (+0000)
Subject: Fix some code and test coverage issues in fts5_index.c.
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fheads%2Ffts5;p=thirdparty%2Fsqlite.git

Fix some code and test coverage issues in fts5_index.c.

FossilOrigin-Name: 7635c68018ce1656a1c5d6eebaf8f3a8e8839b59
---

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 95e4fe3848..85ee7676a7 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -266,7 +266,6 @@ typedef struct Fts5DlidxIter Fts5DlidxIter;
 typedef struct Fts5DlidxLvl Fts5DlidxLvl;
 typedef struct Fts5DlidxWriter Fts5DlidxWriter;
 typedef struct Fts5Iter Fts5Iter;
-typedef struct Fts5PageWriter Fts5PageWriter;
 typedef struct Fts5SegIter Fts5SegIter;
 typedef struct Fts5DoclistIter Fts5DoclistIter;
 typedef struct Fts5SegWriter Fts5SegWriter;
@@ -356,13 +355,6 @@ struct Fts5Structure {
 /*
 ** An object of type Fts5SegWriter is used to write to segments.
 */
-struct Fts5PageWriter {
-  int pgno;                       /* Page number for this page */
-  int iPrevPgidx;                 /* Previous value written into pgidx */
-  Fts5Buffer buf;                 /* Buffer containing leaf data */
-  Fts5Buffer pgidx;               /* Buffer containing page-index */
-  Fts5Buffer term;                /* Buffer containing previous term on page */
-};
 struct Fts5DlidxWriter {
   int pgno;                       /* Page number for this page */
   int bPrevValid;                 /* True if iPrev is valid */
@@ -371,12 +363,15 @@ struct Fts5DlidxWriter {
 };
 struct Fts5SegWriter {
   int iSegid;                     /* Segid to write to */
-  Fts5PageWriter writer;          /* PageWriter object */
+  int pgno;                       /* Page number for current leaf page */
+  int iPrevPgidx;                 /* Previous value written into pgidx */
+  Fts5Buffer buf;                 /* Buffer of current leaf page data */
+  Fts5Buffer pgidx;               /* Buffer of current leaf page-index */
+  Fts5Buffer term;                /* Buffer containing previous term on leaf */
+
   i64 iPrevRowid;                 /* Previous rowid written to current leaf */
   u8 bFirstRowidInDoclist;        /* True if next rowid is first in doclist */
   u8 bFirstRowidInPage;           /* True if next rowid is first in page */
-  /* TODO1: Can use (writer.pgidx.n==0) instead of bFirstTermInPage */
-  u8 bFirstTermInPage;            /* True if next term will be first in leaf */
   int nLeafWritten;               /* Number of leaf pages written */
   int nEmpty;                     /* Number of contiguous term-less nodes */
 
@@ -1004,6 +999,14 @@ static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){
   return pRet;
 }
 
+/*
+** Execute "PRAGMA db.data_version" and return the integer value that 
+** it returns.
+**
+** This function returns 0 if Fts5Index.rc is set to other than SQLITE_OK
+** when it is called. If an error occurs while executing the PRAGMA,
+** Fts5Index.rc is set to an SQLite error code before returning.
+*/
 static i64 fts5IndexDataVersion(Fts5Index *p){
   i64 iVersion = 0;
 
@@ -3047,6 +3050,20 @@ static void fts5PoslistFilterCallback(
   }
 }
 
+/*
+** This function is used to extract the position list associated with
+** the entry that segment iterator pSeg currently points to. If the
+** entire position list resides on a single leaf page, then this 
+** function invokes the xChunk callback exactly once. Or, if the position
+** list spans multiple leaves, xChunk is invoked once for each leaf.
+**
+** The first argument passed to the xChunk callback is a copy of the Fts5Index
+** pointer passed as the first argument to this function. Similarly, the 
+** second argument passed to xChunk is a copy of the third parameter passed
+** to this function. The third and fourth arguments passed to xChunk are
+** a pointer to a blob containing part of the position list and the size
+** in bytes there of.
+*/
 static void fts5ChunkIterate(
   Fts5Index *p,                   /* Index object */
   Fts5SegIter *pSeg,              /* Poslist of this iterator */
@@ -3060,7 +3077,7 @@ static void fts5ChunkIterate(
   int pgno = pSeg->iLeafPgno;
   int pgnoSave = 0;
 
-  /* This function does notmwork with detail=none databases. */
+  /* This function does not work with detail=none databases. */
   assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE );
 
   if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){
@@ -3734,7 +3751,7 @@ static void fts5WriteBtreeTerm(
 ){
   fts5WriteFlushBtree(p, pWriter);
   fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm);
-  pWriter->iBtPage = pWriter->writer.pgno;
+  pWriter->iBtPage = pWriter->pgno;
 }
 
 /*
@@ -3757,10 +3774,16 @@ static void fts5WriteBtreeNoTerm(
   pWriter->nEmpty++;
 }
 
+/*
+** Buffer pBuf contains a doclist-index page. Return the first rowid 
+** value on the page.
+*/
 static i64 fts5DlidxExtractFirstRowid(Fts5Buffer *pBuf){
   i64 iRowid;
   int iOff;
 
+  /* Skip past the flags byte and the page number of the left-most child
+  ** page to the first rowid.  */
   iOff = 1 + fts5GetVarint(&pBuf->p[1], (u64*)&iRowid);
   fts5GetVarint(&pBuf->p[iOff], (u64*)&iRowid);
   return iRowid;
@@ -3768,8 +3791,8 @@ static i64 fts5DlidxExtractFirstRowid(Fts5Buffer *pBuf){
 
 /*
 ** Rowid iRowid has just been appended to the current leaf page. It is the
-** first on the page. This function appends an appropriate entry to the current
-** doclist-index.
+** first on the page. This function appends an appropriate entry to the
+** current doclist-index.
 */
 static void fts5WriteDlidxAppend(
   Fts5Index *p, 
@@ -3818,7 +3841,7 @@ static void fts5WriteDlidxAppend(
     if( pDlidx->bPrevValid ){
       iVal = iRowid - pDlidx->iPrev;
     }else{
-      i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno);
+      i64 iPgno = (i==0 ? pWriter->pgno : pDlidx[-1].pgno);
       assert( pDlidx->buf.n==0 );
       sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone);
       sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno);
@@ -3831,42 +3854,42 @@ static void fts5WriteDlidxAppend(
   }
 }
 
+/*
+** Flush the writer's current leaf page to disk. Initialize various
+** fields ready for the next leaf page.
+*/
 static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
-  static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
-  Fts5PageWriter *pPage = &pWriter->writer;
   i64 iRowid;
 
-  assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) );
-
   /* Set the szLeaf header field. */
-  assert( 0==fts5GetU16(&pPage->buf.p[2]) );
-  fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n);
+  assert( 0==fts5GetU16(&pWriter->buf.p[2]) );
+  fts5PutU16(&pWriter->buf.p[2], (u16)pWriter->buf.n);
 
-  if( pWriter->bFirstTermInPage ){
+  if( pWriter->pgidx.n==0 ){
     /* No term was written to this page. */
-    assert( pPage->pgidx.n==0 );
+    assert( pWriter->pgidx.n==0 );
     fts5WriteBtreeNoTerm(p, pWriter);
   }else{
     /* Append the pgidx to the page buffer. Set the szLeaf header field. */
-    fts5BufferAppendBlob(&p->rc, &pPage->buf, pPage->pgidx.n, pPage->pgidx.p);
+    fts5BufferSafeAppendBlob(&pWriter->buf, pWriter->pgidx.p, pWriter->pgidx.n);
   }
 
   /* Write the page out to disk */
-  iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, pPage->pgno);
-  fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n);
+  iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, pWriter->pgno);
+  fts5DataWrite(p, iRowid, pWriter->buf.p, pWriter->buf.n);
 
   /* Initialize the next page. */
-  fts5BufferZero(&pPage->buf);
-  fts5BufferZero(&pPage->pgidx);
-  fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero);
-  pPage->iPrevPgidx = 0;
-  pPage->pgno++;
+  fts5BufferZero(&pWriter->buf);
+  fts5BufferZero(&pWriter->pgidx);
+  memset(pWriter->buf.p, 0, 4);
+  pWriter->buf.n = 4;
+  pWriter->iPrevPgidx = 0;
+  pWriter->pgno++;
 
   /* Increase the leaves written counter */
   pWriter->nLeafWritten++;
 
   /* The new leaf holds no terms or rowids */
-  pWriter->bFirstTermInPage = 1;
   pWriter->bFirstRowidInPage = 1;
 }
 
@@ -3883,34 +3906,24 @@ static void fts5WriteAppendTerm(
   int nTerm, const u8 *pTerm 
 ){
   int nPrefix;                    /* Bytes of prefix compression for term */
-  Fts5PageWriter *pPage = &pWriter->writer;
-  Fts5Buffer *pPgidx = &pWriter->writer.pgidx;
+  int iOff;
+  Fts5Buffer *pPgidx = &pWriter->pgidx;
 
   assert( p->rc==SQLITE_OK );
-  assert( pPage->buf.n>=4 );
-  assert( pPage->buf.n>4 || pWriter->bFirstTermInPage );
+  assert( pWriter->buf.n>=4 );
 
   /* If the current leaf page is full, flush it to disk. */
-  if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){
-    if( pPage->buf.n>4 ){
+  if( (pWriter->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){
+    if( pWriter->buf.n>4 ){
       fts5WriteFlushLeaf(p, pWriter);
     }
-    fts5BufferGrow(&p->rc, &pPage->buf, nTerm+FTS5_DATA_PADDING);
+    fts5BufferGrow(&p->rc, &pWriter->buf, nTerm+FTS5_DATA_PADDING);
   }
-  
-  /* TODO1: Updating pgidx here. */
-  pPgidx->n += sqlite3Fts5PutVarint(
-      &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx
-  );
-  pPage->iPrevPgidx = pPage->buf.n;
-#if 0
-  fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n);
-  pPgidx->n += 2;
-#endif
 
-  if( pWriter->bFirstTermInPage ){
+  iOff = pWriter->buf.n;
+  if( pPgidx->n==0 ){
     nPrefix = 0;
-    if( pPage->pgno!=1 ){
+    if( pWriter->pgno!=1 ){
       /* This is the first term on a leaf that is not the leftmost leaf in
       ** the segment b-tree. In this case it is necessary to add a term to
       ** the b-tree hierarchy that is (a) larger than the largest term 
@@ -3919,37 +3932,38 @@ static void fts5WriteAppendTerm(
       ** byte longer than the longest prefix (pTerm/nTerm) shares with the
       ** previous term. 
       **
-      ** Usually, the previous term is available in pPage->term. The exception
+      ** Usually, the previous term is available in pWriter->term. The exception
       ** is if this is the first term written in an incremental-merge step.
       ** In this case the previous term is not available, so just write a
       ** copy of (pTerm/nTerm) into the parent node. This is slightly
       ** inefficient, but still correct.  */
       int n = nTerm;
-      if( pPage->term.n ){
-        n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm);
+      if( pWriter->term.n ){
+        n = 1 + fts5PrefixCompress(pWriter->term.n, pWriter->term.p, pTerm);
       }
       fts5WriteBtreeTerm(p, pWriter, n, pTerm);
-      pPage = &pWriter->writer;
     }
   }else{
-    nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm);
-    fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix);
+    nPrefix = fts5PrefixCompress(pWriter->term.n, pWriter->term.p, pTerm);
+    fts5BufferAppendVarint(&p->rc, &pWriter->buf, nPrefix);
   }
 
+  fts5BufferSafeAppendVarint(pPgidx, iOff - pWriter->iPrevPgidx);
+  pWriter->iPrevPgidx = iOff;
+
   /* Append the number of bytes of new data, then the term data itself
   ** to the page. */
-  fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm - nPrefix);
-  fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm - nPrefix, &pTerm[nPrefix]);
+  fts5BufferAppendVarint(&p->rc, &pWriter->buf, nTerm - nPrefix);
+  fts5BufferAppendBlob(&p->rc, &pWriter->buf, nTerm - nPrefix, &pTerm[nPrefix]);
 
-  /* Update the Fts5PageWriter.term field. */
-  fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm);
-  pWriter->bFirstTermInPage = 0;
+  /* Update the Fts5SegWriter.term field. */
+  fts5BufferSet(&p->rc, &pWriter->term, nTerm, pTerm);
 
   pWriter->bFirstRowidInPage = 0;
   pWriter->bFirstRowidInDoclist = 1;
 
   assert( p->rc || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n==0) );
-  pWriter->aDlidx[0].pgno = pPage->pgno;
+  pWriter->aDlidx[0].pgno = pWriter->pgno;
 }
 
 /*
@@ -3961,9 +3975,8 @@ static void fts5WriteAppendRowid(
   i64 iRowid
 ){
   if( p->rc==SQLITE_OK ){
-    Fts5PageWriter *pPage = &pWriter->writer;
 
-    if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){
+    if( (pWriter->buf.n + pWriter->pgidx.n)>=p->pConfig->pgsz ){
       fts5WriteFlushLeaf(p, pWriter);
     }
 
@@ -3971,16 +3984,17 @@ static void fts5WriteAppendRowid(
     ** rowid-pointer in the page-header. Also append a value to the dlidx
     ** buffer, in case a doclist-index is required.  */
     if( pWriter->bFirstRowidInPage ){
-      fts5PutU16(pPage->buf.p, (u16)pPage->buf.n);
+      fts5PutU16(pWriter->buf.p, (u16)pWriter->buf.n);
       fts5WriteDlidxAppend(p, pWriter, iRowid);
     }
 
     /* Write the rowid. */
     if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){
-      fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid);
+      fts5BufferSafeAppendVarint(&pWriter->buf, iRowid);
     }else{
       assert( p->rc || iRowid>pWriter->iPrevRowid );
-      fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid);
+      assert( pWriter->buf.n+9 <= pWriter->buf.nSpace );
+      fts5BufferSafeAppendVarint(&pWriter->buf, iRowid - pWriter->iPrevRowid);
     }
     pWriter->iPrevRowid = iRowid;
     pWriter->bFirstRowidInDoclist = 0;
@@ -3988,33 +4002,35 @@ static void fts5WriteAppendRowid(
   }
 }
 
+/*
+** Append position list data to the writers output.
+*/
 static void fts5WriteAppendPoslistData(
   Fts5Index *p, 
-  Fts5SegWriter *pWriter, 
-  const u8 *aData, 
-  int nData
+  Fts5SegWriter *pWriter,         /* Writer object to write to the output of */
+  const u8 *aData,                /* Buffer containing data to append */
+  int nData                       /* Size of buffer aData[] in bytes */
 ){
-  Fts5PageWriter *pPage = &pWriter->writer;
   const u8 *a = aData;
   int n = nData;
   
   assert( p->pConfig->pgsz>0 );
   while( p->rc==SQLITE_OK 
-     && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz 
+     && (pWriter->buf.n + pWriter->pgidx.n + n)>=p->pConfig->pgsz 
   ){
-    int nReq = p->pConfig->pgsz - pPage->buf.n - pPage->pgidx.n;
+    int nReq = p->pConfig->pgsz - pWriter->buf.n - pWriter->pgidx.n;
     int nCopy = 0;
     while( nCopy<nReq ){
       i64 dummy;
       nCopy += fts5GetVarint(&a[nCopy], (u64*)&dummy);
     }
-    fts5BufferAppendBlob(&p->rc, &pPage->buf, nCopy, a);
+    fts5BufferAppendBlob(&p->rc, &pWriter->buf, nCopy, a);
     a += nCopy;
     n -= nCopy;
     fts5WriteFlushLeaf(p, pWriter);
   }
   if( n>0 ){
-    fts5BufferAppendBlob(&p->rc, &pPage->buf, n, a);
+    fts5BufferAppendBlob(&p->rc, &pWriter->buf, n, a);
   }
 }
 
@@ -4028,20 +4044,19 @@ static void fts5WriteFinish(
   int *pnLeaf                     /* OUT: Number of leaf pages in b-tree */
 ){
   int i;
-  Fts5PageWriter *pLeaf = &pWriter->writer;
   if( p->rc==SQLITE_OK ){
-    assert( pLeaf->pgno>=1 );
-    if( pLeaf->buf.n>4 ){
+    assert( pWriter->pgno>=1 );
+    if( pWriter->buf.n>4 ){
       fts5WriteFlushLeaf(p, pWriter);
     }
-    *pnLeaf = pLeaf->pgno-1;
-    if( pLeaf->pgno>1 ){
+    *pnLeaf = pWriter->pgno-1;
+    if( pWriter->pgno>1 ){
       fts5WriteFlushBtree(p, pWriter);
     }
   }
-  fts5BufferFree(&pLeaf->term);
-  fts5BufferFree(&pLeaf->buf);
-  fts5BufferFree(&pLeaf->pgidx);
+  fts5BufferFree(&pWriter->term);
+  fts5BufferFree(&pWriter->buf);
+  fts5BufferFree(&pWriter->pgidx);
   fts5BufferFree(&pWriter->btterm);
 
   for(i=0; i<pWriter->nDlidx; i++){
@@ -4050,6 +4065,10 @@ static void fts5WriteFinish(
   sqlite3_free(pWriter->aDlidx);
 }
 
+/*
+** Initialize the Fts5SegWriter object indicated by the second argument
+** to write to the segment with seg-id iSegid.
+*/
 static void fts5WriteInit(
   Fts5Index *p, 
   Fts5SegWriter *pWriter, 
@@ -4061,16 +4080,15 @@ static void fts5WriteInit(
   pWriter->iSegid = iSegid;
 
   fts5WriteDlidxGrow(p, pWriter, 1);
-  pWriter->writer.pgno = 1;
-  pWriter->bFirstTermInPage = 1;
+  pWriter->pgno = 1;
   pWriter->iBtPage = 1;
 
-  assert( pWriter->writer.buf.n==0 );
-  assert( pWriter->writer.pgidx.n==0 );
+  assert( pWriter->buf.n==0 );
+  assert( pWriter->pgidx.n==0 );
 
   /* Grow the two buffers to pgsz + padding bytes in size. */
-  sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.pgidx, nBuffer);
-  sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.buf, nBuffer);
+  sqlite3Fts5BufferSize(&p->rc, &pWriter->pgidx, nBuffer);
+  sqlite3Fts5BufferSize(&p->rc, &pWriter->buf, nBuffer);
 
   if( p->pIdxWriter==0 ){
     Fts5Config *pConfig = p->pConfig;
@@ -4082,8 +4100,8 @@ static void fts5WriteInit(
 
   if( p->rc==SQLITE_OK ){
     /* Initialize the 4-byte leaf-page header to 0x00. */
-    memset(pWriter->writer.buf.p, 0, 4);
-    pWriter->writer.buf.n = 4;
+    memset(pWriter->buf.p, 0, 4);
+    pWriter->buf.n = 4;
 
     /* Bind the current output segment id to the index-writer. This is an
     ** optimization over binding the same value over and over as rows are
@@ -4096,6 +4114,8 @@ static void fts5WriteInit(
 ** Iterator pIter was used to iterate through the input segments of on an
 ** incremental merge operation. This function is called if the incremental
 ** merge step has finished but the input has not been completely exhausted.
+** It removes (DELETEs) any input segment leaves that are no longer required
+** from the database.
 */
 static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){
   int i;
@@ -4154,6 +4174,11 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){
   fts5BufferFree(&buf);
 }
 
+/*
+** This function is used as an fts5ChunkIterate() callback when merging
+** levels. The chunk passed to this function is appended to the output
+** segment.
+*/
 static void fts5MergeChunkCallback(
   Fts5Index *p, 
   void *pCtx, 
@@ -4205,7 +4230,7 @@ static void fts5IndexMergeLevel(
     pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1];
 
     fts5WriteInit(p, &writer, pSeg->iSegid);
-    writer.writer.pgno = pSeg->pgnoLast+1;
+    writer.pgno = pSeg->pgnoLast+1;
     writer.iBtPage = 0;
   }else{
     int iSegid = fts5AllocateSegid(p, pStruct);
@@ -4265,15 +4290,15 @@ static void fts5IndexMergeLevel(
 
     if( eDetail==FTS5_DETAIL_NONE ){
       if( pSegIter->bDel ){
-        fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0);
+        fts5BufferAppendVarint(&p->rc, &writer.buf, 0);
         if( pSegIter->nPos>0 ){
-          fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0);
+          fts5BufferAppendVarint(&p->rc, &writer.buf, 0);
         }
       }
     }else{
       /* Append the position-list data to the output */
       nPos = pSegIter->nPos*2 + pSegIter->bDel;
-      fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos);
+      fts5BufferAppendVarint(&p->rc, &writer.buf, nPos);
       fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback);
     }
   }
@@ -4471,8 +4496,8 @@ static void fts5IndexFlush(Fts5Index *p){
       Fts5SegWriter writer;
       fts5WriteInit(p, &writer, iSegid);
 
-      pBuf = &writer.writer.buf;
-      pPgidx = &writer.writer.pgidx;
+      pBuf = &writer.buf;
+      pPgidx = &writer.pgidx;
 
       /* fts5WriteInit() should have initialized the buffers to (most likely)
       ** the maximum space required. */
@@ -5061,10 +5086,11 @@ static void fts5SetupPrefixIter(
     xAppend = fts5AppendPoslist;
   }
 
+  assert( p->pStruct );
   aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
   pStruct = fts5StructureRead(p);
 
-  if( aBuf && pStruct ){
+  if( aBuf ){
     const int flags = FTS5INDEX_QUERY_SCAN 
                     | FTS5INDEX_QUERY_SKIPEMPTY 
                     | FTS5INDEX_QUERY_NOOUTPUT;
@@ -5365,6 +5391,8 @@ int sqlite3Fts5IndexQuery(
   /* If the QUERY_SCAN flag is set, all other flags must be clear. This
   ** flag is used by the fts5vocab module only. */
   assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN );
+  assert( p->rc==SQLITE_OK );
+  assert( p->pStruct!=0 || (flags & FTS5INDEX_QUERY_PREFIX)==0 );
 
   if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){
     int iIdx = 0;                 /* Index to search */
@@ -5567,6 +5595,7 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
 */
 int sqlite3Fts5IndexNewTrans(Fts5Index *p){
   assert( p->pStruct==0 || p->iStructVersion!=0 );
+  assert( p->rc==SQLITE_OK );
   if( p->pConfig->iCookie<0 || fts5IndexDataVersion(p)!=p->iStructVersion ){
     fts5StructureInvalidate(p);
   }
@@ -6457,40 +6486,6 @@ static void fts5DecodeFunction(
   fts5BufferFree(&s);
 }
 
-/*
-** The implementation of user-defined scalar function fts5_rowid().
-*/
-static void fts5RowidFunction(
-  sqlite3_context *pCtx,          /* Function call context */
-  int nArg,                       /* Number of args (always 2) */
-  sqlite3_value **apVal           /* Function arguments */
-){
-  const char *zArg;
-  if( nArg==0 ){
-    sqlite3_result_error(pCtx, "should be: fts5_rowid(subject, ....)", -1);
-  }else{
-    zArg = (const char*)sqlite3_value_text(apVal[0]);
-    if( 0==sqlite3_stricmp(zArg, "segment") ){
-      i64 iRowid;
-      int segid, pgno;
-      if( nArg!=3 ){
-        sqlite3_result_error(pCtx, 
-            "should be: fts5_rowid('segment', segid, pgno))", -1
-        );
-      }else{
-        segid = sqlite3_value_int(apVal[1]);
-        pgno = sqlite3_value_int(apVal[2]);
-        iRowid = FTS5_SEGMENT_ROWID(segid, pgno);
-        sqlite3_result_int64(pCtx, iRowid);
-      }
-    }else{
-      sqlite3_result_error(pCtx, 
-        "first arg to fts5_rowid() must be 'segment'" , -1
-      );
-    }
-  }
-}
-
 /*
 ** This is called as part of registering the FTS5 module with database
 ** connection db. It registers several user-defined scalar functions useful
@@ -6511,11 +6506,6 @@ int sqlite3Fts5IndexInit(sqlite3 *db){
     );
   }
 
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(
-        db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
-    );
-  }
   return rc;
 }
 
diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl
index 0f371dcfd9..50d514dcb8 100644
--- a/ext/fts5/test/fts5_common.tcl
+++ b/ext/fts5/test/fts5_common.tcl
@@ -645,3 +645,10 @@ proc fts5_tclnum_register {db} {
 # End of tokenizer code.
 #-------------------------------------------------------------------------
 
+proc fts5_rowid_func {segid pgno} { return [expr ($segid<<37) + $pgno] }
+proc register_fts5_rowid {db} {
+  $db func fts5_rowid -argcount 2 fts5_rowid_func
+}
+
+
+
diff --git a/ext/fts5/test/fts5corrupt.test b/ext/fts5/test/fts5corrupt.test
index edaafb2379..b9d087384d 100644
--- a/ext/fts5/test/fts5corrupt.test
+++ b/ext/fts5/test/fts5corrupt.test
@@ -41,18 +41,20 @@ db_save
 do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
 set segid [lindex [fts5_level_segids t1] 0]
 
+register_fts5_rowid db
 do_test 1.3 {
   execsql {
-    DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4);
+    DELETE FROM t1_data WHERE rowid = fts5_rowid($segid, 4);
   }
   catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
 } {1 {database disk image is malformed}}
 
 do_test 1.4 {
   db_restore_and_reopen
+  register_fts5_rowid db
   execsql {
     UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE
-    rowid = fts5_rowid('segment', $segid, 4);
+    rowid = fts5_rowid($segid, 4);
   }
   catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
 } {1 {database disk image is malformed}}
diff --git a/ext/fts5/test/fts5faultC.test b/ext/fts5/test/fts5faultC.test
index a9c94a44c0..df51bf9a51 100644
--- a/ext/fts5/test/fts5faultC.test
+++ b/ext/fts5/test/fts5faultC.test
@@ -17,14 +17,13 @@ source $testdir/malloc_common.tcl
 set testprefix fts5faultC
 return_if_no_fts5
 
+if 1 {
+
 #--------------------------------------------------------------------------
 # Test that if an OOM error occurs while trying to set a configuration
 # option, the in-memory and on-disk configurations are not left in an 
 # inconsistent state.
 #
-
-
-
 proc posrowid {cmd} { $cmd xRowid }
 proc negrowid {cmd} { expr -1 * [$cmd xRowid] }
 
@@ -73,7 +72,50 @@ do_faultsim_test 1.1 -faults oom-* -prep {
 }
 
 
+#--------------------------------------------------------------------------
+
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(y);
+}
+do_faultsim_test 2.1 -faults * -prep {
+  sqlite3 db test.db
+  execsql { SELECT * FROM t1('x') }
+} -body {
+  execsql { SELECT * FROM t1('x') }
+  execsql { SELECT * FROM t1('x') }
+} -test {
+  faultsim_test_result [list 0 {}]
+}
+do_faultsim_test 2.2 -faults * -prep {
+  sqlite3 db test.db
+  execsql { INSERT INTO t1 VALUES('yy') }
+} -body {
+  execsql { INSERT INTO t1 VALUES('yy') }
+  execsql { INSERT INTO t1 VALUES('yy') }
+} -test {
+  faultsim_test_result [list 0 {}]
+}
 
+#--------------------------------------------------------------------------
+}
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(y);
+}
+faultsim_save_and_close
+do_faultsim_test 3.1 -faults * -prep {
+  faultsim_restore_and_reopen
+  execsql { 
+    BEGIN;
+      INSERT INTO t1 VALUES('x y z');
+      INSERT INTO t1 VALUES('a b c');
+  }
+} -body {
+  execsql { INSERT INTO t1(t1) VALUES('optimize') }
+} -test {
+  faultsim_test_result [list 0 {}]
+}
 
 finish_test
 
diff --git a/ext/fts5/test/fts5rowid.test b/ext/fts5/test/fts5rowid.test
index 19590cdf0d..7a24a1287f 100644
--- a/ext/fts5/test/fts5rowid.test
+++ b/ext/fts5/test/fts5rowid.test
@@ -15,33 +15,13 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5rowid
 
-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-do_catchsql_test 1.1 {
-  SELECT fts5_rowid()
-} {1 {should be: fts5_rowid(subject, ....)}}
-
-do_catchsql_test 1.2 {
-  SELECT fts5_rowid('segment')
-} {1 {should be: fts5_rowid('segment', segid, pgno))}}
-
-do_execsql_test 1.3 {
-  SELECT fts5_rowid('segment', 1, 1)
-} {137438953473}
-
-do_catchsql_test 1.4 {
-  SELECT fts5_rowid('nosucharg');
-} {1 {first arg to fts5_rowid() must be 'segment'}} 
-
+return_if_no_fts5
 
 #-------------------------------------------------------------------------
 # Tests of the fts5_decode() function.
 #
 reset_db
+register_fts5_rowid db
 do_execsql_test 2.1 { 
   CREATE VIRTUAL TABLE x1 USING fts5(a, b);
   INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
@@ -92,7 +72,7 @@ do_execsql_test 2.6 {
 #} $res
 
 do_execsql_test 2.8 {
-  SELECT fts5_decode(fts5_rowid('segment', 1000, 1), X'AB')
+  SELECT fts5_decode(fts5_rowid(1000, 1), X'AB')
 } {corrupt}
 
 #-------------------------------------------------------------------------
diff --git a/ext/fts5/test/fts5simple3.test b/ext/fts5/test/fts5simple3.test
index 3d956459f9..8ed0e12b82 100644
--- a/ext/fts5/test/fts5simple3.test
+++ b/ext/fts5/test/fts5simple3.test
@@ -96,5 +96,35 @@ do_execsql_test 4.1 { SELECT * FROM sqlite_master } {}
 do_execsql_test 4.2 { COMMIT }
 do_execsql_test 4.3 { SELECT * FROM sqlite_master } {}
 
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a);
+  INSERT INTO t1 VALUES('vtable constructor failed');
+} {}
+do_execsql_test 5.1 {
+  SELECT quote(block) FROM t1_data WHERE rowid=10;
+} {
+  X'000000000101010001010101'
+}
+do_test 5.3 {
+  sqlite3 db2 test.db
+  register_fts5_rowid db2
+  db2 eval {
+    UPDATE t1_data SET block = X'000000000101010001A7080101' WHERE rowid=10;
+    UPDATE t1_data SET rowid=fts5_rowid(5000, 1) WHERE rowid=fts5_rowid(1,1);
+    UPDATE t1_idx SET segid=5000 WHERE segid=1;
+  }
+  db2 close
+} {}
+do_execsql_test 5.4 {
+  SELECT rowid FROM t1('cons*');
+} {1}
+breakpoint
+do_execsql_test 5.5 {
+  INSERT INTO t1 VALUES('hello world');
+}
+
 finish_test
 
diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test
index 7e92822c33..6344a321c3 100644
--- a/ext/fts5/test/fts5synonym2.test
+++ b/ext/fts5/test/fts5synonym2.test
@@ -42,13 +42,13 @@ proc fts5_test_bothlist {cmd} {
 }
 sqlite3_fts5_create_function db fts5_test_bothlist fts5_test_bothlist
 
-proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] }
-sqlite3_fts5_create_function db fts5_rowid fts5_rowid
+proc rowid_func {cmd} { expr [$cmd xColumnText -1] }
+sqlite3_fts5_create_function db rowid_func rowid_func
 
 do_execsql_test 1.$tok.0.1 "
   CREATE VIRTUAL TABLE ss USING fts5(a, b, 
        tokenize='tclnum $tok', detail=%DETAIL%);
-  INSERT INTO ss(ss, rank) VALUES('rank', 'fts5_rowid()');
+  INSERT INTO ss(ss, rank) VALUES('rank', 'rowid_func()');
 "
 
 do_execsql_test 1.$tok.0.2 {
diff --git a/ext/fts5/test/fts5version.test b/ext/fts5/test/fts5version.test
index 7e4d74d114..265b5959a3 100644
--- a/ext/fts5/test/fts5version.test
+++ b/ext/fts5/test/fts5version.test
@@ -59,6 +59,25 @@ do_test 1.7 {
   catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
 } {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}}
 
+#---------------------------------------------------------------------------
+
+do_execsql_test 2.1 {
+  CREATE VIRTUAL TABLE t2 USING fts5(two);
+  INSERT INTO t2 VALUES('a b c d');
+}
+
+do_test 2.2 {
+  sqlite3 db2 test.db
+  execsql {
+    INSERT INTO t2(t2, rank) VALUES('pgsz', 512);
+    UPDATE t2_config SET v=5 WHERE k='version';
+  } db2
+  db2 close
+} {}
+
+do_catchsql_test 2.3 {
+  SELECT * FROM t2('b + c');
+} {1 {SQL logic error or missing database}}
 
 finish_test
 
diff --git a/manifest b/manifest
index 38bb6957dc..4fd5b4c81e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sfurther\stests\sfor\ssavepoint\srollback.\sFix\svarious\scode\sissues\sand\sadd\smissing\scomments\sin\sfts5_index.c.
-D 2016-03-28T20:13:25.897
+C Fix\ssome\scode\sand\stest\scoverage\sissues\sin\sfts5_index.c.
+D 2016-03-29T21:19:04.017
 F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66
@@ -104,7 +104,7 @@ F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 5ca4bafe29aa3d27683c90e836192e4aefd20a3f
 F ext/fts5/fts5_hash.c f3a7217c86eb8f272871be5f6aa1b6798960a337
-F ext/fts5/fts5_index.c 48fed29450d8d38b0c9ee5acc97d9360f35d9b01
+F ext/fts5/fts5_index.c 870fa81c7d08dae287f843cddf8809c5665f3f3e
 F ext/fts5/fts5_main.c 1e1e6e2d6df6b224fe9b2c75bcbe265c73b8712d
 F ext/fts5/fts5_storage.c e0aa8509e01eb22ae8e198c1de9c3200755c0d94
 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
@@ -116,7 +116,7 @@ F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738
 F ext/fts5/fts5_vocab.c dba72ca393d71c2588548b51380387f6b44c77a8
 F ext/fts5/fts5parse.y fcc5e92e570d38cab38488b2109cbf67468923b2
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841
+F ext/fts5/test/fts5_common.tcl 7bae6b2a93bd62be99cc78930c4806afcc152b4b
 F ext/fts5/test/fts5aa.test bd2d88182b9f7f30d300044048ad14683306b745
 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
@@ -139,7 +139,7 @@ F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07
 F ext/fts5/test/fts5config.test 7788b9c058074d640dfcdd81d97b6a9480000368
 F ext/fts5/test/fts5conflict.test 26f4e46c4d31e16221794832a990dc4e30e18de5
 F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
-F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
+F ext/fts5/test/fts5corrupt.test 0d0dbfa10a134ab710f14b9a25a3922e93e43ac3
 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c
 F ext/fts5/test/fts5corrupt3.test f77f65e386231daf62902466b40ff998b2c8ce4f
 F ext/fts5/test/fts5detail.test ef5c690535a797413acaf5ad9b8ab5d49972df69
@@ -159,7 +159,7 @@ F ext/fts5/test/fts5fault8.test 6785af34bd1760de74e2824ea9c161965af78f85
 F ext/fts5/test/fts5fault9.test e10e395428a9ea0596ebe752ff7123d16ab78e08
 F ext/fts5/test/fts5faultA.test fa5d59c0ff62b7125cd14eee38ded1c46e15a7ea
 F ext/fts5/test/fts5faultB.test 92ae906284062bf081b6c854afa54dcb1aa9ef88
-F ext/fts5/test/fts5faultC.test 10da76c6b69df05ff9095c7e56dc3a4b8d0e8f20
+F ext/fts5/test/fts5faultC.test c1eaa71dbf832190a797772682ada0c63af88d58
 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
 F ext/fts5/test/fts5fuzz1.test bece4695fc169b61ab236ada7931c6e4942cbef9
 F ext/fts5/test/fts5hash.test 06f9309ccb4d5050a131594e9e47d0b21456837d
@@ -180,12 +180,12 @@ F ext/fts5/test/fts5query.test f5ec25f5f2fbb70033424113cdffc101b1985a40
 F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
-F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6
+F ext/fts5/test/fts5rowid.test 4092994f3b4b94c071b1866e6854aa705bbc4c1f
 F ext/fts5/test/fts5simple.test f157c8b068f5e68473ad86bfe220b07a84e0209f
 F ext/fts5/test/fts5simple2.test 98377ae1ff7749a42c21fe1a139c1ed312522c46
-F ext/fts5/test/fts5simple3.test 5e00bc009aa0a62190e795383d969b8f31dbde15
+F ext/fts5/test/fts5simple3.test 9508bbfce687d521853a33cb9e096783cfd637b3
 F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48
-F ext/fts5/test/fts5synonym2.test aa4c43bd3b691ff80f658cb064f5ab40690e834e
+F ext/fts5/test/fts5synonym2.test 29335811b8bd6113915b711adbd69b943ce9d301
 F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
@@ -194,7 +194,7 @@ F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
 F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529
-F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e
+F ext/fts5/test/fts5version.test 64c07b0a3d951db35b7984a051144ac499a01f00
 F ext/fts5/test/fts5vocab.test 480d780aa6b699816c5066225fbd86f3a0239477
 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85
 F ext/fts5/tool/fts5txt2db.tcl 526a9979c963f1c54fd50976a05a502e533a4c59
@@ -1460,7 +1460,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 06039d901ad680b8d5abdf31c3799bd971750b5d
-R 4b4e165ed2fcd56f69de6ef3da9c9fa1
+P a805c6f7ea59a74ba3110a058ba6eb9dda8058a7
+R 74c0771422d93ac3c0fc9105cf2a64de
 U dan
-Z 91b6074a9c111559bae6cc0a64059b03
+Z fe5545e02974667cb03b9c402c34ddc2
diff --git a/manifest.uuid b/manifest.uuid
index 5d5004e44b..bf9e810b8b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a805c6f7ea59a74ba3110a058ba6eb9dda8058a7
\ No newline at end of file
+7635c68018ce1656a1c5d6eebaf8f3a8e8839b59
\ No newline at end of file