From: dan <dan@noemail.net>
Date: Wed, 3 Dec 2014 17:27:35 +0000 (+0000)
Subject: Begin testing fts5 OOM and IO error handling.
X-Git-Tag: version-3.8.11~114^2~125
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=845d0ab323b05b920198af4532289fc6621f40f7;p=thirdparty%2Fsqlite.git

Begin testing fts5 OOM and IO error handling.

FossilOrigin-Name: 2037dba62fdd995ad15b642abe499a790f5ffe5c
---

diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c
index 54d3c4bbcd..e80715a4c7 100644
--- a/ext/fts5/fts5.c
+++ b/ext/fts5/fts5.c
@@ -308,24 +308,19 @@ static int fts5InitVtab(
 ){
   Fts5Global *pGlobal = (Fts5Global*)pAux;
   const char **azConfig = (const char**)argv;
-  int rc;                         /* Return code */
+  int rc = SQLITE_OK;             /* Return code */
   Fts5Config *pConfig;            /* Results of parsing argc/argv */
   Fts5Table *pTab = 0;            /* New virtual table object */
 
-  /* Parse the arguments */
-  rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr);
-  assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
-
-  /* Allocate the new vtab object */
+  /* Allocate the new vtab object and parse the configuration */
+  pTab = (Fts5Table*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Table));
   if( rc==SQLITE_OK ){
-    pTab = (Fts5Table*)sqlite3_malloc(sizeof(Fts5Table));
-    if( pTab==0 ){
-      rc = SQLITE_NOMEM;
-    }else{
-      memset(pTab, 0, sizeof(Fts5Table));
-      pTab->pConfig = pConfig;
-      pTab->pGlobal = pGlobal;
-    }
+    rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr);
+    assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
+  }
+  if( rc==SQLITE_OK ){
+    pTab->pConfig = pConfig;
+    pTab->pGlobal = pGlobal;
   }
 
   /* Open the index sub-system */
diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index b3d5eed811..684b34f009 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -187,6 +187,9 @@ int sqlite3Fts5PoslistNext64(
   i64 *piOff                      /* IN/OUT: Current offset */
 );
 
+/* Malloc utility */
+void *sqlite3Fts5MallocZero(int *pRc, int nByte);
+
 /*
 ** End of interface to code in fts5_buffer.c.
 **************************************************************************/
@@ -225,24 +228,25 @@ int sqlite3Fts5IndexClose(Fts5Index *p, int bDestroy);
 ** Open a new iterator to iterate though all docids that match the 
 ** specified token or token prefix.
 */
-Fts5IndexIter *sqlite3Fts5IndexQuery(
+int sqlite3Fts5IndexQuery(
   Fts5Index *p,                   /* FTS index to query */
   const char *pToken, int nToken, /* Token (or prefix) to query for */
-  int flags                       /* Mask of FTS5INDEX_QUERY_X flags */
+  int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
+  Fts5IndexIter **ppIter
 );
 
 /*
 ** Docid list iteration.
 */
-int  sqlite3Fts5IterEof(Fts5IndexIter*);
-void sqlite3Fts5IterNext(Fts5IndexIter*);
-void sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
-i64  sqlite3Fts5IterRowid(Fts5IndexIter*);
+int sqlite3Fts5IterEof(Fts5IndexIter*);
+int sqlite3Fts5IterNext(Fts5IndexIter*);
+int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
+i64 sqlite3Fts5IterRowid(Fts5IndexIter*);
 
 /*
 ** Obtain the position list that corresponds to the current position.
 */
-const u8 *sqlite3Fts5IterPoslist(Fts5IndexIter*, int *pn);
+int sqlite3Fts5IterPoslist(Fts5IndexIter*, const u8 **pp, int *pn);
 
 /*
 ** Close an iterator opened by sqlite3Fts5IndexQuery().
@@ -259,7 +263,7 @@ void sqlite3Fts5IterClose(Fts5IndexIter*);
 ** unique token in the document with an iCol value less than zero. The iPos
 ** argument is ignored for a delete.
 */
-void sqlite3Fts5IndexWrite(
+int sqlite3Fts5IndexWrite(
   Fts5Index *p,                   /* Index to write to */
   int iCol,                       /* Column token appears in (-ve -> delete) */
   int iPos,                       /* Position of token within column */
@@ -270,7 +274,7 @@ void sqlite3Fts5IndexWrite(
 ** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
 ** document iDocid.
 */
-void sqlite3Fts5IndexBeginWrite(
+int sqlite3Fts5IndexBeginWrite(
   Fts5Index *p,                   /* Index to write to */
   i64 iDocid                      /* Docid to add or remove data from */
 );
@@ -321,9 +325,6 @@ int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
 */
 int sqlite3Fts5IndexReads(Fts5Index *p);
 
-/* Malloc utility */
-void *sqlite3Fts5MallocZero(int *pRc, int nByte);
-
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c
index 478b903614..59eb1b8752 100644
--- a/ext/fts5/fts5_buffer.c
+++ b/ext/fts5/fts5_buffer.c
@@ -282,4 +282,18 @@ void sqlite3Fts5BufferAppendListElem(
   *pOut = '\0';
 }
 
+void *sqlite3Fts5MallocZero(int *pRc, int nByte){
+  void *pRet = 0;
+  if( *pRc==SQLITE_OK ){
+    pRet = sqlite3_malloc(nByte);
+    if( pRet==0 && nByte>0 ){
+      *pRc = SQLITE_NOMEM;
+    }else{
+      memset(pRet, 0, nByte);
+    }
+  }
+  return pRet;
+}
+
+
 
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index 3cc1ffda46..9ea78143c5 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -115,8 +115,13 @@ static int fts5ConfigParseSpecial(
 **
 ** Return 0 if an OOM error is encountered.
 */
-static char *fts5Strdup(const char *z){
-  return sqlite3_mprintf("%s", z);
+static char *fts5Strdup(int *pRc, const char *z){
+  char *pRet = 0;
+  if( *pRc==SQLITE_OK ){
+    pRet = sqlite3_mprintf("%s", z);
+    if( pRet==0 ) *pRc = SQLITE_NOMEM;
+  }
+  return pRet;
 }
 
 /*
@@ -159,44 +164,41 @@ int sqlite3Fts5ConfigParse(
   pRet->db = db;
   pRet->iCookie = -1;
 
-  pRet->azCol = (char**)sqlite3_malloc(sizeof(char*) * nArg);
-  pRet->zDb = fts5Strdup(azArg[1]);
-  pRet->zName = fts5Strdup(azArg[2]);
-  if( sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
-    *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
-    rc = SQLITE_ERROR;
-  }else if( pRet->azCol==0 || pRet->zDb==0 || pRet->zName==0 ){
-    rc = SQLITE_NOMEM;
-  }else{
-    int i;
-    for(i=3; rc==SQLITE_OK && i<nArg; i++){
-      char *zDup = fts5Strdup(azArg[i]);
-      if( zDup==0 ){
-        rc = SQLITE_NOMEM;
-      }else{
-
-        /* Check if this is a special directive - "cmd=arg" */
-        if( zDup[0]!='"' && zDup[0]!='\'' && zDup[0]!='[' && zDup[0]!='`' ){
-          char *p = zDup;
-          while( *p && *p!='=' ) p++;
-          if( *p ){
-            char *zArg = &p[1];
-            *p = '\0';
-            sqlite3Fts5Dequote(zArg);
-            rc = fts5ConfigParseSpecial(pRet, zDup, zArg, pzErr);
-            sqlite3_free(zDup);
-            zDup = 0;
+  pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
+  pRet->zDb = fts5Strdup(&rc, azArg[1]);
+  pRet->zName = fts5Strdup(&rc, azArg[2]);
+  if( rc==SQLITE_OK ){
+    if( sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
+      *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
+      rc = SQLITE_ERROR;
+    }else{
+      int i;
+      for(i=3; rc==SQLITE_OK && i<nArg; i++){
+        char *zDup = fts5Strdup(&rc, azArg[i]);
+        if( zDup ){
+          /* Check if this is a special directive - "cmd=arg" */
+          if( zDup[0]!='"' && zDup[0]!='\'' && zDup[0]!='[' && zDup[0]!='`' ){
+            char *p = zDup;
+            while( *p && *p!='=' ) p++;
+            if( *p ){
+              char *zArg = &p[1];
+              *p = '\0';
+              sqlite3Fts5Dequote(zArg);
+              rc = fts5ConfigParseSpecial(pRet, zDup, zArg, pzErr);
+              sqlite3_free(zDup);
+              zDup = 0;
+            }
           }
-        }
 
-        /* If it is not a special directive, it must be a column name. In
-        ** this case, check that it is not the reserved column name "rank". */
-        if( zDup ){
-          sqlite3Fts5Dequote(zDup);
-          pRet->azCol[pRet->nCol++] = zDup;
-          if( sqlite3_stricmp(zDup, FTS5_RANK_NAME)==0 ){
-            *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zDup);
-            rc = SQLITE_ERROR;
+          /* If it is not a special directive, it must be a column name. In
+           ** this case, check that it is not the reserved column name "rank". */
+          if( zDup ){
+            sqlite3Fts5Dequote(zDup);
+            pRet->azCol[pRet->nCol++] = zDup;
+            if( sqlite3_stricmp(zDup, FTS5_RANK_NAME)==0 ){
+              *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zDup);
+              rc = SQLITE_ERROR;
+            }
           }
         }
       }
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index efb91dadb6..5c95eda7e4 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -386,8 +386,11 @@ static int fts5ExprPhraseIsMatch(
   /* Initialize a term iterator for each term in the phrase */
   for(i=0; i<pPhrase->nTerm; i++){
     int n;
-    const u8 *a = sqlite3Fts5IterPoslist(pPhrase->aTerm[i].pIter, &n);
-    if( sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ) goto ismatch_out;
+    const u8 *a;
+    rc = sqlite3Fts5IterPoslist(pPhrase->aTerm[i].pIter, &a, &n);
+    if( rc || sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ){
+      goto ismatch_out;
+    }
   }
 
   while( 1 ){
@@ -576,22 +579,21 @@ static int fts5ExprNearAdvanceAll(
   Fts5ExprNearset *pNear,         /* Near object to advance iterators of */
   int *pbEof                      /* OUT: Set to true if phrase at EOF */
 ){
-  int rc = SQLITE_OK;             /* Return code */
   int i, j;                       /* Phrase and token index, respectively */
 
   for(i=0; i<pNear->nPhrase; i++){
     Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
     for(j=0; j<pPhrase->nTerm; j++){
       Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
-      sqlite3Fts5IterNext(pIter);
-      if( sqlite3Fts5IterEof(pIter) ){
+      int rc = sqlite3Fts5IterNext(pIter);
+      if( rc || sqlite3Fts5IterEof(pIter) ){
         *pbEof = 1;
         return rc;
       }
     }
   }
 
-  return rc;
+  return SQLITE_OK;
 }
 
 /*
@@ -711,21 +713,21 @@ static int fts5ExprNearNextMatch(
     rc = fts5ExprNearNextRowidMatch(pExpr, pNode, bFromValid, iFrom);
     if( pNode->bEof || rc!=SQLITE_OK ) break;
 
-    for(i=0; i<pNear->nPhrase; i++){
+    for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
       Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
       if( pPhrase->nTerm>1 || pNear->iCol>=0 ){
         int bMatch = 0;
         rc = fts5ExprPhraseIsMatch(pExpr, pNear->iCol, pPhrase, &bMatch);
-        if( rc!=SQLITE_OK ) return rc;
         if( bMatch==0 ) break;
       }else{
         int n;
-        const u8 *a = sqlite3Fts5IterPoslist(pPhrase->aTerm[0].pIter, &n);
+        const u8 *a;
+        rc = sqlite3Fts5IterPoslist(pPhrase->aTerm[0].pIter, &a, &n);
         fts5BufferSet(&rc, &pPhrase->poslist, n, a);
       }
     }
 
-    if( i==pNear->nPhrase ){
+    if( rc==SQLITE_OK && i==pNear->nPhrase ){
       int bMatch = 1;
       if( pNear->nPhrase>1 ){
         rc = fts5ExprNearIsMatch(pNear, &bMatch);
@@ -735,7 +737,9 @@ static int fts5ExprNearNextMatch(
 
     /* If control flows to here, then the current rowid is not a match.
     ** Advance all term iterators in all phrases to the next rowid. */
-    rc = fts5ExprNearAdvanceAll(pExpr, pNear, &pNode->bEof);
+    if( rc==SQLITE_OK ){
+      rc = fts5ExprNearAdvanceAll(pExpr, pNear, &pNode->bEof);
+    }
     if( pNode->bEof || rc!=SQLITE_OK ) break;
   }
 
@@ -755,24 +759,26 @@ static int fts5ExprNearInitAll(
   Fts5ExprTerm *pTerm;
   Fts5ExprPhrase *pPhrase;
   int i, j;
+  int rc = SQLITE_OK;
 
-  for(i=0; i<pNear->nPhrase; i++){
+  for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
     pPhrase = pNear->apPhrase[i];
     for(j=0; j<pPhrase->nTerm; j++){
       pTerm = &pPhrase->aTerm[j];
-      pTerm->pIter = sqlite3Fts5IndexQuery(
+      rc = sqlite3Fts5IndexQuery(
           pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm),
           (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
-          (pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0)
+          (pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0),
+          &pTerm->pIter
       );
       if( pTerm->pIter && sqlite3Fts5IterEof(pTerm->pIter) ){
         pNode->bEof = 1;
-        return SQLITE_OK;
+        break;
       }
     }
   }
 
-  return SQLITE_OK;
+  return rc;
 }
 
 /* fts5ExprNodeNext() calls fts5ExprNodeNextMatch(). And vice-versa. */
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index b51e7bad11..7d0d01afdf 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -262,12 +262,6 @@ static int fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
 # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
 #endif
 
-#ifdef SQLITE_DEBUG
-static int fts5MissingData() { return 0; }
-#else
-# define fts5MissingData() 
-#endif
-
 
 typedef struct Fts5BtreeIter Fts5BtreeIter;
 typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel;
@@ -592,78 +586,6 @@ struct Fts5BtreeIter {
 };
 
 
-/*
-** Decode a segment-data rowid from the %_data table. This function is
-** the opposite of macro FTS5_SEGMENT_ROWID().
-*/
-static void fts5DecodeRowid(
-  i64 iRowid,                     /* Rowid from %_data table */
-  int *piIdx,                     /* OUT: Index */
-  int *piSegid,                   /* OUT: Segment id */
-  int *piHeight,                  /* OUT: Height */
-  int *piPgno                     /* OUT: Page number */
-){
-  *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1));
-  iRowid >>= FTS5_DATA_PAGE_B;
-
-  *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1));
-  iRowid >>= FTS5_DATA_HEIGHT_B;
-
-  *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1));
-  iRowid >>= FTS5_DATA_ID_B;
-
-  *piIdx = (int)(iRowid & (((i64)1 << FTS5_DATA_IDX_B) - 1));
-}
-
-static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
-  int iIdx,iSegid,iHeight,iPgno;  /* Rowid compenents */
-  fts5DecodeRowid(iKey, &iIdx, &iSegid, &iHeight, &iPgno);
-
-  if( iSegid==0 ){
-    if( iKey==FTS5_AVERAGES_ROWID ){
-      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(averages) ");
-    }else{
-      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
-          "{structure idx=%d}", (int)(iKey-10)
-      );
-    }
-  }
-  else if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
-    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(dlidx idx=%d segid=%d pgno=%d)",
-        iIdx, iSegid, iPgno
-    );
-  }else{
-    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(idx=%d segid=%d h=%d pgno=%d)",
-        iIdx, iSegid, iHeight, iPgno
-    );
-  }
-}
-
-static void fts5DebugStructure(
-  int *pRc,                       /* IN/OUT: error code */
-  Fts5Buffer *pBuf,
-  Fts5Structure *p
-){
-  int iLvl, iSeg;                 /* Iterate through levels, segments */
-
-  for(iLvl=0; iLvl<p->nLevel; iLvl++){
-    Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
-    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
-        " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge
-    );
-    for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
-      Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
-      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
-          " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, 
-          pSeg->pgnoFirst, pSeg->pgnoLast
-      );
-    }
-    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
-  }
-}
-
-
-
 static void fts5PutU16(u8 *aOut, u16 iVal){
   aOut[0] = (iVal>>8);
   aOut[1] = (iVal&0xFF);
@@ -691,19 +613,6 @@ static void *fts5IdxMalloc(Fts5Index *p, int nByte){
   return pRet;
 }
 
-void *sqlite3Fts5MallocZero(int *pRc, int nByte){
-  void *pRet = 0;
-  if( *pRc==SQLITE_OK ){
-    pRet = sqlite3_malloc(nByte);
-    if( pRet==0 && nByte>0 ){
-      *pRc = SQLITE_NOMEM;
-    }else{
-      memset(pRet, 0, nByte);
-    }
-  }
-  return pRet;
-}
-
 /*
 ** Compare the contents of the pLeft buffer with the pRight/nRight blob.
 **
@@ -794,8 +703,6 @@ sqlite3_free(buf.p);
       );
     }
 
-    if( rc ) fts5MissingData();
-
     if( rc==SQLITE_OK ){
       int nByte = sqlite3_blob_bytes(p->pReader);
       if( pBuf ){
@@ -2475,47 +2382,6 @@ static void fts5AddTermToHash(
   }
 }
 
-/*
-** Insert or remove data to or from the index. Each time a document is 
-** added to or removed from the index, this function is called one or more
-** times.
-**
-** For an insert, it must be called once for each token in the new document.
-** If the operation is a delete, it must be called (at least) once for each
-** unique token in the document with an iCol value less than zero. The iPos
-** argument is ignored for a delete.
-*/
-void sqlite3Fts5IndexWrite(
-  Fts5Index *p,                   /* Index to write to */
-  int iCol,                       /* Column token appears in (-ve -> delete) */
-  int iPos,                       /* Position of token within column */
-  const char *pToken, int nToken  /* Token to add or remove to or from index */
-){
-  int i;                          /* Used to iterate through indexes */
-  Fts5Config *pConfig = p->pConfig;
-
-  /* If an error has already occured this call is a no-op. */
-  if( p->rc!=SQLITE_OK ) return;
-
-  /* Allocate hash tables if they have not already been allocated */
-  if( p->apHash==0 ){
-    int nHash = pConfig->nPrefix + 1;
-    p->apHash = (Fts5Hash**)fts5IdxMalloc(p, sizeof(Fts5Hash*) * nHash);
-    for(i=0; p->rc==SQLITE_OK && i<nHash; i++){
-      p->rc = sqlite3Fts5HashNew(&p->apHash[i], &p->nPendingData);
-    }
-  }
-
-  /* Add the new token to the main terms hash table. And to each of the
-  ** prefix hash tables that it is large enough for. */
-  fts5AddTermToHash(p, 0, iCol, iPos, pToken, nToken);
-  for(i=0; i<pConfig->nPrefix; i++){
-    if( nToken>=pConfig->aPrefix[i] ){
-      fts5AddTermToHash(p, i+1, iCol, iPos, pToken, pConfig->aPrefix[i]);
-    }
-  }
-}
-
 /*
 ** Allocate a new segment-id for the structure pStruct.
 **
@@ -3342,111 +3208,6 @@ static void fts5IndexFlush(Fts5Index *p){
   p->nPendingData = 0;
 }
 
-/*
-** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
-** to the document with rowid iRowid.
-*/
-void sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){
-  if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){
-    fts5IndexFlush(p);
-  }
-  p->iWriteRowid = iRowid;
-}
-
-/*
-** Commit data to disk.
-*/
-int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
-  fts5IndexFlush(p);
-  if( bCommit ) fts5CloseReader(p);
-  return p->rc;
-}
-
-/*
-** Discard any data stored in the in-memory hash tables. Do not write it
-** to the database. Additionally, assume that the contents of the %_data
-** table may have changed on disk. So any in-memory caches of %_data 
-** records must be invalidated.
-*/
-int sqlite3Fts5IndexRollback(Fts5Index *p){
-  fts5CloseReader(p);
-  fts5IndexDiscardData(p);
-  return SQLITE_OK;
-}
-
-/*
-** Open a new Fts5Index handle. If the bCreate argument is true, create
-** and initialize the underlying %_data table.
-**
-** If successful, set *pp to point to the new object and return SQLITE_OK.
-** Otherwise, set *pp to NULL and return an SQLite error code.
-*/
-int sqlite3Fts5IndexOpen(
-  Fts5Config *pConfig, 
-  int bCreate, 
-  Fts5Index **pp,
-  char **pzErr
-){
-  int rc = SQLITE_OK;
-  Fts5Index *p;                   /* New object */
-
-  *pp = p = (Fts5Index*)sqlite3_malloc(sizeof(Fts5Index));
-  if( !p ) return SQLITE_NOMEM;
-
-  memset(p, 0, sizeof(Fts5Index));
-  p->pConfig = pConfig;
-  p->nCrisisMerge = FTS5_CRISIS_MERGE;
-  p->nWorkUnit = FTS5_WORK_UNIT;
-  p->nMaxPendingData = 1024*1024;
-  p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName);
-  if( p->zDataTbl==0 ){
-    rc = SQLITE_NOMEM;
-  }else if( bCreate ){
-    int i;
-    Fts5Structure s;
-    rc = sqlite3Fts5CreateTable(
-        pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
-    );
-    if( rc==SQLITE_OK ){
-      memset(&s, 0, sizeof(Fts5Structure));
-      for(i=0; i<pConfig->nPrefix+1; i++){
-        fts5StructureWrite(p, i, &s);
-      }
-      rc = p->rc;
-    }
-    sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0);
-  }
-
-  if( rc ){
-    sqlite3Fts5IndexClose(p, 0);
-    *pp = 0;
-  }
-  return rc;
-}
-
-/*
-** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen().
-*/
-int sqlite3Fts5IndexClose(Fts5Index *p, int bDestroy){
-  int rc = SQLITE_OK;
-  if( bDestroy ){
-    rc = sqlite3Fts5DropTable(p->pConfig, "data");
-  }
-  assert( p->pReader==0 );
-  sqlite3_finalize(p->pWriter);
-  sqlite3_finalize(p->pDeleter);
-  if( p->apHash ){
-    int i;
-    for(i=0; i<=p->pConfig->nPrefix; i++){
-      sqlite3Fts5HashFree(p->apHash[i]);
-    }
-    sqlite3_free(p->apHash);
-  }
-  sqlite3_free(p->zDataTbl);
-  sqlite3_free(p);
-  return rc;
-}
-
 /*
 ** Return a simple checksum value based on the arguments.
 */
@@ -3465,31 +3226,6 @@ static u64 fts5IndexEntryCksum(
   return ret;
 }
 
-/*
-** Calculate and return a checksum that is the XOR of the index entry
-** checksum of all entries that would be generated by the token specified
-** by the final 5 arguments.
-*/
-u64 sqlite3Fts5IndexCksum(
-  Fts5Config *pConfig,            /* Configuration object */
-  i64 iRowid,                     /* Document term appears in */
-  int iCol,                       /* Column term appears in */
-  int iPos,                       /* Position term appears in */
-  const char *pTerm, int nTerm    /* Term at iPos */
-){
-  u64 ret = 0;                    /* Return value */
-  int iIdx;                       /* For iterating through indexes */
-
-  for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
-    int n = ((iIdx==pConfig->nPrefix) ? nTerm : pConfig->aPrefix[iIdx]);
-    if( n<=nTerm ){
-      ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, n);
-    }
-  }
-
-  return ret;
-}
-
 static void fts5BtreeIterInit(
   Fts5Index *p, 
   int iIdx,
@@ -3736,17 +3472,269 @@ static void fts5IndexIntegrityCheckSegment(
 }
 
 /*
-** Run internal checks to ensure that the FTS index (a) is internally 
-** consistent and (b) contains entries for which the XOR of the checksums
-** as calculated by fts5IndexEntryCksum() is cksum.
+** Iterator pMulti currently points to a valid entry (not EOF). This
+** function appends a copy of the position-list of the entry pMulti 
+** currently points to to buffer pBuf.
 **
-** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
-** checksum does not match. Return SQLITE_OK if all checks pass without
-** error, or some other SQLite error code if another error (e.g. OOM)
-** occurs.
+** If an error occurs, an error code is left in p->rc. It is assumed
+** no error has already occurred when this function is called.
 */
-int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
-  Fts5Config *pConfig = p->pConfig;
+static void fts5MultiIterPoslist(
+  Fts5Index *p,
+  Fts5MultiSegIter *pMulti,
+  int bSz,
+  Fts5Buffer *pBuf
+){
+  Fts5ChunkIter iter;
+  Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1] ];
+  assert( fts5MultiIterEof(p, pMulti)==0 );
+  fts5ChunkIterInit(p, pSeg, &iter);
+  if( fts5ChunkIterEof(p, &iter)==0 ){
+    if( bSz ){
+      fts5BufferAppendVarint(&p->rc, pBuf, iter.nRem);
+    }
+    while( fts5ChunkIterEof(p, &iter)==0 ){
+      fts5BufferAppendBlob(&p->rc, pBuf, iter.n, iter.p);
+      fts5ChunkIterNext(p, &iter);
+    }
+  }
+  fts5ChunkIterRelease(&iter);
+}
+
+static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
+  if( pIter->i<pIter->n ){
+    if( pIter->i ){
+      i64 iDelta;
+      pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&iDelta);
+      if( pIter->bAsc ){
+        pIter->iRowid += iDelta;
+      }else{
+        pIter->iRowid -= iDelta;
+      }
+    }else{
+      pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
+    }
+    pIter->i += getVarint32(&pIter->a[pIter->i], pIter->nPoslist);
+    pIter->aPoslist = &pIter->a[pIter->i];
+    pIter->i += pIter->nPoslist;
+  }else{
+    pIter->aPoslist = 0;
+  }
+}
+
+static void fts5DoclistIterInit(
+  Fts5Buffer *pBuf, 
+  int bAsc, 
+  Fts5DoclistIter *pIter
+){
+  memset(pIter, 0, sizeof(*pIter));
+  pIter->a = pBuf->p;
+  pIter->n = pBuf->n;
+  pIter->bAsc = bAsc;
+  fts5DoclistIterNext(pIter);
+}
+
+/*
+** Append a doclist to buffer pBuf.
+*/
+static void fts5MergeAppendDocid(
+  int *pRc,                       /* IN/OUT: Error code */
+  int bAsc,
+  Fts5Buffer *pBuf,               /* Buffer to write to */
+  i64 *piLastRowid,               /* IN/OUT: Previous rowid written (if any) */
+  i64 iRowid                      /* Rowid to append */
+){
+  if( pBuf->n==0 ){
+    fts5BufferAppendVarint(pRc, pBuf, iRowid);
+  }else if( bAsc==0 ){
+    fts5BufferAppendVarint(pRc, pBuf, *piLastRowid - iRowid);
+  }else{
+    fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid);
+  }
+  *piLastRowid = iRowid;
+}
+
+/*
+** Buffers p1 and p2 contain doclists. This function merges the content
+** of the two doclists together and sets buffer p1 to the result before
+** returning.
+**
+** If an error occurs, an error code is left in p->rc. If an error has
+** already occurred, this function is a no-op.
+*/
+static void fts5MergePrefixLists(
+  Fts5Index *p,                   /* FTS5 backend object */
+  int bAsc,
+  Fts5Buffer *p1,                 /* First list to merge */
+  Fts5Buffer *p2                  /* Second list to merge */
+){
+  if( p2->n ){
+    i64 iLastRowid = 0;
+    Fts5DoclistIter i1;
+    Fts5DoclistIter i2;
+    Fts5Buffer out;
+    Fts5Buffer tmp;
+    memset(&out, 0, sizeof(out));
+    memset(&tmp, 0, sizeof(tmp));
+
+    fts5DoclistIterInit(p1, bAsc, &i1);
+    fts5DoclistIterInit(p2, bAsc, &i2);
+    while( i1.aPoslist!=0 || i2.aPoslist!=0 ){
+      if( i2.aPoslist==0 || (i1.aPoslist && 
+           ( (!bAsc && i1.iRowid>i2.iRowid) || (bAsc && i1.iRowid<i2.iRowid) )
+      )){
+        /* Copy entry from i1 */
+        fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i1.iRowid);
+        fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist);
+        fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist);
+        fts5DoclistIterNext(&i1);
+      }
+      else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){
+        /* Copy entry from i2 */
+        fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i2.iRowid);
+        fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist);
+        fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist);
+        fts5DoclistIterNext(&i2);
+      }
+      else{
+        Fts5PoslistReader r1;
+        Fts5PoslistReader r2;
+        Fts5PoslistWriter writer;
+
+        memset(&writer, 0, sizeof(writer));
+
+        /* Merge the two position lists. */ 
+        fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i2.iRowid);
+        fts5BufferZero(&tmp);
+        sqlite3Fts5PoslistReaderInit(-1, i1.aPoslist, i1.nPoslist, &r1);
+        sqlite3Fts5PoslistReaderInit(-1, i2.aPoslist, i2.nPoslist, &r2);
+        while( p->rc==SQLITE_OK && (r1.bEof==0 || r2.bEof==0) ){
+          i64 iNew;
+          if( r2.bEof || (r1.bEof==0 && r1.iPos<r2.iPos) ){
+            iNew = r1.iPos;
+            sqlite3Fts5PoslistReaderNext(&r1);
+          }else{
+            iNew = r2.iPos;
+            sqlite3Fts5PoslistReaderNext(&r2);
+            if( r1.iPos==r2.iPos ) sqlite3Fts5PoslistReaderNext(&r1);
+          }
+          p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
+        }
+
+        fts5BufferAppendVarint(&p->rc, &out, tmp.n);
+        fts5BufferAppendBlob(&p->rc, &out, tmp.n, tmp.p);
+        fts5DoclistIterNext(&i1);
+        fts5DoclistIterNext(&i2);
+      }
+    }
+
+    fts5BufferSet(&p->rc, p1, out.n, out.p);
+    fts5BufferFree(&tmp);
+    fts5BufferFree(&out);
+  }
+}
+
+static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
+  Fts5Buffer tmp = *p1;
+  *p1 = *p2;
+  *p2 = tmp;
+}
+
+static void fts5SetupPrefixIter(
+  Fts5Index *p,                   /* Index to read from */
+  int bAsc,                       /* True for "ORDER BY rowid ASC" */
+  const u8 *pToken,               /* Buffer containing prefix to match */
+  int nToken,                     /* Size of buffer pToken in bytes */
+  Fts5IndexIter *pIter            /* Populate this object */
+){
+  Fts5Structure *pStruct;
+  Fts5Buffer *aBuf;
+  const int nBuf = 32;
+
+  aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
+  pStruct = fts5StructureRead(p, 0);
+
+  if( aBuf && pStruct ){
+    Fts5DoclistIter *pDoclist;
+    int i;
+    i64 iLastRowid = 0;
+    Fts5MultiSegIter *p1 = 0;     /* Iterator used to gather data from index */
+    Fts5Buffer doclist;
+
+    memset(&doclist, 0, sizeof(doclist));
+    for(fts5MultiIterNew(p, pStruct, 0, 1, pToken, nToken, -1, 0, &p1);
+        fts5MultiIterEof(p, p1)==0;
+        fts5MultiIterNext(p, p1, 0, 0)
+    ){
+      i64 iRowid = fts5MultiIterRowid(p1);
+      int nTerm;
+      const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm);
+      assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
+      if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;
+
+      if( doclist.n>0 
+       && ((!bAsc && iRowid>=iLastRowid) || (bAsc && iRowid<=iLastRowid))
+      ){
+
+        for(i=0; doclist.n && p->rc==SQLITE_OK; i++){
+          assert( i<nBuf );
+          if( aBuf[i].n==0 ){
+            fts5BufferSwap(&doclist, &aBuf[i]);
+            fts5BufferZero(&doclist);
+          }else{
+            fts5MergePrefixLists(p, bAsc, &doclist, &aBuf[i]);
+            fts5BufferZero(&aBuf[i]);
+          }
+        }
+      }
+      if( doclist.n==0 ){
+        fts5BufferAppendVarint(&p->rc, &doclist, iRowid);
+      }else if( bAsc==0 ){
+        fts5BufferAppendVarint(&p->rc, &doclist, iLastRowid - iRowid);
+      }else{
+        fts5BufferAppendVarint(&p->rc, &doclist, iRowid - iLastRowid);
+      }
+      iLastRowid = iRowid;
+      fts5MultiIterPoslist(p, p1, 1, &doclist);
+    }
+
+    for(i=0; i<nBuf; i++){
+      fts5MergePrefixLists(p, bAsc, &doclist, &aBuf[i]);
+      fts5BufferFree(&aBuf[i]);
+    }
+    fts5MultiIterFree(p, p1);
+
+    pDoclist = (Fts5DoclistIter*)fts5IdxMalloc(p, sizeof(Fts5DoclistIter));
+    if( !pDoclist ){
+      fts5BufferFree(&doclist);
+    }else{
+      pIter->pDoclist = pDoclist;
+      fts5DoclistIterInit(&doclist, bAsc, pIter->pDoclist);
+    }
+  }
+
+  fts5StructureRelease(pStruct);
+  sqlite3_free(aBuf);
+}
+
+static int fts5IndexReturn(Fts5Index *p){
+  int rc = p->rc;
+  p->rc = SQLITE_OK;
+  return rc;
+}
+
+/*
+** Run internal checks to ensure that the FTS index (a) is internally 
+** consistent and (b) contains entries for which the XOR of the checksums
+** as calculated by fts5IndexEntryCksum() is cksum.
+**
+** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
+** checksum does not match. Return SQLITE_OK if all checks pass without
+** error, or some other SQLite error code if another error (e.g. OOM)
+** occurs.
+*/
+int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
+  Fts5Config *pConfig = p->pConfig;
   int iIdx;                       /* Used to iterate through indexes */
   int rc;                         /* Return code */
   u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
@@ -3803,652 +3791,673 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
   return rc;
 }
 
+
 /*
-** This is part of the fts5_decode() debugging aid.
-**
-** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This
-** function appends a human-readable representation of the same object
-** to the buffer passed as the second argument. 
+** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
+** to the document with rowid iRowid.
 */
-static void fts5DecodeStructure(
-  int *pRc,                       /* IN/OUT: error code */
-  Fts5Buffer *pBuf,
-  const u8 *pBlob, int nBlob
-){
-  int rc;                         /* Return code */
-  Fts5Structure *p = 0;           /* Decoded structure object */
-
-  rc = fts5StructureDecode(pBlob, nBlob, 0, &p);
-  if( rc!=SQLITE_OK ){
-    *pRc = rc;
-    return;
+int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){
+  assert( p->rc==SQLITE_OK );
+  if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){
+    fts5IndexFlush(p);
   }
+  p->iWriteRowid = iRowid;
+  return fts5IndexReturn(p);
+}
 
-  fts5DebugStructure(pRc, pBuf, p);
-  fts5StructureRelease(p);
+/*
+** Commit data to disk.
+*/
+int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
+  assert( p->rc==SQLITE_OK );
+  fts5IndexFlush(p);
+  if( bCommit ) fts5CloseReader(p);
+  return fts5IndexReturn(p);
 }
 
 /*
-** Buffer (a/n) is assumed to contain a list of serialized varints. Read
-** each varint and append its string representation to buffer pBuf. Return
-** after either the input buffer is exhausted or a 0 value is read.
-**
-** The return value is the number of bytes read from the input buffer.
+** Discard any data stored in the in-memory hash tables. Do not write it
+** to the database. Additionally, assume that the contents of the %_data
+** table may have changed on disk. So any in-memory caches of %_data 
+** records must be invalidated.
 */
-static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
-  int iOff = 0;
-  while( iOff<n ){
-    int iVal;
-    iOff += getVarint32(&a[iOff], iVal);
-    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal);
-  }
-  return iOff;
+int sqlite3Fts5IndexRollback(Fts5Index *p){
+  fts5CloseReader(p);
+  fts5IndexDiscardData(p);
+  assert( p->rc==SQLITE_OK );
+  return SQLITE_OK;
 }
 
 /*
-** The start of buffer (a/n) contains the start of a doclist. The doclist
-** may or may not finish within the buffer. This function appends a text
-** representation of the part of the doclist that is present to buffer
-** pBuf. 
+** Open a new Fts5Index handle. If the bCreate argument is true, create
+** and initialize the underlying %_data table.
 **
-** The return value is the number of bytes read from the input buffer.
+** If successful, set *pp to point to the new object and return SQLITE_OK.
+** Otherwise, set *pp to NULL and return an SQLite error code.
 */
-static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
-  i64 iDocid;
-  int iOff = 0;
+int sqlite3Fts5IndexOpen(
+  Fts5Config *pConfig, 
+  int bCreate, 
+  Fts5Index **pp,
+  char **pzErr
+){
+  int rc = SQLITE_OK;
+  Fts5Index *p;                   /* New object */
 
-  if( iOff<n ){
-    iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDocid);
-    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
-  }
-  while( iOff<n ){
-    int nPos;
-    iOff += getVarint32(&a[iOff], nPos);
-    iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
-    if( iOff<n ){
-      i64 iDelta;
-      iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDelta);
-      if( iDelta==0 ) return iOff;
-      iDocid -= iDelta;
-      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
+  *pp = p = (Fts5Index*)sqlite3_malloc(sizeof(Fts5Index));
+  if( !p ) return SQLITE_NOMEM;
+
+  memset(p, 0, sizeof(Fts5Index));
+  p->pConfig = pConfig;
+  p->nCrisisMerge = FTS5_CRISIS_MERGE;
+  p->nWorkUnit = FTS5_WORK_UNIT;
+  p->nMaxPendingData = 1024*1024;
+  p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName);
+  if( p->zDataTbl==0 ){
+    rc = SQLITE_NOMEM;
+  }else if( bCreate ){
+    int i;
+    Fts5Structure s;
+    rc = sqlite3Fts5CreateTable(
+        pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
+    );
+    if( rc==SQLITE_OK ){
+      memset(&s, 0, sizeof(Fts5Structure));
+      for(i=0; i<pConfig->nPrefix+1; i++){
+        fts5StructureWrite(p, i, &s);
+      }
+      rc = p->rc;
+    }
+    if( rc==SQLITE_OK ){
+      rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0);
     }
   }
 
-  return iOff;
+  assert( p->rc==SQLITE_OK || rc!=SQLITE_OK );
+  if( rc ){
+    sqlite3Fts5IndexClose(p, 0);
+    *pp = 0;
+  }
+  return rc;
 }
 
 /*
-** The implementation of user-defined scalar function fts5_decode().
+** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen().
 */
-static void fts5DecodeFunction(
-  sqlite3_context *pCtx,          /* Function call context */
-  int nArg,                       /* Number of args (always 2) */
-  sqlite3_value **apVal           /* Function arguments */
-){
-  i64 iRowid;                     /* Rowid for record being decoded */
-  int iIdx,iSegid,iHeight,iPgno;  /* Rowid components */
-  const u8 *a; int n;             /* Record to decode */
-  Fts5Buffer s;                   /* Build up text to return here */
-  int rc = SQLITE_OK;             /* Return code */
-
-  assert( nArg==2 );
-  memset(&s, 0, sizeof(Fts5Buffer));
-  iRowid = sqlite3_value_int64(apVal[0]);
-  n = sqlite3_value_bytes(apVal[1]);
-  a = sqlite3_value_blob(apVal[1]);
-  fts5DecodeRowid(iRowid, &iIdx, &iSegid, &iHeight, &iPgno);
-
-  fts5DebugRowid(&rc, &s, iRowid);
-  if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
-    int i = 0;
-    i64 iPrev;
-    if( n>0 ){
-      i = getVarint(&a[i], (u64*)&iPrev);
-      sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", iPrev);
+int sqlite3Fts5IndexClose(Fts5Index *p, int bDestroy){
+  int rc = SQLITE_OK;
+  if( p ){
+    if( bDestroy ){
+      rc = sqlite3Fts5DropTable(p->pConfig, "data");
     }
-    while( i<n ){
-      i64 iVal;
-      i += getVarint(&a[i], (u64*)&iVal);
-      if( iVal==0 ){
-        sqlite3Fts5BufferAppendPrintf(&rc, &s, " x");
-      }else{
-        iPrev = iPrev - iVal;
-        sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", iPrev);
+    assert( p->pReader==0 );
+    sqlite3_finalize(p->pWriter);
+    sqlite3_finalize(p->pDeleter);
+    if( p->apHash ){
+      int i;
+      for(i=0; i<=p->pConfig->nPrefix; i++){
+        sqlite3Fts5HashFree(p->apHash[i]);
       }
+      sqlite3_free(p->apHash);
     }
+    sqlite3_free(p->zDataTbl);
+    sqlite3_free(p);
+  }
+  return rc;
+}
 
-  }else
-  if( iSegid==0 ){
-    if( iRowid==FTS5_AVERAGES_ROWID ){
-      /* todo */
-    }else{
-      fts5DecodeStructure(&rc, &s, a, n);
+/*
+** Calculate and return a checksum that is the XOR of the index entry
+** checksum of all entries that would be generated by the token specified
+** by the final 5 arguments.
+*/
+u64 sqlite3Fts5IndexCksum(
+  Fts5Config *pConfig,            /* Configuration object */
+  i64 iRowid,                     /* Document term appears in */
+  int iCol,                       /* Column term appears in */
+  int iPos,                       /* Position term appears in */
+  const char *pTerm, int nTerm    /* Term at iPos */
+){
+  u64 ret = 0;                    /* Return value */
+  int iIdx;                       /* For iterating through indexes */
+
+  for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
+    int n = ((iIdx==pConfig->nPrefix) ? nTerm : pConfig->aPrefix[iIdx]);
+    if( n<=nTerm ){
+      ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, n);
     }
-  }else{
+  }
 
-    Fts5Buffer term;
-    memset(&term, 0, sizeof(Fts5Buffer));
+  return ret;
+}
 
-    if( iHeight==0 ){
-      int iTermOff = 0;
-      int iRowidOff = 0;
-      int iOff;
-      int nKeep = 0;
+/*
+** Insert or remove data to or from the index. Each time a document is 
+** added to or removed from the index, this function is called one or more
+** times.
+**
+** For an insert, it must be called once for each token in the new document.
+** If the operation is a delete, it must be called (at least) once for each
+** unique token in the document with an iCol value less than zero. The iPos
+** argument is ignored for a delete.
+*/
+int sqlite3Fts5IndexWrite(
+  Fts5Index *p,                   /* Index to write to */
+  int iCol,                       /* Column token appears in (-ve -> delete) */
+  int iPos,                       /* Position of token within column */
+  const char *pToken, int nToken  /* Token to add or remove to or from index */
+){
+  int i;                          /* Used to iterate through indexes */
+  Fts5Config *pConfig = p->pConfig;
+  assert( p->rc==SQLITE_OK );
 
-      iRowidOff = fts5GetU16(&a[0]);
-      iTermOff = fts5GetU16(&a[2]);
+  /* Allocate hash tables if they have not already been allocated */
+  if( p->apHash==0 ){
+    int nHash = pConfig->nPrefix + 1;
+    p->apHash = (Fts5Hash**)fts5IdxMalloc(p, sizeof(Fts5Hash*) * nHash);
+    for(i=0; p->rc==SQLITE_OK && i<nHash; i++){
+      p->rc = sqlite3Fts5HashNew(&p->apHash[i], &p->nPendingData);
+    }
+  }
 
-      if( iRowidOff ){
-        iOff = iRowidOff;
-      }else if( iTermOff ){
-        iOff = iTermOff;
-      }else{
-        iOff = n;
-      }
-      fts5DecodePoslist(&rc, &s, &a[4], iOff-4);
+  /* Add the new token to the main terms hash table. And to each of the
+  ** prefix hash tables that it is large enough for. */
+  fts5AddTermToHash(p, 0, iCol, iPos, pToken, nToken);
+  for(i=0; i<pConfig->nPrefix; i++){
+    if( nToken>=pConfig->aPrefix[i] ){
+      fts5AddTermToHash(p, i+1, iCol, iPos, pToken, pConfig->aPrefix[i]);
+    }
+  }
 
+  return fts5IndexReturn(p);
+}
 
-      assert( iRowidOff==0 || iOff==iRowidOff );
-      if( iRowidOff ){
-        iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
-      }
+/*
+** Open a new iterator to iterate though all docids that match the 
+** specified token or token prefix.
+*/
+int sqlite3Fts5IndexQuery(
+  Fts5Index *p,                   /* FTS index to query */
+  const char *pToken, int nToken, /* Token (or prefix) to query for */
+  int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
+  Fts5IndexIter **ppIter          /* OUT: New iterator object */
+){
+  Fts5IndexIter *pRet;
+  int iIdx = 0;
 
-      assert( iTermOff==0 || iOff==iTermOff );
-      while( iOff<n ){
-        int nByte;
-        iOff += getVarint32(&a[iOff], nByte);
-        term.n= nKeep;
-        fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]);
-        iOff += nByte;
+  if( flags & FTS5INDEX_QUERY_PREFIX ){
+    Fts5Config *pConfig = p->pConfig;
+    for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
+      if( pConfig->aPrefix[iIdx-1]==nToken ) break;
+    }
+    if( iIdx>pConfig->nPrefix ){
+      iIdx = -1;
+    }
+  }
 
-        sqlite3Fts5BufferAppendPrintf(
-            &rc, &s, " term=%.*s", term.n, (const char*)term.p
+  pRet = (Fts5IndexIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5IndexIter));
+  if( pRet ){
+    memset(pRet, 0, sizeof(Fts5IndexIter));
+
+    pRet->pIndex = p;
+    if( iIdx>=0 ){
+      pRet->pStruct = fts5StructureRead(p, iIdx);
+      if( pRet->pStruct ){
+        fts5MultiIterNew(p, pRet->pStruct, 
+            iIdx, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti
         );
-        iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
-        if( iOff<n ){
-          iOff += getVarint32(&a[iOff], nKeep);
-        }
       }
-      fts5BufferFree(&term);
     }else{
-      Fts5NodeIter ss;
-      for(fts5NodeIterInit(a, n, &ss); ss.aData; fts5NodeIterNext(&rc, &ss)){
-        if( ss.term.n==0 ){
-          sqlite3Fts5BufferAppendPrintf(&rc, &s, " left=%d", ss.iChild);
-        }else{
-          sqlite3Fts5BufferAppendPrintf(&rc,&s, " \"%.*s\"", 
-              ss.term.n, ss.term.p
-          );
-        }
-        if( ss.nEmpty ){
-          sqlite3Fts5BufferAppendPrintf(&rc, &s, " empty=%d%s", ss.nEmpty,
-              ss.bDlidx ? "*" : ""
-          );
-        }
-      }
-      fts5NodeIterFree(&ss);
+      int bAsc = (flags & FTS5INDEX_QUERY_ASC)!=0;
+      fts5SetupPrefixIter(p, bAsc, (const u8*)pToken, nToken, pRet);
     }
   }
-  
-  if( rc==SQLITE_OK ){
-    sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT);
-  }else{
-    sqlite3_result_error_code(pCtx, rc);
+
+  if( p->rc ){
+    sqlite3Fts5IterClose(pRet);
+    pRet = 0;
   }
-  fts5BufferFree(&s);
+  *ppIter = pRet;
+  return fts5IndexReturn(p);
 }
 
 /*
-** This is called as part of registering the FTS5 module with database
-** connection db. It registers several user-defined scalar functions useful
-** with FTS5.
-**
-** If successful, SQLITE_OK is returned. If an error occurs, some other
-** SQLite error code is returned instead.
+** Return true if the iterator passed as the only argument is at EOF.
 */
-int sqlite3Fts5IndexInit(sqlite3 *db){
-  int rc = sqlite3_create_function(
-      db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
-  );
-  return rc;
+int sqlite3Fts5IterEof(Fts5IndexIter *pIter){
+  assert( pIter->pIndex->rc==SQLITE_OK );
+  if( pIter->pDoclist ){ 
+    return pIter->pDoclist->aPoslist==0; 
+  }else{
+    return fts5MultiIterEof(pIter->pIndex, pIter->pMulti);
+  }
 }
 
 /*
-** Iterator pMulti currently points to a valid entry (not EOF). This
-** function appends a copy of the position-list of the entry pMulti 
-** currently points to to buffer pBuf.
-**
-** If an error occurs, an error code is left in p->rc. It is assumed
-** no error has already occurred when this function is called.
+** Move to the next matching rowid. 
 */
-static void fts5MultiIterPoslist(
-  Fts5Index *p,
-  Fts5MultiSegIter *pMulti,
-  int bSz,
-  Fts5Buffer *pBuf
-){
-  Fts5ChunkIter iter;
-  Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1] ];
-  assert( fts5MultiIterEof(p, pMulti)==0 );
-  fts5ChunkIterInit(p, pSeg, &iter);
-  if( fts5ChunkIterEof(p, &iter)==0 ){
-    if( bSz ){
-      fts5BufferAppendVarint(&p->rc, pBuf, iter.nRem);
-    }
-    while( fts5ChunkIterEof(p, &iter)==0 ){
-      fts5BufferAppendBlob(&p->rc, pBuf, iter.n, iter.p);
-      fts5ChunkIterNext(p, &iter);
-    }
+int sqlite3Fts5IterNext(Fts5IndexIter *pIter){
+  assert( pIter->pIndex->rc==SQLITE_OK );
+  if( pIter->pDoclist ){
+    fts5DoclistIterNext(pIter->pDoclist);
+  }else{
+    fts5BufferZero(&pIter->poslist);
+    fts5MultiIterNext(pIter->pIndex, pIter->pMulti, 0, 0);
   }
-  fts5ChunkIterRelease(&iter);
+  return fts5IndexReturn(pIter->pIndex);
 }
 
-static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
-  if( pIter->i<pIter->n ){
-    if( pIter->i ){
-      i64 iDelta;
-      pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&iDelta);
-      if( pIter->bAsc ){
-        pIter->iRowid += iDelta;
-      }else{
-        pIter->iRowid -= iDelta;
-      }
-    }else{
-      pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
-    }
-    pIter->i += getVarint32(&pIter->a[pIter->i], pIter->nPoslist);
-    pIter->aPoslist = &pIter->a[pIter->i];
-    pIter->i += pIter->nPoslist;
+/*
+** Move to the next matching rowid that occurs at or after iMatch. The
+** definition of "at or after" depends on whether this iterator iterates
+** in ascending or descending rowid order.
+*/
+int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){
+  if( pIter->pDoclist ){
+    assert( 0 );
+    /* fts5DoclistIterNextFrom(pIter->pDoclist, iMatch); */
   }else{
-    pIter->aPoslist = 0;
+    fts5MultiIterNextFrom(pIter->pIndex, pIter->pMulti, iMatch);
   }
+  return fts5IndexReturn(pIter->pIndex);
 }
 
-static void fts5DoclistIterInit(
-  Fts5Buffer *pBuf, 
-  int bAsc, 
-  Fts5DoclistIter *pIter
-){
-  memset(pIter, 0, sizeof(*pIter));
-  pIter->a = pBuf->p;
-  pIter->n = pBuf->n;
-  pIter->bAsc = bAsc;
-  fts5DoclistIterNext(pIter);
+/*
+** Return the current rowid.
+*/
+i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){
+  if( pIter->pDoclist ){
+    return pIter->pDoclist->iRowid;
+  }else{
+    return fts5MultiIterRowid(pIter->pMulti);
+  }
 }
 
+
 /*
-** Append a doclist to buffer pBuf.
+** Return a pointer to a buffer containing a copy of the position list for
+** the current entry. Output variable *pn is set to the size of the buffer 
+** in bytes before returning.
+**
+** The returned buffer does not include the 0x00 terminator byte stored on
+** disk.
 */
-static void fts5MergeAppendDocid(
-  int *pRc,                       /* IN/OUT: Error code */
-  int bAsc,
-  Fts5Buffer *pBuf,               /* Buffer to write to */
-  i64 *piLastRowid,               /* IN/OUT: Previous rowid written (if any) */
-  i64 iRowid                      /* Rowid to append */
-){
-  if( pBuf->n==0 ){
-    fts5BufferAppendVarint(pRc, pBuf, iRowid);
-  }else if( bAsc==0 ){
-    fts5BufferAppendVarint(pRc, pBuf, *piLastRowid - iRowid);
+int sqlite3Fts5IterPoslist(Fts5IndexIter *pIter, const u8 **pp, int *pn){
+  assert( pIter->pIndex->rc==SQLITE_OK );
+  if( pIter->pDoclist ){
+    *pn = pIter->pDoclist->nPoslist;
+    *pp = pIter->pDoclist->aPoslist;
   }else{
-    fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid);
+    Fts5Index *p = pIter->pIndex;
+    fts5BufferZero(&pIter->poslist);
+    fts5MultiIterPoslist(p, pIter->pMulti, 0, &pIter->poslist);
+    *pn = pIter->poslist.n;
+    *pp = pIter->poslist.p;
   }
-  *piLastRowid = iRowid;
+  return fts5IndexReturn(pIter->pIndex);
 }
 
 /*
-** Buffers p1 and p2 contain doclists. This function merges the content
-** of the two doclists together and sets buffer p1 to the result before
-** returning.
-**
-** If an error occurs, an error code is left in p->rc. If an error has
-** already occurred, this function is a no-op.
+** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
 */
-static void fts5MergePrefixLists(
-  Fts5Index *p,                   /* FTS5 backend object */
-  int bAsc,
-  Fts5Buffer *p1,                 /* First list to merge */
-  Fts5Buffer *p2                  /* Second list to merge */
-){
-  if( p2->n ){
-    i64 iLastRowid = 0;
-    Fts5DoclistIter i1;
-    Fts5DoclistIter i2;
-    Fts5Buffer out;
-    Fts5Buffer tmp;
-    memset(&out, 0, sizeof(out));
-    memset(&tmp, 0, sizeof(tmp));
+void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
+  if( pIter ){
+    if( pIter->pDoclist ){
+      sqlite3_free(pIter->pDoclist->a);
+      sqlite3_free(pIter->pDoclist);
+    }else{
+      fts5MultiIterFree(pIter->pIndex, pIter->pMulti);
+      fts5StructureRelease(pIter->pStruct);
+      fts5BufferFree(&pIter->poslist);
+    }
+    fts5CloseReader(pIter->pIndex);
+    sqlite3_free(pIter);
+  }
+}
 
-    fts5DoclistIterInit(p1, bAsc, &i1);
-    fts5DoclistIterInit(p2, bAsc, &i2);
-    while( i1.aPoslist!=0 || i2.aPoslist!=0 ){
-      if( i2.aPoslist==0 || (i1.aPoslist && 
-           ( (!bAsc && i1.iRowid>i2.iRowid) || (bAsc && i1.iRowid<i2.iRowid) )
-      )){
-        /* Copy entry from i1 */
-        fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i1.iRowid);
-        fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist);
-        fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist);
-        fts5DoclistIterNext(&i1);
-      }
-      else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){
-        /* Copy entry from i2 */
-        fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i2.iRowid);
-        fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist);
-        fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist);
-        fts5DoclistIterNext(&i2);
-      }
-      else{
-        Fts5PoslistReader r1;
-        Fts5PoslistReader r2;
-        Fts5PoslistWriter writer;
+/*
+** Read the "averages" record into the buffer supplied as the second 
+** argument. Return SQLITE_OK if successful, or an SQLite error code
+** if an error occurs.
+*/
+int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){
+  assert( p->rc==SQLITE_OK );
+  fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID);
+  return fts5IndexReturn(p);
+}
 
-        memset(&writer, 0, sizeof(writer));
+/*
+** Replace the current "averages" record with the contents of the buffer 
+** supplied as the second argument.
+*/
+int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){
+  assert( p->rc==SQLITE_OK );
+  fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData);
+  return fts5IndexReturn(p);
+}
 
-        /* Merge the two position lists. */ 
-        fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i2.iRowid);
-        fts5BufferZero(&tmp);
-        sqlite3Fts5PoslistReaderInit(-1, i1.aPoslist, i1.nPoslist, &r1);
-        sqlite3Fts5PoslistReaderInit(-1, i2.aPoslist, i2.nPoslist, &r2);
-        while( p->rc==SQLITE_OK && (r1.bEof==0 || r2.bEof==0) ){
-          i64 iNew;
-          if( r2.bEof || (r1.bEof==0 && r1.iPos<r2.iPos) ){
-            iNew = r1.iPos;
-            sqlite3Fts5PoslistReaderNext(&r1);
-          }else{
-            iNew = r2.iPos;
-            sqlite3Fts5PoslistReaderNext(&r2);
-            if( r1.iPos==r2.iPos ) sqlite3Fts5PoslistReaderNext(&r1);
-          }
-          p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
-        }
+/*
+** Return the total number of blocks this module has read from the %_data
+** table since it was created.
+*/
+int sqlite3Fts5IndexReads(Fts5Index *p){
+  return p->nRead;
+}
 
-        fts5BufferAppendVarint(&p->rc, &out, tmp.n);
-        fts5BufferAppendBlob(&p->rc, &out, tmp.n, tmp.p);
-        fts5DoclistIterNext(&i1);
-        fts5DoclistIterNext(&i2);
-      }
-    }
+/*
+** Set the 32-bit cookie value stored at the start of all structure 
+** records to the value passed as the second argument.
+**
+** Return SQLITE_OK if successful, or an SQLite error code if an error
+** occurs.
+*/
+int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){
+  int rc = SQLITE_OK;
+  Fts5Config *pConfig = p->pConfig;
+  u8 aCookie[4];
+  int i;
 
-    fts5BufferSet(&p->rc, p1, out.n, out.p);
-    fts5BufferFree(&tmp);
-    fts5BufferFree(&out);
+  assert( p->rc==SQLITE_OK );
+  sqlite3Fts5Put32(aCookie, iNew);
+  for(i=0; rc==SQLITE_OK && i<=pConfig->nPrefix; i++){
+    sqlite3_blob *pBlob = 0;
+    i64 iRowid = FTS5_STRUCTURE_ROWID(i);
+    rc = sqlite3_blob_open(
+        pConfig->db, pConfig->zDb, p->zDataTbl, "block", iRowid, 1, &pBlob
+    );
+    if( rc==SQLITE_OK ){
+      sqlite3_blob_write(pBlob, aCookie, 4, 0);
+      rc = sqlite3_blob_close(pBlob);
+    }
   }
-}
 
-static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
-  Fts5Buffer tmp = *p1;
-  *p1 = *p2;
-  *p2 = tmp;
+  return rc;
 }
 
-static void fts5SetupPrefixIter(
-  Fts5Index *p,                   /* Index to read from */
-  int bAsc,                       /* True for "ORDER BY rowid ASC" */
-  const u8 *pToken,               /* Buffer containing prefix to match */
-  int nToken,                     /* Size of buffer pToken in bytes */
-  Fts5IndexIter *pIter            /* Populate this object */
-){
-  Fts5Structure *pStruct;
-  Fts5Buffer *aBuf;
-  const int nBuf = 32;
-
-  aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
-  pStruct = fts5StructureRead(p, 0);
+/*************************************************************************
+**************************************************************************
+** Below this point is the implementation of the fts5_decode() scalar
+** function only.
+*/
 
-  if( aBuf && pStruct ){
-    Fts5DoclistIter *pDoclist;
-    int i;
-    i64 iLastRowid = 0;
-    Fts5MultiSegIter *p1 = 0;     /* Iterator used to gather data from index */
-    Fts5Buffer doclist;
+/*
+** Decode a segment-data rowid from the %_data table. This function is
+** the opposite of macro FTS5_SEGMENT_ROWID().
+*/
+static void fts5DecodeRowid(
+  i64 iRowid,                     /* Rowid from %_data table */
+  int *piIdx,                     /* OUT: Index */
+  int *piSegid,                   /* OUT: Segment id */
+  int *piHeight,                  /* OUT: Height */
+  int *piPgno                     /* OUT: Page number */
+){
+  *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1));
+  iRowid >>= FTS5_DATA_PAGE_B;
 
-    memset(&doclist, 0, sizeof(doclist));
-    for(fts5MultiIterNew(p, pStruct, 0, 1, pToken, nToken, -1, 0, &p1);
-        fts5MultiIterEof(p, p1)==0;
-        fts5MultiIterNext(p, p1, 0, 0)
-    ){
-      i64 iRowid = fts5MultiIterRowid(p1);
-      int nTerm;
-      const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm);
-      assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
-      if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;
+  *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1));
+  iRowid >>= FTS5_DATA_HEIGHT_B;
 
-      if( doclist.n>0 
-       && ((!bAsc && iRowid>=iLastRowid) || (bAsc && iRowid<=iLastRowid))
-      ){
+  *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1));
+  iRowid >>= FTS5_DATA_ID_B;
 
-        for(i=0; doclist.n && p->rc==SQLITE_OK; i++){
-          assert( i<nBuf );
-          if( aBuf[i].n==0 ){
-            fts5BufferSwap(&doclist, &aBuf[i]);
-            fts5BufferZero(&doclist);
-          }else{
-            fts5MergePrefixLists(p, bAsc, &doclist, &aBuf[i]);
-            fts5BufferZero(&aBuf[i]);
-          }
-        }
-      }
-      if( doclist.n==0 ){
-        fts5BufferAppendVarint(&p->rc, &doclist, iRowid);
-      }else if( bAsc==0 ){
-        fts5BufferAppendVarint(&p->rc, &doclist, iLastRowid - iRowid);
-      }else{
-        fts5BufferAppendVarint(&p->rc, &doclist, iRowid - iLastRowid);
-      }
-      iLastRowid = iRowid;
-      fts5MultiIterPoslist(p, p1, 1, &doclist);
-    }
+  *piIdx = (int)(iRowid & (((i64)1 << FTS5_DATA_IDX_B) - 1));
+}
 
-    for(i=0; i<nBuf; i++){
-      fts5MergePrefixLists(p, bAsc, &doclist, &aBuf[i]);
-      fts5BufferFree(&aBuf[i]);
-    }
-    fts5MultiIterFree(p, p1);
+static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
+  int iIdx,iSegid,iHeight,iPgno;  /* Rowid compenents */
+  fts5DecodeRowid(iKey, &iIdx, &iSegid, &iHeight, &iPgno);
 
-    pDoclist = (Fts5DoclistIter*)fts5IdxMalloc(p, sizeof(Fts5DoclistIter));
-    if( !pDoclist ){
-      fts5BufferFree(&doclist);
+  if( iSegid==0 ){
+    if( iKey==FTS5_AVERAGES_ROWID ){
+      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(averages) ");
     }else{
-      pIter->pDoclist = pDoclist;
-      fts5DoclistIterInit(&doclist, bAsc, pIter->pDoclist);
+      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
+          "{structure idx=%d}", (int)(iKey-10)
+      );
     }
   }
-
-  fts5StructureRelease(pStruct);
-  sqlite3_free(aBuf);
+  else if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(dlidx idx=%d segid=%d pgno=%d)",
+        iIdx, iSegid, iPgno
+    );
+  }else{
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(idx=%d segid=%d h=%d pgno=%d)",
+        iIdx, iSegid, iHeight, iPgno
+    );
+  }
 }
 
-/*
-** Open a new iterator to iterate though all docids that match the 
-** specified token or token prefix.
-*/
-Fts5IndexIter *sqlite3Fts5IndexQuery(
-  Fts5Index *p,                   /* FTS index to query */
-  const char *pToken, int nToken, /* Token (or prefix) to query for */
-  int flags                       /* Mask of FTS5INDEX_QUERY_X flags */
+static void fts5DebugStructure(
+  int *pRc,                       /* IN/OUT: error code */
+  Fts5Buffer *pBuf,
+  Fts5Structure *p
 ){
-  Fts5IndexIter *pRet;
-  int iIdx = 0;
-
-  if( flags & FTS5INDEX_QUERY_PREFIX ){
-    Fts5Config *pConfig = p->pConfig;
-    for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
-      if( pConfig->aPrefix[iIdx-1]==nToken ) break;
-    }
-    if( iIdx>pConfig->nPrefix ){
-      iIdx = -1;
-    }
-  }
-
-  pRet = (Fts5IndexIter*)sqlite3_malloc(sizeof(Fts5IndexIter));
-  if( pRet ){
-    memset(pRet, 0, sizeof(Fts5IndexIter));
+  int iLvl, iSeg;                 /* Iterate through levels, segments */
 
-    pRet->pIndex = p;
-    if( iIdx>=0 ){
-      pRet->pStruct = fts5StructureRead(p, iIdx);
-      if( pRet->pStruct ){
-        fts5MultiIterNew(p, pRet->pStruct, 
-            iIdx, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti
-        );
-      }
-    }else{
-      int bAsc = (flags & FTS5INDEX_QUERY_ASC)!=0;
-      fts5SetupPrefixIter(p, bAsc, (const u8*)pToken, nToken, pRet);
+  for(iLvl=0; iLvl<p->nLevel; iLvl++){
+    Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
+        " {lvl=%d nMerge=%d", iLvl, pLvl->nMerge
+    );
+    for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
+      Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
+      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
+          " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, 
+          pSeg->pgnoFirst, pSeg->pgnoLast
+      );
     }
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
   }
-
-  if( p->rc ){
-    sqlite3Fts5IterClose(pRet);
-    pRet = 0;
-  }
-  return pRet;
 }
 
 /*
-** Return true if the iterator passed as the only argument is at EOF.
+** This is part of the fts5_decode() debugging aid.
+**
+** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This
+** function appends a human-readable representation of the same object
+** to the buffer passed as the second argument. 
 */
-int sqlite3Fts5IterEof(Fts5IndexIter *pIter){
-  if( pIter->pDoclist ){ 
-    return pIter->pDoclist->aPoslist==0; 
-  }else{
-    return fts5MultiIterEof(pIter->pIndex, pIter->pMulti);
-  }
-}
+static void fts5DecodeStructure(
+  int *pRc,                       /* IN/OUT: error code */
+  Fts5Buffer *pBuf,
+  const u8 *pBlob, int nBlob
+){
+  int rc;                         /* Return code */
+  Fts5Structure *p = 0;           /* Decoded structure object */
 
-/*
-** Move to the next matching rowid. 
-*/
-void sqlite3Fts5IterNext(Fts5IndexIter *pIter){
-  if( pIter->pDoclist ){
-    fts5DoclistIterNext(pIter->pDoclist);
-  }else{
-    fts5BufferZero(&pIter->poslist);
-    fts5MultiIterNext(pIter->pIndex, pIter->pMulti, 0, 0);
+  rc = fts5StructureDecode(pBlob, nBlob, 0, &p);
+  if( rc!=SQLITE_OK ){
+    *pRc = rc;
+    return;
   }
-}
 
-/*
-** Move to the next matching rowid that occurs at or after iMatch. The
-** definition of "at or after" depends on whether this iterator iterates
-** in ascending or descending rowid order.
-*/
-void sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){
-  if( pIter->pDoclist ){
-    assert( 0 );
-    /* fts5DoclistIterNextFrom(pIter->pDoclist, iMatch); */
-  }else{
-    fts5MultiIterNextFrom(pIter->pIndex, pIter->pMulti, iMatch);
-  }
+  fts5DebugStructure(pRc, pBuf, p);
+  fts5StructureRelease(p);
 }
 
 /*
-** Return the current rowid.
+** Buffer (a/n) is assumed to contain a list of serialized varints. Read
+** each varint and append its string representation to buffer pBuf. Return
+** after either the input buffer is exhausted or a 0 value is read.
+**
+** The return value is the number of bytes read from the input buffer.
 */
-i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){
-  if( pIter->pDoclist ){
-    return pIter->pDoclist->iRowid;
-  }else{
-    return fts5MultiIterRowid(pIter->pMulti);
+static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
+  int iOff = 0;
+  while( iOff<n ){
+    int iVal;
+    iOff += getVarint32(&a[iOff], iVal);
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal);
   }
+  return iOff;
 }
 
-
 /*
-** Return a pointer to a buffer containing a copy of the position list for
-** the current entry. Output variable *pn is set to the size of the buffer 
-** in bytes before returning.
+** The start of buffer (a/n) contains the start of a doclist. The doclist
+** may or may not finish within the buffer. This function appends a text
+** representation of the part of the doclist that is present to buffer
+** pBuf. 
 **
-** The returned buffer does not include the 0x00 terminator byte stored on
-** disk.
+** The return value is the number of bytes read from the input buffer.
 */
-const u8 *sqlite3Fts5IterPoslist(Fts5IndexIter *pIter, int *pn){
-  if( pIter->pDoclist ){
-    *pn = pIter->pDoclist->nPoslist;
-    return pIter->pDoclist->aPoslist;
-  }else{
-    Fts5Index *p = pIter->pIndex;
-    fts5BufferZero(&pIter->poslist);
-    fts5MultiIterPoslist(p, pIter->pMulti, 0, &pIter->poslist);
-    assert( p->rc==SQLITE_OK );
-    if( p->rc ) return 0;
-    *pn = pIter->poslist.n;
-    return pIter->poslist.p;
+static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
+  i64 iDocid;
+  int iOff = 0;
+
+  if( iOff<n ){
+    iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDocid);
+    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
   }
+  while( iOff<n ){
+    int nPos;
+    iOff += getVarint32(&a[iOff], nPos);
+    iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
+    if( iOff<n ){
+      i64 iDelta;
+      iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDelta);
+      if( iDelta==0 ) return iOff;
+      iDocid -= iDelta;
+      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
+    }
+  }
+
+  return iOff;
 }
 
 /*
-** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
+** The implementation of user-defined scalar function fts5_decode().
 */
-void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
-  if( pIter ){
-    if( pIter->pDoclist ){
-      sqlite3_free(pIter->pDoclist->a);
-      sqlite3_free(pIter->pDoclist);
+static void fts5DecodeFunction(
+  sqlite3_context *pCtx,          /* Function call context */
+  int nArg,                       /* Number of args (always 2) */
+  sqlite3_value **apVal           /* Function arguments */
+){
+  i64 iRowid;                     /* Rowid for record being decoded */
+  int iIdx,iSegid,iHeight,iPgno;  /* Rowid components */
+  const u8 *a; int n;             /* Record to decode */
+  Fts5Buffer s;                   /* Build up text to return here */
+  int rc = SQLITE_OK;             /* Return code */
+
+  assert( nArg==2 );
+  memset(&s, 0, sizeof(Fts5Buffer));
+  iRowid = sqlite3_value_int64(apVal[0]);
+  n = sqlite3_value_bytes(apVal[1]);
+  a = sqlite3_value_blob(apVal[1]);
+  fts5DecodeRowid(iRowid, &iIdx, &iSegid, &iHeight, &iPgno);
+
+  fts5DebugRowid(&rc, &s, iRowid);
+  if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
+    int i = 0;
+    i64 iPrev;
+    if( n>0 ){
+      i = getVarint(&a[i], (u64*)&iPrev);
+      sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", iPrev);
+    }
+    while( i<n ){
+      i64 iVal;
+      i += getVarint(&a[i], (u64*)&iVal);
+      if( iVal==0 ){
+        sqlite3Fts5BufferAppendPrintf(&rc, &s, " x");
+      }else{
+        iPrev = iPrev - iVal;
+        sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", iPrev);
+      }
+    }
+
+  }else
+  if( iSegid==0 ){
+    if( iRowid==FTS5_AVERAGES_ROWID ){
+      /* todo */
     }else{
-      fts5MultiIterFree(pIter->pIndex, pIter->pMulti);
-      fts5StructureRelease(pIter->pStruct);
-      fts5BufferFree(&pIter->poslist);
+      fts5DecodeStructure(&rc, &s, a, n);
     }
-    fts5CloseReader(pIter->pIndex);
-    sqlite3_free(pIter);
-  }
-}
+  }else{
 
-/*
-** Read the "averages" record into the buffer supplied as the second 
-** argument. Return SQLITE_OK if successful, or an SQLite error code
-** if an error occurs.
-*/
-int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){
-  fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID);
-  return p->rc;
-}
+    Fts5Buffer term;
+    memset(&term, 0, sizeof(Fts5Buffer));
 
-/*
-** Replace the current "averages" record with the contents of the buffer 
-** supplied as the second argument.
-*/
-int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){
-  fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData);
-  return p->rc;
-}
+    if( iHeight==0 ){
+      int iTermOff = 0;
+      int iRowidOff = 0;
+      int iOff;
+      int nKeep = 0;
 
-/*
-** Return the total number of blocks this module has read from the %_data
-** table since it was created.
-*/
-int sqlite3Fts5IndexReads(Fts5Index *p){
-  return p->nRead;
-}
+      iRowidOff = fts5GetU16(&a[0]);
+      iTermOff = fts5GetU16(&a[2]);
 
-/*
-** Set the 32-bit cookie value at the start of all structure records to
-** the value passed as the second argument.
-**
-** Return SQLITE_OK if successful, or an SQLite error code if an error
-** occurs.
-*/
-int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){
-  int rc = SQLITE_OK;
-  Fts5Config *pConfig = p->pConfig;
-  u8 aCookie[4];
-  int i;
+      if( iRowidOff ){
+        iOff = iRowidOff;
+      }else if( iTermOff ){
+        iOff = iTermOff;
+      }else{
+        iOff = n;
+      }
+      fts5DecodePoslist(&rc, &s, &a[4], iOff-4);
 
-  sqlite3Fts5Put32(aCookie, iNew);
-  for(i=0; rc==SQLITE_OK && i<=pConfig->nPrefix; i++){
-    sqlite3_blob *pBlob = 0;
-    i64 iRowid = FTS5_STRUCTURE_ROWID(i);
-    rc = sqlite3_blob_open(
-        pConfig->db, pConfig->zDb, p->zDataTbl, "block", iRowid, 1, &pBlob
-    );
-    if( rc==SQLITE_OK ){
-      sqlite3_blob_write(pBlob, aCookie, 4, 0);
-      rc = sqlite3_blob_close(pBlob);
+
+      assert( iRowidOff==0 || iOff==iRowidOff );
+      if( iRowidOff ){
+        iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
+      }
+
+      assert( iTermOff==0 || iOff==iTermOff );
+      while( iOff<n ){
+        int nByte;
+        iOff += getVarint32(&a[iOff], nByte);
+        term.n= nKeep;
+        fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]);
+        iOff += nByte;
+
+        sqlite3Fts5BufferAppendPrintf(
+            &rc, &s, " term=%.*s", term.n, (const char*)term.p
+        );
+        iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
+        if( iOff<n ){
+          iOff += getVarint32(&a[iOff], nKeep);
+        }
+      }
+      fts5BufferFree(&term);
+    }else{
+      Fts5NodeIter ss;
+      for(fts5NodeIterInit(a, n, &ss); ss.aData; fts5NodeIterNext(&rc, &ss)){
+        if( ss.term.n==0 ){
+          sqlite3Fts5BufferAppendPrintf(&rc, &s, " left=%d", ss.iChild);
+        }else{
+          sqlite3Fts5BufferAppendPrintf(&rc,&s, " \"%.*s\"", 
+              ss.term.n, ss.term.p
+          );
+        }
+        if( ss.nEmpty ){
+          sqlite3Fts5BufferAppendPrintf(&rc, &s, " empty=%d%s", ss.nEmpty,
+              ss.bDlidx ? "*" : ""
+          );
+        }
+      }
+      fts5NodeIterFree(&ss);
     }
   }
+  
+  if( rc==SQLITE_OK ){
+    sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_error_code(pCtx, rc);
+  }
+  fts5BufferFree(&s);
+}
 
+
+/*
+** This is called as part of registering the FTS5 module with database
+** connection db. It registers several user-defined scalar functions useful
+** with FTS5.
+**
+** If successful, SQLITE_OK is returned. If an error occurs, some other
+** SQLite error code is returned instead.
+*/
+int sqlite3Fts5IndexInit(sqlite3 *db){
+  int rc = sqlite3_create_function(
+      db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
+  );
   return rc;
 }
 
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index 0ce4e50e7a..67bcbe8f1a 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -226,21 +226,23 @@ int sqlite3Fts5StorageOpen(
 */
 int sqlite3Fts5StorageClose(Fts5Storage *p, int bDestroy){
   int rc = SQLITE_OK;
-  int i;
+  if( p ){
+    int i;
 
-  /* Finalize all SQL statements */
-  for(i=0; i<ArraySize(p->aStmt); i++){
-    sqlite3_finalize(p->aStmt[i]);
-  }
+    /* Finalize all SQL statements */
+    for(i=0; i<ArraySize(p->aStmt); i++){
+      sqlite3_finalize(p->aStmt[i]);
+    }
 
-  /* If required, remove the shadow tables from the database */
-  if( bDestroy ){
-    rc = sqlite3Fts5DropTable(p->pConfig, "content");
-    if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "docsize");
-    if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "config");
-  }
+    /* If required, remove the shadow tables from the database */
+    if( bDestroy ){
+      rc = sqlite3Fts5DropTable(p->pConfig, "content");
+      if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "docsize");
+      if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "config");
+    }
 
-  sqlite3_free(p);
+    sqlite3_free(p);
+  }
   return rc;
 }
 
@@ -265,8 +267,7 @@ static int fts5StorageInsertCallback(
   Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
   Fts5Index *pIdx = pCtx->pStorage->pIndex;
   pCtx->szCol = iPos+1;
-  sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken);
-  return SQLITE_OK;
+  return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken);
 }
 
 /*
@@ -288,8 +289,8 @@ static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
       Fts5InsertCtx ctx;
       ctx.pStorage = p;
       ctx.iCol = -1;
-      sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
-      for(iCol=1; iCol<=pConfig->nCol; iCol++){
+      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
+      for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
         rc = sqlite3Fts5Tokenize(pConfig, 
             (const char*)sqlite3_column_text(pSeek, iCol),
             sqlite3_column_bytes(pSeek, iCol),
@@ -475,8 +476,10 @@ int sqlite3Fts5StorageInsert(
   *piRowid = sqlite3_last_insert_rowid(pConfig->db);
 
   /* Add new entries to the FTS index */
-  sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
-  ctx.pStorage = p;
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
+    ctx.pStorage = p;
+  }
   for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
     ctx.szCol = 0;
     rc = sqlite3Fts5Tokenize(pConfig, 
diff --git a/manifest b/manifest
index 60491b268b..a2dd5734ff 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sa\sconfiguration\soption\sto\sremap\sthe\s"rank"\scolumn\sto\san\sauxiliary\sfts5\sfunction.
-D 2014-12-02T20:18:11.604
+C Begin\stesting\sfts5\sOOM\sand\sIO\serror\shandling.
+D 2014-12-03T17:27:35.105
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -104,16 +104,16 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
 F ext/fts5/extract_api_docs.tcl 6320db4a1d0722a4e2069e661381ad75e9889786
-F ext/fts5/fts5.c 572bd5d4d272ca562240dc1905538f060783ab78
+F ext/fts5/fts5.c 1dae34f4a788b5760c52b914d6384d83ee027b35
 F ext/fts5/fts5.h 72fc1e9995b1ddc254a487b9528614a83bd3dfb6
-F ext/fts5/fts5Int.h 9dbf415de032b1cc770dcedaa5a8e434d88ca90c
+F ext/fts5/fts5Int.h 36054b1dfc4881a9b94f945b348ab6cc01c0c7a5
 F ext/fts5/fts5_aux.c 0e3e5fea6bf5772805afe14c95cb5f16e03e4b3f
-F ext/fts5/fts5_buffer.c c79d67a5a611521f1f3b9d495981f22c02ef4bdb
-F ext/fts5/fts5_config.c 664fdc8519b55753f5c24d7b45176f05586b7965
-F ext/fts5/fts5_expr.c d317be07d70223a6865444f17982570260b690a5
+F ext/fts5/fts5_buffer.c 1bc5c762bb2e9b4a40b2e8a820a31b809e72eec1
+F ext/fts5/fts5_config.c 17986112dc76e7e39170e08df68f84180f66a9fe
+F ext/fts5/fts5_expr.c 5db50cd4ae9c3764d7daa8388bf406c0bad15039
 F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279
-F ext/fts5/fts5_index.c 7e7023f3a29f104b44df2ca2474b296b8dfe447c
-F ext/fts5/fts5_storage.c 0198c5976cefa5e8d3f1cfffa3587d0dd594fb2a
+F ext/fts5/fts5_index.c 9233b8b1f519e50d9ec139031032d9211dfcb541
+F ext/fts5/fts5_storage.c bfeedb83b095a1018f4f531c3cc3f9099e9f9081
 F ext/fts5/fts5_tcl.c 5272224faf9be129679da5e19d788f0307afc375
 F ext/fts5/fts5_tokenize.c 8360c0d1ae0d4696f3cc13f7c67a2db6011cdc5b
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
@@ -612,6 +612,7 @@ F test/fts5aj.test bc3d91bd012c7ca175cdf266c2074920bb5fa5ba
 F test/fts5ak.test e55bb0f3fac1291d32bc9485a3ee55a7d76f4d5f
 F test/fts5al.test 61b067f3b0b61679ab164a8a855882dfd313988d
 F test/fts5ea.test afaf3497b43add578384dc1fd26b0342738abe87
+F test/fts5fault1.test 27cb71251f8f2cd710ce4bdc1f0c29fa5db83be7
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
@@ -778,7 +779,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d
 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54
-F test/permutations.test 4f71bc5c9ce9a249cc94ad415cda809ce7f2360b
+F test/permutations.test a762abd3f97809c877c93e6b526ec07bb2a75b96
 F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0
 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13
 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
@@ -1207,7 +1208,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 9c1697a2aa1f601e6eb11704abe63a73c8105447
-R 30808e5592c3e61509564bec30e4914f
+P b5f5971283b9b2f60c16f9675099855af95012cd
+R cb30a6b5c5f7ea511cc1ede93a5d038a
 U dan
-Z 9589e0356694de369bd9f49ee042fc35
+Z f7fa77a51653f0fa8e3497900e76f571
diff --git a/manifest.uuid b/manifest.uuid
index 124c8a0053..f14250e59b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b5f5971283b9b2f60c16f9675099855af95012cd
\ No newline at end of file
+2037dba62fdd995ad15b642abe499a790f5ffe5c
\ No newline at end of file
diff --git a/test/fts5fault1.test b/test/fts5fault1.test
new file mode 100644
index 0000000000..723ae3d22f
--- /dev/null
+++ b/test/fts5fault1.test
@@ -0,0 +1,37 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set testprefix fts5fault1
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+faultsim_save_and_close
+do_faultsim_test 1 -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) }
+} -test {
+  faultsim_test_result {0 {}} 
+}
+
+
+
+finish_test
diff --git a/test/permutations.test b/test/permutations.test
index ca34266a27..4e366ca36f 100644
--- a/test/permutations.test
+++ b/test/permutations.test
@@ -225,8 +225,12 @@ test_suite "fts3" -prefix "" -description {
 test_suite "fts5" -prefix "" -description {
   All FTS5 tests.
 } -files {
-  fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ae.test fts5ea.test
+  fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ae.test 
   fts5af.test fts5ag.test fts5ah.test fts5ai.test fts5aj.test
+  fts5ak.test fts5al.test
+  fts5ea.test
+
+  fts5fault1.test
 }
 
 test_suite "nofaultsim" -prefix "" -description {