From: dan <dan@noemail.net>
Date: Thu, 22 Jan 2015 19:13:08 +0000 (+0000)
Subject: Add further tests and fixes for fts5.
X-Git-Tag: version-3.8.11~114^2~96
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=0b520cc5f925f3d990b0eb2f2479603ac9414d57;p=thirdparty%2Fsqlite.git

Add further tests and fixes for fts5.

FossilOrigin-Name: 5b295897153e9b26cd0d2e7ea112a4d461d0a665
---

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index afe1911180..7b7a86d8eb 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -345,6 +345,9 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p);
 
 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 
+int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
+#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
+
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c
index 97dd0dc0ec..61eed74be4 100644
--- a/ext/fts5/fts5_hash.c
+++ b/ext/fts5/fts5_hash.c
@@ -111,11 +111,14 @@ void sqlite3Fts5HashFree(Fts5Hash *pHash){
 void sqlite3Fts5HashClear(Fts5Hash *pHash){
   int i;
   for(i=0; i<pHash->nSlot; i++){
-    if( pHash->aSlot[i] ){
-      sqlite3_free(pHash->aSlot[i]);
-      pHash->aSlot[i] = 0;
+    Fts5HashEntry *pNext;
+    Fts5HashEntry *pSlot;
+    for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){
+      pNext = pSlot->pNext;
+      sqlite3_free(pSlot);
     }
   }
+  memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*));
   pHash->nEntry = 0;
 }
 
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 780e8e268b..d7cc3da1db 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -604,6 +604,72 @@ static u16 fts5GetU16(const u8 *aIn){
   return ((u16)aIn[0] << 8) + aIn[1];
 }
 
+/*
+** This is a copy of the sqlite3GetVarint32() routine from the SQLite core.
+** Except, this version does handle the single byte case that the core
+** version depends on being handled before its function is called.
+*/
+int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v){
+  u32 a,b;
+
+  /* The 1-byte case. Overwhelmingly the most common. */
+  a = *p;
+  /* a: p0 (unmasked) */
+  if (!(a&0x80))
+  {
+    /* Values between 0 and 127 */
+    *v = a;
+    return 1;
+  }
+
+  /* The 2-byte case */
+  p++;
+  b = *p;
+  /* b: p1 (unmasked) */
+  if (!(b&0x80))
+  {
+    /* Values between 128 and 16383 */
+    a &= 0x7f;
+    a = a<<7;
+    *v = a | b;
+    return 2;
+  }
+
+  /* The 3-byte case */
+  p++;
+  a = a<<14;
+  a |= *p;
+  /* a: p0<<14 | p2 (unmasked) */
+  if (!(a&0x80))
+  {
+    /* Values between 16384 and 2097151 */
+    a &= (0x7f<<14)|(0x7f);
+    b &= 0x7f;
+    b = b<<7;
+    *v = a | b;
+    return 3;
+  }
+
+  /* A 32-bit varint is used to store size information in btrees.
+  ** Objects are rarely larger than 2MiB limit of a 3-byte varint.
+  ** A 3-byte varint is sufficient, for example, to record the size
+  ** of a 1048569-byte BLOB or string.
+  **
+  ** We only unroll the first 1-, 2-, and 3- byte cases.  The very
+  ** rare larger cases can be handled by the slower 64-bit varint
+  ** routine.
+  */
+  {
+    u64 v64;
+    u8 n;
+    p -= 2;
+    n = sqlite3GetVarint(p, &v64);
+    *v = (u32)v64;
+    assert( n>3 && n<=9 );
+    return n;
+  }
+}
+
 /*
 ** Allocate and return a buffer at least nByte bytes in size.
 **
@@ -918,8 +984,8 @@ static int fts5StructureDecode(
 
   /* Read the total number of levels and segments from the start of the
   ** structure record.  */
-  i += getVarint32(&pData[i], nLevel);
-  i += getVarint32(&pData[i], nSegment);
+  i += fts5GetVarint32(&pData[i], nLevel);
+  i += fts5GetVarint32(&pData[i], nSegment);
   nByte = (
       sizeof(Fts5Structure) +                    /* Main structure */
       sizeof(Fts5StructureLevel) * (nLevel)      /* aLevel[] array */
@@ -935,8 +1001,8 @@ static int fts5StructureDecode(
       int nTotal;
       int iSeg;
 
-      i += getVarint32(&pData[i], pLvl->nMerge);
-      i += getVarint32(&pData[i], nTotal);
+      i += fts5GetVarint32(&pData[i], pLvl->nMerge);
+      i += fts5GetVarint32(&pData[i], nTotal);
       assert( nTotal>=pLvl->nMerge );
       pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, 
           nTotal * sizeof(Fts5StructureSegment)
@@ -945,10 +1011,10 @@ static int fts5StructureDecode(
       if( rc==SQLITE_OK ){
         pLvl->nSeg = nTotal;
         for(iSeg=0; iSeg<nTotal; iSeg++){
-          i += getVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid);
-          i += getVarint32(&pData[i], pLvl->aSeg[iSeg].nHeight);
-          i += getVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst);
-          i += getVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast);
+          i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid);
+          i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].nHeight);
+          i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst);
+          i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast);
         }
       }else{
         fts5StructureRelease(pRet);
@@ -1144,6 +1210,7 @@ static void fts5StructurePromoteTo(
 
   for(il=iPromote+1; il<pStruct->nLevel; il++){
     Fts5StructureLevel *pLvl = &pStruct->aLevel[il];
+    if( pLvl->nMerge ) return;
     for(is=pLvl->nSeg-1; is>=0; is--){
       int sz = fts5SegmentSize(&pLvl->aSeg[is]);
       if( sz>szPromote ) return;
@@ -1193,7 +1260,8 @@ static void fts5StructurePromote(
     /* Check for condition (a) */
     for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--);
     pTst = &pStruct->aLevel[iTst];
-    if( iTst>=0 && pTst->nMerge==0 ){
+    assert( pTst->nMerge==0 );
+    if( iTst>=0 ){
       int i;
       int szMax = 0;
       for(i=0; i<pTst->nSeg; i++){
@@ -1208,31 +1276,13 @@ static void fts5StructurePromote(
       }
     }
 
-    /* Check for condition (b) */
+    /* If condition (a) is not met, assume (b) is true. StructurePromoteTo()
+    ** is a no-op if it is not.  */
     if( iPromote<0 ){
-      Fts5StructureLevel *pTst;
-      for(iTst=iLvl+1; iTst<pStruct->nLevel; iTst++){
-        pTst = &pStruct->aLevel[iTst];
-        if( pTst->nSeg ) break;
-      }
-      if( iTst<pStruct->nLevel && pTst->nMerge==0 ){
-        Fts5StructureSegment *pSeg2 = &pTst->aSeg[pTst->nSeg-1];
-        int sz = pSeg2->pgnoLast - pSeg2->pgnoFirst + 1;
-        if( sz<=szSeg ){
-          iPromote = iLvl;
-          szPromote = szSeg;
-        }
-      }
-    }
-
-    /* If iPromote is greater than or equal to zero at this point, then it
-    ** is the level number of a level to which segments that consist of
-    ** szPromote or fewer pages should be promoted. */ 
-    if( iPromote>=0 ){
-      fts5PrintStructure("BEFORE", pStruct);
-      fts5StructurePromoteTo(p, iPromote, szPromote, pStruct);
-      fts5PrintStructure("AFTER", pStruct);
+      iPromote = iLvl;
+      szPromote = szSeg;
     }
+    fts5StructurePromoteTo(p, iPromote, szPromote, pStruct);
   }
 }
 
@@ -1246,7 +1296,7 @@ static void fts5NodeIterGobbleNEmpty(Fts5NodeIter *pIter){
   if( pIter->iOff<pIter->nData && 0==(pIter->aData[pIter->iOff] & 0xfe) ){
     pIter->bDlidx = pIter->aData[pIter->iOff] & 0x01;
     pIter->iOff++;
-    pIter->iOff += getVarint32(&pIter->aData[pIter->iOff], pIter->nEmpty);
+    pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], pIter->nEmpty);
   }else{
     pIter->nEmpty = 0;
     pIter->bDlidx = 0;
@@ -1262,8 +1312,8 @@ static void fts5NodeIterNext(int *pRc, Fts5NodeIter *pIter){
     pIter->iChild += pIter->nEmpty;
   }else{
     int nPre, nNew;
-    pIter->iOff += getVarint32(&pIter->aData[pIter->iOff], nPre);
-    pIter->iOff += getVarint32(&pIter->aData[pIter->iOff], nNew);
+    pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], nPre);
+    pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], nNew);
     pIter->term.n = nPre-2;
     fts5BufferAppendBlob(pRc, &pIter->term, nNew, pIter->aData+pIter->iOff);
     pIter->iOff += nNew;
@@ -1282,7 +1332,7 @@ static void fts5NodeIterInit(const u8 *aData, int nData, Fts5NodeIter *pIter){
   memset(pIter, 0, sizeof(*pIter));
   pIter->aData = aData;
   pIter->nData = nData;
-  pIter->iOff = getVarint32(aData, pIter->iChild);
+  pIter->iOff = fts5GetVarint32(aData, pIter->iChild);
   fts5NodeIterGobbleNEmpty(pIter);
 }
 
@@ -1466,7 +1516,7 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){
   int iOff = pIter->iLeafOffset;  /* Offset to read at */
   int nNew;                       /* Bytes of new data */
 
-  iOff += getVarint32(&a[iOff], nNew);
+  iOff += fts5GetVarint32(&a[iOff], nNew);
   pIter->term.n = nKeep;
   fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]);
   iOff += nNew;
@@ -1548,7 +1598,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){
     i64 iDelta = 0;
     int nPos;
 
-    i += getVarint32(&a[i], nPos);
+    i += fts5GetVarint32(&a[i], nPos);
     i += nPos;
     if( i>=n ) break;
     i += getVarint(&a[i], (u64*)&iDelta);
@@ -1665,7 +1715,7 @@ static void fts5SegIterNext(
         pIter->iRowidOffset--;
 
         pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset];
-        iOff += getVarint32(&a[iOff], nPos);
+        iOff += fts5GetVarint32(&a[iOff], nPos);
         iOff += nPos;
         getVarint(&a[iOff], (u64*)&iDelta);
         pIter->iRowid += iDelta;
@@ -1685,7 +1735,7 @@ static void fts5SegIterNext(
       iOff = pIter->iLeafOffset;
       if( iOff<n ){
         int nPoslist;
-        iOff += getVarint32(&a[iOff], nPoslist);
+        iOff += fts5GetVarint32(&a[iOff], nPoslist);
         iOff += nPoslist;
       }
 
@@ -1700,7 +1750,7 @@ static void fts5SegIterNext(
             fts5SegIterNextPage(p, pIter);
             pIter->iLeafOffset = 4;
           }else if( iOff!=fts5GetU16(&a[2]) ){
-            pIter->iLeafOffset += getVarint32(&a[iOff], nKeep);
+            pIter->iLeafOffset += fts5GetVarint32(&a[iOff], nKeep);
           }
         }else{
           pIter->iRowid -= iDelta;
@@ -1760,7 +1810,7 @@ static void fts5SegIterReverse(Fts5Index *p, int iIdx, Fts5SegIter *pIter){
       i64 iDelta;
 
       /* Position list size in bytes */
-      iOff += getVarint32(&pLeaf->p[iOff], nPos);
+      iOff += fts5GetVarint32(&pLeaf->p[iOff], nPos);
       iOff += nPos;
       if( iOff>=pLeaf->n ) break;
 
@@ -1843,7 +1893,7 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, int iIdx, Fts5SegIter *pIter){
       int nPoslist;
 
       /* iOff is currently the offset of the size field of a position list. */
-      iOff += getVarint32(&pLeaf->p[iOff], nPoslist);
+      iOff += fts5GetVarint32(&pLeaf->p[iOff], nPoslist);
       iOff += nPoslist;
 
       if( iOff<pLeaf->n ){
@@ -2353,7 +2403,7 @@ static void fts5ChunkIterInit(
     pLeaf = pIter->pLeaf;
   }
 
-  iOff += getVarint32(&pLeaf->p[iOff], pIter->nRem);
+  iOff += fts5GetVarint32(&pLeaf->p[iOff], pIter->nRem);
   pIter->n = MIN(pLeaf->n - iOff, pIter->nRem);
   pIter->p = pLeaf->p + iOff;
 
@@ -2383,7 +2433,7 @@ static int fts5PosIterReadVarint(Fts5Index *p, Fts5PosIter *pIter){
       if( fts5ChunkIterEof(p, &pIter->chunk) ) return 0;
       pIter->iOff = 0;
     }
-    pIter->iOff += getVarint32(&pIter->chunk.p[pIter->iOff], iVal);
+    pIter->iOff += fts5GetVarint32(&pIter->chunk.p[pIter->iOff], iVal);
   }
   return iVal;
 }
@@ -2546,20 +2596,25 @@ static void fts5WriteBtreeNEmpty(Fts5Index *p, Fts5SegWriter *pWriter){
 }
 
 static void fts5WriteBtreeGrow(Fts5Index *p, Fts5SegWriter *pWriter){
-  Fts5PageWriter *aNew;
-  Fts5PageWriter *pNew;
-  int nNew = sizeof(Fts5PageWriter) * (pWriter->nWriter+1);
+  if( p->rc==SQLITE_OK ){
+    Fts5PageWriter *aNew;
+    Fts5PageWriter *pNew;
+    int nNew = sizeof(Fts5PageWriter) * (pWriter->nWriter+1);
 
-  aNew = (Fts5PageWriter*)sqlite3_realloc(pWriter->aWriter, nNew);
-  if( aNew==0 ) return;
+    aNew = (Fts5PageWriter*)sqlite3_realloc(pWriter->aWriter, nNew);
+    if( aNew==0 ){
+      p->rc = SQLITE_NOMEM;
+      return;
+    }
 
-  pNew = &aNew[pWriter->nWriter];
-  memset(pNew, 0, sizeof(Fts5PageWriter));
-  pNew->pgno = 1;
-  fts5BufferAppendVarint(&p->rc, &pNew->buf, 1);
+    pNew = &aNew[pWriter->nWriter];
+    memset(pNew, 0, sizeof(Fts5PageWriter));
+    pNew->pgno = 1;
+    fts5BufferAppendVarint(&p->rc, &pNew->buf, 1);
 
-  pWriter->nWriter++;
-  pWriter->aWriter = aNew;
+    pWriter->nWriter++;
+    pWriter->aWriter = aNew;
+  }
 }
 
 /*
@@ -3150,13 +3205,15 @@ static void fts5IndexWork(
 #endif
 
       if( nBest<p->pConfig->nAutomerge 
-          && pStruct->aLevel[iBestLvl].nMerge==0 
-        ){
+       && pStruct->aLevel[iBestLvl].nMerge==0 
+      ){
         break;
       }
       fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem);
-      fts5StructurePromote(p, iBestLvl+1, pStruct);
       assert( nRem==0 || p->rc==SQLITE_OK );
+      if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
+        fts5StructurePromote(p, iBestLvl+1, pStruct);
+      }
       *ppStruct = pStruct;
     }
   }
@@ -3272,6 +3329,7 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
       pSeg->pgnoFirst = 1;
       pSeg->pgnoLast = pgnoLast;
     }
+    fts5StructurePromote(p, 0, pStruct);
   }
 
   if( p->pConfig->nAutomerge>0 ) fts5IndexWork(p, iHash, &pStruct, pgnoLast);
@@ -3543,7 +3601,7 @@ static void fts5IndexIntegrityCheckSegment(
     }else{
       int nTerm;                  /* Size of term on leaf in bytes */
       int res;                    /* Comparison of term and split-key */
-      iOff += getVarint32(&pLeaf->p[iOff], nTerm);
+      iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
       res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n));
       if( res==0 ) res = nTerm - iter.term.n;
       if( res<0 ){
@@ -3667,7 +3725,7 @@ static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
     }else{
       pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
     }
-    pIter->i += getVarint32(&pIter->a[pIter->i], pIter->nPoslist);
+    pIter->i += fts5GetVarint32(&pIter->a[pIter->i], pIter->nPoslist);
     pIter->aPoslist = &pIter->a[pIter->i];
     pIter->i += pIter->nPoslist;
   }else{
@@ -3829,7 +3887,7 @@ static void fts5SetupPrefixIter(
        && ((!bAsc && iRowid>=iLastRowid) || (bAsc && iRowid<=iLastRowid))
       ){
 
-        for(i=0; doclist.n && p->rc==SQLITE_OK; i++){
+        for(i=0; p->rc==SQLITE_OK && doclist.n; i++){
           assert( i<nBuf );
           if( aBuf[i].n==0 ){
             fts5BufferSwap(&doclist, &aBuf[i]);
@@ -4499,7 +4557,7 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
   int iOff = 0;
   while( iOff<n ){
     int iVal;
-    iOff += getVarint32(&a[iOff], iVal);
+    iOff += fts5GetVarint32(&a[iOff], iVal);
     sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal);
   }
   return iOff;
@@ -4523,7 +4581,7 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
   }
   while( iOff<n ){
     int nPos;
-    iOff += getVarint32(&a[iOff], nPos);
+    iOff += fts5GetVarint32(&a[iOff], nPos);
     iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
     if( iOff<n ){
       i64 iDelta;
@@ -4628,7 +4686,7 @@ static void fts5DecodeFunction(
       assert( iTermOff==0 || iOff==iTermOff );
       while( iOff<n ){
         int nByte;
-        iOff += getVarint32(&a[iOff], nByte);
+        iOff += fts5GetVarint32(&a[iOff], nByte);
         term.n= nKeep;
         fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]);
         iOff += nByte;
@@ -4638,7 +4696,7 @@ static void fts5DecodeFunction(
         );
         iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
         if( iOff<n ){
-          iOff += getVarint32(&a[iOff], nKeep);
+          iOff += fts5GetVarint32(&a[iOff], nKeep);
         }
       }
       fts5BufferFree(&term);
diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl
index c8c6a8a944..94e82d8684 100644
--- a/ext/fts5/test/fts5_common.tcl
+++ b/ext/fts5/test/fts5_common.tcl
@@ -114,4 +114,21 @@ proc fts5_aux_test_functions {db} {
   }
 }
 
+proc fts5_level_segs {tbl} {
+  set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
+  set ret [list]
+  foreach L [lrange [db one $sql] 1 end] {
+    lappend ret [expr [llength $L] - 2]
+  }
+  set ret
+} 
+
+proc fts5_rnddoc {n} {
+  set map [list 0 a  1 b  2 c  3 d  4 e  5 f  6 g  7 h  8 i  9 j]
+  set doc [list]
+  for {set i 0} {$i < $n} {incr i} {
+    lappend doc "x[string map $map [format %.3d [expr int(rand()*1000)]]]"
+  }
+  set doc
+}
 
diff --git a/ext/fts5/test/fts5ab.test b/ext/fts5/test/fts5ab.test
index 0d96ea47af..79ebc7eaa5 100644
--- a/ext/fts5/test/fts5ab.test
+++ b/ext/fts5/test/fts5ab.test
@@ -149,5 +149,90 @@ foreach {tn expr res} {
   } $res
 }
 
+#-------------------------------------------------------------------------
+# Documents with more than 2M tokens.
+#
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE s1 USING fts5(x);
+}
+foreach {tn doc} [list \
+  1 [string repeat {a x } 1500000]       \
+  2 "[string repeat {a a } 1500000] x"   \
+] {
+  do_execsql_test 4.$tn { INSERT INTO s1 VALUES($doc) }
+}
+
+do_execsql_test 4.3 {
+  SELECT rowid FROM s1 WHERE s1 MATCH 'x'
+} {2 1}
+
+do_execsql_test 4.4 {
+  SELECT rowid FROM s1 WHERE s1 MATCH '"a x"'
+} {2 1}
+
+#-------------------------------------------------------------------------
+# Check that a special case of segment promotion works. The case is where
+# a new segment is written to level L, but the oldest segment within level
+# (L-2) is larger than it.
+#
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE s2 USING fts5(x);
+  INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
+  INSERT INTO s2(s2, rank) VALUES('automerge', 0);
+}
+
+proc rnddoc {n} {
+  set map [list 0 a  1 b  2 c  3 d  4 e  5 f  6 g  7 h  8 i  9 j]
+  set doc [list]
+  for {set i 0} {$i < $n} {incr i} {
+    lappend doc [string map $map [format %.3d [expr int(rand()*1000)]]]
+  }
+  set doc
+}
+db func rnddoc rnddoc
+
+do_test 5.1 {
+  for {set i 1} {$i <= 65} {incr i} {
+    execsql { INSERT INTO s2 VALUES(rnddoc(10)) }
+  }
+  for {set i 1} {$i <= 63} {incr i} {
+    execsql { DELETE FROM s2 WHERE rowid = $i }
+  }
+  fts5_level_segs s2
+} {0 8}
+
+do_test 5.2 {
+  execsql {
+    INSERT INTO s2(s2, rank) VALUES('automerge', 8);
+  }
+  for {set i 0} {$i < 7} {incr i} {
+    execsql { INSERT INTO s2 VALUES(rnddoc(50)) }
+  }
+  fts5_level_segs s2
+} {8 0 0}
+
+# Test also the other type of segment promotion - when a new segment is written
+# that is larger than segments immediately following it.
+do_test 5.3 {
+  execsql {
+    DROP TABLE s2;
+    CREATE VIRTUAL TABLE s2 USING fts5(x);
+    INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
+    INSERT INTO s2(s2, rank) VALUES('automerge', 0);
+  }
+
+  for {set i 1} {$i <= 16} {incr i} {
+    execsql { INSERT INTO s2 VALUES(rnddoc(5)) }
+  }
+  fts5_level_segs s2
+} {0 1}
+
+do_test 5.4 {
+  execsql { INSERT INTO s2 VALUES(rnddoc(160)) }
+  fts5_level_segs s2
+} {2 0}
+
 
 finish_test
+
diff --git a/ext/fts5/test/fts5fault1.test b/ext/fts5/test/fts5fault1.test
index 4392f13c97..68aed258ec 100644
--- a/ext/fts5/test/fts5fault1.test
+++ b/ext/fts5/test/fts5fault1.test
@@ -210,8 +210,6 @@ do_faultsim_test 5.5.4 -faults oom* -body {
   faultsim_test_result [list 0 1]
 }
 
-}
-
 #-------------------------------------------------------------------------
 #
 reset_db
@@ -276,8 +274,8 @@ do_faultsim_test 6.4 -faults oom-* -prep {
   faultsim_test_result [list 0 {}]
 }
 
-
 #-------------------------------------------------------------------------
+#
 do_faultsim_test 7.0 -faults oom* -prep {
   catch { db close }
 } -body {
@@ -286,6 +284,74 @@ do_faultsim_test 7.0 -faults oom* -prep {
   faultsim_test_result [list 0 {}] [list 1 {}]
 }
 
+#-------------------------------------------------------------------------
+# A prefix query against a large document set.
+#
+proc rnddoc {n} {
+  set map [list 0 a  1 b  2 c  3 d  4 e  5 f  6 g  7 h  8 i  9 j]
+  set doc [list]
+  for {set i 0} {$i < $n} {incr i} {
+    lappend doc "x[string map $map [format %.3d [expr int(rand()*1000)]]]"
+  }
+  set doc
+}
+
+reset_db
+db func rnddoc rnddoc
+
+do_test 8.0 {
+  execsql { CREATE VIRTUAL TABLE x1 USING fts5(a) }
+  set ::res [list]
+  for {set i 100} {$i>0} {incr i -1} {
+    execsql { INSERT INTO x1 VALUES( rnddoc(50) ) }
+    lappend ::res $i
+  }
+} {}
+
+do_faultsim_test 8.1 -faults oom* -prep {
+} -body {
+  execsql { 
+    SELECT rowid FROM x1 WHERE x1 MATCH 'x*'
+  }
+} -test {
+  faultsim_test_result [list 0 $::res]
+}
+
+}
+
+#-------------------------------------------------------------------------
+# Segment promotion.
+#
+do_test 9.0 {
+  reset_db
+  db func rnddoc fts5_rnddoc
+  execsql {
+    CREATE VIRTUAL TABLE s2 USING fts5(x);
+    INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
+    INSERT INTO s2(s2, rank) VALUES('automerge', 0);
+  }
+
+  for {set i 1} {$i <= 16} {incr i} {
+    execsql { INSERT INTO s2 VALUES(rnddoc(5)) }
+  }
+  fts5_level_segs s2
+} {0 1}
+faultsim_save_and_close
+
+do_faultsim_test 9.1 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+  db func rnddoc fts5_rnddoc
+} -body {
+  execsql { INSERT INTO s2 VALUES(rnddoc(160)) }
+} -test {
+  faultsim_test_result {0 {}}
+  if {$testrc==0} {
+    set ls [fts5_level_segs s2]
+    if {$ls != "2 0"} { error "fts5_level_segs says {$ls}" }
+  }
+}
+
+
 
 finish_test
 
diff --git a/manifest b/manifest
index 7f398a24bf..ae6189124b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Further\stests\sand\sfixes\sfor\sfts5.
-D 2015-01-21T20:30:14.830
+C Add\sfurther\stests\sand\sfixes\sfor\sfts5.
+D 2015-01-22T19:13:08.439
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -106,22 +106,22 @@ F ext/fts3/unicode/mkunicode.tcl 4199cb887040ee3c3cd59a5171ddb0566904586e
 F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
 F ext/fts5/fts5.c 0ba5a8f27e1aa4deab82f0fc295d55f67dfe7f34
 F ext/fts5/fts5.h f931954065693898d26c51f23f1d27200184a69a
-F ext/fts5/fts5Int.h b593d5ff5f0cc6493778f88bc19db1dea42e003b
+F ext/fts5/fts5Int.h 99da8551098bb23fd94d0aa3f4ae1a411ee630b4
 F ext/fts5/fts5_aux.c 549aef152b0fd46020f5595d861b1fd60b3f9b4f
 F ext/fts5/fts5_buffer.c 32dd3c950392346ca69a0f1803501766c5c954f9
 F ext/fts5/fts5_config.c 33534ca25198cc62c54ff7d285d455c57ad19399
 F ext/fts5/fts5_expr.c 8a0e643768666dc2bffe74104141274809699808
-F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279
-F ext/fts5/fts5_index.c baf26bfee5bd776194c0e508e3c101964ae851c6
+F ext/fts5/fts5_hash.c 7a87f9f2eae2216c710064821fa0621ac6a8ce7b
+F ext/fts5/fts5_index.c ee7b141adde3dbdaa56f1e198c06a0786d298126
 F ext/fts5/fts5_storage.c d56722960982d0c48ba1b88d9001fefed8cff1a4
 F ext/fts5/fts5_tcl.c 1293fac2bb26903fd3d5cdee59c5885ba7e620d5
 F ext/fts5/fts5_tokenize.c 7c61d5c35c3449597bdeaa54dd48afe26852c7b0
 F ext/fts5/fts5_unicode2.c 9c7dd640d1f014bf5c3ee029759adfbb4d7e95a9
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl 08e939096a07eb77a7a986613e960f31d3cab2cc
+F ext/fts5/test/fts5_common.tcl 7db772d34fa0139d4b58d2b321928c9ccd30f699
 F ext/fts5/test/fts5aa.test 8ddbbcbedab67101dc9a86fd5c39d78b0e06515f
-F ext/fts5/test/fts5ab.test 91a3faac09ad9fab5f71494db6e4071963281536
+F ext/fts5/test/fts5ab.test 3f3ad2fb9ed60a0df57b626fa6fe6ef41d4deee0
 F ext/fts5/test/fts5ac.test 48181b7c873da0e3b4a3316760fcb90d88e7fbd8
 F ext/fts5/test/fts5ad.test 3b01eec8516d5631909716514e2e585a45ef0eb1
 F ext/fts5/test/fts5ae.test 014d5be2f5f70407fb032d4f27704116254797c3
@@ -136,7 +136,7 @@ F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
 F ext/fts5/test/fts5content.test 4234e0b11e003fe1e80472aa637f70464396fdd0
 F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
 F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
-F ext/fts5/test/fts5fault1.test 405886f4ecd255fa7a7077c021a65c9f8eaa4804
+F ext/fts5/test/fts5fault1.test f9bafb61b40061ad19b61d15003c5faeea4a57b5
 F ext/fts5/test/fts5near.test 3f9f64e16cac82725d03d4e04c661090f0b3b947
 F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54
 F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e
@@ -1282,7 +1282,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P e0d614425f7f5ffe266fdc03642931b1ec19ad25
-R c0a23f610b9ed738a5e0737db6938cae
+P c020a291ed293a66d21c5885e50a7fee04aa6366
+R 244beb886a9d1f5f10328b67a9ad3f5b
 U dan
-Z 2e4179b4673c6b65260289e1eab737a6
+Z 4d510e0e441ea9491ac2e8425faae5c2
diff --git a/manifest.uuid b/manifest.uuid
index 0492abfca8..75fced3560 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c020a291ed293a66d21c5885e50a7fee04aa6366
\ No newline at end of file
+5b295897153e9b26cd0d2e7ea112a4d461d0a665
\ No newline at end of file