From: dan <dan@noemail.net>
Date: Fri, 15 Jan 2016 19:54:47 +0000 (+0000)
Subject: Rationalize some code in fts5_storage.c. Add tests to cover recently added branches... 
X-Git-Tag: version-3.11.0~138
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=25247435be528220e2a0e40e966c9b73c8918b3a;p=thirdparty%2Fsqlite.git

Rationalize some code in fts5_storage.c. Add tests to cover recently added branches in fts5.

FossilOrigin-Name: 3b72df405ac9b3a71144f45317d32e25f0084c4b
---

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 1daeefc09f..ee953c399c 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -586,7 +586,7 @@ int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
 int sqlite3Fts5DropAll(Fts5Config*);
 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
 
-int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
 int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
 
@@ -606,8 +606,6 @@ int sqlite3Fts5StorageConfigValue(
     Fts5Storage *p, const char*, sqlite3_value*, int
 );
 
-int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
-
 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
 int sqlite3Fts5StorageOptimize(Fts5Storage *p);
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index fec589efcb..95fb10baeb 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1405,7 +1405,7 @@ static int fts5SpecialDelete(
   int eType1 = sqlite3_value_type(apVal[1]);
   if( eType1==SQLITE_INTEGER ){
     sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
-    rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]);
+    rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]);
   }
   return rc;
 }
@@ -1512,7 +1512,7 @@ static int fts5UpdateMethod(
     /* Case 1: DELETE */
     else if( nArg==1 ){
       i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
-      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
+      rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
     }
 
     /* Case 2: INSERT */
@@ -1522,7 +1522,7 @@ static int fts5UpdateMethod(
        && sqlite3_value_type(apVal[1])==SQLITE_INTEGER 
       ){
         i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
-        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
+        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
       }
       fts5StorageInsert(&rc, pTab, apVal, pRowid);
     }
@@ -1533,22 +1533,22 @@ static int fts5UpdateMethod(
       i64 iNew = sqlite3_value_int64(apVal[1]);  /* New rowid */
       if( iOld!=iNew ){
         if( eConflict==SQLITE_REPLACE ){
-          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
+          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
           if( rc==SQLITE_OK ){
-            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
+            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
           }
           fts5StorageInsert(&rc, pTab, apVal, pRowid);
         }else{
           rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid);
           if( rc==SQLITE_OK ){
-            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
+            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
           }
           if( rc==SQLITE_OK ){
             rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid);
           }
         }
       }else{
-        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
+        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
         fts5StorageInsert(&rc, pTab, apVal, pRowid);
       }
     }
@@ -1747,7 +1747,9 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){
       const u8 *a;
       int n; 
       rc = fts5CsrPoslist(pCsr, i, &a, &n);
-      sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
+      if( rc==SQLITE_OK ){
+        sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
+      }
     }
 
     if( rc==SQLITE_OK ){
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index 4a1ce3a3d9..56383619d1 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -378,39 +378,52 @@ static int fts5StorageInsertCallback(
 ** delete-markers to the FTS index necessary to delete it. Do not actually
 ** remove the %_content row at this time though.
 */
-static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
+static int fts5StorageDeleteFromIndex(
+  Fts5Storage *p, 
+  i64 iDel, 
+  sqlite3_value **apVal
+){
   Fts5Config *pConfig = p->pConfig;
-  sqlite3_stmt *pSeek;            /* SELECT to read row iDel from %_data */
+  sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
   int rc;                         /* Return code */
+  int rc2;                        /* sqlite3_reset() return code */
+  int iCol;
+  Fts5InsertCtx ctx;
 
-  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
-  if( rc==SQLITE_OK ){
-    int rc2;
+  if( apVal==0 ){
+    rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
+    if( rc!=SQLITE_OK ) return rc;
     sqlite3_bind_int64(pSeek, 1, iDel);
-    if( sqlite3_step(pSeek)==SQLITE_ROW ){
-      int iCol;
-      Fts5InsertCtx ctx;
-      ctx.pStorage = p;
-      ctx.iCol = -1;
-      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
-      for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
-        if( pConfig->abUnindexed[iCol-1] ) continue;
-        ctx.szCol = 0;
-        rc = sqlite3Fts5Tokenize(pConfig, 
-            FTS5_TOKENIZE_DOCUMENT,
-            (const char*)sqlite3_column_text(pSeek, iCol),
-            sqlite3_column_bytes(pSeek, iCol),
-            (void*)&ctx,
-            fts5StorageInsertCallback
-        );
-        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
+    if( sqlite3_step(pSeek)!=SQLITE_ROW ){
+      return sqlite3_reset(pSeek);
+    }
+  }
+
+  ctx.pStorage = p;
+  ctx.iCol = -1;
+  rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
+  for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
+    if( pConfig->abUnindexed[iCol-1]==0 ){
+      const char *zText;
+      int nText;
+      if( pSeek ){
+        zText = (const char*)sqlite3_column_text(pSeek, iCol);
+        nText = sqlite3_column_bytes(pSeek, iCol);
+      }else{
+        zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
+        nText = sqlite3_value_bytes(apVal[iCol-1]);
       }
-      p->nTotalRow--;
+      ctx.szCol = 0;
+      rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
+          zText, nText, (void*)&ctx, fts5StorageInsertCallback
+      );
+      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
     }
-    rc2 = sqlite3_reset(pSeek);
-    if( rc==SQLITE_OK ) rc = rc2;
   }
+  p->nTotalRow--;
 
+  rc2 = sqlite3_reset(pSeek);
+  if( rc==SQLITE_OK ) rc = rc2;
   return rc;
 }
 
@@ -490,16 +503,17 @@ static int fts5StorageSaveTotals(Fts5Storage *p){
 /*
 ** Remove a row from the FTS table.
 */
-int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
   Fts5Config *pConfig = p->pConfig;
   int rc;
   sqlite3_stmt *pDel = 0;
 
+  assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 );
   rc = fts5StorageLoadTotals(p, 1);
 
   /* Delete the index records */
   if( rc==SQLITE_OK ){
-    rc = fts5StorageDeleteFromIndex(p, iDel);
+    rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
   }
 
   /* Delete the %_docsize record */
@@ -532,61 +546,6 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
   return rc;
 }
 
-int sqlite3Fts5StorageSpecialDelete(
-  Fts5Storage *p, 
-  i64 iDel, 
-  sqlite3_value **apVal
-){
-  Fts5Config *pConfig = p->pConfig;
-  int rc;
-  sqlite3_stmt *pDel = 0;
-
-  assert( pConfig->eContent!=FTS5_CONTENT_NORMAL );
-  rc = fts5StorageLoadTotals(p, 1);
-
-  /* Delete the index records */
-  if( rc==SQLITE_OK ){
-    int iCol;
-    Fts5InsertCtx ctx;
-    ctx.pStorage = p;
-    ctx.iCol = -1;
-
-    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
-    for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
-      if( pConfig->abUnindexed[iCol] ) continue;
-      ctx.szCol = 0;
-      rc = sqlite3Fts5Tokenize(pConfig, 
-        FTS5_TOKENIZE_DOCUMENT,
-        (const char*)sqlite3_value_text(apVal[iCol]),
-        sqlite3_value_bytes(apVal[iCol]),
-        (void*)&ctx,
-        fts5StorageInsertCallback
-      );
-      p->aTotalSize[iCol] -= (i64)ctx.szCol;
-    }
-    p->nTotalRow--;
-  }
-
-  /* Delete the %_docsize record */
-  if( pConfig->bColumnsize ){
-    if( rc==SQLITE_OK ){
-      rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
-    }
-    if( rc==SQLITE_OK ){
-      sqlite3_bind_int64(pDel, 1, iDel);
-      sqlite3_step(pDel);
-      rc = sqlite3_reset(pDel);
-    }
-  }
-
-  /* Write the averages record */
-  if( rc==SQLITE_OK ){
-    rc = fts5StorageSaveTotals(p);
-  }
-
-  return rc;
-}
-
 /*
 ** Delete all entries in the FTS5 index.
 */
diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c
index 2ed10b7cdb..ddc5576c5d 100644
--- a/ext/fts5/fts5_vocab.c
+++ b/ext/fts5/fts5_vocab.c
@@ -427,6 +427,10 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
                   int ii = FTS5_POS2COLUMN(iPos);
                   pCsr->aCnt[ii]++;
                   if( iCol!=ii ){
+                    if( ii>=nCol ){
+                      rc = FTS5_CORRUPT;
+                      break;
+                    }
                     pCsr->aDoc[ii]++;
                     iCol = ii;
                   }
@@ -444,7 +448,11 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
               if( rc==SQLITE_OK ){
                 while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){
                   assert_nc( iPos>=0 && iPos<nCol );
-                  if( iPos<nCol ) pCsr->aDoc[iPos]++;
+                  if( iPos>=nCol ){
+                    rc = FTS5_CORRUPT;
+                    break;
+                  }
+                  pCsr->aDoc[iPos]++;
                 }
               }
               sqlite3Fts5BufferFree(&buf);
@@ -472,7 +480,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
     }
   }
 
-  if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
+  if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
     while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++;
     assert( pCsr->iCol<pCsr->pConfig->nCol );
   }
diff --git a/ext/fts5/test/fts5fault5.test b/ext/fts5/test/fts5fault5.test
index 41de5208b2..75b7d9af50 100644
--- a/ext/fts5/test/fts5fault5.test
+++ b/ext/fts5/test/fts5fault5.test
@@ -65,19 +65,26 @@ do_faultsim_test 2.2 -faults oom-t* -body {
 }
 
 #-------------------------------------------------------------------------
-# OOM while scanning an fts5vocab table.
+# OOM while scanning fts5vocab tables.
 #
 reset_db
 do_test 3.0 {
   execsql {
     CREATE VIRTUAL TABLE tt USING fts5(x);
     CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'row');
+
+    CREATE VIRTUAL TABLE tt2 USING fts5(x, detail=col);
+    CREATE VIRTUAL TABLE tv2 USING fts5vocab(tt2, 'col');
+
     INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
+    INSERT INTO tt2(tt2, rank) VALUES('pgsz', 32);
     BEGIN;
   }
+
   for {set i 0} {$i < 20} {incr i} {
     set str [string repeat "$i " 50]
     execsql { INSERT INTO tt VALUES($str) }
+    execsql { INSERT INTO tt2 VALUES($str) }
   }
   execsql COMMIT
 } {}
@@ -98,6 +105,28 @@ do_faultsim_test 3.2 -faults oom-t* -body {
   faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}}
 }
 
+breakpoint
+do_execsql_test 3.3.0 {
+  SELECT * FROM tv2;
+} {
+  0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}        
+  14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19  x 1 {}     
+  2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}   
+  9 x 1 {}
+}
+do_faultsim_test 3.3 -faults oom-t* -body {
+  db eval {
+    SELECT * FROM tv2;
+  }
+} -test {
+  faultsim_test_result [list 0 [list                                   \
+      0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}        \
+      14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19  x 1 {}     \
+      2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}   \
+      9 x 1 {}
+  ]]
+}
+
 
 
 finish_test
diff --git a/ext/fts5/test/fts5fault8.test b/ext/fts5/test/fts5fault8.test
index 01a1876955..d93066c6c0 100644
--- a/ext/fts5/test/fts5fault8.test
+++ b/ext/fts5/test/fts5fault8.test
@@ -24,8 +24,6 @@ ifcapable !fts5 {
 
 foreach_detail_mode $testprefix {
 
-if {[detail_is_none]==0} continue
-
 fts5_aux_test_functions db
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
@@ -34,7 +32,7 @@ do_execsql_test 1.0 {
   INSERT INTO t1 VALUES(NULL, '1 2 1 2');
 }
 
-do_faultsim_test 1 -faults oom-t* -body {
+do_faultsim_test 1 -faults oom-* -body {
   execsql { 
     SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' 
   }
@@ -43,6 +41,14 @@ do_faultsim_test 1 -faults oom-t* -body {
                        {1 SQLITE_NOMEM}
 }
 
+do_faultsim_test 2 -faults oom-* -body {
+  execsql { 
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+} -test {
+  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
+}
+
 }
 
 finish_test
diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test
index c61970e154..33b2a73ab9 100644
--- a/ext/fts5/test/fts5simple2.test
+++ b/ext/fts5/test/fts5simple2.test
@@ -267,8 +267,6 @@ do_execsql_test 14.1 {
   SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
 } {0.0.1}
 
-}
-
 #-------------------------------------------------------------------------
 #
 reset_db
@@ -299,6 +297,30 @@ do_execsql_test 15.3.2 {
 do_test 15.4 {
   execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
 } {}
+
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 16.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x, detail=none);
+  BEGIN;
+    INSERT INTO t2(rowid, x) VALUES(1, 'a b c');
+    INSERT INTO t2(rowid, x) VALUES(456, 'a b c');
+    INSERT INTO t2(rowid, x) VALUES(1000, 'a b c');
+  COMMIT;
+  UPDATE t2 SET x=x;
+}
+
+do_execsql_test 16.1 {
+  INSERT INTO t2(t2) VALUES('integrity-check');
+} {}
+
+do_execsql_test 16.2 {
+  SELECT rowid FROM t2('b') ORDER BY rowid DESC
+} {1000 456 1}
+
   
 finish_test
 
diff --git a/ext/fts5/test/fts5update.test b/ext/fts5/test/fts5update.test
new file mode 100644
index 0000000000..9e7debe9da
--- /dev/null
+++ b/ext/fts5/test/fts5update.test
@@ -0,0 +1,93 @@
+# 2016 Jan 16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5update
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set docs {
+  "eight zero iv eight 7"            "ix one 8 one three ii one"        
+  "1 9 9 three viii"                 "5 zero ii 6 nine ix 3"            
+  "3 zero 5 2 seven nine"            "two eight viii eight 1"           
+  "4 six two 5 9 vii"                "viii ii four 8 i i iv"            
+  "vii 0 iv seven 7 viii"            "five 1 nine vi seven"             
+  "1 zero zero iii 1"                "one one six 6 nine seven"         
+  "one v 4 zero 4 iii ii"            "2 3 eight six ix"                 
+  "six iv 7 three 5"                 "ix zero 0 8 ii 7 3"               
+  "four six nine 2 vii 3"            "five viii 5 8 0 7"                
+}
+
+foreach_detail_mode $::testprefix {
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
+} {}
+
+do_test 1.1 {
+  foreach {a b} $docs {
+    execsql {INSERT INTO t1 VALUES($a, $b)}
+  }
+} {}
+
+proc update {iRowid iA iB} {
+  set a [lindex $::docs $iA]
+  set b [lindex $::docs $iB]
+  execsql { UPDATE t1 SET a=$a, b=$b WHERE rowid=$iRowid }
+}
+
+set nDoc [llength $::docs]
+foreach n {1 5 10 50 100} {
+  do_test 1.2.$n {
+    execsql BEGIN
+    for {set i 1} {$i <= 1000} {incr i} {
+      set iRowid [expr {int(rand() * ($nDoc/2)) + 1}]
+      set iA [expr {int(rand() * $nDoc)}]
+      set iB [expr {int(rand() * $nDoc)}]
+      update $iRowid $iA $iB
+
+      if {($i % $n)==0} {
+        execsql { COMMIT; BEGIN }
+      }
+
+      if {($i % $n)==100} {
+        execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+      }
+    }
+    execsql COMMIT
+    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+  } {}
+}
+
+do_execsql_test 1.3 {
+  UPDATE t1 SET a=a AND b=b;
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+do_test 1.4 {
+  execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) }
+  for {set i 0} {$i < 50} {incr i} {
+    execsql { UPDATE t1 SET a=a AND b=b }
+    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+  }
+} {}
+
+}
+finish_test
+
+
diff --git a/ext/fts5/test/fts5vocab.test b/ext/fts5/test/fts5vocab.test
index 823b2b2872..f7278dd5c1 100644
--- a/ext/fts5/test/fts5vocab.test
+++ b/ext/fts5/test/fts5vocab.test
@@ -57,6 +57,8 @@ proc row_to_col {L} {
   set ret
 }
 
+if 1 {
+
 do_execsql_test 1.1.1 {
   CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%);
   CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row');
@@ -391,5 +393,57 @@ if {![detail_is_none]} {
 
 }
 
+#-------------------------------------------------------------------------
+# Test the fts5vocab tables response to a specific types of corruption:
+# where the fts5 index contains hits for columns that do not exist.
+#
+do_execsql_test 8.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b, c, detail=%DETAIL%);
+  INSERT INTO x1 VALUES('a b c', 'd e f', 'g h i');
+  INSERT INTO x1 VALUES('g h i', 'a b c', 'd e f');
+  INSERT INTO x1 VALUES('d e f', 'g h i', 'a b c');
+  CREATE VIRTUAL TABLE x1_r USING fts5vocab(x1, row);
+  CREATE VIRTUAL TABLE x1_c USING fts5vocab(x1, col);
+}
+
+set resr [star_from_row {a 3 3 b 3 3 c 3 3 d 3 3 e 3 3 f 3 3 g 3 3 h 3 3 i 3 3}]
+set resc [star_from_col {
+  a a 1 1 a b 1 1 a c 1 1 b a 1 1 
+  b b 1 1 b c 1 1 c a 1 1 c b 1 1 
+  c c 1 1 d a 1 1 d b 1 1 d c 1 1
+  e a 1 1 e b 1 1 e c 1 1 f a 1 1 
+  f b 1 1 f c 1 1 g a 1 1 g b 1 1 
+  g c 1 1 h a 1 1 h b 1 1 h c 1 1 
+  i a 1 1 i b 1 1 i c 1 1
+}]
+if {[detail_is_none]} { set resc [row_to_col $resr] }
+
+do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr
+do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc
+
+do_execsql_test 8.2 {
+  PRAGMA writable_schema = 1;
+  UPDATE sqlite_master 
+  SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(a, detail=%DETAIL%)'
+  WHERE name = 'x1';
+}
+db close
+sqlite3 db test.db
+sqlite3_fts5_may_be_corrupt 1
+
+do_execsql_test 8.2.1 { SELECT * FROM x1_r } $resr
+
+if {[detail_is_none]} {
+  do_execsql_test 8.2.2 { SELECT * FROM x1_c } $resc
+} else {
+  do_catchsql_test 8.2.2 { 
+    SELECT * FROM x1_c 
+  } {1 {database disk image is malformed}}
+}
+
+sqlite3_fts5_may_be_corrupt 0
+
+}
+
 finish_test
 
diff --git a/main.mk b/main.mk
index 7aaa58fffd..abf4fd2be7 100644
--- a/main.mk
+++ b/main.mk
@@ -334,7 +334,8 @@ TESTSRC += \
   $(TOP)/ext/misc/vfslog.c \
   $(TOP)/ext/fts5/fts5_tcl.c \
   $(TOP)/ext/fts5/fts5_test_mi.c \
-  $(TOP)/ext/fts5/fts5_test_tok.c
+  $(TOP)/ext/fts5/fts5_test_tok.c \
+  $(FTS5_SRC)
 
 
 #TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
diff --git a/manifest b/manifest
index 4bde0b8c82..10927de756 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sunnecessary\sde-initialization\sof\sthe\sParse\sobject.
-D 2016-01-15T16:11:58.808
+C Rationalize\ssome\scode\sin\sfts5_storage.c.\sAdd\stests\sto\scover\srecently\sadded\sbranches\sin\sfts5.
+D 2016-01-15T19:54:47.454
 F Makefile.in a476545d0c8626224d0bacac85c6e2967474af81
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 01e855f958932d0d3ed62ec675fc63e2cef61fcb
@@ -97,22 +97,22 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
-F ext/fts5/fts5Int.h 6c5a332e6add01dd69166a252d1818fb75c42a08
+F ext/fts5/fts5Int.h 313e3276ac9e0245ee722ef803253857a68722b9
 F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e
 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1
 F ext/fts5/fts5_config.c 9c243d04ac0ca997d2d2e2252891f2a10fbd7217
 F ext/fts5/fts5_expr.c 510db45967ca359f64f2ba2c707ab57d740cad56
 F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
 F ext/fts5/fts5_index.c cd036089b22b0340fccef88a3ad62ac1016c7cbb
-F ext/fts5/fts5_main.c 488ceecdb4400ecc6a3d3b2247cedef153955388
-F ext/fts5/fts5_storage.c f7b2d330dd7b29a9f4da09f6d85879ca8c41b2e8
+F ext/fts5/fts5_main.c 1d116f5c44c6f06ec282d33dd6d041c8131b3d6a
+F ext/fts5/fts5_storage.c fb2eaec3aa954b680d43096dc539f8270bd6390e
 F ext/fts5/fts5_tcl.c bcacc05dec0446e7b1a44d5d906057e677bd7ea4
 F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070
 F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be
 F ext/fts5/fts5_tokenize.c 504984ac6993323247221eebe3cd55bead01b5f8
 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c
 F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
-F ext/fts5/fts5_vocab.c da64ecbd217625980a1721fbd588a1e4118a51b6
+F ext/fts5/fts5_vocab.c ee6df1a3be103414d7b7af833ae1885c7b83a9d0
 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl 393882afb225a21edf033043bbf936951e9198c1
@@ -149,10 +149,10 @@ F ext/fts5/test/fts5fault1.test 4b39c47ca3544615daa8a2f733b911fa08022c77
 F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341
 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
 F ext/fts5/test/fts5fault4.test 4864f2b5c2c083440dbe85aff60897bc1aa04603
-F ext/fts5/test/fts5fault5.test f2b8645053d48982e8979749e93994c43011c118
+F ext/fts5/test/fts5fault5.test 10c13a783de3f42a21e3e53e123b62ed0c3a1618
 F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda
 F ext/fts5/test/fts5fault7.test 01be274bfc8d9bf22451a3bf5892e9399d044f1b
-F ext/fts5/test/fts5fault8.test aeb4717b7b293678bc4d2f3c0159206a525375d9
+F ext/fts5/test/fts5fault8.test f2d8a2b673a5f72ca1fa0e85bdbfb2041ffd347d
 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
 F ext/fts5/test/fts5hash.test 7cf4607b8657c383f0b520668a99971e95d8b139
 F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65
@@ -172,7 +172,7 @@ F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821
 F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9
-F ext/fts5/test/fts5simple2.test 843f1f7fe439ff32bf74f4fd6430632f9636ef3a
+F ext/fts5/test/fts5simple2.test 7b51f8d411e9a77fa4519fb09ba5a3afda75c94d
 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
 F ext/fts5/test/fts5synonym2.test d2d9099d9d105b55ea03fd52d61ae2847d534129
 F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7
@@ -182,8 +182,9 @@ F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
 F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
+F ext/fts5/test/fts5update.test 6f0abb89556f0a638b982e010f4d8d1de373a61c
 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e
-F ext/fts5/test/fts5vocab.test e4b12f238f113795615ba6343b63fb326d6a360e
+F ext/fts5/test/fts5vocab.test 480d780aa6b699816c5066225fbd86f3a0239477
 F ext/fts5/tool/fts5speed.tcl aaee41894b552df8fbf8616aad003b2ea9ba3221
 F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84
 F ext/fts5/tool/loadfts5.tcl 4cc2d6af43b58d4fac05bc4fdabd0e5862c3b2c1
@@ -265,7 +266,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 55f0940264e55540773214ee1f2dbba0bf359957
+F main.mk 7575015811ef75a2cfcbc6f09de161cc8236e8a5
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -1415,7 +1416,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P c52ca2c0662bb30ab34574f933429512655b19ff
-R 18101cd5e40446f0b579af30aa8257e8
-U drh
-Z 9e527ccefb4d19cf49503262bc52d994
+P 75ab30c5fcb51e556342dc3112f9acf122f79947
+R 374e508382342a2641c42b70ee61e09e
+U dan
+Z e6d3f908cc39d2673b189171ce4240a7
diff --git a/manifest.uuid b/manifest.uuid
index f3f393f928..7756c1a36c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-75ab30c5fcb51e556342dc3112f9acf122f79947
\ No newline at end of file
+3b72df405ac9b3a71144f45317d32e25f0084c4b
\ No newline at end of file