From: dan <dan@noemail.net>
Date: Fri, 24 Apr 2015 15:56:09 +0000 (+0000)
Subject: Add extra tests for corrupt database handling in fts5.
X-Git-Tag: version-3.8.11~114^2~62
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=def90aae18f406fddb37151fb9aa8cb81eca0aac;p=thirdparty%2Fsqlite.git

Add extra tests for corrupt database handling in fts5.

FossilOrigin-Name: 41449f7a0b5da6332eef48386c91ef63382c4783
---

diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 59d3271212..0bbea3aa7d 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -39,6 +39,18 @@ int sqlite3Fts5Corrupt(void);
 # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
 #endif
 
+/*
+** The assert_nc() macro is similar to the assert() macro, except that it
+** is used for assert() conditions that are true only if it can be 
+** guranteed that the database is not corrupt.
+*/
+#ifdef SQLITE_TEST
+extern int sqlite3_fts5_may_be_corrupt;
+# define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
+#else
+# define assert_nc(x) assert(x)
+#endif
+
 /**************************************************************************
 ** Interface to code in fts5.c. 
 */
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 3708ba58ca..abddf5b30d 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -752,6 +752,46 @@ static void fts5CloseReader(Fts5Index *p){
   }
 }
 
+/*
+** Check if row iRowid exists in the %_data table, and that it contains
+** a blob value. If so, return SQLITE_ERROR (yes - SQLITE_ERROR, not 
+** SQLITE_OK). If not, return SQLITE_CORRUPT_VTAB.
+**
+** If an error occurs (e.g. OOM or IOERR), return the relevant error code.
+**
+** This function does not need to be efficient. It is part of vary rarely
+** invoked error handling code only.
+*/
+#if 0
+static int fts5CheckMissingRowid(Fts5Index *p, i64 iRowid){
+  const char *zFmt = "SELECT typeof(block)=='blob' FROM '%q'.%Q WHERE id=%lld";
+  int bOk = 0;
+  int rc;
+  char *zSql;
+
+  zSql = sqlite3_mprintf(zFmt, p->pConfig->zDb, p->zDataTbl, iRowid);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    sqlite3_stmt *pStmt;
+    rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, &pStmt, 0);
+    if( rc==SQLITE_OK ){
+      if( SQLITE_ROW==sqlite3_step(pStmt) ){
+        bOk = sqlite3_column_int(pStmt, 0);
+      }
+      rc = sqlite3_finalize(pStmt);
+    }
+    sqlite3_free(zSql);
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = bOk ? SQLITE_ERROR : FTS5_CORRUPT;
+  }
+
+  return rc;
+}
+#endif
+
 static Fts5Data *fts5DataReadOrBuffer(
   Fts5Index *p, 
   Fts5Buffer *pBuf, 
@@ -761,13 +801,6 @@ static Fts5Data *fts5DataReadOrBuffer(
   if( p->rc==SQLITE_OK ){
     int rc = SQLITE_OK;
 
-#if 0
-Fts5Buffer buf = {0,0,0};
-fts5DebugRowid(&rc, &buf, iRowid);
-fprintf(stdout, "read: %s\n", buf.p);
-fflush(stdout);
-sqlite3_free(buf.p);
-#endif
     if( p->pReader ){
       /* This call may return SQLITE_ABORT if there has been a savepoint
       ** rollback since it was last used. In this case a new blob handle
@@ -788,6 +821,13 @@ sqlite3_free(buf.p);
       );
     }
 
+    /* If either of the sqlite3_blob_open() or sqlite3_blob_reopen() calls
+    ** above returned SQLITE_ERROR, return SQLITE_CORRUPT_VTAB instead.
+    ** All the reasons those functions might return SQLITE_ERROR - missing
+    ** table, missing row, non-blob/text in block column - indicate 
+    ** backing store corruption.  */
+    if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT;
+
     if( rc==SQLITE_OK ){
       u8 *aOut;                   /* Read blob data into this buffer */
       int nByte = sqlite3_blob_bytes(p->pReader);
@@ -1563,7 +1603,7 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
   if( p->rc==SQLITE_OK ){
     const u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset];
     int iOff = pIter->iLeafOffset;  /* Offset to read at */
-    pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos,&pIter->bDel);
+    pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos, &pIter->bDel);
   }
 }
 
@@ -1577,8 +1617,6 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
 **
 **   Fts5SegIter.term
 **   Fts5SegIter.rowid
-**   Fts5SegIter.nPos
-**   Fts5SegIter.bDel
 **
 ** accordingly and leaves (Fts5SegIter.iLeafOffset) set to the content of
 ** the first position list. The position list belonging to document 
@@ -3912,7 +3950,6 @@ static void fts5BtreeIterNext(Fts5BtreeIter *pIter){
   pIter->nEmpty = pIter->aLvl[0].s.nEmpty;
   pIter->bDlidx = pIter->aLvl[0].s.bDlidx;
   pIter->iLeaf = pIter->aLvl[0].s.iChild;
-  assert( p->rc==SQLITE_OK || pIter->bEof );
 }
 
 static void fts5BtreeIterFree(Fts5BtreeIter *pIter){
@@ -3985,7 +4022,7 @@ static void fts5IndexIntegrityCheckSegment(
 
   /* Iterate through the b-tree hierarchy.  */
   for(fts5BtreeIterInit(p, iIdx, pSeg, &iter);
-      iter.bEof==0;
+      p->rc==SQLITE_OK && iter.bEof==0;
       fts5BtreeIterNext(&iter)
   ){
     i64 iRow;                     /* Rowid for this leaf */
diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c
index f1c2284276..5bbfc821a2 100644
--- a/ext/fts5/fts5_tcl.c
+++ b/ext/fts5/fts5_tcl.c
@@ -22,6 +22,14 @@
 #include <string.h>
 #include <assert.h>
 
+/*
+** This variable is set to true when running corruption tests. Otherwise
+** false. If it is false, extra assert() conditions in the fts5 code are
+** activated - conditions that are only true if it is guaranteed that the
+** fts5 database is not corrupt.
+*/
+int sqlite3_fts5_may_be_corrupt = 0;
+
 /*************************************************************************
 ** This is a copy of the first part of the SqliteDb structure in 
 ** tclsqlite.c.  We need it here so that the get_sqlite_pointer routine
@@ -830,6 +838,33 @@ static void xF5tFree(ClientData clientData){
   ckfree(clientData);
 }
 
+/*
+**      sqlite3_fts5_may_be_corrupt BOOLEAN
+**
+** Set or clear the global "may-be-corrupt" flag. Return the old value.
+*/
+static int f5tMayBeCorrupt(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int bOld = sqlite3_fts5_may_be_corrupt;
+
+  if( objc!=2 && objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?");
+    return TCL_ERROR;
+  }
+  if( objc==2 ){
+    int bNew;
+    if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR;
+    sqlite3_fts5_may_be_corrupt = bNew;
+  }
+
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld));
+  return TCL_OK;
+}
+
 /*
 ** Entry point.
 */
@@ -842,7 +877,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){
     { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 },
     { "sqlite3_fts5_token",            f5tTokenizerReturn, 1 },
     { "sqlite3_fts5_tokenize",         f5tTokenize, 0 },
-    { "sqlite3_fts5_create_function",  f5tCreateFunction, 0 }
+    { "sqlite3_fts5_create_function",  f5tCreateFunction, 0 },
+    { "sqlite3_fts5_may_be_corrupt",   f5tMayBeCorrupt, 0 }
   };
   int i;
   F5tTokenizerContext *pContext;
diff --git a/ext/fts5/test/fts5corrupt.test b/ext/fts5/test/fts5corrupt.test
index a9393de43d..0791ab0cf9 100644
--- a/ext/fts5/test/fts5corrupt.test
+++ b/ext/fts5/test/fts5corrupt.test
@@ -9,6 +9,8 @@
 #
 #***********************************************************************
 #
+# This file tests that the FTS5 'integrity-check' command detects 
+# inconsistencies (corruption) in the on-disk backing tables.
 #
 
 source [file join [file dirname [info script]] fts5_common.tcl]
@@ -38,7 +40,7 @@ do_test 1.3 {
     DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', 0, $segid, 0, 4);
   }
   catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
-} {1 {SQL logic error or missing database}}
+} {1 {database disk image is malformed}}
 
 do_test 1.4 {
   db_restore_and_reopen
diff --git a/ext/fts5/test/fts5corrupt2.test b/ext/fts5/test/fts5corrupt2.test
new file mode 100644
index 0000000000..a5f657b160
--- /dev/null
+++ b/ext/fts5/test/fts5corrupt2.test
@@ -0,0 +1,116 @@
+# 2015 Apr 24
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file tests that FTS5 handles corrupt databases (i.e. internal
+# inconsistencies in the backing tables) correctly. In this case 
+# "correctly" means without crashing.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5corrupt2
+
+# Create a simple FTS5 table containing 100 documents. Each document 
+# contains 10 terms, each of which start with the character "x".
+#
+expr srand(0)
+db func rnddoc fts5_rnddoc
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
+  INSERT INTO t1 SELECT rnddoc(10) FROM ii;
+}
+
+set mask [expr 31 << 31]
+
+# Test 1:
+#
+#   For each page in the t1_data table, open a transaction and DELETE
+#   the t1_data entry. Then run:
+#
+#     * an integrity-check, and
+#     * unless the deleted block was a b-tree node, a query for "t1 MATCH 'x*'"
+#
+#   and check that the corruption is detected in both cases. The 
+#   rollback the transaction.
+#
+# Test 2:
+#
+#   Same thing, except instead of deleting a row from t1_data, replace its
+#   blob content with integer value 14.
+#
+foreach {tno stmt} {
+  1 { DELETE FROM t1_data WHERE rowid=$rowid }
+  2 { UPDATE t1_data SET block=14 WHERE rowid=$rowid }
+} {
+  break
+  set tn 0
+  foreach rowid [db eval {SELECT rowid FROM t1_data WHERE rowid>10}] {
+    incr tn
+    #if {$tn!=224} continue
+  
+    do_test 1.$tno.$tn.1.$rowid {
+      execsql { BEGIN }
+      execsql $stmt
+      catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+    } {1 {database disk image is malformed}}
+  
+    if {($rowid & $mask)==0} {
+      # Node is a leaf node, not a b-tree node.
+      do_catchsql_test 1.$tno.$tn.2.$rowid {
+        SELECT rowid FROM t1 WHERE t1 MATCH 'x*'
+      } {1 {database disk image is malformed}}
+    }
+  
+    do_execsql_test 1.$tno.$tn.3.$rowid {
+      ROLLBACK;
+      INSERT INTO t1(t1) VALUES('integrity-check');
+    } {}
+  }
+}
+
+# Run N-1 tests, where N is the number of bytes in the rightmost leaf page
+# of the fts index. For test $i, truncate the rightmost leafpage to $i
+# bytes. Then test both the integrity-check detects the corruption.
+#
+# Also tested is that "MATCH 'x*'" does not crash and sometimes reports
+# corruption. It may not report the db as corrupt because truncating the
+# final leaf to some sizes may create a valid leaf page.
+#
+set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}] 
+set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
+set all [db eval {SELECT rowid FROM t1}]
+for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
+  do_execsql_test 2.$i.1 {
+    BEGIN;
+      UPDATE t1_data SET block = substr(block, 1, $i) WHERE rowid=$lrowid;
+  }
+
+  do_catchsql_test 2.$i.2 {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  } {1 {database disk image is malformed}}
+
+  do_test 2.$i.3 {
+    set res [catchsql {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'}]
+    expr {
+        $res=="1 {database disk image is malformed}" 
+     || $res=="0 {$all}" 
+    }
+  } 1
+
+  do_execsql_test 2.$i.4 {
+    ROLLBACK;
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  } {}
+}
+
+finish_test
+
diff --git a/ext/fts5/test/fts5rebuild.test b/ext/fts5/test/fts5rebuild.test
index dfaf28bc6e..644a674942 100644
--- a/ext/fts5/test/fts5rebuild.test
+++ b/ext/fts5/test/fts5rebuild.test
@@ -39,7 +39,7 @@ do_execsql_test 1.5 {
 
 do_catchsql_test 1.6 {
   INSERT INTO f1(f1) VALUES('integrity-check');
-} {1 {SQL logic error or missing database}}
+} {1 {database disk image is malformed}}
 
 do_execsql_test 1.7 {
   INSERT INTO f1(f1) VALUES('rebuild');
diff --git a/manifest b/manifest
index 994c259715..9d446ac187 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sfts5\sbuild\sproblem\sin\smain.mk.
-D 2015-04-24T06:02:29.587
+C Add\sextra\stests\sfor\scorrupt\sdatabase\shandling\sin\sfts5.
+D 2015-04-24T15:56:09.379
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in faaf75b89840659d74501bea269c7e33414761c1
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -106,15 +106,15 @@ F ext/fts3/unicode/mkunicode.tcl 159c1194da0bc72f51b3c2eb71022568006dc5ad
 F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
 F ext/fts5/fts5.c 1eb8ca073be5222c43e4eee5408764c2cbb4200b
 F ext/fts5/fts5.h 24a2cc35b5e76eec57b37ba48c12d9d2cb522b3a
-F ext/fts5/fts5Int.h 1b537736f8838df7fca10245c0f70a23cfddc7f5
+F ext/fts5/fts5Int.h 1309320cb233e1c5b38d7f1e2cab2138bbf497d8
 F ext/fts5/fts5_aux.c fcea18b1a2a3f95a498b52aba2983557d7678a22
 F ext/fts5/fts5_buffer.c 3ba56cc6824c9f7b1e0695159e0a9c636f6b4a23
 F ext/fts5/fts5_config.c 0847facc8914f57ea4452c43ce109200dc65e894
 F ext/fts5/fts5_expr.c 05da381ab26031243266069302c6eb4094b2c5dd
 F ext/fts5/fts5_hash.c 3cb5a3d04dd2030eb0ac8d544711dfd37c0e6529
-F ext/fts5/fts5_index.c 7c9615a83e0ca928817e81be65e266f639f45532
+F ext/fts5/fts5_index.c 1663ad6a9ae221f14f27442b9b1a9d5088a2c5fe
 F ext/fts5/fts5_storage.c ac0f0937059c8d4f38a1f13aa5f2c2cd7edf3e0d
-F ext/fts5/fts5_tcl.c 617b6bb96545be8d9045de6967c688cd9cd15541
+F ext/fts5/fts5_tcl.c 10bf0eb678d34c1bfdcfaf653d2e6dd92afa8b38
 F ext/fts5/fts5_tokenize.c c07f2c2f749282c1dbbf46bde1f6d7095c740b8b
 F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
@@ -135,7 +135,8 @@ F ext/fts5/test/fts5al.test 6a5717faaf7f1e0e866360022d284903f3a4eede
 F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
 F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b
 F ext/fts5/test/fts5content.test 8dc302fccdff834d946497e9d862750ea87d4517
-F ext/fts5/test/fts5corrupt.test dbdcfe75749ed2f2eb3915cf68fd55d3dc3b058d
+F ext/fts5/test/fts5corrupt.test 9e8524281aa322c522c1d6e2b347e24e060c2727
+F ext/fts5/test/fts5corrupt2.test 3be48d8a30d30e3ae819f04e957c45d091bfbb85
 F ext/fts5/test/fts5dlidx.test 748a84ceb74a4154725096a26dfa854260b0182f
 F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
 F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
@@ -145,7 +146,7 @@ F ext/fts5/test/fts5near.test 3f9f64e16cac82725d03d4e04c661090f0b3b947
 F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54
 F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e
 F ext/fts5/test/fts5prefix.test 4610dfba4460d92f23a8014874a46493f1be77b5
-F ext/fts5/test/fts5rebuild.test 2a5e98205393487b4a732c8290999af7c0b907b4
+F ext/fts5/test/fts5rebuild.test ee6792715c6c528cc188e7869d67c3c655889ddb
 F ext/fts5/test/fts5rowid.test a1b2a6d76648c734c1aab11ee1a619067e8d90e6
 F ext/fts5/test/fts5tokenizer.test b34ae592db66f6e89546d791ce1f905ba0b3395c
 F ext/fts5/test/fts5unicode.test 79b3e34eb29ce4929628aa514a40cb467fdabe4d
@@ -1301,7 +1302,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 2dd59b5762c944b3bdd270e95c6739cd4f530bfa
-R 16931392bbfaadd753438e7b377b1156
+P 60045cedef109f03317dc878fe6bb3d03867ae69
+R 9c5c238cdd6f30dc8d0223c36173d961
 U dan
-Z b1c66eea22fb17fe989c58ccfd8fe427
+Z ec7cf237df9e7bd8116f5a496704530c
diff --git a/manifest.uuid b/manifest.uuid
index e4bd9e57cc..62f07c3585 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-60045cedef109f03317dc878fe6bb3d03867ae69
\ No newline at end of file
+41449f7a0b5da6332eef48386c91ef63382c4783
\ No newline at end of file