Better detection and handling of corrupt database files. (CVS 1922)

FossilOrigin-Name: 8f5b199e845fa7ae3444ef69bd840716d305cf73

diff --git a/manifest b/manifest
index 17ce26cef1bbfad76709fe8e84c702dcbb6e8c29..5465488ce3c49d128526c95d3cc14190cc34bf91 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Documentation\supdates\s(CVS\s1921)
-D 2004-08-30T14:58:12
+C Better\sdetection\sand\shandling\sof\scorrupt\sdatabase\sfiles.\s(CVS\s1922)
+D 2004-08-30T16:52:18
 F Makefile.in 65a7c43fcaf9a710d62f120b11b6e435eeb4a450
 F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457
 F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd
@@ -28,7 +28,7 @@ F sqlite3.def cf325d366f167029a971de7333f32b74bfe2e375
 F sqlite3.pc.in 985b9bf34192a549d7d370e0f0b6b34a4f61369a
 F src/attach.c 0bd4f11da6999665da30625665a4096ba7898de6
 F src/auth.c 60db23b98bb94c8b0178180faaf49dc116674217
-F src/btree.c 14c20dfb320473a1fd4e37d43eba5e2afd606757
+F src/btree.c 9cf41d26fb175b5d46051ed29c28a6e179094dda
 F src/btree.h 94dfec0a1722d33359b23e7e310f2b64ffedf029
 F src/build.c 7d93b19d03bffd171f7c7873739688cab7e300c5
 F src/date.c edff4aa851eeca8abbc737dc3933a2f0671156ce
@@ -52,7 +52,7 @@ F src/os_unix.c a5625eed7ab071d1715df783f4684945ae538a22
 F src/os_unix.h f3097815e041e82e24d92505e1ff61ba24172d13
 F src/os_win.c 9e2887825b1a32f0ceb1b73b93ffe29a112cd86f
 F src/os_win.h babd4e912967c6b09088cfe38a45e8005a07ba44
-F src/pager.c 6ecf24602f56ac98914685d449f6653903f36fec
+F src/pager.c 616563dc510993ab67baa54fc6a2ddacc05175ca
 F src/pager.h 67739fe649f33be55dba522ca8a9cc4e42d14f71
 F src/parse.y 581a2ce014b843506805b2470c02b7865ad034d5
 F src/pragma.c a7cea75286fcff6666a5412b04478fcf0ecef5c4
@@ -80,7 +80,7 @@ F src/vdbe.h e081c72cd0f7c19d49b1927460aeefcf0fbc85ac
 F src/vdbeInt.h aadadddc8cfad6aa5a5445c849f70d881276fe34
 F src/vdbeapi.c 854732720c2cfc6ff76b28eef6253ac84a5408bc
 F src/vdbeaux.c 022c484dba235d2dcbb1faca0f1943702f4232ed
-F src/vdbemem.c 8971ecc9e56f8b7dbde865906753dbd7812a4f8f
+F src/vdbemem.c ef9ac7d32acfe4bce5c5b408b1294c8d9e0cdb56
 F src/where.c a84eee276cd072158224da6b5f30733df2d56027
 F test/all.test 3b692eb43583b52c99c344b2fa8934512d179016
 F test/attach.test feb2ce54e78688df4c84553416d5aec3b2a0112e
@@ -106,6 +106,7 @@ F test/collate4.test 4a7902b7560686af11d6cace717d876c6937b7ef
 F test/collate5.test 1dd5f0f508c46667f9d4606c7950c414b0bdc0d5
 F test/collate6.test 2a45768914f04c1447a69d1358bbede376552675
 F test/conflict.test c5b849b01cfbe0a4f63a90cba6f68e2fe3a75f87
+F test/corrupt.test 0080ddcece23e8ba47c44608c4fb73fd4d1d8ce2
 F test/crash.test a3f6d27f7cb7f7bd752461db1e14f7c781ecedc3
 F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2
 F test/date.test fd3db29bba089578b1b008e3511ea9e74840377a
@@ -152,7 +153,7 @@ F test/pager3.test 16f546293bb751b8151dc17df613fca938bbec8b
 F test/pragma.test 66a66b7f3b273b93325c9a5794acb418f52fdcbf
 F test/printf.test 5a30fb0d736148fca64cb1b7ed0390be7414e024
 F test/progress.test 76c722f090b1ccb575e7e4e203a71608c5763beb x
-F test/quick.test 5bb4afdb204c57329c86fa11f3f0a5296675fd7f
+F test/quick.test 2e9af7203555ea894af654adf0794157434928fb
 F test/quote.test 6d75cf635d93ba2484dc9cb378d88cbae9dc2c62
 F test/rollback.test 4097328d44510277244ef4fa51b22b2f11d7ef4c
 F test/rowid.test b3d059f5c8d8874fa1c31030e0636f67405d20ea
@@ -245,7 +246,7 @@ F www/tclsqlite.tcl 560ecd6a916b320e59f2917317398f3d59b7cc25
 F www/vdbe.tcl 59288db1ac5c0616296b26dce071c36cb611dfe9
 F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0
 F www/whentouse.tcl a8335bce47cc2fddb07f19052cb0cb4d9129a8e4
-P acfc59186ac7452c0a74dc4ef8df9a1c94fc4f46
-R 075f7776d11844f668c581d38ba0ab40
+P 9322c439c5727f0d65548efdf4de4d7b89b4be66
+R 78b9eb3fdca60240e1b2bb671f0502fe
 U drh
-Z fcffcef9e3415aa6a29c8c8063b208d9
+Z b8b12d29336c5de9dba049e2bc96808c

diff --git a/manifest.uuid b/manifest.uuid
index a58e1e885a4022a5d64d99ad1e232ef4832a5939..283fbb4b9dbee843461498801a87683756ec42c8 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9322c439c5727f0d65548efdf4de4d7b89b4be66
\ No newline at end of file
+8f5b199e845fa7ae3444ef69bd840716d305cf73
\ No newline at end of file

diff --git a/src/btree.c b/src/btree.c
index 0b4ddeb60e3803f099721a8ffd0e2f8bfd804697..64e8aef2528eddadc6c237231e1b09086e1e6fe7 100644 (file)
--- a/src/btree.c
+++ b/src/btree.c
@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btree.c,v 1.182 2004/08/14 19:20:10 drh Exp $
+** $Id: btree.c,v 1.183 2004/08/30 16:52:18 drh Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** For a detailed discussion of BTrees, refer to
@@ -822,8 +822,10 @@ static int initPage(
   assert( pParent==0 || pParent->pBt==pPage->pBt );
   assert( pPage->pgno==sqlite3pager_pagenumber(pPage->aData) );
   assert( pPage->aData == &((unsigned char*)pPage)[-pPage->pBt->pageSize] );
-  assert( pPage->pParent==0 || pPage->pParent==pParent );
-  assert( pPage->pParent==pParent || !pPage->isInit );
+  if( pPage->pParent!=pParent && (pPage->pParent!=0 || pPage->isInit) ){
+    /* The parent page should never change unless the file is corrupt */
+    return SQLITE_CORRUPT; /* bkpt-CORRUPT */
+  }
   if( pPage->isInit ) return SQLITE_OK;
   if( pPage->pParent==0 && pParent!=0 ){
     pPage->pParent = pParent;
@@ -838,6 +840,14 @@ static int initPage(
   pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf;
   top = get2byte(&data[hdr+5]);
   pPage->nCell = get2byte(&data[hdr+3]);
+  if( pPage->nCell>MX_CELL ){
+    /* To many cells for a single page.  The page must be corrupt */
+    return SQLITE_CORRUPT; /* bkpt-CORRUPT */
+  }
+  if( pPage->nCell==0 && pParent!=0 && pParent->pgno!=1 ){
+    /* All pages must have at least one cell, except for root pages */
+    return SQLITE_CORRUPT; /* bkpt-CORRUPT */
+  }
 
   /* Compute the total free space on the page */
   pc = get2byte(&data[hdr+1]);
@@ -845,16 +855,28 @@ static int initPage(
   i = 0;
   while( pc>0 ){
     int next, size;
-    if( pc>=usableSize ) return SQLITE_CORRUPT;
-    if( i++>SQLITE_MAX_PAGE_SIZE ) return SQLITE_CORRUPT;
+    if( pc>usableSize-4 ){
+      /* Free block is off the page */
+      return SQLITE_CORRUPT;  /* bkpt-CORRUPT */
+    }
+    if( i++>SQLITE_MAX_PAGE_SIZE/4 ){
+      /* The free block list forms an infinite loop */
+      return SQLITE_CORRUPT;  /* bkpt-CORRUPT */
+    }
     next = get2byte(&data[pc]);
     size = get2byte(&data[pc+2]);
-    if( next>0 && next<=pc+size+3 ) return SQLITE_CORRUPT;
+    if( next>0 && next<=pc+size+3 ){
+      /* Free blocks must be in accending order */
+      return SQLITE_CORRUPT;  /* bkpt-CORRUPT */
+    }
     nFree += size;
     pc = next;
   }
   pPage->nFree = nFree;
-  if( nFree>=usableSize ) return SQLITE_CORRUPT;
+  if( nFree>=usableSize ){
+    /* Free space cannot exceed total page size */
+    return SQLITE_CORRUPT;  /* bkpt-CORRUPT */
+  }
 
   pPage->isInit = 1;
   pageIntegrity(pPage);
@@ -922,6 +944,9 @@ static int getAndInitPage(
   MemPage *pParent     /* Parent of the page */
 ){
   int rc;
+  if( pgno==0 ){
+    return SQLITE_CORRUPT;  /* bkpt-CORRUPT */
+  }
   rc = getPage(pBt, pgno, ppPage);
   if( rc==SQLITE_OK && (*ppPage)->isInit==0 ){
     rc = initPage(*ppPage, pParent);
@@ -1790,7 +1815,7 @@ static int getPayload(
   }
 
   if( amt>0 ){
-    return SQLITE_CORRUPT;
+    return SQLITE_CORRUPT; /* bkpt-CORRUPT */
   }
   return SQLITE_OK;
 }
@@ -1931,7 +1956,7 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){
   pCur->idx = 0;
   pCur->info.nSize = 0;
   if( pNewPage->nCell<1 ){
-    return SQLITE_CORRUPT;
+    return SQLITE_CORRUPT; /* bkpt-CORRUPT */
   }
   return SQLITE_OK;
 }
@@ -2386,6 +2411,9 @@ static int allocatePage(Btree *pBt, MemPage **ppPage, Pgno *pPgno, Pgno nearby){
       memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4);
       *ppPage = pTrunk;
       TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1));
+    }else if( k>pBt->usableSize/4 - 8 ){
+      /* Value of k is out of range.  Database corruption */
+      return SQLITE_CORRUPT; /* bkpt-CORRUPT */
     }else{
       /* Extract a leaf from the trunk */
       int closest;
@@ -2404,6 +2432,10 @@ static int allocatePage(Btree *pBt, MemPage **ppPage, Pgno *pPgno, Pgno nearby){
         closest = 0;
       }
       *pPgno = get4byte(&aData[8+closest*4]);
+      if( *pPgno>sqlite3pager_pagecount(pBt->pPager) ){
+        /* Free page off the end of the file */
+        return SQLITE_CORRUPT; /* bkpt-CORRUPT */
+      }
       TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d: %d more free pages\n",
              *pPgno, closest+1, k, pTrunk->pgno, n-1));
       if( closest<k-1 ){
@@ -2465,7 +2497,7 @@ static int freePage(MemPage *pPage){
     rc = getPage(pBt, get4byte(&pPage1->aData[32]), &pTrunk);
     if( rc ) return rc;
     k = get4byte(&pTrunk->aData[4]);
-    if( k==pBt->usableSize/4 - 8 ){
+    if( k>=pBt->usableSize/4 - 8 ){
       /* The trunk is full.  Turn the page being freed into a new
       ** trunk page with no leaves. */
       rc = sqlite3pager_write(pPage->aData);
@@ -3565,7 +3597,9 @@ int sqlite3BtreeDelete(BtCursor *pCur){
     getTempCursor(pCur, &leafCur);
     rc = sqlite3BtreeNext(&leafCur, &notUsed);
     if( rc!=SQLITE_OK ){
-      if( rc!=SQLITE_NOMEM ) rc = SQLITE_CORRUPT;
+      if( rc!=SQLITE_NOMEM ){
+        rc = SQLITE_CORRUPT;  /* bkpt-CORRUPT */
+      }
       return rc;
     }
     rc = sqlite3pager_write(leafCur.pPage->aData);
@@ -4041,10 +4075,16 @@ static void checkList(
     }
     if( isFreeList ){
       int n = get4byte(&pOvfl[4]);
-      for(i=0; i<n; i++){
-        checkRef(pCheck, get4byte(&pOvfl[8+i*4]), zContext);
+      if( n>=pCheck->pBt->usableSize/4-8 ){
+        sprintf(zMsg, "freelist leaf count too big on page %d", iPage);
+        checkAppendMsg(pCheck, zContext, zMsg);
+        N--;
+      }else{
+        for(i=0; i<n; i++){
+          checkRef(pCheck, get4byte(&pOvfl[8+i*4]), zContext);
+        }
+        N -= n;
       }
-      N -= n;
     }
     iPage = get4byte(pOvfl);
     sqlite3pager_unref(pOvfl);

diff --git a/src/pager.c b/src/pager.c
index ac760aae9c793bd4b7e74d77433a143ae365a9d6..b53962ad66a4ab1e1792bd850c3cf75432aa3cfd 100644 (file)
--- a/src/pager.c
+++ b/src/pager.c
@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.160 2004/08/21 19:20:42 drh Exp $
+** @(#) $Id: pager.c,v 1.161 2004/08/30 16:52:18 drh Exp $
 */
 #include "os.h"         /* Must be first to enable large file support */
 #include "sqliteInt.h"
@@ -1308,7 +1308,7 @@ static int pager_stmt_playback(Pager *pPager){
 end_stmt_playback:
   if( rc!=SQLITE_OK ){
     pPager->errMask |= PAGER_ERR_CORRUPT;
-    rc = SQLITE_CORRUPT;
+    rc = SQLITE_CORRUPT;  /* bkpt-CORRUPT */
   }else{
     pPager->journalOff = szJ;
     /* pager_reload_cache(pPager); */
@@ -2882,7 +2882,7 @@ int sqlite3pager_rollback(Pager *pPager){
     rc = pager_playback(pPager);
   }
   if( rc!=SQLITE_OK ){
-    rc = SQLITE_CORRUPT;
+    rc = SQLITE_CORRUPT;  /* bkpt-CORRUPT */
     pPager->errMask |= PAGER_ERR_CORRUPT;
   }
   pPager->dbSize = -1;

diff --git a/src/vdbemem.c b/src/vdbemem.c
index 434ec4daa1aa918a58c253e3b084a5fd4fbf683e..c6cd94e6348430472bbfa409d9a618d374d4de40 100644 (file)
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -632,10 +632,14 @@ void sqlite3VdbeMemSanity(Mem *pMem, u8 db_enc){
               pMem->enc==SQLITE_UTF16LE 
       );
       /* If the string is UTF-8 encoded and nul terminated, then pMem->n
-      ** must be the length of the string.
+      ** must be the length of the string.  (Later:)  If the database file
+      ** has been corrupted, '\000' characters might have been inserted
+      ** into the middle of the string.  In that case, the strlen() might
+      ** be less.
       */
       if( pMem->enc==SQLITE_UTF8 && (flags & MEM_Term) ){ 
-        assert( strlen(pMem->z)==pMem->n );
+        assert( strlen(pMem->z)<=pMem->n );
+        assert( pMem->z[pMem->n]==0 );
       }
     }
   }else{

diff --git a/test/corrupt.test b/test/corrupt.test
new file mode 100644 (file)
index 0000000..b3aae8d
--- /dev/null
+++ b/test/corrupt.test
@@ -0,0 +1,109 @@
+# 2004 August 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+# This file implements tests to make sure SQLite does not crash or
+# segfault if it sees a corrupt database file.
+#
+# $Id: corrupt.test,v 1.1 2004/08/30 16:52:19 drh Exp $
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# Construct a large database for testing.
+#
+do_test corrupt-1.1 {
+  execsql {
+    BEGIN;
+    CREATE TABLE t1(x);
+    INSERT INTO t1 VALUES(randstr(10,100));
+    INSERT INTO t1 VALUES(randstr(10,100));
+    INSERT INTO t1 VALUES(randstr(10,100));
+    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
+    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
+    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
+    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
+    INSERT INTO t1 VALUES(randstr(2100,3000));
+    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
+    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
+    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
+    INSERT INTO t1 SELECT x || randstr(5,10) FROM t1;
+    CREATE INDEX t1i1 ON t1(x);
+    CREATE TABLE t2 AS SELECT * FROM t1;
+    DELETE FROM t2 WHERE rowid%5!=0;
+    COMMIT;
+    PRAGMA integrity_check;
+  }
+} {ok}
+
+# Copy a file 
+#
+proc copy_file {from to} {
+  set f [open $from]
+  fconfigure $f -translation binary
+  set t [open $to w]
+  fconfigure $t -translation binary
+  puts -nonewline $t [read $f [file size $from]]
+  close $t
+  close $f
+}
+
+# Setup for the tests.
+copy_file test.db test.bu
+set fsize [file size test.db]
+set junk "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+while {[string length $junk]<256} {append junk $junk}
+set junk [string range $junk 0 255]
+
+
+for {set i [expr {1*256}]} {$i<$fsize-256} {incr i 256} {
+  set tn [expr {$i/256}]
+  db close
+  copy_file test.bu test.db
+  set fd [open test.db r+]
+  fconfigure $fd -translation binary
+  seek $fd $i
+  puts -nonewline $fd $junk
+  close $fd
+  sqlite3 db test.db
+  do_test corrupt-2.$tn.1 {
+    sqlite3 db test.db
+    catchsql {SELECT count(*) FROM sqlite_master}
+    set x {}
+  } {}
+  do_test corrupt-2.$tn.2 {
+    catchsql {SELECT count(*) FROM t1}
+    set x {}
+  } {}
+  do_test corrupt-2.$tn.3 {
+    catchsql {SELECT count(*) FROM t1 WHERE x>'abcdef'}
+    set x {}
+  } {}
+  do_test corrupt-2.$tn.4 {
+    catchsql {SELECT count(*) FROM t2}
+    set x {}
+  } {}
+  do_test corrupt-2.$tn.5 {
+    catchsql {CREATE TABLE t3 AS SELECT * FROM t1}
+    set x {}
+  } {}
+  do_test corrupt-2.$tn.6 {
+    catchsql {DROP TABLE t1}
+    set x {}
+  } {}
+if {$tn==724} btree_breakpoint
+  do_test corrupt-2.$tn.7 {
+    catchsql {PRAGMA integrity_check}
+    set x {}
+  } {}
+}  
+
+finish_test

diff --git a/test/quick.test b/test/quick.test
index 4857daf3bf343478e7ea52309f90ed944a413355..a0338dc0b42aa7cfe263c9a6e5dca71f8f5d57a5 100644 (file)
--- a/test/quick.test
+++ b/test/quick.test
@@ -10,7 +10,7 @@
 #***********************************************************************
 # This file runs all tests.
 #
-# $Id: quick.test,v 1.28 2004/07/22 15:02:26 drh Exp $
+# $Id: quick.test,v 1.29 2004/08/30 16:52:19 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -20,12 +20,13 @@ set ISQUICK 1
 
 set EXCLUDE {
   all.test
-  quick.test
   btree2.test
+  corrupt.test
+  crash.test
   malloc.test
   memleak.test
   misuse.test
-  crash.test
+  quick.test
   utf16.test
 }