Fixes and tests for backup of a WAL database.

FossilOrigin-Name: 480d12db4c0ebcc37598f7620d39193875eab15b

diff --git a/manifest b/manifest
index 49b2e5bcc4a9aa9d063c1b5cbe2c8d9714e4dbac..2f6f6317d8c4f27691d699da6f3737639b6c2d7d 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\svery\ssimple\stest\scases\sfor\sbackup\sand\sVACUUM\sof\sWAL\sdatabases.\sMore\sto\scome.
-D 2010-04-23T11:44:41
+C Fixes\sand\stests\sfor\sbackup\sof\sa\sWAL\sdatabase.
+D 2010-04-23T19:15:00
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -131,8 +131,8 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c
 F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f
 F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e
 F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581
-F src/log.c da486aab7abfab8edbc20eb49e2f68e9d3a902b6
-F src/log.h b8c45a6348d9ef57c5205a08c611d57d07ee9feb
+F src/log.c d9fdaad6b0b5ace54153b85f5bfcf1b9d1abac67
+F src/log.h bc44b0eec723648c8aa0a05ab78e1b76a4032e02
 F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e
 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30
 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f
 F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1
 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053
-F src/pager.c d83d2ea6d2025554e079f891854b133924305e19
+F src/pager.c 485a34834a96863fd709f4d01f886ae70a9689ea
 F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444
 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e
 F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf
@@ -253,7 +253,7 @@ F test/autoindex1.test ffb06a246e2c1f89cfbe3d93eca513c9e78d4063
 F test/autovacuum.test 25f891bc343a8bf5d9229e2e9ddab9f31a9ab5ec
 F test/autovacuum_ioerr2.test 598b0663074d3673a9c1bc9a16e80971313bafe6
 F test/avtrans.test 1e901d8102706b63534dbd2bdd4d8f16c4082650
-F test/backup.test 3549ea8f541a08205c0eb813b21e81ea8301f6ed
+F test/backup.test b1e874fa9b01de9dd5137a8371d060b76a435162
 F test/backup2.test 159419073d9769fdb1780ed7e5b391a046f898d5
 F test/backup_ioerr.test 1f012e692f42c0442ae652443258f70e9f20fa38
 F test/backup_malloc.test 1e063c6d75143d0d6e0ae77971dd690070369387
@@ -759,7 +759,7 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
 F test/wal.test fcb5ec1fbec08c1165b6974f126056f2b4cead49
-F test/walbak.test 3aecf824ee6433bd34673336623c4990aa1346ba
+F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba
 F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1
 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47
 F test/walmode.test c2f4e30ad64910b2d8faf6cf4e940b3f201b41df
@@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 5d6d4423d1def39bd2424703120aa985085c3f8e
-R 7813072bd3855cfe2a094b15eff01c69
+P 1077d8130b8ed5716ad73f78382270909d347963
+R 0d1581e70f8ee05579afbc11de72b135
 U dan
-Z f898bd1b5727c5103088c594c0575550
+Z c6fc40f98d5658443e299ebf167b2a61

diff --git a/manifest.uuid b/manifest.uuid
index b5381319c7bd32bf88649fd58d1592803227d842..4ef5c841105019557d07bd9eaa539933f4002214 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1077d8130b8ed5716ad73f78382270909d347963
\ No newline at end of file
+480d12db4c0ebcc37598f7620d39193875eab15b
\ No newline at end of file

diff --git a/src/log.c b/src/log.c
index f32fdb221d8cb81463a7ab32251bfc9457607848..c1b734f8fe8a70f7d1aedc860f523d62e80ae85b 100644 (file)
--- a/src/log.c
+++ b/src/log.c
@@ -1568,6 +1568,14 @@ int sqlite3LogWriteLock(Log *pLog, int op){
   return SQLITE_OK;
 }
 
+/* 
+** Return true if data has been written but not committed to the log file. 
+*/
+int sqlite3LogDirty(Log *pLog){
+  assert( pLog->isWriteLocked );
+  return( pLog->hdr.iLastPg!=((LogSummaryHdr*)pLog->pSummary->aData)->iLastPg );
+}
+
 /* 
 ** Write a set of frames to the log. The caller must hold at least a
 ** RESERVED lock on the database file.

diff --git a/src/log.h b/src/log.h
index af698ebf5f40b5bf648da24ecbd7fa69a8eb1958..02201c837e8d7f6f816b86e1388065b704fab58c 100644 (file)
--- a/src/log.h
+++ b/src/log.h
@@ -37,6 +37,9 @@ void sqlite3LogDbsize(Log *pLog, Pgno *pPgno);
 /* Obtain or release the WRITER lock. */
 int sqlite3LogWriteLock(Log *pLog, int op);
 
+/* Return true if data has been written but not committed to the log file. */
+int sqlite3LogDirty(Log *pLog);
+
 /* Write a frame or frames to the log. */
 int sqlite3LogFrames(Log *pLog, int, PgHdr *, Pgno, int, int);
 

diff --git a/src/pager.c b/src/pager.c
index 81f0968f546ec2ecb09c6644a559aa61dce3ab42..2db8b9336c0308521e63573204412b26ec85664c 100644 (file)
--- a/src/pager.c
+++ b/src/pager.c
@@ -2234,9 +2234,30 @@ static int readDbPage(PgHdr *pPg){
   return rc;
 }
 
+/*
+** This function is called when a transaction on a WAL database is rolled
+** back. For each dirty page in the cache, do one of the following:
+**
+**   * If the page has no outstanding references, simply discard it.
+**   * Otherwise, if the page has one or more outstanding references, 
+**     reload the original content from the database (or log file).
+*/
 static int pagerRollbackLog(Pager *pPager){
   int rc = SQLITE_OK;
   PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache);
+
+  /* Normally, if a transaction is rolled back, any backup processes are
+  ** updated as data is copied out of the rollback journal and into the
+  ** database. This is not generally possible with a WAL database, as
+  ** rollback involves simply truncating the log file. Therefore, if one
+  ** or more frames have already been written to the log (and therefore 
+  ** also copied into the backup databases) as part of this transaction,
+  ** the backups must be restarted.
+  */
+  if( sqlite3LogDirty(pPager->pLog) ){
+    sqlite3BackupRestart(pPager->pBackup);
+  }
+
   pPager->dbSize = pPager->dbOrigSize;
   while( pList && rc==SQLITE_OK ){
     PgHdr *pNext = pList->pDirty;
@@ -3218,6 +3239,33 @@ static int subjournalPage(PgHdr *pPg){
   return rc;
 }
 
+/*
+** This function is a wrapper around sqlite3LogFrames(). As well as logging
+** the contents of the list of pages headed by pList (connected by pDirty),
+** this function notifies any active backup processes that the pages have
+** changed. 
+*/ 
+static int pagerLogFrames(
+  Pager *pPager,                  /* Pager object */
+  PgHdr *pList,                   /* List of frames to log */
+  Pgno nTruncate,                 /* Database size after this commit */
+  int isCommit,                   /* True if this is a commit */
+  int sync_flags                  /* Flags to pass to OsSync() (or 0) */
+){
+  int rc;                         /* Return code */
+
+  assert( pPager->pLog );
+  rc = sqlite3LogFrames(pPager->pLog, 
+      pPager->pageSize, pList, nTruncate, isCommit, sync_flags
+  );
+  if( rc==SQLITE_OK && pPager->pBackup ){
+    PgHdr *p;
+    for(p=pList; p; p=p->pDirty){
+      sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData);
+    }
+  }
+  return rc;
+}
 
 /*
 ** This function is called by the pcache layer when it has reached some
@@ -3248,7 +3296,7 @@ static int pagerStress(void *p, PgHdr *pPg){
   pPg->pDirty = 0;
   if( pagerUseLog(pPager) ){
     /* Write a single frame for this page to the log. */
-    rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pPg, 0, 0, 0);
+    rc = pagerLogFrames(pPager, pPg, 0, 0, 0);
   }else{
     /* The doNotSync flag is set by the sqlite3PagerWrite() function while it
     ** is journalling a set of two or more database pages that are stored
@@ -4881,8 +4929,8 @@ int sqlite3PagerCommitPhaseOne(
     if( pagerUseLog(pPager) ){
       PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache);
       if( pList ){
-        rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pList,
-            pPager->dbSize, 1, (pPager->fullSync ? pPager->sync_flags : 0)
+        rc = pagerLogFrames(pPager, pList, pPager->dbSize, 1, 
+            (pPager->fullSync ? pPager->sync_flags : 0)
         );
       }
       sqlite3PcacheCleanAll(pPager->pPCache);
@@ -5119,6 +5167,7 @@ int sqlite3PagerRollback(Pager *pPager){
   PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager)));
   if( pagerUseLog(pPager) ){
     int rc2;
+
     rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1);
     rc2 = pager_end_transaction(pPager, pPager->setMaster);
     if( rc==SQLITE_OK ) rc = rc2;

diff --git a/test/backup.test b/test/backup.test
index ec30adb361fc1082527b5dc9d12f09d526af26ff..ca8de8eccab30a3c24f47586419e46165aaf6152 100644 (file)
--- a/test/backup.test
+++ b/test/backup.test
@@ -38,6 +38,8 @@ source $testdir/tester.tcl
 #
 # backup-9.*: Test that passing a negative argument to backup_step() is
 #             interpreted as "copy the whole file".
+# 
+# backup-10.*: Test writing the source database mid backup.
 #
 
 proc data_checksum {db file} { $db one "SELECT md5sum(a, b) FROM ${file}.t1" }
@@ -905,6 +907,7 @@ ifcapable memorymanage {
 }
 
 
+#-----------------------------------------------------------------------
 # Test that if the database is written to via the same database handle being
 # used as the source by a backup operation:
 #

diff --git a/test/walbak.test b/test/walbak.test
index f4998ac7f12db170ff1cc05512d0e911f423b593..11a7040cdc339f3c50f000794b8e0050d6b968c6 100644 (file)
--- a/test/walbak.test
+++ b/test/walbak.test
@@ -20,6 +20,12 @@ proc log_file_size {nFrame pgsz} {
   expr {12 + ($pgsz+16)*$nFrame}
 }
 
+# Test organization:
+# 
+#   walback-1.*: Simple tests.
+#   walback-2.*: Test backups when the source db is modified mid-backup.
+#
+
 # Make sure a simple backup from a WAL database works.
 #
 do_test walbak-1.0 {
@@ -53,20 +59,20 @@ db2 close
 
 # Try a VACUUM on a WAL database.
 #
-do_test walbak-2.1 {
+do_test walbak-1.4 {
   execsql { 
     VACUUM;
     PRAGMA main.journal_mode;
   }
 } {wal}
-do_test walbak-2.2 {
+do_test walbak-1.5 {
   list [file size test.db] [file size test.db-wal]
 } [list 1024 [log_file_size 6 1024]]
-do_test walbak-2.3 {
+do_test walbak-1.6 {
   execsql { PRAGMA checkpoint }
   list [file size test.db] [file size test.db-wal]
 } [list [expr 3*1024] [log_file_size 6 1024]]
-do_test walbak-2.4 {
+do_test walbak-1.7 {
   execsql { 
     CREATE TABLE t2(a, b);
     INSERT INTO t2 SELECT * FROM t1;
@@ -74,13 +80,105 @@ do_test walbak-2.4 {
   }
   list [file size test.db] [file size test.db-wal]
 } [list [expr 3*1024] [log_file_size 6 1024]]
-do_test walbak-2.5 {
+do_test walbak-1.8 {
   execsql { VACUUM }
   list [file size test.db] [file size test.db-wal]
 } [list [expr 3*1024] [log_file_size 8 1024]]
-do_test walbak-2.6 {
+do_test walbak-1.9 {
   execsql { PRAGMA checkpoint }
   list [file size test.db] [file size test.db-wal]
 } [list [expr 2*1024] [log_file_size 8 1024]]
 
+#-------------------------------------------------------------------------
+# Backups when the source db is modified mid-backup.
+#
+proc sig {{db db}} {
+  $db eval { 
+    PRAGMA integrity_check;
+    SELECT md5sum(a, b) FROM t1; 
+  }
+}
+db close
+file delete test.db
+sqlite3 db test.db
+do_test walbak-2.1 {
+  execsql { PRAGMA journal_mode = WAL }
+  execsql {
+    CREATE TABLE t1(a PRIMARY KEY, b);
+    BEGIN;
+      INSERT INTO t1 VALUES(randomblob(500), randomblob(500));
+      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /*  2 */
+      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /*  4 */
+      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /*  8 */
+      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 16 */
+      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 32 */
+      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 64 */
+    COMMIT;
+  }
+} {}
+do_test walbak-2.2 {
+  db backup abc.db
+  sqlite3 db2 abc.db
+  string compare [sig db] [sig db2]
+} {0}
+
+do_test walbak-2.3 {
+  sqlite3_backup B db2 main db main
+  B step 50
+  execsql { UPDATE t1 SET b = randomblob(500) }
+  list [B step 1000] [B finish]
+} {SQLITE_DONE SQLITE_OK}
+do_test walbak-2.4 {
+  string compare [sig db] [sig db2]
+} {0}
+
+do_test walbak-2.5 {
+  db close
+  sqlite3 db test.db
+  execsql { PRAGMA cache_size = 10 }
+  sqlite3_backup B db2 main db main
+  B step 50
+  execsql {
+    BEGIN;
+      UPDATE t1 SET b = randomblob(500);
+  }
+  expr [file size test.db-wal] > 10*1024
+} {1}
+do_test walbak-2.6 {
+  B step 1000
+} {SQLITE_BUSY}
+do_test walbak-2.7 {
+  execsql COMMIT
+  list [B step 1000] [B finish]
+} {SQLITE_DONE SQLITE_OK}
+do_test walbak-2.8 {
+  string compare [sig db] [sig db2]
+} {0}
+
+do_test walbak-2.9 {
+  db close
+  sqlite3 db test.db
+  execsql { PRAGMA cache_size = 10 }
+  sqlite3_backup B db2 main db main
+  B step 50
+  execsql {
+    BEGIN;
+      UPDATE t1 SET b = randomblob(500);
+  }
+  expr [file size test.db-wal] > 10*1024
+} {1}
+do_test walbak-2.10 {
+  B step 1000
+} {SQLITE_BUSY}
+do_test walbak-2.11 {
+  execsql ROLLBACK
+set sigB [sig db]
+  list [B step 1000] [B finish]
+} {SQLITE_DONE SQLITE_OK}
+do_test walbak-2.12 {
+  string compare [sig db] [sig db2]
+} {0}
+db2 close
+
 finish_test
+