From: dan <dan@noemail.net>
Date: Tue, 27 Apr 2010 05:42:32 +0000 (+0000)
Subject: Fix a problem that occurs when one process causes the log-summary file to grow and... 
X-Git-Tag: version-3.7.2~455^2~48
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=31f98fc8e41ae7f8bf0ced0f8572cfcd692e648b;p=thirdparty%2Fsqlite.git

Fix a problem that occurs when one process causes the log-summary file to grow and then a second process attempts to read the database.

FossilOrigin-Name: b51a5f8bc660616aa264025dd7ad4bdab458814b
---

diff --git a/manifest b/manifest
index 74ed04bf37..d538e7c77b 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,5 @@
------BEGIN PGP SIGNED MESSAGE-----
-Hash: SHA1
-
-C Merge\sin\srecent\schanges\sfrom\sthe\strunk
-D 2010-04-27T01:56:22
+C Fix\sa\sproblem\sthat\soccurs\swhen\sone\sprocess\scauses\sthe\slog-summary\sfile\sto\sgrow\sand\sthen\sa\ssecond\sprocess\sattempts\sto\sread\sthe\sdatabase.
+D 2010-04-27T05:42:32
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -224,7 +221,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e
 F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1
 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2
 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda
-F src/wal.c ce03bdd759c7b949188c0b75c46777fe0f7f105e
+F src/wal.c 26d2a7028cc4af995e657431e2dbcea0aac63784
 F src/wal.h 812101dd76610401fbcd44114e7e8b7ce0224645
 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
 F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885
@@ -476,13 +473,13 @@ F test/limit.test 2db7b3b34fb925b8e847d583d2eb67531d0ce67e
 F test/loadext.test 0393ce12d9616aa87597dd0ec88181de181f6db0
 F test/loadext2.test 0bcaeb4d81cd5b6e883fdfea3c1bdbe1f173cbca
 F test/lock.test 842e80b6be816c79525a20b098cca066989feed7
-F test/lock2.test ec208a5f394d92affaf599fde3f374361657d0ff
+F test/lock2.test 64ba8f1c7709abf91287d564984a6b8c5209e755
 F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00
 F test/lock4.test f4f36271aa5ae1da449646bf43c7341f6b2b4c4e
 F test/lock5.test 6b1f78f09ad1522843dad571b76b321e6f439bf7
 F test/lock6.test 862aa71e97b288d6b3f92ba3313f51bd0b003776
 F test/lock7.test 64006c84c1c616657e237c7ad6532b765611cf64
-F test/lock_common.tcl 58aa21f38c28223cc1107b5b2c9d7d61aa428e79
+F test/lock_common.tcl ebc5b9a238d6d9f08a8433e52bdbc67d1478e774
 F test/lookaside.test 1dd350dc6dff015c47c07fcc5a727a72fc5bae02
 F test/main.test 2be2352ac77ac5b238c6337a5469aeeef57677e6
 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9
@@ -535,7 +532,7 @@ F test/pageropt.test 3ee6578891baaca967f0bd349e4abfa736229e1a
 F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb
 F test/pcache.test eebc4420b37cb07733ae9b6e99c9da7c40dd6d58
 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16
-F test/permutations.test 0730dc6c028b620a15c38a0a7d07238da088ecd3
+F test/permutations.test 894a6df3c4394194f84feea8520580dad1b71280
 F test/pragma.test 6960f9efbce476f70ba9ee2171daf5042f9e3d8a
 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47
 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea
@@ -556,7 +553,7 @@ F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7
 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec
 F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0
 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd
-F test/savepoint6.test 30f5c068d7aceacbf9bd6a7df1c99f34b2fc2723
+F test/savepoint6.test bdc653c5c08328c84a74471b5b23fd483b919dfe
 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481
 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5
 F test/securedel.test 328d2921c0ca49bdd3352e516b0377fc07143254
@@ -761,7 +758,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61
 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
-F test/wal.test b5d5fe1b3171eefb25355758e0dc689096bc1f87
+F test/wal.test fbf7a911b35eb9b09a84eee4020e9e4c8b148d6f
 F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba
 F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1
 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47
@@ -811,14 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 30d0134454734fd9335896372c30a903e9adcc64 ca9d86baf70f210d331ce93102177c8005c494cb
-R b0a15978d17250dbff5ca26f38578573
-U drh
-Z 50883e064ff8d31e35ff6084caafc75f
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.6 (GNU/Linux)
-
-iD8DBQFL1kRKoxKgR168RlERAhGtAJ0ZNNVfZUQ2rD2DR5znbRsfpO0FrQCeMGjZ
-bgsVZnQebWJklqCayCQA1Cc=
-=MFJQ
------END PGP SIGNATURE-----
+P 7a0ac682c3bffcb345321fe97434563350ac90f9
+R 8f311e98676985c9f340b1e7e337c8e4
+U dan
+Z 70973fdb159f8a0ed7dee7d674e22de0
diff --git a/manifest.uuid b/manifest.uuid
index e84e36275c..f9a7784eff 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7a0ac682c3bffcb345321fe97434563350ac90f9
\ No newline at end of file
+b51a5f8bc660616aa264025dd7ad4bdab458814b
\ No newline at end of file
diff --git a/src/wal.c b/src/wal.c
index 8c6878a1a4..14ee78d9f1 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -592,6 +592,22 @@ static int logSummaryMap(LogSummary *pSummary, int nByte){
   return SQLITE_OK;
 }
 
+/*
+** The log-summary file is already mapped to pSummary->aData[], but the
+** mapping needs to be resized. Unmap and remap the file so that the mapping 
+** is at least nByte bytes in size, or the size of the entire file if it 
+** is larger than nByte bytes.
+*/
+static int logSummaryRemap(LogSummary *pSummary, int nByte){
+  int rc;
+  sqlite3_mutex_enter(pSummary->mutex);
+  munmap(pSummary->aData, pSummary->nData*4);
+  pSummary->aData = 0;
+  rc = logSummaryMap(pSummary, nByte);
+  sqlite3_mutex_leave(pSummary->mutex);
+  return rc;
+}
+
 /*
 ** Return the index in the LogSummary.aData array that corresponds to 
 ** frame iFrame. The log-summary file consists of a header, followed by
@@ -613,18 +629,18 @@ static int logSummaryEntry(u32 iFrame){
 ** the previous call), but that restriction is not enforced or asserted
 ** here.
 */
-static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){
+static int logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){
   u32 iSlot = logSummaryEntry(iFrame);
 
   while( (iSlot+128)>=pSummary->nData ){
+    int rc;
     int nByte = pSummary->nData*4 + LOGSUMMARY_MMAP_INCREMENT;
 
     /* Unmap and remap the log-summary file. */
-    sqlite3_mutex_enter(pSummary->mutex);
-    munmap(pSummary->aData, pSummary->nData*4);
-    pSummary->aData = 0;
-    logSummaryMap(pSummary, nByte);
-    sqlite3_mutex_leave(pSummary->mutex);
+    rc = logSummaryRemap(pSummary, nByte);
+    if( rc!=SQLITE_OK ){
+      return rc;
+    }
   }
 
   /* Set the log-summary entry itself */
@@ -649,6 +665,8 @@ static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){
     logMergesort8(aFrame, aTmp, aIndex, &nIndex);
     memset(&aIndex[nIndex], aIndex[nIndex-1], 256-nIndex);
   }
+
+  return SQLITE_OK;
 }
 
 
@@ -1496,6 +1514,16 @@ int sqlite3WalOpenSnapshot(Log *pLog, int *pChanged){
     if( rc!=SQLITE_OK ){
       /* An error occured while attempting log recovery. */
       sqlite3WalCloseSnapshot(pLog);
+    }else{
+      /* Check if the mapping needs to grow. */
+      LogSummary *pSummary = pLog->pSummary;
+
+      if( pLog->hdr.iLastPg 
+       && logSummaryEntry(pLog->hdr.iLastPg)>=pSummary->nData 
+      ){
+        rc = logSummaryRemap(pSummary, 0);
+        assert( rc || logSummaryEntry(pLog->hdr.iLastPg)<pSummary->nData );
+      }
     }
   }
   return rc;
@@ -1516,20 +1544,20 @@ void sqlite3WalCloseSnapshot(Log *pLog){
 ** Read a page from the log, if it is present. 
 */
 int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){
+  LogSummary *pSummary = pLog->pSummary;
   u32 iRead = 0;
   u32 *aData; 
   int iFrame = (pLog->hdr.iLastPg & 0xFFFFFF00);
 
   assert( pLog->isLocked );
-
-  sqlite3_mutex_enter(pLog->pSummary->mutex);
-  aData = pLog->pSummary->aData;
+  sqlite3_mutex_enter(pSummary->mutex);
 
   /* Do a linear search of the unindexed block of page-numbers (if any) 
   ** at the end of the log-summary. An alternative to this would be to
   ** build an index in private memory each time a read transaction is
   ** opened on a new snapshot.
   */
+  aData = pSummary->aData;
   if( pLog->hdr.iLastPg ){
     u32 *pi = &aData[logSummaryEntry(pLog->hdr.iLastPg)];
     u32 *piStop = pi - (pLog->hdr.iLastPg & 0xFF);
diff --git a/test/lock2.test b/test/lock2.test
index 63319535d1..e4c5204714 100644
--- a/test/lock2.test
+++ b/test/lock2.test
@@ -34,7 +34,6 @@ source $testdir/lock_common.tcl
 #
 do_test lock2-1.1 {
   set ::tf1 [launch_testfixture]
-  testfixture $::tf1 "sqlite3_test_control_pending_byte $::sqlite_pending_byte"
   testfixture $::tf1 {
     sqlite3 db test.db -key xyzzy
     db eval {select * from sqlite_master}
diff --git a/test/lock_common.tcl b/test/lock_common.tcl
index 31c04e8535..a17de8a3da 100644
--- a/test/lock_common.tcl
+++ b/test/lock_common.tcl
@@ -24,6 +24,7 @@ proc launch_testfixture {} {
   }
   set chan [open "|$prg tf_main.tcl" r+]
   fconfigure $chan -buffering line
+  testfixture $chan "sqlite3_test_control_pending_byte $::sqlite_pending_byte"
   return $chan
 }
 
diff --git a/test/permutations.test b/test/permutations.test
index e4c47a3650..f51cc84422 100644
--- a/test/permutations.test
+++ b/test/permutations.test
@@ -758,6 +758,10 @@ if {[info commands register_demovfs] != ""} {
 
 run_tests "wal" -description {
   Run tests with journal_mode=WAL
+} -initialize {
+  set ::savepoint6_iterations 100
+} -shutdown {
+  unset -nocomplain ::savepoint6_iterations
 } -include {
   savepoint.test
   savepoint2.test
diff --git a/test/savepoint6.test b/test/savepoint6.test
index 2b2bcc8e06..7f60769c63 100644
--- a/test/savepoint6.test
+++ b/test/savepoint6.test
@@ -26,7 +26,9 @@ set DATABASE_SCHEMA {
     CREATE INDEX i2 ON t1(y);
 }
 
-set ::ITERATIONS 1000
+if {0==[info exists ::savepoint6_iterations]} {
+  set ::savepoint6_iterations 1000
+}
 
 #--------------------------------------------------------------------------
 # In memory database state.
@@ -257,7 +259,7 @@ foreach zSetup [list {
     checkdb
   } {ok}
   
-  for {set i 0} {$i < $::ITERATIONS} {incr i} {
+  for {set i 0} {$i < $::savepoint6_iterations} {incr i} {
     do_test savepoint6-$testname.$i.1 {
       savepoint_op
       checkdb
diff --git a/test/wal.test b/test/wal.test
index d533f9e0c7..30586d5cb3 100644
--- a/test/wal.test
+++ b/test/wal.test
@@ -19,7 +19,7 @@ source $testdir/lock_common.tcl
 
 proc reopen_db {} {
   catch { db close }
-  file delete -force test.db test.db-wal
+  file delete -force test.db test.db-wal test.db-wal-summary
   sqlite3_wal db test.db
 }
 
@@ -249,12 +249,10 @@ do_test wal-4.5.2 {
   }
 } {32}
 do_test wal-4.5.3 {
-breakpoint
   execsql { ROLLBACK TO tr }
 } {}
 do_test wal-4.5.4 {
   set logsize [file size test.db-wal]
-breakpoint
   execsql {
       INSERT INTO t1 VALUES('x', 'y');
     RELEASE tr;
@@ -269,7 +267,6 @@ do_test wal-4.5.6 {
   file copy -force test.db test2.db
   file copy -force test.db-wal test2.db-wal
   sqlite3 db2 test2.db
-breakpoint
   execsql { SELECT count(*) FROM t2 ; SELECT count(*) FROM t1 } db2
 } {1 2}
 do_test wal-4.5.7 {
@@ -823,10 +820,11 @@ db close
 #-------------------------------------------------------------------------
 # Test large log summaries.
 #
-do_test wal-13.1 {
+set sqlite_walsummary_mmap_incr 64
+do_test wal-13.1.1 {
   list [file exists test.db] [file exists test.db-wal]
 } {1 0}
-do_test wal-13.2 {
+do_test wal-13.1.2 {
   set fd [open test.db-wal w]
   seek $fd [expr 200*1024*1024]
   puts $fd ""
@@ -834,25 +832,63 @@ do_test wal-13.2 {
   sqlite3 db test.db
   execsql { SELECT * FROM t2 }
 } {B 2}
-do_test wal-13.3 {
+do_test wal-13.1.3 {
   db close
   file exists test.db-wal
 } {0}
-do_test wal-13.4 {
+do_test wal-13.1.4 {
   sqlite3 db test.db
   execsql { SELECT count(*) FROM t2 }
 } {1}
-do_test wal-13.5 {
-  for {set i 0} {$i < 15} {incr i} {
+do_test wal-13.1.5 {
+  for {set i 0} {$i < 6} {incr i} {
     execsql { INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM t2 }
   }
   execsql { SELECT count(*) FROM t2 }
-} [expr int(pow(2, 15))]
-do_test wal-13.6 {
-  set sz [file size test.db-wal-summary]
-  expr {$sz<=(3*64*1024) && $sz>(2*64*1024)}
-} {1}
+} [expr int(pow(2, 6))]
+do_test wal-13.1.6 {
+  file size test.db-wal
+} [log_file_size 80 1024]
+
+foreach code [list {
+  set tn 2
+  proc buddy {tcl} { uplevel #0 $tcl }
+} {
+  set tn 3
+  set ::buddy [launch_testfixture]
+  proc buddy {tcl} { testfixture $::buddy $tcl }
+}] {
 
+  eval $code
+  reopen_db
+
+  do_test wal-13.$tn.0 {
+    buddy { sqlite3 db2 test.db }
+    execsql {
+      PRAGMA journal_mode = WAL;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 SELECT randomblob(400);
+    }
+    execsql { SELECT count(*) FROM t1 }
+  } {1}
+
+  for {set ii 1} {$ii<16} {incr ii} {
+    do_test wal-13.$tn.$ii.a {
+      buddy { db2 eval { INSERT INTO t1 SELECT randomblob(400) FROM t1 } }
+      buddy { db2 eval { SELECT count(*) FROM t1 } }
+    } [expr (1<<$ii)]
+    do_test wal-13.$tn.$ii.b {
+      db eval { SELECT count(*) FROM t1 }
+    } [expr (1<<$ii)]
+    do_test wal-13.$tn.$ii.c {
+      db eval { SELECT count(*) FROM t1 }
+    } [expr (1<<$ii)]
+  }
+
+  catch { db2 close }
+  catch { close $::buddy }
+  db close
+}
 
 finish_test