Add support for crash recovery in multi-process mode. And add test cases for

the same.

FossilOrigin-Name: a8115f95e80cc90c095fdd0a151da51f4d3ee427defdc34780e977585a68776d

diff --git a/manifest b/manifest
index f2e203fee2f6b262cbe4808d4fc181d7c1ad40f8..c181f8613ba8a3a383fae17ddd36e877810ccffa 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Update\sthis\sbranch\swith\sthe\slatest\schanges\sfrom\strunk.
-D 2017-08-16T17:06:42.450
+C Add\ssupport\sfor\scrash\srecovery\sin\smulti-process\smode.\sAnd\sadd\stest\scases\sfor\nthe\ssame.
+D 2017-08-17T19:32:02.420
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -456,7 +456,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 4324a94573b1e29286f8121e4881db59eaedc014afeb274c8d3e07ed282e0e20
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c c9b3d8444bbf6f167d84f41ca6f3672e2521cb163a8c706b19058dc82fffe9b8
-F src/server.c e8d1c1a0e39d508c75e11eafa9a20edc85f5b619b27f7218950f3a77431f5a71
+F src/server.c 481366fff6584dc6754b6e59d3340d379faf0c55fd7e012d46257139af282e1d
 F src/server.h cf1ede28aaa07a30550228582f211327b5ebe5517d2334e35ec09d00fd6d230d
 F src/shell.c bd6a37cbe8bf64ef6a6a74fdc50f067d3148149b4ce2b4d03154663e66ded55f
 F src/shell.c.in b5725acacba95ccefa57b6d068f710e29ba8239c3aa704628a1902a1f729c175
@@ -1094,7 +1094,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test 4d6645be4965438d14297bbf8290aa97fd13cd5bf9fab718bfec6dc8e29da1cb
+F test/permutations.test b3326d58fe5c183ebcb359fe880db6c98ed1d3f4d3b350ec718f74287e063bee
 F test/pragma.test f274259d6393b6681eb433beb8dd39a26ec06a4431052a4880b43b84912a3f58
 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f
 F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed
@@ -1176,7 +1176,7 @@ F test/server3.test c3ae4ca7a6e7df870bfcd2450a9815507eaa80b9cdc44ee6c7975d483115
 F test/server4.test 97040670597948a695b1973537d770417589f1998bcbb3959302aaee3c211250
 F test/server5.test 2e554001145170094a19731a8ce2981d040cf44c947542b35d130e6e31256fca
 F test/server_common.tcl 551923cf8d51fefcdb4444bfd72b88ca5c5228fe1525da5b6528ae4edb7a2f2e
-F test/servercrash.test 816c132b26af008067cab2913783f67006d4003e3988f3f3ee1075742f6e0a6c
+F test/servercrash.test 1cbd2f98cadee2d8d42ed85ad76fbcf48958fedd537c82221838cd9bc6899dae
 F test/session.test 78fa2365e93d3663a6e933f86e7afc395adf18be
 F test/shared.test 1da9dbad400cee0d93f252ccf76e1ae007a63746
 F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879
@@ -1657,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 04e0cb571dbed00e269a890a755e252d7e8204d6d2ed5a7cfdb3d78d990a2876 39543903282409ad3f139f8a0bb376661e7595a33af4f647945b1513a028ccb4
-R 57d1fe32a9abbe4cff206d1840c9818b
+P 380a7b7a458b212b0463c8e128c289cc70f2fdb9b9a63b91fc7542c120eb9c10
+R 70f8b6d0240578b43d191efd6f3e839e
 U dan
-Z 63f3cfe0c27f676329bb592352b611d7
+Z 25c9fec4aa243def4a00985f9de58c4b

diff --git a/manifest.uuid b/manifest.uuid
index b82571434ef655d130b96cc7c0d720e2d9b22464..acbcb6548b000258f13e628fb9153491e2c5dfd3 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-380a7b7a458b212b0463c8e128c289cc70f2fdb9b9a63b91fc7542c120eb9c10
\ No newline at end of file
+a8115f95e80cc90c095fdd0a151da51f4d3ee427defdc34780e977585a68776d
\ No newline at end of file

diff --git a/src/server.c b/src/server.c
index beeffa5e3a6db89810c8b926bd36db61a9934118..3af351e6be255b39aa3e61e970342ccc7541c83a 100644 (file)
--- a/src/server.c
+++ b/src/server.c
@@ -214,19 +214,48 @@ static int serverFindDatabase(Server *pNew, i64 *aFileId){
   return rc;
 }
 
+static int serverClientRollback(Server *p, int iClient){
+  ServerDb *pDb = p->pDb;
+  ServerJournal *pJ = &pDb->aJrnl[iClient];
+  int bExist = 1;
+  int rc = SQLITE_OK;
+
+  if( pJ->jfd->pMethods==0 ){
+    bExist = 0;
+    rc = sqlite3OsAccess(pDb->pVfs, pJ->zJournal, SQLITE_ACCESS_EXISTS,&bExist);
+    if( bExist && rc==SQLITE_OK ){
+      int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL;
+      rc = sqlite3OsOpen(pDb->pVfs, pJ->zJournal, pJ->jfd, flags, &flags);
+    }
+  }
+
+  if( bExist && rc==SQLITE_OK ){
+    rc = sqlite3PagerRollbackJournal(p->pPager, pJ->jfd);
+  }
+  return rc;
+}
+
+
 /*
 ** Free all resources allocated by serverInitDatabase() associated with the
 ** object passed as the only argument.
 */
 static void serverShutdownDatabase(
-  ServerDb *pDb, 
+  Server *p, 
   sqlite3_file *dbfd, 
   int bDelete
 ){
+  ServerDb *pDb = p->pDb;
   int i;
 
   for(i=0; i<HMA_MAX_TRANSACTIONID; i++){
     ServerJournal *pJ = &pDb->aJrnl[i];
+
+    if( pDb->pServerShm && bDelete ){
+      int rc = serverClientRollback(p, i);
+      if( rc!=SQLITE_OK ) bDelete = 0;
+    }
+
     if( pJ->jfd ){
       sqlite3OsClose(pJ->jfd);
       if( bDelete ) sqlite3OsDelete(pDb->pVfs, pJ->zJournal, 0);
@@ -252,6 +281,24 @@ static void serverShutdownDatabase(
   pDb->bInit = 0;
 }
 
+static void serverClientUnlock(Server *p, int iClient){
+  ServerDb *pDb = p->pDb;
+  int i;
+
+  assert( pDb->pServerShm );
+  for(i=0; i<HMA_PAGELOCK_SLOTS; i++){
+    u32 *pSlot = &pDb->aSlot[i];
+    while( 1 ){
+      u32 o = *pSlot;
+      u32 n = o & ~((u32)1 << iClient);
+      if( slotGetWriter(n)==iClient ){
+        n -= ((iClient + 1) << HMA_MAX_TRANSACTIONID);
+      }
+      if( o==n || serverCompareAndSwap(pSlot, o, n) ) break;
+    }
+  }
+}
+
 /*
 ** This function is called when the very first connection to a database
 ** is established. It is responsible for rolling back any hot journal
@@ -308,15 +355,7 @@ static int serverInitDatabase(Server *pNew, int eServer){
       }
 
       if( bRollback ){
-        int bExist = 0;
-        rc = sqlite3OsAccess(pVfs, pJ->zJournal, SQLITE_ACCESS_EXISTS, &bExist);
-        if( rc==SQLITE_OK && bExist ){
-          int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL;
-          rc = sqlite3OsOpen(pVfs, pJ->zJournal, pJ->jfd, flags, &flags);
-          if( rc==SQLITE_OK ){
-            rc = sqlite3PagerRollbackJournal(pNew->pPager, pJ->jfd);
-          }
-        }
+        rc = serverClientRollback(pNew, i);
       }
     }
   }
@@ -333,7 +372,7 @@ static int serverInitDatabase(Server *pNew, int eServer){
   if( rc==SQLITE_OK ){
     pDb->bInit = 1;
   }else{
-    serverShutdownDatabase(pNew->pDb, dbfd, eServer==1);
+    serverShutdownDatabase(pNew, dbfd, eServer==1);
   }
   return rc;
 }
@@ -364,10 +403,10 @@ void sqlite3ServerDisconnect(Server *p, sqlite3_file *dbfd){
   /* In a multi-process setup, release the lock on the client slot and
   ** clear the bit in the ServerDb.transmask bitmask. */
   if( pDb->pServerShm && p->iTransId>=0 ){
+    serverFcntlLock(p, p->iTransId, 0);
     sqlite3_mutex_enter(pDb->mutex);
     pDb->transmask &= ~((u32)1 << p->iTransId);
     sqlite3_mutex_leave(pDb->mutex);
-    serverFcntlLock(p, p->iTransId, 0);
   }
 
   serverEnterMutex();
@@ -391,7 +430,7 @@ void sqlite3ServerDisconnect(Server *p, sqlite3_file *dbfd){
     }else{
       bDelete = 1;
     }
-    serverShutdownDatabase(pDb, dbfd, bDelete);
+    serverShutdownDatabase(p, dbfd, bDelete);
 
     for(pp=&g_server.pDb; *pp!=pDb; pp=&((*pp)->pNext));
     *pp = pDb->pNext;
@@ -454,6 +493,16 @@ int sqlite3ServerConnect(
           }
         }
         sqlite3_mutex_leave(pNew->pDb->mutex);
+
+        /* If this is a multi-process database, it may be that the previous
+        ** user of client-id pNew->iTransId crashed mid transaction. Roll
+        ** back any hot journal file in the file-system and release 
+        ** page locks held by any crashed process. TODO: The call to
+        ** serverClientUnlock() is expensive.  */
+        if( rc==SQLITE_OK && pDb->pServerShm ){
+          serverClientUnlock(pNew, pNew->iTransId);
+          rc = serverClientRollback(pNew, pNew->iTransId);
+        }
       }
     }else{
       rc = SQLITE_NOMEM_BKPT;
@@ -719,6 +768,28 @@ int sqlite3ServerReleaseWriteLocks(Server *p){
   return rc;
 }
 
+static int serverCheckClient(Server *p, int iClient){
+  ServerDb *pDb = p->pDb;
+  int rc = SQLITE_BUSY_DEADLOCK;
+  if( pDb->pServerShm && 0==(pDb->transmask & (1 << iClient)) ){
+
+    /* At this point it is know that client iClient, if it exists, resides in
+    ** some other process. Check that it is still alive by attempting to lock
+    ** its client slot. If the client is not alive, clear all its locks and
+    ** rollback its journal.  */
+    rc = serverFcntlLock(p, iClient, 1);
+    if( rc==SQLITE_OK ){
+      serverClientUnlock(p, iClient);
+      rc = serverClientRollback(p, iClient);
+      serverFcntlLock(p, iClient, 0);
+      pDb->transmask &= ~(1 << iClient);
+    }else if( rc==SQLITE_BUSY ){
+      rc = SQLITE_BUSY_DEADLOCK;
+    }
+  }
+  return rc;
+}
+
 /*
 ** Lock page pgno for reading (bWrite==0) or writing (bWrite==1).
 **
@@ -768,12 +839,18 @@ int sqlite3ServerLock(Server *p, Pgno pgno, int bWrite, int bBlock){
         bSkip = 1;
         break;
       }else if( iWriter>=0 ){
-        rc = SQLITE_BUSY_DEADLOCK;
+        rc = serverCheckClient(p, iWriter);
       }else if( bWrite ){
         if( (slotReaderMask(o) & ~(1 << p->iTransId))==0 ){
           n += ((p->iTransId + 1) << HMA_MAX_TRANSACTIONID);
         }else{
-          rc = SQLITE_BUSY_DEADLOCK;
+          int i;
+          for(i=0; i<HMA_MAX_TRANSACTIONID; i++){
+            if( o & (1 << i) ){
+              rc = serverCheckClient(p, i);
+              break;
+            }
+          }
         }
       }else{
         n |= (1 << p->iTransId);

diff --git a/test/permutations.test b/test/permutations.test
index 58199756006214c82aa2f66808f032602ae4d8b0..736c4ddedb572b215ad6bf95e9a2817c852b6e2c 100644 (file)
--- a/test/permutations.test
+++ b/test/permutations.test
@@ -278,7 +278,8 @@ test_suite "server" -prefix "" -description {
   All server-edition tests.
 } -files [
   test_set \
-      select1.test server2.test server3.test server4.test server5.test
+      select1.test server2.test server3.test server4.test server5.test \
+      servercrash.test
 ]
 
 test_suite "fts5-light" -prefix "" -description {

diff --git a/test/servercrash.test b/test/servercrash.test
index d7cc15f4ea9e3f7a1f9f6fb627717a7fed1a1a0b..5e18a923dc2dce9184cb97f8e61813d23f00fdb2 100644 (file)
--- a/test/servercrash.test
+++ b/test/servercrash.test
@@ -21,8 +21,15 @@ ifcapable !crashtest {
 }
 do_not_use_codec
 
+source $testdir/server_common.tcl
+return_if_no_server
+db close
+
+server_set_vfs unix
+server_reset_db
+
 do_execsql_test 1.0 {
-  PRAGMA page_siBlockze = 4096;
+  PRAGMA page_size = 4096;
   PRAGMA auto_vacuum = OFF;
   CREATE TABLE t1(a, b);
   CREATE TABLE t2(c, d);
@@ -64,5 +71,17 @@ for {set i 0} {$i < 10} {incr i} {
   db close
 }
 
+sqlite3 db test.db
+db eval {SELECT * FROM t1}
+for {set i 0} {$i < 10} {incr i} {
+  do_test 4.$i.1 {
+    crashsql -delay 1 -file test.db { INSERT INTO t1 VALUES(5, 6) }
+  } {1 {child process exited abnormally}}
+
+  db close
+  sqlite3 db test.db
+  do_execsql_test 4.$i.2 { SELECT * FROM t1 } {1 2 3 4}
+}
+
 finish_test