Update the mechanism used to keep track of what kind of syncing to do for

WAL transaction commits and checkpoint operations.  Use the checkpoint-style
of syncing to sync the header of a new or restarted WAL file.

FossilOrigin-Name: bf65dae8d4297c57ac63228ccf0100f9fabf2fb600438c9f2e10a29c4b118168

diff --git a/manifest b/manifest
index e74b367e9705c3d3ae15012922d8113c4101fb53..cfd5eb0e6473f97282aaaeda7807ad1ee1b4c535 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sincorrect\shyperlink\sin\sa\scomment.
-D 2017-08-24T20:54:42.754
+C Update\sthe\smechanism\sused\sto\skeep\strack\sof\swhat\skind\sof\ssyncing\sto\sdo\sfor\nWAL\stransaction\scommits\sand\scheckpoint\soperations.\s\sUse\sthe\scheckpoint-style\nof\ssyncing\sto\ssync\sthe\sheader\sof\sa\snew\sor\srestarted\sWAL\sfile.
+D 2017-08-25T01:14:43.498
 F Makefile.in c644bbe8ebe4aae82ad6783eae6b6beea4c727b99ff97568b847ced5e2ac7afb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 25b154da7f0b3d4924f27378c1f8d006285b80811f1ccf3ed953dbebf6282136
@@ -436,14 +436,14 @@ F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23
 F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d
 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7
-F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24
+F src/os.c 93e0979b9b55df29c0c4923f73b48e9d3fe728f01dd8ed4f6a9d2f1d79779bc8
 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c 489aa972ccc34f7b4770b891694b32101c59ddd4be4ef0ddd9a4da58c145c1a6
 F src/os_win.c 964165b66cde03abc72fe948198b01be608436894732eadb94c8720d2467f223
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c c1dc0609f04a0659519bb2b8ca1440a64b0ad82b6c2afd675f1a50f6c918321a
+F src/pager.c bf51378c57c8e05d7f4d7bb9861f403a2e40cde82e25513401216d1ed30bc3e5
 F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa
 F src/parse.y 52ef3cecd0934e9da4a45b585883a03243ad615d338ad94f44501a05891dcdfa
 F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870
@@ -535,8 +535,8 @@ F src/vdbesort.c fea2bea25f5e9ccd91e0760d7359f0365f9fba1aaeac7216c71cad78765f58e
 F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834
 F src/vtab.c f1d5c23132fb0247af3e86146404112283ddedb6c518de0d4edc91cfb36970ef
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c 4c7c13a3c0d92b0bed157ac234679e4874afcb1990af33510542a0f35f9264b8
-F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71
+F src/wal.c 839db09792fead5052bb35e533fa485e134913d547d05b5f42e537b73e63f07a
+F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6
 F src/where.c ad558533d6fe6578857635218633aa241e2428835763c46801be9e68d6ec0701
 F src/whereInt.h 93bb90b77d39901eda31b44d8e90da1351193ccfe96876f89b58a93a33b84c3d
@@ -1486,7 +1486,7 @@ F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c37
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad
 F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477
-F test/wal2.test 56b0bc95b8693a0be294f8d210c49025dd094bd7
+F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef
 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9
@@ -1650,7 +1650,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7ec72314d6b66e4969833194fd032f01963bb861a7bb30d060acdb82dc5d62fe
-R 0a9008b1e381e229108d828896139715
+P 25292b9a4eb5efc7db551da0b3c9cfd7e79da1594ee14729de44090e188c2b2a
+R 0c45c0346ff1a3d7a9dbb07f83f4f723
+T *branch * wal-sync-refactor
+T *sym-wal-sync-refactor *
+T -sym-trunk *
 U drh
-Z 0f9a691a1b4c4aa8a5eacf3d9e1dd571
+Z 5b1634e4e5e7d2e25acb68caf19e0213

diff --git a/manifest.uuid b/manifest.uuid
index 404de581997a3697d386276098d2c02bb91eb3b8..cfce901f7c50f3ef649c909ef1f51c0a8918faac 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-25292b9a4eb5efc7db551da0b3c9cfd7e79da1594ee14729de44090e188c2b2a
\ No newline at end of file
+bf65dae8d4297c57ac63228ccf0100f9fabf2fb600438c9f2e10a29c4b118168
\ No newline at end of file

diff --git a/src/os.c b/src/os.c
index 5cf001479468d747bcf5ca32406aeb4f594d30e9..18277df3568dbe4a46f7e6b36fdf49faf1c9ed8b 100644 (file)
--- a/src/os.c
+++ b/src/os.c
@@ -98,7 +98,7 @@ int sqlite3OsTruncate(sqlite3_file *id, i64 size){
 }
 int sqlite3OsSync(sqlite3_file *id, int flags){
   DO_OS_MALLOC_TEST(id);
-  return id->pMethods->xSync(id, flags);
+  return flags ? id->pMethods->xSync(id, flags) : SQLITE_OK;
 }
 int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){
   DO_OS_MALLOC_TEST(id);

diff --git a/src/pager.c b/src/pager.c
index 5986afdae10e78d25bb03d316a3faf034d4db5d5..2e5977fe43e7b741ba2301276bd824edce0d21db 100644 (file)
--- a/src/pager.c
+++ b/src/pager.c
@@ -616,6 +616,18 @@ struct PagerSavepoint {
 **   is set to zero in all other states. In PAGER_ERROR state, Pager.errCode 
 **   is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX 
 **   sub-codes.
+**
+** syncFlags, walSyncFlags
+**
+**   syncFlags is either SQLITE_SYNC_NORMAL (0x02) or SQLITE_SYNC_FULL (0x03).
+**   syncFlags is used for rollback mode.  walSyncFlags is used for WAL mode
+**   and contains the flags used to sync the checkpoint operations in the
+**   lower two bits, and sync flags used for transaction commits in the WAL
+**   file in bits 0x04 and 0x08.  In other words, to get the correct sync flags
+**   for checkpoint operations, use (walSyncFlags&0x03) and to get the correct
+**   sync flags for transaction commit, use ((walSyncFlags>>2)&0x03).  Note
+**   that with synchronous=NORMAL in WAL mode, transaction commit is not synced
+**   meaning that the 0x04 and 0x08 bits are both zero.
 */
 struct Pager {
   sqlite3_vfs *pVfs;          /* OS functions to use for IO */
@@ -625,9 +637,8 @@ struct Pager {
   u8 noSync;                  /* Do not sync the journal if true */
   u8 fullSync;                /* Do extra syncs of the journal for robustness */
   u8 extraSync;               /* sync directory after journal delete */
-  u8 ckptSyncFlags;           /* SYNC_NORMAL or SYNC_FULL for checkpoint */
-  u8 walSyncFlags;            /* SYNC_NORMAL or SYNC_FULL for wal writes */
   u8 syncFlags;               /* SYNC_NORMAL or SYNC_FULL otherwise */
+  u8 walSyncFlags;            /* See description above */
   u8 tempFile;                /* zFilename is a temporary or immutable file */
   u8 noLock;                  /* Do not lock (except in WAL mode) */
   u8 readOnly;                /* True for a read-only database */
@@ -3598,20 +3609,17 @@ void sqlite3PagerSetFlags(
   }
   if( pPager->noSync ){
     pPager->syncFlags = 0;
-    pPager->ckptSyncFlags = 0;
   }else if( pgFlags & PAGER_FULLFSYNC ){
     pPager->syncFlags = SQLITE_SYNC_FULL;
-    pPager->ckptSyncFlags = SQLITE_SYNC_FULL;
-  }else if( pgFlags & PAGER_CKPT_FULLFSYNC ){
-    pPager->syncFlags = SQLITE_SYNC_NORMAL;
-    pPager->ckptSyncFlags = SQLITE_SYNC_FULL;
   }else{
     pPager->syncFlags = SQLITE_SYNC_NORMAL;
-    pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL;
   }
-  pPager->walSyncFlags = pPager->syncFlags;
+  pPager->walSyncFlags = (pPager->syncFlags<<2);
   if( pPager->fullSync ){
-    pPager->walSyncFlags |= WAL_SYNC_TRANSACTIONS;
+    pPager->walSyncFlags |= pPager->syncFlags;
+  }
+  if( (pgFlags & PAGER_CKPT_FULLFSYNC) && !pPager->noSync ){
+    pPager->walSyncFlags |= (SQLITE_SYNC_FULL<<2);
   }
   if( pgFlags & PAGER_CACHESPILL ){
     pPager->doNotSpill &= ~SPILLFLAG_OFF;
@@ -4110,7 +4118,7 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){
   pPager->exclusiveMode = 0;
 #ifndef SQLITE_OMIT_WAL
   assert( db || pPager->pWal==0 );
-  sqlite3WalClose(pPager->pWal, db, pPager->ckptSyncFlags, pPager->pageSize,
+  sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,
       (db && (db->flags & SQLITE_NoCkptOnClose) ? 0 : pTmp)
   );
   pPager->pWal = 0;
@@ -4918,13 +4926,11 @@ act_like_temp_file:
     assert( pPager->extraSync==0 );
     assert( pPager->syncFlags==0 );
     assert( pPager->walSyncFlags==0 );
-    assert( pPager->ckptSyncFlags==0 );
   }else{
     pPager->fullSync = 1;
     pPager->extraSync = 0;
     pPager->syncFlags = SQLITE_SYNC_NORMAL;
-    pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS;
-    pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL;
+    pPager->walSyncFlags = SQLITE_SYNC_NORMAL | (SQLITE_SYNC_NORMAL<<2);
   }
   /* pPager->pFirst = 0; */
   /* pPager->pFirstSynced = 0; */
@@ -7372,7 +7378,7 @@ int sqlite3PagerCheckpoint(
     rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode,
         (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler),
         pPager->pBusyHandlerArg,
-        pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace,
+        pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace,
         pnLog, pnCkpt
     );
   }
@@ -7529,7 +7535,7 @@ int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){
   if( rc==SQLITE_OK && pPager->pWal ){
     rc = pagerExclusiveLock(pPager);
     if( rc==SQLITE_OK ){
-      rc = sqlite3WalClose(pPager->pWal, db, pPager->ckptSyncFlags,
+      rc = sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags,
                            pPager->pageSize, (u8*)pPager->pTmpSpace);
       pPager->pWal = 0;
       pagerFixMaplimit(pPager);

diff --git a/src/wal.c b/src/wal.c
index 45f1bc8c84178a1be99d3443156cdf73141907fb..9930b84421bceaf572619617215fa5d767359647 100644 (file)
--- a/src/wal.c
+++ b/src/wal.c
@@ -1798,9 +1798,7 @@ static int walCheckpoint(
       pInfo->nBackfillAttempted = mxSafeFrame;
 
       /* Sync the WAL to disk */
-      if( sync_flags ){
-        rc = sqlite3OsSync(pWal->pWalFd, sync_flags);
-      }
+      rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags));
 
       /* If the database may grow as a result of this checkpoint, hint
       ** about the eventual size of the db file to the VFS layer.
@@ -1841,8 +1839,8 @@ static int walCheckpoint(
           i64 szDb = pWal->hdr.nPage*(i64)szPage;
           testcase( IS_BIG_INT(szDb) );
           rc = sqlite3OsTruncate(pWal->pDbFd, szDb);
-          if( rc==SQLITE_OK && sync_flags ){
-            rc = sqlite3OsSync(pWal->pDbFd, sync_flags);
+          if( rc==SQLITE_OK ){
+            rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags));
           }
         }
         if( rc==SQLITE_OK ){
@@ -2948,8 +2946,8 @@ static int walWriteToLog(
     iOffset += iFirstAmt;
     iAmt -= iFirstAmt;
     pContent = (void*)(iFirstAmt + (char*)pContent);
-    assert( p->syncFlags & (SQLITE_SYNC_NORMAL|SQLITE_SYNC_FULL) );
-    rc = sqlite3OsSync(p->pFd, p->syncFlags & SQLITE_SYNC_MASK);
+    assert( WAL_SYNC_FLAGS(p->syncFlags)!=0 );
+    rc = sqlite3OsSync(p->pFd, WAL_SYNC_FLAGS(p->syncFlags));
     if( iAmt==0 || rc ) return rc;
   }
   rc = sqlite3OsWrite(p->pFd, pContent, iAmt, iOffset);
@@ -3121,8 +3119,8 @@ int sqlite3WalFrames(
     **
     **     https://sqlite.org/src/info/ff5be73dee
     */
-    if( pWal->syncHeader && sync_flags ){
-      rc = sqlite3OsSync(pWal->pWalFd, sync_flags & SQLITE_SYNC_MASK);
+    if( pWal->syncHeader ){
+      rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags));
       if( rc ) return rc;
     }
   }
@@ -3197,7 +3195,7 @@ int sqlite3WalFrames(
   ** sector boundary is synced; the part of the last frame that extends
   ** past the sector boundary is written after the sync.
   */
-  if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){
+  if( isCommit && WAL_SYNC_FLAGS(sync_flags)!=0 ){
     int bSync = 1;
     if( pWal->padToSectorBoundary ){
       int sectorSize = sqlite3SectorSize(pWal->pWalFd);
@@ -3213,7 +3211,7 @@ int sqlite3WalFrames(
     }
     if( bSync ){
       assert( rc==SQLITE_OK );
-      rc = sqlite3OsSync(w.pFd, sync_flags & SQLITE_SYNC_MASK);
+      rc = sqlite3OsSync(w.pFd, WAL_SYNC_FLAGS(sync_flags));
     }
   }
 

diff --git a/src/wal.h b/src/wal.h
index 4f6d01dad6291da039036918a3b4c31296ebc89c..d97300a684762092752d0c278916927ba9fd290e 100644 (file)
--- a/src/wal.h
+++ b/src/wal.h
@@ -19,11 +19,11 @@
 
 #include "sqliteInt.h"
 
-/* Additional values that can be added to the sync_flags argument of
-** sqlite3WalFrames():
+/* Macros for extracting appropriate sync flags for either transaction
+** commits (WAL_SYNC_FLAGS(X)) or for checkpoint ops (CKPT_SYNC_FLAGS(X)):
 */
-#define WAL_SYNC_TRANSACTIONS  0x20   /* Sync at the end of each transaction */
-#define SQLITE_SYNC_MASK       0x13   /* Mask off the SQLITE_SYNC_* values */
+#define WAL_SYNC_FLAGS(X)   ((X)&0x03)
+#define CKPT_SYNC_FLAGS(X)  (((X)>>2)&0x03)
 
 #ifdef SQLITE_OMIT_WAL
 # define sqlite3WalOpen(x,y,z)                   0

diff --git a/test/wal2.test b/test/wal2.test
index 0b15b1546170beb339ab3d101553d9637ef5ad8f..bc170ad3fff61e3feab468d19b8ae959486e81a4 100644 (file)
--- a/test/wal2.test
+++ b/test/wal2.test
@@ -1191,7 +1191,7 @@ if {$::tcl_platform(platform) == "unix"} {
 #
 foreach {tn sql reslist} {
   1 { }                                 {10 0 4 0 6 0}
-  2 { PRAGMA checkpoint_fullfsync = 1 } {10 4 4 2 6 2}
+  2 { PRAGMA checkpoint_fullfsync = 1 } {10 6 4 3 6 3}
   3 { PRAGMA checkpoint_fullfsync = 0 } {10 0 4 0 6 0}
 } {
   ifcapable default_ckptfullfsync {
@@ -1261,8 +1261,8 @@ foreach {tn settings restart_sync commit_sync ckpt_sync} {
   6  {0 1 full}    {0 2}  {0 1}  {0 2}
 
   7  {1 0 off}     {0 0}  {0 0}  {0 0}
-  8  {1 0 normal}  {1 0}  {0 0}  {0 2}
-  9  {1 0 full}    {2 0}  {1 0}  {0 2}
+  8  {1 0 normal}  {0 1}  {0 0}  {0 2}
+  9  {1 0 full}    {1 1}  {1 0}  {0 2}
 
   10 {1 1 off}     {0 0}  {0 0}  {0 0}
   11 {1 1 normal}  {0 1}  {0 0}  {0 2}