Enhance the WAL header sync so that it honors the various synchronous

pragmas, settings, and device characteristics.

FossilOrigin-Name: 9799241f7de952c4d1ea8bf6508b577d2b57a370

diff --git a/manifest b/manifest
index 8c212674dbbe23f163da80f4bb4499ba85906b32..bc2dd4cdeae7a0594b50b394b7ca105cb581b926 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Proposed\schanges\sthat\sensure\sthat\sthe\sWAL\sheader\sis\swritten\sprior\sto\sthe\nfirst\scommit\smark.
-D 2011-12-16T19:34:36.384
+C Enhance\sthe\sWAL\sheader\ssync\sso\sthat\sit\shonors\sthe\svarious\ssynchronous\npragmas,\ssettings,\sand\sdevice\scharacteristics.
+D 2011-12-16T21:26:26.087
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -168,7 +168,7 @@ F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04
 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440
 F src/os_unix.c 7dc7df10331942b139032328449a3723e051979e
 F src/os_win.c 197d23ce8a0dff748e766e034bf95ff756dd3884
-F src/pager.c d981f3bfcc0e4460537d983899620700ccf8f539
+F src/pager.c c7c32a1c279e0bbbde3578172985c41d4c5efc35
 F src/pager.h 5cd760857707529b403837d813d86b68938d6183
 F src/parse.y fabb2e7047417d840e6fdb3ef0988a86849a08ba
 F src/pcache.c 1fdd77978c1525d1ca4b9ef48eb80abca710cb4c
@@ -250,8 +250,8 @@ F src/vdbemem.c 2fc78b3e0fabcc1eaa23cd79dd2e30e6dcfe1e56
 F src/vdbesort.c 468d43c057063e54da4f1988b38b4f46d60e7790
 F src/vdbetrace.c d6e50e04e1ec498150e519058f617d91b8f5c843
 F src/vtab.c e9318d88feac85be8e27ee783ac8f5397933fc8a
-F src/wal.c a1157f289ef700ce9148d8448916799cb364c20f
-F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a
+F src/wal.c 20022b733ce06f77cc1d7f154da32987b57532ae
+F src/wal.h 42f8313f7aaf8913e2d1fdf7b47025c23491ea1d
 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
 F src/where.c af623942514571895818b9b7ae11db95ae3b3d88
 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823
@@ -902,8 +902,8 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
 F test/wal.test c743be787e60c1242fa6cdf73b410e64b2977e25
-F test/wal2.test ad6412596815f553cd30f271d291ab003092bc7e
-F test/wal3.test 18da4e65c30c43c646ad40e145e9a074e4062fc9
+F test/wal2.test 29e2cbe840582fc6efd0487b4f6337caed4b3e80
+F test/wal3.test 29a6e8843e5f5fd13f33cb0407d2923107020d32
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 1bbfaa316dc2a1d0d1fac3f4500c38a90055a41b
 F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3
@@ -984,10 +984,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P 09ccc4a1be7ba81890f10aac6623dd90dab4f990
-R e39509fa33546fba87ec45fa63df451b
-T *branch * wal-header-sync
-T *sym-wal-header-sync *
-T -sym-trunk *
+P 91d0437c0702904d27f0ef7b1b52d0797efe1826
+R 5f78dbf89e78a567743daf43caa5df96
 U drh
-Z 6469301a128bde2c8fea4c5b47d61847
+Z da5e946383fd5dfe4d9a2957cf126f26

diff --git a/manifest.uuid b/manifest.uuid
index cf227b205ff12c77d7f557d4ba1dffb111e05137..433b9f5a98bc34f27cb2c4dfbbc5636cf1ff3888 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-91d0437c0702904d27f0ef7b1b52d0797efe1826
\ No newline at end of file
+9799241f7de952c4d1ea8bf6508b577d2b57a370
\ No newline at end of file

diff --git a/src/pager.c b/src/pager.c
index f3069e52bad0a63568b2b71bc8e64c42491e9ca3..9db6ebd3f4b7047564d60ce19ae7012d7ca47076 100644 (file)
--- a/src/pager.c
+++ b/src/pager.c
@@ -616,6 +616,7 @@ struct Pager {
   u8 noSync;                  /* Do not sync the journal if true */
   u8 fullSync;                /* Do extra syncs of the journal for robustness */
   u8 ckptSyncFlags;           /* SYNC_NORMAL or SYNC_FULL for checkpoint */
+  u8 walSyncFlags;            /* SYNC_NORMAL or SYNC_FULL for wal writes */
   u8 syncFlags;               /* SYNC_NORMAL or SYNC_FULL otherwise */
   u8 tempFile;                /* zFilename is a temporary file */
   u8 readOnly;                /* True for a read-only database */
@@ -786,7 +787,7 @@ static int pagerUseWal(Pager *pPager){
 #else
 # define pagerUseWal(x) 0
 # define pagerRollbackWal(x) 0
-# define pagerWalFrames(v,w,x,y,z) 0
+# define pagerWalFrames(v,w,x,y) 0
 # define pagerOpenWalIfPresent(z) SQLITE_OK
 # define pagerBeginReadTransaction(z) SQLITE_OK
 #endif
@@ -2955,8 +2956,7 @@ static int pagerWalFrames(
   Pager *pPager,                  /* Pager object */
   PgHdr *pList,                   /* List of frames to log */
   Pgno nTruncate,                 /* Database size after this commit */
-  int isCommit,                   /* True if this is a commit */
-  int syncFlags                   /* Flags to pass to OsSync() (or 0) */
+  int isCommit                    /* True if this is a commit */
 ){
   int rc;                         /* Return code */
 #if defined(SQLITE_DEBUG) || defined(SQLITE_CHECK_PAGES)
@@ -2987,7 +2987,7 @@ static int pagerWalFrames(
 
   if( pList->pgno==1 ) pager_write_changecounter(pList);
   rc = sqlite3WalFrames(pPager->pWal, 
-      pPager->pageSize, pList, nTruncate, isCommit, syncFlags
+      pPager->pageSize, pList, nTruncate, isCommit, pPager->walSyncFlags
   );
   if( rc==SQLITE_OK && pPager->pBackup ){
     PgHdr *p;
@@ -3367,6 +3367,10 @@ void sqlite3PagerSetSafetyLevel(
     pPager->syncFlags = SQLITE_SYNC_NORMAL;
     pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL;
   }
+  pPager->walSyncFlags = pPager->syncFlags;
+  if( pPager->fullSync ){
+    pPager->walSyncFlags |= WAL_SYNC_TRANSACTIONS;
+  }
 }
 #endif
 
@@ -4194,7 +4198,7 @@ static int pagerStress(void *p, PgHdr *pPg){
       rc = subjournalPage(pPg); 
     }
     if( rc==SQLITE_OK ){
-      rc = pagerWalFrames(pPager, pPg, 0, 0, 0);
+      rc = pagerWalFrames(pPager, pPg, 0, 0);
     }
   }else{
   
@@ -4533,9 +4537,17 @@ int sqlite3PagerOpen(
   pPager->readOnly = (u8)readOnly;
   assert( useJournal || pPager->tempFile );
   pPager->noSync = pPager->tempFile;
-  pPager->fullSync = pPager->noSync ?0:1;
-  pPager->syncFlags = pPager->noSync ? 0 : SQLITE_SYNC_NORMAL;
-  pPager->ckptSyncFlags = pPager->syncFlags;
+  if( pPager->noSync ){
+    assert( pPager->fullSync==0 );
+    assert( pPager->syncFlags==0 );
+    assert( pPager->walSyncFlags==0 );
+    assert( pPager->ckptSyncFlags==0 );
+  }else{
+    pPager->fullSync = 1;
+    pPager->syncFlags = SQLITE_SYNC_NORMAL;
+    pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS;
+    pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL;
+  }
   /* pPager->pFirst = 0; */
   /* pPager->pFirstSynced = 0; */
   /* pPager->pLast = 0; */
@@ -5765,9 +5777,7 @@ int sqlite3PagerCommitPhaseOne(
       }
       assert( rc==SQLITE_OK );
       if( ALWAYS(pList) ){
-        rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1, 
-            (pPager->fullSync ? pPager->syncFlags : 0)
-        );
+        rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1);
       }
       sqlite3PagerUnref(pPageOne);
       if( rc==SQLITE_OK ){

diff --git a/src/wal.c b/src/wal.c
index 915e014aaa374aec7f36fafc09aec378c01dc658..34ddb1b1e2eec63eb348a2758f08a1ce1e240e17 100644 (file)
--- a/src/wal.c
+++ b/src/wal.c
@@ -418,11 +418,13 @@ struct Wal {
   volatile u32 **apWiData;   /* Pointer to wal-index content in memory */
   u32 szPage;                /* Database page size */
   i16 readLock;              /* Which read lock is being held.  -1 for none */
+  u8 syncFlags;              /* Flags to use to sync header writes */
   u8 exclusiveMode;          /* Non-zero if connection is in exclusive mode */
   u8 writeLock;              /* True if in a write transaction */
   u8 ckptLock;               /* True if holding a checkpoint lock */
   u8 readOnly;               /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */
   u8 truncateOnCommit;       /* True to truncate WAL file on commit */
+  u8 noSyncHeader;           /* Avoid WAL header fsyncs if true */
   WalIndexHdr hdr;           /* Wal-index header for current transaction */
   const char *zWalName;      /* Name of WAL file */
   u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
@@ -1297,6 +1299,8 @@ int sqlite3WalOpen(
     sqlite3OsClose(pRet->pWalFd);
     sqlite3_free(pRet);
   }else{
+    int iDC = sqlite3OsDeviceCharacteristics(pRet->pWalFd);
+    if( iDC & SQLITE_IOCAP_SEQUENTIAL ){ pRet->noSyncHeader = 1; }
     *ppWal = pRet;
     WALTRACE(("WAL%d: opened\n", pRet));
   }
@@ -2633,7 +2637,10 @@ static int walWriteToLog(
   sqlite3_int64 iOffset      /* Start writing at this offset */
 ){
   int rc;
-  if( iOffset>=pWal->szFirstBlock || iOffset+iAmt<pWal->szFirstBlock ){
+  if( iOffset>=pWal->szFirstBlock
+   || iOffset+iAmt<pWal->szFirstBlock
+   || pWal->syncFlags==0
+  ){
     /* The common and fast case.  Just write the data. */
     rc = sqlite3OsWrite(pWal->pWalFd, pContent, iAmt, iOffset);
   }else{
@@ -2643,7 +2650,8 @@ static int walWriteToLog(
     assert( iFirstAmt>0 && iFirstAmt<iAmt );
     rc = sqlite3OsWrite(pWal->pWalFd, pContent, iFirstAmt, iOffset);
     if( rc ) return rc;
-    rc = sqlite3OsSync(pWal->pWalFd, SQLITE_SYNC_NORMAL);
+    assert( pWal->syncFlags & (SQLITE_SYNC_NORMAL|SQLITE_SYNC_FULL) );
+    rc = sqlite3OsSync(pWal->pWalFd, pWal->syncFlags);
     if( rc ) return rc;
     pContent = (void*)(iFirstAmt + (char*)pContent);
     rc = sqlite3OsWrite(pWal->pWalFd, pContent,
@@ -2721,9 +2729,15 @@ int sqlite3WalFrames(
   }
   assert( (int)pWal->szPage==szPage );
 
-  /* The size of the block containing the WAL header */
-  pWal->szFirstBlock = sqlite3OsSectorSize(pWal->pWalFd);
-  if( szPage>pWal->szFirstBlock ) pWal->szFirstBlock = szPage;
+  /* Setup information needed to do the WAL header sync */
+  if( pWal->noSyncHeader ){
+    assert( pWal->szFirstBlock==0 );
+    assert( pWal->syncFlags==0 );
+  }else{
+    pWal->szFirstBlock = sqlite3OsSectorSize(pWal->pWalFd);
+    if( szPage>pWal->szFirstBlock ) pWal->szFirstBlock = szPage;
+    pWal->syncFlags = sync_flags & SQLITE_SYNC_MASK;
+  }
 
   /* Write the log file. */
   for(p=pList; p; p=p->pDirty){
@@ -2756,11 +2770,10 @@ int sqlite3WalFrames(
   }
 
   /* Sync the log file if the 'isSync' flag was specified. */
-  if( sync_flags ){
+  if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){
     i64 iSegment = sqlite3OsSectorSize(pWal->pWalFd);
     i64 iOffset = walFrameOffset(iFrame+1, szPage);
 
-    assert( isCommit );
     assert( iSegment>0 );
 
     iSegment = (((iOffset+iSegment-1)/iSegment) * iSegment);
@@ -2786,7 +2799,7 @@ int sqlite3WalFrames(
       iOffset += szPage;
     }
 
-    rc = sqlite3OsSync(pWal->pWalFd, sync_flags);
+    rc = sqlite3OsSync(pWal->pWalFd, sync_flags & SQLITE_SYNC_MASK);
   }
 
   if( isCommit && pWal->truncateOnCommit && pWal->mxWalSize>=0 ){

diff --git a/src/wal.h b/src/wal.h
index a62b23bbdc99f2551bcb8f8083541b0968113478..7cd84be9972838ba4008569ddc8ffc9e88b13196 100644 (file)
--- a/src/wal.h
+++ b/src/wal.h
@@ -87,6 +87,12 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData);
 /* Write a frame or frames to the log. */
 int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int);
 
+/* Additional values that can be added to the sync_flags argument of
+** sqlite3WalFrames():
+*/
+#define WAL_SYNC_TRANSACTIONS  0x20   /* Sync at the end of each transaction */
+#define SQLITE_SYNC_MASK       0x13   /* Mask off the SQLITE_SYNC_* values */
+
 /* Copy pages from the log to the database file */ 
 int sqlite3WalCheckpoint(
   Wal *pWal,                      /* Write-ahead log connection */

diff --git a/test/wal2.test b/test/wal2.test
index f4887065a56cacd3af017a331b6cb4485a3d016e..90ea9defed106718966562561c892b34892e40f2 100644 (file)
--- a/test/wal2.test
+++ b/test/wal2.test
@@ -1176,9 +1176,9 @@ if {$::tcl_platform(platform) == "unix"} {
 # Test that "PRAGMA checkpoint_fullsync" appears to be working.
 #
 foreach {tn sql reslist} {
-  1 { }                                 {8 0 3 0 5 0}
-  2 { PRAGMA checkpoint_fullfsync = 1 } {8 4 3 2 5 2}
-  3 { PRAGMA checkpoint_fullfsync = 0 } {8 0 3 0 5 0}
+  1 { }                                 {10 0 4 0 6 0}
+  2 { PRAGMA checkpoint_fullfsync = 1 } {10 4 4 2 6 2}
+  3 { PRAGMA checkpoint_fullfsync = 0 } {10 0 4 0 6 0}
 } {
   faultsim_delete_and_reopen
 
@@ -1192,12 +1192,12 @@ foreach {tn sql reslist} {
   do_execsql_test wal2-14.$tn.2 {
     PRAGMA wal_autocheckpoint = 10;
     CREATE TABLE t1(a, b);                -- 2 wal syncs
-    INSERT INTO t1 VALUES(1, 2);          -- 1 wal sync
+    INSERT INTO t1 VALUES(1, 2);          -- 2 wal sync
     PRAGMA wal_checkpoint;                -- 1 wal sync, 1 db sync
     BEGIN;
       INSERT INTO t1 VALUES(3, 4);
       INSERT INTO t1 VALUES(5, 6);
-    COMMIT;                               -- 1 wal sync
+    COMMIT;                               -- 2 wal sync
     PRAGMA wal_checkpoint;                -- 1 wal sync, 1 db sync
   } {10 0 5 5 0 2 2}
 
@@ -1233,22 +1233,22 @@ catch { db close }
 # PRAGMA fullfsync
 # PRAGMA synchronous
 #
-foreach {tn settings commit_sync ckpt_sync} {
-  1  {0 0 off}     {0 0}  {0 0}
-  2  {0 0 normal}  {0 0}  {2 0}
-  3  {0 0 full}    {1 0}  {2 0}
-
-  4  {0 1 off}     {0 0}  {0 0}
-  5  {0 1 normal}  {0 0}  {0 2}
-  6  {0 1 full}    {0 1}  {0 2}
-
-  7  {1 0 off}     {0 0}  {0 0}
-  8  {1 0 normal}  {0 0}  {0 2}
-  9  {1 0 full}    {1 0}  {0 2}
-
-  10 {1 1 off}     {0 0}  {0 0}
-  11 {1 1 normal}  {0 0}  {0 2}
-  12 {1 1 full}    {0 1}  {0 2}
+foreach {tn settings restart_sync commit_sync ckpt_sync} {
+  1  {0 0 off}     {0 0}  {0 0}  {0 0}
+  2  {0 0 normal}  {1 0}  {0 0}  {2 0}
+  3  {0 0 full}    {2 0}  {1 0}  {2 0}
+
+  4  {0 1 off}     {0 0}  {0 0}  {0 0}
+  5  {0 1 normal}  {0 1}  {0 0}  {0 2}
+  6  {0 1 full}    {0 2}  {0 1}  {0 2}
+
+  7  {1 0 off}     {0 0}  {0 0}  {0 0}
+  8  {1 0 normal}  {1 0}  {0 0}  {0 2}
+  9  {1 0 full}    {2 0}  {1 0}  {0 2}
+
+  10 {1 1 off}     {0 0}  {0 0}  {0 0}
+  11 {1 1 normal}  {0 1}  {0 0}  {0 2}
+  12 {1 1 full}    {0 2}  {0 1}  {0 2}
 } {
   forcedelete test.db
 
@@ -1262,27 +1262,35 @@ foreach {tn settings commit_sync ckpt_sync} {
   sqlite3 db test.db
   do_execsql_test 15.$tn.1 "
     CREATE TABLE t1(x);
+    PRAGMA wal_autocheckpoint = OFF;
     PRAGMA journal_mode = WAL;
     PRAGMA checkpoint_fullfsync = [lindex $settings 0];
     PRAGMA fullfsync = [lindex $settings 1];
     PRAGMA synchronous = [lindex $settings 2];
-  " {wal}
+  " {0 wal}
 
   do_test 15.$tn.2 {
     set sync(normal) 0
     set sync(full) 0
     execsql { INSERT INTO t1 VALUES('abc') }
     list $::sync(normal) $::sync(full)
-  } $commit_sync
+  } $restart_sync
 
   do_test 15.$tn.3 {
     set sync(normal) 0
     set sync(full) 0
-    execsql { INSERT INTO t1 VALUES('def') }
+    execsql { INSERT INTO t1 VALUES('abc') }
     list $::sync(normal) $::sync(full)
   } $commit_sync
 
   do_test 15.$tn.4 {
+    set sync(normal) 0
+    set sync(full) 0
+    execsql { INSERT INTO t1 VALUES('def') }
+    list $::sync(normal) $::sync(full)
+  } $commit_sync
+
+  do_test 15.$tn.5 {
     set sync(normal) 0
     set sync(full) 0
     execsql { PRAGMA wal_checkpoint }

diff --git a/test/wal3.test b/test/wal3.test
index ea5e70571bd55869717c4cd88c9ca65bacbdc1f7..82f7d72a97a7011e94f4a5cab8d34a6d07cd0d24 100644 (file)
--- a/test/wal3.test
+++ b/test/wal3.test
@@ -217,6 +217,7 @@ foreach {tn syncmode synccount} {
   
     execsql "PRAGMA synchronous = $syncmode"
     execsql { PRAGMA journal_mode = WAL }
+    execsql { CREATE TABLE filler(a,b,c); }
 
     set ::syncs [list]
     T filter xSync