From: dan <dan@noemail.net>
Date: Thu, 29 Jan 2015 19:12:12 +0000 (+0000)
Subject: Ensure that "PRAGMA wal_checkpoint = TRUNCATE|FULL|RESTART" block on other connection... 
X-Git-Tag: version-3.8.9~147
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=976b003344df8b2d97d0e587c7a1c4256341439c;p=thirdparty%2Fsqlite.git

Ensure that "PRAGMA wal_checkpoint = TRUNCATE|FULL|RESTART" block on other connections and truncate the database file as required even if the entire wal file has already been checkpointed.

FossilOrigin-Name: 53429689d4fcf472edbc89cc50b5e69ba3270634
---

diff --git a/manifest b/manifest
index ca90d2b9bd..1463f0f1c5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Split\sup\sthe\sSRC\svariable\sin\sMakefile.msc\sto\savoid\sover-long\scmd.exe\scommands\swhen\sTOP\sis\sset\sto\sa\slong\spathname.
-D 2015-01-29T18:38:05.899
+C Ensure\sthat\s"PRAGMA\swal_checkpoint\s=\sTRUNCATE|FULL|RESTART"\sblock\son\sother\sconnections\sand\struncate\sthe\sdatabase\sfile\sas\srequired\seven\sif\sthe\sentire\swal\sfile\shas\salready\sbeen\scheckpointed.
+D 2015-01-29T19:12:12.112
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e
 F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770
 F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660
-F src/main.c 05bf368c934cc73d02906030846eb4d1818c10f7
+F src/main.c 341fcc5601a8ca84389fa32bcf5a857f65af8dd0
 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c abe6ee469b6c5a35c7f22bfeb9c9bac664a1c987
@@ -302,7 +302,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f
 F src/vdbesort.c 6d64c5448b64851b99931ede980addc3af70d5e2
 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010
 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9
-F src/wal.c 85353539f2d9d0c91ebd057c32525b1e1aa3335e
+F src/wal.c 39303f2c9db02a4e422cd8eb2c8760420c6a51fe
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804
 F src/where.c d46de821bc604a4fd36fa3928c086950e91aafb1
@@ -1126,7 +1126,7 @@ F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc
 F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada
 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
-F test/wal5.test 11b8658dd4d5448f4604124bebd9b68be5bc3e66
+F test/wal5.test dba8f5f5de95178bc40521d6edf153b2e2829917
 F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e
 F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8
 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd
@@ -1237,7 +1237,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 85dc12625d300fe48f3c096f54ebcb8b6ef4e30a
-R 30ab1d1acabeef9590cca414845b1919
-U drh
-Z 4d0075f03968634835018469d0d0c906
+P 7d70ac65c16f08832a1f0fc4dec0f62a17cbcc85
+R 73adf612681872a358f9dbc8b22308b4
+U dan
+Z 0d0ede6d5c4b5d769c9edb317c2eb34b
diff --git a/manifest.uuid b/manifest.uuid
index d4f0f59f06..ee7430a6bc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7d70ac65c16f08832a1f0fc4dec0f62a17cbcc85
\ No newline at end of file
+53429689d4fcf472edbc89cc50b5e69ba3270634
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 8cf16b001a..5aa37fa492 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1966,6 +1966,7 @@ int sqlite3_wal_checkpoint_v2(
     rc = SQLITE_ERROR;
     sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb);
   }else{
+    db->busyHandler.nBusy = 0;
     rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt);
     sqlite3Error(db, rc);
   }
diff --git a/src/wal.c b/src/wal.c
index f2738a6727..71f4a3d452 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -1694,7 +1694,7 @@ static int walCheckpoint(
   int sync_flags,                 /* Flags for OsSync() (or 0) */
   u8 *zBuf                        /* Temporary buffer to use */
 ){
-  int rc;                         /* Return code */
+  int rc = SQLITE_OK;             /* Return code */
   int szPage;                     /* Database page-size */
   WalIterator *pIter = 0;         /* Wal iterator context */
   u32 iDbpage = 0;                /* Next database page to write */
@@ -1708,104 +1708,107 @@ static int walCheckpoint(
   testcase( szPage<=32768 );
   testcase( szPage>=65536 );
   pInfo = walCkptInfo(pWal);
-  if( pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK;
+  if( pInfo->nBackfill<pWal->hdr.mxFrame ){
 
-  /* Allocate the iterator */
-  rc = walIteratorInit(pWal, &pIter);
-  if( rc!=SQLITE_OK ){
-    return rc;
-  }
-  assert( pIter );
+    /* Allocate the iterator */
+    rc = walIteratorInit(pWal, &pIter);
+    if( rc!=SQLITE_OK ){
+      return rc;
+    }
+    assert( pIter );
 
-  /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked
-  ** in the SQLITE_CHECKPOINT_PASSIVE mode. */
-  assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 );
+    /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked
+    ** in the SQLITE_CHECKPOINT_PASSIVE mode. */
+    assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 );
 
-  /* Compute in mxSafeFrame the index of the last frame of the WAL that is
-  ** safe to write into the database.  Frames beyond mxSafeFrame might
-  ** overwrite database pages that are in use by active readers and thus
-  ** cannot be backfilled from the WAL.
-  */
-  mxSafeFrame = pWal->hdr.mxFrame;
-  mxPage = pWal->hdr.nPage;
-  for(i=1; i<WAL_NREADER; i++){
-    u32 y = pInfo->aReadMark[i];
-    if( mxSafeFrame>y ){
-      assert( y<=pWal->hdr.mxFrame );
-      rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1);
-      if( rc==SQLITE_OK ){
-        pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED);
-        walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
-      }else if( rc==SQLITE_BUSY ){
-        mxSafeFrame = y;
-        xBusy = 0;
-      }else{
-        goto walcheckpoint_out;
+    /* Compute in mxSafeFrame the index of the last frame of the WAL that is
+    ** safe to write into the database.  Frames beyond mxSafeFrame might
+    ** overwrite database pages that are in use by active readers and thus
+    ** cannot be backfilled from the WAL.
+    */
+    mxSafeFrame = pWal->hdr.mxFrame;
+    mxPage = pWal->hdr.nPage;
+    for(i=1; i<WAL_NREADER; i++){
+      u32 y = pInfo->aReadMark[i];
+      if( mxSafeFrame>y ){
+        assert( y<=pWal->hdr.mxFrame );
+        rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1);
+        if( rc==SQLITE_OK ){
+          pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED);
+          walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
+        }else if( rc==SQLITE_BUSY ){
+          mxSafeFrame = y;
+          xBusy = 0;
+        }else{
+          goto walcheckpoint_out;
+        }
       }
     }
-  }
 
-  if( pInfo->nBackfill<mxSafeFrame
-   && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0), 1))==SQLITE_OK
-  ){
-    i64 nSize;                    /* Current size of database file */
-    u32 nBackfill = pInfo->nBackfill;
-
-    /* Sync the WAL to disk */
-    if( sync_flags ){
-      rc = sqlite3OsSync(pWal->pWalFd, sync_flags);
-    }
+    if( pInfo->nBackfill<mxSafeFrame
+     && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK
+    ){
+      i64 nSize;                    /* Current size of database file */
+      u32 nBackfill = pInfo->nBackfill;
 
-    /* If the database may grow as a result of this checkpoint, hint
-    ** about the eventual size of the db file to the VFS layer.
-    */
-    if( rc==SQLITE_OK ){
-      i64 nReq = ((i64)mxPage * szPage);
-      rc = sqlite3OsFileSize(pWal->pDbFd, &nSize);
-      if( rc==SQLITE_OK && nSize<nReq ){
-        sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq);
+      /* Sync the WAL to disk */
+      if( sync_flags ){
+        rc = sqlite3OsSync(pWal->pWalFd, sync_flags);
       }
-    }
 
+      /* If the database may grow as a result of this checkpoint, hint
+      ** about the eventual size of the db file to the VFS layer.
+      */
+      if( rc==SQLITE_OK ){
+        i64 nReq = ((i64)mxPage * szPage);
+        rc = sqlite3OsFileSize(pWal->pDbFd, &nSize);
+        if( rc==SQLITE_OK && nSize<nReq ){
+          sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq);
+        }
+      }
 
-    /* Iterate through the contents of the WAL, copying data to the db file. */
-    while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
-      i64 iOffset;
-      assert( walFramePgno(pWal, iFrame)==iDbpage );
-      if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ) continue;
-      iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE;
-      /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */
-      rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset);
-      if( rc!=SQLITE_OK ) break;
-      iOffset = (iDbpage-1)*(i64)szPage;
-      testcase( IS_BIG_INT(iOffset) );
-      rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset);
-      if( rc!=SQLITE_OK ) break;
-    }
 
-    /* If work was actually accomplished... */
-    if( rc==SQLITE_OK ){
-      if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
-        i64 szDb = pWal->hdr.nPage*(i64)szPage;
-        testcase( IS_BIG_INT(szDb) );
-        rc = sqlite3OsTruncate(pWal->pDbFd, szDb);
-        if( rc==SQLITE_OK && sync_flags ){
-          rc = sqlite3OsSync(pWal->pDbFd, sync_flags);
+      /* Iterate through the contents of the WAL, copying data to the db file */
+      while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
+        i64 iOffset;
+        assert( walFramePgno(pWal, iFrame)==iDbpage );
+        if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){
+          continue;
         }
+        iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE;
+        /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */
+        rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset);
+        if( rc!=SQLITE_OK ) break;
+        iOffset = (iDbpage-1)*(i64)szPage;
+        testcase( IS_BIG_INT(iOffset) );
+        rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset);
+        if( rc!=SQLITE_OK ) break;
       }
+
+      /* If work was actually accomplished... */
       if( rc==SQLITE_OK ){
-        pInfo->nBackfill = mxSafeFrame;
+        if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
+          i64 szDb = pWal->hdr.nPage*(i64)szPage;
+          testcase( IS_BIG_INT(szDb) );
+          rc = sqlite3OsTruncate(pWal->pDbFd, szDb);
+          if( rc==SQLITE_OK && sync_flags ){
+            rc = sqlite3OsSync(pWal->pDbFd, sync_flags);
+          }
+        }
+        if( rc==SQLITE_OK ){
+          pInfo->nBackfill = mxSafeFrame;
+        }
       }
-    }
 
-    /* Release the reader lock held while backfilling */
-    walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
-  }
+      /* Release the reader lock held while backfilling */
+      walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
+    }
 
-  if( rc==SQLITE_BUSY ){
-    /* Reset the return code so as not to report a checkpoint failure
-    ** just because there are active readers.  */
-    rc = SQLITE_OK;
+    if( rc==SQLITE_BUSY ){
+      /* Reset the return code so as not to report a checkpoint failure
+      ** just because there are active readers.  */
+      rc = SQLITE_OK;
+    }
   }
 
   /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the
@@ -1820,7 +1823,7 @@ static int walCheckpoint(
     }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){
       u32 salt1;
       sqlite3_randomness(4, &salt1);
-      assert( mxSafeFrame==pWal->hdr.mxFrame );
+      assert( pInfo->nBackfill==pWal->hdr.mxFrame );
       rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1);
       if( rc==SQLITE_OK ){
         if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){
diff --git a/test/wal5.test b/test/wal5.test
index 8c1ec8bcc7..2d3522b03c 100644
--- a/test/wal5.test
+++ b/test/wal5.test
@@ -390,6 +390,87 @@ foreach {testprefix do_wal_checkpoint} {
     } [wal_file_size 2 1024]
 
   }
+  
+  # Test that FULL, RESTART and TRUNCATE callbacks block on other clients
+  # and truncate the wal file as required even if the entire wal file has
+  # already been checkpointed when they are invoked.
+  #
+  do_multiclient_test tn {
+
+    code1 $do_wal_checkpoint
+    code2 $do_wal_checkpoint
+    code3 $do_wal_checkpoint
+
+    do_test 5.$tn.1 {
+      sql1 {
+        PRAGMA page_size = 1024;
+        PRAGMA auto_vacuum = 0;
+        PRAGMA journal_mode = WAL;
+        PRAGMA synchronous = normal;
+        CREATE TABLE t1(x, y);
+        CREATE INDEX i1 ON t1(x, y);
+        INSERT INTO t1 VALUES(1, 2);
+        INSERT INTO t1 VALUES(3, 4);
+        INSERT INTO t1 VALUES(5, 6);
+      }
+      file size test.db-wal
+    } [wal_file_size 10 1024]
+
+    do_test 5.$tn.2 { 
+      sql2 { BEGIN; SELECT * FROM t1 }
+    } {1 2 3 4 5 6}
+
+    do_test 5.$tn.3 { do_wal_checkpoint db -mode passive } {0 10 10}
+
+    do_test 5.$tn.4 { 
+      sql3 { BEGIN; INSERT INTO t1 VALUES(7, 8); }
+    } {}
+
+    do_test 5.$tn.5 { do_wal_checkpoint db -mode passive  } {0 10 10}
+    do_test 5.$tn.6 { do_wal_checkpoint db -mode full     } {1 10 10}
+
+    do_test 5.$tn.7 { sql3 { ROLLBACK } } {}
+
+    do_test 5.$tn.8 { do_wal_checkpoint db -mode full     } {0 10 10}
+    do_test 5.$tn.9 { do_wal_checkpoint db -mode truncate } {1 10 10}
+
+    do_test 5.$tn.10 { 
+      file size test.db-wal
+    } [wal_file_size 10 1024]
+
+    proc xBusyHandler {n} { sql2 { COMMIT } ; return 0 }
+    db busy xBusyHandler
+
+    do_test 5.$tn.11 { do_wal_checkpoint db -mode truncate } {0 0 0}
+    do_test 5.$tn.12 { file size test.db-wal } 0
+
+    do_test 5.$tn.13 {
+      sql1 {
+        INSERT INTO t1 VALUES(7, 8);
+        INSERT INTO t1 VALUES(9, 10);
+        SELECT * FROM t1;
+      }
+    } {1 2 3 4 5 6 7 8 9 10}
+
+    do_test 5.$tn.14 { 
+      sql2 { BEGIN; SELECT * FROM t1 }
+    } {1 2 3 4 5 6 7 8 9 10}
+
+    proc xBusyHandler {n} { return 1 }
+    do_test 5.$tn.14 { do_wal_checkpoint db -mode truncate } {1 4 4}
+    do_test 5.$tn.15 { file size test.db-wal } [wal_file_size 4 1024]
+
+    do_test 5.$tn.16 { do_wal_checkpoint db -mode restart } {1 4 4}
+
+    proc xBusyHandler {n} { sql2 { COMMIT } ; return 0 }
+    db busy xBusyHandler
+    do_test 5.$tn.17 { do_wal_checkpoint db -mode restart } {0 4 4}
+    do_test 5.$tn.18 { file size test.db-wal } [wal_file_size 4 1024]
+
+    do_test 5.$tn.19 { do_wal_checkpoint db -mode truncate } {0 0 0}
+    do_test 5.$tn.20 { file size test.db-wal } 0
+  }
+
 }