From: drh <drh@noemail.net>
Date: Fri, 1 May 2020 18:37:34 +0000 (+0000)
Subject: Add the SQLITE_FCNTL_CKPT_START file-control.  Use it to optimize the
X-Git-Tag: version-3.32.0~48^2
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fheads%2Fckpt-start-fcntl;p=thirdparty%2Fsqlite.git

Add the SQLITE_FCNTL_CKPT_START file-control.  Use it to optimize the
cksumvfs extension.

FossilOrigin-Name: b40f5aa344ae10cf4da83b3aa9e4866d6f6ffb06ba7e34ec1ce80c92468cf3bf
---

diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c
index 57f91f3c1a..2ea5a67c85 100644
--- a/ext/misc/cksumvfs.c
+++ b/ext/misc/cksumvfs.c
@@ -195,6 +195,7 @@ struct CksmFile {
                         ** Always true if reserve size is 8. */
   char verifyCksm;      /* True to verify checksums */
   char isWal;           /* True if processing a WAL file */
+  char inCkpt;          /* Currently doing a checkpoint */
   CksmFile *pPartner;   /* Ptr from WAL to main-db, or from main-db to WAL */
 };
 
@@ -366,6 +367,20 @@ static int cksmClose(sqlite3_file *pFile){
   return pFile->pMethods->xClose(pFile);
 }
 
+/*
+** Set the computeCkSm and verifyCksm flags, if they need to be
+** changed.
+*/
+static void cksmSetFlags(CksmFile *p, int hasCorrectReserveSize){
+  if( hasCorrectReserveSize!=p->computeCksm ){
+    p->computeCksm = p->verifyCksm = hasCorrectReserveSize;
+    if( p->pPartner ){
+      p->pPartner->verifyCksm = hasCorrectReserveSize;
+      p->pPartner->computeCksm = hasCorrectReserveSize;
+    }
+  }
+}
+
 /*
 ** Read data from a cksm-file.
 */
@@ -383,18 +398,17 @@ static int cksmRead(
     if( iOfst==0 && iAmt>=100 && memcmp(zBuf,"SQLite format 3",16)==0 ){
       u8 *d = (u8*)zBuf;
       char hasCorrectReserveSize = (d[20]==8);
-      if( hasCorrectReserveSize!=p->computeCksm ){
-        p->computeCksm = p->verifyCksm = hasCorrectReserveSize;
-      }
+      cksmSetFlags(p, hasCorrectReserveSize);
     }
-    /* Verify the checksum if (1) the size seems appropriate for a database
-    ** page, and if (2) checksum verification is enabled.  But (3) do not 
-    ** verify the checksums on a WAL page if the main database file
-    ** has subsequently turned off checksums.
+    /* Verify the checksum if
+    **    (1) the size indicates that we are dealing with a complete
+    **        database page
+    **    (2) checksum verification is enabled
+    **    (3) we are not in the middle of checkpoint
     */
     if( iAmt>=512           /* (1) */
      && p->verifyCksm       /* (2) */
-     && (!p->isWal || (p->pPartner!=0 && p->pPartner->verifyCksm))  /* (3) */
+     && !p->inCkpt          /* (3) */
     ){
       u8 cksum[8];
       cksmCompute((u8*)zBuf, iAmt-8, cksum);
@@ -423,9 +437,7 @@ static int cksmWrite(
   if( iOfst==0 && iAmt>=100 && memcmp(zBuf,"SQLite format 3",16)==0 ){
     u8 *d = (u8*)zBuf;
     char hasCorrectReserveSize = (d[20]==8);
-    if( hasCorrectReserveSize!=p->computeCksm ){
-      p->computeCksm = p->verifyCksm = hasCorrectReserveSize;
-    }
+    cksmSetFlags(p, hasCorrectReserveSize);
   }
   /* If the write size is appropriate for a database page and if
   ** checksums where ever enabled, then it will be safe to compute
@@ -433,7 +445,10 @@ static int cksmWrite(
   ** it will never decrease.  And because it cannot decrease, the
   ** checksum will not overwrite anything.
   */
-  if( iAmt>=512 && p->computeCksm ){
+  if( iAmt>=512
+   && p->computeCksm
+   && !p->inCkpt
+  ){
     cksmCompute((u8*)zBuf, iAmt-8, ((u8*)zBuf)+iAmt-8);
   }
   return pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst);
@@ -510,6 +525,7 @@ static int cksmFileControl(sqlite3_file *pFile, int op, void *pArg){
         }else{
           p->verifyCksm = 0;
         }
+        if( p->pPartner ) p->pPartner->verifyCksm = p->verifyCksm;
       }
       azArg[0] = sqlite3_mprintf("%d",p->verifyCksm);
       return SQLITE_OK;
@@ -518,6 +534,9 @@ static int cksmFileControl(sqlite3_file *pFile, int op, void *pArg){
       /* Do not allow page size changes on a checksum database */
       return SQLITE_OK;
     }
+  }else if( op==SQLITE_FCNTL_CKPT_START || op==SQLITE_FCNTL_CKPT_DONE ){
+    p->inCkpt = op==SQLITE_FCNTL_CKPT_START;
+    if( p->pPartner ) p->pPartner->inCkpt = p->inCkpt;
   }
   rc = pFile->pMethods->xFileControl(pFile, op, pArg);
   if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
diff --git a/manifest b/manifest
index feb2f07b97..ef2565abcc 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Clarification\sto\sthe\ssqlite3_uri()\sfamily\sof\sinterfaces.\s\sDocumentation\nenhancement\sonly\s-\sno\schanges\sto\scode.
-D 2020-05-01T13:45:12.131
+C Add\sthe\sSQLITE_FCNTL_CKPT_START\sfile-control.\s\sUse\sit\sto\soptimize\sthe\ncksumvfs\sextension.
+D 2020-05-01T18:37:34.019
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -285,7 +285,7 @@ F ext/misc/appendvfs.c 3777f22ec1057dc4e5fd89f2fbddcc7a29fbeef1ad038c736c54411bb
 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a
 F ext/misc/btreeinfo.c 26004b7a6be320ec08fc20ca8d0f01fccb00a98cbe0f3197446794ff2a506aa3
 F ext/misc/carray.c 91e9a7f512fda934894bed30464552fffa7d3073b5be04189ae0bd0c59f26bfd
-F ext/misc/cksumvfs.c 755627c9112a000bc89653cb5038e080e69c58fa0a5c8e9ebd747f24e5fc01c9
+F ext/misc/cksumvfs.c bdbdf15edd52ed96014122745b445d2c05c968b017f36597990e5670c1b498eb
 F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c85c243
 F ext/misc/completion.c a0efe03edfdc4f717c61e6c9b0bfe2708ff7878010dae3174980a68fdf76aabc
 F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9
@@ -535,7 +535,7 @@ F src/resolve.c d36a2b1639e1c33d7b508abfd3452a63e7fd81737f6f3940bfef085fca6f21f4
 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
 F src/select.c c310de94bf67315054587c18a16e7a3e3dc3a98dc79168f0c2b776548d43f6cd
 F src/shell.c.in 86cd0f0412b9739b769fafdfcad28f731882d522042a95c30ab033a5eba68b03
-F src/sqlite.h.in b20d5dc52765ff82f3701395a7e670611ddf138ee0ae84482452ad3a0b5f24b5
+F src/sqlite.h.in 00fdd0a9cdcb4ca3ea6aed8466a23e158c4a2feb5c2552de62c70f40d2418289
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 2d1af80082edffd71c6f96f70ad1ce6a4fb46615ad10291fc77fe0dea9ff0197
 F src/sqliteInt.h 0f3848c46310d197246003f052985b72d1cdbfc0b31e069db76cb5231062fa1d
@@ -616,7 +616,7 @@ F src/vdbesort.c 2be76d26998ce2b3324cdcc9f6443728e54b6c7677c553ad909c7d7cfab587d
 F src/vdbetrace.c fa3bf238002f0bbbdfb66cc8afb0cea284ff9f148d6439bc1f6f2b4c3b7143f0
 F src/vtab.c 7b704a90515a239c6cdba6a66b1bb3a385e62326cceb5ecb05ec7a091d6b8515
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
-F src/wal.c ea8dad28bb0e2b85ac1ab7618968687ff5fd522af8a1a38d6960ec176ebc8ee6
+F src/wal.c 8efa749ff1e84fb69127bdab1d0f3545afc1706f4dde6f19f2f6237d7dc9c2d2
 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a
 F src/walker.c 7c429c694abd12413a5c17aec9f47cfe9eba6807e6b0a32df883e8e3a14835ed
 F src/where.c 9546c82056e8cdb27291f98cf1adca5d271240b399bb97b32f77fc2bea6146c9
@@ -1862,7 +1862,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 80498b69ea489e8816c80a52c9e55a62699116fcbfdcbfd922ef23c2d9938871
-R 5d328ccb43b2f01beca1bf0c7121f5fa
+P 853703cd6d44d6dd48ef5eda6523e374b8ebdf7c338ddaad31c15a40a8b3fd9b
+R d02abd9a5af9bd115679571d54eadf06
+T *branch * ckpt-start-fcntl
+T *sym-ckpt-start-fcntl *
+T -sym-trunk *
 U drh
-Z 9b72a689cdf698a7000a4037a7a1049c
+Z 843d07fcbc8456358389e1c6f788c09d
diff --git a/manifest.uuid b/manifest.uuid
index 36b7244a29..ce1bb2e80e 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-853703cd6d44d6dd48ef5eda6523e374b8ebdf7c338ddaad31c15a40a8b3fd9b
\ No newline at end of file
+b40f5aa344ae10cf4da83b3aa9e4866d6f6ffb06ba7e34ec1ce80c92468cf3bf
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index f65576a15d..5c082623ce 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -1111,6 +1111,11 @@ struct sqlite3_io_methods {
 ** happen either internally or externally and that are associated with
 ** a particular attached database.
 **
+** <li>[[SQLITE_FCNTL_CKPT_START]]
+** The [SQLITE_FCNTL_CKPT_START] opcode is invoked from within a checkpoint
+** in wal mode before the client starts to copy pages from the wal
+** file to the database file.
+**
 ** <li>[[SQLITE_FCNTL_CKPT_DONE]]
 ** The [SQLITE_FCNTL_CKPT_DONE] opcode is invoked from within a checkpoint
 ** in wal mode after the client has finished copying pages from the wal
@@ -1155,6 +1160,7 @@ struct sqlite3_io_methods {
 #define SQLITE_FCNTL_SIZE_LIMIT             36
 #define SQLITE_FCNTL_CKPT_DONE              37
 #define SQLITE_FCNTL_RESERVE_BYTES          38
+#define SQLITE_FCNTL_CKPT_START             39
 
 /* deprecated names */
 #define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE
diff --git a/src/wal.c b/src/wal.c
index 3e4f4acfde..b230858599 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -1868,6 +1868,7 @@ static int walCheckpoint(
       if( rc==SQLITE_OK ){
         i64 nReq = ((i64)mxPage * szPage);
         i64 nSize;                    /* Current size of database file */
+        sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0);
         rc = sqlite3OsFileSize(pWal->pDbFd, &nSize);
         if( rc==SQLITE_OK && nSize<nReq ){
           sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq);
@@ -1895,6 +1896,7 @@ static int walCheckpoint(
         rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset);
         if( rc!=SQLITE_OK ) break;
       }
+      sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0);
 
       /* If work was actually accomplished... */
       if( rc==SQLITE_OK ){
@@ -1906,10 +1908,6 @@ static int walCheckpoint(
             rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags));
           }
         }
-        if( rc==SQLITE_OK ){
-          rc = sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0);
-          if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
-        }
         if( rc==SQLITE_OK ){
           pInfo->nBackfill = mxSafeFrame;
         }