From: dan <Dan Kennedy>
Date: Mon, 22 Feb 2021 15:44:45 +0000 (+0000)
Subject: When a sub-transaction is released, if no pages required by containing sub-transactio... 
X-Git-Tag: version-3.35.0~68^2
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f43fef29bb78dd5194cf39c12e74552f411cd4d0;p=thirdparty%2Fsqlite.git

When a sub-transaction is released, if no pages required by containing sub-transactions were journaled, truncate the statement journal. This might prevent out-of-control statement journal growth in some cases.

FossilOrigin-Name: e36327fb22db08763a82fb517407ff5ab0dbc053953098033e7e50796a777810
---

diff --git a/manifest b/manifest
index 59fa169b0a..8a3fcf5859 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Rename\sthe\s"struct\sSrcList_item"\sobject\sto\sthe\smore\ssuccinct\s"SrcItem".\nThis\sis\sa\ssymbolic\schange\sonly.\s\sThe\slogic\sis\sunmodified.
-D 2021-02-21T21:04:54.414
+C When\sa\ssub-transaction\sis\sreleased,\sif\sno\spages\srequired\sby\scontaining\ssub-transactions\swere\sjournaled,\struncate\sthe\sstatement\sjournal.\sThis\smight\sprevent\sout-of-control\sstatement\sjournal\sgrowth\sin\ssome\scases.
+D 2021-02-22T15:44:45.441
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -514,7 +514,7 @@ F src/mem2.c b93b8762ab999a29ae7751532dadf0a1ac78040308a5fb1d17fcc365171d67eb
 F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6
 F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944
 F src/memdb.c ab0632d42407e866d2b616bd19d4211ac0ad1b430f04c4e187d60005b8700b98
-F src/memjournal.c 90b2ca7e2f465d57c16b69d15a9f3e3294af61088eb4938f2f7664d5ac50f813
+F src/memjournal.c bb0533253ad4a9657c29742d8504b6813c8367ccf135c0fb553d786850a55ea9
 F src/msvc.h 3a15918220367a8876be3fa4f2abe423a861491e84b864fb2b7426bf022a28f8
 F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25
 F src/mutex.h a7b2293c48db5f27007c3bdb21d438873637d12658f5a0bf8ad025bb96803c4a
@@ -529,7 +529,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c adbbcea4c63d3b400d405f60a5da4c01433753ec4a12e2dc695beb2bbd671fe9
 F src/os_win.c 77d39873836f1831a9b0b91894fec45ab0e9ca8e067dc8c549e1d1eca1566fe9
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c c49952ac5e9cc536778eff528091d79d38b3e45cbeeed4695dc05e207dc6547d
+F src/pager.c 970691daea03f9f15e34de671bd8675c1e136232b529e21bfd36d4dba6d41753
 F src/pager.h 4bf9b3213a4b2bebbced5eaa8b219cf25d4a82f385d093cd64b7e93e5285f66f
 F src/parse.y e6019e934cbbb4108ed3b9a6a225ee388d9a93fd12877ed5ba72c1dd16ebdd3c
 F src/pcache.c 385ff064bca69789d199a98e2169445dc16e4291fa807babd61d4890c3b34177
@@ -1905,7 +1905,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0d2c992f3622a6272ca4a3caff6b21f619fe976b8df8b34eff066086f8df2202
-R 3144ef024c1b147facc59f065688290b
-U drh
-Z 9a2772a6abeb20b71009a7863f85f821
+P bfd5bf2c73110fcb36db9ba2a949ff516131fbd3e89325f88fe9f5c2b4ed87b2
+R 54537f477ff92d6f9883b3b55586620b
+T *branch * stmt-jrnl-truncate
+T *sym-stmt-jrnl-truncate *
+T -sym-trunk *
+U dan
+Z 89681e18982564f6dc1e76e8bd67344b
diff --git a/manifest.uuid b/manifest.uuid
index 08a79a0dbf..48c77f3bcd 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-bfd5bf2c73110fcb36db9ba2a949ff516131fbd3e89325f88fe9f5c2b4ed87b2
\ No newline at end of file
+e36327fb22db08763a82fb517407ff5ab0dbc053953098033e7e50796a777810
\ No newline at end of file
diff --git a/src/memjournal.c b/src/memjournal.c
index 4811f2d8d4..77cd6db7f3 100644
--- a/src/memjournal.c
+++ b/src/memjournal.c
@@ -70,7 +70,6 @@ struct MemJournal {
   int nChunkSize;                 /* In-memory chunk-size */
 
   int nSpill;                     /* Bytes of data before flushing */
-  int nSize;                      /* Bytes of data currently in memory */
   FileChunk *pFirst;              /* Head of in-memory chunk-list */
   FilePoint endpoint;             /* Pointer to the end of the file */
   FilePoint readpoint;            /* Pointer to the end of the last xRead() */
@@ -131,14 +130,13 @@ static int memjrnlRead(
 /*
 ** Free the list of FileChunk structures headed at MemJournal.pFirst.
 */
-static void memjrnlFreeChunks(MemJournal *p){
+static void memjrnlFreeChunks(FileChunk *pFirst){
   FileChunk *pIter;
   FileChunk *pNext;
-  for(pIter=p->pFirst; pIter; pIter=pNext){
+  for(pIter=pFirst; pIter; pIter=pNext){
     pNext = pIter->pNext;
     sqlite3_free(pIter);
   } 
-  p->pFirst = 0;
 }
 
 /*
@@ -165,7 +163,7 @@ static int memjrnlCreateFile(MemJournal *p){
     }
     if( rc==SQLITE_OK ){
       /* No error has occurred. Free the in-memory buffers. */
-      memjrnlFreeChunks(&copy);
+      memjrnlFreeChunks(copy.pFirst);
     }
   }
   if( rc!=SQLITE_OK ){
@@ -248,7 +246,6 @@ static int memjrnlWrite(
         nWrite -= iSpace;
         p->endpoint.iOffset += iSpace;
       }
-      p->nSize = iAmt + iOfst;
     }
   }
 
@@ -256,22 +253,30 @@ static int memjrnlWrite(
 }
 
 /*
-** Truncate the file.
-**
-** If the journal file is already on disk, truncate it there. Or, if it
-** is still in main memory but is being truncated to zero bytes in size,
-** ignore 
+** Truncate the in-memory file.
 */
 static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){
   MemJournal *p = (MemJournal *)pJfd;
-  if( ALWAYS(size==0) ){
-    memjrnlFreeChunks(p);
-    p->nSize = 0;
-    p->endpoint.pChunk = 0;
-    p->endpoint.iOffset = 0;
-    p->readpoint.pChunk = 0;
-    p->readpoint.iOffset = 0;
+  FileChunk *pIter = 0;
+
+  if( size==0 ){
+    memjrnlFreeChunks(p->pFirst);
+    p->pFirst = 0;
+  }else{
+    i64 iOff = p->nChunkSize;
+    for(pIter=p->pFirst; ALWAYS(pIter) && iOff<=size; pIter=pIter->pNext){
+      iOff += p->nChunkSize;
+    }
+    if( pIter ){
+      memjrnlFreeChunks(pIter->pNext);
+      pIter->pNext = 0;
+    }
   }
+
+  p->endpoint.pChunk = pIter;
+  p->endpoint.iOffset = size;
+  p->readpoint.pChunk = 0;
+  p->readpoint.iOffset = 0;
   return SQLITE_OK;
 }
 
@@ -280,7 +285,7 @@ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){
 */
 static int memjrnlClose(sqlite3_file *pJfd){
   MemJournal *p = (MemJournal *)pJfd;
-  memjrnlFreeChunks(p);
+  memjrnlFreeChunks(p->pFirst);
   return SQLITE_OK;
 }
 
diff --git a/src/pager.c b/src/pager.c
index a5510e7eb8..cd9096ed1c 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -435,6 +435,7 @@ struct PagerSavepoint {
   Bitvec *pInSavepoint;        /* Set of pages in this savepoint */
   Pgno nOrig;                  /* Original number of pages in file */
   Pgno iSubRec;                /* Index of first record in sub-journal */
+  int bTruncateOnRelease;      /* If stmt journal may be truncated on RELEASE */
 #ifndef SQLITE_OMIT_WAL
   u32 aWalData[WAL_SAVEPOINT_NDATA];        /* WAL savepoint context */
 #endif
@@ -1070,6 +1071,9 @@ static int subjRequiresPage(PgHdr *pPg){
   for(i=0; i<pPager->nSavepoint; i++){
     p = &pPager->aSavepoint[i];
     if( p->nOrig>=pgno && 0==sqlite3BitvecTestNotNull(p->pInSavepoint, pgno) ){
+      for(i=i+1; i<pPager->nSavepoint; i++){
+        pPager->aSavepoint[i].bTruncateOnRelease = 0;
+      }
       return 1;
     }
   }
@@ -6848,6 +6852,7 @@ static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){
     }
     aNew[ii].iSubRec = pPager->nSubRec;
     aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize);
+    aNew[ii].bTruncateOnRelease = 1;
     if( !aNew[ii].pInSavepoint ){
       return SQLITE_NOMEM_BKPT;
     }
@@ -6929,13 +6934,15 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){
     /* If this is a release of the outermost savepoint, truncate 
     ** the sub-journal to zero bytes in size. */
     if( op==SAVEPOINT_RELEASE ){
-      if( nNew==0 && isOpen(pPager->sjfd) ){
+      PagerSavepoint *pRel = &pPager->aSavepoint[nNew];
+      if( pRel->bTruncateOnRelease && isOpen(pPager->sjfd) ){
         /* Only truncate if it is an in-memory sub-journal. */
         if( sqlite3JournalIsInMemory(pPager->sjfd) ){
-          rc = sqlite3OsTruncate(pPager->sjfd, 0);
+          i64 sz = (pPager->pageSize+4)*pRel->iSubRec;
+          rc = sqlite3OsTruncate(pPager->sjfd, sz);
           assert( rc==SQLITE_OK );
         }
-        pPager->nSubRec = 0;
+        pPager->nSubRec = pRel->iSubRec;
       }
     }
     /* Else this is a rollback operation, playback the specified savepoint.