Auto-vacuum: Ensure pages to be removed by database truncation are in the journal...

FossilOrigin-Name: 081676e491760a45325e2349b177d6382faab9f5

diff --git a/manifest b/manifest
index fdb56ae965b3eab23baea717caaf956f7ffe7063..86dee7d327d20e4099c48825e3cf90bec7ac65fa 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Compile\sand\spass\sthe\squick\sregression\stests\swith\sautovacuum\sdisabled.\s(CVS\s2073)
-D 2004-11-06T00:02:48
+C Auto-vacuum:\sEnsure\spages\sto\sbe\sremoved\sby\sdatabase\struncation\sare\sin\sthe\sjournal\sfile.\sAlso\sfix\san\ssqlite3pager_movepage()\sbug.\s(CVS\s2074)
+D 2004-11-06T12:26:08
 F Makefile.in c4d2416860f472a1e3393714d0372074197565df
 F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457
 F README a01693e454a00cc117967e3f9fdab2d4d52e9bc1
@@ -29,7 +29,7 @@ F sqlite3.def dbaeb20c153e1d366e8f421b55a573f5dfc00863
 F sqlite3.pc.in 985b9bf34192a549d7d370e0f0b6b34a4f61369a
 F src/attach.c e49d09dad9f5f9fb10b4b0c1be5a70ae4c45e689
 F src/auth.c 3b81f2a42f48a62c2c9c9b0eda31a157c681edea
-F src/btree.c 32b5c8d45744d8710475ae99744a0ca9a402d2a0
+F src/btree.c 8ae13efe790fc462ee590ba96e1940f4c943fe19
 F src/btree.h 861e40b759a195ba63819740e484390012cf81ab
 F src/build.c fba59507d36d27effbeba59bdd2a720071b7673a
 F src/date.c 34bdb0082db7ec2a83ef00063f7b44e61ee19dad
@@ -46,13 +46,13 @@ F src/os.h 5834a404b7c8318dc1928c9fc0137a65c9c0416c
 F src/os_common.h 0e7f428ba0a6c40a61bc56c4e96f493231301b73
 F src/os_mac.c 7367dab0c44ab0b2c4337e73ac6f6f97f171c2cb
 F src/os_mac.h 608fdf39eafa1ce25fc8cb223b8b0a073341d4da
-F src/os_test.c d9a21852d170f20adeae6f224d1beba2cb5af8aa
+F src/os_test.c 91e5f22dd89491e5e1554820e715805f43fa4ece
 F src/os_test.h 6a26a4978492e4bbdbf385554958418ff02db162
 F src/os_unix.c 5824b22ba41fe9d514ef9169aac1b5fde73af229
 F src/os_unix.h f3097815e041e82e24d92505e1ff61ba24172d13
 F src/os_win.c 9482dfc92f289b68205bb2c9315757c7e3946bfb
 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
-F src/pager.c 868c67e4ff8a1785c06caaf483fddb5a95013af0
+F src/pager.c 0523c8a122f95b41008b87cae2cc1d368452dbfc
 F src/pager.h 9eba8c53dd91eae7f3f90743b2ee242da02a9862
 F src/parse.y 8456726833755ecd6dac9bcd8af205c8dc419d01
 F src/pragma.c 6a0ae7721e614c5a921e918ab5206d5e654f1a6f
@@ -253,7 +253,7 @@ F www/tclsqlite.tcl 560ecd6a916b320e59f2917317398f3d59b7cc25
 F www/vdbe.tcl 59288db1ac5c0616296b26dce071c36cb611dfe9
 F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0
 F www/whentouse.tcl fdacb0ba2d39831e8a6240d05a490026ad4c4e4c
-P 11dba47e61279bdf3be6f64a6259b877f3bf6155
-R 8f5f401812cf89b0176d6494be1ddd1b
-U drh
-Z 7c63afdfe773d316c41b855a4f0f6dac
+P 89b9026a5371f1c5589f04e3b69171ec1f605b7b
+R 8ad60b55265f20042b59f34c42d3206d
+U danielk1977
+Z 6d567165f557419cc4b6243e52dc95f0

diff --git a/manifest.uuid b/manifest.uuid
index 20bda459a3a427209474130f131b3749689904f7..c71233b920ca3f2dedcfd60cc27ad34ea27bad46 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-89b9026a5371f1c5589f04e3b69171ec1f605b7b
\ No newline at end of file
+081676e491760a45325e2349b177d6382faab9f5
\ No newline at end of file

diff --git a/src/btree.c b/src/btree.c
index 652e65e0e99cac88c430aaaaf1e14e1d37092162..acc731542af62cc2f97ef682cfd22f8b5b85684a 100644 (file)
--- a/src/btree.c
+++ b/src/btree.c
@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** $Id: btree.c,v 1.212 2004/11/06 00:02:48 drh Exp $
+** $Id: btree.c,v 1.213 2004/11/06 12:26:08 danielk1977 Exp $
 **
 ** This file implements a external (disk-based) database using BTrees.
 ** For a detailed discussion of BTrees, refer to
@@ -5072,6 +5072,8 @@ static int checkTreePage(
   char zContext[100];
   char *hit;
 
+  sprintf(zContext, "Page %d: ", iPage);
+
   /* Check that the page exists
   */
   cur.pBt = pBt = pCheck->pBt;

diff --git a/src/os_test.c b/src/os_test.c
index 0e292bc428ee496dfeb8f2870dd0aedcadfc0a95..8199f5b18327af8eeac7008d344dacaf275d711d 100644 (file)
--- a/src/os_test.c
+++ b/src/os_test.c
@@ -239,10 +239,12 @@ printf("Writing block %d of %s\n", i, pFile->zName);
         if( BLOCK_OFFSET(i+1)>nMax ){
           len = nMax-BLOCK_OFFSET(i);
         }
-        if( trash ){
-          sqlite3Randomness(len, p);
+        if( len>0 ){
+          if( trash ){
+            sqlite3Randomness(len, p);
+          }
+          rc = sqlite3RealWrite(&pFile->fd, p, len);
         }
-        rc = sqlite3RealWrite(&pFile->fd, p, len);
       }
       sqliteFree(p);
     }

diff --git a/src/pager.c b/src/pager.c
index ee2258fd705ad227f23de8e582b3f4d7cb4d5305..b2365034d938517788a4e34fb8e4b20542a2c97e 100644 (file)
--- a/src/pager.c
+++ b/src/pager.c
@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.173 2004/11/05 16:37:03 danielk1977 Exp $
+** @(#) $Id: pager.c,v 1.174 2004/11/06 12:26:08 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -41,6 +41,13 @@
 #define TRACE4(X,Y,Z,W)
 #endif
 
+#ifdef OS_TEST
+#define PAGERID(p) (p->fd->fd.h)
+#define FILEHANDLEID(fd) (fd->fd.h)
+#else
+#define PAGERID(p) (p->fd.h)
+#define FILEHANDLEID(fd) (fd.h)
+#endif
 
 /*
 ** The page cache as a whole is always in one of the following
@@ -908,7 +915,7 @@ static int pager_playback_one_page(Pager *pPager, OsFile *jfd, int useCksum){
   */
   pPg = pager_lookup(pPager, pgno);
   assert( pPager->state>=PAGER_EXCLUSIVE || pPg );
-  TRACE3("PLAYBACK %d page %d\n", pPager->fd.h, pgno);
+  TRACE3("PLAYBACK %d page %d\n", PAGERID(pPager), pgno);
   if( pPager->state>=PAGER_EXCLUSIVE ){
     sqlite3OsSeek(&pPager->fd, (pgno-1)*(i64)pPager->pageSize);
     rc = sqlite3OsWrite(&pPager->fd, aData, pPager->pageSize);
@@ -1037,7 +1044,7 @@ static int pager_reload_cache(Pager *pPager){
     if( (int)pPg->pgno <= pPager->origDbSize ){
       sqlite3OsSeek(&pPager->fd, pPager->pageSize*(i64)(pPg->pgno-1));
       rc = sqlite3OsRead(&pPager->fd, zBuf, pPager->pageSize);
-      TRACE3("REFETCH %d page %d\n", pPager->fd.h, pPg->pgno);
+      TRACE3("REFETCH %d page %d\n", PAGERID(pPager), pPg->pgno);
       if( rc ) break;
       CODEC(pPager, zBuf, pPg->pgno, 2);
     }else{
@@ -1506,7 +1513,7 @@ int sqlite3pager_open(
     sqliteFree(zFullPathname);
     return SQLITE_NOMEM;
   }
-  TRACE3("OPEN %d %s\n", fd.h, zFullPathname);
+  TRACE3("OPEN %d %s\n", FILEHANDLEID(fd), zFullPathname);
   pPager->zFilename = (char*)&pPager[1];
   pPager->zDirectory = &pPager->zFilename[nameLen+1];
   pPager->zJournal = &pPager->zDirectory[nameLen+1];
@@ -1693,19 +1700,6 @@ static void unlinkPage(PgHdr *pPg){
 
   /* Unlink from the pgno hash table */
   unlinkHashChain(pPager, pPg);
-/*
-  if( pPg->pNextHash ){
-    pPg->pNextHash->pPrevHash = pPg->pPrevHash;
-  }
-  if( pPg->pPrevHash ){
-    pPg->pPrevHash->pNextHash = pPg->pNextHash;
-  }else{
-    int h = pager_hash(pPg->pgno);
-    assert( pPager->aHash[h]==pPg );
-    pPager->aHash[h] = pPg->pNextHash;
-  }
-  pPg->pNextHash = pPg->pPrevHash = 0;
-*/
 }
 
 /*
@@ -1808,7 +1802,7 @@ int sqlite3pager_close(Pager *pPager){
     pNext = pPg->pNextAll;
     sqliteFree(pPg);
   }
-  TRACE2("CLOSE %d\n", pPager->fd.h);
+  TRACE2("CLOSE %d\n", PAGERID(pPager));
   sqlite3OsClose(&pPager->fd);
   assert( pPager->journalOpen==0 );
   /* Temp files are automatically deleted by the OS
@@ -1939,7 +1933,7 @@ static int syncJournal(Pager *pPager){
         ** it as a candidate for rollback. 
         */
         if( pPager->fullSync ){
-          TRACE2("SYNC journal of %d\n", pPager->fd.h);
+          TRACE2("SYNC journal of %d\n", PAGERID(pPager));
           rc = sqlite3OsSync(&pPager->jfd);
           if( rc!=0 ) return rc;
         }
@@ -1949,7 +1943,7 @@ static int syncJournal(Pager *pPager){
 
         sqlite3OsSeek(&pPager->jfd, pPager->journalOff);
       }
-      TRACE2("SYNC journal of %d\n", pPager->fd.h);
+      TRACE2("SYNC journal of %d\n", PAGERID(pPager));
       rc = sqlite3OsSync(&pPager->jfd);
       if( rc!=0 ) return rc;
       pPager->journalStarted = 1;
@@ -2054,13 +2048,13 @@ static int pager_write_pagelist(PgHdr *pList){
     */
     if( pList->pgno<=pPager->dbSize ){
       CODEC(pPager, PGHDR_TO_DATA(pList), pList->pgno, 6);
-      TRACE3("STORE %d page %d\n", pPager->fd.h, pList->pgno);
+      TRACE3("STORE %d page %d\n", PAGERID(pPager), pList->pgno);
       rc = sqlite3OsWrite(&pPager->fd, PGHDR_TO_DATA(pList), pPager->pageSize);
       CODEC(pPager, PGHDR_TO_DATA(pList), pList->pgno, 0);
     }
 #ifndef NDEBUG
     else{
-      TRACE3("NOSTORE %d page %d\n", pPager->fd.h, pList->pgno);
+      TRACE3("NOSTORE %d page %d\n", PAGERID(pPager), pList->pgno);
     }
 #endif
     if( rc ) return rc;
@@ -2329,7 +2323,7 @@ int sqlite3pager_get(Pager *pPager, Pgno pgno, void **ppPage){
       assert( MEMDB==0 );
       sqlite3OsSeek(&pPager->fd, (pgno-1)*(i64)pPager->pageSize);
       rc = sqlite3OsRead(&pPager->fd, PGHDR_TO_DATA(pPg), pPager->pageSize);
-      TRACE3("FETCH %d page %d\n", pPager->fd.h, pPg->pgno);
+      TRACE3("FETCH %d page %d\n", PAGERID(pPager), pPg->pgno);
       CODEC(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3);
       if( rc!=SQLITE_OK ){
         i64 fileSize;
@@ -2540,7 +2534,7 @@ int sqlite3pager_begin(void *pData, int exFlag){
         return rc;
       }
       pPager->dirtyCache = 0;
-      TRACE2("TRANSACTION %d\n", pPager->fd.h);
+      TRACE2("TRANSACTION %d\n", PAGERID(pPager));
       if( pPager->useJournal && !pPager->tempFile ){
         rc = pager_open_journal(pPager);
       }
@@ -2620,7 +2614,7 @@ int sqlite3pager_write(void *pData){
         u32 saved;
         if( MEMDB ){
           PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager);
-          TRACE3("JOURNAL %d page %d\n", pPager->fd.h, pPg->pgno);
+          TRACE3("JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);
           assert( pHist->pOrig==0 );
           pHist->pOrig = sqliteMallocRaw( pPager->pageSize );
           if( pHist->pOrig ){
@@ -2637,7 +2631,7 @@ int sqlite3pager_write(void *pData){
           rc = sqlite3OsWrite(&pPager->jfd, &((char*)pData)[-4], szPg);
           pPager->journalOff += szPg;
           TRACE4("JOURNAL %d page %d needSync=%d\n",
-                  pPager->fd.h, pPg->pgno, pPg->needSync);
+                  PAGERID(pPager), pPg->pgno, pPg->needSync);
           CODEC(pPager, pData, pPg->pgno, 0);
           *(u32*)PGHDR_TO_EXTRA(pPg, pPager) = saved;
           if( rc!=SQLITE_OK ){
@@ -2657,7 +2651,7 @@ int sqlite3pager_write(void *pData){
       }else{
         pPg->needSync = !pPager->journalStarted && !pPager->noSync;
         TRACE4("APPEND %d page %d needSync=%d\n",
-                pPager->fd.h, pPg->pgno, pPg->needSync);
+                PAGERID(pPager), pPg->pgno, pPg->needSync);
       }
       if( pPg->needSync ){
         pPager->needSync = 1;
@@ -2679,12 +2673,12 @@ int sqlite3pager_write(void *pData){
         if( pHist->pStmt ){
           memcpy(pHist->pStmt, PGHDR_TO_DATA(pPg), pPager->pageSize);
         }
-        TRACE3("STMT-JOURNAL %d page %d\n", pPager->fd.h, pPg->pgno);
+        TRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);
       }else{
         store32bits(pPg->pgno, pPg, -4);
         CODEC(pPager, pData, pPg->pgno, 7);
         rc = sqlite3OsWrite(&pPager->stfd,((char*)pData)-4, pPager->pageSize+4);
-        TRACE3("STMT-JOURNAL %d page %d\n", pPager->fd.h, pPg->pgno);
+        TRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno);
         CODEC(pPager, pData, pPg->pgno, 0);
         if( rc!=SQLITE_OK ){
           sqlite3pager_rollback(pPager);
@@ -2781,7 +2775,7 @@ void sqlite3pager_dont_write(Pager *pPager, Pgno pgno){
       ** corruption during the next transaction.
       */
     }else{
-      TRACE3("DONT_WRITE page %d of %d\n", pgno, pPager->fd.h);
+      TRACE3("DONT_WRITE page %d of %d\n", pgno, PAGERID(pPager));
       pPg->dirty = 0;
     }
   }
@@ -2807,7 +2801,7 @@ void sqlite3pager_dont_rollback(void *pData){
       pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7);
       page_add_to_stmt_list(pPg);
     }
-    TRACE3("DONT_ROLLBACK page %d of %d\n", pPg->pgno, pPager->fd.h);
+    TRACE3("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager));
   }
   if( pPager->stmtInUse && !pPg->inStmt && (int)pPg->pgno<=pPager->stmtSize ){
     assert( pPg->inJournal || (int)pPg->pgno>pPager->origDbSize );
@@ -2853,7 +2847,7 @@ int sqlite3pager_commit(Pager *pPager){
   if( pPager->state<PAGER_RESERVED ){
     return SQLITE_ERROR;
   }
-  TRACE2("COMMIT %d\n", pPager->fd.h);
+  TRACE2("COMMIT %d\n", PAGERID(pPager));
   if( MEMDB ){
     pPg = pager_get_all_dirty_pages(pPager);
     while( pPg ){
@@ -2914,7 +2908,7 @@ commit_abort:
 */
 int sqlite3pager_rollback(Pager *pPager){
   int rc;
-  TRACE2("ROLLBACK %d\n", pPager->fd.h);
+  TRACE2("ROLLBACK %d\n", PAGERID(pPager));
   if( MEMDB ){
     PgHdr *p;
     for(p=pPager->pAll; p; p=p->pNextAll){
@@ -2929,9 +2923,9 @@ int sqlite3pager_rollback(Pager *pPager){
       pHist = PGHDR_TO_HIST(p, pPager);
       if( pHist->pOrig ){
         memcpy(PGHDR_TO_DATA(p), pHist->pOrig, pPager->pageSize);
-        TRACE3("ROLLBACK-PAGE %d of %d\n", p->pgno, pPager->fd.h);
+        TRACE3("ROLLBACK-PAGE %d of %d\n", p->pgno, PAGERID(pPager));
       }else{
-        TRACE3("PAGE %d is clean on %d\n", p->pgno, pPager->fd.h);
+        TRACE3("PAGE %d is clean on %d\n", p->pgno, PAGERID(pPager));
       }
       clearHistory(pHist);
       p->dirty = 0;
@@ -3021,7 +3015,7 @@ int sqlite3pager_stmt_begin(Pager *pPager){
   char zTemp[SQLITE_TEMPNAME_SIZE];
   assert( !pPager->stmtInUse );
   assert( pPager->dbSize>=0 );
-  TRACE2("STMT-BEGIN %d\n", pPager->fd.h);
+  TRACE2("STMT-BEGIN %d\n", PAGERID(pPager));
   if( MEMDB ){
     pPager->stmtInUse = 1;
     pPager->stmtSize = pPager->dbSize;
@@ -3069,7 +3063,7 @@ stmt_begin_failed:
 int sqlite3pager_stmt_commit(Pager *pPager){
   if( pPager->stmtInUse ){
     PgHdr *pPg, *pNext;
-    TRACE2("STMT-COMMIT %d\n", pPager->fd.h);
+    TRACE2("STMT-COMMIT %d\n", PAGERID(pPager));
     if( !MEMDB ){
       sqlite3OsSeek(&pPager->stfd, 0);
       /* sqlite3OsTruncate(&pPager->stfd, 0); */
@@ -3101,7 +3095,7 @@ int sqlite3pager_stmt_commit(Pager *pPager){
 int sqlite3pager_stmt_rollback(Pager *pPager){
   int rc;
   if( pPager->stmtInUse ){
-    TRACE2("STMT-ROLLBACK %d\n", pPager->fd.h);
+    TRACE2("STMT-ROLLBACK %d\n", PAGERID(pPager));
     if( MEMDB ){
       PgHdr *pPg;
       for(pPg=pPager->pStmt; pPg; pPg=pPg->pNextStmt){
@@ -3208,6 +3202,9 @@ static int pager_incr_changecounter(Pager *pPager){
 int sqlite3pager_sync(Pager *pPager, const char *zMaster, Pgno nTrunc){
   int rc = SQLITE_OK;
 
+  TRACE4("DATABASE SYNC: File=%s zMaster=%s nTrunc=%d\n", 
+      pPager->zFilename, zMaster, nTrunc);
+
   /* If this is an in-memory db, or no pages have been written to, or this
   ** function has already been called, it is a no-op.
   */
@@ -3224,6 +3221,25 @@ int sqlite3pager_sync(Pager *pPager, const char *zMaster, Pgno nTrunc){
     if( !pPager->setMaster ){
       rc = pager_incr_changecounter(pPager);
       if( rc!=SQLITE_OK ) goto sync_exit;
+#ifndef SQLITE_OMIT_AUTOVACUUM
+      if( nTrunc!=0 ){
+        /* If this transaction has made the database smaller, then all pages
+        ** being discarded by the truncation must be written to the journal
+        ** file.
+        */
+        Pgno i;
+        void *pPage;
+        for( i=nTrunc+1; i<=pPager->origDbSize; i++ ){
+          if( !(pPager->aInJournal[i/8] & (1<<(i&7))) ){
+            rc = sqlite3pager_get(pPager, i, &pPage);
+            if( rc!=SQLITE_OK ) goto sync_exit;
+            rc = sqlite3pager_write(pPage);
+            sqlite3pager_unref(pPage);
+            if( rc!=SQLITE_OK ) goto sync_exit;
+          }
+        } 
+      }
+#endif
       rc = writeMasterJournal(pPager, zMaster);
       if( rc!=SQLITE_OK ) goto sync_exit;
       rc = syncJournal(pPager);
@@ -3279,10 +3295,12 @@ int sqlite3pager_movepage(Pager *pPager, void *pData, Pgno pgno){
   assert( !pPager->stmtInUse );
   assert( pPg->nRef>0 );
 
+  TRACE4("MOVE %d page %d moves to %d\n", PAGERID(pPager), pPg->pgno, pgno);
+
   /* Unlink pPg from it's hash-chain */
   unlinkHashChain(pPager, pPg);
 
-  /* If the cache contains a page with page-number pgno exists, remove it
+  /* If the cache contains a page with page-number pgno, remove it
   ** from it's hash chain.
   */
   pPgOld = pager_lookup(pPager, pgno);
@@ -3290,6 +3308,9 @@ int sqlite3pager_movepage(Pager *pPager, void *pData, Pgno pgno){
     assert( pPgOld->nRef==0 );
     unlinkHashChain(pPager, pPgOld);
     pPgOld->dirty = 0;
+    if( pPgOld->needSync ){
+      pPg->needSync = 1;
+    }
   }
 
   /* Change the page number for pPg and insert it into the new hash-chain. */