From b6e099a973559f52bda1485ee6126041ae8687d7 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 4 May 2010 14:47:39 +0000 Subject: [PATCH] Fix problems with recovering wal files that use a page-size other than the default. FossilOrigin-Name: 1a391f3c55dc9d4266552fa26d2a9839c6bafce4 --- manifest | 20 +++---- manifest.uuid | 2 +- src/pager.c | 25 +++++--- src/wal.c | 33 +++++++--- src/wal.h | 7 ++- test/wal.test | 146 ++++++++++++++++++++++++++++++++++++++++++++- test/walcrash.test | 20 ++++++- 7 files changed, 220 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 9830981e05..34285aebae 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\swalfault.test. -D 2010-05-04T11:06:03 +C Fix\sproblems\swith\srecovering\swal\sfiles\sthat\suse\sa\spage-size\sother\sthan\sthe\sdefault. +D 2010-05-04T14:47:40 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -152,7 +152,7 @@ F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 F src/os_unix.c 7ab8d2fe59954dca619af3e1bbd4789f5032fece F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 -F src/pager.c e9c7fe979b32a3c5bf4216d8fbe1cf8beff8a1b8 +F src/pager.c f030536592582d29a761b75988f409c894c37d5e F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -221,8 +221,8 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 1d0abe01ae9836f207ecfe2d3f1ece97b744fc3e -F src/wal.h d6d4a6809151e30bed5b01dd05cf27858f5a7bc8 +F src/wal.c 558700d2d596af389305c193fe98e79406a50982 +F src/wal.h b4c42014b5fa3b4e6244ac8c65de7ff67adeb27c F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 75fee9e255b62f773fcadd1d1f25b6f63ac7a356 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -758,9 +758,9 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 511c630993d5adc170f56e581909b6eb580b0f53 +F test/wal.test 2b3626a6c839ef281e2a51f6223578e5fa7616c2 F test/walbak.test a0e45187c7d8928df035dfea29b99b016b21ca3c -F test/walcrash.test 63edc6a9e05f645b54d649186a5818fc82953e2e +F test/walcrash.test f6d5fb2bb108876f04848720a488065d9deef69f F test/walfault.test 2504c5c50d8f9a9e48969de381eb98e2a8b89195 F test/walhook.test 5f18e0fc8787f1f8889d7a9971af18f334f83786 F test/walmode.test bac6f06544a8554588a1543def996bbe2fc41792 @@ -809,7 +809,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a60104aa7e38e7d9f2ff2eae02687dc9c5dd5d77 -R 9692cfa3edc87873be9342b2b7fc3777 +P 232dbe8ecec16485be5c5995fdf7a0ed951c2097 +R 8e6184a7c7d1eb3d58e7d51edd48bf22 U dan -Z 6f47b056126e5089750d5cf38e878789 +Z 520c27e9585f17af9827fbfa6794dca0 diff --git a/manifest.uuid b/manifest.uuid index ca7a765c19..4e513266ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -232dbe8ecec16485be5c5995fdf7a0ed951c2097 \ No newline at end of file +1a391f3c55dc9d4266552fa26d2a9839c6bafce4 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 00d07caff0..7f0e4f7fe9 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2196,8 +2196,8 @@ static int readDbPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - i64 iOffset; /* Byte offset of file to read from */ int isInWal = 0; /* True if page is in log file */ + int pgsz = pPager->pageSize; /* Number of bytes to read */ assert( pPager->state>=PAGER_SHARED && !MEMDB ); assert( isOpen(pPager->fd) ); @@ -2210,11 +2210,11 @@ static int readDbPage(PgHdr *pPg){ if( pagerUseWal(pPager) ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pPg->pData); + rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); } if( rc==SQLITE_OK && !isInWal ){ - iOffset = (pgno-1)*(i64)pPager->pageSize; - rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset); + i64 iOffset = (pgno-1)*(i64)pPager->pageSize; + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); if( rc==SQLITE_IOERR_SHORT_READ ){ rc = SQLITE_OK; } @@ -2832,6 +2832,15 @@ int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ int rc = SQLITE_OK; memset(pDest, 0, N); assert( isOpen(pPager->fd) || pPager->tempFile ); + + if( pagerUseWal(pPager) ){ + int isInWal = 0; + rc = sqlite3WalRead(pPager->pWal, 1, &isInWal, N, pDest); + if( rc!=SQLITE_OK || isInWal ){ + return rc; + } + } + if( isOpen(pPager->fd) ){ IOTRACE(("DBHDR %p 0 %d\n", pPager, N)) rc = sqlite3OsRead(pPager->fd, pDest, N, 0); @@ -3055,7 +3064,8 @@ int sqlite3PagerClose(Pager *pPager){ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL sqlite3WalClose(pPager->pWal, pPager->fd, - (pPager->noSync ? 0 : pPager->sync_flags), pTmp + (pPager->noSync ? 0 : pPager->sync_flags), + pPager->pageSize, pTmp ); pPager->pWal = 0; #endif @@ -5833,7 +5843,8 @@ int sqlite3PagerCheckpoint(Pager *pPager){ u8 *zBuf = (u8 *)pPager->pTmpSpace; rc = sqlite3WalCheckpoint(pPager->pWal, pPager->fd, (pPager->noSync ? 0 : pPager->sync_flags), - zBuf, pPager->xBusyHandler, pPager->pBusyHandlerArg + pPager->pageSize, zBuf, + pPager->xBusyHandler, pPager->pBusyHandlerArg ); } return rc; @@ -5908,7 +5919,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ if( rc==SQLITE_OK ){ rc = sqlite3WalClose(pPager->pWal, pPager->fd, (pPager->noSync ? 0 : pPager->sync_flags), - (u8*)pPager->pTmpSpace + pPager->pageSize, (u8*)pPager->pTmpSpace ); pPager->pWal = 0; } diff --git a/src/wal.c b/src/wal.c index b92a021d2d..09f31a9939 100644 --- a/src/wal.c +++ b/src/wal.c @@ -710,6 +710,7 @@ static int walCheckpoint( Wal *pWal, /* Wal connection */ sqlite3_file *pFd, /* File descriptor open on db file */ int sync_flags, /* Flags for OsSync() (or 0) */ + int nBuf, /* Size of zBuf in bytes */ u8 *zBuf /* Temporary buffer to use */ ){ int rc; /* Return code */ @@ -718,14 +719,20 @@ static int walCheckpoint( u32 iDbpage = 0; /* Next database page to write */ u32 iFrame = 0; /* Wal frame containing data for iDbpage */ - if( pWal->hdr.iLastPg==0 ){ - return SQLITE_OK; - } - /* Allocate the iterator */ pIter = walIteratorInit(pWal); if( !pIter ) return SQLITE_NOMEM; + if( pWal->hdr.iLastPg==0 ){ + rc = SQLITE_OK; + goto out; + } + + if( pWal->hdr.pgsz!=nBuf ){ + rc = SQLITE_CORRUPT_BKPT; + goto out; + } + /* Sync the log file to disk */ if( sync_flags ){ rc = sqlite3OsSync(pWal->pFd, sync_flags); @@ -788,7 +795,8 @@ int sqlite3WalClose( Wal *pWal, /* Wal to close */ sqlite3_file *pFd, /* Database file */ int sync_flags, /* Flags to pass to OsSync() (or 0) */ - u8 *zBuf /* Buffer of at least page-size bytes */ + int nBuf, + u8 *zBuf /* Buffer of at least nBuf bytes */ ){ int rc = SQLITE_OK; if( pWal ){ @@ -804,7 +812,7 @@ int sqlite3WalClose( */ rc = sqlite3OsLock(pFd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ - rc = walCheckpoint(pWal, pFd, sync_flags, zBuf); + rc = walCheckpoint(pWal, pFd, sync_flags, nBuf, zBuf); if( rc==SQLITE_OK ){ isDelete = 1; } @@ -953,7 +961,13 @@ void sqlite3WalCloseSnapshot(Wal *pWal){ /* ** Read a page from the log, if it is present. */ -int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut){ +int sqlite3WalRead( + Wal *pWal, + Pgno pgno, + int *pInWal, + int nOut, + u8 *pOut +){ u32 iRead = 0; u32 *aData; int iFrame = (pWal->hdr.iLastPg & 0xFFFFFF00); @@ -1011,7 +1025,7 @@ int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut){ if( iRead ){ i64 iOffset = walFrameOffset(iRead, pWal->hdr.pgsz) + WAL_FRAME_HDRSIZE; *pInWal = 1; - return sqlite3OsRead(pWal->pFd, pOut, pWal->hdr.pgsz, iOffset); + return sqlite3OsRead(pWal->pFd, pOut, nOut, iOffset); } *pInWal = 0; @@ -1266,6 +1280,7 @@ int sqlite3WalCheckpoint( Wal *pWal, /* Wal connection */ sqlite3_file *pFd, /* File descriptor open on db file */ int sync_flags, /* Flags to sync db file with (or 0) */ + int nBuf, /* Size of temporary buffer */ u8 *zBuf, /* Temporary buffer to use */ int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ @@ -1288,7 +1303,7 @@ int sqlite3WalCheckpoint( /* Copy data from the log to the database file. */ rc = walIndexReadHdr(pWal, &isChanged); if( rc==SQLITE_OK ){ - rc = walCheckpoint(pWal, pFd, sync_flags, zBuf); + rc = walCheckpoint(pWal, pFd, sync_flags, nBuf, zBuf); } if( isChanged ){ /* If a new wal-index header was loaded before the checkpoint was diff --git a/src/wal.h b/src/wal.h index fde28d4799..c1b662d09b 100644 --- a/src/wal.h +++ b/src/wal.h @@ -24,7 +24,7 @@ # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalOpenSnapshot(y,z) 0 # define sqlite3WalCloseSnapshot(z) -# define sqlite3WalRead(w,x,y,z) 0 +# define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y,z) # define sqlite3WalWriteLock(y,z) 0 # define sqlite3WalUndo(x,y,z) 0 @@ -42,7 +42,7 @@ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ int sqlite3WalOpen(sqlite3_vfs*, const char *zDb, Wal **ppWal); -int sqlite3WalClose(Wal *pWal, sqlite3_file *pFd, int sync_flags, u8 *zBuf); +int sqlite3WalClose(Wal *pWal, sqlite3_file *pFd, int sync_flags, int, u8 *); /* Used by readers to open (lock) and close (unlock) a snapshot. A ** snapshot is like a read-transaction. It is the state of the database @@ -55,7 +55,7 @@ int sqlite3WalOpenSnapshot(Wal *pWal, int *); void sqlite3WalCloseSnapshot(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ -int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut); +int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); /* Return the size of the database as it existed at the beginning ** of the snapshot */ @@ -83,6 +83,7 @@ int sqlite3WalCheckpoint( Wal *pWal, /* Write-ahead log connection */ sqlite3_file *pFd, /* File descriptor open on db file */ int sync_flags, /* Flags to sync db file with (or 0) */ + int nBuf, /* Size of buffer nBuf */ u8 *zBuf, /* Temporary buffer to use */ int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ diff --git a/test/wal.test b/test/wal.test index 344f0bafed..2ee8005251 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1193,8 +1193,152 @@ foreach {tn sectorsize logsize} { } sqlite3_test_control_pending_byte $old_pending_byte +#------------------------------------------------------------------------- +# This test - wal-18.* - verifies a couple of specific conditions that +# may be encountered while recovering a log file are handled correctly: +# +# wal-18.1.* When the first 32-bits of a frame checksum is correct but +# the second 32-bits are false, and +# +# wal-18.2.* When the page-size field that occurs at the start of a log +# file is a power of 2 greater than 16384 or smaller than 512. +# +file delete -force test.db test.db-wal test.db-journal +do_test wal-18.0 { + sqlite3 db test.db + execsql { + PRAGMA page_size = 1024; + PRAGMA auto_vacuum = 0; + PRAGMA journal_mode = WAL; + PRAGMA synchronous = OFF; + + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(0, 0); + PRAGMA wal_checkpoint; + + INSERT INTO t1 VALUES(1, 2); -- frames 1 and 2 + INSERT INTO t1 VALUES(3, 4); -- frames 3 and 4 + INSERT INTO t1 VALUES(5, 6); -- frames 5 and 6 + } + + file copy -force test.db testX.db + file copy -force test.db-wal testX.db-wal + db close + list [file size testX.db] [file size testX.db-wal] +} [list [expr 3*1024] [log_file_size 6 1024]] + +foreach {nFrame result} { + 0 {0 0} + 1 {0 0} + 2 {0 0 1 2} + 3 {0 0 1 2} + 4 {0 0 1 2 3 4} + 5 {0 0 1 2 3 4} + 6 {0 0 1 2 3 4 5 6} +} { + do_test wal-18.1.$nFrame { + file copy -force testX.db test.db + file copy -force testX.db-wal test.db-wal + + hexio_write test.db-wal [expr 12 + $nFrame*(16+1024) + 12] 00000000 + + sqlite3 db test.db + execsql { + SELECT * FROM t1; + PRAGMA integrity_check; + } + } [concat $result ok] + db close +} + +proc randomblob {pgsz} { + sqlite3 rbdb :memory: + set blob [rbdb one {SELECT randomblob($pgsz)}] + rbdb close + set blob +} + +proc logcksum {ckv1 ckv2 blob} { + upvar $ckv1 c1 + upvar $ckv2 c2 + + binary scan $blob iu* values + foreach v $values { + incr c1 $v + incr c2 $c1 + } + + set c1 [expr ($c1 + ($c1>>24))&0xFFFFFFFF] + set c2 [expr ($c2 + ($c2>>24))&0xFFFFFFFF] +} + +file copy -force test.db testX.db +foreach {tn pgsz works} { + 1 128 0 + 2 256 0 + 3 512 1 + 4 1024 1 + 5 2048 1 + 6 4096 1 + 7 8192 1 + 8 16384 1 + 9 32768 1 + 10 65536 0 +} { + + for {set pg 1} {$pg <= 3} {incr pg} { + file copy -force testX.db test.db + file delete -force test.db-wal + + # Check that the database now exists and consists of three pages. And + # that there is no associated wal file. + # + do_test wal-18.2.$tn.$pg.1 { file exists test.db-wal } 0 + do_test wal-18.2.$tn.$pg.2 { file exists test.db } 1 + do_test wal-18.2.$tn.$pg.3 { file size test.db } [expr 1024*3] + + do_test wal-18.2.$tn.$pg.4 { + + # Create a wal file that contains a single frame (database page + # number $pg) with the commit flag set. The frame checksum is + # correct, but the contents of the database page are corrupt. + # + # The page-size in the log file header is set to $pgsz. If the + # WAL code considers $pgsz to be a valid SQLite database file page-size, + # the database will be corrupt (because the garbage frame contents + # will be treated as valid content). If $pgsz is invalid (too small + # or too large), the db will not be corrupt as the log file will + # be ignored. + # + set c1 22 + set c2 23 + set walhdr [binary format III $pgsz $c1 $c2] + set framebody [randomblob $pgsz] + set framehdr [binary format II $pg 5] + logcksum c1 c2 $framehdr + logcksum c1 c2 $framebody + set framehdr [binary format IIII $pg 5 $c1 $c2] + set fd [open test.db-wal w] + fconfigure $fd -encoding binary -translation binary + puts -nonewline $fd $walhdr + puts -nonewline $fd $framehdr + puts -nonewline $fd $framebody + close $fd + + file size test.db-wal + } [log_file_size 1 $pgsz] + + do_test wal-18.2.$tn.$pg.5 { + sqlite3 db test.db + set rc [catch { db one {PRAGMA integrity_check} } msg] + expr { $rc!=0 || $msg!="ok" } + } $works + + db close + } +} + catch { db2 close } catch { db close } finish_test - diff --git a/test/walcrash.test b/test/walcrash.test index 47f454c1e9..07c4c8c18a 100644 --- a/test/walcrash.test +++ b/test/walcrash.test @@ -250,11 +250,27 @@ for {set i 1} {$i < $REPEATS} {incr i} { db close } +#------------------------------------------------------------------------- +# This test case simulates a crash while checkpointing the database. Page +# 1 is one of the pages overwritten by the checkpoint. This is a special +# case because it means the content of page 1 may be damaged. SQLite will +# have to determine: +# +# (a) that the database is a WAL database, and +# (b) the database page-size +# +# based on the log file. +# for {set i 1} {$i < $REPEATS} {incr i} { file delete -force test.db test.db-wal + # Select a page-size for this test. + # + set pgsz [lindex {512 1024 2048 4096 8192 16384} [expr $i%6]] + do_test walcrash-7.$i.1 { - crashsql -delay 3 -file test.db -seed [incr seed] -blocksize 512 { + crashsql -delay 3 -file test.db -seed [incr seed] -blocksize 512 " + PRAGMA page_size = $pgsz; PRAGMA journal_mode = wal; BEGIN; CREATE TABLE t1(a, b); @@ -263,7 +279,7 @@ for {set i 1} {$i < $REPEATS} {incr i} { PRAGMA wal_checkpoint; CREATE INDEX i1 ON t1(a); PRAGMA wal_checkpoint; - } + " } {1 {child process exited abnormally}} do_test walcrash-7.$i.2 { -- 2.47.2