-C Add\sa\scouple\sof\smissing\smethods\sto\stest_osinst.c..
-D 2010-05-22T08:22:40
+C Change\sthe\sWAL\sfile\sformat\sto\ssupport\stwo\skinds\sof\schecksums\s-\sone\sthat\sis\sfast\sto\scalculate\son\slittle-endian\sarchitectures\sand\sanother\sthat\sis\sfast\son\sbig-endian\sarchitectures.\sA\sflag\sin\sthe\swal-header\sindicates\swhich\sthe\sfile\suses.
+D 2010-05-24T10:39:36
F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1
F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2
F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda
-F src/wal.c 8371cedf1f9fb8b0d99f31598e276d7a03fb2b05
+F src/wal.c 93e13dfc60e036091eec8c90da4ff9595e97e932
F src/wal.h 434f76f51225bb614e43ccb6bd2341541ba6a06e
F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
F src/where.c 75fee9e255b62f773fcadd1d1f25b6f63ac7a356
F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
-F test/wal.test 90afd254ece957a716751b1c35fac02d6353c2a7
+F test/wal.test 3b8ad018c1faf89d3f5bb23704775f5d20e486de
F test/wal2.test 053c9ea94194c5bce5b742429be75ff2432794ab
F test/walbak.test e7650a26eb4b8abeca9b145b1af1e63026dde432
+F test/walcksum.test cc41a85d8b6f1471ebdf847f82f39dd0003a37bc
F test/walcrash.test f6d5fb2bb108876f04848720a488065d9deef69f
F test/walfault.test f71d4c9a13d4e27086aef55f1e0e94734ffa2f6a
F test/walhook.test 67e675127f4acb72f061a12667ce6e5460b06b78
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 7aade899e55f4565f02d301e1e83fb0bac2ea500
-R cf1514dd18e6972433d06e5339115fa6
+P 5c9e9c06ae350043e66f36087da4021a52e6ee17
+R 7dee910e14836a9af936b980d0803808
U dan
-Z 8423981b7570a11d3c3148c90911df31
+Z e618b798a3fca06f0d820ccfd6fdc7f4
-5c9e9c06ae350043e66f36087da4021a52e6ee17
\ No newline at end of file
+65ba804dd1d31d1eef6ae3f40a3ade344a410b84
\ No newline at end of file
*/
struct WalIndexHdr {
u32 iChange; /* Counter incremented each transaction */
- u32 szPage; /* Database page size in bytes */
+ u16 bigEndCksum; /* True if checksums in WAL are big-endian */
+ u16 szPage; /* Database page size in bytes */
u32 mxFrame; /* Index of last valid frame in the WAL */
u32 nPage; /* Size of database in pages */
u32 aSalt[2]; /* Salt-1 and salt-2 values copied from WAL header */
/* Size of write ahead log header */
#define WAL_HDRSIZE 24
+/* WAL magic value. Either this value, or the same value with the least
+** significant bit also set (WAL_MAGIC | 0x00000001) is stored in 32-bit
+** big-endian format in the first 4 bytes of a WAL file.
+**
+** If the LSB is set, then the checksums for each frame within the WAL
+** file are calculated by treating all data as an array of 32-bit
+** big-endian words. Otherwise, they are calculated by interpreting
+** all data as 32-bit little-endian words.
+*/
+#define WAL_MAGIC 0x377f0682
+
/*
** Return the offset of frame iFrame in the write-ahead log file,
** assuming a database page size of szPage bytes. The offset returned
} aSegment[1]; /* One for every 256 entries in the WAL */
};
+/*
+** The argument to this macro must be of type u32. On a little-endian
+** architecture, it returns the u32 value that results from interpreting
+** the 4 bytes as a big-endian value. On a big-endian architecture, it
+** returns the value that would be produced by intepreting the 4 bytes
+** of the input value as a little-endian integer.
+*/
+#define BYTESWAP32(x) ( \
+ (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \
+ + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \
+)
/*
** Generate or extend an 8 byte checksum based on the data in
** nByte must be a positive multiple of 8.
*/
static void walChecksumBytes(
+ int nativeCksum, /* True for native byte-order, false for non-native */
u8 *a, /* Content to be checksummed */
int nByte, /* Bytes of content in a[]. Must be a multiple of 8. */
const u32 *aIn, /* Initial checksum value input */
u32 *aOut /* OUT: Final checksum value output */
){
u32 s1, s2;
- u8 *aEnd = (u8*)&a[nByte];
+ u32 *aData = (u32 *)a;
+ u32 *aEnd = (u32 *)&a[nByte];
+
if( aIn ){
s1 = aIn[0];
s2 = aIn[1];
}
assert( nByte>=8 );
- assert( (nByte&0x00000003)==0 );
+ assert( (nByte&0x00000007)==0 );
+
+ if( nativeCksum ){
+ do {
+ s1 += *aData++ + s2;
+ s2 += *aData++ + s1;
+ }while( aData<aEnd );
+ }else{
+ do {
+ s1 += BYTESWAP32(aData[0]) + s2;
+ s2 += BYTESWAP32(aData[1]) + s1;
+ aData += 2;
+ }while( aData<aEnd );
+ }
- do {
- s1 += (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3] + s2;
- s2 += (a[4]<<24) + (a[5]<<16) + (a[6]<<8) + a[7] + s1;
- a += 8;
- }while( a<aEnd );
aOut[0] = s1;
aOut[1] = s2;
}
*/
static void walIndexWriteHdr(Wal *pWal){
WalIndexHdr *aHdr;
- walChecksumBytes((u8*)&pWal->hdr,
+ walChecksumBytes(1, (u8*)&pWal->hdr,
sizeof(pWal->hdr) - sizeof(pWal->hdr.aCksum),
0, pWal->hdr.aCksum);
aHdr = (WalIndexHdr*)pWal->pWiData;
u8 *aData, /* Pointer to page data */
u8 *aFrame /* OUT: Write encoded frame here */
){
+ int nativeCksum; /* True for native byte-order checksums */
u32 aCksum[2];
assert( WAL_FRAME_HDRSIZE==24 );
sqlite3Put4byte(&aFrame[0], iPage);
sqlite3Put4byte(&aFrame[4], nTruncate);
memcpy(&aFrame[8], pWal->hdr.aSalt, 8);
- walChecksumBytes(aFrame, 16, 0, aCksum);
- walChecksumBytes(aData, pWal->szPage, aCksum, aCksum);
+ nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN);
+ walChecksumBytes(nativeCksum, aFrame, 16, 0, aCksum);
+ walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum);
sqlite3Put4byte(&aFrame[16], aCksum[0]);
sqlite3Put4byte(&aFrame[20], aCksum[1]);
u8 *aData, /* Pointer to page data (for checksum) */
u8 *aFrame /* Frame data */
){
+ int nativeCksum; /* True for native byte-order checksums */
u32 aCksum[2];
assert( WAL_FRAME_HDRSIZE==24 );
** of the frame-header, and the frame-data matches
** the checksum in the last 8 bytes of the frame-header.
*/
- walChecksumBytes(aFrame, 16, 0, aCksum);
- walChecksumBytes(aData, pWal->szPage, aCksum, aCksum);
+ nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN);
+ walChecksumBytes(nativeCksum, aFrame, 16, 0, aCksum);
+ walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum);
if( aCksum[0]!=sqlite3Get4byte(&aFrame[16])
|| aCksum[1]!=sqlite3Get4byte(&aFrame[20])
){
static int walIndexRecover(Wal *pWal){
int rc; /* Return Code */
i64 nSize; /* Size of log file */
- WalIndexHdr hdr; /* Recovered wal-index header */
+ WalIndexHdr hdr; /* Recovered wal-index header */
assert( pWal->lockState>SQLITE_SHM_READ );
memset(&hdr, 0, sizeof(hdr));
return rc;
}
- if( nSize>WAL_FRAME_HDRSIZE ){
- u8 aBuf[WAL_HDRSIZE]; /* Buffer to load first frame header into */
+ if( nSize>WAL_HDRSIZE ){
+ u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */
u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */
int szFrame; /* Number of bytes in buffer aFrame[] */
u8 *aData; /* Pointer to data part of aFrame buffer */
int iFrame; /* Index of last frame read */
i64 iOffset; /* Next offset to read from log file */
int szPage; /* Page size according to the log */
+ u32 magic; /* Magic value read from WAL header */
- /* Read in the first frame header in the file (to determine the
- ** database page size).
- */
+ /* Read in the WAL header. */
rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0);
if( rc!=SQLITE_OK ){
return rc;
}
/* If the database page size is not a power of two, or is greater than
- ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid data.
+ ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid
+ ** data. Similarly, if the 'magic' value is invalid, ignore the whole
+ ** WAL file.
*/
+ magic = sqlite3Get4byte(&aBuf[0]);
szPage = sqlite3Get4byte(&aBuf[8]);
- if( szPage&(szPage-1) || szPage>SQLITE_MAX_PAGE_SIZE || szPage<512 ){
+ if( (magic&0xFFFFFFFE)!=WAL_MAGIC
+ || szPage&(szPage-1)
+ || szPage>SQLITE_MAX_PAGE_SIZE
+ || szPage<512
+ ){
goto finished;
}
+ hdr.bigEndCksum = pWal->hdr.bigEndCksum = (magic&0x00000001);
pWal->szPage = szPage;
pWal->nCkpt = sqlite3Get4byte(&aBuf[12]);
memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);
if( h1.szPage==0 ){
return 1; /* Malformed header - probably all zeros */
}
- walChecksumBytes((u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum);
+ walChecksumBytes(1, (u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum);
if( aCksum[0]!=h1.aCksum[0] || aCksum[1]!=h1.aCksum[1] ){
return 1; /* Checksum does not match */
}
iFrame = pWal->hdr.mxFrame;
if( iFrame==0 ){
u8 aWalHdr[WAL_HDRSIZE]; /* Buffer to assembly wal-header in */
- sqlite3Put4byte(&aWalHdr[0], 0x377f0682);
+ sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN));
sqlite3Put4byte(&aWalHdr[4], 3007000);
sqlite3Put4byte(&aWalHdr[8], szPage);
pWal->szPage = szPage;
+ pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN;
sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt);
memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8);
rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0);
upvar $ckv1 c1
upvar $ckv2 c2
+ set scanpattern I*
+ if {$::tcl_platform(byteOrder) eq "littleEndian"} {
+ set scanpattern i*
+ }
- binary scan $blob I* values
+ binary scan $blob $scanpattern values
foreach {v1 v2} $values {
set c1 [expr {($c1 + $v1 + $c2)&0xFFFFFFFF}]
set c2 [expr {($c2 + $v2 + $c1)&0xFFFFFFFF}]
logcksum c1 c2 $framehdr
logcksum c1 c2 $framebody
set framehdr [binary format IIIIII $pg 5 22 23 $c1 $c2]
+
set fd [open test.db-wal w]
fconfigure $fd -encoding binary -translation binary
puts -nonewline $fd $walhdr
--- /dev/null
+# 2010 May 24
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+
+ifcapable !wal {finish_test ; return }
+
+# This proc calculates checksums in the same way as those used by SQLite
+# in WAL files. If the $endian argument is "big", then checksums are
+# calculated by interpreting data as an array of big-endian integers. If
+# it is "little", data is interpreted as an array of little-endian integers.
+#
+proc log_cksum {endian ckv1 ckv2 blob} {
+ upvar $ckv1 c1
+ upvar $ckv2 c2
+
+ if {$endian!="big" && $endian!="little"} {
+ return -error "Bad value \"$endian\" - must be \"big\" or \"little\""
+ }
+ set scanpattern I*
+ if {$endian == "little"} { set scanpattern i* }
+
+ binary scan $blob $scanpattern values
+ foreach {v1 v2} $values {
+ set c1 [expr {($c1 + $v1 + $c2)&0xFFFFFFFF}]
+ set c2 [expr {($c2 + $v2 + $c1)&0xFFFFFFFF}]
+ }
+}
+
+proc log_file_size {nFrame pgsz} {
+ expr {24 + ($pgsz+24)*$nFrame}
+}
+
+# Read and return the contents of file $filename. Treat the content as
+# binary data.
+#
+proc readfile {filename} {
+ set fd [open $filename]
+ fconfigure $fd -encoding binary
+ fconfigure $fd -translation binary
+ set data [read $fd]
+ close $fd
+ return $data
+}
+
+#
+# File $filename must be a WAL file on disk. Check that the checksum of frame
+# $iFrame in the file is correct when interpreting data as $endian-endian
+# integers ($endian must be either "big" or "little"). If the checksum looks
+# correct, return 1. Otherwise 0.
+#
+proc log_checksum_verify {filename iFrame endian} {
+ set data [readfile $filename]
+ set c1 0
+ set c2 0
+
+ binary scan [string range $data 8 11] I pgsz
+
+ set n [log_file_size [expr $iFrame-1] $pgsz]
+ binary scan [string range $data [expr $n+16] [expr $n+23]] II expect1 expect2
+ log_cksum $endian c1 c2 [string range $data $n [expr $n+15]]
+ log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]]
+
+ set expect1 [expr $expect1&0xFFFFFFFF]
+ set expect2 [expr $expect2&0xFFFFFFFF]
+ expr {$c1==$expect1 && $c2==$expect2}
+}
+
+#
+# File $filename must be a WAL file on disk. Compute the checksum for frame
+# $iFrame in the file by interpreting data as $endian-endian integers
+# ($endian must be either "big" or "little"). Then write the computed
+# checksum into the file.
+#
+proc log_checksum_write {filename iFrame endian} {
+ set data [readfile $filename]
+ set c1 0
+ set c2 0
+
+ binary scan [string range $data 8 11] I pgsz
+
+ set n [log_file_size [expr $iFrame-1] $pgsz]
+ log_cksum $endian c1 c2 [string range $data $n [expr $n+15]]
+ log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]]
+
+ set bin [binary format II $c1 $c2]
+ set fd [open $filename r+]
+ fconfigure $fd -encoding binary
+ fconfigure $fd -translation binary
+ seek $fd [expr $n+16]
+ puts -nonewline $fd $bin
+ close $fd
+}
+
+#
+# File $filename must be a WAL file on disk. Set the 'magic' field of the
+# WAL header to indicate that checksums are $endian-endian ($endian must be
+# either "big" or "little").
+#
+proc log_checksum_writemagic {filename endian} {
+ set val [expr {0x377f0682 | ($endian == "big" ? 1 : 0)}]
+ set bin [binary format I $val]
+ set fd [open $filename r+]
+ fconfigure $fd -encoding binary
+ fconfigure $fd -translation binary
+ puts -nonewline $fd $bin
+ close $fd
+}
+
+#-------------------------------------------------------------------------
+# Test cases walcksum-1.* attempt to verify the following:
+#
+# * That both native and non-native order checksum log files can
+# be recovered.
+#
+# * That when appending to native or non-native checksum log files
+# SQLite continues to use the right kind of checksums.
+#
+# * Test point 2 when the appending process is not one that recovered
+# the log file.
+#
+# * Test that both native and non-native checksum log files can be
+# checkpointed. And that after doing so the next write to the log
+# file occurs using native byte-order checksums.
+#
+set native "big"
+if {$::tcl_platform(byteOrder) == "littleEndian"} { set native "little" }
+foreach endian {big little} {
+
+ # Create a database. Leave some data in the log file.
+ #
+ do_test walcksum-1.$endian.1 {
+ catch { db close }
+ file delete -force test.db test.db-wal test.db-journal
+ sqlite3 db test.db
+ execsql {
+ PRAGMA page_size = 1024;
+ PRAGMA auto_vacuum = 0;
+ PRAGMA synchronous = NORMAL;
+
+ CREATE TABLE t1(a PRIMARY KEY, b);
+ INSERT INTO t1 VALUES(1, 'one');
+ INSERT INTO t1 VALUES(2, 'two');
+ INSERT INTO t1 VALUES(3, 'three');
+ INSERT INTO t1 VALUES(5, 'five');
+
+ PRAGMA journal_mode = WAL;
+ INSERT INTO t1 VALUES(8, 'eight');
+ INSERT INTO t1 VALUES(13, 'thirteen');
+ INSERT INTO t1 VALUES(21, 'twentyone');
+ }
+
+ file copy -force test.db test2.db
+ file copy -force test.db-wal test2.db-wal
+ db close
+
+ list [file size test2.db] [file size test2.db-wal]
+ } [list [expr 1024*3] [log_file_size 6 1024]]
+
+ # Verify that the checksums are valid for all frames and that they
+ # are calculated by interpreting data in native byte-order.
+ #
+ for {set f 1} {$f <= 6} {incr f} {
+ do_test walcksum-1.$endian.2.$f {
+ log_checksum_verify test2.db-wal $f $native
+ } 1
+ }
+
+ # Replace all checksums in the current WAL file with $endian versions.
+ # Then check that it is still possible to recover and read the database.
+ #
+ for {set f 1} {$f <= 6} {incr f} {
+ do_test walcksum-1.$endian.3.$f {
+ log_checksum_write test2.db-wal $f $endian
+ log_checksum_verify test2.db-wal $f $endian
+ } {1}
+ }
+ do_test walcksum-1.$endian.4.1 {
+ log_checksum_writemagic test2.db-wal $endian
+ file copy -force test2.db test.db
+ file copy -force test2.db-wal test.db-wal
+ sqlite3 db test.db
+ execsql { SELECT a FROM t1 }
+ } {1 2 3 5 8 13 21}
+
+ # Following recovery, any frames written to the log should use the same
+ # endianness as the existing frames. Check that this is the case.
+ #
+ do_test walcksum-1.$endian.5.0 {
+ execsql {
+ PRAGMA synchronous = NORMAL;
+ INSERT INTO t1 VALUES(34, 'thirtyfour');
+ }
+ list [file size test.db] [file size test.db-wal]
+ } [list [expr 1024*3] [log_file_size 8 1024]]
+ for {set f 1} {$f <= 8} {incr f} {
+ do_test walcksum-1.$endian.5.$f {
+ log_checksum_verify test.db-wal $f $endian
+ } {1}
+ }
+
+ # Now connect a second connection to the database. Check that this one
+ # (not the one that did recovery) also appends frames to the log using
+ # the same endianness for checksums as the existing frames.
+ #
+ do_test walcksum-1.$endian.6 {
+ sqlite3 db2 test.db
+ execsql {
+ PRAGMA integrity_check;
+ SELECT a FROM t1;
+ } db2
+ } {ok 1 2 3 5 8 13 21 34}
+ do_test walcksum-1.$endian.7.0 {
+ execsql {
+ PRAGMA synchronous = NORMAL;
+ INSERT INTO t1 VALUES(55, 'fiftyfive');
+ } db2
+ list [file size test.db] [file size test.db-wal]
+ } [list [expr 1024*3] [log_file_size 10 1024]]
+ for {set f 1} {$f <= 10} {incr f} {
+ do_test walcksum-1.$endian.7.$f {
+ log_checksum_verify test.db-wal $f $endian
+ } {1}
+ }
+
+ # Now that both the recoverer and non-recoverer have added frames to the
+ # log file, check that it can still be recovered.
+ #
+ file copy -force test.db test2.db
+ file copy -force test.db-wal test2.db-wal
+ do_test walcksum-1.$endian.7.11 {
+ sqlite3 db3 test2.db
+ execsql {
+ PRAGMA integrity_check;
+ SELECT a FROM t1;
+ } db3
+ } {ok 1 2 3 5 8 13 21 34 55}
+ db3 close
+
+ # Run a checkpoint on the database file. Then, check that any frames written
+ # to the start of the log use native byte-order checksums.
+ #
+ do_test walcksum-1.$endian.8.1 {
+ execsql {
+ PRAGMA wal_checkpoint;
+ INSERT INTO t1 VALUES(89, 'eightynine');
+ }
+ log_checksum_verify test.db-wal 1 $native
+ } {1}
+ do_test walcksum-1.$endian.8.2 {
+ log_checksum_verify test.db-wal 2 $native
+ } {1}
+ do_test walcksum-1.$endian.8.3 {
+ log_checksum_verify test.db-wal 3 $native
+ } [expr {$native == $endian}]
+
+ do_test walcksum-1.$endian.9 {
+ execsql {
+ PRAGMA integrity_check;
+ SELECT a FROM t1;
+ } db2
+ } {ok 1 2 3 5 8 13 21 34 55 89}
+
+ catch { db close }
+ catch { db2 close }
+}
+
+finish_test
projects / thirdparty / sqlite.git / commitdiff
