From: dan Date: Fri, 6 Oct 2017 13:43:42 +0000 (+0000) Subject: Fix frame overwriting in wal2 mode. X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=24f1b25c7b7b3b73033693c42bd9af956aaf3263;p=thirdparty%2Fsqlite.git Fix frame overwriting in wal2 mode. FossilOrigin-Name: a4b02bc9388226da21b3837a20c6c7eb0d13854dde62b7136e04f4978528dc71 --- diff --git a/manifest b/manifest index e9519e9d3f..ebdae660fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stest\scase\sfailures\son\sthis\sbranch. -D 2017-10-05T18:14:46.713 +C Fix\sframe\soverwriting\sin\swal2\smode. +D 2017-10-06T13:43:42.775 F Makefile.in 4bc36d913c2e3e2d326d588d72f618ac9788b2fd4b7efda61102611a6495c3ff F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6033b51b6aea702ea059f6ab2d47b1d3cef648695f787247dd4fb395fe60673f @@ -537,7 +537,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 3455865cef56441a90c3688b65e091df3343e18c36bd4ad9da1154ecb22f2216 +F src/wal.c 1fdb379a36877a21fa97220bbe075957c024a6d06716db692e1d35145738a5a1 F src/wal.h b6063e6be1b03389372f3f32240e99b8ab92c32cdd05aa0e31b30a21e4e41654 F src/walker.c 3ccfa8637f95355bff61144e01a615b8ef26f79c312880848da73f03367da1e6 F src/where.c 049522adcf5426f1a8c3ed07be15e1ffa3266afd34e8e7bee64b63e2fbfad0b5 @@ -1492,6 +1492,7 @@ F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 6ac39b94a284ebac6efb6be93b0cdfe73ee6083f129555e3144d8a615e9900ef +F test/wal2rewrite.test 467a66722b3e68b2c5c50f98fb62491c5f5e3f54b1d5728c7f8f1d5afe4976fe F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1656,7 +1657,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e2fc5c814cf6862d536aacb9eca66ecd31ba7e3e3033fa4c5564d533f4a18dfc -R 69b71d3158015f2955b522d7c6fc4b0d +P 16decc13af908087fb8aa34eeccf43e8da1b8f2e4b808028986d1ef08134c72c +R 756fa931da3c4e04a97cc0362e6290b7 U dan -Z 23af8b038261b8f3163841b50db40cad +Z 4ba52753cca3b375d5de2490acf53ff4 diff --git a/manifest.uuid b/manifest.uuid index 2d0fb63ef8..d9d56c6538 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -16decc13af908087fb8aa34eeccf43e8da1b8f2e4b808028986d1ef08134c72c \ No newline at end of file +a4b02bc9388226da21b3837a20c6c7eb0d13854dde62b7136e04f4978528dc71 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index db4fca8b94..54486f9969 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3006,6 +3006,30 @@ static int walSearchHash( return SQLITE_OK; } +static int walSearchWal( + Wal *pWal, + int iWal, + Pgno pgno, + u32 *piRead +){ + int rc = SQLITE_OK; + int bWal2 = isWalMode2(pWal); + u32 iLast = walidxGetMxFrame(&pWal->hdr, iWal); + if( iLast ){ + int iHash; + int iMinHash = walFramePage(pWal->minFrame); + u32 iExternal = bWal2 ? walExternalEncode(iWal, iLast) : iLast; + assert( bWal2==0 || pWal->minFrame==0 ); + for(iHash=walFramePage(iExternal); + iHash>=iMinHash && *piRead==0; + iHash-=(1+bWal2) + ){ + rc = walSearchHash(pWal, iExternal, iHash, pgno, piRead); + if( rc!=SQLITE_OK ) break; + } + } + return rc; +} /* ** Search the wal file for page pgno. If found, set *piRead to the frame that @@ -3024,8 +3048,6 @@ int sqlite3WalFindFrame( int iApp = walidxGetFile(&pWal->hdr); int rc = SQLITE_OK; u32 iRead = 0; /* If !=0, WAL frame to return data from */ - u32 iLast; /* Last frame in wal file */ - int iHash; /* Used to loop through N hash tables */ /* This routine is only be called from within a read transaction. */ assert( pWal->readLock!=WAL_LOCK_NONE ); @@ -3041,18 +3063,8 @@ int sqlite3WalFindFrame( assert( bWal2 || iApp==0 ); /* Search the wal file that the client holds a partial lock on first */ - iLast = walidxGetMxFrame(&pWal->hdr, iApp); - if( iLast ){ - u32 iExternal = bWal2 ? walExternalEncode(iApp, iLast) : iLast; - int iMinHash = walFramePage(pWal->minFrame); - for(iHash=walFramePage(iExternal); - iHash>=iMinHash && iRead==0; - iHash-=(1+bWal2) - ){ - rc = walSearchHash(pWal, iExternal, iHash, pgno, &iRead); - if( rc!=SQLITE_OK ) break; - } - } + + rc = walSearchWal(pWal, iApp, pgno, &iRead); /* If the requested page was not found, no error has occured, and ** the client holds a full-wal lock on the other wal file, search it @@ -3061,14 +3073,7 @@ int sqlite3WalFindFrame( pWal->readLock==WAL_LOCK_PART1_FULL2 || pWal->readLock==WAL_LOCK_PART2_FULL1 )){ - iLast = walidxGetMxFrame(&pWal->hdr, !iApp); - if( iLast ){ - u32 iExternal = walExternalEncode(!iApp, iLast); - for(iHash=walFramePage2(!iApp, iLast); iHash>=0 && iRead==0; iHash -= 2){ - rc = walSearchHash(pWal, iExternal, iHash, pgno, &iRead); - if( rc!=SQLITE_OK ) break; - } - } + rc = walSearchWal(pWal, !iApp, pgno, &iRead); } #if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) @@ -3526,6 +3531,7 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-headers in */ u32 iRead; /* Next frame to read from wal file */ i64 iCksumOff; + sqlite3_file *pWalFd = pWal->apWalFd[walidxGetFile(&pWal->hdr)]; assert( isWalMode2(pWal)==0 ); @@ -3589,6 +3595,7 @@ int sqlite3WalFrames( u32 iFirst = 0; /* First frame that may be overwritten */ WalIndexHdr *pLive; /* Pointer to shared header */ int iApp; + int bWal2 = isWalMode2(pWal); assert( pList ); assert( pWal->writeLock ); @@ -3599,9 +3606,8 @@ int sqlite3WalFrames( pLive = (WalIndexHdr*)walIndexHdr(pWal); if( memcmp(&pWal->hdr, (void *)pLive, sizeof(WalIndexHdr))!=0 ){ - if( isWalMode2(pWal)==0 ){ - iFirst = pLive->mxFrame+1; - } + /* if( isWalMode2(pWal)==0 ) */ + iFirst = walidxGetMxFrame(pLive, walidxGetFile(pLive))+1; } /* See if it is possible to write these frames into the start of the @@ -3617,7 +3623,7 @@ int sqlite3WalFrames( */ iApp = walidxGetFile(&pWal->hdr); iFrame = walidxGetMxFrame(&pWal->hdr, iApp); - assert( iApp==0 || isWalMode2(pWal) ); + assert( iApp==0 || bWal2 ); #if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){} @@ -3634,7 +3640,7 @@ int sqlite3WalFrames( sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN)); sqlite3Put4byte(&aWalHdr[4], pWal->hdr.iVersion); sqlite3Put4byte(&aWalHdr[8], szPage); - if( isWalMode2(pWal) ){ + if( bWal2 ){ if( walidxGetMxFrame(&pWal->hdr, !iApp)>0 ){ u8 aPrev[4]; rc = sqlite3OsRead(pWal->apWalFd[!iApp], aPrev, 4, 12); @@ -3697,8 +3703,11 @@ int sqlite3WalFrames( ** checksums must be recomputed when the transaction is committed. */ if( iFirst && (p->pDirty || isCommit==0) ){ u32 iWrite = 0; - VVA_ONLY(rc =) sqlite3WalFindFrame(pWal, p->pgno, &iWrite); + VVA_ONLY(rc =) walSearchWal(pWal, iApp, p->pgno, &iWrite); assert( rc==SQLITE_OK || iWrite==0 ); + if( iWrite && bWal2 ){ + walExternalDecode(iWrite, &iWrite); + } if( iWrite>=iFirst ){ i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE; void *pData; @@ -3810,7 +3819,7 @@ int sqlite3WalFrames( /* If this is a commit, update the wal-index header too. */ if( isCommit ){ walIndexWriteHdr(pWal); - if( isWalMode2(pWal) ){ + if( bWal2 ){ int iOther = !walidxGetFile(&pWal->hdr); if( walidxGetMxFrame(&pWal->hdr, iOther) && !walCkptInfo(pWal)->nBackfill diff --git a/test/wal2rewrite.test b/test/wal2rewrite.test new file mode 100644 index 0000000000..dea4f8c4f5 --- /dev/null +++ b/test/wal2rewrite.test @@ -0,0 +1,80 @@ +# 2017 September 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2rewrite +ifcapable !wal {finish_test ; return } + +proc filesize {filename} { + if {[file exists $filename]} { + return [file size $filename] + } + return 0 +} + +foreach {tn jrnlmode} { + 1 wal + 2 wal2 +} { + reset_db + execsql "PRAGMA journal_mode = $jrnlmode" + do_execsql_test $tn.1 { + PRAGMA journal_size_limit = 10000; + PRAGMA cache_size = 5; + PRAGMA wal_autocheckpoint = 10; + + CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER, c BLOB); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + + WITH s(i) AS ( + SELECT 1 UNION SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO t1 SELECT i, i, randomblob(800) FROM s; + } {10000 10} + + for {set i 0} {$i < 4} {incr i} { + do_execsql_test $tn.$i.1 { + UPDATE t1 SET c=randomblob(800) WHERE (b%10)==5 AND ($i%2) + } + do_execsql_test $tn.$i.2 { + BEGIN; + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + UPDATE t1 SET b=b+10, c=randomblob(800); + } + execsql COMMIT + + do_test $tn.$i.3 { expr [filesize test.db-wal] < 100000 } 1 + do_test $tn.$i.4 { expr [filesize test.db-wal2] < 100000 } 1 + } + +} + + + +finish_test