From: dan Date: Tue, 11 Aug 2020 11:34:00 +0000 (+0000) Subject: Merge latest trunk changes into this branch. X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=174f4fa241b6c76b792f9f0e27e707fdfd05b0be;p=thirdparty%2Fsqlite.git Merge latest trunk changes into this branch. FossilOrigin-Name: 91262e665755a1430f3b667867b9c4fa0cc3aa0388c21e8ba436485beefa9f4a --- 174f4fa241b6c76b792f9f0e27e707fdfd05b0be diff --cc manifest index 7bd5393dab,e9d8b99f0d..89f4cd0170 --- a/manifest +++ b/manifest @@@ -1,5 -1,5 +1,5 @@@ -C Simplify\s#ifdefs\sassociated\swith\sParse.eParseMode.\s\sFix\san\s#ifdef\serror\nassociated\swith\sSQLITE_OMIT_AUTOVACUUM. -D 2020-08-10T21:01:32.620 +C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. - D 2020-07-31T16:01:33.582 ++D 2020-08-11T11:34:00.038 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@@ -478,10 -476,10 +478,10 @@@ F src/auth.c a3d5bfdba83d25abed1013a8c7 F src/backup.c b1c90cd4110248c8e1273ff4578d3a84c0c34725e1b96dacd4a6294a908702de F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 - F src/btree.c 8a28f5051eefadffaf13e6b7dc2c34fb6dbba50fdd870e2c984208445438472f -F src/btree.c 1439fd9b45d4d1883c53752daef42af489adaa1a1508fa39dedbc9c80ea21a2f ++F src/btree.c e0f00b5a9533ba7c4a5fe1765bae56c8257bc4a27ce3d38e72d8a05b42621a1b F src/btree.h 7af72bbb4863c331c8f6753277ab40ee67d2a2125a63256d5c25489722ec162b F src/btreeInt.h 83166f6daeb91062b6ae9ee6247b3ad07e40eba58f3c05ba9e8dedad4ab1ea38 - F src/build.c c2e1390b8bff21daf9da03d097e4f824cdf4c15cd823ba072a7820694c8728eb + F src/build.c dde514651cf3cf3a3a64a6e5c62384c01486c6faeabd165b3fe6dec6d2a0e113 F src/callback.c d0b853dd413255d2e337b34545e54d888ea02f20da5ad0e63585b389624c4a6c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c e98518d2d3d4029a13c805e07313fb60c877be56db76e90dd5f3af73085d0ce6 @@@ -524,21 -522,21 +524,21 @@@ F src/os_setup.h 0dbaea40a7d36bf311613d F src/os_unix.c 9b1b860163fd2d4d7679b5260d384d1a9f88ef917a90f28963eca8acd472d8c8 F src/os_win.c a2149ff0a85c1c3f9cc102a46c673ce87e992396ba3411bfb53db66813b32f1d F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a - F src/pager.c 9bf6108cb79596f28051fe8681c5df9d6d6f66a90959b4919573f387445212eb -F src/pager.c 3700a1c55427a3d4168ad1f1b8a8b0cb9ace1d107e4506e30a8f1e66d8a1195e -F src/pager.h 4bf9b3213a4b2bebbced5eaa8b219cf25d4a82f385d093cd64b7e93e5285f66f ++F src/pager.c 35fcb2d9400404d0d8f719461b411076bcd5597574b7a238f3a04f959842a88d +F src/pager.h 9d38ac4b9f0e1361f7ddb6aa0675a54f5e4f085703108d3aafe7868872113f92 F src/parse.y 5bdb760a29c0b25caf7e80e82210b81cd2ea3066d5199ca29e6eac40b34bc184 F src/pcache.c 385ff064bca69789d199a98e2169445dc16e4291fa807babd61d4890c3b34177 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 6596e10baf3d8f84cc1585d226cf1ab26564a5f5caf85a15757a281ff977d51a -F src/pragma.c bdb600be936f66b9fe69d26dfbba4528beaaf4f95c479c85b328a92484e0bf71 +F src/pragma.c eedd779f8dd3c0c6328454510311286da41c298b85848105ed88d00b886daad8 F src/pragma.h 8dc78ab7e9ec6ce3ded8332810a2066f1ef6267e2e03cd7356ee00276125c6cf F src/prepare.c 3d5a761d026052bc888d1b803a06dd2bfe245e8e836d4689f927003549148b0f - F src/printf.c 94b5419ad0a17269f76a9e968ca19cf9fa37617abed2e246fc48844e511b6bc6 + F src/printf.c 9efcd4e984f22bcccb1ded37a1178cac98f6e3a0534e1e0629f64899971f8838 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 - F src/resolve.c 2dd6821aac2cd27de9fcf6aa6d1f8c41b4b5841c9bc58bf1c9109008009a3a2e + F src/resolve.c d74715aceed2a8f493ba244d535646fa93132042a4400a29dfd26ec841514048 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 - F src/select.c d977d463bdeb9be4bec16829c7c2c6193330bf0962bfdf23113e8f6ba548cd86 - F src/shell.c.in 352a0a6399ccae40a30f72ea06f52f3791a062bde9b8929a97f345e1584ba310 + F src/select.c 510fdf819f218be3dac2683d3eaaf64e5080f548061a4dd12205590beda976bb + F src/shell.c.in b9b819feede7b85585ab0826490a352e04e2ee46e8132c92597d29972b2be1d7 F src/sqlite.h.in d2c03414a8ee5d4a6855c04dd7cd5998e45139b0fe66b65bae86d4223edd091f F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 2d1af80082edffd71c6f96f70ad1ce6a4fb46615ad10291fc77fe0dea9ff0197 @@@ -609,11 -607,11 +609,11 @@@ F src/upsert.c 2920de71b20f04fe25eb00b6 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c c8bf30c4356b091bcc3b624d0e24b2b4d11b8be4d6c90d8e0705971e15cc819b F src/vacuum.c 1c4f8e2f39d950037f4cf946b6858c993d3a54c3101f78e05c76460a073afcf0 - F src/vdbe.c a8a69553e46a50f3cb104753b13e3b65e94c289735af60c8ef9a6d72473a7177 -F src/vdbe.c e9f7f818f128c8600058c0eabb6b3975974c95153a104d340f419adabbc15b9f ++F src/vdbe.c d4caea3cc077c74c385dc5c25993ef699c6bd0c572f188748b2b2438162782dd F src/vdbe.h 83603854bfa5851af601fc0947671eb260f4363e62e960e8a994fb9bbcd2aaa1 F src/vdbeInt.h 762abffb7709f19c2cb74af1bba73a900f762e64f80d69c31c9ae89ed1066b60 - F src/vdbeapi.c c1a9004ac554d8d48794d2ce5f80397f8e419fd28643a543cc1e004c7713c3ef - F src/vdbeaux.c 1cbbbffdb874c6f3e7aab40f3deb48abac4a71df1043cd95bb0d652d4e053871 + F src/vdbeapi.c c5e7cb2ab89a24d7f723e87b508f21bfb1359a04db5277d8a99fd1e015c12eb9 + F src/vdbeaux.c 73854da7a9a4f12db72a855758214173c82f46a14be6cb19e63677ba02c97cae F src/vdbeblob.c 253ed82894924c362a7fa3079551d3554cd1cdace39aa833da77d3bc67e7c1b1 F src/vdbemem.c 947f2a65910edb4014dc981d33e414a68c51f169f9df8c4c493a0ba840b6eb1f F src/vdbesort.c 2be76d26998ce2b3324cdcc9f6443728e54b6c7677c553ad909c7d7cfab587df @@@ -621,8 -619,8 +621,8 @@@ F src/vdbetrace.c fa3bf238002f0bbbdfb66 F src/vdbevtab.c f99b275366c5fc5e2d99f734729880994ab9500bdafde7fae3b02d562b9d323c F src/vtab.c 5f5fc793092f53bbdfde296c50f563fb7bda58cf48e9cf6a8bdfbc5abd409845 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 - F src/wal.c 47092a65e8eed2ebf52c606578e0f76303faf899de1a89aec1317d2c61c36a01 -F src/wal.c 7a05a519a02ffb7f2a458838a25853c7300c9e6d9ef546ee48469378ac0404f9 -F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a ++F src/wal.c dafdd709d1e01f77719a983f5b3071ccd82808d59a044f8ce48cd0bfa1c1fea5 +F src/wal.h d01234e828943e002040c22a7e017642962f9fd9b2dc142fa599769ae4e459e9 F src/walker.c 3df26a33dc4f54e8771600fb7fdebe1ece0896c2ad68c30ab40b017aa4395049 F src/where.c 2ea911238674e9baaeddf105dddabed92692a01996073c4d4983f9a7efe481f9 F src/whereInt.h 6b874aa15f94e43a2cec1080be64d955b04deeafeac90ffb5d6975c0d511be3c @@@ -1245,7 -1243,7 +1245,7 @@@ F test/parser1.test 6ccdf5e459a5dc4673d F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff - F test/permutations.test 8ae7eadb43511f8c5b6b76860a78d31828c2856d0ea44499fc6215321d21bef6 -F test/permutations.test 4d174cfc92e31aff96a383dac767a94d649566b73857875afb7511e3e430d5f5 ++F test/permutations.test 1665c3f92c00cfe4643fbc39fc23a72e4a5fb673944807a5fa95b0c6573e1a80 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f F test/pragma.test 50b91bedea9324d3ab48e793f908ee7d2c7dcf84bfa2281e792838be59641ec8 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@@ -1674,18 -1672,8 +1674,18 @@@ F test/vtab_alter.test 736e66fb5ec7b4fe F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7edca12 -F test/wal.test 16180bc4becda176428ad02eaea437b4b8f5ae099314de443a4e12b2dcc007a2 +F test/wal.test 09c9e05591a5e513ecf14f72b4ca86726fca778a7abab1063a16f3ceb8cf2a86 F test/wal2.test 31f6e2c404b9f2cdf9ca19b105a1742fdc19653c2c936da39e3658c617524046 +F test/wal2big.test 0b4ec526f9ca4bbabc355042c38045ae2e253fb46eb327bb7693d0122bc6968b +F test/wal2fault.test 2e8e60cacd5bcd451618aeffd05f676894d17202d3e2986e288d36e2c5993249 +F test/wal2lock.test 0ef98d72dc6bcf7711dedd684760488400d9a9a6eec0dc5d3822060437793552 +F test/wal2recover.test ba8f4bc9397c838734619f9e759bd98b00e355347b3cf80a2e677610d231d5d8 +F test/wal2recover2.test 0c46afc759e4392a3c12fba17432b880c93a13bf4246d1be5101b00bae4c5f01 +F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c +F test/wal2rollback.test 23adc4a099b23f6aaea8b04fdca1c35861d887dd80f8be7da2d5273eb777e428 +F test/wal2savepoint.test 2c82bd6a6ee5066c156040d2e9c2415646fcf96116ae7ad127eaf0c0b4a85f22 - F test/wal2simple.test 96206c98bf64ab20ec00a1c0f6c709e258b98b39f2149889361f31966ce5a703 ++F test/wal2simple.test 4a9bfc34dc888e75acb4e7bdce0bf15e960cbee02f185c07a396b35c49d4431d +F test/wal2snapshot.test 7a5f4629a3c43a43c3440b8b2ea9f07de91a46b0b9eea5f08f62b5bf5b6468df F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@@ -1891,7 -1879,7 +1891,7 @@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 - P 5c1837572586902313702c46057ef5dc84030683ff3467c6f3f25903b6ab22d1 96e3dba2ed3ab0c5b2ecf65a3408633e0767c884d48c270e9ef10ab9fa3ec051 - R 698ea5a1c25e8c3f1a013177c3d20738 -P 680bdc6524ad6af0e74401e96e3a576145042fb865c1582bfaffc11d4ff76a4f -R b720a86de77c44208b3db65976426d46 -U drh -Z 737e0d499b7474145d7a36af76d98f48 ++P 0c0d0a77bc8fa68ee584ca6d7a0408dcd00734a9b6b69d127ee8477c230a749c 5bbd4bddd3b9fa64d134ed62bce3eb4a09456bf24dec2474b5d764a3a3775964 ++R 19eea2388f833c601e8554bc39b427c6 +U dan - Z 739b0ae425836d1ff5201d4d98a57ab1 ++Z e3452c6286d56d7ce7f02a59585758e0 diff --cc manifest.uuid index fbc799809b,3f92f7acc7..7bc87f2c5c --- a/manifest.uuid +++ b/manifest.uuid @@@ -1,1 -1,1 +1,1 @@@ - 0c0d0a77bc8fa68ee584ca6d7a0408dcd00734a9b6b69d127ee8477c230a749c -5bbd4bddd3b9fa64d134ed62bce3eb4a09456bf24dec2474b5d764a3a3775964 ++91262e665755a1430f3b667867b9c4fa0cc3aa0388c21e8ba436485beefa9f4a diff --cc src/wal.c index ab7a91ab2b,0ef08b1377..622264a15f --- a/src/wal.c +++ b/src/wal.c @@@ -1464,184 -1135,6 +1464,211 @@@ static int walIndexAppend(Wal *pWal, in return rc; } +/* +** Recover a single wal file - *-wal if iWal==0, or *-wal2 if iWal==1. +*/ +static int walIndexRecoverOne(Wal *pWal, int iWal, u32 *pnCkpt, int *pbZero){ + i64 nSize; /* Size of log file */ + u32 aFrameCksum[2] = {0, 0}; + int rc; + sqlite3_file *pWalFd = pWal->apWalFd[iWal]; + + assert( iWal==0 || iWal==1 ); + + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + sqlite3_randomness(8, pWal->hdr.aSalt); + + rc = sqlite3OsFileSize(pWalFd, &nSize); + if( rc==SQLITE_OK ){ + if( nSize>WAL_HDRSIZE ){ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u32 *aPrivate = 0; /* Heap copy of *-shm pg being populated */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int szFrame; /* Number of bytes in buffer aFrame[] */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + int szPage; /* Page size according to the log */ + u32 magic; /* Magic value read from WAL header */ + u32 version; /* Magic value read from WAL header */ + int isValid; /* True if this frame is valid */ + int iPg; /* Current 32KB wal-index page */ + int iLastFrame; /* Last frame in wal, based on size alone */ + int iLastPg; /* Last shm page used by this wal */ + + /* Read in the WAL header. */ + rc = sqlite3OsRead(pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* If the database page size is not a power of two, or is greater than + ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid + ** data. Similarly, if the 'magic' value is invalid, ignore the whole + ** WAL file. + */ + magic = sqlite3Get4byte(&aBuf[0]); + szPage = sqlite3Get4byte(&aBuf[8]); + if( (magic&0xFFFFFFFE)!=WAL_MAGIC + || szPage&(szPage-1) + || szPage>SQLITE_MAX_PAGE_SIZE + || szPage<512 + ){ + return SQLITE_OK; + } + pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); + pWal->szPage = szPage; + + /* Verify that the WAL header checksum is correct */ + walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, + aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum + ); + if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) + || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) + ){ + return SQLITE_OK; + } + + memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); + *pnCkpt = sqlite3Get4byte(&aBuf[12]); + + /* Verify that the version number on the WAL format is one that + ** are able to understand */ + version = sqlite3Get4byte(&aBuf[4]); + if( version!=WAL_VERSION1 && version!=WAL_VERSION2 ){ + return SQLITE_CANTOPEN_BKPT; + } + pWal->hdr.iVersion = version; + + /* Malloc a buffer to read frames into. */ + szFrame = szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc64(szFrame + WALINDEX_PGSZ); + if( !aFrame ){ + return SQLITE_NOMEM_BKPT; + } + aData = &aFrame[WAL_FRAME_HDRSIZE]; + aPrivate = (u32*)&aData[szPage]; + + /* Read all frames from the log file. */ + iLastFrame = (nSize - WAL_HDRSIZE) / szFrame; + if( version==WAL_VERSION2 ){ + iLastPg = walFramePage2(iWal, iLastFrame); + }else{ + iLastPg = walFramePage(iLastFrame); + } + for(iPg=iWal; iPg<=iLastPg; iPg+=(version==WAL_VERSION2 ? 2 : 1)){ + u32 *aShare; + int iFrame; /* Index of last frame read */ + int iLast; + int iFirst; + int nHdr, nHdr32; + + rc = walIndexPage(pWal, iPg, (volatile u32**)&aShare); + if( rc ) break; + pWal->apWiData[iPg] = aPrivate; + + if( iWal ){ + assert( version==WAL_VERSION2 ); + iFirst = 1 + (iPg/2)*HASHTABLE_NPAGE; + iLast = iFirst + HASHTABLE_NPAGE - 1; + }else{ + int i2 = (version==WAL_VERSION2) ? (iPg/2) : iPg; + iLast = HASHTABLE_NPAGE_ONE+i2*HASHTABLE_NPAGE; + iFirst = 1 + (i2==0?0:HASHTABLE_NPAGE_ONE+(i2-1)*HASHTABLE_NPAGE); + } + iLast = MIN(iLast, iLastFrame); + + for(iFrame=iFirst; iFrame<=iLast; iFrame++){ + i64 iOffset = walFrameOffset(iFrame, szPage); + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ + + /* Read and decode the next log frame. */ + rc = sqlite3OsRead(pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ) break; + isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); + if( !isValid ) break; + rc = walIndexAppend(pWal, iWal, iFrame, pgno); + if( NEVER(rc!=SQLITE_OK) ) break; + + /* If nTruncate is non-zero, this is a commit record. */ + if( nTruncate ){ + pWal->hdr.mxFrame = iFrame; + pWal->hdr.nPage = nTruncate; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; + aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + } + } + pWal->apWiData[iPg] = aShare; + nHdr = (iPg==0 ? WALINDEX_HDR_SIZE : 0); + nHdr32 = nHdr / sizeof(u32); ++#ifndef SQLITE_SAFER_WALINDEX_RECOVERY ++ /* Memcpy() should work fine here, on all reasonable implementations. ++ ** Technically, memcpy() might change the destination to some ++ ** intermediate value before setting to the final value, and that might ++ ** cause a concurrent reader to malfunction. Memcpy() is allowed to ++ ** do that, according to the spec, but no memcpy() implementation that ++ ** we know of actually does that, which is why we say that memcpy() ++ ** is safe for this. Memcpy() is certainly a lot faster. ++ */ + memcpy(&aShare[nHdr32], &aPrivate[nHdr32], WALINDEX_PGSZ-nHdr); ++#else ++ /* In the event that some platform is found for which memcpy() ++ ** changes the destination to some intermediate value before ++ ** setting the final value, this alternative copy routine is ++ ** provided. ++ */ ++ { ++ int i; ++ for(i=nHdr32; ihdr.aFrameCksum[0] = aFrameCksum[0]; + pWal->hdr.aFrameCksum[1] = aFrameCksum[1]; + + return rc; +} + +static int walOpenWal2(Wal *pWal){ + int rc = SQLITE_OK; + if( !isOpen(pWal->apWalFd[1]) ){ + int f = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + rc = sqlite3OsOpen(pWal->pVfs, pWal->zWalName2, pWal->apWalFd[1], f, &f); + } + return rc; +} + +static int walTruncateWal2(Wal *pWal){ + int bIs; + int rc; + assert( !isOpen(pWal->apWalFd[1]) ); + rc = sqlite3OsAccess(pWal->pVfs, pWal->zWalName2, SQLITE_ACCESS_EXISTS, &bIs); + if( rc==SQLITE_OK && bIs ){ + rc = walOpenWal2(pWal); + if( rc==SQLITE_OK ){ + rc = sqlite3OsTruncate(pWal->apWalFd[1], 0); + sqlite3OsClose(pWal->apWalFd[1]); + } + } + return rc; +} /* ** Recover the wal-index by reading the write-ahead log file. @@@ -2579,8 -2009,16 +2606,17 @@@ static int walCheckpoint sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0); rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); - if( (nSize+(i64)pWal->hdr.mxFrame*szPage)hdr.mxFrame + (bWal2?walidxGetMxFrame(&pWal->hdr,1):0); ++ if( (nSize+mx*szPage)pDbFd, SQLITE_FCNTL_SIZE_HINT,&nReq); + } } - } /* Iterate through the contents of the WAL, copying data to the db file */ diff --cc test/wal2simple.test index 3d35b026c6,0000000000..e17fb66ec9 mode 100644,000000..100644 --- a/test/wal2simple.test +++ b/test/wal2simple.test @@@ -1,474 -1,0 +1,475 @@@ +# 2017 September 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2simple +ifcapable !wal {finish_test ; return } + +#------------------------------------------------------------------------- +# The following tests verify that a client can switch in and out of wal +# and wal2 mode. But that it is not possible to change directly from wal +# to wal2, or from wal2 to wal mode. +# +do_execsql_test 1.1.0 { + PRAGMA journal_mode = wal2 +} {wal2} +execsql { SELECT * FROM sqlite_master} +do_execsql_test 1.x { + PRAGMA journal_mode; + PRAGMA main.journal_mode; +} {wal2 wal2} +db close +do_test 1.1.1 { file size test.db } {1024} +do_test 1.1.2 { hexio_read test.db 18 2 } 0303 + +sqlite3 db test.db +do_execsql_test 1.2.0 { + SELECT * FROM sqlite_master; + PRAGMA journal_mode = delete; +} {delete} +db close +do_test 1.2.1 { file size test.db } {1024} +do_test 1.2.2 { hexio_read test.db 18 2 } 0101 + +sqlite3 db test.db +do_execsql_test 1.3.0 { + SELECT * FROM sqlite_master; + PRAGMA journal_mode = wal; +} {wal} +db close +do_test 1.3.1 { file size test.db } {1024} +do_test 1.3.2 { hexio_read test.db 18 2 } 0202 + +sqlite3 db test.db +do_catchsql_test 1.4.0 { + PRAGMA journal_mode = wal2; +} {1 {cannot change from wal to wal2 mode}} +do_execsql_test 1.4.1 { + PRAGMA journal_mode = wal; + PRAGMA journal_mode = delete; + PRAGMA journal_mode = wal2; + PRAGMA journal_mode = wal2; +} {wal delete wal2 wal2} +do_catchsql_test 1.4.2 { + PRAGMA journal_mode = wal; +} {1 {cannot change from wal2 to wal mode}} +db close +do_test 1.4.3 { hexio_read test.db 18 2 } 0303 + +#------------------------------------------------------------------------- +# Test that recovery in wal2 mode works. +# +forcedelete test.db test.db-wal test.db-wal2 +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 5000; +} {wal2 5000} + +proc wal_hook {DB nm nFrame} { $DB eval { PRAGMA wal_checkpoint } } +db wal_hook {wal_hook db} + +for {set i 1} {$i <= 200} {incr i} { + execsql { INSERT INTO t1 VALUES(NULL, randomblob(100)) } + set res [db eval { SELECT sum(a), md5sum(b) FROM t1 }] + + do_test 2.1.$i { + foreach f [glob -nocomplain test.db2*] { forcedelete $f } + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-wal2 test.db2-wal2 + + sqlite3 db2 test.db2 + db2 eval { SELECT sum(a), md5sum(b) FROM t1 } + } $res + + db2 close +} + +#------------------------------------------------------------------------- + +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(x BLOB, y INTEGER PRIMARY KEY); + CREATE INDEX i1 ON t1(x); + PRAGMA cache_size = 5; + PRAGMA journal_mode = wal2; +} {wal2} + +do_test 3.1 { + execsql BEGIN + for {set i 1} {$i < 1000} {incr i} { + execsql { INSERT INTO t1 VALUES(randomblob(800), $i) } + } + execsql COMMIT +} {} + +do_execsql_test 3.2 { + PRAGMA integrity_check; +} {ok} + +#------------------------------------------------------------------------- +catch { db close } +foreach f [glob -nocomplain test.db*] { forcedelete $f } +reset_db +do_execsql_test 4.0 { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = wal2; +} {wal2} + +do_execsql_test 4.1 { + SELECT * FROM t1; +} {} + +do_execsql_test 4.2 { + INSERT INTO t1 VALUES(1, 2); +} {} + +do_execsql_test 4.3 { + SELECT * FROM t1; +} {1 2} + +do_test 4.4 { + sqlite3 db2 test.db + execsql { SELECT * FROM t1 } db2 +} {1 2} + +do_test 4.5 { + lsort [glob test.db*] +} {test.db test.db-shm test.db-wal test.db-wal2} + +do_test 4.6 { + db close + db2 close + sqlite3 db test.db + execsql { SELECT * FROM t1 } +} {1 2} + +do_execsql_test 4.7 { + PRAGMA journal_size_limit = 4000; + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + INSERT INTO t1 VALUES(9, 10); + INSERT INTO t1 VALUES(11, 12); + INSERT INTO t1 VALUES(13, 14); + INSERT INTO t1 VALUES(15, 16); + INSERT INTO t1 VALUES(17, 18); + SELECT * FROM t1; +} {4000 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18} + +do_test 4.8 { + sqlite3 db2 test.db + execsql { SELECT * FROM t1 } db2 +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18} + +do_test 4.9 { + db close + db2 close + lsort [glob test.db*] +} {test.db} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 4000; +} {wal2 4000} + +proc wal_hook {DB nm nFrame} { + $DB eval { PRAGMA wal_checkpoint } +} +db wal_hook [list wal_hook db] + + +foreach js {4000 8000 12000} { + foreach NROW [list 100 200 300 400 500 600 1000] { + do_test 5.$js.$NROW.1 { + db eval "DELETE FROM t1" + db eval "PRAGMA journal_size_limit = $js" + set nTotal 0 + for {set i 0} {$i < $NROW} {incr i} { + db eval { INSERT INTO t1 VALUES($i, $i, randomblob(abs(random()%50))) } + incr nTotal $i + } + set {} {} + } {} + + do_test 5.$js.$NROW.2 { + sqlite3 db2 test.db + db2 eval { + PRAGMA integrity_check; + SELECT count(*), sum(b) FROM t1; + } + } [list ok $NROW $nTotal] + + db2 close + } +} + + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE TABLE tx(x); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 3500; +} {wal2 3500} + +do_test 6.1 { + for {set i 0} {$i < 10} {incr i} { + execsql "CREATE TABLE t$i (x);" + } +} {} + +do_test 6.2.1 { + foreach f [glob -nocomplain test.db2*] { forcedelete $f } + forcecopy test.db-wal2 test.db2-wal2 + sqlite3 db2 test.db2 + db2 eval { SELECT * FROM sqlite_master } +} {} +do_test 6.2.2 { + db2 eval { + PRAGMA journal_mode = wal2; + SELECT * FROM sqlite_master; + } +} {wal2} + +do_test 6.3.1 { + db2 close + foreach f [glob -nocomplain test.db2*] { forcedelete $f } + forcecopy test.db-wal2 test.db2-wal2 + forcecopy test.db test.db2 + sqlite3 db2 test.db2 + db2 eval { SELECT * FROM sqlite_master } +} {table tx tx 2 {CREATE TABLE tx(x)}} +do_test 6.3.2 { + db2 eval { + PRAGMA journal_mode = wal2; + SELECT * FROM sqlite_master; + } +} {wal2 table tx tx 2 {CREATE TABLE tx(x)}} + +do_test 6.4.1 { + db2 close + foreach f [glob -nocomplain test.db2*] { forcedelete $f } + forcecopy test.db-wal2 test.db2-wal2 + forcecopy test.db-wal test.db2-wal + sqlite3 db2 test.db2 + db2 eval { SELECT * FROM sqlite_master } +} {} +do_test 6.4.2 { + db2 eval { + PRAGMA journal_mode = wal2; + SELECT * FROM sqlite_master; + } +} {wal2} +db2 close + +#------------------------------------------------------------------------- +reset_db +sqlite3 db2 test.db +do_execsql_test 7.0 { + PRAGMA journal_size_limit = 10000; + PRAGMA journal_mode = wal2; + PRAGMA wal_autocheckpoint = 0; + BEGIN; + CREATE TABLE t1(a); + INSERT INTO t1 VALUES( randomblob(8000) ); + COMMIT; +} {10000 wal2 0} + +do_test 7.1 { + list [file size test.db-wal] [file size test.db-wal2] +} {9464 0} + +# Connection db2 is holding a PART1 lock. +# +# 7.2.2: Test that the PART1 does not prevent db from switching to the +# other wal file. +# +# 7.2.3: Test that the PART1 does prevent a checkpoint of test.db-wal. +# +# 7.2.4: Test that after the PART1 is released the checkpoint is possible. +# +do_test 7.2.1 { + execsql { + BEGIN; + SELECT count(*) FROM t1; + } db2 +} {1} +do_test 7.2.2 { + execsql { + INSERT INTO t1 VALUES( randomblob(800) ); + INSERT INTO t1 VALUES( randomblob(800) ); + } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {13656 3176 1024} +do_test 7.2.3 { + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {13656 3176 1024} +do_test 7.2.4 { + execsql { END } db2 + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {13656 3176 11264} + +# Connection db2 is holding a PART2_FULL1 lock. +# +# 7.3.2: Test that the lock does not prevent checkpointing. +# +# 7.3.3: Test that the lock does prevent the writer from overwriting +# test.db-wal. +# +# 7.3.4: Test that after the PART2_FULL1 is released the writer can +# switch wal files and overwrite test.db-wal +# +db close +db2 close +sqlite3 db test.db +sqlite3 db2 test.db +do_test 7.3.1 { + execsql { + PRAGMA wal_autocheckpoint = 0; + PRAGMA journal_size_limit = 10000; + INSERT INTO t1 VALUES(randomblob(10000)); + INSERT INTO t1 VALUES(randomblob(500)); + } + execsql { + BEGIN; + SELECT count(*) FROM t1; + } db2 + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 3176 11264} +do_test 7.3.2 { + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 3176 21504} +do_test 7.3.3 { + execsql { + INSERT INTO t1 VALUES(randomblob(10000)); + INSERT INTO t1 VALUES(randomblob(500)); + } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 18896 21504} +do_test 7.3.4 { + execsql END db2 + execsql { INSERT INTO t1 VALUES(randomblob(5000)); } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 18896 21504} + +# Connection db2 is holding a PART2 lock. +# +# 7.4.2: Test that the lock does not prevent writer switching to test.db-wal. +# +# 7.3.3: Test that the lock does prevent checkpointing of test.db-wal2. +# +# 7.3.4: Test that after the PART2 is released test.db-wal2 can be +# checkpointed. +# +db close +db2 close ++breakpoint +sqlite3 db test.db +sqlite3 db2 test.db +do_test 7.4.1 { + execsql { + PRAGMA wal_autocheckpoint = 0; + PRAGMA journal_size_limit = 10000; + INSERT INTO t1 VALUES(randomblob(10000)); + INSERT INTO t1 VALUES(randomblob(10000)); + PRAGMA wal_checkpoint; + } + execsql { + BEGIN; + SELECT count(*) FROM t1; + } db2 + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 44032} +do_test 7.4.2 { + execsql { + INSERT INTO t1 VALUES(randomblob(5000)); + } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 44032} +do_test 7.4.3 { + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 44032} +do_test 7.4.4 { + execsql END db2 + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 54272} + +# Connection db2 is holding a PART1_FULL2 lock. +# +# 7.5.2: Test that the lock does not prevent a checkpoint of test.db-wal2. +# +# 7.5.3: Test that the lock does prevent the writer from overwriting +# test.db-wal2. +# +# 7.5.4: Test that after the PART1_FULL2 lock is released, the writer +# can switch to test.db-wal2. +# +db close +db2 close +sqlite3 db test.db +sqlite3 db2 test.db +do_test 7.5.1 { + execsql { + PRAGMA wal_autocheckpoint = 0; + PRAGMA journal_size_limit = 10000; + INSERT INTO t1 VALUES(randomblob(10000)); + INSERT INTO t1 VALUES(randomblob(10000)); + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES(randomblob(5000)); + } + execsql { + BEGIN; + SELECT count(*) FROM t1; + } db2 + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 64512} +do_test 7.5.2 { + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 75776} +do_test 7.5.3.1 { + execsql { INSERT INTO t1 VALUES(randomblob(5000)) } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {14704 12608 75776} +do_test 7.5.3.2 { + execsql { INSERT INTO t1 VALUES(randomblob(5000)) } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {22040 12608 75776} +do_test 7.5.4 { + execsql END db2 + execsql { INSERT INTO t1 VALUES(randomblob(5000)) } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {22040 12608 75776} + + +finish_test +