From: dan Date: Mon, 3 Dec 2018 20:49:34 +0000 (+0000) Subject: Merge the wal2 and begin-concurrent code. Both features work, but not at the X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4d8517e5003e3195bf402fd6c5440eb913c66e77;p=thirdparty%2Fsqlite.git Merge the wal2 and begin-concurrent code. Both features work, but not at the same time. FossilOrigin-Name: b7281a1caa574870a071bea3e96b1d8210c28c17f9094449b3ce1a42b311e6a1 --- 4d8517e5003e3195bf402fd6c5440eb913c66e77 diff --cc manifest index 801f2cffb1,adbd28acea..dd9771b7e3 --- a/manifest +++ b/manifest @@@ -1,5 -1,5 +1,5 @@@ - C Minor\schange\sto\swal.c\son\sthis\sbranch\sto\smake\sit\smore\ssimilar\sto\strunk. - D 2018-12-03T20:38:15.728 -C Increase\sa\stimeout\sin\stest\sfile\swalprotocol2.test.\sTo\saccount\sfor\sunix\sbuilds\nwithout\sHAVE_USLEEP. -D 2018-12-03T18:13:46.107 ++C Merge\sthe\swal2\sand\sbegin-concurrent\scode.\sBoth\sfeatures\swork,\sbut\snot\sat\sthe\nsame\stime. ++D 2018-12-03T20:49:34.015 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@@ -432,7 -429,7 +433,7 @@@ F ext/userauth/userauth.c f81aa5a3ecacf F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 - F main.mk c4070221684a71edd3846c05a5cfc648762f3044ef4d91648d33f1f692e15c3c -F main.mk 1274d58d63cb050586fbe0e06b22dcc69972ccb16e57f39a033f37d294316d12 ++F main.mk 3657e7786d4f3b945acc33ea20c1b940eb686d43070a3839bd23b4c7beb758ed F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@@ -449,12 -446,12 +450,12 @@@ F src/analyze.c 3dc6b98cf007b005af89df1 F src/attach.c 92b51739a885da8bd84bc9a05485f1e48148bce5c15432f059b45af98fff75cd F src/auth.c 0fac71038875693a937e506bceb492c5f136dd7b1249fbd4ae70b4e8da14f9df F src/backup.c 78d3cecfbe28230a3a9a1793e2ead609f469be43e8f486ca996006be551857ab -F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 +F src/bitvec.c 8433d9e98dd6f2ea3286e0d2fe5d65de1bfc18a706486eb2026b01be066b5806 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 - F src/btree.c 4a2184be69d491d4b0228d4e397d67cb0802bbec06e7615b485ea1af69a131f6 -F src/btree.c 3887a4f5513a831c445063ff1d81d9d6a50ad47522600dab0ef0656f4389fe5a -F src/btree.h febb2e817be499570b7a2e32a9bbb4b607a9234f6b84bb9ae84916d4806e96f2 -F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 -F src/build.c 127d33ad57b455a9339e9fabff41284c8b030cc6247ca7a2a6c0ad7abfc1ce85 ++F src/btree.c f5041e98e9016cb3f3871dae6e8e6c29f6c9867add38d8e8d463033287545565 +F src/btree.h 1ed41c71481a1196a520064f2282bc13d768bbd8ae2850e319a3048f8ee7cb3d +F src/btreeInt.h 6c65e6c96f561596f6870c79a64d4706af81613881d7947e3f063e923f14115f +F src/build.c 5e04fb8528a4a915ed9af94b5cd068e53f2cfa9824d37464c4059279ea9bc8a0 F src/callback.c 789bd33d188146f66c0dd8306472a72d1c05f71924b24a91caf6bd45cf9aba73 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b @@@ -494,12 -491,12 +495,12 @@@ F src/os.c 8aeb0b0f40f8f5b0da03fe497066 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 - F src/os_unix.c 86eca42c3d955bebea0082450f978e5633448235f03f86b27a02538bb26e7fff -F src/os_unix.c 711480e9152f221098ec2b0d4ef94dc798f08af649c34f5cd4dc2bbf40c4f556 ++F src/os_unix.c 1aa113b261a0ad44fd410a001f6e39bfc1ebd4279b2fb8c2d636a7620d76a45a F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a - F src/pager.c 79b8ced46a3c32be19ab4b00d8836292e460411d283a333cb008923c69fbca6a - F src/pager.h 389ba8f526d13026aa7081dc581aa742eb7207e3277e7106c522c5b65ad92590 -F src/pager.c dd88ccf7fa519d6e325a115af28cd528713ea4d33a510d6f202386df002f2e74 -F src/pager.h 3abf6d65199fd0680b26a047c6167a96a4d6ead7535e02522b79f0fb27a3edec -F src/parse.y 6840fe7c0b5eb4dd25ee5d075213bc8255ed4c0678d71bfb6744d0520d91c179 ++F src/pager.c 224f8ad64b8243cd92f79e4c28d94507c114a745a755545bc1d153f577d7e433 ++F src/pager.h d8cf37b3415c742d1f267ae2e0e6495826a72d403cbdbefdab2e2f5ff2a1dde7 +F src/parse.y 9e69c380ac16423a1f373cde66d1be0d14a789f93464705a48dcc812d04d9210 F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c bf9fcea656dce1cd2cca6b77a1d1d3552050d55a31c98bf0d9f405930a83bc95 @@@ -580,8 -577,8 +581,8 @@@ F src/update.c 1816d56c1bca1ba4e0ef98ca F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 -F src/vacuum.c 836cadc922de866c849e23a75f93d344cdc143d388339305d09a3fed27e8798d -F src/vdbe.c ee46b31b88015e1ba5b38b9405cada3c8d264bc5e19764ecc751e63aeb2edf66 +F src/vacuum.c 8747a99e0687ae5cb3515b70a9d82bbd20370de9f097c7bd93a392ece3dea03c - F src/vdbe.c d2672a54cc283e1425cc8c7c45271530b3686b8b5f06bf4426d9a5812e951abe ++F src/vdbe.c accf86634537d2d1ef72cac8b9b5281943f2711853659e479e3d10addd2550f3 F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980 F src/vdbeapi.c ecccfce6f614c33a95952efeec969d163e8349eac314ee2b7b163eda921b5eb0 @@@ -592,8 -589,8 +593,8 @@@ F src/vdbesort.c 90aad5a92608f2dd771c96 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 - F src/wal.c 8a12219e699ed737fa4dd8f0449f1bb5dee765502b48710dbd5be747ea58e65b - F src/wal.h f325a5856b669f5ba449157485915816103857c8574efc746ac55eba3335c5e0 -F src/wal.c e5b19ec1ce95882a2ae610d2657b441b079734427757c1a4d069f5f57bb7e9a2 -F src/wal.h fc6113057f2950fc14631176b748293e216fb385ea8df665e6d259e37f8f7d21 ++F src/wal.c 88e424b4de9cb836166a967b5b8a6de7f90cdd493dd1d04a1b855fa58dbe7212 ++F src/wal.h b42fc8081cd1765d4d4dd99b33f2db2f71128f4e25ff8c08d1a346f5af62e27a F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e F src/whereInt.h f125f29fca80890768e0b2caa14f95db74b2dacd3a122a168f97aa7b64d6968f @@@ -1604,7 -1592,9 +1605,9 @@@ F test/vtab_alter.test 736e66fb5ec7b4fe F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 -F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918 +F test/wal2.test a225bafac35a47765b890bacdeb57e5e81039f21cc18a1e8ce88eb76e56b843c + F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c + F test/wal2simple.test 8c9dfb8f1bca01a0deb57f7074cdb83865c2292e89b13f7a51a1c160dca3f5f4 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@@ -1792,8 -1782,7 +1795,10 @@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 - P a56506b9387a067ef259504d127694ad20223f4b08781d1676ff7f5fdd9443d8 - R d66e284ab90799b5d6b7d2957b4e83a3 - T +closed 0d12f49feb78a94a1b188e80379b51dfe9bf6c8e60225134e15216192cabed21 -P 7a44fa5a350a3f19b8e9f5196d22535788885f8c0e849572202bf64a055ddc2d -R 8dcc78dd7464cc5597695c90813a01e4 ++P 6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 480be916c840e9e28010c22cf29a8396502c9e6387f31f750260c6290f58e9a1 ++R 41c3c1de1e62db08c33d5cf08c11006b ++T *branch * begin-concurrent-wal2 ++T *sym-begin-concurrent-wal2 * ++T -sym-begin-concurrent * U dan - Z 035c0f4c166e3fab14296a764b5a28cd -Z 7d2b6bedf7bcc33a553c67266fb7675b ++Z f5d7b930a621f73c3ef64e6a734bda83 diff --cc manifest.uuid index dd8dc8b081,2987cabc9a..a398ac8e87 --- a/manifest.uuid +++ b/manifest.uuid @@@ -1,1 -1,1 +1,1 @@@ - 6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 -480be916c840e9e28010c22cf29a8396502c9e6387f31f750260c6290f58e9a1 ++b7281a1caa574870a071bea3e96b1d8210c28c17f9094449b3ce1a42b311e6a1 diff --cc src/wal.c index 673a159b19,42001bf2d2..5b50c99dfa --- a/src/wal.c +++ b/src/wal.c @@@ -471,8 -702,8 +702,9 @@@ struct Wal WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ + u32 nPriorFrame; /* For sqlite3WalInfo() */ const char *zWalName; /* Name of WAL file */ + char *zWalName2; /* Name of second WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ @@@ -2772,8 -3320,11 +3337,12 @@@ int sqlite3WalBeginReadTransaction(Wal testcase( (rc&0xff)==SQLITE_IOERR ); testcase( rc==SQLITE_PROTOCOL ); testcase( rc==SQLITE_OK ); + + if( rc==SQLITE_OK && pWal->hdr.iVersion==WAL_VERSION2 ){ + rc = walOpenWal2(pWal); + } + pWal->nPriorFrame = pWal->hdr.mxFrame; #ifdef SQLITE_ENABLE_SNAPSHOT if( rc==SQLITE_OK ){ if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ @@@ -2938,8 -3555,11 +3573,11 @@@ int sqlite3WalFindFrame #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* If expensive assert() statements are available, do a linear search ** of the wal-index file content. Make sure the results agree with the - ** result obtained using the hash indexes above. */ + ** result obtained using the hash indexes above. + ** + ** TODO: This is broken for wal2. + */ - { + if( rc==SQLITE_OK ){ u32 iRead2 = 0; u32 iTest; assert( pWal->bShmUnreliable || pWal->minFrame>0 ); @@@ -3033,202 -3638,38 +3685,202 @@@ static int walWriteLock(Wal *pWal) ** There can only be a single writer active at a time. */ int sqlite3WalBeginWriteTransaction(Wal *pWal){ + int rc = walWriteLock(pWal); + if( rc==SQLITE_OK ){ + /* If another connection has written to the database file since the + ** time the read transaction on this connection was started, then + ** the write is disallowed. Release the WRITER lock and return + ** SQLITE_BUSY_SNAPSHOT in this case. */ + if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + rc = SQLITE_BUSY_SNAPSHOT; + } + } + return rc; +} + +/* +** This function is called by a writer that has a read-lock on aReadmark[0] +** (pWal->readLock==0). This function relinquishes that lock and takes a +** lock on a different aReadmark[] slot. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int walUpgradeReadlock(Wal *pWal){ + int cnt; int rc; + assert( pWal->writeLock && pWal->readLock==0 ); + walUnlockShared(pWal, WAL_READ_LOCK(0)); + pWal->readLock = -1; + cnt = 0; + do{ + int notUsed; + rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); + }while( rc==WAL_RETRY ); + assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ + testcase( (rc&0xff)==SQLITE_IOERR ); + testcase( rc==SQLITE_PROTOCOL ); + testcase( rc==SQLITE_OK ); + return rc; +} - /* Cannot start a write transaction without first holding a read - ** transaction. */ - assert( pWal->readLock!=WAL_LOCK_NONE ); - assert( pWal->writeLock==0 && pWal->iReCksum==0 ); - if( pWal->readOnly ){ - return SQLITE_READONLY; - } +#ifndef SQLITE_OMIT_CONCURRENT +/* +** This function is only ever called when committing a "BEGIN CONCURRENT" +** transaction. It may be assumed that no frames have been written to +** the wal file. The second parameter is a pointer to the in-memory +** representation of page 1 of the database (which may or may not be +** dirty). The third is a bitvec with a bit set for each page in the +** database file that was read by the current concurrent transaction. +** +** This function performs three tasks: +** +** 1) It obtains the WRITER lock on the wal file, +** +** 2) It checks that there are no conflicts between the current +** transaction and any transactions committed to the wal file since +** it was opened, and +** +** 3) It ejects any non-dirty pages from the page-cache that have been +** written by another client since the CONCURRENT transaction was started +** (so as to avoid ending up with an inconsistent cache after the +** current transaction is committed). +** +** If no error occurs and the caller may proceed with committing the +** transaction, SQLITE_OK is returned. SQLITE_BUSY is returned if the WRITER +** lock cannot be obtained. Or, if the WRITER lock can be obtained but there +** are conflicts with a committed transaction, SQLITE_BUSY_SNAPSHOT. Finally, +** if an error (i.e. an OOM condition or IO error), an SQLite error code +** is returned. +*/ +int sqlite3WalLockForCommit( + Wal *pWal, + PgHdr *pPage1, + Bitvec *pAllRead, + Pgno *piConflict +){ + Pager *pPager = pPage1->pPager; + int rc = walWriteLock(pWal); - /* Only one writer allowed at a time. Get the write lock. Return - ** SQLITE_BUSY if unable. + /* If the database has been modified since this transaction was started, + ** check if it is still possible to commit. The transaction can be + ** committed if: + ** + ** a) None of the pages in pList have been modified since the + ** transaction opened, and + ** + ** b) The database schema cookie has not been modified since the + ** transaction was started. */ - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); - if( rc ){ - return rc; - } - pWal->writeLock = 1; + if( rc==SQLITE_OK ){ + WalIndexHdr head; + + if( walIndexLoadHdr(pWal, &head) ){ + /* This branch is taken if the wal-index header is corrupted. This + ** occurs if some other writer has crashed while committing a + ** transaction to this database since the current concurrent transaction + ** was opened. */ + rc = SQLITE_BUSY_SNAPSHOT; + }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){ + int iHash; + int iLastHash = walFramePage(head.mxFrame); + u32 iFirst = pWal->hdr.mxFrame+1; /* First wal frame to check */ + if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){ + assert( pWal->readLock==0 ); + iFirst = 1; + } + for(iHash=walFramePage(iFirst); iHash<=iLastHash; iHash++){ + WalHashLoc sLoc; - /* If another connection has written to the database file since the - ** time the read transaction on this connection was started, then - ** the write is disallowed. - */ - if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); - pWal->writeLock = 0; - rc = SQLITE_BUSY_SNAPSHOT; + rc = walHashGet(pWal, iHash, &sLoc); + if( rc==SQLITE_OK ){ + u32 i, iMin, iMax; + assert( head.mxFrame>=sLoc.iZero ); + iMin = (sLoc.iZero >= iFirst) ? 1 : (iFirst - sLoc.iZero); + iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; + if( iMax>(head.mxFrame-sLoc.iZero) ) iMax = (head.mxFrame-sLoc.iZero); + for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){ + PgHdr *pPg; + if( sLoc.aPgno[i]==1 ){ + /* Check that the schema cookie has not been modified. If + ** it has not, the commit can proceed. */ + u8 aNew[4]; + u8 *aOld = &((u8*)pPage1->pData)[40]; + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + iOffset = walFrameOffset(i+sLoc.iZero, sz) + WAL_FRAME_HDRSIZE+40; - rc = sqlite3OsRead(pWal->pWalFd, aNew, sizeof(aNew), iOffset); ++ rc = sqlite3OsRead(pWal->apWalFd[0], aNew, sizeof(aNew), iOffset); + if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ + rc = SQLITE_BUSY_SNAPSHOT; + } + }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i]) ){ + *piConflict = sLoc.aPgno[i]; + rc = SQLITE_BUSY_SNAPSHOT; + }else if( (pPg = sqlite3PagerLookup(pPager, sLoc.aPgno[i])) ){ + /* Page aPgno[i], which is present in the pager cache, has been + ** modified since the current CONCURRENT transaction was started. + ** However it was not read by the current transaction, so is not + ** a conflict. There are two possibilities: (a) the page was + ** allocated at the of the file by the current transaction or + ** (b) was present in the cache at the start of the transaction. + ** + ** For case (a), do nothing. This page will be moved within the + ** database file by the commit code to avoid the conflict. The + ** call to PagerUnref() is to release the reference grabbed by + ** the sqlite3PagerLookup() above. + ** + ** In case (b), drop the page from the cache - otherwise + ** following the snapshot upgrade the cache would be inconsistent + ** with the database as stored on disk. */ + if( sqlite3PagerIswriteable(pPg) ){ + sqlite3PagerUnref(pPg); + }else{ + sqlite3PcacheDrop(pPg); + } + } + } + } + if( rc!=SQLITE_OK ) break; + } + } } + pWal->nPriorFrame = pWal->hdr.mxFrame; + return rc; +} + +/* !defined(SQLITE_OMIT_CONCURRENT) +** +** This function is called as part of committing an CONCURRENT transaction. +** It is assumed that sqlite3WalLockForCommit() has already been successfully +** called and so (a) the WRITER lock is held and (b) it is known that the +** wal-index-header stored in shared memory is not corrupt. +** +** Before returning, this function upgrades the client so that it is +** operating on the database snapshot currently at the head of the wal file +** (even if the CONCURRENT transaction ran against an older snapshot). +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +int sqlite3WalUpgradeSnapshot(Wal *pWal){ + int rc = SQLITE_OK; + assert( pWal->writeLock ); + memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); + + /* If this client has its read-lock on slot aReadmark[0] and the entire + ** wal has not been checkpointed, switch it to a different slot. Otherwise + ** any reads performed between now and committing the transaction will + ** read from the old snapshot - not the one just upgraded to. */ + if( pWal->readLock==0 && pWal->hdr.mxFrame!=walCkptInfo(pWal)->nBackfill ){ + rc = walUpgradeReadlock(pWal); + } return rc; } +#endif /* SQLITE_OMIT_CONCURRENT */ /* ** End a write transaction. The commit has already been done. This @@@ -3258,10 -3699,14 +3910,14 @@@ int sqlite3WalEndWriteTransaction(Wal * */ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ int rc = SQLITE_OK; - if( ALWAYS(pWal->writeLock) ){ + if( pWal->writeLock ){ - Pgno iMax = pWal->hdr.mxFrame; + int iWal = walidxGetFile(&pWal->hdr); + Pgno iMax = walidxGetMxFrame(&pWal->hdr, iWal); + Pgno iNew; Pgno iFrame; - + + assert( isWalMode2(pWal) || iWal==0 ); + /* Restore the clients cache of the wal-index header to the state it ** was in before the client began writing to the database. */ @@@ -3311,11 -3764,14 +3975,14 @@@ void sqlite3WalSavepoint(Wal *pWal, u3 */ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ int rc = SQLITE_OK; + int iWal = walidxGetFile(&pWal->hdr); + int iCmp = isWalMode2(pWal) ? iWal : pWal->nCkpt; - assert( pWal->writeLock ); + assert( pWal->writeLock || aWalData[0]==pWal->hdr.mxFrame ); - assert( aWalData[3]!=pWal->nCkpt || aWalData[0]<=pWal->hdr.mxFrame ); + assert( isWalMode2(pWal) || iWal==0 ); + assert( aWalData[3]!=iCmp || aWalData[0]<=walidxGetMxFrame(&pWal->hdr,iWal) ); - if( aWalData[3]!=pWal->nCkpt ){ + if( aWalData[3]!=iCmp ){ /* This savepoint was opened immediately after the write-transaction ** was started. Right after that, the writer decided to wrap around ** to the start of the log. Update the savepoint values to match. @@@ -3349,7 -3832,38 +4043,37 @@@ static int walRestartOk(Wal *pWal) static int walRestartLog(Wal *pWal){ int rc = SQLITE_OK; - if( pWal->readLock==0 ){ + if( isWalMode2(pWal) ){ + int iApp = walidxGetFile(&pWal->hdr); + int nWalSize = WAL_DEFAULT_WALSIZE; + if( pWal->mxWalSize>0 ){ + nWalSize = (pWal->mxWalSize-WAL_HDRSIZE+pWal->szPage+WAL_FRAME_HDRSIZE-1) + / (pWal->szPage+WAL_FRAME_HDRSIZE); + nWalSize = MAX(nWalSize, 1); + } + + if( walidxGetMxFrame(&pWal->hdr, iApp)>=nWalSize ){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + if( walidxGetMxFrame(&pWal->hdr, !iApp)==0 || pInfo->nBackfill ){ + rc = walRestartOk(pWal); + if( rc==SQLITE_OK ){ + iApp = !iApp; + pWal->nCkpt++; + walidxSetFile(&pWal->hdr, iApp); + walidxSetMxFrame(&pWal->hdr, iApp, 0); + sqlite3Put4byte((u8*)&pWal->hdr.aSalt[0], pWal->hdr.aFrameCksum[0]); + sqlite3Put4byte((u8*)&pWal->hdr.aSalt[1], pWal->hdr.aFrameCksum[1]); + walIndexWriteHdr(pWal); + pInfo->nBackfill = 0; + walLockReader(pWal, pWal->readLock, 0); + pWal->readLock = iApp ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2; + rc = walLockReader(pWal, pWal->readLock, 1); + }else if( rc==SQLITE_BUSY ){ + rc = SQLITE_OK; + } + } + } + }else if( pWal->readLock==0 ){ - int cnt; volatile WalCkptInfo *pInfo = walCkptInfo(pWal); assert( pInfo->nBackfill==pWal->hdr.mxFrame ); if( pInfo->nBackfill>0 ){ @@@ -3373,17 -3886,19 +4097,18 @@@ return rc; } } - walUnlockShared(pWal, WAL_READ_LOCK(0)); - pWal->readLock = WAL_LOCK_NONE; - cnt = 0; - do{ - int notUsed; - rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); - }while( rc==WAL_RETRY ); - assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ - testcase( (rc&0xff)==SQLITE_IOERR ); - testcase( rc==SQLITE_PROTOCOL ); - testcase( rc==SQLITE_OK ); + + /* Regardless of whether or not the wal file was restarted, change the + ** read-lock held by this client to a slot other than aReadmark[0]. + ** Clients with a lock on aReadmark[0] read from the database file + ** only - never from the wal file. This means that if a writer holding + ** a lock on aReadmark[0] were to commit a transaction but not close the + ** read-transaction, subsequent read operations would read directly from + ** the database file - ignoring the new pages just appended + ** to the wal file. */ + rc = walUpgradeReadlock(pWal); } + return rc; } @@@ -3826,7 -4384,10 +4595,9 @@@ int sqlite3WalCheckpoint /* Copy data from the log to the database file. */ if( rc==SQLITE_OK ){ - if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ - + if( (walPagesize(pWal)!=nBuf) + && (walidxGetMxFrame(&pWal->hdr, 0) || walidxGetMxFrame(&pWal->hdr, 1)) + ){ rc = SQLITE_CORRUPT_BKPT; }else{ rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags, zBuf); @@@ -4044,19 -4627,7 +4837,19 @@@ int sqlite3WalFramesize(Wal *pWal) /* Return the sqlite3_file object for the WAL file */ sqlite3_file *sqlite3WalFile(Wal *pWal){ - return pWal->pWalFd; + return pWal->apWalFd[0]; } +/* +** Return the values required by sqlite3_wal_info(). +*/ +int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame){ + int rc = SQLITE_OK; + if( pWal ){ + *pnFrame = pWal->hdr.mxFrame; + *pnPrior = pWal->nPriorFrame; + } + return rc; +} + #endif /* #ifndef SQLITE_OMIT_WAL */