From: dan Date: Tue, 18 Dec 2018 18:01:17 +0000 (+0000) Subject: Merge the wal2 feature into this branch. X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0ba73a6e4d4d1eebc052820d1297a72f61eafeb7;p=thirdparty%2Fsqlite.git Merge the wal2 feature into this branch. FossilOrigin-Name: a2c618c0a45e1b1d271a4a981ee1e3b368dfcce94eda5c2d7d08f794784046d9 --- 0ba73a6e4d4d1eebc052820d1297a72f61eafeb7 diff --cc manifest index 735edc7a3f,4bc575194a..fecfd96dc2 --- a/manifest +++ b/manifest @@@ -1,5 -1,5 +1,5 @@@ - C Merge\slatest\sbegin-concurrent\schanges\sinto\sthis\sbranch. - D 2018-12-18T17:47:53.703 -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2018-12-18T17:32:08.747 ++C Merge\sthe\swal2\sfeature\sinto\sthis\sbranch. ++D 2018-12-18T18:01:17.673 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in d8b254f8bb81bab43c340d70d17dc3babab40fcc8a348c8255881f780a45fee6 @@@ -504,14 -505,14 +505,14 @@@ F src/parse.y 8206217fe7fa96652aa1b8a79 F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ddc9fc7d9861cf3a1f30660264b76b1ae9e1dce5dbba085cf001d5cb6b41cf8c - F src/pragma.c 96ce7dce4dc9cb2b7aa0e1b2ce7536870bdc00b10becc278245e775489447ea0 + F src/pragma.c 3b23e9eb390888a4d2c71c86be6c710c34179657f4f039fae019ba660be00ca9 -F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 +F src/pragma.h 7003ea8e45e5da0a7cd6d35846214f9ae9ecf5be66b268415ceea5855324af11 F src/prepare.c 0e8fc0deaf36da104e08d07ce7d97bc09ab57d078b399381532fec3fa1d3f2bb F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 -F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 +F src/random.c f27af4099afaea7284ade5c206224dcfdb2334cfd119d018b470d46356b3f27d F src/resolve.c 72fe8cae7326b979e7258ab4c531956951e1a5f3fe8644c646abaec1b2eb6d95 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 -F src/select.c 8c7317d5ee920516a56b8b4ca79fbfca70a1f8b52d67e884c808ea3a016c04e3 +F src/select.c ca760497a7a0263b29ca4001760eac53de7c05b0b9edb12adeae1a1c44466779 F src/shell.c.in 207da30342db0b6fac8b2487abd60b059a5ea80cc9494bd1db76a1dd4aae7cca F src/sqlite.h.in 846968c2880c2223e0d65c544698fedcf45bbddc52693a9b020519725690e1fd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@@ -593,8 -594,8 +594,8 @@@ F src/vdbesort.c 90aad5a92608f2dd771c96 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 - F src/wal.c 59b20f8fd0d7420e8896a1ef7cbd5db7352f383235ec25db08d9c22b55f4b603 - F src/wal.h f325a5856b669f5ba449157485915816103857c8574efc746ac55eba3335c5e0 -F src/wal.c 2640020e19d6d2f14ac02fdc468ea965388d956880221d4c8088f20077a85598 -F src/wal.h c398e0269e8f37495cedb63b5e288c2aac6f6d103d05fb55f4affec21311615d ++F src/wal.c dca50fc1279f726751947aaa9334ad946094f55f6b0a850e70e58e41037f9433 ++F src/wal.h 4ef31f6defdd12877eff7832eb5f56c1f898bc074ac5aab67a539f9c59cb8118 F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e F src/whereInt.h f125f29fca80890768e0b2caa14f95db74b2dacd3a122a168f97aa7b64d6968f @@@ -1195,7 -1196,7 +1196,7 @@@ F test/pagesize.test 5769fc62d8c890a83a F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff - F test/permutations.test 4a991666f8840fe1c7cbf20d29ea75cdc76f4a6fc6b647a760924f8eb2f4e88b -F test/permutations.test 2a639a69a3ddee195b36cb5118e7a6ac2a1a0a320b08fdbb32239bf88189ec30 ++F test/permutations.test d0f878173c36562b50898ecccc1a021f06dbb17c3b41ee5b6e620eb4c872a73c F test/pg_common.tcl 301ac19c1a52fd55166d26db929b3b89165c634d52b5f8ad76ea8cb06960db30 F test/pragma.test c267bf02742c823a191960895b3d52933cebd7beee26757d1ed694f213fcd867 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@@ -1800,7 -1812,7 +1812,10 @@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 - P f2083ee410b7504e8478f7373e76ded913e3a6a92cb7345b8c1ac27554f6edf8 123cbb3312917ad5b3c32556547c5b7e8ba4e2d2def8651ff80f9fc1bdc1875c - R 121415d4c182782d4168d3f43c3faca4 -P fa46fa3bfcd4b6d1d219db4179ce69d0edb0786a521acaa034fdb74b312274ff d64f248da3ce7762fe2c17fbc83f7bea9ffca73723bb3ad0982a85320839da90 -R 8beca6502a060f919ffe486db5f7e63b ++P a93ca38b432e0c9fb2e00499cfc88d09f59f27f908f377d8ae99b6717e548a60 b3a163b46cad19a2e6a90cdd5b4c2e94d66e80b7ad2a0c9857580fe74748a604 ++R 8df706f43d7960b329e6ae5fae0ee12e ++T *branch * begin-concurrent-pnu-wal2 ++T *sym-begin-concurrent-pnu-wal2 * ++T -sym-begin-concurrent-pnu * U dan - Z c5e975ad0db2f2825cbe9c283ff86d0a -Z a47178a1de728bf0e9a383c129165be6 ++Z ca2e4b85418d60538bab17e132700aa3 diff --cc manifest.uuid index fbd978a0c4,df45b7c9ce..811cd2f076 --- a/manifest.uuid +++ b/manifest.uuid @@@ -1,1 -1,1 +1,1 @@@ - a93ca38b432e0c9fb2e00499cfc88d09f59f27f908f377d8ae99b6717e548a60 -b3a163b46cad19a2e6a90cdd5b4c2e94d66e80b7ad2a0c9857580fe74748a604 ++a2c618c0a45e1b1d271a4a981ee1e3b368dfcce94eda5c2d7d08f794784046d9 diff --cc src/wal.c index 4b22ba42ef,b7e2391041..5e9ca3691f --- a/src/wal.c +++ b/src/wal.c @@@ -473,8 -707,8 +707,9 @@@ struct Wal u32 iReCksum; /* On commit, recalculate checksums from here */ u32 nPriorFrame; /* For sqlite3WalInfo() */ const char *zWalName; /* Name of WAL file */ + char *zWalName2; /* Name of second WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ + FastPrng sPrng; /* Random number generator */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ #endif @@@ -1122,6 -1425,148 +1426,148 @@@ static int walIndexAppend(Wal *pWal, in return rc; } + /* + ** Recover a single wal file - *-wal if iWal==0, or *-wal2 if iWal==1. + */ + static int walIndexRecoverOne(Wal *pWal, int iWal, u32 *pnCkpt, int *pbZero){ + i64 nSize; /* Size of log file */ + u32 aFrameCksum[2] = {0, 0}; + int rc; + sqlite3_file *pWalFd = pWal->apWalFd[iWal]; + + assert( iWal==0 || iWal==1 ); + + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); - sqlite3_randomness(8, pWal->hdr.aSalt); ++ sqlite3FastRandomness(&pWal->sPrng, 8, pWal->hdr.aSalt); + + rc = sqlite3OsFileSize(pWalFd, &nSize); + if( rc==SQLITE_OK ){ + if( nSize>WAL_HDRSIZE ){ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int szFrame; /* Number of bytes in buffer aFrame[] */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + int iFrame; /* Index of last frame read */ + i64 iOffset; /* Next offset to read from log file */ + int szPage; /* Page size according to the log */ + u32 magic; /* Magic value read from WAL header */ + u32 version; /* Magic value read from WAL header */ + int isValid; /* True if this frame is valid */ + + /* Read in the WAL header. */ + rc = sqlite3OsRead(pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* If the database page size is not a power of two, or is greater than + ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid + ** data. Similarly, if the 'magic' value is invalid, ignore the whole + ** WAL file. + */ + magic = sqlite3Get4byte(&aBuf[0]); + szPage = sqlite3Get4byte(&aBuf[8]); + if( (magic&0xFFFFFFFE)!=WAL_MAGIC + || szPage&(szPage-1) + || szPage>SQLITE_MAX_PAGE_SIZE + || szPage<512 + ){ + return SQLITE_OK; + } + pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); + pWal->szPage = szPage; + + /* Verify that the WAL header checksum is correct */ + walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, + aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum + ); + if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) + || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) + ){ + return SQLITE_OK; + } + + memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); + *pnCkpt = sqlite3Get4byte(&aBuf[12]); + + /* Verify that the version number on the WAL format is one that + ** are able to understand */ + version = sqlite3Get4byte(&aBuf[4]); + if( version!=WAL_VERSION1 && version!=WAL_VERSION2 ){ + return SQLITE_CANTOPEN_BKPT; + } + pWal->hdr.iVersion = version; + + /* Malloc a buffer to read frames into. */ + szFrame = szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc64(szFrame); + if( !aFrame ){ + return SQLITE_NOMEM_BKPT; + } + aData = &aFrame[WAL_FRAME_HDRSIZE]; + + /* Read all frames from the log file. */ + iFrame = 0; + for(iOffset=WAL_HDRSIZE; (iOffset+szFrame)<=nSize; iOffset+=szFrame){ + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ + + /* Read and decode the next log frame. */ + iFrame++; + rc = sqlite3OsRead(pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ) break; + isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); + if( !isValid ) break; + rc = walIndexAppend(pWal, iWal, iFrame, pgno); + if( rc!=SQLITE_OK ) break; + + /* If nTruncate is non-zero, this is a commit record. */ + if( nTruncate ){ + pWal->hdr.mxFrame = iFrame; + pWal->hdr.nPage = nTruncate; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; + aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + } + } + + sqlite3_free(aFrame); + }else if( pbZero ){ + *pbZero = 1; + } + } + + pWal->hdr.aFrameCksum[0] = aFrameCksum[0]; + pWal->hdr.aFrameCksum[1] = aFrameCksum[1]; + + return rc; + } + + static int walOpenWal2(Wal *pWal){ + int rc = SQLITE_OK; + if( !isOpen(pWal->apWalFd[1]) ){ + int f = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + rc = sqlite3OsOpen(pWal->pVfs, pWal->zWalName2, pWal->apWalFd[1], f, &f); + } + return rc; + } + + static int walTruncateWal2(Wal *pWal){ + int bIs; + int rc; + assert( !isOpen(pWal->apWalFd[1]) ); + rc = sqlite3OsAccess(pWal->pVfs, pWal->zWalName2, SQLITE_ACCESS_EXISTS, &bIs); + if( rc==SQLITE_OK && bIs ){ + rc = walOpenWal2(pWal); + if( rc==SQLITE_OK ){ + rc = sqlite3OsTruncate(pWal->apWalFd[1], 0); + sqlite3OsClose(pWal->apWalFd[1]); + } + } + return rc; + } /* ** Recover the wal-index by reading the write-ahead log file. @@@ -1381,11 -1821,16 +1822,17 @@@ int sqlite3WalOpen pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); + sqlite3FastPrngInit(&pRet->sPrng); + pRet->bWal2 = bWal2; - /* Open file handle on the write-ahead log file. */ + pRet->zWalName2 = (char*)pRet + sizeof(Wal) + 2*pVfs->szOsFile; + memcpy(pRet->zWalName2, zWalName, nWalName); + pRet->zWalName2[nWalName] = '2'; + pRet->zWalName2[nWalName+1] = '\0'; + + /* Open a file handle on the first write-ahead log file. */ flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); - rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); + rc = sqlite3OsOpen(pVfs, zWalName, pRet->apWalFd[0], flags, &flags); if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ pRet->readOnly = WAL_RDONLY; } diff --cc src/wal.h index 70247f1c2b,7e700692a4..c7f6c4b28c --- a/src/wal.h +++ b/src/wal.h @@@ -155,8 -156,11 +156,14 @@@ int sqlite3WalFramesize(Wal *pWal) /* Return the sqlite3_file object for the WAL file */ sqlite3_file *sqlite3WalFile(Wal *pWal); + /* Return the journal mode (WAL or WAL2) used by this Wal object. */ + int sqlite3WalJournalMode(Wal *pWal); + + /* sqlite3_wal_info() data */ + int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame); + +/* sqlite3_wal_info() data */ +int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame); + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* SQLITE_WAL_H */