From: dan Date: Mon, 21 Aug 2023 18:02:57 +0000 (+0000) Subject: Merge latest trunk changes into this branch. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=cc74eafee549a6e1dfc1f09c9396662ee0c4b2dc;p=thirdparty%2Fsqlite.git Merge latest trunk changes into this branch. FossilOrigin-Name: 135bf72c6b5f436b11463dd40b3a73bcc610bf9c2dcbe6646dbedd0d9325f6a9 --- cc74eafee549a6e1dfc1f09c9396662ee0c4b2dc diff --cc manifest index 5adfa1337d,63a6e39eb0..d1c64c4551 --- a/manifest +++ b/manifest @@@ -1,12 -1,12 +1,12 @@@ - C Merge\slatest\strunk\schanges,\sincluding\ssupport\sfor\shandling\sof\sstructured-exceptions\sin\sMSVC\sbuilds,\swith\sthis\sbranch. - D 2023-08-11T19:33:54.234 -C Fix\san\sundefined\sinteger\soverflow\sin\sfts5. -D 2023-08-21T16:14:58.145 ++C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. ++D 2023-08-21T18:02:57.762 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 - F Makefile.in e1282d59cf2dffef4a69ca4b0364129fdaefd2f1718bf789f6b5d27992261bad -F Makefile.in 6f391d54fa01f8a49b024fef5cce1ab8234c281164641cf9a52694b432bdec1b ++F Makefile.in ef816e3c4a4893cbf84589903fa5428214726302557e2fa7eabe6a6046ae65f6 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 - F Makefile.msc 0b57ab2867b1fdc90c4e35a6777a3b24b81006cc7ba947c8db13929b4edd3800 - F README.md c1c4218efcc4071a6e26db2b517fdbc1035696a29b370edd655faddbef02b224 -F Makefile.msc 26c2d196391a285c279adb10fd6001774d9b243af94b700b681e4a49cd476684 ++F Makefile.msc 01eac287493d2ea7b0ced738dcf19a2234c60195d39e1c86f2b0fa20c9d7d14e + F README.md 093d7054271141a0a8518558e3d49087cb71f84d33b50ee10053946ed85dcac8 F VERSION c6366dc72582d3144ce87b013cc35fe48d62f6d07d5be0c9716ea33c862144aa F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@@ -489,11 -528,11 +530,11 @@@ F ext/session/test_session.c 5285482f83 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb -F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c +F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 - F ext/wasm/GNUmakefile 8159bc5f9433fe21022c1a8e8c30cb1a523530ba9ef53bdf5d1e0a2186554806 + F ext/wasm/GNUmakefile d02f3c8798b754f68b1f6b422ccff894a10bf352fc9c4eb8945baeace1acac28 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 5eb44fa02e9c693a1884a3692428647894b0380b24bca120866b7a24c8786134 @@@ -565,7 -604,7 +606,7 @@@ F ext/wasm/wasmfs.make 8a4955882aaa0783 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 - F main.mk b7cf0350144fdf7551ecca064272b88098dcf206cb0d4c79754ad734a8f1c24b -F main.mk e18c03071dbb9da424212ba2a1ea331ae0e8f6d7d51283b7ad610c52ea11229c ++F main.mk 5280f58fa05801656870b486cc130e0d9a600ce53275171a0b1c07eaf121660f F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@@ -630,10 -669,10 +671,10 @@@ F src/os_common.h 6c0eb8dd40ef3e12fe585 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 F src/os_unix.c 2e8b12107f75d1bd16412f312b4c5d5103191807a37836d3b81beb26436ad81b - F src/os_win.c 7038223a1cda0a47e2ab4db47f63bf1833fe53ba0542f0f283a062ea13894103 + F src/os_win.c 4a50a154aeebc66a1f8fb79c1ff6dd5fe3d005556533361e0d460d41cb6a45a8 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 993445a19b611d473ca007542ab3149840661a4c7e9f2d9e1ec008b7cc2abe78 -F src/pager.h f4d33fec8052603758792045493423b8871a996da2d0973927b7d36cd6070473 +F src/pager.c e7af170f5799a5edd77fc267e81cb0292e1a1cb7b488b62c6e2fff78aedc2efb - F src/pager.h ca3b415a3f03359d5692b71ec0ba068a58ce447fac7312f89293280d95ee38da ++F src/pager.h 1da654b51a4a002e6d8bf68fbdb454e1f31ebe43cc4d399a4dfb8da4ad2fe182 F src/parse.y aeb7760d41cfa86465e3adba506500c021597049fd55f82a30e5b7045862c28c F src/pcache.c 4cd4a0043167da9ba7e19b4d179a0e6354e7fe32c16f781ecf9bf0a5ff63b40b F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 @@@ -710,15 -749,15 +751,15 @@@ F src/threads.c 4ae07fa022a3dc7c5beb373 F src/tokenize.c 23d9f4539880b40226254ad9072f4ecf12eb1902e62aea47aac29928afafcfd5 F src/treeview.c 1d52fbc4e97161e65858d36e3424ea6e3fc045dd8a679c82b4b9593dc30de3bd F src/trigger.c ad6ab9452715fa9a8075442e15196022275b414b9141b566af8cdb7a1605f2b0 - F src/update.c 0bb9171afaa4d0b100ad946873bccda7aef90ffe083ef5c63668fce08c4df9da - F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 + F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 + F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 - F src/util.c a40062117e705eb3339201842717a022092816b92479eead6397cde28af32ff9 + F src/util.c 278b81c3b33db1b5a5f3859adf8905c165b910080043061d44d3c5a25b4b406d F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c 346d848a0bf8128e3e3722c5406f4bde6c32d7093b93402c6f8e0718d19305c3 +F src/vdbe.c 11f1a00443991bd12bd0c97050b2614f7fd3e82b9cc0ffdb13d14d424d733e77 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c - F src/vdbeapi.c f37822f215740ede2a8fcae99bc13f2cc3a72dd0e1d22b81b9298c5ca67dbc38 + F src/vdbeapi.c 37341acd781fda162e8cf4d9fc2eaea2febad3b365877a9d7233b8c6d0960d85 F src/vdbeaux.c e3aa5c46827cd95e0fc4d0f302fa3e901ab5f07258fdbb42709eeef40f63018d F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce F src/vdbemem.c 317b9f48708139db6239ade40c7980b4bc8233168383690d588dad6d8437f722 @@@ -727,8 -766,8 +768,8 @@@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1 F src/vdbevtab.c 57fa8f56478e5b5cb558cb425e7878515e0a105c54f96f1d1bbf4b9433529254 F src/vtab.c 1ecf8c3745d29275688d583e12822fa984d421e0286b5ef50c137bc3bf6d7a64 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 - F src/wal.c 1ea71da11648befe8be082d0ef6adad144435a827d7ae1593da1bf375482c21d - F src/wal.h a3e64f463acf027f041c2081841e15e8c81c56d0daa45e308df423d9d66fd5a7 -F src/wal.c 01e051a1e713d9eabdb25df38602837cec8f4c2cae448ce2cf6accc87af903e9 -F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 ++F src/wal.c 7b887817194adfdf627df2c403a73c83fab833551009b876d3bcd7f0323859ed ++F src/wal.h 97b8a9903387401377b59507e86b93a148ef1ad4e5ce0f23659a12dcdce56af2 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 F src/where.c b8917792f1e0dbfa28fb29e6cd3d560060d69667be0ba4c491cbc772363264f5 F src/whereInt.h c7d19902863beadec1d04e66aca39c0bcd60b74f05f0eaa7422c7005dfc5d51a @@@ -1372,7 -1411,7 +1413,7 @@@ F test/pcache.test c8acbedd3b6fd0f9a7ca F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/pendingrace.test cbdf0f74bc939fb43cebad64dda7a0b5a3941a10b7e9cc2b596ff3e423a18156 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff - F test/permutations.test 1f9b143250923886db3d5793e703a51b892e7f3fcd0e8cff5b6e30915687afe0 -F test/permutations.test f7caf8dd5c7b1da74842a48df116f7f193399c656d4ffc805cd0d9658568c675 ++F test/permutations.test 58188152f3e0e319cd916f5f6e1afb34c8d3a5219fe49fe62c0c706514e54970 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f F test/pragma.test 57a36226218c03cfb381019fe43234b2cefbd8a1f12825514f906a17ccf7991e F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@@ -1563,9 -1602,9 +1604,9 @@@ F test/temptable.test d2c9b87a54147161b F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 68454ef88508c196d19e8694daa27bff7107a91857799eaa12f417188ae53ede +F test/tester.tcl e6b9adcf4e56aa5c658501c10650646fee467b85d51a4ac80263d2cdae3993ba - F test/testrunner.tcl 59490f189cac99b16b0376d0cc0a7ecfb753a84b89c9f4c361af337d88db53ac - F test/testrunner_data.tcl 8169c68654ac8906833b8a6aadca973358a441ebf88270dd05c153e5f96f76b8 + F test/testrunner.tcl 56a744d4e6e516b2091c2ca6b7b27b9600e9ded136a2c860c350515511ebe20a + F test/testrunner_data.tcl 8afa4eeec7f7421db3d3af64cdd5544015e887c4289cf871a730cca5a7a8d934 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@@ -1829,20 -1868,8 +1870,20 @@@ F test/vtab_shared.test 5253bff2355a9a3 F test/vtabdistinct.test 7688f0889358f849fd60bbfde1ded38b014b18066076d4bfbb75395804dfe072 F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7edca12 F test/vtabrhs1.test 9b5ecbc74a689500c33a4b2b36761f9bcc22fcc4e3f9d21066ee0c9c74cf5f6c -F test/wal.test b7cc6984709f54afbf8441747ced1f646af120bf0c1b1d847bfa39306fbea089 +F test/wal.test 07d61ee522bc203235a7979645d6c5d84f86882afad5b678532ee37b5b85b133 F test/wal2.test 31f6e2c404b9f2cdf9ca19b105a1742fdc19653c2c936da39e3658c617524046 +F test/wal2big.test 829141cbecdda4329db8fa38705424c4a73db72a06b9540b06811a825d330409 +F test/wal2fault.test 2e8e60cacd5bcd451618aeffd05f676894d17202d3e2986e288d36e2c5993249 +F test/wal2lock.test 0ef98d72dc6bcf7711dedd684760488400d9a9a6eec0dc5d3822060437793552 +F test/wal2openclose.test 2b26be723ea7f4263c8d5d70b37efd1c359561a0526e39466c45fe8e6478daee +F test/wal2recover.test 28b3fa0acb4e8725211a02803cd51116cd048841a876224f244af78039ba34fa - F test/wal2recover2.test af7e3bc08690918257da1bb37c94029a780bb17a65113c11b1a8bc9498fe82af ++F test/wal2recover2.test fbbbf251639ab22eba0f0f09721a6824fadfb8f0214afdece2c5cf704e7fbc43 +F test/wal2recover3.test 4a91689e165a38bc401736e6518188c2b0ff4fa1566d1810b8867536db128177 +F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c +F test/wal2rollback.test 23adc4a099b23f6aaea8b04fdca1c35861d887dd80f8be7da2d5273eb777e428 +F test/wal2savepoint.test 2c82bd6a6ee5066c156040d2e9c2415646fcf96116ae7ad127eaf0c0b4a85f22 +F test/wal2simple.test 320a08927f307d0ead26fa3fcef4e3f64279f49be9504f918cb33294f75aeec8 +F test/wal2snapshot.test 7a5f4629a3c43a43c3440b8b2ea9f07de91a46b0b9eea5f08f62b5bf5b6468df F test/wal3.test 5de023bb862fd1eb9d2ad26fa8d9c43abb5370582e5b08b2ae0d6f93661bc310 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@@ -2065,8 -2092,8 +2106,8 @@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 - P fe1bf30ca0c529c3c68f2531e28aca5659aef5d15c2e3a6925ecd35a5098386b 8a6b0c24937e855b710f97b4aea973eff53e6d43e1182842731547aa4b37db2a - R cb51b81316a6337126aa844ebf9c925d -P 0462a2612d1fc1d077acf60ef415f358aa2c44174c4ef28f4bbfdbcbf03d3b7c -R 31562660b5d535c24c7b617332f56384 ++P 00bfae9ffafd71dfd0bbdbdc174e4ed7d7a850b385416f83be36a4f4a55d1873 f5c50349b1e4da233b554f371be49297b8b40ca5fe0188899ed906f682bf70d6 ++R 7e793d8e89e24019908c77c46fb6019c U dan - Z cdba8e6176109f758d02fe30f64aa30c -Z 1c716a1366090725a450505596ff38c0 ++Z 70bb9b566c677361e446902664d9825d # Remove this line to create a well-formed Fossil manifest. diff --cc manifest.uuid index 5492359300,8e78c67764..e3e4b64496 --- a/manifest.uuid +++ b/manifest.uuid @@@ -1,1 -1,1 +1,1 @@@ - 00bfae9ffafd71dfd0bbdbdc174e4ed7d7a850b385416f83be36a4f4a55d1873 -f5c50349b1e4da233b554f371be49297b8b40ca5fe0188899ed906f682bf70d6 ++135bf72c6b5f436b11463dd40b3a73bcc610bf9c2dcbe6646dbedd0d9325f6a9 diff --cc src/wal.c index b96a0c6be5,d63c13ffc4..a3d8dbb97a --- a/src/wal.c +++ b/src/wal.c @@@ -971,8 -721,10 +985,9 @@@ static void sehInjectFault(Wal *pWal) # define SEH_EXCEPT(X) VVA_ONLY(pWal->nSehTry--); assert( pWal->nSehTry==0 ); # define SEH_INJECT_FAULT assert( pWal->nSehTry>0 ); # define SEH_FREE_ON_ERROR(X,Y) + # define SEH_SET_ON_ERROR(X,Y) #endif /* ifdef SQLITE_USE_SEH */ - /* ** Obtain a pointer to the iPage'th page of the wal-index. The wal-index ** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are @@@ -1678,215 -1362,6 +1693,217 @@@ static int walIndexAppend(Wal *pWal, in return rc; } +/* +** Recover a single wal file - *-wal if iWal==0, or *-wal2 if iWal==1. +*/ +static int walIndexRecoverOne(Wal *pWal, int iWal, u32 *pnCkpt, int *pbZero){ + i64 nSize; /* Size of log file */ + u32 aFrameCksum[2] = {0, 0}; + int rc; + sqlite3_file *pWalFd = pWal->apWalFd[iWal]; + + assert( iWal==0 || iWal==1 ); + + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + sqlite3_randomness(8, pWal->hdr.aSalt); + + rc = sqlite3OsFileSize(pWalFd, &nSize); + if( rc==SQLITE_OK ){ + if( nSize>WAL_HDRSIZE ){ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u32 *aPrivate = 0; /* Heap copy of *-shm pg being populated */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int szFrame; /* Number of bytes in buffer aFrame[] */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + int szPage; /* Page size according to the log */ + u32 magic; /* Magic value read from WAL header */ + u32 version; /* Magic value read from WAL header */ + int isValid; /* True if this frame is valid */ + int iPg; /* Current 32KB wal-index page */ + int iLastFrame; /* Last frame in wal, based on size alone */ + int iLastPg; /* Last shm page used by this wal */ + + /* Read in the WAL header. */ + rc = sqlite3OsRead(pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* If the database page size is not a power of two, or is greater than + ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid + ** data. Similarly, if the 'magic' value is invalid, ignore the whole + ** WAL file. + */ + magic = sqlite3Get4byte(&aBuf[0]); + szPage = sqlite3Get4byte(&aBuf[8]); + if( (magic&0xFFFFFFFE)!=WAL_MAGIC + || szPage&(szPage-1) + || szPage>SQLITE_MAX_PAGE_SIZE + || szPage<512 + ){ + return SQLITE_OK; + } + pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); + pWal->szPage = szPage; + + /* Verify that the WAL header checksum is correct */ + walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, + aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum + ); + if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) + || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) + ){ + return SQLITE_OK; + } + + memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); + *pnCkpt = sqlite3Get4byte(&aBuf[12]); + + /* Verify that the version number on the WAL format is one that + ** are able to understand */ + version = sqlite3Get4byte(&aBuf[4]); + if( version!=WAL_VERSION1 && version!=WAL_VERSION2 ){ + return SQLITE_CANTOPEN_BKPT; + } + pWal->hdr.iVersion = version; + + /* Malloc a buffer to read frames into. */ + szFrame = szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc64(szFrame + WALINDEX_PGSZ); + SEH_FREE_ON_ERROR(0, aFrame); + if( !aFrame ){ + return SQLITE_NOMEM_BKPT; + } + aData = &aFrame[WAL_FRAME_HDRSIZE]; + aPrivate = (u32*)&aData[szPage]; + + /* Read all frames from the log file. */ + iLastFrame = (nSize - WAL_HDRSIZE) / szFrame; + if( version==WAL_VERSION2 ){ + iLastPg = walFramePage2(iWal, iLastFrame); + }else{ + iLastPg = walFramePage(iLastFrame); + } + for(iPg=iWal; iPg<=iLastPg; iPg+=(version==WAL_VERSION2 ? 2 : 1)){ + u32 *aShare; + int iFrame; /* Index of last frame read */ + int iLast; + int iFirst; + int nHdr, nHdr32; + + rc = walIndexPage(pWal, iPg, (volatile u32**)&aShare); + assert( aShare!=0 || rc!=SQLITE_OK ); + if( aShare==0 ) break; ++ SEH_SET_ON_ERROR(iPg, aShare); + pWal->apWiData[iPg] = aPrivate; + + if( iWal ){ + assert( version==WAL_VERSION2 ); + iFirst = 1 + (iPg/2)*HASHTABLE_NPAGE; + iLast = iFirst + HASHTABLE_NPAGE - 1; + }else{ + int i2 = (version==WAL_VERSION2) ? (iPg/2) : iPg; + iLast = HASHTABLE_NPAGE_ONE+i2*HASHTABLE_NPAGE; + iFirst = 1 + (i2==0?0:HASHTABLE_NPAGE_ONE+(i2-1)*HASHTABLE_NPAGE); + } + iLast = MIN(iLast, iLastFrame); + + for(iFrame=iFirst; iFrame<=iLast; iFrame++){ + i64 iOffset = walFrameOffset(iFrame, szPage); + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ + + /* Read and decode the next log frame. */ + rc = sqlite3OsRead(pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ) break; + isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); + if( !isValid ) break; + rc = walIndexAppend(pWal, iWal, iFrame, pgno); + if( NEVER(rc!=SQLITE_OK) ) break; + + /* If nTruncate is non-zero, this is a commit record. */ + if( nTruncate ){ + pWal->hdr.mxFrame = iFrame; + pWal->hdr.nPage = nTruncate; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; + aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + } + } + pWal->apWiData[iPg] = aShare; ++ SEH_SET_ON_ERROR(0, 0); + nHdr = (iPg==0 ? WALINDEX_HDR_SIZE : 0); + nHdr32 = nHdr / sizeof(u32); +#ifndef SQLITE_SAFER_WALINDEX_RECOVERY + /* Memcpy() should work fine here, on all reasonable implementations. + ** Technically, memcpy() might change the destination to some + ** intermediate value before setting to the final value, and that might + ** cause a concurrent reader to malfunction. Memcpy() is allowed to + ** do that, according to the spec, but no memcpy() implementation that + ** we know of actually does that, which is why we say that memcpy() + ** is safe for this. Memcpy() is certainly a lot faster. + */ + memcpy(&aShare[nHdr32], &aPrivate[nHdr32], WALINDEX_PGSZ-nHdr); +#else + /* In the event that some platform is found for which memcpy() + ** changes the destination to some intermediate value before + ** setting the final value, this alternative copy routine is + ** provided. + */ + { + int i; + for(i=nHdr32; ihdr.aFrameCksum[0] = aFrameCksum[0]; + pWal->hdr.aFrameCksum[1] = aFrameCksum[1]; + + return rc; +} + +static int walOpenWal2(Wal *pWal){ + int rc = SQLITE_OK; + if( !isOpen(pWal->apWalFd[1]) ){ + int f = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + rc = sqlite3OsOpen(pWal->pVfs, pWal->zWalName2, pWal->apWalFd[1], f, &f); + } + return rc; +} + +static int walTruncateWal2(Wal *pWal){ + int bIs; + int rc; + assert( !isOpen(pWal->apWalFd[1]) ); + rc = sqlite3OsAccess(pWal->pVfs, pWal->zWalName2, SQLITE_ACCESS_EXISTS, &bIs); + if( rc==SQLITE_OK && bIs ){ + rc = walOpenWal2(pWal); + if( rc==SQLITE_OK ){ + rc = sqlite3OsTruncate(pWal->apWalFd[1], 0); + sqlite3OsClose(pWal->apWalFd[1]); + } + } + return rc; +} /* ** Recover the wal-index by reading the write-ahead log file. diff --cc src/wal.h index 9a047d0536,1b17d2dfbe..2f320b66b0 --- a/src/wal.h +++ b/src/wal.h @@@ -45,7 -45,7 +45,8 @@@ # define sqlite3WalFramesize(z) 0 # define sqlite3WalFindFrame(x,y,z) 0 # define sqlite3WalFile(x) 0 +# define sqlite3WalJournalMode(x) 0 + # undef SQLITE_USE_SEH #else #define WAL_SAVEPOINT_NDATA 4 diff --cc test/wal2recover2.test index 60ecfc28bf,0000000000..9b8058c1d7 mode 100644,000000..100644 --- a/test/wal2recover2.test +++ b/test/wal2recover2.test @@@ -1,267 -1,0 +1,267 @@@ +# 2018 December 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2recover2 +ifcapable !wal {finish_test ; return } + +do_execsql_test 1.0 { + CREATE TABLE t1(x); + CREATE TABLE t2(x); + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 ) + INSERT INTO t1 SELECT i FROM s; + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 ) + INSERT INTO t2 SELECT i FROM s; + + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 10000; +} {wal2 10000} + +set ::L 1125750 +set ::M 1126500 +set ::H 1127250 + +do_execsql_test 1.1 { + UPDATE t1 SET x=x+1; + UPDATE t2 SET x=x+1 WHERE rowid<=750; + + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; +} [list $H $M] + +do_test 1.2 { + list [file size test.db] [file size test.db-wal] [file size test.db-wal2] +} {31744 14704 7368} + +proc cksum {zIn data} { + if {[string length $zIn]==0} { + set s0 0 + set s1 0 + } else { + set s0 [hexio_get_int [string range $zIn 0 7]] + set s1 [hexio_get_int [string range $zIn 8 15]] + } + set n [expr [string length $data] / 8] + + for {set i 0} {$i < $n} {incr i 2} { + set x0 [hexio_get_int -l [string range $data [expr $i*8] [expr $i*8+7]]] + set x1 [hexio_get_int -l [string range $data [expr $i*8+8] [expr $i*8+8+7]]] + + set s0 [expr ($s0 + $x0 + $s1) & 0xFFFFFFFF] + set s1 [expr ($s1 + $x1 + $s0) & 0xFFFFFFFF] + } + + return "[hexio_render_int32 $s0][hexio_render_int32 $s1]" +} + +proc fix_wal_cksums {file} { + # Fix the checksum on the wal header. + set data [hexio_read $file 0 32] + set cksum [cksum {} [string range $data 0 47]] + set salt [hexio_read $file 16 8] + hexio_write $file 24 $cksum + + # Fix the checksums for all pages in the wal file. + set pgsz [hexio_get_int [hexio_read $file 8 4]] + set sz [file size $file] + for {set off 32} {$off < $sz} {incr off [expr $pgsz+24]} { + set e [hexio_read $file $off 8] + set cksum [cksum $cksum $e] + + set p [hexio_read $file [expr $off+24] $pgsz] + set cksum [cksum $cksum $p] + + hexio_write $file [expr $off+8] $salt + hexio_write $file [expr $off+16] $cksum + } +} + +proc wal_incr_hdrfield {file field} { + switch -- $field { + nCkpt { set offset 12 } + salt0 { set offset 16 } + salt1 { set offset 20 } + default { + error "unknown field $field - should be \"nCkpt\", \"salt0\" or \"salt1\"" + } + } + + # Increment the value in the wal header. + set v [hexio_get_int [hexio_read $file $offset 4]] + incr v + hexio_write $file $offset [hexio_render_int32 $v] + + # Fix various checksums + fix_wal_cksums $file +} + +proc wal_set_nckpt {file val} { + # Increment the value in the wal header. + hexio_write $file 12 [hexio_render_int32 $val] + + # Fix various checksums + fix_wal_cksums $file +} + +proc wal_set_follow {file prevfile} { + set pgsz [hexio_get_int [hexio_read $prevfile 8 4]] + set sz [file size $prevfile] + set cksum [hexio_read $prevfile [expr $sz-$pgsz-8] 8] + + hexio_write $file 16 $cksum + fix_wal_cksums $file +} + +foreach {tn file field} { + 1 test.db2-wal salt0 + 2 test.db2-wal salt1 + 3 test.db2-wal nCkpt + 4 test.db2-wal2 salt0 + 5 test.db2-wal2 salt1 + 6 test.db2-wal2 nCkpt +} { + do_test 1.3.$tn { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-wal2 test.db2-wal2 + wal_incr_hdrfield $file $field + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 + } [list $H $L] + db2 close +} + +do_test 1.4 { + forcecopy test.db test.db2 + forcecopy test.db-wal2 test.db2-wal + forcedelete test.db2-wal2 + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 +} [list $L $M] + +do_test 1.5 { + db2 close + forcecopy test.db test.db2 + forcecopy test.db-wal2 test.db2-wal + forcecopy test.db-wal test.db2-wal2 + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 +} [list $H $M] + +db2 close +foreach {tn file field} { + 1 test.db2-wal salt0 + 2 test.db2-wal salt1 + 3 test.db2-wal2 salt0 + 4 test.db2-wal2 salt1 +} { + do_test 1.6.$tn { + forcecopy test.db test.db2 + forcecopy test.db-wal2 test.db2-wal + forcecopy test.db-wal test.db2-wal2 + wal_incr_hdrfield $file $field + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 + } [list $H $L] + db2 close +} + +foreach {tn nCkpt1 nCkpt2 res} [list \ + 1 2 1 "$H $M" \ + 2 2 2 "$L $M" \ + 3 3 1 "$H $L" \ + 4 15 14 "$H $M" \ + 5 0 15 "$H $M" \ + 6 1 15 "$L $M" \ +] { + do_test 1.7.$tn { + forcecopy test.db test.db2 + forcecopy test.db-wal2 test.db2-wal + forcecopy test.db-wal test.db2-wal2 + + wal_set_nckpt test.db2-wal2 $nCkpt2 + wal_set_nckpt test.db2-wal $nCkpt1 + wal_set_follow test.db2-wal test.db2-wal2 + + + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 + } $res + db2 close +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 1.8.1 { + PRAGMA autovacuum = 0; + PRAGMA page_size = 4096; + CREATE TABLE t1(x); + CREATE TABLE t2(x); + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 ) + INSERT INTO t1 SELECT i FROM s; + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 ) + INSERT INTO t2 SELECT i FROM s; + + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 10000; + + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 ) + INSERT INTO t2 SELECT i FROM s; +} {wal2 10000} + +do_test 1.8.2 { + list [file size test.db-wal] [file size test.db-wal2] +} {24752 0} + +do_execsql_test 1.8.3 { PRAGMA user_version = 123 } +do_test 1.8.4 { + list [file size test.db-wal] [file size test.db-wal2] +} {24752 4152} + +do_test 1.8.5 { + hexio_write test.db-wal2 [expr 56+16] 0400 + fix_wal_cksums test.db-wal2 +} {} + +do_test 1.8.6 { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-wal2 test.db2-wal2 + sqlite3 db2 test.db2 + catchsql { SELECT * FROM sqlite_master } db2 - } {1 {malformed database schema (?)}} ++} {1 {database disk image is malformed}} +db2 close + +finish_test