From: drh <> Date: Mon, 16 Dec 2024 18:29:31 +0000 (+0000) Subject: Merge divergence-reduction changes into the bedrock branch. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=602f13bdc94b2e24098528a50401e6e5d68859d3;p=thirdparty%2Fsqlite.git Merge divergence-reduction changes into the bedrock branch. FossilOrigin-Name: ec5d7025cba9f4acaea984d5ec29b05b7f4b01f0e36e5287f27a16895ec42bf7 --- 602f13bdc94b2e24098528a50401e6e5d68859d3 diff --cc manifest index 5a2fb1eb3e,77c5a06e93..0d1905e7e0 --- a/manifest +++ b/manifest @@@ -1,5 -1,5 +1,5 @@@ - C Merge\sall\sthe\slatest\strunk/wal2\senhancements\sand\sfixes\sinto\sthe\sbedrock\sbranch. - D 2024-12-16T13:38:33.681 -C Sync\swal2\swith\strunk. -D 2024-12-16T18:16:09.524 ++C Merge\sdivergence-reduction\schanges\sinto\sthe\sbedrock\sbranch. ++D 2024-12-16T18:29:31.774 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@@ -853,8 -847,8 +853,8 @@@ F src/update.c 2dd1b745acc9253df1b210ac F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c 8b29d9a5956569ea2700f869669b8ef67a9662ee5e724ff77ab3c387e27094ba F src/util.c e5f6a5eeaa26b69054a43bbd0048cfe3d2851f6961052b35aed8f695df922850 -F src/vacuum.c b763b6457bd058d2072ef9364832351fd8d11e8abf70cbb349657360f7d55c40 -F src/vdbe.c b2d91fce8b2d357b69d3c4aa9d9175736132a1db52baa091ee8cc54f0926120a +F src/vacuum.c 25e407a6dc8b288fa4295b3d92fa9ce9318503e84df53cdf403a50fccbc1ba31 - F src/vdbe.c 310284edd21f8471f945038a8fce25da0e025b568ebde5c16b11b32a956641e0 ++F src/vdbe.c f567c19f5029f3e221931ced6436c718a1609d2de7fb158de57eac1a0f1e61ce F src/vdbe.h 9676348d342bd04e21e384c63b57224171ce84fac77853357334ef94c4d33cf4 F src/vdbeInt.h bf294a0c8fc4cc80779e74b04b8bd82c6e1197b3137cefe0b16cdf002fc7dfd6 F src/vdbeapi.c 38c252a202d70b56cfb734460bc888ddbd581afec1a10cd4d6c894c9e0b5baea @@@ -866,8 -860,8 +866,8 @@@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1 F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 F src/vtab.c 316cd48e9320660db3047cd306cd056e4361180cebb4d0f10a39244e10c11422 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 - F src/wal.c 3dfcce08758eec841bab07320c2b4e495c1c0621c2fd6327b3c3b19f52da48b6 -F src/wal.c 3ff22607c591f208bf81b5ea2160e5715d12bbb5b762394f79b2bd13d3f7568b -F src/wal.h 97b8a9903387401377b59507e86b93a148ef1ad4e5ce0f23659a12dcdce56af2 ++F src/wal.c 964c785ac8324de1ae742c44aa72ddd4401d4abab11ef6999aec21d7221b3e67 +F src/wal.h 8d02ab8c2a93a941f5898eb3345bf711c1d3f8f86f4be8d5428fb6c074962d8a F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 F src/where.c 9ad3dea8003a8913da6a4ca8322e2fe30773f46e88a0d4fbf9db13bdb999efa2 F src/whereInt.h 1e36ec50392f7cc3d93d1152d4338064cd522b87156a0739388b7e273735f0ca @@@ -2239,8 -2216,8 +2239,8 @@@ F tool/version-info.c 3b36468a90faf1bbd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f - P eb8449ea9ac8e29425f7127535a1db328d56c86382919fb1fcf42324b71de013 eab619453abf6979962a68d298c44553ef0d9eb18112e61420d4de8a9c16f620 - R 0be70ab68e68f62d1526e079bfae1838 -P e4406a6e2660f1f42614d0034d024cc206109ac71e9f8b289aeed2eba20be8e9 8f725472b0fe62359a4cd3237b43d7b834e042d8ce425abde06e3ed6c62dbafa -R a440747468661c427837f99ee412b9ae ++P a0cf2621c4586ddfa43ec5a2a6469ddb8528adff78a80063be007cf76cf8d98a caadbe0c0c3dee411140df7d13f6e8275f9c13562bb384be38520ee2305c32bd ++R 69e1b203e08eb71e1b73ec7264118057 U drh - Z 5899f9e916b0dbf68fc97b35b2f61daf -Z fa1fbe4ac13afbbbd055627cdf93d6dd ++Z a1145c781041a58de3461e5eb972096e # Remove this line to create a well-formed Fossil manifest. diff --cc manifest.uuid index 74e8550b33,97514e3705..1b7f0dc3cb --- a/manifest.uuid +++ b/manifest.uuid @@@ -1,1 -1,1 +1,1 @@@ - a0cf2621c4586ddfa43ec5a2a6469ddb8528adff78a80063be007cf76cf8d98a -caadbe0c0c3dee411140df7d13f6e8275f9c13562bb384be38520ee2305c32bd ++ec5d7025cba9f4acaea984d5ec29b05b7f4b01f0e36e5287f27a16895ec42bf7 diff --cc src/wal.c index 1f2db82f05,891552b2d1..fc0a24510f --- a/src/wal.c +++ b/src/wal.c @@@ -4476,266 -4410,40 +4475,265 @@@ int sqlite3WalBeginWriteTransaction(Wa return SQLITE_OK; } #endif + + rc = walWriteLock(pWal); + if( rc==SQLITE_OK ){ + /* If another connection has written to the database file since the + ** time the read transaction on this connection was started, then + ** the write is disallowed. Release the WRITER lock and return + ** SQLITE_BUSY_SNAPSHOT in this case. */ + SEH_TRY { + if( memcmp(&pWal->hdr, (void*)walIndexHdr(pWal),sizeof(WalIndexHdr))!=0 ){ + rc = SQLITE_BUSY_SNAPSHOT; + } + } + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + if( rc!=SQLITE_OK ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + } + } + return rc; +} - /* Cannot start a write transaction without first holding a read - ** transaction. */ - assert( pWal->readLock!=WAL_LOCK_NONE ); - assert( pWal->writeLock==0 && pWal->iReCksum==0 ); +/* +** This function is called by a writer that has a read-lock on aReadmark[0] +** (pWal->readLock==0). This function relinquishes that lock and takes a +** lock on a different aReadmark[] slot. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int walUpgradeReadlock(Wal *pWal){ + int cnt; + int rc; + assert( pWal->writeLock && pWal->readLock==0 ); + assert( isWalMode2(pWal)==0 ); + walUnlockShared(pWal, WAL_READ_LOCK(0)); + pWal->readLock = -1; + cnt = 0; + do{ + int notUsed; + rc = walTryBeginRead(pWal, ¬Used, 1, &cnt); + }while( rc==WAL_RETRY ); + assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ + testcase( (rc&0xff)==SQLITE_IOERR ); + testcase( rc==SQLITE_PROTOCOL ); + testcase( rc==SQLITE_OK ); + return rc; +} - if( pWal->readOnly ){ - return SQLITE_READONLY; - } - /* Only one writer allowed at a time. Get the write lock. Return - ** SQLITE_BUSY if unable. - */ - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); - if( rc ){ - return rc; +#ifndef SQLITE_OMIT_CONCURRENT +/* +** A concurrent transaction has conflicted with external frame iExternal. +** Transform this value to the one required by SQLITE_COMMIT_CONFLICT_FRAME - +** the frame offset within its wal file, with the 0x80000000 bit set for +** wal2, clear for the default wal file. +*/ +static u32 walConflictFrame(Wal *pWal, u32 iExternal){ + u32 iRet = iExternal; + if( isWalMode2(pWal) ){ + int bFile = walExternalDecode(iExternal, &iRet); + iRet = (iRet | (bFile ? 0x80000000 : 0)); } - pWal->writeLock = 1; + return iRet; +} + +/* +** This function does the work of sqlite3WalLockForCommit(). The difference +** between this function and sqlite3WalLockForCommit() is that the latter +** encloses everything in a SEH_TRY {} block. +*/ +static int walLockForCommit( + Wal *pWal, + PgHdr *pPg1, + Bitvec *pAllRead, + u32 *aConflict +){ + int rc = walWriteLock(pWal); - /* If another connection has written to the database file since the - ** time the read transaction on this connection was started, then - ** the write is disallowed. + /* If the database has been modified since this transaction was started, + ** check if it is still possible to commit. The transaction can be + ** committed if: + ** + ** a) None of the pages in pList have been modified since the + ** transaction opened, and + ** + ** b) The database schema cookie has not been modified since the + ** transaction was started. */ - SEH_TRY { - if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + if( rc==SQLITE_OK ){ + WalIndexHdr head; + + if( walIndexLoadHdr(pWal, &head) ){ + /* This branch is taken if the wal-index header is corrupted. This + ** occurs if some other writer has crashed while committing a + ** transaction to this database since the current concurrent transaction + ** was opened. */ rc = SQLITE_BUSY_SNAPSHOT; + }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){ + int bWal2 = isWalMode2(pWal); + int iHash; + int nLoop = 1+(bWal2 && walidxGetFile(&head)!=walidxGetFile(&pWal->hdr)); + int iLoop; + + if( pPg1==0 ){ + /* If pPg1==0, then the current transaction modified the database + ** schema. This means it conflicts with all other transactions. */ + u32 bFile = walidxGetFile(&pWal->hdr); + u32 iFrame = walidxGetMxFrame(&head, bFile) | (bFile << 31); + aConflict[SQLITE_COMMIT_CONFLICT_PGNO] = 1; + aConflict[SQLITE_COMMIT_CONFLICT_FRAME] = iFrame; + rc = SQLITE_BUSY_SNAPSHOT; + } + + assert( nLoop==1 || nLoop==2 ); + for(iLoop=0; rc==SQLITE_OK && iLoophdr.mxFrame (which will be + ** set to the size of the old, now overwritten, wal file). This + ** doesn't come up in wal2 mode, as in wal2 mode the client always + ** has a PART lock on one of the wal files, preventing it from being + ** checkpointed or overwritten. */ + iFirst = pWal->hdr.mxFrame+1; + if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){ + assert( pWal->readLock==0 ); + iFirst = 1; + } + mxFrame = head.mxFrame; + }else{ + int iA = walidxGetFile(&pWal->hdr); + if( iLoop==0 ){ + iFirst = walExternalEncode(iA, 1+walidxGetMxFrame(&pWal->hdr, iA)); + mxFrame = walExternalEncode(iA, walidxGetMxFrame(&head, iA)); + }else{ + iFirst = walExternalEncode(!iA, 1); + mxFrame = walExternalEncode(!iA, walidxGetMxFrame(&head, !iA)); + } + } + iLastHash = walFramePage(mxFrame); + + for(iHash=walFramePage(iFirst); iHash<=iLastHash; iHash += (1+bWal2)){ + WalHashLoc sLoc; + + rc = walHashGet(pWal, iHash, &sLoc); + if( rc==SQLITE_OK ){ + u32 i, iMin, iMax; + assert( mxFrame>=sLoc.iZero ); + iMin = (sLoc.iZero >= iFirst) ? 1 : (iFirst - sLoc.iZero); + iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; + if( iMax>(mxFrame-sLoc.iZero) ) iMax = (mxFrame-sLoc.iZero); + for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){ + PgHdr *pPg; + if( sLoc.aPgno[i-1]==1 ){ + /* Check that the schema cookie has not been modified. If + ** it has not, the commit can proceed. */ + u8 aNew[4]; + u8 *aOld = &((u8*)pPg1->pData)[40]; + int sz; + i64 iOff; + u32 iFrame = sLoc.iZero + i; + int iWal = 0; + if( bWal2 ){ + iWal = walExternalDecode(iFrame, &iFrame); + } + sz = head.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + iOff = walFrameOffset(iFrame, sz) + WAL_FRAME_HDRSIZE + 40; + rc = sqlite3OsRead(pWal->apWalFd[iWal],aNew,sizeof(aNew),iOff); + if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ + u32 iFrame = walConflictFrame(pWal, sLoc.iZero+i); + aConflict[SQLITE_COMMIT_CONFLICT_PGNO] = 1; + aConflict[SQLITE_COMMIT_CONFLICT_FRAME] = iFrame; + rc = SQLITE_BUSY_SNAPSHOT; + } + }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i-1]) ){ + u32 iFrame = walConflictFrame(pWal, sLoc.iZero+i); + aConflict[SQLITE_COMMIT_CONFLICT_PGNO] = sLoc.aPgno[i-1]; + aConflict[SQLITE_COMMIT_CONFLICT_FRAME] = iFrame; + rc = SQLITE_BUSY_SNAPSHOT; + }else + if( (pPg = sqlite3PagerLookup(pPg1->pPager, sLoc.aPgno[i-1])) ){ + /* Page aPgno[i], which is present in the pager cache, has been + ** modified since the current CONCURRENT transaction was + ** started. However it was not read by the current + ** transaction, so is not a conflict. There are two + ** possibilities: (a) the page was allocated at the of the file + ** by the current transaction or (b) was present in the cache + ** at the start of the transaction. + ** + ** For case (a), do nothing. This page will be moved within the + ** database file by the commit code to avoid the conflict. The + ** call to PagerUnref() is to release the reference grabbed by + ** the sqlite3PagerLookup() above. + ** + ** In case (b), drop the page from the cache - otherwise + ** following the snapshot upgrade the cache would be + ** inconsistent with the database as stored on disk. */ + if( sqlite3PagerIswriteable(pPg) ){ + sqlite3PagerUnref(pPg); + }else{ + sqlite3PcacheDrop(pPg); + } + } + } + } + if( rc!=SQLITE_OK ) break; + } + } } } - - SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + pWal->nPriorFrame = walGetPriorFrame(&pWal->hdr); + return rc; +} - if( rc!=SQLITE_OK ){ - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); - pWal->writeLock = 0; - } +/* +** This function is only ever called when committing a "BEGIN CONCURRENT" +** transaction. It may be assumed that no frames have been written to +** the wal file. The second parameter is a pointer to the in-memory +** representation of page 1 of the database (which may or may not be +** dirty). The third is a bitvec with a bit set for each page in the +** database file that was read by the current concurrent transaction. +** +** This function performs three tasks: +** +** 1) It obtains the WRITER lock on the wal file, +** +** 2) It checks that there are no conflicts between the current +** transaction and any transactions committed to the wal file since +** it was opened, and +** +** 3) It ejects any non-dirty pages from the page-cache that have been +** written by another client since the CONCURRENT transaction was started +** (so as to avoid ending up with an inconsistent cache after the +** current transaction is committed). +** +** If no error occurs and the caller may proceed with committing the +** transaction, SQLITE_OK is returned. SQLITE_BUSY is returned if the WRITER +** lock cannot be obtained. Or, if the WRITER lock can be obtained but there +** are conflicts with a committed transaction, SQLITE_BUSY_SNAPSHOT. Finally, +** if an error (i.e. an OOM condition or IO error), an SQLite error code +** is returned. +*/ +int sqlite3WalLockForCommit( + Wal *pWal, + PgHdr *pPg1, + Bitvec *pAllRead, + Pgno *piConflict +){ + int rc = SQLITE_OK; + SEH_TRY { + rc = walLockForCommit(pWal, pPg1, pAllRead, piConflict); + } SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) return rc; } @@@ -5028,19 -4659,18 +5026,18 @@@ static int walRestartLog(Wal *pWal) return rc; } } - walUnlockShared(pWal, WAL_READ_LOCK(0)); - pWal->readLock = WAL_LOCK_NONE; - cnt = 0; - do{ - int notUsed; - rc = walTryBeginRead(pWal, ¬Used, 1, &cnt); - }while( rc==WAL_RETRY ); - assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ - testcase( (rc&0xff)==SQLITE_IOERR ); - testcase( rc==SQLITE_PROTOCOL ); - testcase( rc==SQLITE_OK ); - } + + /* Regardless of whether or not the wal file was restarted, change the + ** read-lock held by this client to a slot other than aReadmark[0]. + ** Clients with a lock on aReadmark[0] read from the database file + ** only - never from the wal file. This means that if a writer holding + ** a lock on aReadmark[0] were to commit a transaction but not close the + ** read-transaction, subsequent read operations would read directly from + ** the database file - ignoring the new pages just appended + ** to the wal file. */ + rc = walUpgradeReadlock(pWal); + } - + pWal->nPriorFrame = walGetPriorFrame(&pWal->hdr); return rc; }