From: drh <> Date: Tue, 9 Nov 2021 14:52:22 +0000 (+0000) Subject: Merge the walIndexPage() enhancement from trunk into the begin-concurrent X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6a608be78c0c452f2832174fcaaeb37f0c19cd34;p=thirdparty%2Fsqlite.git Merge the walIndexPage() enhancement from trunk into the begin-concurrent branch. FossilOrigin-Name: ff2238397f78a3c276fa96f2a9cafb6f99c8e7eeaa69cf513a9f7b7581778ec5 --- 6a608be78c0c452f2832174fcaaeb37f0c19cd34 diff --cc manifest index a44875959a,7abb13825f..aabc5fcce6 --- a/manifest +++ b/manifest @@@ -1,5 -1,5 +1,5 @@@ - C Merge\sall\strunk\senhancements\sprior\sto\sthe\swalIndexPage()\sfix\sinto\sthe\nbegin-concurrent\sbranch. - D 2021-11-09T14:36:58.717 -C Ensure\sthat\sthe\sWAL\scode\scorrectly\shandles\sall\spossible\soutcomes\sfrom\sthe\nwalIndexPage()\sroutine. -D 2021-10-28T00:09:31.896 ++C Merge\sthe\swalIndexPage()\senhancement\sfrom\strunk\sinto\sthe\sbegin-concurrent\nbranch. ++D 2021-11-09T14:52:22.435 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@@ -638,8 -633,8 +638,8 @@@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1 F src/vdbevtab.c f99b275366c5fc5e2d99f734729880994ab9500bdafde7fae3b02d562b9d323c F src/vtab.c d07cc24dd84b0b51bf05adb187b0d2e6b0cac56cfbc0197995a26d4f8fa5c7e2 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 - F src/wal.c e3a21abbd23d5c8f30fb03e545ba292d3640890968e62ef1604c4ff30c0f043a -F src/wal.c 6ae14d4797f31c67fc2be659d24fbc6e1a6f5f423bdfb5ef831ea171bce42320 -F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a ++F src/wal.c b8668cec19412f78968159abdc6d17e32d99d0f7dfb7eb5a7d732eae682f64df +F src/wal.h 7ffe787437f20a098af347011967a6d3bb8e5c3dc645e6be59eff44d2b2c5297 F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c ecabef93b0f14442a73eca205fc960428984d75fbdc7d60226b7fc9cac127187 F src/whereInt.h 83877a75a1bce056ea44aff02f1dfa958ad1d6038c213ddadb8652003b45151d @@@ -1944,7 -1929,8 +1944,7 @@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 - P ae4eed0b7eccbba0ea91ae209fd94a9aefad41d57c1e5e2d07a796fc7b4f17db 7238d58051bfdcea8f7a4aeab89145849d0659c987df9063aacafe97be6657fe - R f72428c1b043951223fa6602387b12e3 -P 7238d58051bfdcea8f7a4aeab89145849d0659c987df9063aacafe97be6657fe 12715c6b234a04627ca27e94bfa8bd456998360a9f0117480e0038f4747818d6 -R fe9d125153f206ef4cb83374e184623d -T +closed 12715c6b234a04627ca27e94bfa8bd456998360a9f0117480e0038f4747818d6 ++P 6603f00581ed29b3fa9e10f22357a8e5154934c0df4737d7b6b6c0f8d4f22a80 6979efbf07d93e7afad508165df684dcc6fe33b91ca772397c8afa00d16d1a0d ++R febe73fbea83d026b9a9cf44666fdcac U drh - Z 4baa6d53c991c50daa02bdb231c04d6c -Z b79ac0d17a59f6e7c71e03a206543e27 ++Z 3ffee0f1219deb9e392061f07a1c5359 diff --cc manifest.uuid index 53f2f8a0d2,11f72edd9c..6973cd8f76 --- a/manifest.uuid +++ b/manifest.uuid @@@ -1,1 -1,1 +1,1 @@@ - 6603f00581ed29b3fa9e10f22357a8e5154934c0df4737d7b6b6c0f8d4f22a80 -6979efbf07d93e7afad508165df684dcc6fe33b91ca772397c8afa00d16d1a0d ++ff2238397f78a3c276fa96f2a9cafb6f99c8e7eeaa69cf513a9f7b7581778ec5 diff --cc src/wal.c index 095380f3dd,4c37560798..80105721f1 --- a/src/wal.c +++ b/src/wal.c @@@ -3321,176 -3287,35 +3329,177 @@@ int sqlite3WalBeginWriteTransaction(Wa return SQLITE_OK; } #endif + + rc = walWriteLock(pWal); + if( rc==SQLITE_OK ){ + /* If another connection has written to the database file since the + ** time the read transaction on this connection was started, then + ** the write is disallowed. Release the WRITER lock and return + ** SQLITE_BUSY_SNAPSHOT in this case. */ + if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + rc = SQLITE_BUSY_SNAPSHOT; + } + } + return rc; +} - /* Cannot start a write transaction without first holding a read - ** transaction. */ - assert( pWal->readLock>=0 ); - assert( pWal->writeLock==0 && pWal->iReCksum==0 ); +/* +** This function is called by a writer that has a read-lock on aReadmark[0] +** (pWal->readLock==0). This function relinquishes that lock and takes a +** lock on a different aReadmark[] slot. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int walUpgradeReadlock(Wal *pWal){ + int cnt; + int rc; + assert( pWal->writeLock && pWal->readLock==0 ); + walUnlockShared(pWal, WAL_READ_LOCK(0)); + pWal->readLock = -1; + cnt = 0; + do{ + int notUsed; + rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); + }while( rc==WAL_RETRY ); + assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ + testcase( (rc&0xff)==SQLITE_IOERR ); + testcase( rc==SQLITE_PROTOCOL ); + testcase( rc==SQLITE_OK ); + return rc; +} - if( pWal->readOnly ){ - return SQLITE_READONLY; - } - /* Only one writer allowed at a time. Get the write lock. Return - ** SQLITE_BUSY if unable. - */ - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); - if( rc ){ - return rc; - } - pWal->writeLock = 1; +#ifndef SQLITE_OMIT_CONCURRENT +/* +** This function is only ever called when committing a "BEGIN CONCURRENT" +** transaction. It may be assumed that no frames have been written to +** the wal file. The second parameter is a pointer to the in-memory +** representation of page 1 of the database (which may or may not be +** dirty). The third is a bitvec with a bit set for each page in the +** database file that was read by the current concurrent transaction. +** +** This function performs three tasks: +** +** 1) It obtains the WRITER lock on the wal file, +** +** 2) It checks that there are no conflicts between the current +** transaction and any transactions committed to the wal file since +** it was opened, and +** +** 3) It ejects any non-dirty pages from the page-cache that have been +** written by another client since the CONCURRENT transaction was started +** (so as to avoid ending up with an inconsistent cache after the +** current transaction is committed). +** +** If no error occurs and the caller may proceed with committing the +** transaction, SQLITE_OK is returned. SQLITE_BUSY is returned if the WRITER +** lock cannot be obtained. Or, if the WRITER lock can be obtained but there +** are conflicts with a committed transaction, SQLITE_BUSY_SNAPSHOT. Finally, +** if an error (i.e. an OOM condition or IO error), an SQLite error code +** is returned. +*/ +int sqlite3WalLockForCommit( + Wal *pWal, + PgHdr *pPg1, + Bitvec *pAllRead, + Pgno *piConflict +){ + int rc = walWriteLock(pWal); - /* If another connection has written to the database file since the - ** time the read transaction on this connection was started, then - ** the write is disallowed. + /* If the database has been modified since this transaction was started, + ** check if it is still possible to commit. The transaction can be + ** committed if: + ** + ** a) None of the pages in pList have been modified since the + ** transaction opened, and + ** + ** b) The database schema cookie has not been modified since the + ** transaction was started. */ - if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); - pWal->writeLock = 0; - rc = SQLITE_BUSY_SNAPSHOT; + if( rc==SQLITE_OK ){ + WalIndexHdr head; + + if( walIndexLoadHdr(pWal, &head) ){ + /* This branch is taken if the wal-index header is corrupted. This + ** occurs if some other writer has crashed while committing a + ** transaction to this database since the current concurrent transaction + ** was opened. */ + rc = SQLITE_BUSY_SNAPSHOT; + }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){ + int iHash; + int iLast = walFramePage(head.mxFrame); + u32 iFirst = pWal->hdr.mxFrame+1; /* First wal frame to check */ + if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){ + assert( pWal->readLock==0 ); + iFirst = 1; + } + if( pPg1==0 ){ + /* If pPg1==0, then the current transaction modified the database + ** schema. This means it conflicts with all other transactions. */ + *piConflict = 1; + rc = SQLITE_BUSY_SNAPSHOT; + } + for(iHash=walFramePage(iFirst); rc==SQLITE_OK && iHash<=iLast; iHash++){ + WalHashLoc sLoc; + + rc = walHashGet(pWal, iHash, &sLoc); + if( rc==SQLITE_OK ){ + u32 i, iMin, iMax; + assert( head.mxFrame>=sLoc.iZero ); + iMin = (sLoc.iZero >= iFirst) ? 1 : (iFirst - sLoc.iZero); + iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; + if( iMax>(head.mxFrame-sLoc.iZero) ) iMax = (head.mxFrame-sLoc.iZero); + for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){ + PgHdr *pPg; - if( sLoc.aPgno[i]==1 ){ ++ if( sLoc.aPgno[i-1]==1 ){ + /* Check that the schema cookie has not been modified. If + ** it has not, the commit can proceed. */ + u8 aNew[4]; + u8 *aOld = &((u8*)pPg1->pData)[40]; + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + iOffset = walFrameOffset(i+sLoc.iZero, sz) + WAL_FRAME_HDRSIZE+40; + rc = sqlite3OsRead(pWal->pWalFd, aNew, sizeof(aNew), iOffset); + if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ + rc = SQLITE_BUSY_SNAPSHOT; + } - }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i]) ){ - *piConflict = sLoc.aPgno[i]; ++ }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i-1]) ){ ++ *piConflict = sLoc.aPgno[i-1]; + rc = SQLITE_BUSY_SNAPSHOT; - }else if( (pPg = sqlite3PagerLookup(pPg1->pPager, sLoc.aPgno[i])) ){ - /* Page aPgno[i], which is present in the pager cache, has been ++ }else ++ if( (pPg = sqlite3PagerLookup(pPg1->pPager, sLoc.aPgno[i-1])) ){ ++ /* Page aPgno[i-1], which is present in the pager cache, has been + ** modified since the current CONCURRENT transaction was started. + ** However it was not read by the current transaction, so is not + ** a conflict. There are two possibilities: (a) the page was + ** allocated at the of the file by the current transaction or + ** (b) was present in the cache at the start of the transaction. + ** + ** For case (a), do nothing. This page will be moved within the + ** database file by the commit code to avoid the conflict. The + ** call to PagerUnref() is to release the reference grabbed by + ** the sqlite3PagerLookup() above. + ** + ** In case (b), drop the page from the cache - otherwise + ** following the snapshot upgrade the cache would be inconsistent + ** with the database as stored on disk. */ + if( sqlite3PagerIswriteable(pPg) ){ + sqlite3PagerUnref(pPg); + }else{ + sqlite3PcacheDrop(pPg); + } + } + } + } + } + } } + pWal->nPriorFrame = pWal->hdr.mxFrame; return rc; }