From: dan Date: Tue, 4 Dec 2018 19:41:07 +0000 (+0000) Subject: First attempt at making features work together. Only the most minimal testing X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=7a13e692be60a25104daeb2e921aa2a0a82cd244;p=thirdparty%2Fsqlite.git First attempt at making features work together. Only the most minimal testing so far. FossilOrigin-Name: fd707001f0afb1cf32cfeeda3ec7b5622eb49ddedf8fec1a7aa4c8841c77c37a --- diff --git a/manifest b/manifest index dd9771b7e3..dc2ae521ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\swal2\sand\sbegin-concurrent\scode.\sBoth\sfeatures\swork,\sbut\snot\sat\sthe\nsame\stime. -D 2018-12-03T20:49:34.015 +C First\sattempt\sat\smaking\sfeatures\swork\stogether.\sOnly\sthe\smost\sminimal\stesting\nso\sfar. +D 2018-12-04T19:41:07.389 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -593,7 +593,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 88e424b4de9cb836166a967b5b8a6de7f90cdd493dd1d04a1b855fa58dbe7212 +F src/wal.c 4b7ad0e623e01ede1ecb2cae24548c29e51ce5a21b2fd91af26b4bb447b72cc4 F src/wal.h b42fc8081cd1765d4d4dd99b33f2db2f71128f4e25ff8c08d1a346f5af62e27a F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1606,6 +1606,7 @@ F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bd F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test a225bafac35a47765b890bacdeb57e5e81039f21cc18a1e8ce88eb76e56b843c +F test/wal2concurrent.test d0af64c3b113993553acfeeb64d207fb6d924e498fd953dbf31040a566ec10df F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c F test/wal2simple.test 8c9dfb8f1bca01a0deb57f7074cdb83865c2292e89b13f7a51a1c160dca3f5f4 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 @@ -1795,10 +1796,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 480be916c840e9e28010c22cf29a8396502c9e6387f31f750260c6290f58e9a1 -R 41c3c1de1e62db08c33d5cf08c11006b -T *branch * begin-concurrent-wal2 -T *sym-begin-concurrent-wal2 * -T -sym-begin-concurrent * +P b7281a1caa574870a071bea3e96b1d8210c28c17f9094449b3ce1a42b311e6a1 +R db024f31bd5bbdadf08ded59e2f54e6f U dan -Z f5d7b930a621f73c3ef64e6a734bda83 +Z abc3921392e51c645845b41c65979599 diff --git a/manifest.uuid b/manifest.uuid index a398ac8e87..e66f70d089 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b7281a1caa574870a071bea3e96b1d8210c28c17f9094449b3ce1a42b311e6a1 \ No newline at end of file +fd707001f0afb1cf32cfeeda3ec7b5622eb49ddedf8fec1a7aa4c8841c77c37a \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 5b50c99dfa..3c0e73967d 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2850,7 +2850,7 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ ** even if some external agent does a "chmod" to make the shared-memory ** writable by us, until sqlite3OsShmUnmap() has been called. ** This is a requirement on the VFS implementation. - */ + */ rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */ if( rc!=SQLITE_READONLY_CANTINIT ){ @@ -3531,23 +3531,32 @@ int sqlite3WalFindFrame( /* This routine is only be called from within a read transaction. */ assert( pWal->readLock!=WAL_LOCK_NONE ); - /* If this is a wal2 system, the client must have a partial-wal lock - ** on wal file iApp. Or if it is a wal system, iApp==0 must be true. */ - assert( bWal2==0 || iApp==1 - || pWal->readLock==WAL_LOCK_PART1 || pWal->readLock==WAL_LOCK_PART1_FULL2 - ); - assert( bWal2==0 || iApp==0 - || pWal->readLock==WAL_LOCK_PART2 || pWal->readLock==WAL_LOCK_PART2_FULL1 - ); - assert( bWal2 || iApp==0 ); + /* If this is a regular wal system, then iApp must be set to 0 (there is + ** only one wal file, after all). Or, if this is a wal2 system and the + ** write-lock is not held, the client must have a partial-wal lock on wal + ** file iApp. This is not always true if the write-lock is held and this + ** function is being called after WalLockForCommit() as part of committing + ** a CONCURRENT transaction. */ +#ifdef SQLITE_DEBUG + if( bWal2 ){ + if( pWal->writeLock==0 ){ + int l = pWal->readLock; + assert( iApp==1 || l==WAL_LOCK_PART1 || l==WAL_LOCK_PART1_FULL2 ); + assert( iApp==0 || l==WAL_LOCK_PART2 || l==WAL_LOCK_PART2_FULL1 ); + } + }else{ + assert( iApp==0 ); + } +#endif /* Return early if read-lock 0 is held. */ if( (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ + assert( !bWal2 ); *piRead = 0; return SQLITE_OK; } - /* Search the wal file that the client holds a partial lock on first */ + /* Search the wal file that the client holds a partial lock on first. */ rc = walSearchWal(pWal, iApp, pgno, &iRead); /* If the requested page was not found, no error has occured, and @@ -3556,6 +3565,10 @@ int sqlite3WalFindFrame( if( rc==SQLITE_OK && bWal2 && iRead==0 && ( pWal->readLock==WAL_LOCK_PART1_FULL2 || pWal->readLock==WAL_LOCK_PART2_FULL1 +#ifndef SQLITE_OMIT_CONCURRENT + || (pWal->readLock==WAL_LOCK_PART1 && iApp==1) + || (pWal->readLock==WAL_LOCK_PART2 && iApp==0) +#endif )){ rc = walSearchWal(pWal, !iApp, pgno, &iRead); } @@ -3784,67 +3797,104 @@ int sqlite3WalLockForCommit( ** was opened. */ rc = SQLITE_BUSY_SNAPSHOT; }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){ + int bWal2 = isWalMode2(pWal); int iHash; int iLastHash = walFramePage(head.mxFrame); - u32 iFirst = pWal->hdr.mxFrame+1; /* First wal frame to check */ - if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){ - assert( pWal->readLock==0 ); - iFirst = 1; - } - for(iHash=walFramePage(iFirst); iHash<=iLastHash; iHash++){ - WalHashLoc sLoc; + int nLoop = 1+(bWal2 && walidxGetFile(&head)!=walidxGetFile(&pWal->hdr)); + int iLoop; + + + assert( nLoop==1 || nLoop==2 ); + for(iLoop=0; iLoophdr.mxFrame (which will be + ** set to the size of the old, now overwritten, wal file). This + ** doesn't come up in wal2 mode, as in wal2 mode the client always + ** has a PART lock on one of the wal files, preventing it from being + ** checkpointed or overwritten. */ + iFirst = pWal->hdr.mxFrame+1; + if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){ + assert( pWal->readLock==0 ); + iFirst = 1; + } + mxFrame = head.mxFrame; + }else{ + int iA = walidxGetFile(&pWal->hdr); + if( iLoop==0 ){ + iFirst = walExternalEncode(iA, 1+walidxGetMxFrame(&pWal->hdr, iA)); + mxFrame = walExternalEncode(iA, walidxGetMxFrame(&head, iA)); + }else{ + iFirst = walExternalEncode(!iA, 1); + mxFrame = walExternalEncode(!iA, walidxGetMxFrame(&head, !iA)); + } + } + iLastHash = walFramePage(mxFrame); - rc = walHashGet(pWal, iHash, &sLoc); - if( rc==SQLITE_OK ){ - u32 i, iMin, iMax; - assert( head.mxFrame>=sLoc.iZero ); - iMin = (sLoc.iZero >= iFirst) ? 1 : (iFirst - sLoc.iZero); - iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; - if( iMax>(head.mxFrame-sLoc.iZero) ) iMax = (head.mxFrame-sLoc.iZero); - for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){ - PgHdr *pPg; - if( sLoc.aPgno[i]==1 ){ - /* Check that the schema cookie has not been modified. If - ** it has not, the commit can proceed. */ - u8 aNew[4]; - u8 *aOld = &((u8*)pPage1->pData)[40]; - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (sz&0xfe00) + ((sz&0x0001)<<16); - iOffset = walFrameOffset(i+sLoc.iZero, sz) + WAL_FRAME_HDRSIZE+40; - rc = sqlite3OsRead(pWal->apWalFd[0], aNew, sizeof(aNew), iOffset); - if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ + for(iHash=walFramePage(iFirst); iHash<=iLastHash; iHash += (1+bWal2)){ + WalHashLoc sLoc; + + rc = walHashGet(pWal, iHash, &sLoc); + if( rc==SQLITE_OK ){ + u32 i, iMin, iMax; + assert( mxFrame>=sLoc.iZero ); + iMin = (sLoc.iZero >= iFirst) ? 1 : (iFirst - sLoc.iZero); + iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; + if( iMax>(mxFrame-sLoc.iZero) ) iMax = (mxFrame-sLoc.iZero); + for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){ + PgHdr *pPg; + if( sLoc.aPgno[i]==1 ){ + /* Check that the schema cookie has not been modified. If + ** it has not, the commit can proceed. */ + u8 aNew[4]; + u8 *aOld = &((u8*)pPage1->pData)[40]; + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + iOffset = walFrameOffset(i+sLoc.iZero, sz)+WAL_FRAME_HDRSIZE+40; + rc = sqlite3OsRead(pWal->apWalFd[0], aNew,sizeof(aNew),iOffset); + if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ + rc = SQLITE_BUSY_SNAPSHOT; + } + }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i]) ){ + *piConflict = sLoc.aPgno[i]; rc = SQLITE_BUSY_SNAPSHOT; - } - }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i]) ){ - *piConflict = sLoc.aPgno[i]; - rc = SQLITE_BUSY_SNAPSHOT; - }else if( (pPg = sqlite3PagerLookup(pPager, sLoc.aPgno[i])) ){ - /* Page aPgno[i], which is present in the pager cache, has been - ** modified since the current CONCURRENT transaction was started. - ** However it was not read by the current transaction, so is not - ** a conflict. There are two possibilities: (a) the page was - ** allocated at the of the file by the current transaction or - ** (b) was present in the cache at the start of the transaction. - ** - ** For case (a), do nothing. This page will be moved within the - ** database file by the commit code to avoid the conflict. The - ** call to PagerUnref() is to release the reference grabbed by - ** the sqlite3PagerLookup() above. - ** - ** In case (b), drop the page from the cache - otherwise - ** following the snapshot upgrade the cache would be inconsistent - ** with the database as stored on disk. */ - if( sqlite3PagerIswriteable(pPg) ){ - sqlite3PagerUnref(pPg); - }else{ - sqlite3PcacheDrop(pPg); + }else if( (pPg = sqlite3PagerLookup(pPager, sLoc.aPgno[i])) ){ + /* Page aPgno[i], which is present in the pager cache, has been + ** modified since the current CONCURRENT transaction was + ** started. However it was not read by the current + ** transaction, so is not a conflict. There are two + ** possibilities: (a) the page was allocated at the of the file + ** by the current transaction or (b) was present in the cache + ** at the start of the transaction. + ** + ** For case (a), do nothing. This page will be moved within the + ** database file by the commit code to avoid the conflict. The + ** call to PagerUnref() is to release the reference grabbed by + ** the sqlite3PagerLookup() above. + ** + ** In case (b), drop the page from the cache - otherwise + ** following the snapshot upgrade the cache would be + ** inconsistent with the database as stored on disk. */ + if( sqlite3PagerIswriteable(pPg) ){ + sqlite3PagerUnref(pPg); + }else{ + sqlite3PcacheDrop(pPg); + } } } } + if( rc!=SQLITE_OK ) break; } - if( rc!=SQLITE_OK ) break; } } } @@ -3876,6 +3926,7 @@ int sqlite3WalUpgradeSnapshot(Wal *pWal){ ** any reads performed between now and committing the transaction will ** read from the old snapshot - not the one just upgraded to. */ if( pWal->readLock==0 && pWal->hdr.mxFrame!=walCkptInfo(pWal)->nBackfill ){ + assert( isWalMode2(pWal)==0 ); rc = walUpgradeReadlock(pWal); } return rc; @@ -3959,7 +4010,6 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ */ void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ int iWal = walidxGetFile(&pWal->hdr); - assert( pWal->writeLock ); assert( isWalMode2(pWal) || iWal==0 ); aWalData[0] = walidxGetMxFrame(&pWal->hdr, iWal); aWalData[1] = pWal->hdr.aFrameCksum[0]; diff --git a/test/wal2concurrent.test b/test/wal2concurrent.test new file mode 100644 index 0000000000..db0c8485b9 --- /dev/null +++ b/test/wal2concurrent.test @@ -0,0 +1,84 @@ +# 2015 July 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix wal2concurrent + +ifcapable !concurrent { + finish_test + return +} + + +#------------------------------------------------------------------------- +# Warm-body test. +# +sqlite3 db2 test.db +do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + CREATE TABLE t2(y); + PRAGMA journal_size_limit = 5000; + PRAGMA journal_mode = wal2; +} {5000 wal2} + +do_execsql_test 1.1 { + INSERT INTO t1 VALUES(1); + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(2); +} {} + +do_test 1.2 { + execsql { + PRAGMA journal_size_limit = 5000; + INSERT INTO t1 VALUES(3) + } db2 + catchsql { COMMIT } +} {1 {database is locked}} + +do_catchsql_test 1.3 { COMMIT } {1 {database is locked}} +do_catchsql_test 1.4 { ROLLBACK } {0 {}} + +do_test 1.5 { + list [file size test.db-wal] [file size test.db-wal2] +} {2128 0} + +do_execsql_test 1.6 { + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(2); +} {} + +do_test 1.7 { + execsql { INSERT INTO t2 VALUES(randomblob(4000)) } db2 + list [file size test.db-wal] [file size test.db-wal2] +} {7368 0} + +do_test 1.8 { + execsql { + INSERT INTO t2 VALUES(1); + INSERT INTO t1 VALUES(5); + } db2 + list [file size test.db-wal] [file size test.db-wal2] +} {7368 2128} + +do_catchsql_test 1.9 { COMMIT } {1 {database is locked}} +do_catchsql_test 1.10 { ROLLBACK } {0 {}} + +db close +sqlite3 db test.db +do_execsql_test 1.11 { SELECT * FROM t1 } {1 3 5} +do_execsql_test 1.12 { SELECT count(*) FROM t2 } {2} + +finish_test +