From: drh Date: Tue, 17 Jul 2012 17:46:21 +0000 (+0000) Subject: Cherrypick [8c9ee1d78f] and [e416359633] from trunk: X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3314d12e03da62821ebe025d7aa0690cd155964d;p=thirdparty%2Fsqlite.git Cherrypick [8c9ee1d78f] and [e416359633] from trunk: Ensure that there is always at least one aReadMark slot usable by an unprivileged reader while a checkpoint is running. Also, if one or more transactions are recovered from a log file, initialize one of the aReadMark slots to contain mxFrame as part of the recovery process. FossilOrigin-Name: 65035912264e3acbced5a3e16793327f0a2f17bb --- diff --git a/manifest b/manifest index 108794aa5c..8a72fe106a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\svtab1\stest\sscript\sfix\sand\sthe\sFTS3\smemory\sleak\sfix\sfrom\strunk\ninto\sthe\sapple-osx\sbranch. -D 2012-06-08T14:11:30.151 +C Cherrypick\s[8c9ee1d78f]\sand\s[e416359633]\sfrom\strunk:\nEnsure\sthat\sthere\sis\salways\sat\sleast\sone\saReadMark\sslot\susable\sby\san\sunprivileged\sreader\swhile\sa\scheckpoint\sis\srunning.\sAlso,\sif\sone\sor\smore\stransactions\sare\srecovered\sfrom\sa\slog\sfile,\sinitialize\sone\sof\sthe\saReadMark\sslots\sto\scontain\smxFrame\sas\spart\sof\sthe\srecovery\sprocess. +D 2012-07-17T17:46:21.303 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f4e42073f1092a09a9cbd42aecf2f33790208cb5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -233,7 +233,7 @@ F src/test_superlock.c 12e2bc484c6c2ba837327d37f2e6a6fd9d1464f8 F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 -F src/test_vfs.c 9d934e111021d56c629efc73a796648c9519ad12 +F src/test_vfs.c da6d0d982b11756c94c1760196355d33d03ff745 F src/test_vfstrace.c 6b28adb2a0e8ecd0f2e3581482e1f658b11b4067 F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 @@ -253,7 +253,7 @@ F src/vdbemem.c cb55e84b8e2c15704968ee05f0fae25883299b74 F src/vdbesort.c b25814d385895544ebc8118245c8311ded7f81c9 F src/vdbetrace.c 6700008a6a05e6e39531ed252c32557ceb962b91 F src/vtab.c bb8ea3a26608bb1357538a5d2fc72beba6638998 -F src/wal.c 1ff2ebb47ba29610abd768bf74cecd9c33049ebf +F src/wal.c c981d242bbb28ae3fc4889528a5621cc564d3ed2 F src/wal.h ce626f1f9000caf09a99a6634a8d794686f92e1b F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 24c7494d8875ead994b4dfe5461340c27fd424ca @@ -931,8 +931,8 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 F test/wal.test 5759631b0e610d616c33cc21124b3395f39cd0c7 -F test/wal2.test 0f616db53cbf93bbd7a8ef8b33f048a2b43d2f0a -F test/wal3.test ae86a92d41d81730278fca0b71368f3e78e9c64a +F test/wal2.test 583ce66003086387a7301c0b62e549026f2a50ed +F test/wal3.test de822707fbc8e9d056edc895adfb8abcccee4590 F test/wal4.test 5755887f321baa4c55de0b91066fa7d0cafcac9d F test/wal5.test 187ae92cc9ba1ec6803681b9025cad89af1a8c69 F test/wal6.test c561d1e44c89f9cb458a7b03003ed4baac08ba07 @@ -950,7 +950,7 @@ F test/walhook.test 5d2bdb04fd3e220e2f96e6b566d57e00020bdaec F test/walmode.test aa45339b4afa435dde5d88e71a95459cc221a3f4 F test/walnoshm.test 559b878f3aab838971d820329ca35f1caa7b038e F test/walpersist.test abd956d66e2f36d2d9d05d3a969f48be6d2ddbec -F test/walro.test 180321fa4e7cf7e98b5df232763c3e0781673a41 +F test/walro.test 78f389d11e4a32e329feb9b63859ebc3081bbea0 F test/walshared.test 04590b10c677f75318701818c50bc0dda5da64ab F test/walslow.test 658066419a92d3bd85be71a11ce477af4ffe9153 F test/walthread.test c3aaf9ef7ad21ae79c2345425bfddb39cdac954f @@ -1010,7 +1010,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 9d1b8515625ce7fbde170180126bd1a273bb4994 025227be5495f950c466dfabac140cba69e498be -R 73fc9d898a1621cfcd2d0602a720ceb8 +P 892d8779cc3c4037d6e21c04c39796284d811157 +R ab4b327459cf45e2530a4cdced0d9405 U drh -Z 70bd5ce96a21e994af0322978beb4b1b +Z 7a54c87c3e8e79189a731257e2592a5b diff --git a/manifest.uuid b/manifest.uuid index 36f8bad2e1..e07a5f6bdf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -892d8779cc3c4037d6e21c04c39796284d811157 \ No newline at end of file +65035912264e3acbced5a3e16793327f0a2f17bb \ No newline at end of file diff --git a/src/test_vfs.c b/src/test_vfs.c index d1c34a38e4..fd2aa9fb07 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -81,6 +81,7 @@ struct Testvfs { Tcl_Obj *pScript; /* Script to execute */ TestvfsBuffer *pBuffer; /* List of shared buffers */ int isNoshm; + int isFullshm; int mask; /* Mask controlling [script] and [ioerr] */ @@ -760,6 +761,7 @@ static int tvfsShmOpen(sqlite3_file *pFile){ pFd = tvfsGetFd(pFile); p = (Testvfs *)pFd->pVfs->pAppData; + assert( 0==p->isFullshm ); assert( pFd->pShmId && pFd->pShm==0 && pFd->pNext==0 ); /* Evaluate the Tcl script: @@ -820,6 +822,10 @@ static int tvfsShmMap( TestvfsFd *pFd = tvfsGetFd(pFile); Testvfs *p = (Testvfs *)(pFd->pVfs->pAppData); + if( p->isFullshm ){ + return sqlite3OsShmMap(pFd->pReal, iPage, pgsz, isWrite, pp); + } + if( 0==pFd->pShm ){ rc = tvfsShmOpen(pFile); if( rc!=SQLITE_OK ){ @@ -864,6 +870,10 @@ static int tvfsShmLock( int nLock; char zLock[80]; + if( p->isFullshm ){ + return sqlite3OsShmLock(pFd->pReal, ofst, n, flags); + } + if( p->pScript && p->mask&TESTVFS_SHMLOCK_MASK ){ sqlite3_snprintf(sizeof(zLock), zLock, "%d %d", ofst, n); nLock = (int)strlen(zLock); @@ -919,6 +929,11 @@ static void tvfsShmBarrier(sqlite3_file *pFile){ TestvfsFd *pFd = tvfsGetFd(pFile); Testvfs *p = (Testvfs *)(pFd->pVfs->pAppData); + if( p->isFullshm ){ + sqlite3OsShmBarrier(pFd->pReal); + return; + } + if( p->pScript && p->mask&TESTVFS_SHMBARRIER_MASK ){ tvfsExecTcl(p, "xShmBarrier", Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0 @@ -936,6 +951,10 @@ static int tvfsShmUnmap( TestvfsBuffer *pBuffer = pFd->pShm; TestvfsFd **ppFd; + if( p->isFullshm ){ + return sqlite3OsShmUnmap(pFd->pReal, deleteFlag); + } + if( !pBuffer ) return SQLITE_OK; assert( pFd->pShmId && pFd->pShm ); @@ -1350,6 +1369,7 @@ static int testvfs_cmd( int i; int isNoshm = 0; /* True if -noshm is passed */ + int isFullshm = 0; /* True if -fullshm is passed */ int isDefault = 0; /* True if -default is passed */ int szOsFile = 0; /* Value passed to -szosfile */ int mxPathname = -1; /* Value passed to -mxpathname */ @@ -1365,6 +1385,7 @@ static int testvfs_cmd( if( Tcl_GetBooleanFromObj(interp, objv[i+1], &isNoshm) ){ return TCL_ERROR; } + if( isNoshm ) isFullshm = 0; } else if( nSwitch>2 && 0==strncmp("-default", zSwitch, nSwitch) ){ if( Tcl_GetBooleanFromObj(interp, objv[i+1], &isDefault) ){ @@ -1386,6 +1407,12 @@ static int testvfs_cmd( return TCL_ERROR; } } + else if( nSwitch>2 && 0==strncmp("-fullshm", zSwitch, nSwitch) ){ + if( Tcl_GetBooleanFromObj(interp, objv[i+1], &isFullshm) ){ + return TCL_ERROR; + } + if( isFullshm ) isNoshm = 0; + } else{ goto bad_args; } @@ -1427,6 +1454,7 @@ static int testvfs_cmd( pVfs->szOsFile = szOsFile; p->pVfs = pVfs; p->isNoshm = isNoshm; + p->isFullshm = isFullshm; p->mask = TESTVFS_ALL_MASK; sqlite3_vfs_register(pVfs, isDefault); diff --git a/src/wal.c b/src/wal.c index 808075d3fc..480b79fe4e 100644 --- a/src/wal.c +++ b/src/wal.c @@ -142,8 +142,9 @@ ** byte order of the host computer. ** ** The purpose of the wal-index is to answer this question quickly: Given -** a page number P, return the index of the last frame for page P in the WAL, -** or return NULL if there are no frames for page P in the WAL. +** a page number P and a maximum frame index M, return the index of the +** last frame in the wal before frame M for page P in the WAL, or return +** NULL if there are no frames for page P in the WAL prior to M. ** ** The wal-index consists of a header region, followed by an one or ** more index blocks. @@ -1198,6 +1199,7 @@ finished: pInfo->nBackfill = 0; pInfo->aReadMark[0] = 0; for(i=1; iaReadMark[i] = READMARK_NOT_USED; + if( pWal->hdr.mxFrame ) pInfo->aReadMark[1] = pWal->hdr.mxFrame; /* If more than one frame was recovered from the log file, report an ** event via sqlite3_log(). This is to help with identifying performance @@ -1703,7 +1705,7 @@ static int walCheckpoint( assert( y<=pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = READMARK_NOT_USED; + pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); }else if( rc==SQLITE_BUSY ){ mxSafeFrame = y; @@ -2628,7 +2630,8 @@ static int walRestartLog(Wal *pWal){ aSalt[1] = salt1; walIndexWriteHdr(pWal); pInfo->nBackfill = 0; - for(i=1; iaReadMark[i] = READMARK_NOT_USED; + pInfo->aReadMark[1] = 0; + for(i=2; iaReadMark[i] = READMARK_NOT_USED; assert( pInfo->aReadMark[0]==0 ); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); }else if( rc!=SQLITE_BUSY ){ diff --git a/test/wal2.test b/test/wal2.test index 59ebec798d..6a05c757b5 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -134,9 +134,11 @@ set RECOVER [list \ {1 7 unlock exclusive} {0 1 unlock exclusive} \ ] set READ [list \ - {4 1 lock exclusive} {4 1 unlock exclusive} \ {4 1 lock shared} {4 1 unlock shared} \ ] +set INITSLOT [list \ + {4 1 lock exclusive} {4 1 unlock exclusive} \ +] foreach {tn iInsert res wal_index_hdr_mod wal_locks} " 2 5 {5 15} 0 {$RECOVER $READ} @@ -149,7 +151,7 @@ foreach {tn iInsert res wal_index_hdr_mod wal_locks} " 9 12 {12 78} 7 {$RECOVER $READ} 10 13 {13 91} 8 {$RECOVER $READ} 11 14 {14 105} 9 {$RECOVER $READ} - 12 15 {15 120} -1 {$READ} + 12 15 {15 120} -1 {$INITSLOT $READ} " { do_test wal2-1.$tn.1 { diff --git a/test/wal3.test b/test/wal3.test index 229be5f7e9..b530f5f5f0 100644 --- a/test/wal3.test +++ b/test/wal3.test @@ -664,7 +664,7 @@ T filter xShmLock T script lock_callback proc lock_callback {method file handle spec} { - if {$spec == "4 1 unlock exclusive"} { + if {$spec == "1 7 unlock exclusive"} { T filter {} set ::r [catchsql { SELECT * FROM b } db2] } diff --git a/test/walro.test b/test/walro.test index 8362e0e4c4..27d73e176b 100644 --- a/test/walro.test +++ b/test/walro.test @@ -175,8 +175,133 @@ do_multiclient_test tn { do_test 1.3.2.4 { code1 { sqlite3_extended_errcode db } } {SQLITE_READONLY_RECOVERY} + + #----------------------------------------------------------------------- + # Test cases 1.4.* check that checkpoints and log wraps don't prevent + # read-only connections from reading the database. + do_test 1.4.1 { + code1 { db close } + forcedelete test.db-shm + file exists test.db-shm + } {0} + + # Open one read-only and one read-write connection. Write some data + # and then run a checkpoint using the read-write connection. Then + # check the read-only connection can still read. + do_test 1.4.2 { + code1 { sqlite3 db file:test.db?readonly_shm=1 } + code2 { sqlite3 db2 test.db } + csql2 { + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + PRAGMA wal_checkpoint; + } + } {0 {0 3 3}} + do_test 1.4.3 { + csql1 { SELECT * FROM t1 } + } {0 {a b c d e f g h i j k l 1 2 3 4 5 6}} + + # Using the read-write connection, open a transaction and write lots + # of data - causing a cache spill and a log wrap. Then check that the + # read-only connection can still read the database. + do_test 1.4.4.1 { + csql2 { + PRAGMA cache_size = 10; + BEGIN; + CREATE TABLE t2(x, y); + INSERT INTO t2 VALUES('abc', 'xyz'); + INSERT INTO t2 SELECT x||y, y||x FROM t2; + INSERT INTO t2 SELECT x||y, y||x FROM t2; + INSERT INTO t2 SELECT x||y, y||x FROM t2; + INSERT INTO t2 SELECT x||y, y||x FROM t2; + INSERT INTO t2 SELECT x||y, y||x FROM t2; + INSERT INTO t2 SELECT x||y, y||x FROM t2; + INSERT INTO t2 SELECT x||y, y||x FROM t2; + INSERT INTO t2 SELECT x||y, y||x FROM t2; + INSERT INTO t2 SELECT x||y, y||x FROM t2; + } + file size test.db-wal + } {147800} + do_test 1.4.4.2 { + csql1 { SELECT * FROM t1 } + } {0 {a b c d e f g h i j k l 1 2 3 4 5 6}} + do_test 1.4.4.3 { + csql2 COMMIT + csql1 { SELECT count(*) FROM t2 } + } {0 512} + do_test 1.4.5 { + code2 { db2 close } + code1 { db close } + } {} +} + +forcedelete test.db + +#----------------------------------------------------------------------- +# Test cases 2.* check that a read-only connection may read the +# database file while a checkpoint operation is ongoing. +# +do_multiclient_test tn { + # Do not run tests with the connections in the same process. + # + if {$tn==2} continue + + # Close all connections and delete the database. + # + code1 { db close } + code2 { db2 close } + code3 { db3 close } + forcedelete test.db + forcedelete walro + + foreach c {code1 code2 code3} { + $c { + sqlite3_shutdown + sqlite3_config_uri 1 + } + } + + proc tv_hook {x file args} { + if {[file tail $file]=="test.db-wal"} { + do_test 2.1.2 { + code2 { sqlite3 db2 file:test.db?readonly_shm=1 } + csql2 { SELECT count(*) FROM t2 } + } {0 4} + do_test 2.1.3 { + code2 { db2 close } + } {} + } + } + + do_test 2.1.1 { + testvfs tv -default 1 -fullshm 1 + tv script tv_hook + tv filter {} + code1 { sqlite3 db test.db } + csql1 { + PRAGMA journal_mode = WAL; + BEGIN; + CREATE TABLE t2(x, y); + INSERT INTO t2 VALUES('abc', 'xyz'); + INSERT INTO t2 SELECT x||y, y||x FROM t2; + INSERT INTO t2 SELECT x||y, y||x FROM t2; + COMMIT; + } + } {0 wal} + + tv filter xSync + set res [csql1 { PRAGMA wal_checkpoint }] + do_test 2.1.4 { set res } {0 {0 2 2}} + + do_test 2.1.5 { + code1 { db close } + code1 { tv delete } + } {} } forcedelete $shmpath finish_test + +