From: dan Date: Thu, 18 Nov 2010 19:28:01 +0000 (+0000) Subject: Fixes for SQLITE_BUSY handling in blocking checkpoint code. X-Git-Tag: version-3.7.6~174^2~3 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f2b8dd588d9899c9a4f3afd28ed64dce08d6705b;p=thirdparty%2Fsqlite.git Fixes for SQLITE_BUSY handling in blocking checkpoint code. FossilOrigin-Name: 4c663a4dcc77e00558edd94f58410605b95db33a --- diff --git a/manifest b/manifest index de24bf1209..4cb48f95f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\swith\slatest\strunk\sfix. -D 2010-11-18T16:59:24 +C Fixes\sfor\sSQLITE_BUSY\shandling\sin\sblocking\scheckpoint\scode. +D 2010-11-18T19:28:02 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -141,7 +141,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 8af9fcc75708d60b88636ccba38b4a7b3c155c3e -F src/main.c 91465f2658911ddb51be89e7b8ee01af8584308f +F src/main.c 5927ac9ca30c46df68148ad68227329d1be41a21 F src/malloc.c 3d7284cd9346ab6e3945535761e68c23c6cf40ef F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206 @@ -236,7 +236,7 @@ F src/vdbeblob.c e0ce3c54cc0c183af2ec67b63a289acf92251df4 F src/vdbemem.c 23723a12cd3ba7ab3099193094cbb2eb78956aa9 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c b297e8fa656ab5e66244ab15680d68db0adbec30 -F src/wal.c 8eca619a28a70a667c913e5927131250836377a2 +F src/wal.c 23facfd0f148ac72729fe28bbf973fe0458757b6 F src/wal.h 7a5fbb00114b7f2cd40c7e1003d4c41ce9d26840 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c fa22d45b2577c77146f2e894d58011d472d64103 @@ -824,11 +824,11 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test dea22218fd68c61fe206f53d508a0552894144b7 +F test/wal.test f060cae4b2164c4375109a8f803873187234661d F test/wal2.test 894d55dda774340fe7bebe239bed9b6130ff23d7 F test/wal3.test 55529a3fbf0a04670558dbf0b06f04a2f3508db4 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 -F test/wal5.test e5330471fc284d9ae62a2a18c9dfd10b6605d8b6 +F test/wal5.test 79963972107e4cabda4c4b44a64e89c3e9af463d F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0 @@ -888,7 +888,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P e376480f0855c598c91529abacbd73e31d9f4713 0a95589f2166f9ce420e647b73e8c797fe8f4833 -R 87db3bd5bf4a3735ef4309e5d5955901 +P a8910e89dee326d7788b29e77820eb1e114739ca +R a15fbbfd076e96cb91858a5a2944b7af U dan -Z 9fccaf5d4045282a697f1275247ad8b5 +Z 77692281a1477b941972542152e8834d diff --git a/manifest.uuid b/manifest.uuid index b70e7c96b6..81ad5c3317 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8910e89dee326d7788b29e77820eb1e114739ca \ No newline at end of file +4c663a4dcc77e00558edd94f58410605b95db33a \ No newline at end of file diff --git a/src/main.c b/src/main.c index 4333a5873b..0870a6bcee 100644 --- a/src/main.c +++ b/src/main.c @@ -1413,16 +1413,23 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; /* Return code */ int i; /* Used to iterate through attached dbs */ + int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ assert( sqlite3_mutex_held(db->mutex) ); for(i=0; inDb && rc==SQLITE_OK; i++){ if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); + pnLog = 0; + pnCkpt = 0; + if( rc==SQLITE_BUSY ){ + bBusy = 1; + rc = SQLITE_OK; + } } } - return rc; + return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; } #endif /* SQLITE_OMIT_WAL */ diff --git a/src/wal.c b/src/wal.c index 0619822434..2cb0e415ee 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1543,6 +1543,14 @@ static int walBusyLock( return rc; } +/* +** The cache of the wal-index header must be valid to call this function. +** Return the page-size in bytes used by the database. +*/ +static int walPagesize(Wal *pWal){ + return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); +} + /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -1577,10 +1585,9 @@ static int walBusyLock( static int walCheckpoint( Wal *pWal, /* Wal connection */ int eMode, /* One of PASSIVE, FULL or RESTART */ - int (*xBusy)(void*), /* Function to call when busy */ + int (*xBusyCall)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ - int nBuf, /* Size of zBuf in bytes */ u8 *zBuf, /* Temporary buffer to use */ int *pnCkpt /* Total frames checkpointed */ ){ @@ -1593,11 +1600,11 @@ static int walCheckpoint( u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ + int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ - szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + szPage = walPagesize(pWal); testcase( szPage<=32768 ); testcase( szPage>=65536 ); - if( pWal->hdr.mxFrame==0 ) return SQLITE_OK; /* Allocate the iterator */ rc = walIteratorInit(pWal, &pIter); @@ -1606,40 +1613,33 @@ static int walCheckpoint( } assert( pIter ); - /*** TODO: Move this test out to the caller. Make it an assert() here ***/ - if( szPage!=nBuf ){ - rc = SQLITE_CORRUPT_BKPT; - goto walcheckpoint_out; - } - pInfo = walCkptInfo(pWal); mxPage = pWal->hdr.nPage; if( pnCkpt ) *pnCkpt = pInfo->nBackfill; + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall; /* Compute in mxSafeFrame the index of the last frame of the WAL that is ** safe to write into the database. Frames beyond mxSafeFrame might ** overwrite database pages that are in use by active readers and thus ** cannot be backfilled from the WAL. */ - do { - mxSafeFrame = pWal->hdr.mxFrame; - for(i=1; iaReadMark[i]; - if( mxSafeFrame>=y ){ - assert( y<=pWal->hdr.mxFrame ); - rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1); - if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = READMARK_NOT_USED; - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - }else if( rc==SQLITE_BUSY ){ - mxSafeFrame = y; - }else{ - goto walcheckpoint_out; - } + mxSafeFrame = pWal->hdr.mxFrame; + for(i=1; iaReadMark[i]; + if( mxSafeFrame>y ){ + assert( y<=pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ + pInfo->aReadMark[i] = READMARK_NOT_USED; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc==SQLITE_BUSY ){ + mxSafeFrame = y; + xBusy = 0; + }else{ + goto walcheckpoint_out; } } - }while( eMode!=SQLITE_CHECKPOINT_PASSIVE - && xBusy && mxSafeFramehdr.mxFrame && xBusy(pBusyArg) ); + } if( pInfo->nBackfillhdr.mxFrame==mxSafeFrame - ){ + /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal + ** file has been copied into the database file, then block until all + ** readers have finished using the wal file. This ensures that the next + ** process to write to the database restarts the wal file. + */ + if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); - rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); - if( rc==SQLITE_OK ){ - walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + if( pInfo->nBackfillhdr.mxFrame ){ + rc = SQLITE_BUSY; + }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ + assert( mxSafeFrame==pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc==SQLITE_OK ){ + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + } } } @@ -2680,6 +2687,7 @@ int sqlite3WalCheckpoint( ){ int rc; /* Return code */ int isChanged = 0; /* True if a new wal-index header is loaded */ + int eMode2 = eMode; /* Mode to pass to walCheckpoint() */ assert( pWal->ckptLock==0 ); assert( pWal->writeLock==0 ); @@ -2697,21 +2705,37 @@ int sqlite3WalCheckpoint( /* If this is a blocking-checkpoint, then obtain the write-lock as well ** to prevent any writers from running while the checkpoint is underway. ** This has to be done before the call to walIndexReadHdr() below. + ** + ** If the writer lock cannot be obtained, then a passive checkpoint is + ** run instead. Since the checkpointer is not holding the writer lock, + ** there is no point in blocking waiting for any readers. Assuming no + ** other error occurs, this function will return SQLITE_BUSY to the caller. */ if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); - if( rc==SQLITE_OK ) pWal->writeLock = 1; + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + }else if( rc==SQLITE_BUSY ){ + eMode2 = SQLITE_CHECKPOINT_PASSIVE; + rc = SQLITE_OK; + } } - /* Copy data from the log to the database file. */ + /* Read the wal-index header. */ if( rc==SQLITE_OK ){ rc = walIndexReadHdr(pWal, &isChanged); } - if( rc==SQLITE_OK ){ - if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; - rc = walCheckpoint( - pWal, eMode, xBusy, pBusyArg, sync_flags, nBuf, zBuf, pnCkpt); + + /* Copy data from the log to the database file. */ + if( rc==SQLITE_OK && pWal->hdr.mxFrame ){ + if( walPagesize(pWal)!=nBuf ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; + rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags,zBuf,pnCkpt); + } } + if( isChanged ){ /* If a new wal-index header was loaded before the checkpoint was ** performed, then the pager-cache associated with pWal is now @@ -2727,7 +2751,7 @@ int sqlite3WalCheckpoint( walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); pWal->ckptLock = 0; WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); - return rc; + return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc); } /* Return the value to pass to a sqlite3_wal_hook callback, the diff --git a/test/wal.test b/test/wal.test index 70dabda6b3..339661e2ee 100644 --- a/test/wal.test +++ b/test/wal.test @@ -845,6 +845,7 @@ do_test wal-13.1.2 { sqlite3 db test.db execsql { SELECT * FROM t2 } } {B 2} +breakpoint do_test wal-13.1.3 { db close file exists test.db-wal @@ -1029,7 +1030,7 @@ catch { db close } foreach {tn ckpt_cmd ckpt_res ckpt_main ckpt_aux} { 1 {sqlite3_wal_checkpoint db} SQLITE_OK 1 1 2 {sqlite3_wal_checkpoint db ""} SQLITE_OK 1 1 - 3 {db eval "PRAGMA wal_checkpoint"} {0 16 16} 1 1 + 3 {db eval "PRAGMA wal_checkpoint"} {0 10 10} 1 1 4 {sqlite3_wal_checkpoint db main} SQLITE_OK 1 0 5 {sqlite3_wal_checkpoint db aux} SQLITE_OK 0 1 diff --git a/test/wal5.test b/test/wal5.test index 49cd839761..ffcf044866 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -21,10 +21,12 @@ ifcapable !wal {finish_test ; return } set testprefix wal5 +proc db_page_count {{file test.db}} { expr [file size $file] / 1024 } +proc wal_page_count {{file test.db}} { wal_frame_count ${file}-wal 1024 } + + do_multiclient_test tn { - proc db_page_count {} { expr [file size test.db] / 1024 } - proc wal_page_count {} { wal_frame_count test.db-wal 1024 } set ::nBusyHandler 0 set ::busy_handler_script "" @@ -115,4 +117,87 @@ do_multiclient_test tn { do_test 1.$tn.12 { set ::db_file_size } 10 } + +#------------------------------------------------------------------------- +# This block of tests explores checkpoint operations on more than one +# database file. +# +proc setup_and_attach_aux {} { + sql1 { ATTACH 'test.db2' AS aux } + sql2 { ATTACH 'test.db2' AS aux } + sql3 { ATTACH 'test.db2' AS aux } + sql1 { + PRAGMA main.page_size=1024; PRAGMA main.journal_mode=WAL; + PRAGMA aux.page_size=1024; PRAGMA aux.journal_mode=WAL; + } +} + +proc file_page_counts {} { + list [db_page_count test.db ] \ + [wal_page_count test.db ] \ + [db_page_count test.db2] \ + [wal_page_count test.db2] +} + +do_multiclient_test tn { + setup_and_attach_aux + do_test 2.1.$tn.1 { + sql1 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + CREATE TABLE aux.t2(a, b); + INSERT INTO t2 VALUES(1, 2); + } + } {} + + do_test 2.2.$tn.2 { file_page_counts } {1 5 1 5} + do_test 2.1.$tn.3 { sql1 { PRAGMA wal_checkpoint } } {0 5 5} + do_test 2.1.$tn.4 { file_page_counts } {2 5 2 5} +} + +do_multiclient_test tn { + + setup_and_attach_aux + + do_test 2.2.$tn.1 { + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + CREATE TABLE aux.t2(a, b); + INSERT INTO t2 VALUES(1, 2); + INSERT INTO t2 VALUES(3, 4); + } + } {} + + do_test 2.2.$tn.2 { file_page_counts } {1 5 1 7} + do_test 2.2.$tn.3 { sql2 { BEGIN; SELECT * FROM t1 } } {1 2} + do_test 2.2.$tn.4 { sql1 { PRAGMA wal_checkpoint = RESTART } } {1 5 5} + do_test 2.2.$tn.5 { file_page_counts } {2 5 2 7} +} + +do_multiclient_test tn { + + setup_and_attach_aux + + do_test 2.3.$tn.1 { + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + CREATE TABLE aux.t2(a, b); + INSERT INTO t2 VALUES(1, 2); + } + } {} + + do_test 2.3.$tn.2 { file_page_counts } {1 5 1 5} + do_test 2.3.$tn.3 { sql2 { BEGIN; SELECT * FROM t1 } } {1 2} + do_test 2.3.$tn.4 { sql1 { INSERT INTO t1 VALUES(3, 4) } } {} + do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {} + do_test 2.3.$tn.6 { file_page_counts } {1 7 1 7} + + do_test 2.3.$tn.7 { sql1 { PRAGMA wal_checkpoint = FULL } } {1 7 5} + do_test 2.3.$tn.8 { file_page_counts } {1 7 2 7} +} + + + finish_test