From: drh Date: Wed, 19 May 2010 01:53:53 +0000 (+0000) Subject: Add a large comment to wal.c describing the WAL and wal-index file formats. X-Git-Tag: version-3.7.2~364 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=027a128a78f64a254ede87e3c7ac415598555268;p=thirdparty%2Fsqlite.git Add a large comment to wal.c describing the WAL and wal-index file formats. FossilOrigin-Name: a71a22b52f4570e934063553a81b39268127dc44 --- diff --git a/manifest b/manifest index 8b64e2736c..762dea9f6b 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Update\sthe\swal-index\shash\sformat\sso\sthat\shash-table\sspace\sis\sreused\sfollowing\na\srollback,\sthus\spreventing\shash\stable\soverflows.\s\sAdd\sassert()s\sto\sverify\nthat\shash\stables\sdo\snot\soverfill.\s\sFurther\srefactoring\sof\sthe\swal-index\scode. -D 2010-05-18T23:29:53 +C Add\sa\slarge\scomment\sto\swal.c\sdescribing\sthe\sWAL\sand\swal-index\sfile\sformats. +D 2010-05-19T01:53:54 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -227,7 +227,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 6ef6731346daf2461462ea20d5fc44682feb1a28 +F src/wal.c 59463e08693a9fddf516197e0053ebcd58f74c66 F src/wal.h 434f76f51225bb614e43ccb6bd2341541ba6a06e F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 75fee9e255b62f773fcadd1d1f25b6f63ac7a356 @@ -816,14 +816,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P b5b60fdcc5dcf41f2c79912075ac241f7ce220d6 -R 4929b4583dfb4e1eb01782532f6827c1 +P ada9a8c7b69c5dd2d66bbf62b61181651e6d2142 +R 18fadfda9fbc33cddb219c42d944f3cc U drh -Z d3a6efc94e4a817e6e627477c12bae31 +Z da5a3c11958bee0d8152271c179253e5 -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFL8yL1oxKgR168RlERAvCcAJ9c+PHCm9rZ4hPCfVE6HbCjS/YEFgCdF7L6 -bHpr3zEADy01V3PS/VC1PCE= -=u4O9 +iD8DBQFL80S1oxKgR168RlERAoLoAJoDDynU57b4fyCkRG5FEJ7stDA9hQCcDNGs +5upq5UH5aI43a8UF2LKqxus= +=50kN -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 1ddb537f42..a5efb95401 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ada9a8c7b69c5dd2d66bbf62b61181651e6d2142 \ No newline at end of file +a71a22b52f4570e934063553a81b39268127dc44 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index cdac2fb0f7..3de43ddc6e 100644 --- a/src/wal.c +++ b/src/wal.c @@ -10,13 +10,13 @@ ** ************************************************************************* ** -** This file contains the implementation of a write-ahead log file used in -** "journal_mode=wal" mode. +** This file contains the implementation of a write-ahead log (WAL) used in +** "journal_mode=WAL" mode. ** ** WRITE-AHEAD LOG (WAL) FILE FORMAT ** ** A wal file consists of a header followed by zero or more "frames". -** Each frame records the revised content of a single page within the +** Each frame records the revised content of a single page from the ** database file. All changes to the database are recorded by writing ** frames into the WAL. Transactions commit when a frame is written that ** contains a commit marker. A single WAL can and usually does record @@ -25,7 +25,7 @@ ** "checkpoint". ** ** A single WAL file can be used multiple times. In other words, the -** WAL can fill up with frames and then be checkpointed. Then new +** WAL can fill up with frames and then be checkpointed and then new ** frames can overwrite the old ones. A WAL always grows from beginning ** toward the end. Checksums and counters attached to each frame are ** used to determine which frames within the WAL are valid and which @@ -39,7 +39,7 @@ ** 8: Randomly selected salt value 2. ** ** Immediately following the header are zero or more frames. Each -** frame itself consists of a 16-byte header followed by a bytes +** frame consists of a 16-byte header followed by a bytes ** of page data. The header is broken into 4 big-endian 32-bit unsigned ** integer values, as follows: ** @@ -60,10 +60,10 @@ ** ** The reader algorithm in the previous paragraph works correctly, but ** because frames for page P can appear anywhere within the WAL, the -** reader has to scan the either WAL looking for page P frames. If the +** reader has to scan the entire WAL looking for page P frames. If the ** WAL is large (multiple megabytes is typical) that scan can be slow, -** and read performanc suffers. To overcome this problem, a separate -** datastructure called the wal-index is maintained to expedite the +** and read performance suffers. To overcome this problem, a separate +** data structure called the wal-index is maintained to expedite the ** search for frames of a particular page. ** ** WAL-INDEX FORMAT @@ -90,7 +90,84 @@ ** The wal-index consists of a header region, followed by an one or ** more index blocks. ** -** To be completed.... +** The wal-index header contains the total number of frames within the WAL +** in the the mxFrame field. Each index block contains information on +** HASHTABLE_NPAGE frames. Each index block contains two sections, a +** mapping which is a database page number for each frame, and a hash +** table used to look up frames by page number. The mapping section is +** an array of HASHTABLE_NPAGE 32-bit page numbers. The first entry on the +** array is the page number for the first frame; the second entry is the +** page number for the second frame; and so forth. The last index block +** holds a total of (mxFrame%HASHTABLE_NPAGE) page numbers. All index +** blocks other than the last are completely full with HASHTABLE_NPAGE +** page numbers. All index blocks are the same size; the mapping section +** of the last index block merely contains unused entries if mxFrame is +** not an even multiple of HASHTABLE_NPAGE. +** +** Even without using the hash table, the last frame for page P +** can be found by scanning the mapping sections of each index block +** starting with the last index block and moving toward the first, and +** within each index block, starting at the end and moving toward the +** beginning. The first entry that equals P corresponds to the frame +** holding the content for that page. +** +** The hash table consists of HASHTABLE_NSLOT 16-bit unsigned integers. +** HASHTABLE_NSLOT = 2*HASHTABLE_NPAGE, and there is one entry in the +** hash table for each page number in the mapping section, so the hash +** table is never more than half full. The expected number of collisions +** prior to finding a match is 1. Each entry of the hash table is an +** 1-based index of an entry in the mapping section of the same +** index block. Let K be the 1-based index of the largest entry in +** the mapping section. (For index blocks other than the last, K will +** always be exactly HASHTABLE_NPAGE (4096) and for the last index block +** K will be (mxFrame%HASHTABLE_NPAGE).) Unused slots of the hash table +** contain a value greater than K. Note that no hash table slot ever +** contains a zero value. +** +** To look for page P in the hash table, first compute a hash iKey on +** P as follows: +** +** iKey = (P * 383) % HASHTABLE_NSLOT +** +** Then start scanning entries of the hash table, starting with iKey +** (wrapping around to the beginning when the end of the hash table is +** reached) until an unused hash slot is found. Let the first unused slot +** be at index iUnused. (iUnused might be less than iKey if there was +** wrap-around.) Because the hash table is never more than half full, +** the search is guaranteed to eventually hit an unused entry. Let +** iMax be the value between iKey and iUnused, closest to iUnused, +** where aHash[iMax]==P. If there is no iMax entry (if there exists +** no hash slot such that aHash[i]==p) then page P is not in the +** current index block. Otherwise the iMax-th mapping entry of the +** current index block corresponds to the last entry that references +** page P. +** +** A hash search begins with the last index block and moves toward the +** first index block, looking for entries corresponding to page P. On +** average, only two or three slots in each index block need to be +** examined in order to either find the last entry for page P, or to +** establish that no such entry exists in the block. Each index block +** holds over 4000 entries. So two or three index blocks are sufficient +** to cover a typical 10 megabyte WAL file, assuming 1K pages. 8 or 10 +** comparisons (on average) suffice to either locate a frame in the +** WAL or to establish that the frame does not exist in the WAL. This +** is much faster than scanning the entire 10MB WAL. +** +** Note that entries are added in order of increasing K. Hence, one +** reader might be using some value K0 and a second reader that started +** at a later time (after additional transactions were added to the WAL +** and to the wal-index) might be using a different value K1, where K1>K0. +** Both readers can use the same hash table and mapping section to get +** the correct result. There may be entries in the hash table with +** K>K0 but to the first reader, those entries will appear to be unused +** slots in the hash table and so the first reader will get an answer as +** if no values greater than K0 had ever been inserted into the hash table +** in the first place - which is what reader one wants. Meanwhile, the +** second reader using K1 will see additional values that were inserted +** later, which is exactly what reader two wants. +** +** When a rollback occurs, the value of K is decreased. This has the +** effect of automatically removing entries from the hash table. */ #ifndef SQLITE_OMIT_WAL @@ -112,7 +189,7 @@ typedef struct WalIterator WalIterator; struct WalIndexHdr { u32 iChange; /* Counter incremented each transaction */ u32 pgsz; /* Database page size in bytes */ - u32 iLastPg; /* Index of last valid frame in the WAL */ + u32 mxFrame; /* Index of last valid frame in the WAL */ u32 nPage; /* Size of database in pages */ u32 iCheck1; /* Checkpoint value 1 */ u32 iCheck2; /* Checkpoint value 2 */ @@ -620,7 +697,7 @@ static int walIndexRecover(Wal *pWal){ if( nTruncate ){ hdr.iCheck1 = aCksum[0]; hdr.iCheck2 = aCksum[1]; - hdr.iLastPg = iFrame; + hdr.mxFrame = iFrame; hdr.nPage = nTruncate; hdr.pgsz = nPgsz; } @@ -633,7 +710,7 @@ static int walIndexRecover(Wal *pWal){ } finished: - if( rc==SQLITE_OK && hdr.iLastPg==0 ){ + if( rc==SQLITE_OK && hdr.mxFrame==0 ){ rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT); } if( rc==SQLITE_OK ){ @@ -841,7 +918,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ u8 *aSpace; /* Surplus space on the end of the allocation */ /* Make sure the wal-index is mapped into local memory */ - rc = walIndexMap(pWal, walMappingSize(pWal->hdr.iLastPg)); + rc = walIndexMap(pWal, walMappingSize(pWal->hdr.mxFrame)); if( rc!=SQLITE_OK ){ return rc; } @@ -855,7 +932,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ aData = (u32*)pWal->pWiData; /* Allocate space for the WalIterator object */ - iLast = pWal->hdr.iLastPg; + iLast = pWal->hdr.mxFrame; nSegment = (iLast >> 8) + 1; nFinal = (iLast & 0x000000FF); nByte = sizeof(WalIterator) + (nSegment+1)*(sizeof(struct WalSegment)+256); @@ -915,7 +992,7 @@ static int walCheckpoint( /* Allocate the iterator */ rc = walIteratorInit(pWal, &pIter); - if( rc!=SQLITE_OK || pWal->hdr.iLastPg==0 ){ + if( rc!=SQLITE_OK || pWal->hdr.mxFrame==0 ){ goto out; } @@ -949,7 +1026,7 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pDbFd, sync_flags); if( rc!=SQLITE_OK ) goto out; } - pWal->hdr.iLastPg = 0; + pWal->hdr.mxFrame = 0; pWal->hdr.iCheck1 = 2; pWal->hdr.iCheck2 = 3; walIndexWriteHdr(pWal, &pWal->hdr); @@ -1206,7 +1283,7 @@ int sqlite3WalRead( ){ int rc; /* Return code */ u32 iRead = 0; /* If !=0, WAL frame to return data from */ - u32 iLast = pWal->hdr.iLastPg; /* Last page in WAL for this reader */ + u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ /* If the "last page" field of the wal-index header snapshot is 0, then @@ -1382,12 +1459,12 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ int rc = SQLITE_OK; if( pWal->lockState==SQLITE_SHM_WRITE ){ int unused; - Pgno iMax = pWal->hdr.iLastPg; + Pgno iMax = pWal->hdr.mxFrame; Pgno iFrame; assert( pWal->pWiData==0 ); rc = walIndexReadHdr(pWal, &unused); - for(iFrame=pWal->hdr.iLastPg+1; rc==SQLITE_OK && iFrame<=iMax; iFrame++){ + for(iFrame=pWal->hdr.mxFrame+1; rc==SQLITE_OK && iFrame<=iMax; iFrame++){ assert( pWal->lockState==SQLITE_SHM_WRITE ); rc = xUndo(pUndoCtx, pWal->pWiData[walIndexEntry(iFrame)]); } @@ -1401,7 +1478,7 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ */ u32 sqlite3WalSavepoint(Wal *pWal){ assert( pWal->lockState==SQLITE_SHM_WRITE ); - return pWal->hdr.iLastPg; + return pWal->hdr.mxFrame; } /* Move the write position of the WAL back to iFrame. Called in @@ -1412,7 +1489,7 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame){ u8 aCksum[8]; assert( pWal->lockState==SQLITE_SHM_WRITE ); - pWal->hdr.iLastPg = iFrame; + pWal->hdr.mxFrame = iFrame; if( iFrame>0 ){ i64 iOffset = walFrameOffset(iFrame, pWal->hdr.pgsz) + sizeof(u32)*2; rc = sqlite3OsRead(pWal->pWalFd, aCksum, sizeof(aCksum), iOffset); @@ -1453,7 +1530,7 @@ int sqlite3WalFrames( ** this source file for a description of the WAL header format. */ assert( WAL_FRAME_HDRSIZE>=WAL_HDRSIZE ); - iFrame = pWal->hdr.iLastPg; + iFrame = pWal->hdr.mxFrame; if( iFrame==0 ){ sqlite3Put4byte(aFrame, nPgsz); sqlite3_randomness(8, &aFrame[4]); @@ -1525,7 +1602,7 @@ int sqlite3WalFrames( ** guarantees that there are no other writers, and no data that may ** be in use by existing readers is being overwritten. */ - iFrame = pWal->hdr.iLastPg; + iFrame = pWal->hdr.mxFrame; for(p=pList; p && rc==SQLITE_OK; p=p->pDirty){ iFrame++; rc = walIndexAppend(pWal, iFrame, p->pgno); @@ -1539,7 +1616,7 @@ int sqlite3WalFrames( if( rc==SQLITE_OK ){ /* Update the private copy of the header. */ pWal->hdr.pgsz = nPgsz; - pWal->hdr.iLastPg = iFrame; + pWal->hdr.mxFrame = iFrame; if( isCommit ){ pWal->hdr.iChange++; pWal->hdr.nPage = nTruncate;