From: dan Date: Fri, 22 Mar 2013 18:20:14 +0000 (+0000) Subject: Allow the database file to be memory mapped in wal mode. X-Git-Tag: version-3.7.17~114^2~58 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=99bd10979abd01877d6c213235c0ad22ab142a8f;p=thirdparty%2Fsqlite.git Allow the database file to be memory mapped in wal mode. FossilOrigin-Name: d190ddabc386bc9654b99e33fb81b2f6e67b54d6 --- diff --git a/manifest b/manifest index 1be3bebe19..cab9a67764 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sfix\sfor\sthe\sassert()\sstatements\sadded\sby\sthe\sprevious\scommit. -D 2013-03-22T17:46:11.890 +C Allow\sthe\sdatabase\sfile\sto\sbe\smemory\smapped\sin\swal\smode. +D 2013-03-22T18:20:14.143 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,8 +162,8 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 -F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c +F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c +F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -249,8 +249,8 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 -F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 +F src/wal.c 7fec703bb0770d9faad5190f49897a8bc8eb239f +F src/wal.h d99ce512ac60f9147a0640e9e6fb67dd1057b781 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 765615f9fba7c1765eb741cb98a09a28b464ee55 -R 125e508e18383c5f0a5b411a39a78cab +P 19345416ed5e1ab5b0b35993b0b9069c2fb1683b +R 7243a86b7c03fc5d49450319f9d9b1bf U dan -Z 4d1b2a2cebcc3fe5bdd2b5af2205540b +Z 7b1a30ba76756800b3f32db2afc843bf diff --git a/manifest.uuid b/manifest.uuid index 8efe087450..4f8ecfff16 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19345416ed5e1ab5b0b35993b0b9069c2fb1683b \ No newline at end of file +d190ddabc386bc9654b99e33fb81b2f6e67b54d6 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 0ff894bd87..660514d508 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2095,7 +2095,7 @@ static void pagerReportSize(Pager *pPager){ ** Return SQLITE_OK if successful, or an SQLite error code if an error ** occurs. */ -static int pagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ +int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ int rc = SQLITE_OK; if( pPager->nMapValid>=(iOff+nBuf) ){ memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf); @@ -2279,7 +2279,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = pagerWriteData(pPager, aData, pPager->pageSize, ofst); + rc = sqlite3PagerWriteData(pPager, aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2864,11 +2864,10 @@ end_playback: ** If an IO error occurs, then the IO error is returned to the caller. ** Otherwise, SQLITE_OK is returned. */ -static int readDbPage(PgHdr *pPg){ +static int readDbPage(PgHdr *pPg, u32 iFrame){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ assert( pPager->eState>=PAGER_READER && !MEMDB ); @@ -2880,11 +2879,10 @@ static int readDbPage(PgHdr *pPg){ return SQLITE_OK; } - if( pagerUseWal(pPager) ){ + if( iFrame ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); - } - if( rc==SQLITE_OK && !isInWal ){ + rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); + }else{ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){ memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize); @@ -2972,7 +2970,13 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ - rc = readDbPage(pPg); + u32 iFrame = 0; + if( pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + } + if( rc==SQLITE_OK ){ + rc = readDbPage(pPg, iFrame); + } if( rc==SQLITE_OK ){ pPager->xReiniter(pPg); } @@ -3388,7 +3392,6 @@ static void pagerFixMaplimit(Pager *pPager){ || pPager->fd->pMethods->iVersion<3 || pPager->fd->pMethods->xMremap==0 || pPager->tempFile - || pPager->pWal ){ pPager->nMapLimit = 0; }else if( pPager->nMapCfgLimit<0 ){ @@ -3888,9 +3891,9 @@ static int pagerMap(Pager *pPager, int bExtend){ Pgno nPg; /* Size of mapping to request in pages */ i64 sz; /* Size of mapping to request in bytes */ - assert( pPager->pWal==0 && isOpen(pPager->fd) && pPager->tempFile==0 ); + assert( isOpen(pPager->fd) && pPager->tempFile==0 ); assert( pPager->pMap==0 || pPager->nMap>0 ); - assert( pPager->eState>=1 ); + /* assert( pPager->eState>=1 ); */ assert( pPager->nMmapOut==0 ); assert( pPager->nMapLimit>0 ); @@ -4213,6 +4216,46 @@ static int syncJournal(Pager *pPager, int newHdr){ return SQLITE_OK; } +/* +** This is called by the wal.c module at the start of a checkpoint. If the +** checkpoint runs to completion, it will set the database file size to +** szReq bytes. This function performs two tasks: +** +** * If the file is currently less than szReq bytes in size, an +** xFileControl(SQLITE_FNCTL_SIZE_HINT) is issued to inform the OS +** layer of the expected file size, and +** +** * If mmap is being used, then the mapping is extended to szReq +** bytes in size. +** +** SQLITE_OK is returned if successful, or an error code if an error occurs. +*/ +int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ + int rc; + sqlite3_int64 sz; + + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->nMmapOut==0 ); + + rc = sqlite3OsFileSize(pPager->fd, &sz); + if( rc==SQLITE_OK ){ + if( sz>szReq ){ + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &sz); + } + } + + if( rc==SQLITE_OK + && pPager->nMapLimit>0 + && pPager->nMapValidnMapValidnMapLimit + ){ + pPager->dbFileSize = (szReq / pPager->pageSize); + rc = pagerMap(pPager, 1); + } + + return rc; +} + /* ** The argument is the first in a linked list of dirty pages connected ** by the PgHdr.pDirty pointer. This function writes each one of the @@ -4275,7 +4318,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ pPager->dbFileSize = pPager->dbSize; - pagerMap(pPager, 1); + rc = pagerMap(pPager, 1); } } @@ -4301,7 +4344,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - pagerWriteData(pPager, pData, pPager->pageSize, offset); + rc = sqlite3PagerWriteData(pPager, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -5275,6 +5318,7 @@ int sqlite3PagerAcquire( ){ int rc = SQLITE_OK; PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT); /* It is acceptable to use a read-only (mmap) page for any page except @@ -5287,6 +5331,7 @@ int sqlite3PagerAcquire( assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); + assert( noContent==0 || bMmapOk==0 ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; @@ -5298,7 +5343,12 @@ int sqlite3PagerAcquire( rc = pPager->errCode; }else{ - if( bMmapOk ){ + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + + if( iFrame==0 && bMmapOk ){ if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ rc = pagerMap(pPager, 0); } @@ -5378,9 +5428,13 @@ int sqlite3PagerAcquire( memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ + if( pagerUseWal(pPager) && bMmapOk==0 ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } assert( pPg->pPager==pPager ); pPager->aStat[PAGER_STAT_MISS]++; - rc = readDbPage(pPg); + rc = readDbPage(pPg, iFrame); if( rc!=SQLITE_OK ){ goto pager_acquire_err; } @@ -7063,7 +7117,7 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, + rc = sqlite3WalOpen(pPager->pVfs, pPager, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); diff --git a/src/pager.h b/src/pager.h index 81ab30c115..970b5035bf 100644 --- a/src/pager.h +++ b/src/pager.h @@ -174,6 +174,10 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); +int sqlite3PagerSetFilesize(Pager *, i64); + +/* Write data to the database file */ +int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); diff --git a/src/wal.c b/src/wal.c index 0d7271bf04..8757003d78 100644 --- a/src/wal.c +++ b/src/wal.c @@ -412,6 +412,7 @@ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ + Pager *pPager; /* Pager object */ u32 iCallback; /* Value to pass to log callback (or 0) */ i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ @@ -1251,6 +1252,7 @@ static void walIndexClose(Wal *pWal, int isDelete){ */ int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ + Pager *pPager, /* Pager object handle */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ @@ -1291,6 +1293,7 @@ int sqlite3WalOpen( pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; + pRet->pPager = pPager; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ @@ -1727,10 +1730,7 @@ static int walCheckpoint( */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); - if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); - } + rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); } /* Iterate through the contents of the WAL, copying data to the db file. */ @@ -1744,7 +1744,7 @@ static int walCheckpoint( if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + rc = sqlite3PagerWriteData(pWal->pPager, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } @@ -2287,19 +2287,17 @@ void sqlite3WalEndReadTransaction(Wal *pWal){ } /* -** Read a page from the WAL, if it is present in the WAL and if the -** current read transaction is configured to use the WAL. +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. ** -** The *pInWal is set to 1 if the requested page is in the WAL and -** has been loaded. Or *pInWal is set to 0 if the page was not in -** the WAL and needs to be read out of the database. +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. */ -int sqlite3WalRead( +int sqlite3WalFindFrame( Wal *pWal, /* WAL handle */ Pgno pgno, /* Database page number to read data for */ - int *pInWal, /* OUT: True if data is read from WAL */ - int nOut, /* Size of buffer pOut in bytes */ - u8 *pOut /* Buffer to write page data to */ + u32 *piRead /* OUT: Frame number (or zero) */ ){ u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ @@ -2315,7 +2313,7 @@ int sqlite3WalRead( ** WAL were empty. */ if( iLast==0 || pWal->readLock==0 ){ - *pInWal = 0; + *piRead = 0; return SQLITE_OK; } @@ -2386,26 +2384,31 @@ int sqlite3WalRead( } #endif - /* If iRead is non-zero, then it is the log frame number that contains the - ** required page. Read and return data from the log file. - */ - if( iRead ){ - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (sz&0xfe00) + ((sz&0x0001)<<16); - testcase( sz<=32768 ); - testcase( sz>=65536 ); - iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; - *pInWal = 1; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); - } - - *pInWal = 0; + *piRead = iRead; return SQLITE_OK; } +/* +** Read the contents of frame iRead from the wal file into buffer pOut +** (which is nOut bytes in size). Return SQLITE_OK if successful, or an +** error code otherwise. +*/ +int sqlite3WalReadFrame( + Wal *pWal, /* WAL handle */ + u32 iRead, /* Frame to read */ + int nOut, /* Size of buffer pOut in bytes */ + u8 *pOut /* Buffer to write page data to */ +){ + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); +} /* ** Return the size of the database in pages (or zero, if unknown). diff --git a/src/wal.h b/src/wal.h index a848de1527..96d4ff9adf 100644 --- a/src/wal.h +++ b/src/wal.h @@ -31,7 +31,6 @@ # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) -# define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 @@ -54,7 +53,8 @@ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); +int sqlite3WalOpen( + sqlite3_vfs*, Pager *, sqlite3_file*, const char *, int, i64, Wal**); int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */ @@ -71,7 +71,8 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *); void sqlite3WalEndReadTransaction(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ -int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); +int sqlite3WalFindFrame(Wal *, Pgno, u32 *); +int sqlite3WalReadFrame(Wal *, u32, int, u8 *); /* If the WAL is not empty, return the size of the database. */ Pgno sqlite3WalDbsize(Wal *pWal);