From: dan Date: Sat, 23 Mar 2013 21:00:41 +0000 (+0000) Subject: Replace the sqlite3_io_methods.xMremap interface with sqlite3_io_methods.xFetch and... X-Git-Tag: version-3.7.17~114^2~49 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f23da96636f9de1a81c939bd7a0cc25ee1d37f49;p=thirdparty%2Fsqlite.git Replace the sqlite3_io_methods.xMremap interface with sqlite3_io_methods.xFetch and xUnfetch. FossilOrigin-Name: 1431be95579160fb70408d43e17fc23c7b69ab4a --- diff --git a/manifest b/manifest index 6660b0b860..8d71dc88ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sa\scomment\sin\swal.c.\sNo\scode\schanges. -D 2013-03-23T17:29:06.036 +C Replace\sthe\ssqlite3_io_methods.xMremap\sinterface\swith\ssqlite3_io_methods.xFetch\sand\sxUnfetch. +D 2013-03-23T21:00:41.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,7 +118,7 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c d5f895810e8ff9737c9ec7b76abc3dcff5860335 F src/attach.c ea5247f240e2c08afd608e9beb380814b86655e1 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c 6256400ab8be4a15a2512277d8b214dbd44ff8ce +F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c f5ea8d3d658887b5cae0369ef10a427d7469a768 @@ -157,13 +157,13 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 -F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b +F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 +F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf +F src/os_unix.c d8cdf331ad08650fc8397eee79b55548070077b0 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c cffcfe753445ef3a3d3830efd5f807fc171e7262 -F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 +F src/pager.c 2e63de7185c2a573d26e5087869a9e48326076f9 +F src/pager.h faf4bed79e80cb6527209dfe5cfb08aa5764b941 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in d63c7fb5832287af7e8b903c4a4c30c90414876f +F src/sqlite.h.in f41949e1e778043dd9fc23b33b96e7a6e20de4d6 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -249,8 +249,8 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c 923d90992fc3b069571cf3fbd635c54c3e839e55 -F src/wal.h d99ce512ac60f9147a0640e9e6fb67dd1057b781 +F src/wal.c e84eff498c57ec2d79ca4496a3f4a638af378fb3 +F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8dbe89d05ce91428c69003f0da79d883fa23e2b5 -R b063a8588944a5f842812de15381707f +P 60b9f5e4dd2af54975ba78437239f0bebd472fd2 +R eee4d88d6400cf65980c5b201532d2c9 U dan -Z 350cbe536c1fd5aa73b476fafae6db5c +Z eabd20cdc7aabbc65591f96e4f6dff04 diff --git a/manifest.uuid b/manifest.uuid index 7aea0f37ae..f378aa2e8b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -60b9f5e4dd2af54975ba78437239f0bebd472fd2 \ No newline at end of file +1431be95579160fb70408d43e17fc23c7b69ab4a \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 8939bc67bc..71a8a1a3e7 100644 --- a/src/backup.c +++ b/src/backup.c @@ -520,7 +520,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); if( rc==SQLITE_OK ){ u8 *zData = sqlite3PagerGetData(pSrcPg); - rc = sqlite3PagerWriteData(pDestPager, zData, pgszSrc, iOff); + rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff); } sqlite3PagerUnref(pSrcPg); } diff --git a/src/os.c b/src/os.c index 38757dcdb1..08e0d42172 100644 --- a/src/os.c +++ b/src/os.c @@ -140,15 +140,12 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } -int sqlite3OsMremap( - sqlite3_file *id, /* Database file handle */ - int flags, /* SQLITE_MREMAP_XXX flags */ - i64 iOff, /* Offset at which mapping(s) start */ - i64 nOld, /* Size of old mapping */ - i64 nNew, /* Size of requested mapping */ - void **pp /* IN/OUT: Pointer to mapped region */ -){ - return id->pMethods->xMremap(id, flags, iOff, nOld, nNew, pp); + +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +int sqlite3OsUnfetch(sqlite3_file *id, void *p){ + return id->pMethods->xUnfetch(id, p); } /* diff --git a/src/os.h b/src/os.h index f08b92dcf0..a7dacfb6ea 100644 --- a/src/os.h +++ b/src/os.h @@ -259,7 +259,8 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); -int sqlite3OsMremap(sqlite3_file *id, int, i64, i64, i64, void **); +int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +int sqlite3OsUnfetch(sqlite3_file *, void *); /* diff --git a/src/os_unix.c b/src/os_unix.c index 2f03c39670..334719200a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -249,9 +249,14 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ - sqlite3_int64 mmapSize; /* Size of xMremap() */ - void *pMapRegion; /* Area memory mapped */ + #endif + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_SIZE value */ + void *pMapRegion; /* Memory mapped region */ + int nFetchOut; /* Number of outstanding xFetch refs */ + #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. @@ -1805,6 +1810,9 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ return posixUnlock(id, eFileLock, 0); } +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); + /* ** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file @@ -1817,6 +1825,7 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ */ static int closeUnixFile(sqlite3_file *id){ unixFile *pFile = (unixFile*)id; + unixUnmapfile(pFile); if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); pFile->h = -1; @@ -3074,7 +3083,6 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); - assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3085,6 +3093,21 @@ static int unixRead( ); #endif + /* Deal with as much of this write request as possible by transfering + ** data to the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ return SQLITE_OK; @@ -3157,7 +3180,6 @@ static int unixWrite( int wrote = 0; assert( id ); assert( amt>0 ); - assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3190,6 +3212,21 @@ static int unixWrite( } #endif + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; offset += wrote; @@ -3470,6 +3507,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( pFile->inNormalWrite && nByte==0 ){ pFile->transCntrChng = 1; } +#endif /* If the file was just truncated to a size smaller than the currently ** mapped region, reduce the effective mapping size as well. SQLite will @@ -3478,7 +3516,6 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( nBytemmapSize ){ pFile->mmapSize = nByte; } -#endif return SQLITE_OK; } @@ -3568,6 +3605,19 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } + if( pFile->mmapLimit>0 ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } + return SQLITE_OK; } @@ -3635,8 +3685,8 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_FCNTL_GETFD: { - *(int*)pArg = pFile->h; + case SQLITE_FCNTL_MMAP_SIZE: { + pFile->mmapLimit = *(i64*)pArg; return SQLITE_OK; } #ifdef SQLITE_DEBUG @@ -4451,91 +4501,86 @@ static int unixShmUnmap( */ #define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) -/* -** Map, remap or unmap part of the database file. -*/ -static int unixMremap( - sqlite3_file *fd, /* Main database file */ - int flags, /* Mask of SQLITE_MREMAP_XXX flags */ - sqlite3_int64 iOff, /* Offset to start mapping at */ - sqlite3_int64 nOld, /* Size of old mapping, or zero */ - sqlite3_int64 nNew, /* Size of new mapping, or zero */ - void **ppMap /* IN/OUT: Old/new mappings */ -){ - unixFile *p = (unixFile *)fd; /* The underlying database file */ - int rc = SQLITE_OK; /* Return code */ - void *pNew = 0; /* New mapping */ - i64 nNewRnd; /* nNew rounded up */ - i64 nOldRnd; /* nOld rounded up */ - - assert( iOff==0 ); - /* assert( p->mmapSize==nOld ); */ - assert( p->pMapRegion==0 || p->pMapRegion==(*ppMap) ); - - /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested - ** mapping (nNew bytes) may be greater than the size of the database file. - ** If this is the case, extend the file on disk using ftruncate(). */ - assert( nNew>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); - if( flags & SQLITE_MREMAP_EXTEND ){ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); + if( pFd->pMapRegion ){ + munmap(pFd->pMapRegion, pFd->mmapOrigsize); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapOrigsize = 0; + } +} + +static int unixMapfile(unixFile *pFd, i64 nByte){ + i64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ struct stat statbuf; /* Low-level file information */ - rc = osFstat(p->h, &statbuf); - if( rc==SQLITE_OK && nNew>statbuf.st_size ){ - rc = robust_ftruncate(p->h, nNew); + rc = osFstat(pFd->h, &statbuf); + if( rc!=SQLITE_OK ){ + return SQLITE_IOERR_FSTAT; } - if( rc!=SQLITE_OK ) return rc; + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapLimit ){ + nMap = pFd->mmapLimit; } - /* According to some sources, the effect of changing the size of the - ** underlying file on mapped regions that correspond to the added or - ** removed pages is undefined. However, there is reason to believe that - ** on modern platforms like Linux or OSX, things just work. For example, - ** it is possible to create a mapping larger than the file on disk and - ** extend the file on disk later on. - ** - ** Exploit this on Linux and OSX to reduce the number of munmap()/mmap() - ** calls required if the file size is changing. In this case all mappings - ** are rounded up to the nearest 4MB. And if a new mapping is requested - ** that has the same rounded size as an old mapping, the old mapping can - ** be reused as is. */ -#if defined(__APPLE__) || defined(__linux__) - nNewRnd = ROUNDUP(nNew, 4096*1024); - nOldRnd = ROUNDUP(nOld, 4096*1024); -#else - nNewRnd = ROUNDUP(nNew, 4096*1); - nOldRnd = ROUNDUP(nOld, 4096*1); -#endif + if( nMap!=pFd->mmapSize ){ + void *pNew; + unixUnmapfile(pFd); - /* On OSX or Linux, reuse the old mapping if it is the right size. */ -#if defined(__APPLE__) || defined(__linux__) - if( nNewRnd==nOldRnd ){ - VVA_ONLY( p->mmapSize = nNew; ) - return SQLITE_OK; + if( nMap>0 ){ + void *pNew; + int flags = PROT_READ; + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + pNew = mmap(0, ROUNDUP(nMap, 4096), flags, MAP_SHARED, pFd->h, 0); + if( pNew==MAP_FAILED ){ + return SQLITE_IOERR_MREMAP; + } + + pFd->pMapRegion = pNew; + pFd->mmapOrigsize = pFd->mmapSize = nMap; + } } -#endif - /* If we get this far, unmap any old mapping. */ - if( nOldRnd!=0 ){ - void *pOld = *ppMap; - munmap(pOld, nOldRnd); - VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ); - } - - /* And, if required, use mmap() to create a new mapping. */ - if( nNewRnd>0 ){ - int flags = PROT_READ; - if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, nNewRnd, flags, MAP_SHARED, p->h, iOff); - if( pNew==MAP_FAILED ){ - pNew = 0; - VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ) - rc = SQLITE_IOERR_MREMAP; - }else{ - VVA_ONLY( p->mmapSize = nNew; p->pMapRegion = pNew; ) + return SQLITE_OK; +} + +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + *pp = 0; + + if( pFd->mmapLimit>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } + return SQLITE_OK; +} + +static int unixUnfetch(sqlite3_file *fd, void *p){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + + assert( (p==0)==(pFd->nFetchOut==0) ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); } - *ppMap = pNew; - return rc; + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; } /* @@ -4597,7 +4642,8 @@ static const sqlite3_io_methods METHOD = { \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ unixShmUnmap, /* xShmUnmap */ \ - unixMremap, /* xMremap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4865,7 +4911,6 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; - VVA_ONLY( pNew->mmapSize = 0; ) if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/pager.c b/src/pager.c index 09cc4a2375..7e00e1431a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -656,14 +656,10 @@ struct Pager { int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ - void *pMap; /* Memory mapped prefix of database file */ - i64 nMap; /* Size of mapping at pMap in bytes */ - i64 nMapValid; /* Bytes at pMap known to be valid */ - i64 nMapLimit; /* Maximum permitted mapping size */ + u8 bUseFetch; /* True to use xFetch() */ int nMapCfgLimit; /* Configured limit value */ int nMmapOut; /* Number of mmap pages currently outstanding */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ - int bMapResize; /* Check if the mapping should be resized */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -2087,24 +2083,6 @@ static void pagerReportSize(Pager *pPager){ # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif -/* -** Write nBuf bytes of data from buffer pBuf to offset iOff of the -** database file. If this part of the database file is memory mapped, -** use memcpy() to do so. Otherwise, call sqlite3OsWrite(). -** -** Return SQLITE_OK if successful, or an SQLite error code if an error -** occurs. -*/ -int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ - int rc = SQLITE_OK; - if( pPager->nMapValid>=(iOff+nBuf) ){ - memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf); - }else{ - rc = sqlite3OsWrite(pPager->fd, pBuf, nBuf, iOff); - } - return rc; -} - /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -2279,7 +2257,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3PagerWriteData(pPager, aData, pPager->pageSize, ofst); + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2534,9 +2512,6 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ if( rc==SQLITE_OK && currentSize!=newSize ){ if( currentSize>newSize ){ rc = sqlite3OsTruncate(pPager->fd, newSize); - if( newSizenMapValid ){ - pPager->nMapValid = newSize; - } }else if( (currentSize+szPage)<=newSize ){ char *pTmp = pPager->pTmpSpace; memset(pTmp, 0, szPage); @@ -2884,13 +2859,9 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); }else{ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; - if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){ - memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize); - }else{ - rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; - } + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; } } @@ -3120,6 +3091,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); + if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0); } return rc; @@ -3382,22 +3354,24 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* -** Set Pager.nMapLimit, the maximum permitted mapping size, based on the -** current values of Pager.nMapCfgLimit and Pager.pageSize. -** -** If this connection should not use mmap at all, set nMapLimit to zero. +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of nMapCfgLimit. */ static void pagerFixMaplimit(Pager *pPager){ - if( isOpen(pPager->fd)==0 - || pPager->fd->pMethods->iVersion<3 - || pPager->fd->pMethods->xMremap==0 - || pPager->tempFile - ){ - pPager->nMapLimit = 0; - }else if( pPager->nMapCfgLimit<0 ){ - pPager->nMapLimit = (i64)pPager->nMapCfgLimit * -1024; - }else{ - pPager->nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) ){ + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->nMapCfgLimit!=0; + if( pPager->bUseFetch ){ + void *p; + i64 nMapLimit; + if( pPager->nMapCfgLimit<0 ){ + nMapLimit = (i64)pPager->nMapCfgLimit * -1024; + }else{ + nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + } + + p = (void *)&nMapLimit; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, p); + } } } @@ -3870,67 +3844,22 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } -/* -** Unmap any memory mapping of the database file. -*/ -static int pagerUnmap(Pager *pPager){ - assert( pPager->nMmapOut==0 ); - if( pPager->pMap ){ - sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); - pPager->nMap = 0; - pPager->nMapValid = 0; - } - return SQLITE_OK; -} - -/* -** Create, or recreate, the memory mapping of the database file. -*/ -static int pagerMap(Pager *pPager, int bExtend){ - int rc = SQLITE_OK; /* Return code */ - Pgno nPg; /* Size of mapping to request in pages */ - i64 sz; /* Size of mapping to request in bytes */ - - assert( isOpen(pPager->fd) && pPager->tempFile==0 ); - assert( pPager->pMap==0 || pPager->nMap>0 ); - /* assert( pPager->eState>=1 ); */ - assert( pPager->nMmapOut==0 ); - assert( pPager->nMapLimit>0 ); - - /* Figure out how large a mapping to request. Set variable sz to this - ** value in bytes. */ - nPg = (pPager->eState==1) ? pPager->dbSize : pPager->dbFileSize; - sz = (i64)nPg * pPager->pageSize; - if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; - - if( sz!=pPager->nMapValid ){ - int flags = (bExtend ? SQLITE_MREMAP_EXTEND : 0); - rc = sqlite3OsMremap(pPager->fd, flags, 0, pPager->nMap, sz, &pPager->pMap); - if( rc==SQLITE_OK ){ - assert( pPager->pMap!=0 ); - pPager->nMap = sz; - }else{ - assert( pPager->pMap==0 ); - pPager->nMap = 0; - } - pPager->nMapValid = pPager->nMap; - } - pPager->bMapResize = 0; - - return rc; -} - /* ** Obtain a reference to a memory mapped page object for page number pgno. -** The caller must ensure that page pgno lies within the currently mapped -** region. If successful, set *ppPage to point to the new page reference +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference ** and return SQLITE_OK. Otherwise, return an SQLite error code and set ** *ppPage to zero. ** ** Page references obtained by calling this function should be released ** by calling pagerReleaseMapPage(). */ -static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ PgHdr *p; /* Memory mapped page to return */ if( pPager->pFree ){ @@ -3955,8 +3884,8 @@ static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ assert( p->pPager==pPager ); assert( p->nRef==1 ); - p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; p->pgno = pgno; + p->pData = pData; pPager->nMmapOut++; return SQLITE_OK; @@ -3971,6 +3900,9 @@ static void pagerReleaseMapPage(PgHdr *pPg){ pPager->nMmapOut--; pPg->pDirty = pPager->pFree; pPager->pFree = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, pPg->pData); } /* @@ -4006,7 +3938,6 @@ int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); - pagerUnmap(pPager); pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; @@ -4216,46 +4147,6 @@ static int syncJournal(Pager *pPager, int newHdr){ return SQLITE_OK; } -/* -** This is called by the wal.c module at the start of a checkpoint. If the -** checkpoint runs to completion, it will set the database file size to -** szReq bytes. This function performs two tasks: -** -** * If the file is currently less than szReq bytes in size, an -** xFileControl(SQLITE_FNCTL_SIZE_HINT) is issued to inform the OS -** layer of the expected file size, and -** -** * If mmap is being used, then the mapping is extended to szReq -** bytes in size. -** -** SQLITE_OK is returned if successful, or an error code if an error occurs. -*/ -int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ - int rc; - i64 sz; /* Size of file on disk in bytes */ - - assert( pPager->eState==PAGER_OPEN ); - assert( pPager->nMmapOut==0 ); - - rc = sqlite3OsFileSize(pPager->fd, &sz); - if( rc==SQLITE_OK ){ - if( sz>szReq ){ - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &sz); - } - } - - - if( rc==SQLITE_OK ){ - i64 szMap = (szReq > pPager->nMapLimit) ? pPager->nMapLimit : szReq; - if( pPager->nMapValid!=pPager->nMap || szMap!=pPager->nMap ){ - pPager->dbFileSize = (szReq / pPager->pageSize); - rc = pagerMap(pPager, 1); - } - } - - return rc; -} - /* ** The argument is the first in a linked list of dirty pages connected ** by the PgHdr.pDirty pointer. This function writes each one of the @@ -4315,11 +4206,6 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; - - if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ - pPager->dbFileSize = pPager->dbSize; - rc = pagerMap(pPager, 1); - } } while( rc==SQLITE_OK && pList ){ @@ -4344,7 +4230,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - rc = sqlite3PagerWriteData(pPager, pData, pPager->pageSize, offset); + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -5164,7 +5050,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( !pPager->tempFile && ( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 - || pPager->pMap + || pPager->bUseFetch /* TODO: Currently required for xUnfetch(0) only. */ )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous @@ -5188,13 +5074,9 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerPagecount(pPager, &nPage); if( rc ) goto failed; - if( nPage>0 || pPager->pMap ){ + if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); - if( pPager->pMap ){ - memcpy(&dbFileVers, &((u8 *)(pPager->pMap))[24], sizeof(dbFileVers)); - }else{ - rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - } + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); if( rc!=SQLITE_OK ){ goto failed; } @@ -5211,9 +5093,9 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** In this case there may exist a Pager.pMap mapping that appears ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ - pagerUnmap(pPager); - }else if( pPager->pMap ){ - pPager->bMapResize = 1; + if( pPager->bUseFetch ){ + sqlite3OsUnfetch(pPager->fd, 0); + } } } @@ -5325,7 +5207,7 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pPager->nMapLimit>0 && pgno!=1 + const int bMmapOk = (pgno!=1 && pPager->bUseFetch && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) ); @@ -5349,15 +5231,20 @@ int sqlite3PagerAcquire( } if( iFrame==0 && bMmapOk ){ - if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ - rc = pagerMap(pPager, 0); - } - if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ + void *pData = 0; + + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + + if( rc==SQLITE_OK && pData ){ if( pPager->eState>PAGER_READER ){ (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); } if( pPg==0 ){ - rc = pagerAcquireMapPage(pPager, pgno, &pPg); + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, pData); } if( pPg ){ assert( rc==SQLITE_OK ); @@ -7117,7 +7004,7 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, pPager, + rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); @@ -7155,8 +7042,6 @@ int sqlite3PagerOpenWal( assert( pbOpen==0 || *pbOpen==0 ); assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); - pagerUnmap(pPager); - if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; diff --git a/src/pager.h b/src/pager.h index 970b5035bf..066d1f11d7 100644 --- a/src/pager.h +++ b/src/pager.h @@ -176,9 +176,6 @@ int sqlite3SectorSize(sqlite3_file *); void sqlite3PagerTruncateImage(Pager*,Pgno); int sqlite3PagerSetFilesize(Pager *, i64); -/* Write data to the database file */ -int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff); - #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c0a2803787..666a2e084d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -707,24 +707,6 @@ struct sqlite3_file { ** fails to zero-fill short reads might seem to work. However, ** failure to zero-fill short reads will eventually lead to ** database corruption. -** -** Assuming parameter nNew is non-zero, the xMremap method should attempt -** to memory map a region nNew bytes in size starting at offset iOffset -** of the file. If successful, it should set *ppMap to point to the -** mapping and return SQLITE_OK. If the file is opened for read-write -** access, then the mapping should also be read-write. -** -** If nOld is non-zero, then the initial value of *ppMap points to a -** mapping returned by a previous call to xMremap. The existing mapping -** is nOld bytes in size and starts at offset iOffset of the file. In -** this case the xMremap method is expected to unmap the existing mapping -** and overwrite *ppMap with the pointer to the new mapping. If nOld is -** zero, then the initial value of *ppMap is undefined. -** -** If nNew is zero, then no new mapping should be created. Any old -** mapping must still be unmapped if nOld is non-zero. If the nOld -** parameter is non-zero, then the existing mapping is always unmapped - -** even if an error occurs. */ typedef struct sqlite3_io_methods sqlite3_io_methods; struct sqlite3_io_methods { @@ -747,14 +729,12 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ - int (*xMremap)(sqlite3_file *fd, int flags, - sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, void *p); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; -#define SQLITE_MREMAP_EXTEND 0x0001 /* xMremap call may extend file */ - /* ** CAPI3REF: Standard File Control Opcodes ** @@ -906,6 +886,10 @@ struct sqlite3_io_methods { ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** +**
  • [[SQLITE_FCNTL_MMAP_SIZE]] +** The argument is assumed to point to a value of type sqlite3_int64. An +** advisory maximum amount of this file to memory map in bytes. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -924,7 +908,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 -#define SQLITE_FCNTL_GETFD 17 +#define SQLITE_FCNTL_MMAP_SIZE 18 /* ** CAPI3REF: Mutex Handle diff --git a/src/wal.c b/src/wal.c index 051b77f515..4602eb2c34 100644 --- a/src/wal.c +++ b/src/wal.c @@ -412,7 +412,6 @@ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ - Pager *pPager; /* Pager object */ u32 iCallback; /* Value to pass to log callback (or 0) */ i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ @@ -1252,7 +1251,6 @@ static void walIndexClose(Wal *pWal, int isDelete){ */ int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ - Pager *pPager, /* Pager object handle */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ @@ -1293,7 +1291,6 @@ int sqlite3WalOpen( pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; - pRet->pPager = pPager; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ @@ -1725,19 +1722,18 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - /* If the database file is currently smaller than mxPage pages in size, - ** the call below issues an SQLITE_FCNTL_SIZE_HINT to the OS layer to - ** inform it that it is likely to grow to that size. - ** - ** Additionally, if the pager is using mmap(), then the call to - ** SetFilesize() guarantees that the mapping is not larger than mxPage - ** pages. This makes the sqlite3OsTruncate() call below safe - no pages - ** that are part of the mapped region will be truncated away. */ + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } } + /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; @@ -1749,7 +1745,7 @@ static int walCheckpoint( if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3PagerWriteData(pWal->pPager, zBuf, szPage, iOffset); + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } diff --git a/src/wal.h b/src/wal.h index 96d4ff9adf..ff7624af63 100644 --- a/src/wal.h +++ b/src/wal.h @@ -53,8 +53,7 @@ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen( - sqlite3_vfs*, Pager *, sqlite3_file*, const char *, int, i64, Wal**); +int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */