From: dan Date: Tue, 19 Mar 2013 19:28:06 +0000 (+0000) Subject: Add the sqlite3_io_methods.xMremap() method to the VFS interface. Also "PRAGMA mmap_s... X-Git-Tag: version-3.7.17~114^2~68 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5d8a137218d43070a2c635900d1ae7d7a7ef0f69;p=thirdparty%2Fsqlite.git Add the sqlite3_io_methods.xMremap() method to the VFS interface. Also "PRAGMA mmap_size". FossilOrigin-Name: 6183f1bd86ceed76d22d9762f3d7eb33262c62d1 --- diff --git a/manifest b/manifest index a6549b18c2..45c3ee6fc5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\spossible,\suse\smemcpy()\sto\sand\sfrom\sthe\smapped\sregion\sinstead\sof\sxWrite()\sand\sxRead(). -D 2013-03-16T20:19:21.766 +C Add\sthe\ssqlite3_io_methods.xMremap()\smethod\sto\sthe\sVFS\sinterface.\sAlso\s"PRAGMA\smmap_size". +D 2013-03-19T19:28:06.473 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,8 +121,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 934921ec91456c264f61ce671ca62cb826af977a -F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd +F src/btree.c d4d551f05a555926a7c0f49c2e263f7ee2b1c59f +F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc @@ -157,18 +157,18 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c -F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 +F src/os.c 81a82a736b8a461a656f9b3e401a39768fe73a79 +F src/os.h 4681261aa24a9d2187aaf4cb963880e6cddb1f48 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 +F src/os_unix.c 0c7b0d076f2ac6279b0b280a26bcae8c89f36f4f F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 495c5344392d5932ea5072f20bfbd8a58cf19d67 +F src/pager.c 78b65bf9685bf21b787ce2a7389c2b96102942dc F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 9f0ee3d74a7f33eeeff40a4b014fc3abf8182ce2 +F src/pragma.c 86c8088ac6a12d3f3be5f7394542651f03fa9a38 F src/prepare.c 78cd7ae8cd0d8de8ef8a8b5352fc5f38693a0852 F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,15 +176,15 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in 9a5c737a1feb4495d351c56883587d4fda52e81e +F src/sqlite.h.in fd75f5bcf479b315b1c717fa1d07f018bd919f79 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a +F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3213f3101e3b85f047d6e389da5a53d76d3d7540 -F src/test1.c 3dac8d76be8852d65ff8b9ce4b50ed08b999ed59 +F src/test1.c 39378e3e14a8162e29dc90d1e05399d12e8a569e F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -369,7 +369,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test f3228180c87bbe5d39c9397bf001c0095c3821b9 F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b -F test/dbstatus2.test bf7396af964b89e39435babbcdf296ae8fc5f10a +F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -403,7 +403,7 @@ F test/eqp.test 46aa946dd55c90635327898275d3e533d23a9845 F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test a1b324cb21834a490cd052d409d34789cfef57cb -F test/exclusive2.test 372be98f6de44dd78734e364b7b626ea211761a6 +F test/exclusive2.test 881193eccec225cfed9d7f744b65e57f26adee25 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d @@ -508,7 +508,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4unicode.test 25ccad45896f8e50f6a694cff738a35f798cdb40 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test b058483c17952eff7797b837bbb61e27e6b05606 +F test/func.test b0fc34fdc36897769651975a2b0a606312753643 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 @@ -526,7 +526,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 -F test/incrblob.test 34765fa6fb5d8e0f256fc7d6497c04b205398849 +F test/incrblob.test bf210bea512474d4e1d94fbb9b0fcb386cd65dea F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 @@ -652,11 +652,11 @@ F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 -F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 +F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test eb49937dca270b2c3f62d4c91fc7034ca905b7f1 +F test/permutations.test 694f4a2667242bab49cce05c54c2adfcc2727d9e F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1038,7 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 022fdc986b33701abfd39621072ac3d9f9f7d43e -R e9d1b00daf0ae79f0e154b8607fa8ad8 +P f8ca5622d99bedca957caa9ad311d798f63b3ce9 +R 1add721bf7e9669d3b4d4c28d41bb13b U dan -Z 82853c015b40f8bb1665068f9eea77d4 +Z 3a3766130d03f0bab6c4936a9c26c7a5 diff --git a/manifest.uuid b/manifest.uuid index 500dcc6707..8d198c909c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8ca5622d99bedca957caa9ad311d798f63b3ce9 \ No newline at end of file +6183f1bd86ceed76d22d9762f3d7eb33262c62d1 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 5104625c35..2b7872fa8c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2130,6 +2130,19 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } +/* +** Change the limit on the amount of the database file that may be +** memory mapped. +*/ +int sqlite3BtreeSetMmapSize(Btree *p, int nMap){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetMmapsize(pBt->pPager, nMap); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + /* ** Change the way data is synced to disk in order to increase or decrease ** how well the database resists damage due to OS crashes and power diff --git a/src/btree.h b/src/btree.h index d4c9fe37d7..f2cca2fc71 100644 --- a/src/btree.h +++ b/src/btree.h @@ -63,6 +63,7 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); +int sqlite3BtreeSetMmapSize(Btree*, int); int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); diff --git a/src/os.c b/src/os.c index b5e918a727..45d8e0c121 100644 --- a/src/os.c +++ b/src/os.c @@ -140,6 +140,9 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } +int sqlite3OsMremap(sqlite3_file *id, i64 iOff, i64 nOld, i64 nNew, void **pp){ + return id->pMethods->xMremap(id, iOff, nOld, nNew, pp); +} /* ** The next group of routines are convenience wrappers around the diff --git a/src/os.h b/src/os.h index 1ec7d4ba11..76a60effdb 100644 --- a/src/os.h +++ b/src/os.h @@ -259,6 +259,7 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); +int sqlite3OsMremap(sqlite3_file *id, i64, i64, i64, void **); /* diff --git a/src/os_unix.c b/src/os_unix.c index 89326783d7..f37f2404ee 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4429,6 +4429,42 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Map, remap or unmap part of the database file. +*/ +static int unixMremap( + sqlite3_file *fd, /* Main database file */ + sqlite3_int64 iOff, /* Offset to start mapping at */ + sqlite3_int64 nOld, /* Size of old mapping, or zero */ + sqlite3_int64 nNew, /* Size of new mapping, or zero */ + void **ppMap /* IN/OUT: Old/new mappings */ +){ + unixFile *p = (unixFile *)fd; /* The underlying database file */ + int rc = SQLITE_OK; /* Return code */ + void *pNew = 0; /* New mapping */ + + assert( iOff==0 ); + + if( nOld!=0 ){ + void *pOld = *ppMap; + munmap(pOld, nOld); + } + + if( nNew>0 ){ + int flags = PROT_READ; + if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + nNew = (nNew+4095) & ~(i64)((1 << 12)-1); + pNew = mmap(0, nNew, flags, MAP_SHARED, p->h, iOff); + if( pNew==MAP_FAILED ){ + pNew = 0; + rc = SQLITE_IOERR; + } + } + + *ppMap = pNew; + return rc; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -4487,7 +4523,8 @@ static const sqlite3_io_methods METHOD = { \ unixShmMap, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ - unixShmUnmap /* xShmUnmap */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixMremap, /* xMremap */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4504,7 +4541,7 @@ static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ IOMETHODS( posixIoFinder, /* Finder function name */ posixIoMethods, /* sqlite3_io_methods object name */ - 2, /* shared memory is enabled */ + 3, /* shared memory and mmap are enabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ diff --git a/src/pager.c b/src/pager.c index d9b9ed5052..9f7b349382 100644 --- a/src/pager.c +++ b/src/pager.c @@ -659,8 +659,11 @@ struct Pager { void *pMap; /* Memory mapped prefix of database file */ i64 nMap; /* Size of mapping at pMap in bytes */ i64 nMapValid; /* Bytes at pMap known to be valid */ + i64 nMapLimit; /* Maximum permitted mapping size */ + int nMapCfgLimit; /* Configured limit value */ int nMmapOut; /* Number of mmap pages currently outstanding */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ + int bMapResize; /* Check if the mapping should be resized */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -3356,6 +3359,35 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } +/* +** Set Pager.nMapLimit, the maximum permitted mapping size, based on the +** current values of Pager.nMapCfgLimit and Pager.pageSize. +** +** If this connection should not use mmap at all, set nMapLimit to zero. +*/ +static void pagerFixMaplimit(Pager *pPager){ + if( isOpen(pPager->fd)==0 + || pPager->fd->pMethods->iVersion<3 + || pPager->fd->pMethods->xMremap==0 + || pPager->tempFile + || pPager->pWal + ){ + pPager->nMapLimit = 0; + }else if( pPager->nMapCfgLimit<0 ){ + pPager->nMapLimit = (i64)pPager->nMapCfgLimit * -1024; + }else{ + pPager->nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + } +} + +/* +** Change the maximum size of any memory mapping made of the database file. +*/ +void sqlite3PagerSetMmapsize(Pager *pPager, int nMap){ + pPager->nMapCfgLimit = nMap; + pagerFixMaplimit(pPager); +} + /* ** Free as much memory as possible from the pager. */ @@ -3591,6 +3623,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; pagerReportSize(pPager); + pagerFixMaplimit(pPager); } return rc; } @@ -3816,108 +3849,100 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } -#include - /* -** Unmap any mapping of the database file. +** Unmap any memory mapping of the database file. */ static int pagerUnmap(Pager *pPager){ if( pPager->pMap ){ - munmap(pPager->pMap, pPager->nMap); - pPager->pMap = 0; + sqlite3OsMremap(pPager->fd, 0, pPager->nMap, 0, &pPager->pMap); pPager->nMap = 0; pPager->nMapValid = 0; } return SQLITE_OK; } +/* +** Create, or recreate, the memory mapping of the database file. +*/ static int pagerMap(Pager *pPager){ - int rc; - i64 sz = 0; - - assert( pPager->pMap==0 && pPager->nMap==0 ); - - rc = sqlite3OsFileSize(pPager->fd, &sz); - sz = sz & ~(4096-1); - - if( rc==SQLITE_OK && sz>0 ){ - int fd; - rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_GETFD, (void *)&fd); + int rc = SQLITE_OK; /* Return code */ + Pgno nPg; /* Size of mapping to request in pages */ + i64 sz; /* Size of mapping to request in bytes */ + + assert( pPager->pWal==0 && isOpen(pPager->fd) && pPager->tempFile==0 ); + assert( pPager->pMap==0 || pPager->nMap>0 ); + assert( pPager->eState>=1 ); + assert( pPager->nMmapOut==0 ); + assert( pPager->nMapLimit>0 ); + + /* Figure out how large a mapping to request. Set variable sz to this + ** value in bytes. */ + nPg = (pPager->eState==1) ? pPager->dbSize : pPager->dbFileSize; + sz = (i64)nPg * pPager->pageSize; + if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; + + if( sz!=pPager->nMapValid ){ + rc = sqlite3OsMremap(pPager->fd, 0, pPager->nMap, sz, &pPager->pMap); if( rc==SQLITE_OK ){ - void *pMap = mmap(0, sz, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); - if( pMap==MAP_FAILED ){ - return SQLITE_IOERR; - } - pPager->pMap = pMap; - pPager->nMapValid = pPager->nMap = sz; + assert( pPager->pMap!=0 ); + pPager->nMap = sz; + }else{ + assert( pPager->pMap==0 ); + pPager->nMap = 0; } + pPager->nMapValid = pPager->nMap; } + pPager->bMapResize = 0; return rc; } -static int pagerRemap(Pager *pPager, Pgno nPage){ - i64 sz = (i64)nPage * pPager->pageSize; - sz = sz & ~(4096-1); - - if( pPager->nMap!=sz ){ - void *pMap = mremap(pPager->pMap, pPager->nMap, sz, MREMAP_MAYMOVE); - if( pMap==MAP_FAILED ){ - return SQLITE_IOERR; - } - pPager->pMap = pMap; - pPager->nMapValid = pPager->nMap = sz; - } - - return SQLITE_OK; -} - +/* +** Obtain a reference to a memory mapped page object for page number pgno. +** The caller must ensure that page pgno lies within the currently mapped +** region. If successful, set *ppPage to point to the new page reference +** and return SQLITE_OK. Otherwise, return an SQLite error code and set +** *ppPage to zero. +** +** Page references obtained by calling this function should be released +** by calling pagerReleaseMapPage(). +*/ static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ - int rc; - *ppPage = 0; - - assert( pPager->pWal==0 ); + PgHdr *p; /* Memory mapped page to return */ - if( MEMDB==0 && pPager->tempFile==0 ){ - if( pPager->pMap==0 ){ - rc = pagerMap(pPager); - if( rc!=SQLITE_OK ) return rc; + if( pPager->pFree ){ + *ppPage = p = pPager->pFree; + pPager->pFree = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ){ + return SQLITE_NOMEM; } + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; + } - if( pgno!=1 && pPager->pMap - && pPager->nMapValid>=((i64)pgno*pPager->pageSize) - ){ - PgHdr *p; - if( pPager->pFree ){ - p = pPager->pFree; - pPager->pFree = p->pDirty; - p->pDirty = 0; - memset(p->pExtra, 0, pPager->nExtra); - }else{ - p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); - if( p==0 ) return SQLITE_NOMEM; - p->pExtra = (void *)&p[1]; - p->flags = PGHDR_MMAP; - p->nRef = 1; - p->pPager = pPager; - } - - assert( p->pExtra==(void *)&p[1] ); - assert( p->pPage==0 ); - assert( p->flags==PGHDR_MMAP ); - assert( p->pPager==pPager ); - assert( p->nRef==1 ); + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); - p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; - p->pgno = pgno; - pPager->nMmapOut++; - *ppPage = p; - } - } + p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; + p->pgno = pgno; + pPager->nMmapOut++; return SQLITE_OK; } +/* +** Release a reference to page pPg. pPg must have been returned by an +** earlier call to pagerAcquireMapPage(). +*/ static void pagerReleaseMapPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; pPager->nMmapOut--; @@ -3925,6 +3950,9 @@ static void pagerReleaseMapPage(PgHdr *pPg){ pPager->pFree = pPg; } +/* +** Free all PgHdr objects stored in the Pager.pFree list. +*/ static void pagerFreeMapHdrs(Pager *pPager){ PgHdr *p; PgHdr *pNext; @@ -5115,7 +5143,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** possibility by unmapping the db here. */ pagerUnmap(pPager); }else if( pPager->pMap ){ - pagerRemap(pPager, nPage); + pPager->bMapResize = 1; } } @@ -5226,8 +5254,7 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = ( - (pgno!=1 && pPager->pWal==0 && !pPager->tempFile && !MEMDB) + const int bMmapOk = (pPager->nMapLimit>0 && pgno!=1 && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) ); @@ -5245,7 +5272,7 @@ int sqlite3PagerAcquire( }else{ if( bMmapOk ){ - if( pPager->pMap==0 ){ + if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ rc = pagerMap(pPager); } if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ @@ -7014,6 +7041,7 @@ static int pagerOpenWal(Pager *pPager){ pPager->journalSizeLimit, &pPager->pWal ); } + pagerFixMaplimit(pPager); return rc; } @@ -7104,6 +7132,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; + pagerFixMaplimit(pPager); } } return rc; diff --git a/src/pragma.c b/src/pragma.c index f6dadbd4a8..9b31797d3f 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -744,6 +744,30 @@ void sqlite3Pragma( } }else + /* + ** PRAGMA [database.]mmap_size + ** PRAGMA [database.]mmap_size=N + ** + ** Used to set or query the mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If it is set to a positive value, then it is + ** interpreted as a maximum size in pages. If set to less than zero, then + ** the absolute value is interpreted as a size limit in KB. + ** + ** The default value is zero (do not use memory mapped IO). + */ + if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !zRight ){ + returnSingleInt(pParse, "mmap_size", pDb->pSchema->mmap_size); + }else{ + int size = sqlite3Atoi(zRight); + pDb->pSchema->mmap_size = size; + sqlite3BtreeSetMmapSize(pDb->pBt, pDb->pSchema->mmap_size); + } + }else + /* ** PRAGMA temp_store ** PRAGMA temp_store = "default"|"memory"|"file" diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0ddb0dd105..3fa30e01de 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -706,6 +706,24 @@ struct sqlite3_file { ** fails to zero-fill short reads might seem to work. However, ** failure to zero-fill short reads will eventually lead to ** database corruption. +** +** Assuming parameter nNew is non-zero, the xMremap method should attempt +** to memory map a region nNew bytes in size starting at offset iOffset +** of the file. If successful, it should set *ppMap to point to the +** mapping and return SQLITE_OK. If the file is opened for read-write +** access, then the mapping should also be read-write. +** +** If nOld is non-zero, then the initial value of *ppMap points to a +** mapping returned by a previous call to xMremap. The existing mapping +** is nOld bytes in size and starts at offset iOffset of the file. In +** this case the xMremap method is expected to unmap the existing mapping +** and overwrite *ppMap with the pointer to the new mapping. If nOld is +** zero, then the initial value of *ppMap is undefined. +** +** If nNew is zero, then no new mapping should be created. Any old +** mapping must still be unmapped if nOld is non-zero. If the nOld +** parameter is non-zero, then the existing mapping is always unmapped - +** even if an error occurs. */ typedef struct sqlite3_io_methods sqlite3_io_methods; struct sqlite3_io_methods { @@ -728,6 +746,9 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xMremap)(sqlite3_file *fd, + sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 129c4c5ea5..59483bc87b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -746,6 +746,7 @@ struct Schema { u8 enc; /* Text encoding used by this database */ u16 flags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ + int mmap_size; /* Number of pages to memory map */ }; /* diff --git a/src/test1.c b/src/test1.c index a89b874a3b..500d60e0fe 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5859,9 +5859,9 @@ static int test_getrusage( getrusage(RUSAGE_SELF, &r); sprintf(buf, "ru_utime=%d.%06d ru_stime=%d.%06d ru_minflt=%d ru_majflt=%d", - r.ru_utime.tv_sec, r.ru_utime.tv_usec, - r.ru_stime.tv_sec, r.ru_stime.tv_usec, - r.ru_minflt, r.ru_majflt + (int)r.ru_utime.tv_sec, (int)r.ru_utime.tv_usec, + (int)r.ru_stime.tv_sec, (int)r.ru_stime.tv_usec, + (int)r.ru_minflt, (int)r.ru_majflt ); Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); return TCL_OK; diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 18bb0870bd..2541a1a823 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -40,6 +40,7 @@ proc db_write {db {reset 0}} { do_test 1.1 { db close sqlite3 db test.db + execsql { PRAGMA mmap_size = 0 } expr {[file size test.db] / 1024} } 6 diff --git a/test/exclusive2.test b/test/exclusive2.test index 2208da5101..54203e3d8f 100644 --- a/test/exclusive2.test +++ b/test/exclusive2.test @@ -25,6 +25,14 @@ ifcapable {!pager_pragmas} { return } +# Tests in this file verify that locking_mode=exclusive causes SQLite to +# use cached pages even if the database is changed on disk. This doesn't +# work with mmap. +if {[permutation]=="mmap"} { + finish_test + return +} + # This module does not work right if the cache spills at unexpected # moments. So disable the soft-heap-limit. # diff --git a/test/func.test b/test/func.test index f09ff49805..4ab7688461 100644 --- a/test/func.test +++ b/test/func.test @@ -1273,11 +1273,13 @@ do_test func-29.3 { sqlite3_db_status db CACHE_MISS 1 db eval {SELECT typeof(+x) FROM t29 ORDER BY id} } {integer null real blob text} -do_test func-29.4 { - set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] - if {$x>100} {set x many} - set x -} {many} +if {[permutation] != "mmap"} { + do_test func-29.4 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x>100} {set x many} + set x + } {many} +} do_test func-29.5 { db close sqlite3 db test.db diff --git a/test/incrblob.test b/test/incrblob.test index 7cc99dd983..f51f5852d4 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -123,6 +123,7 @@ foreach AutoVacuumMode [list 0 1] { forcedelete test.db test.db-journal sqlite3 db test.db + execsql "PRAGMA mmap_size = 0" execsql "PRAGMA auto_vacuum = $AutoVacuumMode" do_test incrblob-2.$AutoVacuumMode.1 { @@ -149,6 +150,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql "PRAGMA mmap_size = 0" # Read the last 20 bytes of the blob via a blob handle. set ::blob [db incrblob blobs v 1] @@ -171,6 +173,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql "PRAGMA mmap_size = 0" # Write the second-to-last 20 bytes of the blob via a blob handle. # @@ -200,6 +203,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql { PRAGMA mmap_size = 0 } execsql { SELECT i FROM blobs } } {45} diff --git a/test/pageropt.test b/test/pageropt.test index 82311965a5..de39bbaf05 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -92,7 +92,7 @@ do_test pageropt-1.5 { pagercount_sql { SELECT hex(x) FROM t1 } -} [list 6 0 0 $blobcontent] +} [list [expr {[permutation]=="mmap" ? 1 : 6}] 0 0 $blobcontent] do_test pageropt-1.6 { pagercount_sql { SELECT hex(x) FROM t1 diff --git a/test/permutations.test b/test/permutations.test index d9014d77fe..fd05b58e4f 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -134,6 +134,14 @@ test_suite "veryquick" -prefix "" -description { "Very" quick test suite. Runs in less than 5 minutes on a workstation. This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. +} -files [ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* +] + +test_suite "mmap" -prefix "mm-" -description { + Similar to veryquick. Except with memory mapping enabled. +} -presql { + pragma mmap_size = -65536; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ multiplex* server1.test shared2.test shared6.test