From: dan Date: Wed, 20 Mar 2013 14:26:59 +0000 (+0000) Subject: When possible, use memory mapping when appending new pages to a database file. X-Git-Tag: version-3.7.17~114^2~66 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=eb97b29345592cf094b6639a985f9253653a5063;p=thirdparty%2Fsqlite.git When possible, use memory mapping when appending new pages to a database file. FossilOrigin-Name: 14135da3cdbafd699563a29608f32347cda28338 --- diff --git a/manifest b/manifest index 1a8ba2228d..d519ad3e2a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sfile\smmap1.test. -D 2013-03-20T10:07:43.411 +C When\spossible,\suse\smemory\smapping\swhen\sappending\snew\spages\sto\sa\sdatabase\sfile. +D 2013-03-20T14:26:59.370 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,13 +157,13 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 81a82a736b8a461a656f9b3e401a39768fe73a79 -F src/os.h 4681261aa24a9d2187aaf4cb963880e6cddb1f48 +F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 +F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 0c7b0d076f2ac6279b0b280a26bcae8c89f36f4f +F src/os_unix.c aedd47c2145edd49051452afbe82fa6f54a49b2a F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 78b65bf9685bf21b787ce2a7389c2b96102942dc -F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 +F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 +F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in fd75f5bcf479b315b1c717fa1d07f018bd919f79 +F src/sqlite.h.in d63c7fb5832287af7e8b903c4a4c30c90414876f F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6183f1bd86ceed76d22d9762f3d7eb33262c62d1 -R 1501a2a578b96ab1d025c2aefaaa8f35 +P aee1f53a74e636776cbbc11bdd5516432ad50533 +R 041b09b1e247c8054516280d9526a179 U dan -Z a7aa76ce5b116a8f55025f0891a0dfbb +Z 8c309d7c56b08fcba504ca346091b196 diff --git a/manifest.uuid b/manifest.uuid index 68ed5bb6a8..f40916822a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aee1f53a74e636776cbbc11bdd5516432ad50533 \ No newline at end of file +14135da3cdbafd699563a29608f32347cda28338 \ No newline at end of file diff --git a/src/os.c b/src/os.c index 45d8e0c121..38757dcdb1 100644 --- a/src/os.c +++ b/src/os.c @@ -140,8 +140,15 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } -int sqlite3OsMremap(sqlite3_file *id, i64 iOff, i64 nOld, i64 nNew, void **pp){ - return id->pMethods->xMremap(id, iOff, nOld, nNew, pp); +int sqlite3OsMremap( + sqlite3_file *id, /* Database file handle */ + int flags, /* SQLITE_MREMAP_XXX flags */ + i64 iOff, /* Offset at which mapping(s) start */ + i64 nOld, /* Size of old mapping */ + i64 nNew, /* Size of requested mapping */ + void **pp /* IN/OUT: Pointer to mapped region */ +){ + return id->pMethods->xMremap(id, flags, iOff, nOld, nNew, pp); } /* diff --git a/src/os.h b/src/os.h index 76a60effdb..f08b92dcf0 100644 --- a/src/os.h +++ b/src/os.h @@ -259,7 +259,7 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); -int sqlite3OsMremap(sqlite3_file *id, i64, i64, i64, void **); +int sqlite3OsMremap(sqlite3_file *id, int, i64, i64, i64, void **); /* diff --git a/src/os_unix.c b/src/os_unix.c index f37f2404ee..1c26c6badf 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4434,6 +4434,7 @@ static int unixShmUnmap( */ static int unixMremap( sqlite3_file *fd, /* Main database file */ + int flags, /* Mask of SQLITE_MREMAP_XXX flags */ sqlite3_int64 iOff, /* Offset to start mapping at */ sqlite3_int64 nOld, /* Size of old mapping, or zero */ sqlite3_int64 nNew, /* Size of new mapping, or zero */ @@ -4442,8 +4443,35 @@ static int unixMremap( unixFile *p = (unixFile *)fd; /* The underlying database file */ int rc = SQLITE_OK; /* Return code */ void *pNew = 0; /* New mapping */ + i64 nRnd; /* nNew rounded up to 4096 */ assert( iOff==0 ); + nRnd = (nNew+4095) & ~(i64)((1 << 12)-1); + + /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested + ** mapping (nNew bytes) may be greater than the size of the database file. + ** If this is the case, extend the file on disk using ftruncate(). */ + assert( nNew>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); + if( flags & SQLITE_MREMAP_EXTEND ){ + struct stat statbuf; /* Low-level file information */ + rc = osFstat(p->h, &statbuf); + if( rc==SQLITE_OK && nNew>statbuf.st_size ){ + rc = robust_ftruncate(p->h, nNew); + } + if( rc!=SQLITE_OK ) return rc; + } + +#if defined(_GNU_SOURCE) && defined(__linux__) + if( nRnd && nOld ){ + void *pOld = *ppMap; + *ppMap = pNew = mremap(pOld, nOld, nNew, MREMAP_MAYMOVE); + if( pNew==MAP_FAILED ){ + *ppMap = 0; + return SQLITE_IOERR_MREMAP; + } + return SQLITE_OK; + } +#endif if( nOld!=0 ){ void *pOld = *ppMap; @@ -4453,11 +4481,10 @@ static int unixMremap( if( nNew>0 ){ int flags = PROT_READ; if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - nNew = (nNew+4095) & ~(i64)((1 << 12)-1); - pNew = mmap(0, nNew, flags, MAP_SHARED, p->h, iOff); + pNew = mmap(0, nRnd, flags, MAP_SHARED, p->h, iOff); if( pNew==MAP_FAILED ){ pNew = 0; - rc = SQLITE_IOERR; + rc = SQLITE_IOERR_MREMAP; } } diff --git a/src/pager.c b/src/pager.c index 9f7b349382..2848f9ad05 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3854,7 +3854,7 @@ static int pagerSyncHotJournal(Pager *pPager){ */ static int pagerUnmap(Pager *pPager){ if( pPager->pMap ){ - sqlite3OsMremap(pPager->fd, 0, pPager->nMap, 0, &pPager->pMap); + sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); pPager->nMap = 0; pPager->nMapValid = 0; } @@ -3864,7 +3864,7 @@ static int pagerUnmap(Pager *pPager){ /* ** Create, or recreate, the memory mapping of the database file. */ -static int pagerMap(Pager *pPager){ +static int pagerMap(Pager *pPager, int bExtend){ int rc = SQLITE_OK; /* Return code */ Pgno nPg; /* Size of mapping to request in pages */ i64 sz; /* Size of mapping to request in bytes */ @@ -3882,7 +3882,8 @@ static int pagerMap(Pager *pPager){ if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; if( sz!=pPager->nMapValid ){ - rc = sqlite3OsMremap(pPager->fd, 0, pPager->nMap, sz, &pPager->pMap); + int flags = (bExtend ? SQLITE_MREMAP_EXTEND : 0); + rc = sqlite3OsMremap(pPager->fd, flags, 0, pPager->nMap, sz, &pPager->pMap); if( rc==SQLITE_OK ){ assert( pPager->pMap!=0 ); pPager->nMap = sz; @@ -4246,10 +4247,17 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ + if( rc==SQLITE_OK + && (pList->pDirty ? pPager->dbSize : pList->pgno+1)>pPager->dbHintSize + ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; + + if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ + pPager->dbFileSize = pPager->dbSize; + pagerMap(pPager, 1); + } } while( rc==SQLITE_OK && pList ){ @@ -5273,7 +5281,7 @@ int sqlite3PagerAcquire( if( bMmapOk ){ if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ - rc = pagerMap(pPager); + rc = pagerMap(pPager, 0); } if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ if( pPager->eState>PAGER_READER ){ diff --git a/src/pager.h b/src/pager.h index 94957dac60..81ab30c115 100644 --- a/src/pager.h +++ b/src/pager.h @@ -108,6 +108,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); +void sqlite3PagerSetMmapsize(Pager *, int); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 3fa30e01de..c0a2803787 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -470,6 +470,7 @@ int sqlite3_exec( #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MREMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) @@ -746,12 +747,14 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ - int (*xMremap)(sqlite3_file *fd, + int (*xMremap)(sqlite3_file *fd, int flags, sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; +#define SQLITE_MREMAP_EXTEND 0x0001 /* xMremap call may extend file */ + /* ** CAPI3REF: Standard File Control Opcodes **