From: drh Date: Fri, 30 Apr 2010 14:39:50 +0000 (+0000) Subject: Separate the concepts of underlying storage size and mapped size in the X-Git-Tag: version-3.7.2~455^2~27 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5530b764a83f174000cb720b4edf3235c6d989fc;p=thirdparty%2Fsqlite.git Separate the concepts of underlying storage size and mapped size in the VFS shared-memory implementation. FossilOrigin-Name: 4cbe49f13fed288f94ff305bcfd99df907bf7baf --- diff --git a/manifest b/manifest index 2b907e980d..63d18faa80 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C Add\sa\smissing\swalIndexUnmap()\scall\sto\sthe\scheckpoint\scode.\sChange\sa\scouple\sof\sSQLITE_CANTOPEN\sconstants\sto\sSQLITE_CANTOPEN_BKPT. -D 2010-04-30T11:43:29 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Separate\sthe\sconcepts\sof\sunderlying\sstorage\ssize\sand\smapped\ssize\sin\sthe\nVFS\sshared-memory\simplementation. +D 2010-04-30T14:39:51 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -150,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c 7451a717574c9b9809ad7c429345482c29239230 +F src/os_unix.c b2322d8fb389b4bc7834b1596bd8afac776a2f20 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c 434f9751fc2dfc11ade004282deda5f8560bcba2 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -166,7 +169,7 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 -F src/sqlite.h.in a710846e91159742ee3ddcbabb56eb7c7478e3a5 +F src/sqlite.h.in 6e91727c0c3f9e1365e8fea2b07369a09830f47f F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h 32dfe6294b46914fb567db493994bf902fb0f8c9 F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 @@ -221,7 +224,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 9327e0d4e2b19dcf5614d3bb8cdff25ebbe0e290 +F src/wal.c a4c643582fd9956b46dca187161d466f584b47af F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 2e7a0050e192dd197d4db92393ab120897c14836 -R c2c9e5e0c48b0291d2627d6aeb69855a -U dan -Z e7fa4c17c25ce2a31997877294b65d50 +P 1f9e8c5c27ae2c68bc01d8e2bb2c662be7b3f161 +R 2fbe76e11b4a2b6e620f52387007559a +U drh +Z 868ea682fbe4fab342b578dc69caee7c +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFL2uu6oxKgR168RlERAqI3AJ4jq6FSOAaisShm+qyJakBJL+pVmwCgg1Vd +tQBOoEYMwVK9lQZSibwuRzM= +=5suj +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 4ba7670f10..0313645415 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f9e8c5c27ae2c68bc01d8e2bb2c662be7b3f161 \ No newline at end of file +4cbe49f13fed288f94ff305bcfd99df907bf7baf \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index d991ec9d5d..006b0d182d 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4594,11 +4594,9 @@ struct unixShmFile { sqlite3_mutex *mutexBuf; /* Mutex to access zBuf[] */ sqlite3_mutex *mutexRecov; /* The RECOVER mutex */ char *zFilename; /* Name of the file */ - int size; /* Size of the file */ int h; /* Open file descriptor */ - char *pMMapBuf; /* Where currently mmapped() */ - int nReadPrefix; /* Number of SQLITE_SHM_READ_PREFIX locks */ - int nReadFull; /* Number of SQLITE_SHM_READ_FULL locks */ + int szMap; /* Size of the mapping of file into memory */ + char *pMMapBuf; /* Where currently mmapped(). NULL if unmapped */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ unixShmFile *pNext; /* Next in list of all unixShmFile objects */ @@ -4995,8 +4993,6 @@ static int unixShmOpen( } pFile->fid.dev = sStat.st_dev; pFile->fid.ino = sStat.st_ino; - pFile->size = (int)sStat.st_size; - pFile->size = (pFile->size/SQLITE_UNIX_SHM_INCR)*SQLITE_UNIX_SHM_INCR; /* Check to see if another process is holding the dead-man switch. ** If not, truncate the file to zero length. @@ -5010,7 +5006,6 @@ static int unixShmOpen( rc = SQLITE_IOERR; goto shm_open_err; } - pFile->size = 0; } rc = unixShmSystemLock(pFile, F_RDLCK, UNIX_SHM_DMS); if( rc ) goto shm_open_err; @@ -5078,46 +5073,86 @@ static int unixShmClose(sqlite3_shm *pSharedMem){ } /* -** Query and/or changes the size of a shared-memory segment. -** The reqSize parameter is the new size of the segment, or -1 to -** do just a query. The size of the segment after resizing is -** written into pNewSize. A writer lock is held on the shared memory -** segment while resizing it. +** Query and/or changes the size of the underlying storage for +** a shared-memory segment. The reqSize parameter is the new size +** of the underlying storage, or -1 to do just a query. The size +** of the underlying storage (after resizing if resizing occurs) is +** written into pNewSize. ** -** If ppBuffer is not NULL, the a reader lock is acquired on the shared -** memory segment and *ppBuffer is made to point to the start of the -** shared memory segment. xShmRelease() must be called to release the -** lock. +** This routine does not (necessarily) change the size of the mapping +** of the underlying storage into memory. Use xShmGet() to change +** the mapping size. +** +** The reqSize parameter is the minimum size requested. The implementation +** is free to expand the storage to some larger amount if it chooses. */ static int unixShmSize( sqlite3_shm *pSharedMem, /* Pointer returned by unixShmOpen() */ int reqSize, /* Requested size. -1 for query only */ - int *pNewSize, /* Write new size here */ - void **ppBuf /* Write new buffer origin here */ + int *pNewSize /* Write new size here */ ){ unixShm *p = (unixShm*)pSharedMem; unixShmFile *pFile = p->pFile; int rc = SQLITE_OK; + struct stat sStat; - sqlite3_mutex_enter(pFile->mutexBuf); - sqlite3_mutex_enter(pFile->mutex); if( reqSize>=0 ){ reqSize = (reqSize + SQLITE_UNIX_SHM_INCR - 1)/SQLITE_UNIX_SHM_INCR; reqSize *= SQLITE_UNIX_SHM_INCR; - if( reqSize!=pFile->size ){ - if( pFile->pMMapBuf ) munmap(pFile->pMMapBuf, pFile->size); - rc = ftruncate(pFile->h, reqSize); - if( rc ){ - pFile->pMMapBuf = 0; - pFile->size = 0; - }else{ - pFile->pMMapBuf = mmap(0, reqSize, PROT_READ|PROT_WRITE, MAP_SHARED, - pFile->h, 0); - pFile->size = pFile->pMMapBuf ? reqSize : 0; - } + rc = ftruncate(pFile->h, reqSize); + } + if( fstat(pFile->h, &sStat)==0 ){ + *pNewSize = (int)sStat.st_size; + }else{ + *pNewSize = 0; + rc = SQLITE_IOERR; + } + return rc; +} + + +/* +** Map the shared storage into memory. The minimum size of the +** mapping should be reqMapSize if reqMapSize is positive. If +** reqMapSize is zero or negative, the implementation can choose +** whatever mapping size is convenient. +** +** *ppBuf is made to point to the memory which is a mapping of the +** underlying storage. This segment is locked. unixShmRelease() +** must be called to release the lock. +** +** *pNewMapSize is set to the size of the mapping. +** +** *ppBuf and *pNewMapSize might be NULL and zero if no space has +** yet been allocated to the underlying storage. +*/ +static int unixShmGet( + sqlite3_shm *pSharedMem, /* Pointer returned by unixShmOpen() */ + int reqMapSize, /* Requested size of mapping. -1 means don't care */ + int *pNewMapSize, /* Write new size of mapping here */ + void **ppBuf /* Write mapping buffer origin here */ +){ + unixShm *p = (unixShm*)pSharedMem; + unixShmFile *pFile = p->pFile; + int rc = SQLITE_OK; + + sqlite3_mutex_enter(pFile->mutexBuf); + sqlite3_mutex_enter(pFile->mutex); + if( pFile->szMap==0 || reqMapSize>pFile->szMap ){ + int actualSize; + if( unixShmSize(pSharedMem, -1, &actualSize)==SQLITE_OK + && reqMapSizepMMapBuf ){ + munmap(pFile->pMMapBuf, pFile->szMap); } + pFile->pMMapBuf = mmap(0, reqMapSize, PROT_READ|PROT_WRITE, MAP_SHARED, + pFile->h, 0); + pFile->szMap = pFile->pMMapBuf ? reqMapSize : 0; } - *pNewSize = pFile->size; + *pNewMapSize = pFile->szMap; *ppBuf = pFile->pMMapBuf; sqlite3_mutex_leave(pFile->mutex); return rc; @@ -6490,6 +6525,7 @@ int sqlite3_os_init(void){ unixGetLastError, /* xGetLastError */ \ unixShmOpen, /* xShmOpen */ \ unixShmSize, /* xShmSize */ \ + unixShmGet, /* xShmGet */ \ unixShmRelease, /* xShmRelease */ \ 0, /* xShmPush */ \ 0, /* xShmPull */ \ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 35ac5bd008..d2ce5396f2 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -844,7 +844,8 @@ struct sqlite3_vfs { ** definition. Those that follow are added in version 2 or later */ int (*xShmOpen)(sqlite3_vfs*, const char *zName, sqlite3_shm**); - int (*xShmSize)(sqlite3_shm*, int reqSize, int *pNewSize, void**); + int (*xShmSize)(sqlite3_shm*, int reqSize, int *pNewSize); + int (*xShmGet)(sqlite3_shm*, int reqMapSize, int *pMapSize, void**); int (*xShmRelease)(sqlite3_shm*); int (*xShmPush)(sqlite3_shm*); int (*xShmPull)(sqlite3_shm*); diff --git a/src/wal.c b/src/wal.c index 30af69ff6e..033d1c5232 100644 --- a/src/wal.c +++ b/src/wal.c @@ -126,7 +126,7 @@ struct Wal { sqlite3_file *pFd; /* File handle for WAL file */ u32 iCallback; /* Value to pass to log callback (or 0) */ sqlite3_shm *pWIndex; /* The open wal-index file */ - int szWIndex; /* Size of the wal-index */ + int szWIndex; /* Size of the wal-index that is mapped in mem */ u32 *pWiData; /* Pointer to wal-index content in memory */ u8 lockState; /* SQLITE_SHM_xxxx constant showing lock state */ u8 readerType; /* SQLITE_SHM_READ or SQLITE_SHM_READ_FULL */ @@ -368,7 +368,8 @@ static int walIndexEntry(u32 iFrame){ } /* -** Release our reference to the wal-index memory map. +** Release our reference to the wal-index memory map, if we are holding +** it. */ static void walIndexUnmap(Wal *pWal){ if( pWal->pWiData ){ @@ -378,26 +379,42 @@ static void walIndexUnmap(Wal *pWal){ } /* -** Resize the wal-index file. If newSize is negative, leave the size -** unchanged. +** Map the wal-index file into memory if it isn't already. +** +** The reqSize parameter is the minimum required size of the mapping. +** A value of -1 means "don't care". The reqSize parameter is ignored +** if the mapping is already held. */ -static int walIndexRemap(Wal *pWal, int newSize){ - int rc; - walIndexUnmap(pWal); - rc = pWal->pVfs->xShmSize(pWal->pWIndex, newSize, - &pWal->szWIndex, (void**)(char*)&pWal->pWiData); - if( rc==SQLITE_OK && pWal->pWiData==0 ){ - assert( pWal->szWIndex==0 ); - pWal->pWiData = &pWal->iCallback; +static int walIndexMap(Wal *pWal, int reqSize){ + int rc = SQLITE_OK; + if( pWal->pWiData==0 ){ + rc = pWal->pVfs->xShmGet(pWal->pWIndex, reqSize, &pWal->szWIndex, + (void**)(char*)&pWal->pWiData); + if( rc==SQLITE_OK && pWal->pWiData==0 ){ + /* Make sure pWal->pWiData is not NULL while we are holding the + ** lock on the mapping. */ + assert( pWal->szWIndex==0 ); + pWal->pWiData = &pWal->iCallback; + } } return rc; } /* -** Map the wal-index file into memory if it isn't already. +** Remap the wal-index so that the mapping covers the full size +** of the underlying file. +** +** If enlargeTo is non-negative, then increase the size of the underlying +** storage to be at least as big as enlargeTo before remapping. */ -static int walIndexMap(Wal *pWal){ - int rc = walIndexRemap(pWal, -1); +static int walIndexRemap(Wal *pWal, int enlargeTo){ + int rc; + int sz; + rc = pWal->pVfs->xShmSize(pWal->pWIndex, enlargeTo, &sz); + if( rc==SQLITE_OK && sz>pWal->szWIndex ){ + walIndexUnmap(pWal); + rc = walIndexMap(pWal, sz); + } return rc; } @@ -416,12 +433,12 @@ static int walIndexMap(Wal *pWal){ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ u32 iSlot = walIndexEntry(iFrame); - walIndexMap(pWal); + walIndexMap(pWal, -1); while( (iSlot+128)>=pWal->szWIndex ){ int rc; int nByte = pWal->szWIndex*4 + WALINDEX_MMAP_INCREMENT; - /* Unmap and remap the wal-index file. */ + /* Enlarge the storage, then remap it. */ rc = walIndexRemap(pWal, nByte); if( rc!=SQLITE_OK ){ return rc; @@ -640,7 +657,7 @@ static WalIterator *walIteratorInit(Wal *pWal){ struct WalSegment *pFinal; /* Final (unindexed) segment */ u8 *aTmp; /* Temp space used by merge-sort */ - walIndexMap(pWal); + walIndexMap(pWal, -1); aData = pWal->pWiData; iLast = pWal->hdr.iLastPg; nSegment = (iLast >> 8) + 1; @@ -789,7 +806,8 @@ int walIndexTryHdr(Wal *pWal, int *pChanged){ u32 aHdr[WALINDEX_HDR_NFIELD+2]; if( pWal->szWIndex==0 ){ - int rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT); + int rc; + rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT); if( rc ) return rc; } @@ -830,7 +848,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ int rc; assert( pWal->lockState>=SQLITE_SHM_READ ); - walIndexMap(pWal); + walIndexMap(pWal, -1); /* First try to read the header without a lock. Verify the checksum ** before returning. This will almost always work. @@ -881,11 +899,10 @@ int sqlite3WalOpenSnapshot(Wal *pWal, int *pChanged){ sqlite3WalCloseSnapshot(pWal); }else{ /* Check if the mapping needs to grow. */ - if( pWal->hdr.iLastPg - && walIndexEntry(pWal->hdr.iLastPg)>=pWal->szWIndex - ){ - rc = walIndexRemap(pWal, 0); - assert( rc || walIndexEntry(pWal->hdr.iLastPg)szWIndex ); + if( pWal->hdr.iLastPg + && walIndexEntry(pWal->hdr.iLastPg)>=pWal->szWIndex + ){ + walIndexRemap(pWal, -1); } } } @@ -913,7 +930,7 @@ int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut){ int iFrame = (pWal->hdr.iLastPg & 0xFFFFFF00); assert( pWal->lockState==SQLITE_SHM_READ||pWal->lockState==SQLITE_SHM_WRITE ); - walIndexMap(pWal); + walIndexMap(pWal, -1); /* Do a linear search of the unindexed block of page-numbers (if any) ** at the end of the wal-index. An alternative to this would be to