From: drh Date: Mon, 14 Nov 2011 01:55:02 +0000 (+0000) Subject: Begin making experimental changes to use mmap() for reading content from X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=387dd1bffc2a81608c44a047d3f52ab084d7c41d;p=thirdparty%2Fsqlite.git Begin making experimental changes to use mmap() for reading content from a database. The code compiles, but crashes on the test suite. FossilOrigin-Name: 09be42d5fa7ef692428579d4d8d48b3316190945 --- diff --git a/manifest b/manifest index 2988a3c3a3..6c0a57aeeb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sversion\snumber\sto\sthe\ssqlite3_pcache_methods2\sobject.\s\sOther\sPCACHE2\ndocumentation\simprovements. -D 2011-11-13T21:44:03.995 +C Begin\smaking\sexperimental\schanges\sto\suse\smmap()\sfor\sreading\scontent\sfrom\na\sdatabase.\s\sThe\scode\scompiles,\sbut\scrashes\son\sthe\stest\ssuite. +D 2011-11-14T01:55:02.731 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -125,7 +125,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 4368158da74d4711888e03264105c5c527d76caf F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 149cccf2134f555583d0825e47b17104aa06cb84 +F src/btree.c 2d943dc55e9828b017a2f3fe696a54ff91f51eaf F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h ea863a819224d3e6845ad1e39954d41558b8cd8b F src/build.c 8af67a08a852ff4c63701963cb1ab7166f577814 @@ -162,17 +162,17 @@ F src/mutex_os2.c 882d735098c07c8c6a5472b8dd66e19675fe117f F src/mutex_unix.c b4f4e923bb8de93ec3f251fadb50855f23df9579 F src/mutex_w32.c 5e54f3ba275bcb5d00248b8c23107df2e2f73e33 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 5d9b02782ed36345348d6fe21d7762ed3a9cfd2a -F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 +F src/os.c ac54d2199b0bd16543a179d8376deccbe2b4f699 +F src/os.h 1e4f937e1315710bd4a34347d7ddffc1f8cb9055 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 4fbb91726165e105c1679a2660f49a3f4c376e4f +F src/os_unix.c f2c4cfdc350f9b1c5b6ff5079b65c875dc5ea35d F src/os_win.c d6cf718667c4d89d930f30caa6cdc7f7753e259a -F src/pager.c db33d4bf1e3e019c34c220971cc6c3aa07c30f54 -F src/pager.h 9f81b08efb06db4ba8be69446e10b005c351373d +F src/pager.c 6b3a7765ac98a32e125a7f0214fe65e9b578aac8 +F src/pager.h a29eabed35775666fbbcdf91d6a6811f42efdd54 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 -F src/pcache.c b9d52e9f844d91c27c161279234f273fc02abc71 -F src/pcache.h c770382f9fae4ca5025c5523bd0aa13cd6ddc6f8 +F src/pcache.c ccbd5b12806d8fd8a4652b26a5be42d8eee1e962 +F src/pcache.h f8a98a37dd4f7c8bb4ffbdbd6284457c6b78b16e F src/pcache1.c 0ac7b63db83a705787f4ababf1e4cff27b5f8064 F src/pragma.c 65d1d63d64f8b7350f28d5ee6d40f7985deccdfe F src/prepare.c ec4989f7f480544bdc4192fe663470d2a2d7d61e @@ -182,7 +182,7 @@ F src/resolve.c 365ab1c870e38596d6869e76fb544fe6e4ffc809 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 80f3ac44a8514b1d107b80f5df4a424ae059d2b6 F src/shell.c 29812a900a780eb0f835c4bc65e216272689def8 -F src/sqlite.h.in 557f4113a649f15d13e566aaa85820509b4daa52 +F src/sqlite.h.in e08f84145c049bef7a235221f1b5dc65a79db9ad F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 F src/sqliteInt.h f87f241b9821a9d466c2711b7345d30ef3624249 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d @@ -699,7 +699,7 @@ F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 7b1167925e9d30a5d1f0701d24812fdda42c3a86 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 -F test/syscall.test 2a922050dbee032f587249b070fb42692f5e1e22 +F test/syscall.test e97f53ebfcc93fe89c61e1a3b5f120f28ec671b7 F test/sysfault.test c79441d88d23696fbec7b147dba98d42a04f523f F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 @@ -974,7 +974,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P ceee03c79a55ea39866758aa76b78b10e5f4246d -R d08ae8acf53f2cb149aeb898f50b49c8 +P 9f839ac05a9f3cfe587d2ccdccd50dac41baedbe +R b9e47a24ac1e99bdba7ee295a969cbe5 +T *branch * mmap-experimental +T *sym-mmap-experimental * +T -sym-experimental-pcache * U drh -Z af6f24cf5b51e20dd59385851602a05e +Z 12fb6642b5e5f67ae442ddd38187142a diff --git a/manifest.uuid b/manifest.uuid index 140730cf5c..ec7f112f30 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f839ac05a9f3cfe587d2ccdccd50dac41baedbe \ No newline at end of file +09be42d5fa7ef692428579d4d8d48b3316190945 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 43933fa27d..9dbf0c493b 100644 --- a/src/btree.c +++ b/src/btree.c @@ -728,6 +728,19 @@ int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ return SQLITE_OK; } +/* +** Make a btree page is writable. +*/ +static int btreeMakePageWriteable(MemPage *pPage){ + int rc; + if( sqlite3PagerIswriteable(pPage->pDbPage) ) return SQLITE_OK; + rc = sqlite3PagerWrite(pPage->pDbPage); + pPage->aData = sqlite3PagerGetData(pPage->pDbPage); + pPage->aDataEnd = &pPage->aData[pPage->pBt->usableSize]; + pPage->aCellIdx = &pPage->aData[pPage->cellOffset]; + return rc; +} + #ifndef SQLITE_OMIT_AUTOVACUUM /* ** Given a page number of a regular database page, return the page @@ -1545,7 +1558,7 @@ static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ ** If the noContent flag is set, it means that we do not care about ** the content of the page at this time. So do not go to the disk ** to fetch the content. Just fill in the content with zeros for now. -** If in the future we call sqlite3PagerWrite() on this page, that +** If in the future we call btreeMakePageWriteable() on this page, that ** means we have started to be concerned about content and the disk ** read should occur at that point. */ @@ -2482,7 +2495,7 @@ static int newDatabase(BtShared *pBt){ } pP1 = pBt->pPage1; assert( pP1!=0 ); - rc = sqlite3PagerWrite(pP1->pDbPage); + rc = btreeMakePageWriteable(pP1); if( rc ) return rc; data = pP1->aData; memcpy(data, zMagicHeader, sizeof(zMagicHeader)); @@ -2655,7 +2668,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ ** rollback occurs within the transaction. */ if( pBt->nPage!=get4byte(&pPage1->aData[28]) ){ - rc = sqlite3PagerWrite(pPage1->pDbPage); + rc = btreeMakePageWriteable(pPage1); if( rc==SQLITE_OK ){ put4byte(&pPage1->aData[28], pBt->nPage); } @@ -2854,7 +2867,7 @@ static int relocatePage( if( rc!=SQLITE_OK ){ return rc; } - rc = sqlite3PagerWrite(pPtrPage->pDbPage); + rc = btreeMakePageWriteable(pPtrPage); if( rc!=SQLITE_OK ){ releasePage(pPtrPage); return rc; @@ -2956,7 +2969,7 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ }while( nFin!=0 && iFreePg>nFin ); assert( iFreePgpDbPage); + rc = btreeMakePageWriteable(pLastPg); if( rc==SQLITE_OK ){ rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, nFin!=0); } @@ -2976,7 +2989,7 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ if( rc!=SQLITE_OK ){ return rc; } - rc = sqlite3PagerWrite(pPg->pDbPage); + rc = btreeMakePageWriteable(pPg); releasePage(pPg); if( rc!=SQLITE_OK ){ return rc; @@ -3010,7 +3023,7 @@ int sqlite3BtreeIncrVacuum(Btree *p){ invalidateAllOverflowCache(pBt); rc = incrVacuumStep(pBt, 0, btreePagecount(pBt)); if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + rc = btreeMakePageWriteable(pBt->pPage1); put4byte(&pBt->pPage1->aData[28], pBt->nPage); } } @@ -3068,7 +3081,7 @@ static int autoVacuumCommit(BtShared *pBt){ rc = incrVacuumStep(pBt, nFin, iFree); } if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + rc = btreeMakePageWriteable(pBt->pPage1); put4byte(&pBt->pPage1->aData[32], 0); put4byte(&pBt->pPage1->aData[36], 0); put4byte(&pBt->pPage1->aData[28], nFin); @@ -3848,7 +3861,7 @@ static int accessPayload( a = pCur->info.nLocal - offset; } if( eOp ){ - if( (rc = sqlite3PagerWrite(pPage->pDbPage))!=SQLITE_OK ) return rc; + if( (rc = btreeMakePageWriteable(pPage))!=SQLITE_OK ) return rc; getCellInfo(pCur); aPayload = pCur->info.pCell + pCur->info.nHeader; memcpy(aPayload+offset, pBuf, a); @@ -4776,7 +4789,7 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ /* ** Allocate a new page from the database file. ** -** The new page is marked as dirty. (In other words, sqlite3PagerWrite() +** The new page is marked as dirty. (In other words, btreeMakePageWriteable() ** has already been called on the new page.) The new page has also ** been referenced and the calling routine is responsible for calling ** sqlite3PagerUnref() on the new page when it is done. @@ -4843,7 +4856,7 @@ static int allocateBtreePage( /* Decrement the free-list count by 1. Set iTrunk to the index of the ** first free-list trunk page. iPrevTrunk is initially 1. */ - rc = sqlite3PagerWrite(pPage1->pDbPage); + rc = btreeMakePageWriteable(pPage1); if( rc ) return rc; put4byte(&pPage1->aData[36], n-1); @@ -4877,7 +4890,7 @@ static int allocateBtreePage( ** So extract the trunk page itself and use it as the newly ** allocated page */ assert( pPrevTrunk==0 ); - rc = sqlite3PagerWrite(pTrunk->pDbPage); + rc = btreeMakePageWriteable(pTrunk); if( rc ){ goto end_allocate_page; } @@ -4898,7 +4911,7 @@ static int allocateBtreePage( assert( *pPgno==iTrunk ); *ppPage = pTrunk; searchList = 0; - rc = sqlite3PagerWrite(pTrunk->pDbPage); + rc = btreeMakePageWriteable(pTrunk); if( rc ){ goto end_allocate_page; } @@ -4906,7 +4919,7 @@ static int allocateBtreePage( if( !pPrevTrunk ){ memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); }else{ - rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + rc = btreeMakePageWriteable(pPrevTrunk); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -4928,7 +4941,7 @@ static int allocateBtreePage( if( rc!=SQLITE_OK ){ goto end_allocate_page; } - rc = sqlite3PagerWrite(pNewTrunk->pDbPage); + rc = btreeMakePageWriteable(pNewTrunk); if( rc!=SQLITE_OK ){ releasePage(pNewTrunk); goto end_allocate_page; @@ -4941,7 +4954,7 @@ static int allocateBtreePage( assert( sqlite3PagerIswriteable(pPage1->pDbPage) ); put4byte(&pPage1->aData[32], iNewTrunk); }else{ - rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + rc = btreeMakePageWriteable(pPrevTrunk); if( rc ){ goto end_allocate_page; } @@ -4985,7 +4998,7 @@ static int allocateBtreePage( TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d" ": %d more free pages\n", *pPgno, closest+1, k, pTrunk->pgno, n-1)); - rc = sqlite3PagerWrite(pTrunk->pDbPage); + rc = btreeMakePageWriteable(pTrunk); if( rc ) goto end_allocate_page; if( closestpDbPage); + rc = btreeMakePageWriteable(*ppPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); } @@ -5008,7 +5021,7 @@ static int allocateBtreePage( }else{ /* There are no pages on the freelist, so create a new page at the ** end of the file */ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + rc = btreeMakePageWriteable(pBt->pPage1); if( rc ) return rc; pBt->nPage++; if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ) pBt->nPage++; @@ -5024,7 +5037,7 @@ static int allocateBtreePage( assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); rc = btreeGetPage(pBt, pBt->nPage, &pPg, 1); if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pPg->pDbPage); + rc = btreeMakePageWriteable(pPg); releasePage(pPg); } if( rc ) return rc; @@ -5038,7 +5051,7 @@ static int allocateBtreePage( assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); rc = btreeGetPage(pBt, *pPgno, ppPage, 1); if( rc ) return rc; - rc = sqlite3PagerWrite((*ppPage)->pDbPage); + rc = btreeMakePageWriteable(*ppPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); } @@ -5095,7 +5108,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ } /* Increment the free page count on pPage1 */ - rc = sqlite3PagerWrite(pPage1->pDbPage); + rc = btreeMakePageWriteable(pPage1); if( rc ) goto freepage_out; nFree = get4byte(&pPage1->aData[36]); put4byte(&pPage1->aData[36], nFree+1); @@ -5105,7 +5118,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** always fully overwrite deleted information with zeros. */ if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) - || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) + || ((rc = btreeMakePageWriteable(pPage))!=0) ){ goto freepage_out; } @@ -5157,7 +5170,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** to 3.6.0 or later) we should consider fixing the conditional above ** to read "usableSize/4-2" instead of "usableSize/4-8". */ - rc = sqlite3PagerWrite(pTrunk->pDbPage); + rc = btreeMakePageWriteable(pTrunk); if( rc==SQLITE_OK ){ put4byte(&pTrunk->aData[4], nLeaf+1); put4byte(&pTrunk->aData[8+nLeaf*4], iPage); @@ -5180,7 +5193,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ goto freepage_out; } - rc = sqlite3PagerWrite(pPage->pDbPage); + rc = btreeMakePageWriteable(pPage); if( rc!=SQLITE_OK ){ goto freepage_out; } @@ -5540,7 +5553,7 @@ static void insertCell( pPage->aOvfl[j].pCell = pCell; pPage->aOvfl[j].idx = (u16)i; }else{ - int rc = sqlite3PagerWrite(pPage->pDbPage); + int rc = btreeMakePageWriteable(pPage); if( rc!=SQLITE_OK ){ *pRC = rc; return; @@ -6209,7 +6222,7 @@ static int balance_nonroot( if( ipDbPage); + rc = btreeMakePageWriteable(pNew); nNew++; if( rc ) goto balance_cleanup; }else{ @@ -6538,7 +6551,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ ** page that will become the new right-child of pPage. Copy the contents ** of the node stored on pRoot into the new child page. */ - rc = sqlite3PagerWrite(pRoot->pDbPage); + rc = btreeMakePageWriteable(pRoot); if( rc==SQLITE_OK ){ rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); copyNodeContent(pRoot, pChild, &rc); @@ -6616,7 +6629,7 @@ static int balance(BtCursor *pCur){ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; - rc = sqlite3PagerWrite(pParent->pDbPage); + rc = btreeMakePageWriteable(pParent); if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_QUICKBALANCE if( pPage->hasData @@ -6793,7 +6806,7 @@ int sqlite3BtreeInsert( if( loc==0 ){ u16 szOld; assert( idxnCell ); - rc = sqlite3PagerWrite(pPage->pDbPage); + rc = btreeMakePageWriteable(pPage); if( rc ){ goto end_insert; } @@ -6909,7 +6922,7 @@ int sqlite3BtreeDelete(BtCursor *pCur){ */ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; - rc = sqlite3PagerWrite(pPage->pDbPage); + rc = btreeMakePageWriteable(pPage); if( rc ) return rc; rc = clearCell(pPage, pCell); dropCell(pPage, iCellIdx, cellSizePtr(pPage, pCell), &rc); @@ -6933,7 +6946,7 @@ int sqlite3BtreeDelete(BtCursor *pCur){ allocateTempSpace(pBt); pTmp = pBt->pTmpSpace; - rc = sqlite3PagerWrite(pLeaf->pDbPage); + rc = btreeMakePageWriteable(pLeaf); insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); if( rc ) return rc; @@ -7070,7 +7083,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ if( rc!=SQLITE_OK ){ return rc; } - rc = sqlite3PagerWrite(pRoot->pDbPage); + rc = btreeMakePageWriteable(pRoot); if( rc!=SQLITE_OK ){ releasePage(pRoot); return rc; @@ -7162,7 +7175,7 @@ static int clearDatabasePage( } if( freePageFlag ){ freePage(pPage, &rc); - }else if( (rc = sqlite3PagerWrite(pPage->pDbPage))==0 ){ + }else if( (rc = btreeMakePageWriteable(pPage))==0 ){ zeroPage(pPage, pPage->aData[0] | PTF_LEAF); } @@ -7379,7 +7392,7 @@ int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ sqlite3BtreeEnter(p); assert( p->inTrans==TRANS_WRITE ); assert( pBt->pPage1!=0 ); - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + rc = btreeMakePageWriteable(pBt->pPage1); if( rc==SQLITE_OK ){ put4byte(&pBt->pPage1->aData[36 + idx*4], iMeta); #ifndef SQLITE_OMIT_AUTOVACUUM @@ -8208,7 +8221,7 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){ rc = sqlite3BtreeBeginTrans(pBtree, 2); if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + rc = btreeMakePageWriteable(pBt->pPage1); if( rc==SQLITE_OK ){ aData = pBt->pPage1->aData; aData[18] = (u8)iVersion; diff --git a/src/os.c b/src/os.c index 0b13c86e9e..2a28428540 100644 --- a/src/os.c +++ b/src/os.c @@ -199,6 +199,52 @@ int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ return rc; } +/* Attempt to map all or part of a file into memory. VFSes are not +** required to implement this. The VFS might be an older version (less then +** 3) that does not have an xMap pointer. Or the xMap pointer might be NULL. +*/ +int sqlite3OsMap( + sqlite3_file *pFile, /* The file to be mapped into memory */ + sqlite3_int64 ofst, /* Index of the first byte to map */ + sqlite3_int64 len, /* Number of bytes to be mapped */ + int mmapFlags, /* Map control flags */ + void **ppMemObj, /* Write a mapping object here */ + void **ppMem /* Write the start of the mapped file here */ +){ + int rc; + + /* The current implementation only does read-only mmap. This could change + ** in the future. */ + assert( mmapFlags==SQLITE_OPEN_READONLY ); + + /* The current implementation currently only maps the whole file. This + ** could change in the future. */ + assert( ofst==0 ); + + if( pFile->pMethods==0 || pFile->pMethods->iVersion<3 + || pFile->pMethods->xMap==0 ){ + *ppMemObj = 0; + *ppMem = 0; + rc = SQLITE_CANTOPEN; + }else{ + rc = pFile->pMethods->xMap(pFile, ofst, len, mmapFlags, ppMemObj, ppMem); + } + return rc; +} + +/* Undo a mapping. +** +** The pMemObj parameter will have been obtained by a prior call to +** sqlite3OsMap(). So if pMemObj is not NULL, we know that the current +** VFS does support xMap and xUnmap. +*/ +int sqlite3OsUnmap(sqlite3_file *pFile, void *pMemObj){ + int rc = SQLITE_OK; + if( pMemObj ) rc = pFile->pMethods->xUnmap(pFile, pMemObj); + return rc; +} + + int sqlite3OsOpenMalloc( sqlite3_vfs *pVfs, const char *zFile, diff --git a/src/os.h b/src/os.h index 7f17c203ef..e721d21463 100644 --- a/src/os.h +++ b/src/os.h @@ -251,6 +251,8 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); +int sqlite3OsMap(sqlite3_file*,sqlite3_int64,sqlite3_int64,int,void**,void**); +int sqlite3OsUnmap(sqlite3_file*,void*); /* ** Functions for accessing sqlite3_vfs methods diff --git a/src/os_unix.c b/src/os_unix.c index 51d144906a..1bb7b2df49 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -119,9 +119,7 @@ #include #include #include -#ifndef SQLITE_OMIT_WAL #include -#endif #if SQLITE_ENABLE_LOCKING_STYLE # include @@ -413,6 +411,14 @@ static struct unix_syscall { { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, #define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#define osMmap ((int(*)(void*,size_t,int,int,int,off_t))aSyscall[20].pCurrent) + + { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, +#define osMunmap ((int(*)(void*,size_t))aSyscall[21].pCurrent) + + + }; /* End of the overrideable system calls */ /* @@ -4229,6 +4235,63 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** An object used to record enough information about a file mapping to +** undo that mapping. +*/ +struct unixMapping { + sqlite3_int64 len; + void *p; +}; + +/* +** Try to map some or all of a file into memory +*/ +static int unixMap( + sqlite3_file *pFile, /* File to be mapped */ + sqlite3_int64 ofst, /* Offset of start of section to be mapped */ + sqlite3_int64 len, /* Length of the section to be mapped */ + int mmapFlags, /* Flags controlling the mapping */ + void **ppMapObj, /* Write here an object to undo the mapping */ + void **ppMem /* Write here a pointer to the mapped file */ +){ + struct unixMapping *pNew; + unixFile *pUFile = (unixFile*)pFile; + + assert( mmapFlags==SQLITE_OPEN_READONLY ); + sqlite3BeginBenignMalloc(); + pNew = sqlite3_malloc( sizeof(*pNew) ); + sqlite3EndBenignMalloc(); + if( pNew==0 ){ + *ppMapObj = 0; + *ppMem = 0; + return SQLITE_CANTOPEN; + } + pNew->len = len; + pNew->p = *ppMem = mmap(0, len, PROT_READ, MAP_SHARED, pUFile->h, 0); + if( pNew->p==0 ){ + sqlite3_free(pNew); + return SQLITE_CANTOPEN; + }else{ + *ppMapObj = pNew; + return SQLITE_OK; + } +} + +/* +** Undo a prior memory mapping. +*/ +static int unixUnmap( + sqlite3_file *pFile, + void *pMapObj +){ + struct unixMapping *pMap = (struct unixMapping*)pMapObj; + assert( pMap!=0 ); + munmap(pMap->p, pMap->len); + sqlite3_free(pMap); + return SQLITE_OK; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -4269,9 +4332,9 @@ static int unixShmUnmap( ** * An I/O method finder function called FINDER that returns a pointer ** to the METHOD object in the previous bullet. */ -#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK) \ +#define IOMETHODS(FINDER, METHOD, CLOSE, LOCK, UNLOCK, CKLOCK, SHMMAP) \ static const sqlite3_io_methods METHOD = { \ - VERSION, /* iVersion */ \ + 3, /* iVersion */ \ CLOSE, /* xClose */ \ unixRead, /* xRead */ \ unixWrite, /* xWrite */ \ @@ -4284,10 +4347,12 @@ static const sqlite3_io_methods METHOD = { \ unixFileControl, /* xFileControl */ \ unixSectorSize, /* xSectorSize */ \ unixDeviceCharacteristics, /* xDeviceCapabilities */ \ - unixShmMap, /* xShmMap */ \ + SHMMAP, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ - unixShmUnmap /* xShmUnmap */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixMap, /* xMap */ \ + unixUnmap /* xUnmap */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4304,40 +4369,40 @@ static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ IOMETHODS( posixIoFinder, /* Finder function name */ posixIoMethods, /* sqlite3_io_methods object name */ - 2, /* shared memory is enabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ - unixCheckReservedLock /* xCheckReservedLock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + unixShmMap /* Shared memory enabled */ ) IOMETHODS( nolockIoFinder, /* Finder function name */ nolockIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ nolockClose, /* xClose method */ nolockLock, /* xLock method */ nolockUnlock, /* xUnlock method */ - nolockCheckReservedLock /* xCheckReservedLock method */ + nolockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* Shared memory disabled */ ) IOMETHODS( dotlockIoFinder, /* Finder function name */ dotlockIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ dotlockClose, /* xClose method */ dotlockLock, /* xLock method */ dotlockUnlock, /* xUnlock method */ - dotlockCheckReservedLock /* xCheckReservedLock method */ + dotlockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* Shared memory disabled */ ) #if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS IOMETHODS( flockIoFinder, /* Finder function name */ flockIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ flockClose, /* xClose method */ flockLock, /* xLock method */ flockUnlock, /* xUnlock method */ - flockCheckReservedLock /* xCheckReservedLock method */ + flockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* Shared memory disabled */ ) #endif @@ -4345,11 +4410,11 @@ IOMETHODS( IOMETHODS( semIoFinder, /* Finder function name */ semIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ semClose, /* xClose method */ semLock, /* xLock method */ semUnlock, /* xUnlock method */ - semCheckReservedLock /* xCheckReservedLock method */ + semCheckReservedLock, /* xCheckReservedLock method */ + 0 /* Shared memory disabled */ ) #endif @@ -4357,11 +4422,11 @@ IOMETHODS( IOMETHODS( afpIoFinder, /* Finder function name */ afpIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ afpClose, /* xClose method */ afpLock, /* xLock method */ afpUnlock, /* xUnlock method */ - afpCheckReservedLock /* xCheckReservedLock method */ + afpCheckReservedLock, /* xCheckReservedLock method */ + 0 /* Shared memory disabled */ ) #endif @@ -4382,11 +4447,11 @@ static int proxyCheckReservedLock(sqlite3_file*, int*); IOMETHODS( proxyIoFinder, /* Finder function name */ proxyIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ proxyClose, /* xClose method */ proxyLock, /* xLock method */ proxyUnlock, /* xUnlock method */ - proxyCheckReservedLock /* xCheckReservedLock method */ + proxyCheckReservedLock, /* xCheckReservedLock method */ + 0 /* Shared memory disabled */ ) #endif @@ -4395,11 +4460,11 @@ IOMETHODS( IOMETHODS( nfsIoFinder, /* Finder function name */ nfsIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ nfsUnlock, /* xUnlock method */ - unixCheckReservedLock /* xCheckReservedLock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + 0 /* Shared memory disabled */ ) #endif @@ -6771,7 +6836,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==20 ); + assert( ArraySize(aSyscall)==22 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/src/pager.c b/src/pager.c index 6900610b20..9e2323a181 100644 --- a/src/pager.c +++ b/src/pager.c @@ -655,6 +655,9 @@ struct Pager { PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ + u8 *aFileContent; /* File mapped into memory */ + sqlite3_int64 nFileContent; /* Bytes of memory mapped into aFileContent */ + void *pMapObject; /* Used to unmap the file */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -1782,17 +1785,22 @@ static void pager_unlock(Pager *pPager){ ** it can safely move back to PAGER_OPEN state. This happens in both ** normal and exclusive-locking mode. */ - if( pPager->errCode ){ + if( pPager->errCode || pPager->pMapObject ){ assert( !MEMDB ); pager_reset(pPager); pPager->changeCountDone = pPager->tempFile; pPager->eState = PAGER_OPEN; pPager->errCode = SQLITE_OK; + sqlite3OsUnmap(pPager->fd, pPager->pMapObject); + pPager->pMapObject = 0; + pPager->aFileContent = 0; + pPager->nFileContent = 0; } pPager->journalOff = 0; pPager->journalHdr = 0; pPager->setMaster = 0; + } /* @@ -2800,22 +2808,21 @@ static int readDbPage(PgHdr *pPg){ assert( pPager->eState>=PAGER_READER && !MEMDB ); assert( isOpen(pPager->fd) ); - - if( NEVER(!isOpen(pPager->fd)) ){ - assert( pPager->tempFile ); - memset(pPg->pData, 0, pPager->pageSize); - return SQLITE_OK; - } + assert( pPg->pBuf==pPg->pData ); if( pagerUseWal(pPager) ){ /* Try to pull the page from the write-ahead log. */ rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); } if( rc==SQLITE_OK && !isInWal ){ - i64 iOffset = (pgno-1)*(i64)pPager->pageSize; - rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; + i64 iOffset = (pgno-1)*(i64)pgsz; + if( iOffset+pgsz <= pPager->nFileContent ){ + pPg->pData = &pPager->aFileContent[iOffset]; + }else{ + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } } } @@ -4892,6 +4899,13 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerPagecount(pPager, &pPager->dbSize); } + assert( pPager->aFileContent==0 ); + pPager->nFileContent = pPager->dbSize*(sqlite3_int64)pPager->pageSize; + sqlite3OsMap(pPager->fd, 0, pPager->nFileContent, + SQLITE_OPEN_READONLY, (void**)&pPager->pMapObject, + (void**)&pPager->aFileContent); + if( pPager->aFileContent==0 ) pPager->nFileContent = 0; + failed: if( rc!=SQLITE_OK ){ assert( !MEMDB ); @@ -5325,6 +5339,12 @@ static int pager_write(PgHdr *pPg){ assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); assert( assert_pager_state(pPager) ); + /* Make sure page content is held in malloced memory */ + if( pPg->pData!=pPg->pBuf ){ + memcpy(pPg->pBuf, pPg->pData, pPager->pageSize); + pData = pPg->pData = pPg->pBuf; + } + /* Mark the page as dirty. If the page has already been written ** to the journal then we can return right away. */ @@ -5525,11 +5545,9 @@ int sqlite3PagerWrite(DbPage *pDbPage){ ** to sqlite3PagerWrite(). In other words, return TRUE if it is ok ** to change the content of the page. */ -#ifndef NDEBUG int sqlite3PagerIswriteable(DbPage *pPg){ return pPg->flags&PGHDR_DIRTY; } -#endif /* ** A call to this routine tells the pager that it is not necessary to diff --git a/src/pager.h b/src/pager.h index e36e6c2e86..553cf2954b 100644 --- a/src/pager.h +++ b/src/pager.h @@ -168,8 +168,8 @@ void *sqlite3PagerCodec(DbPage *); /* Functions to support testing and debugging. */ #if !defined(NDEBUG) || defined(SQLITE_TEST) Pgno sqlite3PagerPagenumber(DbPage*); - int sqlite3PagerIswriteable(DbPage*); #endif +int sqlite3PagerIswriteable(DbPage*); #ifdef SQLITE_TEST int *sqlite3PagerStats(Pager*); void sqlite3PagerRefdump(Pager*); diff --git a/src/pcache.c b/src/pcache.c index a504633919..c8b46ace07 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -287,7 +287,7 @@ int sqlite3PcacheFetch( if( !pPgHdr->pPage ){ memset(pPgHdr, 0, sizeof(PgHdr)); pPgHdr->pPage = pPage; - pPgHdr->pData = pPage->pBuf; + pPgHdr->pData = pPgHdr->pBuf = pPage->pBuf; pPgHdr->pExtra = (void *)&pPgHdr[1]; memset(pPgHdr->pExtra, 0, pCache->szExtra); pPgHdr->pCache = pCache; @@ -295,7 +295,7 @@ int sqlite3PcacheFetch( } assert( pPgHdr->pCache==pCache ); assert( pPgHdr->pgno==pgno ); - assert( pPgHdr->pData==pPage->pBuf ); + assert( pPgHdr->pBuf==pPage->pBuf ); assert( pPgHdr->pExtra==(void *)&pPgHdr[1] ); if( 0==pPgHdr->nRef ){ diff --git a/src/pcache.h b/src/pcache.h index c4a25b251f..87be12fcf2 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -24,7 +24,8 @@ typedef struct PCache PCache; */ struct PgHdr { sqlite3_pcache_page *pPage; /* Pcache object page handle */ - void *pData; /* Page data */ + void *pData; /* Page data to actually use */ + void *pBuf; /* Malloced buffer to hold pData */ void *pExtra; /* Extra content */ PgHdr *pDirty; /* Transient list of dirty pages */ Pgno pgno; /* Page number for this page */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 27c2b75db3..6ff25ecb8e 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -689,6 +689,10 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xMap)(sqlite3_file*, sqlite3_int64 ofst, sqlite3_int64 len, + int mmapFlags, void **ppMemObj, void **ppMem); + int (*xUnmap)(sqlite3_file*, void *pMemObj); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; diff --git a/test/syscall.test b/test/syscall.test index b67bead7da..08e304aee2 100644 --- a/test/syscall.test +++ b/test/syscall.test @@ -60,6 +60,7 @@ foreach s { open close access getcwd stat fstat ftruncate fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir + mmap munmap } { if {[test_syscall exists $s]} {lappend syscall_list $s} }