From: drh Date: Mon, 14 Nov 2011 12:34:15 +0000 (+0000) Subject: Further work on mmap(). Still does not work right - autovacuum tests are the X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fheads%2Fmmap-experimental;p=thirdparty%2Fsqlite.git Further work on mmap(). Still does not work right - autovacuum tests are the first to fail. FossilOrigin-Name: 638a39bbaa034a08d73459a36bbe14b76b34189f --- diff --git a/manifest b/manifest index 6c0a57aeeb..0a35f476fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\smaking\sexperimental\schanges\sto\suse\smmap()\sfor\sreading\scontent\sfrom\na\sdatabase.\s\sThe\scode\scompiles,\sbut\scrashes\son\sthe\stest\ssuite. -D 2011-11-14T01:55:02.731 +C Further\swork\son\smmap().\s\sStill\sdoes\snot\swork\sright\s-\sautovacuum\stests\sare\sthe\nfirst\sto\sfail. +D 2011-11-14T12:34:15.531 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -125,7 +125,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 4368158da74d4711888e03264105c5c527d76caf F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 2d943dc55e9828b017a2f3fe696a54ff91f51eaf +F src/btree.c c683806cd69ae845d5093d6e2906a7a73a2d9775 F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h ea863a819224d3e6845ad1e39954d41558b8cd8b F src/build.c 8af67a08a852ff4c63701963cb1ab7166f577814 @@ -168,10 +168,10 @@ F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c f2c4cfdc350f9b1c5b6ff5079b65c875dc5ea35d F src/os_win.c d6cf718667c4d89d930f30caa6cdc7f7753e259a -F src/pager.c 6b3a7765ac98a32e125a7f0214fe65e9b578aac8 -F src/pager.h a29eabed35775666fbbcdf91d6a6811f42efdd54 +F src/pager.c 196b112bb00473f719f49fa777f609c135fdd886 +F src/pager.h 13fd98ed10e7d23b23721dd2eafa474c26c9a234 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 -F src/pcache.c ccbd5b12806d8fd8a4652b26a5be42d8eee1e962 +F src/pcache.c e9246c20744e13268e7e02230446bc748c5cc470 F src/pcache.h f8a98a37dd4f7c8bb4ffbdbd6284457c6b78b16e F src/pcache1.c 0ac7b63db83a705787f4ababf1e4cff27b5f8064 F src/pragma.c 65d1d63d64f8b7350f28d5ee6d40f7985deccdfe @@ -974,10 +974,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 9f839ac05a9f3cfe587d2ccdccd50dac41baedbe -R b9e47a24ac1e99bdba7ee295a969cbe5 -T *branch * mmap-experimental -T *sym-mmap-experimental * -T -sym-experimental-pcache * +P 09be42d5fa7ef692428579d4d8d48b3316190945 +R befae33617c94c7a45bbbb00f51b97b6 U drh -Z 12fb6642b5e5f67ae442ddd38187142a +Z ea236fcfb430914e9125ff2cd702c553 diff --git a/manifest.uuid b/manifest.uuid index ec7f112f30..a9d3eb8f63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09be42d5fa7ef692428579d4d8d48b3316190945 \ No newline at end of file +638a39bbaa034a08d73459a36bbe14b76b34189f \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9dbf0c493b..54c3d302d3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -728,16 +728,23 @@ int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ return SQLITE_OK; } +/* +** Set up the correct data pointers for a MemPage +*/ +static u8 *btreeGetData(MemPage *pPage){ + pPage->aData = sqlite3PagerGetData(pPage->pDbPage); + pPage->aDataEnd = &pPage->aData[pPage->pBt->usableSize]; + pPage->aCellIdx = &pPage->aData[pPage->cellOffset]; + return pPage->aData; +} + /* ** Make a btree page is writable. */ static int btreeMakePageWriteable(MemPage *pPage){ int rc; - if( sqlite3PagerIswriteable(pPage->pDbPage) ) return SQLITE_OK; rc = sqlite3PagerWrite(pPage->pDbPage); - pPage->aData = sqlite3PagerGetData(pPage->pDbPage); - pPage->aDataEnd = &pPage->aData[pPage->pBt->usableSize]; - pPage->aCellIdx = &pPage->aData[pPage->cellOffset]; + btreeGetData(pPage); return rc; } @@ -810,6 +817,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); *pRC= rc = sqlite3PagerWrite(pDbPage); + pPtrmap = sqlite3PagerGetData(pDbPage); if( rc==SQLITE_OK ){ pPtrmap[offset] = eType; put4byte(&pPtrmap[offset+1], parent); @@ -1648,7 +1656,7 @@ static void releasePage(MemPage *pPage){ assert( pPage->aData ); assert( pPage->pBt ); assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); - assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + /* assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); sqlite3PagerUnref(pPage->pDbPage); } @@ -2323,6 +2331,7 @@ static int lockBtree(BtShared *pBt){ int nPage; /* Number of pages in the database */ int nPageFile = 0; /* Number of pages in the database file */ int nPageHeader; /* Number of pages in the database according to hdr */ + u8 *page1; /* Content of page 1 */ assert( sqlite3_mutex_held(pBt->mutex) ); assert( pBt->pPage1==0 ); @@ -2334,15 +2343,15 @@ static int lockBtree(BtShared *pBt){ /* Do some checking to help insure the file we opened really is ** a valid database file. */ - nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData); + page1 = btreeGetData(pPage1); + nPage = nPageHeader = get4byte(&page1[28]); sqlite3PagerPagecount(pBt->pPager, &nPageFile); - if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){ + if( nPage==0 || memcmp(24+page1, 92+page1, 4)!=0 ){ nPage = nPageFile; } if( nPage>0 ){ u32 pageSize; u32 usableSize; - u8 *page1 = pPage1->aData; rc = SQLITE_NOTADB; if( memcmp(page1, zMagicHeader, 16)!=0 ){ goto page1_init_failed; @@ -2497,7 +2506,7 @@ static int newDatabase(BtShared *pBt){ assert( pP1!=0 ); rc = btreeMakePageWriteable(pP1); if( rc ) return rc; - data = pP1->aData; + data = btreeGetData(pP1); memcpy(data, zMagicHeader, sizeof(zMagicHeader)); assert( sizeof(zMagicHeader)==16 ); data[16] = (u8)((pBt->pageSize>>8)&0xff); @@ -2667,6 +2676,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ ** re-read the database size from page 1 if a savepoint or transaction ** rollback occurs within the transaction. */ + btreeGetData(pPage1); if( pBt->nPage!=get4byte(&pPage1->aData[28]) ){ rc = btreeMakePageWriteable(pPage1); if( rc==SQLITE_OK ){ @@ -2913,6 +2923,7 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ u8 eType; Pgno iPtrPage; + btreeGetData(pBt->pPage1); nFreeList = get4byte(&pBt->pPage1->aData[36]); if( nFreeList==0 ){ return SQLITE_DONE; @@ -3065,6 +3076,7 @@ static int autoVacuumCommit(BtShared *pBt){ return SQLITE_CORRUPT_BKPT; } + btreeGetData(pBt->pPage1); nFree = get4byte(&pBt->pPage1->aData[36]); nEntry = pBt->usableSize/5; nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; @@ -3424,7 +3436,7 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ if( rc==SQLITE_OK ){ if( iSavepoint<0 && pBt->initiallyEmpty ) pBt->nPage = 0; rc = newDatabase(pBt); - pBt->nPage = get4byte(28 + pBt->pPage1->aData); + pBt->nPage = get4byte(28 + btreeGetData(pBt->pPage1)); /* The database size was written into the offset 28 of the header ** when the transaction started, so we know that the value at offset @@ -3963,7 +3975,7 @@ static int accessPayload( && offset==0 /* (2) */ && pBt->inTransaction==TRANS_READ /* (4) */ && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ - && pBt->pPage1->aData[19]==0x01 /* (5) */ + && btreeGetData(pBt->pPage1)[19]==0x01 /* (5) */ ){ u8 aSave[4]; u8 *aWrite = &pBuf[-4]; @@ -4825,7 +4837,7 @@ static int allocateBtreePage( assert( sqlite3_mutex_held(pBt->mutex) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); - n = get4byte(&pPage1->aData[36]); + n = get4byte(&btreeGetData(pPage1)[36]); testcase( n==mxPage-1 ); if( n>=mxPage ){ return SQLITE_CORRUPT_BKPT; @@ -5000,6 +5012,7 @@ static int allocateBtreePage( *pPgno, closest+1, k, pTrunk->pgno, n-1)); rc = btreeMakePageWriteable(pTrunk); if( rc ) goto end_allocate_page; + aData = pTrunk->aData; if( closestpPage1 ); assert( idx>=0 && idx<=15 ); - *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); + *pMeta = get4byte(&btreeGetData(pBt->pPage1)[36 + idx*4]); /* If auto-vacuum is disabled in this build and this is an auto-vacuum ** database, mark the database as read-only. */ @@ -7930,6 +7943,7 @@ char *sqlite3BtreeIntegrityCheck( /* Check the integrity of the freelist */ + btreeGetData(pBt->pPage1); checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), get4byte(&pBt->pPage1->aData[36]), "Main freelist: "); @@ -8217,13 +8231,13 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ rc = sqlite3BtreeBeginTrans(pBtree, 0); if( rc==SQLITE_OK ){ - u8 *aData = pBt->pPage1->aData; + u8 *aData = btreeGetData(pBt->pPage1); if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){ rc = sqlite3BtreeBeginTrans(pBtree, 2); if( rc==SQLITE_OK ){ rc = btreeMakePageWriteable(pBt->pPage1); if( rc==SQLITE_OK ){ - aData = pBt->pPage1->aData; + aData = btreeGetData(pBt->pPage1); aData[18] = (u8)iVersion; aData[19] = (u8)iVersion; } diff --git a/src/pager.c b/src/pager.c index 9e2323a181..ae1c493d8f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1726,6 +1726,7 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ ** is opened (by this or by any other connection). */ static void pager_unlock(Pager *pPager){ + PgHdr *pPg; assert( pPager->eState==PAGER_READER || pPager->eState==PAGER_OPEN @@ -1785,22 +1786,31 @@ static void pager_unlock(Pager *pPager){ ** it can safely move back to PAGER_OPEN state. This happens in both ** normal and exclusive-locking mode. */ - if( pPager->errCode || pPager->pMapObject ){ + if( pPager->errCode ){ assert( !MEMDB ); pager_reset(pPager); pPager->changeCountDone = pPager->tempFile; pPager->eState = PAGER_OPEN; pPager->errCode = SQLITE_OK; - sqlite3OsUnmap(pPager->fd, pPager->pMapObject); - pPager->pMapObject = 0; - pPager->aFileContent = 0; - pPager->nFileContent = 0; } pPager->journalOff = 0; pPager->journalHdr = 0; pPager->setMaster = 0; + pPg = 0; + sqlite3PcacheFetch(pPager->pPCache, 1, 0, &pPg); + if( pPg ){ + /* assert( sqlite3PcachePagecount(pPager->pPCache)==1 ); */ + pPg->pData = pPg->pBuf; + sqlite3PcacheRelease(pPg); + }else{ + /*assert( sqlite3PcachePagecount(pPager->pPCache)==0 );*/ + } + sqlite3OsUnmap(pPager->fd, pPager->pMapObject); + pPager->pMapObject = 0; + pPager->aFileContent = 0; + pPager->nFileContent = 0; } /* @@ -5291,6 +5301,16 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ return rc; } +/* +** Make a copy of page content into malloced space. +*/ +void makePageWriteable(Pager *pPager, PgHdr *pPg){ + if( pPg->pData!=pPg->pBuf ){ + memcpy(pPg->pBuf, pPg->pData, pPager->pageSize); + pPg->pData = pPg->pBuf; + } +} + /* ** Mark a single data page as writeable. The page is written into the ** main journal or sub-journal as required. If the page is written into @@ -5340,10 +5360,8 @@ static int pager_write(PgHdr *pPg){ assert( assert_pager_state(pPager) ); /* Make sure page content is held in malloced memory */ - if( pPg->pData!=pPg->pBuf ){ - memcpy(pPg->pBuf, pPg->pData, pPager->pageSize); - pData = pPg->pData = pPg->pBuf; - } + makePageWriteable(pPager, pPg); + pData = pPg->pData; /* Mark the page as dirty. If the page has already been written ** to the journal then we can return right away. @@ -6437,6 +6455,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ } origPgno = pPg->pgno; + makePageWriteable(pPager, pPg); sqlite3PcacheMove(pPg, pgno); sqlite3PcacheMakeDirty(pPg); @@ -6486,9 +6505,9 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ /* ** Return a pointer to the data for the specified page. */ -void *sqlite3PagerGetData(DbPage *pPg){ +u8 *sqlite3PagerGetData(DbPage *pPg){ assert( pPg->nRef>0 || pPg->pPager->memDb ); - return pPg->pData; + return (u8*)pPg->pData; } /* diff --git a/src/pager.h b/src/pager.h index 553cf2954b..cf6f0798d9 100644 --- a/src/pager.h +++ b/src/pager.h @@ -123,7 +123,7 @@ int sqlite3PagerWrite(DbPage*); void sqlite3PagerDontWrite(DbPage*); int sqlite3PagerMovepage(Pager*,DbPage*,Pgno,int); int sqlite3PagerPageRefcount(DbPage*); -void *sqlite3PagerGetData(DbPage *); +u8 *sqlite3PagerGetData(DbPage *); void *sqlite3PagerGetExtra(DbPage *); /* Functions used to manage pager transactions and savepoints. */ diff --git a/src/pcache.c b/src/pcache.c index c8b46ace07..a7032b574a 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -131,7 +131,7 @@ static void pcacheUnpin(PgHdr *p){ if( p->pgno==1 ){ pCache->pPage1 = 0; } - sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, p->pPage, 0); + sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, p->pPage, 1); } }