From: drh Date: Fri, 13 Apr 2007 02:14:30 +0000 (+0000) Subject: Fix multiple performance regressions (ticket #2298 among them) X-Git-Tag: version-3.6.10~2331 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=538f570cd1bf87aa8f922b82b1e490dadd688797;p=thirdparty%2Fsqlite.git Fix multiple performance regressions (ticket #2298 among them) and add tests to make sure they do not come back. (CVS 3839) FossilOrigin-Name: 32bb2d5859906b4fb0f6083eedd7f3a81b9cf5e2 --- diff --git a/manifest b/manifest index 24355b8809..ab101d8d40 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\stoward\sfixes\sfor\stickets\s#2296\sand\s#2291.\s(CVS\s3838) -D 2007-04-12T21:25:02 +C Fix\smultiple\sperformance\sregressions\s(ticket\s#2298\samong\sthem)\r\nand\sadd\stests\sto\smake\ssure\sthey\sdo\snot\scome\sback.\s(CVS\s3839) +D 2007-04-13T02:14:30 F Makefile.in 8cab54f7c9f5af8f22fd97ddf1ecfd1e1860de62 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -58,7 +58,7 @@ F src/alter.c 2c79ec40f65e33deaf90ca493422c74586e481a3 F src/analyze.c 4bbf5ddf9680587c6d4917e02e378b6037be3651 F src/attach.c a16ada4a4654a0d126b8223ec9494ebb81bc5c3c F src/auth.c 902f4722661c796b97f007d9606bd7529c02597f -F src/btree.c 7adc16a10b42d3a3c39e64af85b748c68156f6b2 +F src/btree.c 2023a8371bd23c300571a4ce9673b8859c44be36 F src/btree.h 9b2cc0d113c0bc2d37d244b9a394d56948c9acbf F src/build.c 7c2efa468f0c404ef5aa648d43c383318390937f F src/callback.c 31d22b4919c7645cbcbb1591ce2453e8c677c558 @@ -77,7 +77,7 @@ F src/main.c c8915777ae8e50823d01eefe2b674ef68c32bf61 F src/md5.c c5fdfa5c2593eaee2e32a5ce6c6927c986eaf217 F src/os.c 4650e98aadd27abfe1698ff58edf6893c58d4881 F src/os.h 9240adf088fd55732f8926f9017601ba3430ead8 -F src/os_common.h 0969285cc1e0b1ccc7a2cd7ce8eb144176ce3369 +F src/os_common.h a38233cd3b1f260db6f01f1093295d5708130065 F src/os_os2.c 2ce97909b926a598823f97338027dbec1dcf4165 F src/os_os2.h e5f17dd69333632bbc3112881ea407c37d245eb3 F src/os_test.c 49833426101f99aee4bb5f6a44b7c4b2029fda1c @@ -86,8 +86,8 @@ F src/os_unix.c 426b4c03c304ad78746d65d9ba101e0b72e18e23 F src/os_unix.h 5768d56d28240d3fe4537fac08cc85e4fb52279e F src/os_win.c e94903c7dc1c0599c8ddce42efa0b6928068ddc5 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b -F src/pager.c 655073dd7b32eade2f10f69e1d51fece380d45e1 -F src/pager.h e79a24cf200b8771366217f5bca414f5b7823f42 +F src/pager.c 2a5df21f1fc4acfa51815305892df13c7f90bc76 +F src/pager.h d652ddf092d2318d00e41f8539760fe8e57c157c F src/parse.y b6cfbadb6d5b21b5087d30698ee5af0ebb098767 F src/pragma.c 3b992b5b2640d6ae25cef05aa6a42cd1d6c43234 F src/prepare.c 37207b2b2ccb41d379b01dd62231686bcc48ef1f @@ -101,7 +101,7 @@ F src/sqlite3ext.h 7d0d363ea7327e817ef0dfe1b7eee1f171b72890 F src/sqliteInt.h 6c25db3cdf91d26ffe66c685812c808291c241de F src/table.c 6d0da66dde26ee75614ed8f584a1996467088d06 F src/tclsqlite.c ec69eb9ad56d03fbf7570ca1ca5ea947d1ec4b6f -F src/test1.c 9f85126e66a9a1ec463b609cd0221c151a723e2c +F src/test1.c b96f80e68cd50e10e10cda8bb762152bae52a8d2 F src/test2.c 24458b17ab2f3c90cbc1c8446bd7ffe69be62f88 F src/test3.c 65f92247cf8592854e9bf5115b3fb711f8b33280 F src/test4.c 8b784cd82de158a2317cb4ac4bc86f91ad315e25 @@ -247,7 +247,7 @@ F test/insert4.test 1e27f0a3e5670d5f03c1636f699aa44270945bca F test/interrupt.test c38b7f7c17914f0cd6a119beed5d03bc3f47f9eb F test/intpkey.test af4fd826c4784ec5c93b444de07adea0254d0d30 F test/ioerr.test 491d42c49bbec598966d26b01ed7901f55e5ee2d -F test/ioerr2.test 23c5f19304625d48c12775ee4a23d54e125c47cb +F test/ioerr2.test 65ede6b5f073b2f173228ed9f08b60f309a63d5f F test/join.test af0443185378b64878750aa1cf4b83c216f246b4 F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 @@ -289,6 +289,7 @@ F test/null.test 9503e1f63e959544c006d9f01709c5b5eab67d54 F test/pager.test 6c644725db2a79528f67a6f3472b9c9ddee17f05 F test/pager2.test c025f91b75fe65e85febda64d9416428b8a5cab5 F test/pager3.test 2323bf27fd5bd887b580247e5bce500ceee994b4 +F test/pageropt.test 05d2a3cf1934bb215ecf084a93243a42fe0f8b94 F test/pagesize.test e0a8b3fe80f8b8e808d94a00734c7a18c76c407e F test/pragma.test fecb7085f58d9fb5172a5c0b63fd3b25c7bfb414 F test/printf.test 483b9fe75ffae1fb27328bdce5560b452ba83577 @@ -457,7 +458,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513 -P 4062ddf3c7f4fd150292304fa33ca76dc35571a1 -R c4a1410422b835866a41c3cffa966fab +P 0dd3e2e47b09156838edfa4dea0d82f9cf22d94d +R 2a770dc94eebbe8757f249788620bf30 U drh -Z ee14464e551b6f16d7f21d5424a51d91 +Z a0fae9ba10e5c203140db1181856fb83 diff --git a/manifest.uuid b/manifest.uuid index d0310eae0e..ad6cd3f975 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0dd3e2e47b09156838edfa4dea0d82f9cf22d94d \ No newline at end of file +32bb2d5859906b4fb0f6083eedd7f3a81b9cf5e2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 8665c51042..7c700d9bd2 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.354 2007/04/09 12:45:03 drh Exp $ +** $Id: btree.c,v 1.355 2007/04/13 02:14:30 drh Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to @@ -1396,13 +1396,20 @@ static void zeroPage(MemPage *pPage, int flags){ /* ** Get a page from the pager. Initialize the MemPage.pBt and ** MemPage.aData elements if needed. +** +** If the noContent flag is set, it means that we do not care about +** the content of the page at this time. So do not go to the disk +** to fetch the content. Just fill in the content with zeros for now. +** If in the future we call sqlite3PagerWrite() on this page, that +** means we have started to be concerned about content and the disk +** read should occur at that point. */ -static int getPage(BtShared *pBt, Pgno pgno, MemPage **ppPage, int clrFlag){ +static int getPage(BtShared *pBt, Pgno pgno, MemPage **ppPage, int noContent){ int rc; MemPage *pPage; DbPage *pDbPage; - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, clrFlag); + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent); if( rc ) return rc; pPage = (MemPage *)sqlite3PagerGetExtra(pDbPage); pPage->aData = sqlite3PagerGetData(pDbPage); @@ -1411,9 +1418,6 @@ static int getPage(BtShared *pBt, Pgno pgno, MemPage **ppPage, int clrFlag){ pPage->pgno = pgno; pPage->hdrOffset = pPage->pgno==1 ? 100 : 0; *ppPage = pPage; - if( clrFlag ){ - sqlite3PagerDontRollback(pPage->pDbPage); - } return SQLITE_OK; } @@ -3830,6 +3834,7 @@ static int allocateBtreePage( put4byte(&aData[4], k-1); rc = getPage(pBt, *pPgno, ppPage, 1); if( rc==SQLITE_OK ){ + sqlite3PagerDontRollback((*ppPage)->pDbPage); rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ releasePage(*ppPage); @@ -3948,7 +3953,7 @@ static int freePage(MemPage *pPage){ put4byte(&pTrunk->aData[4], k+1); put4byte(&pTrunk->aData[8+k*4], pPage->pgno); #ifndef SQLITE_SECURE_DELETE - sqlite3PagerDontWrite(pBt->pPager, pPage->pgno); + sqlite3PagerDontWrite(pPage->pDbPage); #endif } TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); @@ -3982,7 +3987,7 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ if( ovflPgno==0 || ovflPgno>sqlite3PagerPagecount(pBt->pPager) ){ return SQLITE_CORRUPT_BKPT; } - rc = getPage(pBt, ovflPgno, &pOvfl, 0); + rc = getPage(pBt, ovflPgno, &pOvfl, nOvfl==0); if( rc ) return rc; if( nOvfl ){ ovflPgno = get4byte(pOvfl->aData); @@ -6561,18 +6566,31 @@ int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ rc = sqlite3PagerOverwrite(pBtTo->pPager, i, sqlite3PagerGetData(pDbPage)); sqlite3PagerUnref(pDbPage); } + + /* If the file is shrinking, journal the pages that are being truncated + ** so that they can be rolled back if the commit fails. + */ for(i=nPage+1; rc==SQLITE_OK && i<=nToPage; i++){ DbPage *pDbPage; if( i==iSkip ) continue; rc = sqlite3PagerGet(pBtTo->pPager, i, &pDbPage); if( rc ) break; rc = sqlite3PagerWrite(pDbPage); + sqlite3PagerDontWrite(pDbPage); + /* Yeah. It seems wierd to call DontWrite() right after Write(). But + ** that is because the names of those procedures do not exactly + ** represent what they do. Write() really means "put this page in the + ** rollback journal and mark it as dirty so that it will be written + ** to the database file later." DontWrite() undoes the second part of + ** that and prevents the page from being written to the database. The + ** page is still on the rollback journal, though. And that is the whole + ** point of this loop: to put pages on the rollback journal. */ sqlite3PagerUnref(pDbPage); - sqlite3PagerDontWrite(pBtTo->pPager, i); } if( !rc && nPagepPager, nPage); } + if( rc ){ sqlite3BtreeRollback(pTo); } diff --git a/src/os_common.h b/src/os_common.h index 6aa244f9c8..ba52314ce8 100644 --- a/src/os_common.h +++ b/src/os_common.h @@ -97,6 +97,7 @@ int sqlite3_diskfull = 0; || (sqlite3_io_error_persist && sqlite3_io_error_hit) ) \ { local_ioerr(); CODE; } static void local_ioerr(){ + IOTRACE(("IOERR\n")); sqlite3_io_error_hit = 1; } #define SimulateDiskfullError(CODE) \ diff --git a/src/pager.c b/src/pager.c index 6f3d029570..478a7014d7 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,7 +18,7 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.326 2007/04/09 11:20:54 danielk1977 Exp $ +** @(#) $Id: pager.c,v 1.327 2007/04/13 02:14:30 drh Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" @@ -162,6 +162,7 @@ struct PgHdr { u8 dirty; /* TRUE if we need to write back changes */ u8 needSync; /* Sync journal before writing this page */ u8 alwaysRollback; /* Disable DontRollback() for this page */ + u8 needRead; /* Read content if PagerWrite() is called */ short int nRef; /* Number of users of this page */ PgHdr *pDirty, *pPrevDirty; /* Dirty pages */ u32 notUsed; /* Buffer space */ @@ -297,15 +298,22 @@ struct Pager { }; /* -** If SQLITE_TEST is defined then increment the variable given in -** the argument +** The following global variables hold counters used for +** testing purposes only. These variables do not exist in +** a non-testing build. These variables are not thread-safe. */ #ifdef SQLITE_TEST -# define TEST_INCR(x) x++ +int sqlite3_pager_readdb_count = 0; /* Number of full pages read from DB */ +int sqlite3_pager_writedb_count = 0; /* Number of full pages written to DB */ +int sqlite3_pager_writej_count = 0; /* Number of pages written to journal */ +int sqlite3_pager_pgfree_count = 0; /* Number of cache pages freed */ +# define PAGER_INCR(v) v++ #else -# define TEST_INCR(x) +# define PAGER_INCR(v) #endif + + /* ** Journal files begin with the following magic string. The data ** was obtained from /dev/random. It is used only as a sanity check. @@ -898,6 +906,8 @@ static void pager_reset(Pager *pPager){ PgHdr *pPg, *pNext; if( pPager->errCode ) return; for(pPg=pPager->pAll; pPg; pPg=pNext){ + IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno)); + PAGER_INCR(sqlite3_pager_pgfree_count); pNext = pPg->pNextAll; sqliteFree(pPg); } @@ -1219,51 +1229,6 @@ delmaster_out: return rc; } -#if 0 -/* -** Make every page in the cache agree with what is on disk. In other words, -** reread the disk to reset the state of the cache. -** -** This routine is called after a rollback in which some of the dirty cache -** pages had never been written out to disk. We need to roll back the -** cache content and the easiest way to do that is to reread the old content -** back from the disk. -*/ -static int pager_reload_cache(Pager *pPager){ - PgHdr *pPg; - int rc = SQLITE_OK; - for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ - char *zBuf = pPager->pTmpSpace; /* Temp storage for one page */ - if( !pPg->dirty ) continue; - if( (int)pPg->pgno <= pPager->origDbSize ){ - rc = sqlite3OsSeek(pPager->fd, pPager->pageSize*(i64)(pPg->pgno-1)); - if( rc==SQLITE_OK ){ - rc = sqlite3OsRead(pPager->fd, zBuf, pPager->pageSize); - } - PAGERTRACE3("REFETCH %d page %d\n", PAGERID(pPager), pPg->pgno); - if( rc ) break; - CODEC1(pPager, zBuf, pPg->pgno, 2); - }else{ - memset(zBuf, 0, pPager->pageSize); - } - if( pPg->nRef==0 || memcmp(zBuf, PGHDR_TO_DATA(pPg), pPager->pageSize) ){ - memcpy(PGHDR_TO_DATA(pPg), zBuf, pPager->pageSize); - if( pPager->xReiniter ){ - pPager->xReiniter(pPg, pPager->pageSize); - }else{ - memset(PGHDR_TO_EXTRA(pPg, pPager), 0, pPager->nExtra); - } - } - pPg->needSync = 0; - pPg->dirty = 0; -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif - } - pPager->pDirty = 0; - return rc; -} -#endif static void pager_truncate_cache(Pager *pPager); @@ -2049,6 +2014,8 @@ static void pager_truncate_cache(Pager *pPager){ ppPg = &pPg->pNextAll; }else{ *ppPg = pPg->pNextAll; + IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno)); + PAGER_INCR(sqlite3_pager_pgfree_count); unlinkPage(pPg); makeClean(pPg); sqliteFree(pPg); @@ -2470,9 +2437,10 @@ static int pager_write_pagelist(PgHdr *pList){ if( pList->pgno<=pPager->dbSize ){ char *pData = CODEC2(pPager, PGHDR_TO_DATA(pList), pList->pgno, 6); PAGERTRACE3("STORE %d page %d\n", PAGERID(pPager), pList->pgno); - IOTRACE(("PGOUT %p %d\n", pPager, pList->pgno)) + IOTRACE(("PGOUT %p %d\n", pPager, pList->pgno)); rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize); - TEST_INCR(pPager->nWrite); + PAGER_INCR(sqlite3_pager_writedb_count); + PAGER_INCR(pPager->nWrite); } #ifndef NDEBUG else{ @@ -2674,6 +2642,8 @@ int sqlite3PagerReleaseMemory(int nReq){ pTmp->pNextAll = pPg->pNextAll; } nReleased += sqliteAllocSize(pPg); + IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno)); + PAGER_INCR(sqlite3_pager_pgfree_count); sqliteFree(pPg); } @@ -2706,7 +2676,9 @@ static int readDbPage(Pager *pPager, PgHdr *pPg, Pgno pgno){ rc = sqlite3OsRead(pPager->fd, PGHDR_TO_DATA(pPg), pPager->pageSize); } - IOTRACE(("PGIN %p %d\n", pPager, pgno)) + PAGER_INCR(sqlite3_pager_readdb_count); + PAGER_INCR(pPager->nRead); + IOTRACE(("PGIN %p %d\n", pPager, pgno)); PAGERTRACE3("FETCH %d page %d\n", PAGERID(pPager), pPg->pgno); CODEC1(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3); return rc; @@ -2839,6 +2811,7 @@ static int pagerSharedLock(Pager *pPager){ if( iChangeCounter!=pPager->iChangeCount ){ pager_reset(pPager); + pPager->iChangeCount = iChangeCounter; } } } @@ -2959,12 +2932,20 @@ pager_allocate_out: ** Since _lookup() never goes to disk, it never has to deal with locks ** or journal files. ** -** If clrFlag is false, the page contents are actually read from disk. -** If clfFlag is true, it means the page is about to be erased and -** rewritten without first being read so there is no point it doing -** the disk I/O. +** If noContent is false, the page contents are actually read from disk. +** If noContent is true, it means that we do not care about the contents +** of the page at this time, so do not do a disk read. Just fill in the +** page content with zeros. But mark the fact that we have not read the +** content by setting the PgHdr.needRead flag. Later on, if +** sqlite3PagerWrite() is called on this page, that means that the +** content is needed and the disk read should occur at that point. */ -int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){ +int sqlite3PagerAcquire( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int noContent /* Do not bother reading content from disk if true */ +){ PgHdr *pPg; int rc; @@ -3000,7 +2981,7 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){ /* The requested page is not in the page cache. */ int nMax; int h; - TEST_INCR(pPager->nMiss); + PAGER_INCR(pPager->nMiss); rc = pagerAllocatePage(pPager, &pPg); if( rc!=SQLITE_OK ){ return rc; @@ -3036,18 +3017,22 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){ /* Populate the page with data, either by reading from the database ** file, or by setting the entire page to zero. */ - if( nMax<(int)pgno || MEMDB || (clrFlag && !pPager->alwaysRollback) ){ + if( nMax<(int)pgno || MEMDB || noContent ){ memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize); + pPg->needRead = noContent; + IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ rc = readDbPage(pPager, pPg, pgno); if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ pPg->pgno = 0; sqlite3PagerUnref(pPg); return rc; - }else{ - TEST_INCR(pPager->nRead); } } + /* If this was page 1, then restore the value of Pager.iChangeCount */ + if( pgno==1 ){ + pPager->iChangeCount = retrieve32bits(pPg, 24); + } /* Link the page into the page hash table */ h = pgno & (pPager->nHash-1); @@ -3065,7 +3050,7 @@ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag){ }else{ /* The requested page is in the page cache. */ assert(pPager->nRef>0 || pgno==1); - TEST_INCR(pPager->nHit); + PAGER_INCR(pPager->nHit); page_ref(pPg); } *ppPage = pPg; @@ -3365,6 +3350,23 @@ static int pager_write(PgHdr *pPg){ CHECK_PAGE(pPg); + /* If this page was previously acquired with noContent==1, that means + ** we didn't really read in the content of the page. This can happen + ** (for example) when the page is being moved to the freelist. But + ** now we are (perhaps) moving the page off of the freelist for + ** reuse and we need to know its original content so that content + ** can be stored in the rollback journal. So do the read at this + ** time. + */ + if( pPg->needRead ){ + rc = readDbPage(pPager, pPg, pPg->pgno); + if( rc==SQLITE_OK ){ + pPg->needRead = 0; + }else{ + return rc; + } + } + /* Mark the page as dirty. If the page has already been written ** to the journal then we can return right away. */ @@ -3425,7 +3427,8 @@ static int pager_write(PgHdr *pPg){ put32bits(pData2, pPg->pgno); rc = sqlite3OsWrite(pPager->jfd, pData2, szPg); IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, - pPager->journalOff, szPg)) + pPager->journalOff, szPg)); + PAGER_INCR(sqlite3_pager_writej_count); pPager->journalOff += szPg; PAGERTRACE4("JOURNAL %d page %d needSync=%d\n", PAGERID(pPager), pPg->pgno, pPg->needSync); @@ -3608,7 +3611,7 @@ int sqlite3PagerOverwrite(Pager *pPager, Pgno pgno, void *pData){ /* ** A call to this routine tells the pager that it is not necessary to -** write the information on page "pgno" back to the disk, even though +** write the information on page pPg back to the disk, even though ** that page might be marked as dirty. ** ** The overlying software layer calls this routine when all of the data @@ -3630,13 +3633,11 @@ int sqlite3PagerOverwrite(Pager *pPager, Pgno pgno, void *pData){ ** page contains critical data, we still need to be sure it gets ** rolled back in spite of the sqlite3PagerDontRollback() call. */ -void sqlite3PagerDontWrite(Pager *pPager, Pgno pgno){ - PgHdr *pPg; +void sqlite3PagerDontWrite(DbPage *pDbPage){ + PgHdr *pPg = pDbPage; + Pager *pPager = pPg->pPager; if( MEMDB ) return; - - pPg = pager_lookup(pPager, pgno); - assert( pPg!=0 ); /* We never call _dont_write unless the page is in mem */ pPg->alwaysRollback = 1; if( pPg->dirty && !pPager->stmtInUse ){ assert( pPager->state>=PAGER_SHARED ); @@ -3650,8 +3651,8 @@ void sqlite3PagerDontWrite(Pager *pPager, Pgno pgno){ ** corruption during the next transaction. */ }else{ - PAGERTRACE3("DONT_WRITE page %d of %d\n", pgno, PAGERID(pPager)); - IOTRACE(("CLEAN %p %d\n", pPager, pgno)) + PAGERTRACE3("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager)); + IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) makeClean(pPg); #ifdef SQLITE_CHECK_PAGES pPg->pageHash = pager_pagehash(pPg); @@ -3665,6 +3666,11 @@ void sqlite3PagerDontWrite(Pager *pPager, Pgno pgno){ ** it is not necessary to restore the data on the given page. This ** means that the pager does not have to record the given page in the ** rollback journal. +** +** If we have not yet actually read the content of this page (if +** the PgHdr.needRead flag is set) then this routine acts as a promise +** that we will never need to read the page content in the future. +** so the needRead flag can be cleared at this point. */ void sqlite3PagerDontRollback(DbPage *pPg){ Pager *pPager = pPg->pPager; @@ -3676,6 +3682,7 @@ void sqlite3PagerDontRollback(DbPage *pPg){ assert( pPager->aInJournal!=0 ); pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7); pPg->inJournal = 1; + pPg->needRead = 0; if( pPager->stmtInUse ){ pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7); } diff --git a/src/pager.h b/src/pager.h index a6f639e05d..96b0cd342d 100644 --- a/src/pager.h +++ b/src/pager.h @@ -13,7 +13,7 @@ ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** -** @(#) $Id: pager.h,v 1.57 2007/03/30 14:06:34 drh Exp $ +** @(#) $Id: pager.h,v 1.58 2007/04/13 02:14:30 drh Exp $ */ #ifndef _PAGER_H_ @@ -109,7 +109,7 @@ int sqlite3PagerStmtBegin(Pager*); int sqlite3PagerStmtCommit(Pager*); int sqlite3PagerStmtRollback(Pager*); void sqlite3PagerDontRollback(DbPage*); -void sqlite3PagerDontWrite(Pager*, Pgno); +void sqlite3PagerDontWrite(DbPage*); int sqlite3PagerRefcount(Pager*); int *sqlite3PagerStats(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int); diff --git a/src/test1.c b/src/test1.c index 5f30a2805b..07c8bffa3f 100644 --- a/src/test1.c +++ b/src/test1.c @@ -13,7 +13,7 @@ ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** -** $Id: test1.c,v 1.236 2007/04/09 12:45:03 drh Exp $ +** $Id: test1.c,v 1.237 2007/04/13 02:14:30 drh Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -197,6 +197,57 @@ static int exec_printf_cb(void *pArg, int argc, char **argv, char **name){ return 0; } +/* +** The I/O tracing callback. +*/ +static FILE *iotrace_file = 0; +static void io_trace_callback(const char *zFormat, ...){ + va_list ap; + va_start(ap, zFormat); + vfprintf(iotrace_file, zFormat, ap); + va_end(ap); + fflush(iotrace_file); +} + +/* +** Usage: io_trace FILENAME +** +** Turn I/O tracing on or off. If FILENAME is not an empty string, +** I/O tracing begins going into FILENAME. If FILENAME is an empty +** string, I/O tracing is turned off. +*/ +static int test_io_trace( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " FILENAME\"", 0); + return TCL_ERROR; + } + if( iotrace_file ){ + if( iotrace_file!=stdout && iotrace_file!=stderr ){ + fclose(iotrace_file); + } + iotrace_file = 0; + sqlite3_io_trace = 0; + } + if( argv[1][0] ){ + if( strcmp(argv[1],"stdout")==0 ){ + iotrace_file = stdout; + }else if( strcmp(argv[1],"stderr")==0 ){ + iotrace_file = stderr; + }else{ + iotrace_file = fopen(argv[1], "w"); + } + sqlite3_io_trace = io_trace_callback; + } + return SQLITE_OK; +} + + /* ** Usage: sqlite3_exec_printf DB FORMAT STRING ** @@ -4216,6 +4267,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_stack_used", (Tcl_CmdProc*)test_stack_used }, { "sqlite3_busy_timeout", (Tcl_CmdProc*)test_busy_timeout }, { "printf", (Tcl_CmdProc*)test_printf }, + { "sqlite3_io_trace", (Tcl_CmdProc*)test_io_trace }, }; static struct { char *zName; @@ -4338,6 +4390,10 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ extern int sqlite3_like_count; extern int sqlite3_tsd_count; extern int sqlite3_xferopt_count; + extern int sqlite3_pager_readdb_count; + extern int sqlite3_pager_writedb_count; + extern int sqlite3_pager_writej_count; + extern int sqlite3_pager_pgfree_count; #if OS_UNIX && defined(SQLITE_TEST) && defined(THREADSAFE) && THREADSAFE extern int threadsOverrideEachOthersLocks; #endif @@ -4377,6 +4433,14 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ (char*)&sqlite3_tsd_count, TCL_LINK_INT); Tcl_LinkVar(interp, "sqlite3_xferopt_count", (char*)&sqlite3_xferopt_count, TCL_LINK_INT); + Tcl_LinkVar(interp, "sqlite3_pager_readdb_count", + (char*)&sqlite3_pager_readdb_count, TCL_LINK_INT); + Tcl_LinkVar(interp, "sqlite3_pager_writedb_count", + (char*)&sqlite3_pager_writedb_count, TCL_LINK_INT); + Tcl_LinkVar(interp, "sqlite3_pager_writej_count", + (char*)&sqlite3_pager_writej_count, TCL_LINK_INT); + Tcl_LinkVar(interp, "sqlite3_pager_pgfree_count", + (char*)&sqlite3_pager_pgfree_count, TCL_LINK_INT); #ifndef SQLITE_OMIT_UTF16 Tcl_LinkVar(interp, "unaligned_string_counter", (char*)&unaligned_string_counter, TCL_LINK_INT); diff --git a/test/ioerr2.test b/test/ioerr2.test index 394d59d4af..e73586f9b7 100644 --- a/test/ioerr2.test +++ b/test/ioerr2.test @@ -15,7 +15,7 @@ # The tests in this file use special facilities that are only # available in the SQLite test fixture. # -# $Id: ioerr2.test,v 1.3 2007/04/09 12:45:03 drh Exp $ +# $Id: ioerr2.test,v 1.4 2007/04/13 02:14:30 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -82,10 +82,11 @@ set sql { ROLLBACK; } -breakpoint foreach bPersist [list 0 1] { set ::go 1 for {set ::N 1} {$::go} {incr ::N} { + db close + sqlite3 db test.db set ::sqlite_io_error_hit 0 set ::sqlite_io_error_persist $bPersist set ::sqlite_io_error_pending $::N @@ -94,5 +95,16 @@ foreach bPersist [list 0 1] { check_db ioerr2-3.$bPersist.$::N } } +foreach bPersist [list 0 1] { + set ::go 1 + for {set ::N 1} {$::go} {incr ::N} { + set ::sqlite_io_error_hit 0 + set ::sqlite_io_error_persist $bPersist + set ::sqlite_io_error_pending $::N + + foreach {::go res} [catchsql $sql] {} + check_db ioerr2-3.[expr {$bPersist+2}].$::N + } +} finish_test diff --git a/test/pageropt.test b/test/pageropt.test new file mode 100644 index 0000000000..ccb2225678 --- /dev/null +++ b/test/pageropt.test @@ -0,0 +1,138 @@ +# 2007 April 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# The focus of the tests in this file are to verify that the +# pager optimizations implemented in version 3.3.14 work. +# +# $Id: pageropt.test,v 1.1 2007/04/13 02:14:30 drh Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +ifcapable {!pager_pragmas} { + finish_test + return +} + +# Run the SQL statement supplied by the argument and return +# the results. Prepend four integers to the beginning of the +# result which are +# +# (1) The number of page reads from the database +# (2) The number of page writes to the database +# (3) The number of page writes to the journal +# (4) The number of cache pages freed +# +proc pagercount_sql {sql {db db}} { + global sqlite3_pager_readdb_count + global sqlite3_pager_writedb_count + global sqlite3_pager_writej_count + global sqlite3_pager_pgfree_count + set sqlite3_pager_readdb_count 0 + set sqlite3_pager_writedb_count 0 + set sqlite3_pager_writej_count 0 + set sqlite3_pager_pgfree_count 0 + set r [$db eval $sql] + set cnt [list $sqlite3_pager_readdb_count \ + $sqlite3_pager_writedb_count \ + $sqlite3_pager_writej_count \ + $sqlite3_pager_pgfree_count] + return [concat $cnt $r] +} + +# Setup the test database +# +do_test pageropt-1.1 { + execsql { + PRAGMA auto_vacuum = OFF; + PRAGMA page_size = 1024; + } + pagercount_sql { + CREATE TABLE t1(x); + } +} {0 2 0 0} +do_test pageropt-1.2 { + pagercount_sql { + INSERT INTO t1 VALUES(randomblob(5000)); + } +} {0 6 2 0} + +# Verify that values remain in cache on for subsequent reads. +# We should not have to go back to disk. +# +do_test pageropt-1.3 { + pagercount_sql { + SELECT length(x) FROM t1 + } +} {0 0 0 0 5000} + +# If another thread reads the database, the original cache +# remains valid. +# +sqlite3 db2 test.db +set blobcontent [db2 one {SELECT hex(x) FROM t1}] +do_test pageropt-1.4 { + pagercount_sql { + SELECT hex(x) FROM t1 + } +} [list 0 0 0 0 $blobcontent] + +# But if the other thread modifies the database, then the cache +# must refill. +# +do_test pageropt-1.5 { + db2 eval {CREATE TABLE t2(y)} + pagercount_sql { + SELECT hex(x) FROM t1 + } +} [list 6 0 0 6 $blobcontent] +do_test pageropt-1.6 { + pagercount_sql { + SELECT hex(x) FROM t1 + } +} [list 0 0 0 0 $blobcontent] + +# Verify that the last page of an overflow chain is not read from +# disk when deleting a row. The one row of t1(x) has four pages +# of overflow. So deleting that row from t1 should involve reading +# the sqlite_master table (1 page) the main page of t1 (1 page) and +# the three overflow pages of t1 for a total of 5 pages. +# +# Pages written are page 1 (for the freelist pointer), the root page +# of the table, and one of the overflow chain pointers because it +# becomes the trunk of the freelist. Total 3. +# +do_test pageropt-2.1 { + db close + sqlite3 db test.db + pagercount_sql { + DELETE FROM t1 WHERE rowid=1 + } +} {5 3 3 0} + +# When pulling pages off of the freelist, there is no reason +# to actually bring in the old content. +# +do_test pageropt-2.2 { + db close + sqlite3 db test.db + pagercount_sql { + INSERT INTO t1 VALUES(randomblob(1500)); + } +} {3 4 3 0} +do_test pageropt-2.3 { + pagercount_sql { + INSERT INTO t1 VALUES(randomblob(1500)); + } +} {0 4 3 0} + +catch {db2 close} +finish_test