From: drh Date: Fri, 26 Aug 2011 00:34:45 +0000 (+0000) Subject: Create a new pager type, PAGER_SORTER, for use in the external merge sort. X-Git-Tag: version-3.7.8~43 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=2a5d9908f991edaba953155655eb345bdc90999a;p=thirdparty%2Fsqlite.git Create a new pager type, PAGER_SORTER, for use in the external merge sort. Such pagers are always held in memory but do report when they are under memory pressure by calling pagerStress. FossilOrigin-Name: c71d73201d950355862dd8d5de142c9673888755 --- diff --git a/manifest b/manifest index bd728d0cf3..592d23a24a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reorder\ssome\sof\sthe\sbranches\sin\sbackup.c\sin\sorder\sto\smake\sthe\scode\neasier\sto\stest. -D 2011-08-25T20:18:47.446 +C Create\sa\snew\spager\stype,\sPAGER_SORTER,\sfor\suse\sin\sthe\sexternal\smerge\ssort.\nSuch\spagers\sare\salways\sheld\sin\smemory\sbut\sdo\sreport\swhen\sthey\sare\sunder\nmemory\spressure\sby\scalling\spagerStress. +D 2011-08-26T00:34:45.360 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,10 +124,10 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 97cf3ba4ff067e716753b33661035e50853aebba -F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce +F src/btree.c ed13fdefdbe671d5777773dcfb3a162ddb4623ae +F src/btree.h 9ddf04226eac592d4cc3709c5a8b33b2351ff5f7 F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c 20784c6b4e4514c90aeeec2bee0fb9d79a4e2189 +F src/build.c 2d5de52df616a3bf5a659cbca85211c46e2ba9bd F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c caf51429be3e0d4114056a8273b0fff812ff8ae9 @@ -167,8 +167,8 @@ F src/os_common.h 65a897143b64667d23ed329a7984b9b405accb58 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 1a34ca3794ced80e4a4ebcc3ba1f4c516762e534 F src/os_win.c 19fa09046f1f86590a188abdcf5630b8fe8279cf -F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 -F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 +F src/pager.c 817f7f7140c9fa2641f28e6330e924708ddd870d +F src/pager.h 2bab1b2ea4eac58663b5833e3522e36b5ff63447 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 49e718c095810c6b3334e3a6d89970aceaddefce F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 @@ -238,14 +238,14 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 8f18a857e7cc1eba1a59d2ab011d4cf0d05f48ad +F src/vdbe.c 4a7191c0f8e918b74e8c84cbdd77746d6b7e3bcf F src/vdbe.h 2bf6ec77d8b9980fc19da6e0b0a36d0dbf884ce4 F src/vdbeInt.h f9250326f264ca5f100acc19e9c07096bb889096 F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c 11b0df8822ecf61e543562247207df75e2ebb617 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b -F src/vdbesort.c d2c872322c94caae7abd39fe88eef177f66240cf +F src/vdbesort.c 8a61a6d731cbe612217edf9eece6197f37c9489e F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 472c74b3452c5a07dfb006010441232b09599ad5 -R 8c5e28ffe9a7799230a8adf4f20dce05 +P 2c443d47ecee7b43a89f0a4bf299c46c66e3f80d +R 5271767b893f002dc634ff718e14ca01 U drh -Z ae50a88981aafd0a9a503f248a20233b +Z 986c77c3e73f3db4a3199ce94583c644 diff --git a/manifest.uuid b/manifest.uuid index 046b6493d3..d151ff9b84 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c443d47ecee7b43a89f0a4bf299c46c66e3f80d \ No newline at end of file +c71d73201d950355862dd8d5de142c9673888755 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 547ef83707..0c5fa38e4a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1734,11 +1734,22 @@ int sqlite3BtreeOpen( /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ assert( (flags & BTREE_SINGLE)==0 || isTempDb ); + /* The BTREE_SORTER flag is only used if SQLITE_OMIT_MERGE_SORT is undef */ +#ifdef SQLITE_OMIT_MERGE_SORT + assert( (flags & BTREE_SORTER)==0 ); +#endif + + /* BTREE_SORTER is always on a BTREE_SINGLE, BTREE_OMIT_JOURNAL */ + assert( (flags & BTREE_SORTER)==0 || + (flags & (BTREE_SINGLE|BTREE_OMIT_JOURNAL)) + ==(BTREE_SINGLE|BTREE_OMIT_JOURNAL) ); + if( db->flags & SQLITE_NoReadlock ){ flags |= BTREE_NO_READLOCK; } if( isMemdb ){ flags |= BTREE_MEMORY; + flags &= ~BTREE_SORTER; } if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; @@ -8174,5 +8185,3 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ pBt->doNotUseWAL = 0; return rc; } - - diff --git a/src/btree.h b/src/btree.h index 9e3a73b3b6..ce19826ad8 100644 --- a/src/btree.h +++ b/src/btree.h @@ -61,6 +61,7 @@ int sqlite3BtreeOpen( #define BTREE_MEMORY 4 /* This is an in-memory DB */ #define BTREE_SINGLE 8 /* The file contains at most 1 b-tree */ #define BTREE_UNORDERED 16 /* Use of a hash implementation is OK */ +#define BTREE_SORTER 32 /* Used as accumulator in external merge sort */ int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); diff --git a/src/build.c b/src/build.c index 32c23855c1..29fbf92713 100644 --- a/src/build.c +++ b/src/build.c @@ -2372,6 +2372,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ if( bUseSorter ){ iSorter = pParse->nTab++; sqlite3VdbeAddOp4(v, OP_OpenSorter, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); + sqlite3VdbeChangeP5(v, BTREE_SORTER); } /* Open the table. Loop through all rows of the table, inserting index diff --git a/src/pager.c b/src/pager.c index 7ff9a9a000..373d06aeca 100644 --- a/src/pager.c +++ b/src/pager.c @@ -620,6 +620,8 @@ struct Pager { u8 tempFile; /* zFilename is a temporary file */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ + u8 hasSeenStress; /* pagerStress() called one or more times */ + u8 isSorter; /* True for a PAGER_SORTER */ /************************************************************************** ** The following block contains those class members that change during @@ -843,6 +845,15 @@ static int assert_pager_state(Pager *p){ assert( pagerUseWal(p)==0 ); } + /* A sorter is a temp file that never spills to disk and always has + ** the doNotSpill flag set + */ + if( p->isSorter ){ + assert( p->tempFile ); + assert( p->doNotSpill ); + assert( p->fd->pMethods==0 ); + } + /* If changeCountDone is set, a RESERVED lock or greater must be held ** on the file. */ @@ -3739,6 +3750,7 @@ static int pagerSyncHotJournal(Pager *pPager){ int sqlite3PagerClose(Pager *pPager){ u8 *pTmp = (u8 *)pPager->pTmpSpace; + assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); /* pPager->errCode = 0; */ @@ -4173,6 +4185,7 @@ static int pagerStress(void *p, PgHdr *pPg){ ** be called in the error state. Nevertheless, we include a NEVER() ** test for the error state as a safeguard against future changes. */ + pPager->hasSeenStress = 1; if( NEVER(pPager->errCode) ) return SQLITE_OK; if( pPager->doNotSpill ) return SQLITE_OK; if( pPager->doNotSyncSpill && (pPg->flags & PGHDR_NEED_SYNC)!=0 ){ @@ -4544,6 +4557,12 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ +#ifndef SQLITE_OMIT_MERGE_SORT + if( flags & PAGER_SORTER ){ + pPager->doNotSpill = 1; + pPager->isSorter = 1; + } +#endif *ppPager = pPager; return SQLITE_OK; @@ -6088,6 +6107,17 @@ int sqlite3PagerIsMemdb(Pager *pPager){ return MEMDB; } +#ifndef SQLITE_OMIT_MERGE_SORT +/* +** Return true if the pager has seen a pagerStress callback. +*/ +int sqlite3PagerUnderStress(Pager *pPager){ + assert( pPager->isSorter ); + assert( pPager->doNotSpill ); + return pPager->hasSeenStress; +} +#endif + /* ** Check that there are at least nSavepoint savepoints open. If there are ** currently less than nSavepoints open, then open one or more savepoints diff --git a/src/pager.h b/src/pager.h index eab7ddaf80..ccd7467d6a 100644 --- a/src/pager.h +++ b/src/pager.h @@ -60,6 +60,7 @@ typedef struct PgHdr DbPage; #define PAGER_OMIT_JOURNAL 0x0001 /* Do not use a rollback journal */ #define PAGER_NO_READLOCK 0x0002 /* Omit readlocks on readonly files */ #define PAGER_MEMORY 0x0004 /* In-memory database */ +#define PAGER_SORTER 0x0020 /* Accumulator in external merge sort */ /* ** Valid values for the second argument to sqlite3PagerLockingMode(). @@ -155,6 +156,9 @@ const char *sqlite3PagerJournalname(Pager*); int sqlite3PagerNosync(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); +#ifndef SQLITE_OMIT_MERGE_SORT +int sqlite3PagerUnderStress(Pager*); +#endif /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); diff --git a/src/vdbe.c b/src/vdbe.c index be07f60872..8491306507 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3131,7 +3131,7 @@ case OP_OpenWrite: { break; } -/* Opcode: OpenEphemeral P1 P2 * P4 * +/* Opcode: OpenEphemeral P1 P2 * P4 P5 ** ** Open a new cursor P1 to a transient table. ** The cursor is always opened read/write even if @@ -3148,6 +3148,11 @@ case OP_OpenWrite: { ** to a TEMP table at the SQL level, or to a table opened by ** this opcode. Then this opcode was call OpenVirtual. But ** that created confusion with the whole virtual-table idea. +** +** The P5 parameter can be a mask of the BTREE_* flags defined +** in btree.h. These flags control aspects of the operation of +** the btree. The BTREE_OMIT_JOURNAL and BTREE_SINGLE flags are +** added automatically. */ /* Opcode: OpenAutoindex P1 P2 * P4 * ** @@ -3174,6 +3179,7 @@ case OP_OpenEphemeral: { SQLITE_OPEN_TRANSIENT_DB; assert( pOp->p1>=0 ); + assert( (pOp->opcode==OP_OpenSorter)==((pOp->p5 & BTREE_SORTER)!=0) ); pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; diff --git a/src/vdbesort.c b/src/vdbesort.c index 9e8d5cda07..be99d397d3 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -388,8 +388,10 @@ static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; int res = 0; + /* sqlite3BtreeFirst() cannot fail because sorter btrees are always held + ** in memory and so an I/O error is not possible. */ rc = sqlite3BtreeFirst(pCsr->pCursor, &res); - if( rc!=SQLITE_OK || res ) return rc; + if( NEVER(rc!=SQLITE_OK) || res ) return rc; assert( pSorter->nBtree>0 ); /* If the first temporary PMA file has not been opened, open it now. */ @@ -429,8 +431,9 @@ static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){ /* Write the record itself to the output file */ if( rc==SQLITE_OK ){ + /* sqlite3BtreeKey() cannot fail because sorter btrees held in memory */ rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc); - if( rc==SQLITE_OK ){ + if( ALWAYS(rc==SQLITE_OK) ){ rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff); iWriteOff += nKey; } @@ -474,6 +477,9 @@ int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){ Pager *pPager = sqlite3BtreePager(pCsr->pBt); int nPage; /* Current size of temporary file in pages */ + /* Sorters never spill to disk */ + assert( sqlite3PagerFile(pPager)->pMethods==0 ); + /* Determine how many pages the temporary b-tree has grown to */ sqlite3PagerPagecount(pPager, &nPage); @@ -483,7 +489,7 @@ int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){ ** also possible that sqlite3_release_memory() was called). So set the ** size of the working set to a little less than the current size of the ** file in pages. */ - if( pSorter->nWorking==0 && sqlite3PagerFile(pPager)->pMethods ){ + if( pSorter->nWorking==0 && sqlite3PagerUnderStress(pPager) ){ pSorter->nWorking = nPage-5; if( pSorter->nWorkingnWorking = SORTER_MIN_WORKING;