From: drh Date: Sat, 3 Sep 2011 00:17:51 +0000 (+0000) Subject: The build works again with -DSQLITE_OMIT_MERGE_SORT. The merge-sorter now X-Git-Tag: version-3.7.8~22^2~2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ca892a725284f1ce7d9f4f5e3c4f75f4f01f055b;p=thirdparty%2Fsqlite.git The build works again with -DSQLITE_OMIT_MERGE_SORT. The merge-sorter now avoids spilling to disk (letting the in-memory linked list grow without bound) if PRAGMA temp_store=3. FossilOrigin-Name: 68e26c4487696d194ee85370380e4b0e56d206ee --- diff --git a/manifest b/manifest index 1f98484757..6c9743bfa6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sdead\scode.\s\sFix\sa\sfaulty\sassert().\s\sImprove\ssome\svariable\snames. -D 2011-09-02T21:42:33.425 +C The\sbuild\sworks\sagain\swith\s-DSQLITE_OMIT_MERGE_SORT.\s\sThe\smerge-sorter\snow\navoids\sspilling\sto\sdisk\s(letting\sthe\sin-memory\slinked\slist\sgrow\swithout\nbound)\sif\sPRAGMA\stemp_store=3. +D 2011-09-03T00:17:51.023 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -127,7 +127,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 4d46fe30b8bc920f68b7d58a5f45316fa5d023ec F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c dc367138cb3625e6d42b389e05d7267aece5753c +F src/build.c 851e81f26a75abbb98bd99a7c5f10e8670d867bb F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c e3132ec65240b2e2f3d50831021eac387f27584d @@ -183,7 +183,7 @@ F src/select.c 32d0f4e5513362706b8973e7f1b87cd0885dfbf5 F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 -F src/sqliteInt.h 723cda73a33c91f5a0a145f4c0ced45d94921079 +F src/sqliteInt.h c7e37ee49b1a922ddcd18fa98dd750efa4d2db14 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -238,14 +238,14 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c da9c7efc48dc79d7785f3f17a1c3df514bf18489 +F src/vdbe.c d4c8224cc931c6082557501d7f822fb12f273922 F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 -F src/vdbeInt.h a255da14be8c2794ce38e0d2142877bb29df9105 +F src/vdbeInt.h 693d6ac6810298fc6b4c503cfbe3f99a240f40af F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 2ae618bec4bec5faa075e82078008763666c655d +F src/vdbesort.c aa7ad2ef91ad7d33eb614d3fb1c4cbd068b321ad F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P cf48ad8353e28339d00f448bb729e10a7f2aad72 -R d2091eb5dd8da1e99612b5b3cd229b4c +P a9a64592cf88580cb254fb0aac65a2f2085976ec +R 4edd7e2341e47b4ce23f768c53615b8c U drh -Z e655a0d761c2ed1b9d5806522d72983b +Z cbdc1caefa14410ba02a8248a6138d65 diff --git a/manifest.uuid b/manifest.uuid index 2cb8cd7e4f..2bc60c00a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9a64592cf88580cb254fb0aac65a2f2085976ec \ No newline at end of file +68e26c4487696d194ee85370380e4b0e56d206ee \ No newline at end of file diff --git a/src/build.c b/src/build.c index 0de1020b9b..27130ef281 100644 --- a/src/build.c +++ b/src/build.c @@ -2335,15 +2335,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); - /* Set bUseSorter to use OP_OpenSorter, or clear it to insert directly - ** into the index. The sorter is used unless either OMIT_MERGE_SORT is - ** defined or the system is configured to store temp files in-memory. */ -#ifdef SQLITE_OMIT_MERGE_SORT - static const int bUseSorter = 0; -#else - const int bUseSorter = !sqlite3TempInMemory(pParse->db); -#endif - #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, db->aDb[iDb].zName ) ){ @@ -2369,11 +2360,11 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeChangeP5(v, 1); } +#ifndef SQLITE_OMIT_MERGE_SORT /* Open the sorter cursor if we are to use one. */ - if( bUseSorter ){ - iSorter = pParse->nTab++; - sqlite3VdbeAddOp4(v, OP_OpenSorter, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); - } + iSorter = pParse->nTab++; + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); +#endif /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ @@ -2383,24 +2374,27 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ regRecord = sqlite3GetTempReg(pParse); regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); - if( bUseSorter ){ - sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); - sqlite3VdbeJumpHere(v, addr1); - addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); - if( pIndex->onError!=OE_None ){ - int j2 = sqlite3VdbeCurrentAddr(v) + 3; - sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); - addr2 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC - ); - }else{ - addr2 = sqlite3VdbeCurrentAddr(v); - } - sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); - }else if( pIndex->onError!=OE_None ){ +#ifndef SQLITE_OMIT_MERGE_SORT + sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); + sqlite3VdbeJumpHere(v, addr1); + addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); + if( pIndex->onError!=OE_None ){ + int j2 = sqlite3VdbeCurrentAddr(v) + 3; + sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); + addr2 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); + sqlite3HaltConstraint( + pParse, OE_Abort, "indexed columns are not unique", P4_STATIC + ); + }else{ + addr2 = sqlite3VdbeCurrentAddr(v); + } + sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); +#else + if( pIndex->onError!=OE_None ){ const int regRowid = regIdxKey + pIndex->nColumn; const int j2 = sqlite3VdbeCurrentAddr(v) + 2; void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); @@ -2418,10 +2412,11 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3HaltConstraint( pParse, OE_Abort, "indexed columns are not unique", P4_STATIC); } - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, bUseSorter); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); +#endif sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3VdbeAddOp2(v, bUseSorter ? OP_SorterNext : OP_Next, iSorter, addr2); + sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_Close, iTab); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 694f4694eb..f27fea2b3a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -372,14 +372,6 @@ # define SQLITE_TEMP_STORE 1 #endif -/* -** If all temporary storage is in-memory, then omit the external merge-sort -** logic since it is superfluous. -*/ -#if SQLITE_TEMP_STORE==3 && !defined(SQLITE_OMIT_MERGE_SORT) -# define SQLITE_OMIT_MERGE_SORT -#endif - /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. diff --git a/src/vdbe.c b/src/vdbe.c index d9310e0563..acf113515f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3215,15 +3215,19 @@ case OP_OpenEphemeral: { ** a transient index that is specifically designed to sort large ** tables using an external merge-sort algorithm. */ -case OP_SorterOpen: -case OP_OpenSorter: { +case OP_SorterOpen: { VdbeCursor *pCx; +#ifndef SQLITE_OMIT_MERGE_SORT pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; pCx->pKeyInfo->enc = ENC(p->db); pCx->isSorter = 1; rc = sqlite3VdbeSorterInit(db, pCx); +#else + pOp->opcode = OP_OpenEphemeral; + pc--; +#endif break; } @@ -4102,10 +4106,15 @@ case OP_SorterCompare: { */ case OP_SorterData: { VdbeCursor *pC; +#ifndef SQLITE_OMIT_MERGE_SORT pOut = &aMem[pOp->p2]; pC = p->apCsr[pOp->p1]; assert( pC->isSorter ); rc = sqlite3VdbeSorterRowkey(pC, pOut); +#else + pOp->opcode = OP_RowKey; + pc--; +#endif break; } @@ -4148,8 +4157,7 @@ case OP_RowData: { assert( pC!=0 ); assert( pC->nullRow==0 ); assert( pC->pseudoTableReg==0 ); - assert( pC->isSorter==(pOp->opcode==OP_SorterData) ); - + assert( !pC->isSorter ); assert( pC->pCursor!=0 ); pCrsr = pC->pCursor; assert( sqlite3BtreeCursorIsValid(pCrsr) ); @@ -4307,6 +4315,9 @@ case OP_Last: { /* jump */ ** correctly optimizing out sorts. */ case OP_SorterSort: /* jump */ +#ifdef SQLITE_OMIT_MERGE_SORT + pOp->opcode = OP_Sort; +#endif case OP_Sort: { /* jump */ #ifdef SQLITE_TEST sqlite3_sort_count++; @@ -4385,6 +4396,9 @@ case OP_Rewind: { /* jump */ ** number P5-1 in the prepared statement is incremented. */ case OP_SorterNext: /* jump */ +#ifdef SQLITE_OMIT_MERGE_SORT + pOp->opcode = OP_Next; +#endif case OP_Prev: /* jump */ case OP_Next: { /* jump */ VdbeCursor *pC; @@ -4434,7 +4448,10 @@ case OP_Next: { /* jump */ ** This instruction only works for indices. The equivalent instruction ** for tables is OP_Insert. */ -case OP_SorterInsert: +case OP_SorterInsert: /* in2 */ +#ifdef SQLITE_OMIT_MERGE_SORT + pOp->opcode = OP_IdxInsert; +#endif case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; BtCursor *pCrsr; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 68ad8dcbf7..7fbd5693fb 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -65,7 +65,7 @@ struct VdbeCursor { i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ - VdbeSorter *pSorter; /* Sorter object for OP_OpenSorter cursors */ + VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ /* Result of last sqlite3BtreeMoveto() done by an OP_NotExists or ** OP_IsUnique opcode on this cursor. */ diff --git a/src/vdbesort.c b/src/vdbesort.c index c1ea8cadd3..e8135f0f14 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -409,11 +409,13 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ return SQLITE_NOMEM; } - pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; - mxCache = db->aDb[0].pSchema->cache_size; - if( mxCachemxPmaSize = mxCache * pgsz; + if( !sqlite3TempInMemory(db) ){ + pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; + mxCache = db->aDb[0].pSchema->cache_size; + if( mxCachemxPmaSize = mxCache * pgsz; + } return SQLITE_OK; } @@ -666,7 +668,7 @@ int sqlite3VdbeSorterWrite( ** * The total memory allocated for the in-memory list is greater ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ - if( rc==SQLITE_OK && ( + if( rc==SQLITE_OK && pSorter->mxPmaSize>0 && ( (pSorter->nInMemory>pSorter->mxPmaSize) || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) )){