-C Use\sOP_SorterOpen\sinstead\sof\sOP_OpenEphemeral\sto\simplement\sGROUP\sBY.
-D 2011-09-01T16:01:27.777
+C Instead\sof\sa\stemporary\sb-tree,\suse\sa\slinked-list\sand\smerge-sort\sto\ssort\srecords\sin\smain\smemory\sin\svdbesort.c.
+D 2011-09-02T10:31:11.173
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3
F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef
F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
-F src/btree.c 4a2856b3bde9959986a7b9327841b3ff94023784
-F src/btree.h 9ddf04226eac592d4cc3709c5a8b33b2351ff5f7
+F src/btree.c bc2099e7d3c22c52b2c54349b9c07c04f2a810d0
+F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce
F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3
-F src/build.c 2d5de52df616a3bf5a659cbca85211c46e2ba9bd
+F src/build.c dc367138cb3625e6d42b389e05d7267aece5753c
F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a
F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
F src/ctime.c e3132ec65240b2e2f3d50831021eac387f27584d
F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3
F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8
-F src/expr.c 4bbdfaf66bc614be9254ce0c26a17429067a3e07
+F src/expr.c cbcd8c2f1588a9862291a081699854c5e1cb28ab
F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
F src/fkey.c 9f00ea98f6b360d477b5a78b5b59a1fbde82431c
F src/func.c 59bb046d7e3df1ab512ac339ccb0a6f996a17cb7
F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50
F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4
F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697
-F src/select.c 037ee5501fe0e743fa98936902c200ed9ed69156
+F src/select.c 32d0f4e5513362706b8973e7f1b87cd0885dfbf5
F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd
F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb
F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93
-F src/sqliteInt.h f6debf9a9eb8463ab2ef8be4b2b740ea9af5afba
+F src/sqliteInt.h 723cda73a33c91f5a0a145f4c0ced45d94921079
F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf
F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0
F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7
F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e
-F src/vdbe.c 9260e5138855399bea2611a29da336688bfa1b79
+F src/vdbe.c da9c7efc48dc79d7785f3f17a1c3df514bf18489
F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86
-F src/vdbeInt.h 51a902e12c7d571e3b516e5407e30f996494aafe
+F src/vdbeInt.h a255da14be8c2794ce38e0d2142877bb29df9105
F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98
F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88
F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3
F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9
-F src/vdbesort.c f3d043a1bab7409d4a23cd7a35287c3ac440a167
+F src/vdbesort.c 9c2e8ca23c2413a5162a4433cb72377588d896f8
F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114
F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582
F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9
F test/index.test b5429732b3b983fa810e3ac867d7ca85dae35097
F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6
F test/index3.test 423a25c789fc8cc51aaf2a4370bbdde2d9e9eed7
-F test/index4.test c82a59c9ae2ac01804bdb100162dca057318f40f
+F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026
F test/indexedby.test be501e381b82b2f8ab406309ba7aac46e221f4ad
F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2
-P bab2e560f6cb989c83a96aad60f666960ede7abe
-R 47ae22012e17b717f429ee6a9d305e96
-U drh
-Z 4edda214a8d1e652ef01c1f98ad8fd39
+P ebf819aaa555bd79fddfc0a6f9827a2539095d6c
+R c9d892df81c49366e067933871c3b776
+U dan
+Z b3581bd03fb5a181801f57c568eb418b
int iIdx = pParse->nTab++; /* Btree cursor used for pIndex */
int iSorter = iTab; /* Cursor opened by OpenSorter (if in use) */
int addr1; /* Address of top of loop */
+ int addr2; /* Address to jump to for next iteration */
int tnum; /* Root page of index */
Vdbe *v; /* Generate code into this virtual machine */
KeyInfo *pKey; /* KeyInfo for index */
if( bUseSorter ){
iSorter = pParse->nTab++;
sqlite3VdbeAddOp4(v, OP_OpenSorter, iSorter, 0, 0, (char*)pKey, P4_KEYINFO);
- sqlite3VdbeChangeP5(v, BTREE_SORTER);
}
/* Open the table. Loop through all rows of the table, inserting index
** records into the sorter. */
sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead);
addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0);
+ addr2 = addr1 + 1;
regRecord = sqlite3GetTempReg(pParse);
regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1);
if( bUseSorter ){
- sqlite3VdbeAddOp2(v, OP_IdxInsert, iSorter, regRecord);
+ sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord);
sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1);
sqlite3VdbeJumpHere(v, addr1);
- addr1 = sqlite3VdbeAddOp2(v, OP_Sort, iSorter, 0);
- sqlite3VdbeAddOp2(v, OP_RowKey, iSorter, regRecord);
- }
-
- if( pIndex->onError!=OE_None ){
+ addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0);
+ if( pIndex->onError!=OE_None ){
+ int j2 = sqlite3VdbeCurrentAddr(v) + 3;
+ sqlite3VdbeAddOp2(v, OP_Goto, 0, j2);
+ addr2 = sqlite3VdbeCurrentAddr(v);
+ sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord);
+ sqlite3HaltConstraint(
+ pParse, OE_Abort, "indexed columns are not unique", P4_STATIC
+ );
+ }else{
+ addr2 = sqlite3VdbeCurrentAddr(v);
+ }
+ sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord);
+ }else if( pIndex->onError!=OE_None ){
const int regRowid = regIdxKey + pIndex->nColumn;
const int j2 = sqlite3VdbeCurrentAddr(v) + 2;
void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey);
sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, bUseSorter);
sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
sqlite3ReleaseTempReg(pParse, regRecord);
- sqlite3VdbeAddOp2(v, OP_Next, iSorter, addr1+1);
+ sqlite3VdbeAddOp2(v, bUseSorter ? OP_SorterNext : OP_Next, iSorter, addr2);
sqlite3VdbeJumpHere(v, addr1);
sqlite3VdbeAddOp1(v, OP_Close, iTab);
** by this opcode will be used for automatically created transient
** indices in joins.
*/
-/* Opcode: OpenSorter P1 P2 * P4 *
-**
-** This opcode works like OP_OpenEphemeral except that it opens
-** a transient index that is specifically designed to sort large
-** tables using an external merge-sort algorithm.
-*/
-case OP_OpenSorter:
case OP_OpenAutoindex:
-case OP_SorterOpen:
case OP_OpenEphemeral: {
VdbeCursor *pCx;
static const int vfsFlags =
SQLITE_OPEN_TRANSIENT_DB;
assert( pOp->p1>=0 );
- assert( (pOp->opcode==OP_OpenSorter)==((pOp->p5 & BTREE_SORTER)!=0) );
pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1);
if( pCx==0 ) goto no_mem;
pCx->nullRow = 1;
}
pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED);
pCx->isIndex = !pCx->isTable;
- pCx->isSorter = pOp->opcode==OP_SorterOpen;
-#ifndef SQLITE_OMIT_MERGE_SORT
- if( rc==SQLITE_OK && pOp->opcode==OP_OpenSorter ){
- rc = sqlite3VdbeSorterInit(db, pCx);
- }
-#endif
+ break;
+}
+
+/* Opcode: OpenSorter P1 P2 * P4 *
+**
+** This opcode works like OP_OpenEphemeral except that it opens
+** a transient index that is specifically designed to sort large
+** tables using an external merge-sort algorithm.
+*/
+case OP_SorterOpen:
+case OP_OpenSorter: {
+ VdbeCursor *pCx;
+ pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1);
+ if( pCx==0 ) goto no_mem;
+ pCx->pKeyInfo = pOp->p4.pKeyInfo;
+ pCx->pKeyInfo->enc = ENC(p->db);
+ pCx->isSorter = 1;
+ rc = sqlite3VdbeSorterInit(db, pCx);
break;
}
break;
}
+/* Opcode: SorterCompare P1 P2 P3
+**
+** P1 is a sorter cursor. This instruction compares the record blob in
+** register P3 with the entry that the sorter cursor currently points to.
+** If, excluding the rowid fields at the end, the two records are a match,
+** fall through to the next instruction. Otherwise, jump to instruction P2.
+*/
+case OP_SorterCompare: {
+ VdbeCursor *pC;
+ int res;
+
+ pC = p->apCsr[pOp->p1];
+ assert( isSorter(pC) );
+ pIn3 = &aMem[pOp->p3];
+ rc = sqlite3VdbeSorterCompare(pC, pIn3, &res);
+ if( res ){
+ pc = pOp->p2-1;
+ }
+ break;
+};
+
+/* Opcode: SorterData P1 P2 * * *
+**
+** Write into register P2 the current sorter data for sorter cursor P1.
+*/
+case OP_SorterData: {
+ VdbeCursor *pC;
+ pOut = &aMem[pOp->p2];
+ pC = p->apCsr[pOp->p1];
+ assert( pC->isSorter );
+ rc = sqlite3VdbeSorterRowkey(pC, pOut);
+ break;
+}
+
/* Opcode: RowData P1 P2 * * *
**
** Write into register P2 the complete row data for cursor P1.
** If the P1 cursor must be pointing to a valid row (not a NULL row)
** of a real table, not a pseudo-table.
*/
-case OP_SorterData:
case OP_RowKey:
case OP_RowData: {
VdbeCursor *pC;
/* Note that RowKey and RowData are really exactly the same instruction */
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
pC = p->apCsr[pOp->p1];
+ assert( pC->isSorter==0 );
assert( pC->isTable || pOp->opcode!=OP_RowData );
assert( pC->isIndex || pOp->opcode==OP_RowData );
assert( pC!=0 );
assert( pC->pseudoTableReg==0 );
assert( pC->isSorter==(pOp->opcode==OP_SorterData) );
- if( isSorter(pC) ){
- assert( pOp->opcode==OP_RowKey );
- rc = sqlite3VdbeSorterRowkey(pC, pOut);
- break;
- }
-
assert( pC->pCursor!=0 );
pCrsr = pC->pCursor;
assert( sqlite3BtreeCursorIsValid(pCrsr) );
}
assert( pC->isSorter==(pOp->opcode==OP_SorterNext) );
if( isSorter(pC) ){
- assert( pOp->opcode==OP_Next );
+ assert( pOp->opcode==OP_SorterNext );
rc = sqlite3VdbeSorterNext(db, pC, &res);
}else{
res = 1;
assert( pC->isTable==0 );
rc = ExpandBlob(pIn2);
if( rc==SQLITE_OK ){
- nKey = pIn2->n;
- zKey = pIn2->z;
- rc = sqlite3VdbeSorterWrite(db, pC, nKey);
- if( rc==SQLITE_OK ){
+ if( isSorter(pC) ){
+ rc = sqlite3VdbeSorterWrite(db, pC, pIn2);
+ }else{
+ nKey = pIn2->n;
+ zKey = pIn2->z;
rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3,
((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0)
- );
+ );
assert( pC->deferredMoveto==0 );
+ pC->cacheStatus = CACHE_STALE;
}
- pC->cacheStatus = CACHE_STALE;
}
}
break;
#ifndef SQLITE_OMIT_MERGE_SORT
typedef struct VdbeSorterIter VdbeSorterIter;
+typedef struct SorterRecord SorterRecord;
/*
** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES:
** being merged (rounded up to the next power of 2).
*/
struct VdbeSorter {
- int nWorking; /* Start a new b-tree after this many pages */
- int nBtree; /* Current size of b-tree contents as PMA */
+ int nInMemory; /* Current size of b-tree contents as PMA */
int nTree; /* Used size of aTree/aIter (power of 2) */
VdbeSorterIter *aIter; /* Array of iterators to merge */
int *aTree; /* Current state of incremental merge */
i64 iReadOff; /* Current read offset within file pTemp1 */
sqlite3_file *pTemp1; /* PMA file 1 */
int nPMA; /* Number of PMAs stored in pTemp1 */
+ SorterRecord *pRecord; /* Head of in-memory record list */
+ int nLimit1; /* Minimum PMA size, in bytes */
+ int nLimit2; /* Maximum PMA size, in bytes */
};
/*
u8 *aKey; /* Pointer to current key */
};
+/*
+** A structure to store a single record. All in-memory records are connected
+** together into a linked list headed at VdbeSorter.pRecord using the
+** SorterRecord.pNext pointer.
+*/
+struct SorterRecord {
+ void *pVal;
+ int nVal;
+ SorterRecord *pNext;
+};
+
/* Minimum allowable value for the VdbeSorter.nWorking variable */
#define SORTER_MIN_WORKING 10
return rc;
}
+
+/*
+** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2,
+** size nKey2 bytes). Argument pKeyInfo supplies the collation functions
+** used by the comparison. If an error occurs, return an SQLite error code.
+** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive
+** value, depending on whether key1 is smaller, equal to or larger than key2.
+**
+** If the bOmitRowid argument is non-zero, assume both keys end in a rowid
+** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid
+** is true and key1 contains even a single NULL value, it is considered to
+** be less than key2. Even if key2 also contains NULL values.
+*/
+static int vdbeSorterCompare(
+ KeyInfo *pKeyInfo, /* Collation functions to use in comparison */
+ int bOmitRowid, /* Ignore rowid field at end of keys */
+ void *pKey1, int nKey1, /* Left side of comparison */
+ void *pKey2, int nKey2, /* Right side of comparison */
+ int *pRes /* OUT: Result of comparison */
+){
+ char aSpace[150];
+ UnpackedRecord *r2;
+ int i;
+
+ r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, sizeof(aSpace));
+ if( r2==0 ) return SQLITE_NOMEM;
+ if( bOmitRowid ){
+ for(i=0; i<r2->nField-1; i++){
+ if( r2->aMem[i].flags & MEM_Null ){
+ *pRes = -1;
+ sqlite3VdbeDeleteUnpackedRecord(r2);
+ return SQLITE_OK;
+ }
+ }
+ r2->flags |= UNPACKED_PREFIX_MATCH;
+ r2->nField--;
+ assert( r2->nField>0 );
+ }
+
+ *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2);
+ sqlite3VdbeDeleteUnpackedRecord(r2);
+ return SQLITE_OK;
+}
+
/*
** This function is called to compare two iterator keys when merging
** multiple b-tree segments. Parameter iOut is the index of the aTree[]
}else if( p2->pFile==0 ){
iRes = i1;
}else{
- char aSpace[150];
- UnpackedRecord *r1;
-
- r1 = sqlite3VdbeRecordUnpack(
- pCsr->pKeyInfo, p1->nKey, p1->aKey, aSpace, sizeof(aSpace)
+ int res;
+ int rc = vdbeSorterCompare(
+ pCsr->pKeyInfo, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res
);
- if( r1==0 ) return SQLITE_NOMEM;
-
- if( sqlite3VdbeRecordCompare(p2->nKey, p2->aKey, r1)>=0 ){
+ if( rc!=SQLITE_OK ) return rc;
+ if( res<=0 ){
iRes = i1;
}else{
iRes = i2;
}
- sqlite3VdbeDeleteUnpackedRecord(r1);
}
pSorter->aTree[iOut] = iRes;
** Initialize the temporary index cursor just opened as a sorter cursor.
*/
int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){
- assert( pCsr->pKeyInfo && pCsr->pBt );
+ int pgsz; /* Page size of main database */
+
+ assert( pCsr->pKeyInfo && pCsr->pBt==0 );
pCsr->pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter));
- return (pCsr->pSorter ? SQLITE_OK : SQLITE_NOMEM);
+ if( pCsr->pSorter==0 ){
+ return SQLITE_NOMEM;
+ }
+
+ pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
+ pCsr->pSorter->nLimit1 = 10 * pgsz;
+ pCsr->pSorter->nLimit2 = db->aDb[0].pSchema->cache_size * pgsz;
+
+ return SQLITE_OK;
+}
+
+/*
+** Free the list of sorted records starting at pRecord.
+*/
+static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){
+ SorterRecord *p;
+ SorterRecord *pNext;
+ for(p=pRecord; p; p=pNext){
+ pNext = p->pNext;
+ sqlite3DbFree(db, p->pVal);
+ sqlite3DbFree(db, p);
+ }
}
/*
if( pSorter->pTemp1 ){
sqlite3OsCloseFree(pSorter->pTemp1);
}
+ vdbeSorterRecordFree(db, pSorter->pRecord);
sqlite3DbFree(db, pSorter);
pCsr->pSorter = 0;
}
);
}
+/*
+** Attemp to merge the two sorted lists p1 and p2 into a single list. If no
+** error occurs set *ppOut to the head of the new list and return SQLITE_OK.
+*/
+static int vdbeSorterMerge(
+ sqlite3 *db, /* Database handle */
+ KeyInfo *pKeyInfo, /* Collation functions to use in comparison */
+ SorterRecord *p1, /* First list to merge */
+ SorterRecord *p2, /* Second list to merge */
+ SorterRecord **ppOut /* OUT: Head of merged list */
+){
+ int rc = SQLITE_OK;
+ SorterRecord *pFinal = 0;
+ SorterRecord **pp = &pFinal;
+
+ while( p1 || p2 ){
+ if( p1==0 ){
+ *pp = p2;
+ p2 = 0;
+ }else if( p2==0 ){
+ *pp = p1;
+ p1 = 0;
+ }else{
+ int res;
+ rc = vdbeSorterCompare(
+ pKeyInfo, 0, p1->pVal, p1->nVal, p2->pVal, p2->nVal, &res
+ );
+ if( rc!=SQLITE_OK ){
+ vdbeSorterRecordFree(db, p1);
+ vdbeSorterRecordFree(db, p2);
+ vdbeSorterRecordFree(db, pFinal);
+ pFinal = 0;
+ break;
+ }
+ if( res<=0 ){
+ *pp = p1;
+ pp = &p1->pNext;
+ p1 = p1->pNext;
+ }else{
+ *pp = p2;
+ pp = &p2->pNext;
+ p2 = p2->pNext;
+ }
+ *pp = 0;
+ }
+ }
+
+ *ppOut = pFinal;
+ return rc;
+}
+
+/*
+** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK
+** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error
+** occurs.
+*/
+static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){
+ int rc = SQLITE_OK;
+ int i;
+ SorterRecord **aSlot;
+ SorterRecord *p;
+ VdbeSorter *pSorter = pCsr->pSorter;
+ KeyInfo *pKeyInfo = pCsr->pKeyInfo;
+
+ aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *));
+ if( !aSlot ){
+ return SQLITE_NOMEM;
+ }
+
+ p = pSorter->pRecord;
+ while( p ){
+ SorterRecord *pNext = p->pNext;
+ p->pNext = 0;
+ for(i=0; rc==SQLITE_OK && aSlot[i]; i++){
+ rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p);
+ aSlot[i] = 0;
+ }
+ if( rc!=SQLITE_OK ){
+ vdbeSorterRecordFree(db, pNext);
+ break;
+ }
+ aSlot[i] = p;
+ p = pNext;
+ }
+
+ p = 0;
+ for(i=0; i<64; i++){
+ if( rc==SQLITE_OK ){
+ rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p);
+ }else{
+ vdbeSorterRecordFree(db, aSlot[i]);
+ }
+ }
+ pSorter->pRecord = p;
+
+#if 0
+ {
+ SorterRecord *pTmp1 = 0;
+ SorterRecord *pTmp2;
+ for(pTmp2=pSorter->pRecord; pTmp2 && rc==SQLITE_OK; pTmp2=pTmp2->pNext){
+ if( pTmp1 ){
+ int res;
+ rc = vdbeSorterCompare(pKeyInfo,
+ 0, pTmp1->pVal, pTmp1->nVal, pTmp2->pVal, pTmp2->nVal, &res
+ );
+ assert( rc!=SQLITE_OK || res<0 );
+ }
+ pTmp1 = pTmp2;
+ }
+ }
+#endif
+
+ if( rc!=SQLITE_OK ){
+ }
+ sqlite3_free(aSlot);
+ return rc;
+}
+
/*
-** Write the current contents of the b-tree to a PMA. Return SQLITE_OK
-** if successful, or an SQLite error code otherwise.
+** Write the current contents of the in-memory linked-list to a PMA. Return
+** SQLITE_OK if successful, or an SQLite error code otherwise.
**
** The format of a PMA is:
**
** Each record consists of a varint followed by a blob of data (the
** key). The varint is the number of bytes in the blob of data.
*/
-static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){
+static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){
int rc = SQLITE_OK; /* Return code */
VdbeSorter *pSorter = pCsr->pSorter;
- int res = 0;
- /* sqlite3BtreeFirst() cannot fail because sorter btrees are always held
- ** in memory and so an I/O error is not possible. */
- rc = sqlite3BtreeFirst(pCsr->pCursor, &res);
- if( NEVER(rc!=SQLITE_OK) || res ) return rc;
- assert( pSorter->nBtree>0 );
+ if( pSorter->nInMemory==0 ){
+ assert( pSorter->pRecord==0 );
+ return rc;
+ }
+
+ rc = vdbeSorterSort(db, pCsr);
/* If the first temporary PMA file has not been opened, open it now. */
- if( pSorter->pTemp1==0 ){
+ if( rc==SQLITE_OK && pSorter->pTemp1==0 ){
rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1);
assert( rc!=SQLITE_OK || pSorter->pTemp1 );
assert( pSorter->iWriteOff==0 );
}
if( rc==SQLITE_OK ){
- i64 iWriteOff = pSorter->iWriteOff;
- void *aMalloc = 0; /* Array used to hold a single record */
- int nMalloc = 0; /* Allocated size of aMalloc[] in bytes */
+ i64 iOff = pSorter->iWriteOff;
+ SorterRecord *p;
+ SorterRecord *pNext = 0;
pSorter->nPMA++;
- for(
- rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nBtree, &iWriteOff);
- rc==SQLITE_OK && res==0;
- rc = sqlite3BtreeNext(pCsr->pCursor, &res)
- ){
- i64 nKey; /* Size of this key in bytes */
-
- /* Write the size of the record in bytes to the output file */
- (void)sqlite3BtreeKeySize(pCsr->pCursor, &nKey);
- rc = vdbeSorterWriteVarint(pSorter->pTemp1, nKey, &iWriteOff);
-
- /* Make sure the aMalloc[] buffer is large enough for the record */
- if( rc==SQLITE_OK && nKey>nMalloc ){
- aMalloc = sqlite3DbReallocOrFree(db, aMalloc, nKey);
- if( !aMalloc ){
- rc = SQLITE_NOMEM;
- }else{
- nMalloc = nKey;
- }
- }
+ rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nInMemory, &iOff);
+ for(p=pSorter->pRecord; rc==SQLITE_OK && p; p=pNext){
+ pNext = p->pNext;
+ rc = vdbeSorterWriteVarint(pSorter->pTemp1, p->nVal, &iOff);
- /* Write the record itself to the output file */
if( rc==SQLITE_OK ){
- /* sqlite3BtreeKey() cannot fail because sorter btrees held in memory */
- rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc);
- if( ALWAYS(rc==SQLITE_OK) ){
- rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff);
- iWriteOff += nKey;
- }
+ rc = sqlite3OsWrite(pSorter->pTemp1, p->pVal, p->nVal, iOff);
+ iOff += p->nVal;
}
- if( rc!=SQLITE_OK ) break;
+ sqlite3DbFree(db, p->pVal);
+ sqlite3DbFree(db, p);
}
/* This assert verifies that unless an error has occurred, the size of
** the PMA on disk is the same as the expected size stored in
- ** pSorter->nBtree. */
- assert( rc!=SQLITE_OK || pSorter->nBtree==(
- iWriteOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nBtree)
+ ** pSorter->nInMemory. */
+ assert( rc!=SQLITE_OK || pSorter->nInMemory==(
+ iOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nInMemory)
));
- pSorter->iWriteOff = iWriteOff;
- sqlite3DbFree(db, aMalloc);
+ pSorter->iWriteOff = iOff;
+ pSorter->pRecord = p;
}
- pSorter->nBtree = 0;
return rc;
}
/*
-** This function is called on a sorter cursor by the VDBE before each row
-** is inserted into VdbeCursor.pCsr. Argument nKey is the size of the key, in
-** bytes, about to be inserted.
-**
-** If it is determined that the temporary b-tree accessed via VdbeCursor.pCsr
-** is large enough, its contents are written to a sorted PMA on disk and the
-** tree emptied. This prevents the b-tree (which must be small enough to
-** fit entirely in the cache in order to support efficient inserts) from
-** growing too large.
-**
-** An SQLite error code is returned if an error occurs. Otherwise, SQLITE_OK.
+** Add a record to the sorter.
*/
-int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){
- int rc = SQLITE_OK; /* Return code */
+int sqlite3VdbeSorterWrite(
+ sqlite3 *db, /* Database handle */
+ VdbeCursor *pCsr, /* Sorter cursor */
+ Mem *pVal /* Memory cell containing record */
+){
VdbeSorter *pSorter = pCsr->pSorter;
- if( pSorter ){
- Pager *pPager = sqlite3BtreePager(pCsr->pBt);
- int nPage; /* Current size of temporary file in pages */
-
- /* Sorters never spill to disk */
- assert( sqlite3PagerFile(pPager)->pMethods==0 );
-
- /* Determine how many pages the temporary b-tree has grown to */
- sqlite3PagerPagecount(pPager, &nPage);
-
- /* If pSorter->nWorking is still zero, but the temporary file has been
- ** created in the file-system, then the most recent insert into the
- ** current b-tree segment probably caused the cache to overflow (it is
- ** also possible that sqlite3_release_memory() was called). So set the
- ** size of the working set to a little less than the current size of the
- ** file in pages. */
- if( pSorter->nWorking==0 && sqlite3PagerUnderStress(pPager) ){
- pSorter->nWorking = nPage-5;
- if( pSorter->nWorking<SORTER_MIN_WORKING ){
- pSorter->nWorking = SORTER_MIN_WORKING;
- }
- }
-
- /* If the number of pages used by the current b-tree segment is greater
- ** than the size of the working set (VdbeSorter.nWorking), start a new
- ** segment b-tree. */
- if( pSorter->nWorking && nPage>=pSorter->nWorking ){
- BtCursor *p = pCsr->pCursor;/* Cursor structure to close and reopen */
- int iRoot; /* Root page of new tree */
+ int rc;
+ SorterRecord *pNew;
- /* Copy the current contents of the b-tree into a PMA in sorted order.
- ** Close the currently open b-tree cursor. */
- rc = vdbeSorterBtreeToPMA(db, pCsr);
- sqlite3BtreeCloseCursor(p);
+ assert( pSorter );
+ pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n;
- if( rc==SQLITE_OK ){
- rc = sqlite3BtreeDropTable(pCsr->pBt, 2, 0);
-#ifdef SQLITE_DEBUG
- sqlite3PagerPagecount(pPager, &nPage);
- assert( rc!=SQLITE_OK || nPage==1 );
-#endif
- }
- if( rc==SQLITE_OK ){
- rc = sqlite3BtreeCreateTable(pCsr->pBt, &iRoot, BTREE_BLOBKEY);
- }
- if( rc==SQLITE_OK ){
- assert( iRoot==2 );
- rc = sqlite3BtreeCursor(pCsr->pBt, iRoot, 1, pCsr->pKeyInfo, p);
- }
+ pNew = (SorterRecord *)sqlite3DbMallocZero(db, sizeof(SorterRecord));
+ if( pNew==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3VdbeMemMakeWriteable(pVal);
+ if( rc==SQLITE_OK ){
+ pNew->pVal = pVal->z;
+ pNew->nVal = pVal->n;
+ pVal->zMalloc = 0;
+ sqlite3VdbeMemSetNull(pVal);
+ pNew->pNext = pSorter->pRecord;
+ pSorter->pRecord = pNew;
+ }else{
+ sqlite3DbFree(db, pNew);
+ rc = SQLITE_NOMEM;
}
+ }
- pSorter->nBtree += sqlite3VarintLen(nKey) + nKey;
+ /* See if the contents of the sorter should now be written out. They
+ ** are written out when either of the following are true:
+ **
+ ** * The total memory allocated for the in-memory list is greater
+ ** than (page-size * cache-size), or
+ **
+ ** * The total memory allocated for the in-memory list is greater
+ ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true.
+ */
+ if( rc==SQLITE_OK && (
+ (pSorter->nInMemory>pSorter->nLimit2)
+ || (pSorter->nInMemory>pSorter->nLimit1 && sqlite3HeapNearlyFull())
+ )){
+ rc = vdbeSorterListToPMA(db, pCsr);
+ pSorter->nInMemory = 0;
}
+
return rc;
}
assert( pSorter );
/* Write the current b-tree to a PMA. Close the b-tree cursor. */
- rc = vdbeSorterBtreeToPMA(db, pCsr);
- sqlite3BtreeCloseCursor(pCsr->pCursor);
+ rc = vdbeSorterListToPMA(db, pCsr);
if( rc!=SQLITE_OK ) return rc;
if( pSorter->nPMA==0 ){
*pbEof = 1;
VdbeSorterIter *pIter;
pIter = &pSorter->aIter[ pSorter->aTree[1] ];
-
- /* Coverage testing note: As things are currently, this call will always
- ** succeed. This is because the memory cell passed by the VDBE layer
- ** happens to be the same one as was used to assemble the keys before they
- ** were passed to the sorter - meaning it is always large enough for the
- ** largest key. But this could change very easily, so we leave the call
- ** to sqlite3VdbeMemGrow() in. */
- if( NEVER(sqlite3VdbeMemGrow(pOut, pIter->nKey, 0)) ){
+ if( sqlite3VdbeMemGrow(pOut, pIter->nKey, 0) ){
return SQLITE_NOMEM;
}
pOut->n = pIter->nKey;
return SQLITE_OK;
}
+/*
+** Compare the key in memory cell pVal with the key that the sorter cursor
+** passed as the first argument currently points to. For the purposes of
+** the comparison, ignore the rowid field at the end of each record.
+**
+** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM).
+** Otherwise, set *pRes to a negative, zero or positive value if the
+** key in pVal is smaller than, equal to or larger than the current sorter
+** key.
+*/
+int sqlite3VdbeSorterCompare(
+ VdbeCursor *pCsr, /* Sorter cursor */
+ Mem *pVal, /* Value to compare to current sorter key */
+ int *pRes /* OUT: Result of comparison */
+){
+ int rc;
+ VdbeSorter *pSorter = pCsr->pSorter;
+ VdbeSorterIter *pIter;
+ pIter = &pSorter->aIter[ pSorter->aTree[1] ];
+ rc = vdbeSorterCompare(pCsr->pKeyInfo, 1,
+ pVal->z, pVal->n, pIter->aKey, pIter->nKey, pRes
+ );
+ assert( rc!=SQLITE_OK || *pRes<=0 );
+ return rc;
+}
+
#endif /* #ifndef SQLITE_OMIT_MERGE_SORT */
projects / thirdparty / sqlite.git / commitdiff
