From: drh Date: Thu, 16 Aug 2012 20:05:43 +0000 (+0000) Subject: Attempt to use two cores to do sorting. Unfortunately, instead of making sorts X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=486e31f1b3b4fd48cbafc8b5305aa34cf5ad4d3e;p=thirdparty%2Fsqlite.git Attempt to use two cores to do sorting. Unfortunately, instead of making sorts go faster as was hoped, this changes slows sorting down by about 10%. FossilOrigin-Name: 11dd05e5984f0d5f98052458b94cbaf7d18c2e8f --- diff --git a/manifest b/manifest index 8609fc7751..21e5194d4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sthreads\sbranch\sto\sinclude\sall\sthe\slatest\strunk\schanges. -D 2012-08-16T11:24:22.980 +C Attempt\sto\suse\stwo\scores\sto\sdo\ssorting.\s\sUnfortunately,\sinstead\sof\smaking\ssorts\ngo\sfaster\sas\swas\shoped,\sthis\schanges\sslows\ssorting\sdown\sby\sabout\s10%. +D 2012-08-16T20:05:43.061 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in adec39f15a9c7000f634b87a535b95279b0cbd09 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeapi.c 88ea823bbcb4320f5a6607f39cd7c2d3cc4c26b1 F src/vdbeaux.c dce80038c3c41f2680e5ab4dd0f7e0d8b7ff9071 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c cb55e84b8e2c15704968ee05f0fae25883299b74 -F src/vdbesort.c 0dc1b274dcb4d4c8e71b0b2b15261f286caba39b +F src/vdbesort.c 3945ae71fe43e52727ec52103a07492e9df1776f F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c bb8ea3a26608bb1357538a5d2fc72beba6638998 F src/wal.c 9294df6f96aae5909ae1a9b733fd1e1b4736978b @@ -1011,7 +1011,10 @@ F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P ed3dc7a89f3416622fcd741ae5fba437929d06d6 31c07db2560ee867723c41cdb634e2aa7993634d -R 8983882407637c60aee542aa1d4c9bca +P f4125771e21f1ca29d5442b5441dacfc06b8032b +R 57dfe237bc5133c34791ea3daa8a6629 +T *branch * threads-sort-ex1 +T *sym-threads-sort-ex1 * +T -sym-threads * U drh -Z 9b3c8e4e1baa5816be48b30862221038 +Z 0c902f962545bc2a76e593bb4b7dad94 diff --git a/manifest.uuid b/manifest.uuid index 0a036ded25..44f9d1753b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4125771e21f1ca29d5442b5441dacfc06b8032b \ No newline at end of file +11dd05e5984f0d5f98052458b94cbaf7d18c2e8f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index ba1e9f0f23..4b546e7a90 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -105,6 +105,7 @@ struct VdbeSorter { int *aTree; /* Current state of incremental merge */ sqlite3_file *pTemp1; /* PMA file 1 */ SorterRecord *pRecord; /* Head of in-memory record list */ + int nRecord; /* Number of elements on the pRecord list */ UnpackedRecord *pUnpacked; /* Used to unpack keys */ }; @@ -387,14 +388,14 @@ static void vdbeSorterCompare( int bOmitRowid, /* Ignore rowid field at end of keys */ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2, /* Right side of comparison */ - int *pRes /* OUT: Result of comparison */ + int *pRes, /* OUT: Result of comparison */ + UnpackedRecord *r2 /* Space to hold the unpacked Key2 record */ ){ KeyInfo *pKeyInfo = pCsr->pKeyInfo; - VdbeSorter *pSorter = pCsr->pSorter; - UnpackedRecord *r2 = pSorter->pUnpacked; int i; if( pKey2 ){ + assert( r2!=0 ); sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2); } @@ -445,9 +446,9 @@ static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ iRes = i1; }else{ int res; - assert( pCsr->pSorter->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ vdbeSorterCompare( - pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res + pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res, + pSorter->pUnpacked ); if( res<=0 ){ iRes = i1; @@ -547,7 +548,8 @@ static void vdbeSorterMerge( const VdbeCursor *pCsr, /* For pKeyInfo */ SorterRecord *p1, /* First list to merge */ SorterRecord *p2, /* Second list to merge */ - SorterRecord **ppOut /* OUT: Head of merged list */ + SorterRecord **ppOut, /* OUT: Head of merged list */ + UnpackedRecord *pUnpacked /* Space to hold an unpacked record */ ){ SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; @@ -555,7 +557,8 @@ static void vdbeSorterMerge( while( p1 && p2 ){ int res; - vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res, + pUnpacked); if( res<=0 ){ *pp = p1; pp = &p1->pNext; @@ -574,41 +577,113 @@ static void vdbeSorterMerge( } /* -** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK -** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error -** occurs. +** Background sorting task */ -static int vdbeSorterSort(const VdbeCursor *pCsr){ - int i; - SorterRecord **aSlot; - SorterRecord *p; - VdbeSorter *pSorter = pCsr->pSorter; +typedef struct SortTask { + SorterRecord *pList; /* List of elements to be sorted */ + const VdbeCursor *pCsr; /* Cursor. Needed for pCur->pKeyInfo */ + UnpackedRecord *pUnpacked; /* Space to hold an unpacked key */ + SorterRecord **apSlot; /* Temp memory for the merge sort */ +} SortTask; - aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); - if( !aSlot ){ - return SQLITE_NOMEM; - } - - p = pSorter->pRecord; +/* +** Do a sort in a background thread +*/ +void *vdbeSorterBackgroundSort(SortTask *pTask){ + SorterRecord *p = pTask->pList; + SorterRecord **a = pTask->apSlot; + int i; + for(i=0; i<64; i++) a[i] = 0; while( p ){ SorterRecord *pNext = p->pNext; p->pNext = 0; - for(i=0; aSlot[i]; i++){ - vdbeSorterMerge(pCsr, p, aSlot[i], &p); - aSlot[i] = 0; + for(i=0; a[i]; i++){ + if( a[i]==0 ) break; + vdbeSorterMerge(pTask->pCsr, a[i], p, &p, pTask->pUnpacked); + a[i] = 0; } - aSlot[i] = p; + a[i] = p; p = pNext; } - p = 0; for(i=0; i<64; i++){ - vdbeSorterMerge(pCsr, p, aSlot[i], &p); + vdbeSorterMerge(pTask->pCsr, a[i], p, &p, pTask->pUnpacked); } - pSorter->pRecord = p; + pTask->pList = p; + return p; +} - sqlite3_free(aSlot); - return SQLITE_OK; +/* +** Divide a linked list of SorterRecord objects into two separate +** linked lists +*/ +static void vdbeSorterDivideList( + SorterRecord *pIn, /* The list to be divided */ + SorterRecord **ppOut1, /* Write the first list here */ + SorterRecord **ppOut2 /* Write the second list here */ +){ + int i = 0; + *ppOut1 = *ppOut2 = 0; + while( pIn ){ + SorterRecord *pNext = pIn->pNext; + pIn->pNext = 0; + if( i & 1 ){ + *ppOut1 = pIn; + ppOut1 = &pIn->pNext; + }else{ + *ppOut2 = pIn; + ppOut2 = &pIn->pNext; + } + i++; + pIn = pNext; + } +} + +/* +** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK +** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error +** occurs. +*/ +static int vdbeSorterSort(const VdbeCursor *pCsr){ + int rc; + SorterRecord *p; + VdbeSorter *pSorter = pCsr->pSorter; + char *pDummy = 0; + int nByteA, nByteB; + SortTask aTask[2]; + SQLiteThread *pThread; + + nByteA = 64*sizeof(SorterRecord*); + nByteB = ROUND8(sizeof(UnpackedRecord)); + nByteB += sizeof(Mem)*(pCsr->pKeyInfo->nField+1); + + aTask[0].apSlot = (SorterRecord **)sqlite3MallocZero(2*(nByteA + nByteB)); + if( !aTask[0].apSlot ){ + return SQLITE_NOMEM; + } + aTask[0].pCsr = pCsr; + aTask[0].pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, + (char*)&aTask[0].apSlot[64], nByteB, &pDummy); + assert( pDummy==0 ); + aTask[1].apSlot = (nByteA+nByteB)+(char*)aTask[0].apSlot; + aTask[1].pCsr = pCsr; + aTask[1].pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, + (char*)&aTask[1].apSlot[64], nByteB, &pDummy); + assert( pDummy==0 ); + + vdbeSorterDivideList(pSorter->pRecord, &aTask[0].pList, &aTask[1].pList); + rc = sqlite3ThreadCreate(&pThread, + (void*(*)(void*))vdbeSorterBackgroundSort, &aTask[0]); + vdbeSorterBackgroundSort(&aTask[1]); + if( rc==SQLITE_NOMEM ){ + vdbeSorterBackgroundSort(&aTask[0]); + }else{ + rc = sqlite3ThreadJoin(pThread, &pDummy); + } + vdbeSorterMerge(pCsr, aTask[0].pList, aTask[1].pList, &pSorter->pRecord, + aTask[0].pUnpacked); + sqlite3_free(aTask[0].apSlot); + return rc; } /* @@ -719,6 +794,7 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ if( pSorter->nInMemory==0 ){ assert( pSorter->pRecord==0 ); + assert( pSorter->nRecord==0 ); return rc; } @@ -745,7 +821,8 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ fileWriterWrite(&writer, p->pVal, p->nVal); sqlite3DbFree(db, p); } - pSorter->pRecord = p; + pSorter->pRecord = 0; + pSorter->nRecord = 0; rc = fileWriterFinish(db, &writer, &pSorter->iWriteOff); } @@ -776,6 +853,7 @@ int sqlite3VdbeSorterWrite( pNew->nVal = pVal->n; pNew->pNext = pSorter->pRecord; pSorter->pRecord = pNew; + pSorter->nRecord++; } /* See if the contents of the sorter should now be written out. They @@ -965,6 +1043,7 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ }else{ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->pNext; + pSorter->nRecord--; pFree->pNext = 0; vdbeSorterRecordFree(db, pFree); *pbEof = !pSorter->pRecord; @@ -1031,7 +1110,8 @@ int sqlite3VdbeSorterCompare( void *pKey; int nKey; /* Sorter key to compare pVal with */ pKey = vdbeSorterRowkey(pSorter, &nKey); - vdbeSorterCompare(pCsr, 1, pVal->z, pVal->n, pKey, nKey, pRes); + vdbeSorterCompare(pCsr, 1, pVal->z, pVal->n, pKey, nKey, pRes, + pSorter->pUnpacked); return SQLITE_OK; }