From: dan Date: Sat, 28 Mar 2015 19:56:41 +0000 (+0000) Subject: Further optimizations for sorting records that begin with integer or text values. X-Git-Tag: version-3.8.10~152^2~8 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a9d9111c4aa590e197878515c7a9de526ef6feb9;p=thirdparty%2Fsqlite.git Further optimizations for sorting records that begin with integer or text values. FossilOrigin-Name: 24fe9f25d64ee516633fed1ae7ebc21554aa69ca --- diff --git a/manifest b/manifest index 65ccebc2d0..aba8fb17d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimize\scases\swhere\sall\sthe\ssorter\sis\ssorting\sa\sset\sof\srecords\sthat\sall\sbegin\swith\sinteger\svalues,\sor\sthat\sall\sbegin\swith\stext\svalues\sto\sbe\scompared\susing\sBINARY. -D 2015-03-26T11:55:03.488 +C Further\soptimizations\sfor\ssorting\srecords\sthat\sbegin\swith\sinteger\sor\stext\svalues. +D 2015-03-28T19:56:41.373 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 88a3e6261286db378fdffa1124cad11b3c05f5bb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -300,7 +300,7 @@ F src/vdbeapi.c 583d56b129dd27f12bed518270de9ebe521e6a75 F src/vdbeaux.c 413dc496248ac18eb0c19e35e86bb1ffd47b8907 F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90 F src/vdbemem.c c0dc81285b7571b0a31c40f17846fe2397ec1cd9 -F src/vdbesort.c a8d82ed4b09f3b1f5390c7546a9bcad72d483abf +F src/vdbesort.c b3d16bbd66ff8d26b42670a8aa5dfda0e0d2e45e F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 62d49237bd8f3be4863815a39387b0f9897fa5e1 F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb @@ -1247,10 +1247,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fb076b28c36975ff2e41440f22fe5de115c195da -R 1e1c6c1ab8dfb515d00c85e16ae436f0 -T *branch * sorter-opt -T *sym-sorter-opt * -T -sym-trunk * +P ce5ad17c25cf2f8274ce304c51e4421faae0b32b +R 5a4cff4b803c75214d0e7f89bc9331f1 U dan -Z 9752268c90db4ae1847219cb63314e99 +Z 8e7d5cc07144236ce8ed9e6cb6b634d0 diff --git a/manifest.uuid b/manifest.uuid index dfa9f94353..1d45093594 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ce5ad17c25cf2f8274ce304c51e4421faae0b32b \ No newline at end of file +24fe9f25d64ee516633fed1ae7ebc21554aa69ca \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 4faeebf963..b8ef783a06 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -291,6 +291,7 @@ struct MergeEngine { ** after the thread has finished are not dire. So we don't worry about ** memory barriers and such here. */ +typedef int (*SorterCompare)(SortSubtask*,int*,const void*,int,const void*,int); struct SortSubtask { SQLiteThread *pThread; /* Background thread, if any */ int bDone; /* Set if thread is finished but not joined */ @@ -298,11 +299,12 @@ struct SortSubtask { UnpackedRecord *pUnpacked; /* Space to unpack a record */ SorterList list; /* List for thread to write to a PMA */ int nPMA; /* Number of PMAs currently in file */ - RecordCompare xCompare; /* Compare function to use */ + SorterCompare xCompare; /* Compare function to use */ SorterFile file; /* Temp file for level-0 PMAs */ SorterFile file2; /* Space for other PMAs */ }; + /* ** Main sorter structure. A single instance of this is allocated for each ** sorter cursor created by the VDBE. @@ -747,30 +749,136 @@ static int vdbePmaReaderInit( return rc; } - /* ** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, ** size nKey2 bytes). Use (pTask->pKeyInfo) for the collation sequences ** used by the comparison. Return the result of the comparison. ** -** Before returning, object (pTask->pUnpacked) is populated with the -** unpacked version of key2. Or, if pKey2 is passed a NULL pointer, then it -** is assumed that the (pTask->pUnpacked) structure already contains the -** unpacked key to use as key2. +** If IN/OUT parameter *pbKey2Cached is true when this function is called, +** it is assumed that (pTask->pUnpacked) contains the unpacked version +** of key2. If it is false, (pTask->pUnpacked) is populated with the unpacked +** version of key2 and *pbKey2Cached set to true before returning. ** ** If an OOM error is encountered, (pTask->pUnpacked->error_rc) is set ** to SQLITE_NOMEM. */ static int vdbeSorterCompare( SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2 /* Right side of comparison */ ){ UnpackedRecord *r2 = pTask->pUnpacked; - if( pKey2 ){ + if( !*pbKey2Cached ){ sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + *pbKey2Cached = 1; + } + return sqlite3VdbeRecordCompare(nKey1, pKey1, r2); +} + +/* +** A specially optimized version of vdbeSorterCompare() that assumes that +** the first field of each key is a TEXT value and that the collation +** sequence to compare them with is BINARY. +*/ +static int vdbeSorterCompareText( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + const u8 * const p1 = (const u8 * const)pKey1; + const u8 * const p2 = (const u8 * const)pKey2; + const u8 * const v1 = &p1[ p1[0] ]; /* Pointer to value 1 */ + const u8 * const v2 = &p2[ p2[0] ]; /* Pointer to value 2 */ + + int n1; + int n2; + int res; + + getVarint32(&p1[1], n1); n1 = (n1 - 13) / 2; + getVarint32(&p2[1], n2); n2 = (n2 - 13) / 2; + res = memcmp(v1, v2, MIN(n1, n2)); + if( res==0 ){ + res = n1 - n2; + } + + if( res==0 ){ + if( pTask->pSorter->pKeyInfo->nField>1 ){ + res = vdbeSorterCompare(pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2); + } + }else{ + if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + res = res * -1; + } } - return pTask->xCompare(nKey1, pKey1, r2); + + return res; +} + +/* +** A specially optimized version of vdbeSorterCompare() that assumes that +** the first field of each key is an INTEGER value. +*/ +static int vdbeSorterCompareInt( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + const u8 * const p1 = (const u8 * const)pKey1; + const u8 * const p2 = (const u8 * const)pKey2; + const int s1 = p1[1]; /* Left hand serial type */ + const int s2 = p2[1]; /* Right hand serial type */ + const u8 * const v1 = &p1[ p1[0] ]; /* Pointer to value 1 */ + const u8 * const v2 = &p2[ p2[0] ]; /* Pointer to value 2 */ + int res; /* Return value */ + + assert( (s1>0 && s1<7) || s1==8 || s1==9 ); + assert( (s2>0 && s2<7) || s2==8 || s2==9 ); + + if( s1>7 && s2>7 ){ + res = s1 - s2; + }else{ + if( s1==s2 ){ + if( (*v1 ^ *v2) & 0x80 ){ + /* The two values have different signs */ + res = (*v1 & 0x80) ? -1 : +1; + }else{ + /* The two values have the same sign. Compare using memcmp(). */ + static const u8 aLen[] = {0, 1, 2, 3, 4, 6, 8 }; + int i; + res = 0; + for(i=0; i7 ){ + res = +1; + }else if( s1>7 ){ + res = -1; + }else{ + res = s1 - s2; + } + + if( res>0 ){ + if( *v1 & 0x80 ) res = -1; + }else if( res<0 ){ + if( *v2 & 0x80 ) res = +1; + } + } + } + + if( res==0 ){ + if( pTask->pSorter->pKeyInfo->nField>1 ){ + res = vdbeSorterCompare(pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2); + } + }else if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + res = res * -1; + } + + return res; } /* @@ -873,7 +981,9 @@ int sqlite3VdbeSorterInit( } } - if( (pKeyInfo->nField+pKeyInfo->nXField)<13 && pKeyInfo->aColl[0]==0 ){ + if( (pKeyInfo->nField+pKeyInfo->nXField)<13 + && (pKeyInfo->aColl[0]==0 || pKeyInfo->aColl[0]==db->pDfltColl) + ){ pSorter->typeMask = SORTER_TYPE_INTEGER | SORTER_TYPE_TEXT; } } @@ -1219,28 +1329,42 @@ static void vdbeSorterMerge( ){ SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; - void *pVal2 = p2 ? SRVAL(p2) : 0; + int bCached = 0; while( p1 && p2 ){ int res; - res = vdbeSorterCompare(pTask, SRVAL(p1), p1->nVal, pVal2, p2->nVal); + res = pTask->xCompare( + pTask, &bCached, SRVAL(p1), p1->nVal, SRVAL(p2), p2->nVal + ); + if( res<=0 ){ *pp = p1; pp = &p1->u.pNext; p1 = p1->u.pNext; - pVal2 = 0; }else{ *pp = p2; - pp = &p2->u.pNext; + pp = &p2->u.pNext; p2 = p2->u.pNext; - if( p2==0 ) break; - pVal2 = SRVAL(p2); + bCached = 0; } } *pp = p1 ? p1 : p2; *ppOut = pFinal; } +/* +** Return the SorterCompare function to compare values collected by the +** sorter object passed as the only argument. +*/ +static SorterCompare vdbeSorterGetCompare(VdbeSorter *p){ + if( p->typeMask==SORTER_TYPE_INTEGER ){ + return vdbeSorterCompareInt; + }else if( p->typeMask==SORTER_TYPE_TEXT ){ + return vdbeSorterCompareText; + } + return vdbeSorterCompare; +} + /* ** Sort the linked list of records headed at pTask->pList. Return ** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if @@ -1256,13 +1380,7 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ if( rc!=SQLITE_OK ) return rc; p = pList->pList; - if( pTask->pSorter->typeMask==0 ){ - pTask->xCompare = sqlite3VdbeRecordCompare; - }else{ - UnpackedRecord *pRec = pTask->pUnpacked; - sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, p->nVal, SRVAL(p), pRec); - pTask->xCompare = sqlite3VdbeFindCompare(pRec); - } + pTask->xCompare = vdbeSorterGetCompare(pTask->pSorter); aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ @@ -1482,13 +1600,12 @@ static int vdbeMergeEngineStep( int i; /* Index of aTree[] to recalculate */ PmaReader *pReadr1; /* First PmaReader to compare */ PmaReader *pReadr2; /* Second PmaReader to compare */ - u8 *pKey2; /* To pReadr2->aKey, or 0 if record cached */ + int bCached = 0; /* Find the first two PmaReaders to compare. The one that was just ** advanced (iPrev) and the one next to it in the array. */ pReadr1 = &pMerger->aReadr[(iPrev & 0xFFFE)]; pReadr2 = &pMerger->aReadr[(iPrev | 0x0001)]; - pKey2 = pReadr2->aKey; for(i=(pMerger->nTree+iPrev)/2; i>0; i=i/2){ /* Compare pReadr1 and pReadr2. Store the result in variable iRes. */ @@ -1498,8 +1615,8 @@ static int vdbeMergeEngineStep( }else if( pReadr2->pFd==0 ){ iRes = -1; }else{ - iRes = vdbeSorterCompare(pTask, - pReadr1->aKey, pReadr1->nKey, pKey2, pReadr2->nKey + iRes = pTask->xCompare(pTask, &bCached, + pReadr1->aKey, pReadr1->nKey, pReadr2->aKey, pReadr2->nKey ); } @@ -1521,9 +1638,9 @@ static int vdbeMergeEngineStep( if( iRes<0 || (iRes==0 && pReadr1aTree[i] = (int)(pReadr1 - pMerger->aReadr); pReadr2 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; - pKey2 = pReadr2->aKey; + bCached = 0; }else{ - if( pReadr1->pFd ) pKey2 = 0; + bCached = (pReadr1->pFd!=0); pMerger->aTree[i] = (int)(pReadr2 - pMerger->aReadr); pReadr1 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; } @@ -1635,7 +1752,7 @@ int sqlite3VdbeSorterWrite( getVarint32((const u8*)&pVal->z[1], t); if( t>0 && t<10 && t!=7 ){ pSorter->typeMask &= SORTER_TYPE_INTEGER; - }else if( t & 0x01 ){ + }else if( t>10 && (t & 0x01) ){ pSorter->typeMask &= SORTER_TYPE_TEXT; }else{ pSorter->typeMask = 0; @@ -1905,10 +2022,12 @@ static void vdbeMergeEngineCompare( }else if( p2->pFd==0 ){ iRes = i1; }else{ + SortSubtask *pTask = pMerger->pTask; + int bCached = 0; int res; - assert( pMerger->pTask->pUnpacked!=0 ); /* from vdbeSortSubtaskMain() */ - res = vdbeSorterCompare( - pMerger->pTask, p1->aKey, p1->nKey, p2->aKey, p2->nKey + assert( pTask->pUnpacked!=0 ); /* from vdbeSortSubtaskMain() */ + res = pTask->xCompare( + pTask, &bCached, p1->aKey, p1->nKey, p2->aKey, p2->nKey ); if( res<=0 ){ iRes = i1; @@ -2326,10 +2445,8 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ MergeEngine *pMain = 0; #if SQLITE_MAX_WORKER_THREADS sqlite3 *db = pTask0->pSorter->db; - RecordCompare xCompare = (pSorter->typeMask==0 ? - sqlite3VdbeRecordCompare : pSorter->aTask[0].xCompare - ); int i; + SorterCompare xCompare = vdbeSorterGetCompare(pSorter); for(i=0; inTask; i++){ pSorter->aTask[i].xCompare = xCompare; }