From: dan Date: Mon, 6 Aug 2012 19:28:20 +0000 (+0000) Subject: Merge the sorter-coalesce-writes branch into the trunk. This improves CREATE INDEX... X-Git-Tag: version-3.7.14~46 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=09ac7ec544ae02d5a68076f8112f514d99f1301b;p=thirdparty%2Fsqlite.git Merge the sorter-coalesce-writes branch into the trunk. This improves CREATE INDEX performance on some platforms. FossilOrigin-Name: e1e9cb08b011e67b767091e42225f22ec862fa64 --- 09ac7ec544ae02d5a68076f8112f514d99f1301b diff --cc manifest index 830c9c3fb7,39fe08007e..45bf47c50d --- a/manifest +++ b/manifest @@@ -1,5 -1,5 +1,5 @@@ - C Update\sdescription\sstrings\sin\sthe\sVSIX\spackage. - D 2012-08-06T10:51:55.331 -C Fix\sa\scrash\sthat\scould\sfollow\san\sOOM\scondition. -D 2012-08-06T19:12:17.500 ++C Merge\sthe\ssorter-coalesce-writes\sbranch\sinto\sthe\strunk.\sThis\simproves\sCREATE\sINDEX\sperformance\son\ssome\splatforms. ++D 2012-08-06T19:28:20.956 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in abd5c10d21d1395f140d9e50ea999df8fa4d6376 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@@ -244,7 -244,7 +244,7 @@@ F src/vdbeapi.c 88ea823bbcb4320f5a6607f F src/vdbeaux.c dce80038c3c41f2680e5ab4dd0f7e0d8b7ff9071 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c cb55e84b8e2c15704968ee05f0fae25883299b74 - F src/vdbesort.c 628b2bc0cc82cae0e9946f70c5c81986e9fba91f -F src/vdbesort.c 1de867bfa04a54c217bfe467f43206c801912921 ++F src/vdbesort.c 4897215f0a0c4e731aa5ac5fc0317b62a4919e79 F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c bb8ea3a26608bb1357538a5d2fc72beba6638998 F src/wal.c 9294df6f96aae5909ae1a9b733fd1e1b4736978b @@@ -530,7 -530,8 +530,8 @@@ F test/incrvacuum_ioerr.test 22f208d01c F test/index.test b5429732b3b983fa810e3ac867d7ca85dae35097 F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 423a25c789fc8cc51aaf2a4370bbdde2d9e9eed7 -F test/index4.test 1e299862024012e0165531cce251572f7f084d15 +F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026 + F test/index5.test edc8c64ca78bee140c21ce3836820fadf47906bb F test/indexedby.test be501e381b82b2f8ab406309ba7aac46e221f4ad F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@@ -1008,7 -1009,7 +1009,7 @@@ F tool/vdbe-compress.tcl d70ea6d8a19e35 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 - P 335e91e599555d9f4e42f90576d1676c381314f4 - R eadc6c87ec15f9db6747eb42f404a18d - U mistachkin - Z 9bc32025e8412eea1e17719cbb9188be -P d045f8b2d44e388d8c4549ff02d4ca7eff4e2038 -R 271a84ad323420f7636b49c9d4c7979c ++P 541e9310a7b88e0b40c6530947803527f28e51de 2e5741f774248abc678b50711c43e38ca30c9091 ++R 9ea05a2580296dc700acf20c4ae23876 + U dan -Z da2600161cd5536c31009989e6b616b4 ++Z ec36f3ad854c62e55cc3fc27576cdc7b diff --cc manifest.uuid index f7027c4945,5e7b941fbe..5a53d65817 --- a/manifest.uuid +++ b/manifest.uuid @@@ -1,1 -1,1 +1,1 @@@ - 541e9310a7b88e0b40c6530947803527f28e51de -2e5741f774248abc678b50711c43e38ca30c9091 ++e1e9cb08b011e67b767091e42225f22ec862fa64 diff --cc src/vdbesort.c index 5faeca8dc7,7433e3e08c..6ec30954aa --- a/src/vdbesort.c +++ b/src/vdbesort.c @@@ -148,55 -166,90 +166,91 @@@ static void vdbeSorterIterZero(sqlite3 } /* - ** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if - ** no error occurs, or an SQLite error code if one does. + ** Read nByte bytes of data from the stream of data iterated by object p. + ** If successful, set *ppOut to point to a buffer containing the data + ** and return SQLITE_OK. Otherwise, if an error occurs, return an SQLite + ** error code. + ** + ** The buffer indicated by *ppOut may only be considered valid until the + ** next call to this function. */ - static int vdbeSorterIterNext( - sqlite3 *db, /* Database handle (for sqlite3DbMalloc() ) */ - VdbeSorterIter *pIter /* Iterator to advance */ + static int vdbeSorterIterRead( + sqlite3 *db, /* Database handle (for malloc) */ + VdbeSorterIter *p, /* Iterator */ + int nByte, /* Bytes of data to read */ + u8 **ppOut /* OUT: Pointer to buffer containing data */ ){ - int rc; /* Return Code */ - int nRead; /* Number of bytes read */ - int nRec = 0; /* Size of record in bytes */ - int iOff = 0; /* Size of serialized size varint in bytes */ - - assert( pIter->iEof>=pIter->iReadOff ); - if( pIter->iEof-pIter->iReadOff>5 ){ - nRead = 5; - }else{ - nRead = (int)(pIter->iEof - pIter->iReadOff); - } - if( nRead<=0 ){ - /* This is an EOF condition */ - vdbeSorterIterZero(db, pIter); - return SQLITE_OK; + int iBuf; /* Offset within buffer to read from */ + int nAvail; /* Bytes of data available in buffer */ + assert( p->aBuffer ); + + /* If there is no more data to be read from the buffer, read the next + ** p->nBuffer bytes of data from the file into it. Or, if there are less + ** than p->nBuffer bytes remaining in the PMA, read all remaining data. */ + iBuf = p->iReadOff % p->nBuffer; + if( iBuf==0 ){ + int nRead; /* Bytes to read from disk */ + int rc; /* sqlite3OsRead() return code */ + + /* Determine how many bytes of data to read. */ + nRead = p->iEof - p->iReadOff; + if( nRead>p->nBuffer ) nRead = p->nBuffer; + assert( nRead>0 ); + + /* Read data from the file. Return early if an error occurs. */ + rc = sqlite3OsRead(p->pFile, p->aBuffer, nRead, p->iReadOff); + assert( rc!=SQLITE_IOERR_SHORT_READ ); + if( rc!=SQLITE_OK ) return rc; } + nAvail = p->nBuffer - iBuf; + + if( nByte<=nAvail ){ + /* The requested data is available in the in-memory buffer. In this + ** case there is no need to make a copy of the data, just return a + ** pointer into the buffer to the caller. */ + *ppOut = &p->aBuffer[iBuf]; + p->iReadOff += nByte; + }else{ + /* The requested data is not all available in the in-memory buffer. + ** In this case, allocate space at p->aAlloc[] to copy the requested + ** range into. Then return a copy of pointer p->aAlloc to the caller. */ + int nRem; /* Bytes remaining to copy */ + + /* Extend the p->aAlloc[] allocation if required. */ + if( p->nAllocnAlloc*2; + while( nByte>nNew ) nNew = nNew*2; + p->aAlloc = sqlite3DbReallocOrFree(db, p->aAlloc, nNew); + if( !p->aAlloc ) return SQLITE_NOMEM; ++ p->nAlloc = nNew; + } - rc = sqlite3OsRead(pIter->pFile, pIter->aAlloc, nRead, pIter->iReadOff); - if( rc==SQLITE_OK ){ - iOff = getVarint32(pIter->aAlloc, nRec); - if( (iOff+nRec)>nRead ){ - int nRead2; /* Number of extra bytes to read */ - if( (iOff+nRec)>pIter->nAlloc ){ - int nNew = pIter->nAlloc*2; - while( (iOff+nRec)>nNew ) nNew = nNew*2; - pIter->aAlloc = sqlite3DbReallocOrFree(db, pIter->aAlloc, nNew); - if( !pIter->aAlloc ) return SQLITE_NOMEM; - pIter->nAlloc = nNew; - } - - nRead2 = iOff + nRec - nRead; - rc = sqlite3OsRead( - pIter->pFile, &pIter->aAlloc[nRead], nRead2, pIter->iReadOff+nRead - ); + /* Copy as much data as is available in the buffer into the start of + ** p->aAlloc[]. */ + memcpy(p->aAlloc, &p->aBuffer[iBuf], nAvail); + p->iReadOff += nAvail; + nRem = nByte - nAvail; + + /* The following loop copies up to p->nBuffer bytes per iteration into + ** the p->aAlloc[] buffer. */ + while( nRem>0 ){ + int rc; /* vdbeSorterIterRead() return code */ + int nCopy; /* Number of bytes to copy */ + u8 *aNext; /* Pointer to buffer to copy data from */ + + nCopy = nRem; + if( nRem>p->nBuffer ) nCopy = p->nBuffer; + rc = vdbeSorterIterRead(db, p, nCopy, &aNext); + if( rc!=SQLITE_OK ) return rc; + assert( aNext!=p->aAlloc ); + memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); + nRem -= nCopy; } + + *ppOut = p->aAlloc; } - assert( rc!=SQLITE_OK || nRec>0 ); - pIter->iReadOff += iOff+nRec; - pIter->nKey = nRec; - pIter->aKey = &pIter->aAlloc[iOff]; - return rc; + return SQLITE_OK; } /*