From: dan Date: Fri, 12 Aug 2011 16:30:30 +0000 (+0000) Subject: Merge latest trunk changes into experimental branch. X-Git-Tag: version-3.7.8~76^2~1 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=15bf39034a0fb020bb4629b30961e3541ce03410;p=thirdparty%2Fsqlite.git Merge latest trunk changes into experimental branch. FossilOrigin-Name: 7e515055f219b01dd72df4e27bdcabfa2f9be5c2 --- 15bf39034a0fb020bb4629b30961e3541ce03410 diff --cc manifest index a78c62937a,83e86d4850..988b61b35a --- a/manifest +++ b/manifest @@@ -1,12 -1,12 +1,12 @@@ - C Remove\san\sunused\sparameter\sfrom\sa\sfunction\sin\svdbesort.c.\sFix\ssome\scomments\sand\sother\sdetails\sin\sthe\ssame\sfile. - D 2011-08-12T16:11:43.093 -C Make\sthe\sopenDirectory\sroutine\sin\sos_unix.c\soverrideable\sso\sthat\sit\scan\nbe\sturned\sinto\sa\sharmless\sno-op\sfor\sthe\schromium\ssandbox. -D 2011-08-10T01:52:12.736 ++C Merge\slatest\strunk\schanges\sinto\sexperimental\sbranch. ++D 2011-08-12T16:30:30.031 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f - F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e + F Makefile.in 1e6988b3c11dee9bd5edc0c804bd4468d74a9cdc F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 - F Makefile.msc 9c76731c20561a33ce1257c70be3f029cedb62a6 + F Makefile.msc 9fd0401b0134b565a9c7d8ed49dc044cc89f2096 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 - F VERSION 3fcdd7fbe3eb282df3978fe77288544543767961 + F VERSION f724de7326e87b7f3b0a55f16ef4b4d993680d54 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 17dc593f791f874d2c23a0f9360850ded0286531 F art/2005osaward.gif 0d1851b2a7c1c9d0ccce545f3e14bca42d7fd248 @@@ -104,7 -104,7 +104,7 @@@ F ext/rtree/tkt3363.test 142ab96eded44a F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 - F main.mk df1e47e4bc886f556b39a8cdb9dc3f6fb6810d64 -F main.mk 50adb7368bd0faa888eb4634b874d542d44f3de3 ++F main.mk a812efc287a904c3e04098f090de036b264ba779 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@@ -124,13 -124,13 +124,13 @@@ F src/auth.c 523da7fb4979469955d822ff92 F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 8c46f0ab69ad9549c75a3a91fed87abdaa743e2f +F src/btree.c a30bdcc27eedc36a38a3a11e1ba83de9a6729f7e F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 - F src/build.c ac39a62c06fbc3d6ef1d5f784e06d806bc821edc -F src/build.c 19a8957a442d922a0d6ed1a5dd67b63202fc3260 ++F src/build.c 77b40abf79e1541356f552c9681b0b431aba672b F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 +F src/ctime.c 7f283795650dd4122cc07bd02193c40193b32cc6 F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3 F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8 F src/expr.c 4bbdfaf66bc614be9254ce0c26a17429067a3e07 @@@ -236,23 -236,22 +236,23 @@@ F src/tokenize.c c819d9f72168a035d545a5 F src/trigger.c 1cfb80e2290ef66ea89cb4e821caae65a02c0d56 F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 - F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 + F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e - F src/vdbe.c 1f5bb8b8a0f24bd93e66958eaebd060ccee272a2 -F src/vdbe.c 49d834f0fe49d305e07f9c212e94007fda2028e9 ++F src/vdbe.c 22d0d5bc8e8e467e1e0fa20a0a6b21c297f5601f F src/vdbe.h 5cf09e7ee8a3f7d93bc51f196a96550786afe7a1 -F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45 +F src/vdbeInt.h f9250326f264ca5f100acc19e9c07096bb889096 F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 -F src/vdbeaux.c 4d100407e3c72e163854aff8903d19d5ecdf46c0 +F src/vdbeaux.c 8fb978eb73a97b34d352dd3ef3bff35b1b3fa7e9 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b - F src/vdbesort.c a756d20db3ff596dd3dd10f6a13fd832f5715540 ++F src/vdbesort.c 3e6b2268326ee075ae5e8cece4e469e5d1d4a859 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 - F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 + F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f - F src/where.c ce7cce80c5cb07ba40e9bf38a33ef806c61f55f2 - F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 + F src/where.c 7d09f4c1512affb60cc1190a4b33d121d4ce039a + F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 52fc8dee494092031a556911d404ca30a749a30b @@@ -616,12 -613,12 +616,12 @@@ F test/pagerfault.test 452f2cc23e3bfcfa F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 - F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb + F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 - F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 - F test/permutations.test bfe51414b7c355a6dfb8803176808dc2e00e3783 - F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850 - F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 + F test/pcache2.test 9f9357bb0f463b87bdf695646024ed2031a0c85a -F test/permutations.test 2c845b5c8cd2abb32f07076734d10e6a9fd19f26 ++F test/permutations.test ad17319066a90e2db71823c3ff104795ffc71b31 + F test/pragma.test c8108e01da04f16e67e5754e610bc62c1b993f6c + F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc @@@ -880,27 -879,28 +882,28 @@@ F test/vtabF.test fd5ad376f5a34fe0891df F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d - F test/wal.test 5617ad308bfdb8a8885220d8a261a6096a8d7e57 - F test/wal2.test ceba91f0d82c4fd87f342aa5504eb1f8fa6418f3 - F test/wal3.test 5c396cc22497244d627306f4c1d360167353f8dd - F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 - F test/wal5.test f06a0427e06db00347e32eb9fa99d6a5c0f2d088 - F test/wal6.test 07aa31ca8892d0527f2c5c5a9a2a87aa421dfaa8 + F test/wal.test e11da8d5ea8a38a247339455098357e9adf63d76 -F test/wal2.test a2caa432d056596c6ee815d2743b2ced86f67a59 ++F test/wal2.test ad6412596815f553cd30f271d291ab003092bc7e + F test/wal3.test 18da4e65c30c43c646ad40e145e9a074e4062fc9 + F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c + F test/wal5.test 08e145a352b1223930c7f0a1de82a8747a99c322 + F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe - F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 + F test/walbak.test 53be94bccbc3f592d22a5909588ccc81b319ce83 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0 - F test/walcksum.test a37b36375c595e61bdb7e1ec49b5f0979b6fc7ce - F test/walcrash.test e763841551d6b23677ccb419797c1589dcbdbaf5 + F test/walcksum.test f5447800a157c9e2234fbb8e80243f0813941bde + F test/walcrash.test 4fcb661faf71db91214156d52d43ee327f52bde1 F test/walcrash2.test 019d60b89d96c1937adb2b30b850ac7e86e5a142 - F test/walfault.test 58fce626359c9376fe35101b5c0f2df8040aa839 + F test/walfault.test efb0d5724893133e71b8d9d90abdb781845a6bb0 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 - F test/walmode.test 22ddccd073c817ac9ead62b88ac446e8dedc7d2c - F test/walnoshm.test a074428046408f4eb5c6a00e09df8cc97ff93317 + F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c + F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 + F test/walpersist.test 45fb0c94fb63908e2d66b1d99ce4645bfce0fa1e F test/walro.test 2d5d69e2e99da19ce6faab340330234fc4ca0720 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 - F test/walslow.test d21625e2e99e11c032ce949e8a94661576548933 - F test/walthread.test a25a393c068a2b42b44333fa3fdaae9072f1617c + F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a + F test/walthread.test a2ed5270eb695284d4ad27d252517bdc3317ee2a F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554 F test/where3.test 8e1175c7ef710c70502858fc4fb08d784b3620b9 @@@ -927,9 -928,10 +931,10 @@@ F tool/lempar.c 01ca97f87610d1dac6d8cd9 F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c.tcl 1fa0ed9cfdc768bf5de7e65fda8d97a46dd2a7e6 +F tool/mksqlite3c.tcl d8c0d3065bc23fd9e27d59bfebd34df203fe6b08 F tool/mksqlite3h.tcl 78013ad79a5e492e5f764f3c7a8ef834255061f8 F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87 + F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 8086c014cbae90f1f2b564d59d05a5e4ac1783c9 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a @@@ -955,7 -957,7 +960,7 @@@ F tool/symbols.sh caaf6ccc7300fd4335331 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 - P 4ced2394b10d0a4f86422ff893bcdf3cf32591e3 - R ced8374c50fa0d83b7bc754994eec7c3 -P dfa22ed4387f9526b74d5265503c7c8e9d559903 -R bf22196e9aa98f18724e5d2624f7dcbf -U drh -Z 2ec35109792cc0dc5d4f7cebd4d85034 ++P 1a8498d8037a1b93e56951bbdbb76291bd5a4f87 6b236069e1ea3c99ff0a007a790d4baebda70b13 ++R 31c6a782e4904c59693691553f2ae0bd +U dan - Z 2a841cd45d02eb2ca25a3f8996849530 ++Z d620fa181c3729f4f9367beda130e667 diff --cc manifest.uuid index 100963f261,ec7b7e54a0..e351990383 --- a/manifest.uuid +++ b/manifest.uuid @@@ -1,1 -1,1 +1,1 @@@ - 1a8498d8037a1b93e56951bbdbb76291bd5a4f87 -6b236069e1ea3c99ff0a007a790d4baebda70b13 ++7e515055f219b01dd72df4e27bdcabfa2f9be5c2 diff --cc src/vdbesort.c index 72b147f7bc,0000000000..4526f01b45 mode 100644,000000..100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@@ -1,705 -1,0 +1,705 @@@ +/* +** 2011 July 9 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code for the VdbeSorter object, used in concert with +** a VdbeCursor to sort large numbers of keys (as may be required, for +** example, by CREATE INDEX statements on tables too large to fit in main +** memory). +*/ + +#include "sqliteInt.h" +#include "vdbeInt.h" + +#ifndef SQLITE_OMIT_MERGE_SORT + +typedef struct VdbeSorterIter VdbeSorterIter; + +/* +** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES: +** +** As keys are added to the sorter, they are written to disk in a series +** of sorted packed-memory-arrays (PMAs). The size of each PMA is roughly +** the same as the cache-size allowed for temporary databases. In order +** to allow the caller to extract keys from the sorter in sorted order, +** all PMAs currently stored on disk must be merged together. This comment +** describes the data structure used to do so. The structure supports +** merging any number of arrays in a single pass with no redundant comparison +** operations. +** +** The aIter[] array contains an iterator for each of the PMAs being merged. +** An aIter[] iterator either points to a valid key or else is at EOF. For +** the purposes of the paragraphs below, we assume that the array is actually +** N elements in size, where N is the smallest power of 2 greater to or equal +** to the number of iterators being merged. The extra aIter[] elements are +** treated as if they are empty (always at EOF). +** +** The aTree[] array is also N elements in size. The value of N is stored in +** the VdbeSorter.nTree variable. +** +** The final (N/2) elements of aTree[] contain the results of comparing +** pairs of iterator keys together. Element i contains the result of +** comparing aIter[2*i-N] and aIter[2*i-N+1]. Whichever key is smaller, the +** aTree element is set to the index of it. +** +** For the purposes of this comparison, EOF is considered greater than any +** other key value. If the keys are equal (only possible with two EOF +** values), it doesn't matter which index is stored. +** +** The (N/4) elements of aTree[] that preceed the final (N/2) described +** above contains the index of the smallest of each block of 4 iterators. +** And so on. So that aTree[1] contains the index of the iterator that +** currently points to the smallest key value. aTree[0] is unused. +** +** Example: +** +** aIter[0] -> Banana +** aIter[1] -> Feijoa +** aIter[2] -> Elderberry +** aIter[3] -> Currant +** aIter[4] -> Grapefruit +** aIter[5] -> Apple +** aIter[6] -> Durian +** aIter[7] -> EOF +** +** aTree[] = { X, 5 0, 5 0, 3, 5, 6 } +** +** The current element is "Apple" (the value of the key indicated by +** iterator 5). When the Next() operation is invoked, iterator 5 will +** be advanced to the next key in its segment. Say the next key is +** "Eggplant": +** +** aIter[5] -> Eggplant +** +** The contents of aTree[] are updated first by comparing the new iterator +** 5 key to the current key of iterator 4 (still "Grapefruit"). The iterator +** 5 value is still smaller, so aTree[6] is set to 5. And so on up the tree. +** The value of iterator 6 - "Durian" - is now smaller than that of iterator +** 5, so aTree[3] is set to 6. Key 0 is smaller than key 6 (BananaaAlloc); + memset(pIter, 0, sizeof(VdbeSorterIter)); +} + +/* +** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if +** no error occurs, or an SQLite error code if one does. +*/ +static int vdbeSorterIterNext( + sqlite3 *db, /* Database handle (for sqlite3DbMalloc() ) */ + VdbeSorterIter *pIter /* Iterator to advance */ +){ + int rc; /* Return Code */ + int nRead; /* Number of bytes read */ + int nRec; /* Size of record in bytes */ + int iOff; /* Size of serialized size varint in bytes */ + + nRead = pIter->iEof - pIter->iReadOff; + if( nRead>5 ) nRead = 5; + if( nRead<=0 ){ + /* This is an EOF condition */ + vdbeSorterIterZero(db, pIter); + return SQLITE_OK; + } + + rc = sqlite3OsRead(pIter->pFile, pIter->aAlloc, nRead, pIter->iReadOff); + iOff = getVarint32(pIter->aAlloc, nRec); + + if( rc==SQLITE_OK && (iOff+nRec)>nRead ){ + int nRead2; /* Number of extra bytes to read */ + if( (iOff+nRec)>pIter->nAlloc ){ + int nNew = pIter->nAlloc*2; + while( (iOff+nRec)>nNew ) nNew = nNew*2; + pIter->aAlloc = sqlite3DbReallocOrFree(db, pIter->aAlloc, nNew); + if( !pIter->aAlloc ) return SQLITE_NOMEM; + pIter->nAlloc = nNew; + } + + nRead2 = iOff + nRec - nRead; + rc = sqlite3OsRead( + pIter->pFile, &pIter->aAlloc[nRead], nRead2, pIter->iReadOff+nRead + ); + } + + assert( nRec>0 || rc!=SQLITE_OK ); + pIter->iReadOff += iOff+nRec; + pIter->nKey = nRec; + pIter->aKey = &pIter->aAlloc[iOff]; + return rc; +} + +/* +** Write a single varint, value iVal, to file-descriptor pFile. Return +** SQLITE_OK if successful, or an SQLite error code if some error occurs. +** +** The value of *piOffset when this function is called is used as the byte +** offset in file pFile to write to. Before returning, *piOffset is +** incremented by the number of bytes written. +*/ +static int vdbeSorterWriteVarint( + sqlite3_file *pFile, /* File to write to */ + i64 iVal, /* Value to write as a varint */ + i64 *piOffset /* IN/OUT: Write offset in file pFile */ +){ + u8 aVarint[9]; /* Buffer large enough for a varint */ + int nVarint; /* Number of used bytes in varint */ + int rc; /* Result of write() call */ + + nVarint = sqlite3PutVarint(aVarint, iVal); + rc = sqlite3OsWrite(pFile, aVarint, nVarint, *piOffset); + *piOffset += nVarint; + + return rc; +} + +/* +** Read a single varint from file-descriptor pFile. Return SQLITE_OK if +** successful, or an SQLite error code if some error occurs. +** +** The value of *piOffset when this function is called is used as the +** byte offset in file pFile from whence to read the varint. If successful +** (i.e. if no IO error occurs), then *piOffset is set to the offset of +** the first byte past the end of the varint before returning. *piVal is +** set to the integer value read. If an error occurs, the final values of +** both *piOffset and *piVal are undefined. +*/ +static int vdbeSorterReadVarint( + sqlite3_file *pFile, /* File to read from */ + i64 iEof, /* Total number of bytes in file */ + i64 *piOffset, /* IN/OUT: Read offset in pFile */ + i64 *piVal /* OUT: Value read from file */ +){ + u8 aVarint[9]; /* Buffer large enough for a varint */ + i64 iOff = *piOffset; /* Offset in file to read from */ + int nRead = 9; /* Number of bytes to read from file */ + int rc; /* Return code */ + + assert( iEof>iOff ); + if( (iEof-iOff)iWriteOff>iStart ); + assert( pIter->aAlloc==0 ); + pIter->pFile = pSorter->pTemp1; + pIter->iReadOff = iStart; + pIter->nAlloc = 128; + pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc); + if( !pIter->aAlloc ){ + rc = SQLITE_NOMEM; + }else{ + i64 iEof = pSorter->iWriteOff; /* EOF of file pSorter->pTemp1 */ + i64 nByte; /* Total size of PMA in bytes */ + rc = vdbeSorterReadVarint(pSorter->pTemp1, iEof, &pIter->iReadOff, &nByte); + *pnByte += nByte; + pIter->iEof = pIter->iReadOff + nByte; + } + if( rc==SQLITE_OK ){ + rc = vdbeSorterIterNext(db, pIter); + } + return rc; +} + +/* +** This function is called to compare two iterator keys when merging +** multiple b-tree segments. Parameter iOut is the index of the aTree[] +** value to recalculate. +*/ +static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ + VdbeSorter *pSorter = pCsr->pSorter; + int i1; + int i2; + int iRes; + VdbeSorterIter *p1; + VdbeSorterIter *p2; + + assert( iOutnTree && iOut>0 ); + + if( iOut>=(pSorter->nTree/2) ){ + i1 = (iOut - pSorter->nTree/2) * 2; + i2 = i1 + 1; + }else{ + i1 = pSorter->aTree[iOut*2]; + i2 = pSorter->aTree[iOut*2+1]; + } + + p1 = &pSorter->aIter[i1]; + p2 = &pSorter->aIter[i2]; + + if( p1->pFile==0 ){ + iRes = i2; + }else if( p2->pFile==0 ){ + iRes = i1; + }else{ + char aSpace[150]; + UnpackedRecord *r1; + + r1 = sqlite3VdbeRecordUnpack( + pCsr->pKeyInfo, p1->nKey, p1->aKey, aSpace, sizeof(aSpace) + ); + if( r1==0 ) return SQLITE_NOMEM; + + if( sqlite3VdbeRecordCompare(p2->nKey, p2->aKey, r1)>=0 ){ + iRes = i1; + }else{ + iRes = i2; + } + sqlite3VdbeDeleteUnpackedRecord(r1); + } + + pSorter->aTree[iOut] = iRes; + return SQLITE_OK; +} + +/* +** Initialize the temporary index cursor just opened as a sorter cursor. +*/ +int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ + assert( pCsr->pKeyInfo && pCsr->pBt ); + pCsr->pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); - return (pCsr->pSorter ? SQLITE_NOMEM : SQLITE_OK); ++ return (pCsr->pSorter ? SQLITE_OK : SQLITE_NOMEM); +} + +/* +** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. +*/ +void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ + VdbeSorter *pSorter = pCsr->pSorter; + if( pSorter ){ + if( pSorter->aIter ){ + int i; + for(i=0; inTree; i++){ + vdbeSorterIterZero(db, &pSorter->aIter[i]); + } + sqlite3DbFree(db, pSorter->aIter); + } + if( pSorter->pTemp1 ){ + sqlite3OsCloseFree(pSorter->pTemp1); + } + sqlite3DbFree(db, pSorter); + pCsr->pSorter = 0; + } +} + +/* +** Allocate space for a file-handle and open a temporary file. If successful, +** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK. +** Otherwise, set *ppFile to 0 and return an SQLite error code. +*/ +static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ + int dummy; + return sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, + SQLITE_OPEN_TEMP_DB | + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &dummy + ); +} + + +/* +** Write the current contents of the b-tree to a PMA. Return SQLITE_OK +** if successful, or an SQLite error code otherwise. +** +** The format of a PMA is: +** +** * A varint. This varint contains the total number of bytes of content +** in the PMA (not including the varint itself). +** +** * One or more records packed end-to-end in order of ascending keys. +** Each record consists of a varint followed by a blob of data (the +** key). The varint is the number of bytes in the blob of data. +*/ +static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){ + int rc = SQLITE_OK; /* Return code */ + VdbeSorter *pSorter = pCsr->pSorter; + int res = 0; + + rc = sqlite3BtreeFirst(pCsr->pCursor, &res); + if( rc!=SQLITE_OK || res ) return rc; + assert( pSorter->nBtree>0 ); + + /* If the first temporary PMA file has not been opened, open it now. */ + if( pSorter->pTemp1==0 ){ + rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1); + assert( rc!=SQLITE_OK || pSorter->pTemp1 ); + assert( pSorter->iWriteOff==0 ); + assert( pSorter->nPMA==0 ); + } + + if( rc==SQLITE_OK ){ + i64 iWriteOff = pSorter->iWriteOff; + void *aMalloc = 0; /* Array used to hold a single record */ + int nMalloc = 0; /* Allocated size of aMalloc[] in bytes */ + + pSorter->nPMA++; + for( + rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nBtree, &iWriteOff); + rc==SQLITE_OK && res==0; + rc = sqlite3BtreeNext(pCsr->pCursor, &res) + ){ + i64 nKey; /* Size of this key in bytes */ + + /* Write the size of the record in bytes to the output file */ + (void)sqlite3BtreeKeySize(pCsr->pCursor, &nKey); + rc = vdbeSorterWriteVarint(pSorter->pTemp1, nKey, &iWriteOff); + + /* Make sure the aMalloc[] buffer is large enough for the record */ + if( rc==SQLITE_OK && nKey>nMalloc ){ + aMalloc = sqlite3DbReallocOrFree(db, aMalloc, nKey); + if( !aMalloc ){ + rc = SQLITE_NOMEM; + }else{ + nMalloc = nKey; + } + } + + /* Write the record itself to the output file */ + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff); + iWriteOff += nKey; + } + } + + if( rc!=SQLITE_OK ) break; + } + + /* This assert verifies that unless an error has occurred, the size of + ** the PMA on disk is the same as the expected size stored in + ** pSorter->nBtree. */ + assert( rc!=SQLITE_OK || pSorter->nBtree==( + iWriteOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nBtree) + )); + + pSorter->iWriteOff = iWriteOff; + sqlite3DbFree(db, aMalloc); + } + + pSorter->nBtree = 0; + return rc; +} + +/* +** This function is called on a sorter cursor by the VDBE before each row +** is inserted into VdbeCursor.pCsr. Argument nKey is the size of the key, in +** bytes, about to be inserted. +** +** If it is determined that the temporary b-tree accessed via VdbeCursor.pCsr +** is large enough, its contents are written to a sorted PMA on disk and the +** tree emptied. This prevents the b-tree (which must be small enough to +** fit entirely in the cache in order to support efficient inserts) from +** growing too large. +** +** An SQLite error code is returned if an error occurs. Otherwise, SQLITE_OK. +*/ +int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){ + int rc = SQLITE_OK; /* Return code */ + VdbeSorter *pSorter = pCsr->pSorter; + if( pSorter ){ + Pager *pPager = sqlite3BtreePager(pCsr->pBt); + int nPage; /* Current size of temporary file in pages */ + + /* Determine how many pages the temporary b-tree has grown to */ + sqlite3PagerPagecount(pPager, &nPage); + + /* If pSorter->nWorking is still zero, but the temporary file has been + ** created in the file-system, then the most recent insert into the + ** current b-tree segment probably caused the cache to overflow (it is + ** also possible that sqlite3_release_memory() was called). So set the + ** size of the working set to a little less than the current size of the + ** file in pages. */ + if( pSorter->nWorking==0 && sqlite3PagerFile(pPager)->pMethods ){ + pSorter->nWorking = nPage-5; + if( pSorter->nWorkingnWorking = SORTER_MIN_WORKING; + } + } + + /* If the number of pages used by the current b-tree segment is greater + ** than the size of the working set (VdbeSorter.nWorking), start a new + ** segment b-tree. */ + if( pSorter->nWorking && nPage>=pSorter->nWorking ){ + BtCursor *p = pCsr->pCursor;/* Cursor structure to close and reopen */ + int iRoot; /* Root page of new tree */ + + /* Copy the current contents of the b-tree into a PMA in sorted order. + ** Close the currently open b-tree cursor. */ + rc = vdbeSorterBtreeToPMA(db, pCsr); + sqlite3BtreeCloseCursor(p); + + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeDropTable(pCsr->pBt, 2, 0); +#ifdef SQLITE_DEBUG + sqlite3PagerPagecount(pPager, &nPage); + assert( rc!=SQLITE_OK || nPage==1 ); +#endif + } + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeCreateTable(pCsr->pBt, &iRoot, BTREE_BLOBKEY); + } + if( rc==SQLITE_OK ){ + assert( iRoot==2 ); + rc = sqlite3BtreeCursor(pCsr->pBt, iRoot, 1, pCsr->pKeyInfo, p); + } + } + + pSorter->nBtree += sqlite3VarintLen(nKey) + nKey; + } + return rc; +} + +/* +** Helper function for sqlite3VdbeSorterRewind(). +*/ +static int vdbeSorterInitMerge( + sqlite3 *db, /* Database handle */ + VdbeCursor *pCsr, /* Cursor handle for this sorter */ + i64 *pnByte /* Sum of bytes in all opened PMAs */ +){ + VdbeSorter *pSorter = pCsr->pSorter; + int rc = SQLITE_OK; /* Return code */ + int i; /* Used to iterator through aIter[] */ + i64 nByte = 0; /* Total bytes in all opened PMAs */ + + /* Initialize the iterators. */ + for(i=0; rc==SQLITE_OK && iaIter[i]; + rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte); + pSorter->iReadOff = pIter->iEof; + assert( pSorter->iReadOff<=pSorter->iWriteOff ); + if( pSorter->iReadOff>=pSorter->iWriteOff ) break; + } + + /* Initialize the aTree[] array. */ + for(i=pSorter->nTree-1; rc==SQLITE_OK && i>0; i--){ + rc = vdbeSorterDoCompare(pCsr, i); + } + + *pnByte = nByte; + return rc; +} + +/* +** Once the sorter has been populated, this function is called to prepare +** for iterating through its contents in sorted order. +*/ +int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ + VdbeSorter *pSorter = pCsr->pSorter; + int rc; /* Return code */ + sqlite3_file *pTemp2 = 0; /* Second temp file to use */ + i64 iWrite2 = 0; /* Write offset for pTemp2 */ + int nIter; /* Number of iterators used */ + int nByte; /* Bytes of space required for aIter/aTree */ + int N = 2; /* Power of 2 >= nIter */ + + assert( pSorter ); + + /* Write the current b-tree to a PMA. Close the b-tree cursor. */ + rc = vdbeSorterBtreeToPMA(db, pCsr); + sqlite3BtreeCloseCursor(pCsr->pCursor); + if( rc!=SQLITE_OK ) return rc; + if( pSorter->nPMA==0 ){ + *pbEof = 1; + return SQLITE_OK; + } + + /* Allocate space for aIter[] and aTree[]. */ + nIter = pSorter->nPMA; + if( nIter>SORTER_MAX_MERGE_COUNT ) nIter = SORTER_MAX_MERGE_COUNT; + assert( nIter>0 ); + while( NaIter = (VdbeSorterIter *)sqlite3DbMallocZero(db, nByte); + if( !pSorter->aIter ) return SQLITE_NOMEM; + pSorter->aTree = (int *)&pSorter->aIter[N]; + pSorter->nTree = N; + + do { + int iNew; /* Index of new, merged, PMA */ + + for(iNew=0; + rc==SQLITE_OK && iNew*SORTER_MAX_MERGE_COUNTnPMA; + iNew++ + ){ + i64 nWrite; /* Number of bytes in new PMA */ + + /* If there are SORTER_MAX_MERGE_COUNT or less PMAs in file pTemp1, + ** initialize an iterator for each of them and break out of the loop. + ** These iterators will be incrementally merged as the VDBE layer calls + ** sqlite3VdbeSorterNext(). + ** + ** Otherwise, if pTemp1 contains more than SORTER_MAX_MERGE_COUNT PMAs, + ** initialize interators for SORTER_MAX_MERGE_COUNT of them. These PMAs + ** are merged into a single PMA that is written to file pTemp2. + */ + rc = vdbeSorterInitMerge(db, pCsr, &nWrite); + assert( rc!=SQLITE_OK || pSorter->aIter[ pSorter->aTree[1] ].pFile ); + if( rc!=SQLITE_OK || pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ + break; + } + + /* Open the second temp file, if it is not already open. */ + if( pTemp2==0 ){ + assert( iWrite2==0 ); + rc = vdbeSorterOpenTempFile(db, &pTemp2); + } + + if( rc==SQLITE_OK ){ + rc = vdbeSorterWriteVarint(pTemp2, nWrite, &iWrite2); + } + + if( rc==SQLITE_OK ){ + int bEof = 0; + while( rc==SQLITE_OK && bEof==0 ){ + int nByte; + VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ]; + assert( pIter->pFile ); + nByte = pIter->nKey + sqlite3VarintLen(pIter->nKey); + rc = sqlite3OsWrite(pTemp2, pIter->aAlloc, nByte, iWrite2); + iWrite2 += nByte; + if( rc==SQLITE_OK ){ + rc = sqlite3VdbeSorterNext(db, pCsr, &bEof); + } + } + } + } + + if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ + break; + }else{ + sqlite3_file *pTmp = pSorter->pTemp1; + pSorter->nPMA = iNew; + pSorter->pTemp1 = pTemp2; + pTemp2 = pTmp; + pSorter->iWriteOff = iWrite2; + pSorter->iReadOff = 0; + iWrite2 = 0; + } + }while( rc==SQLITE_OK ); + + if( pTemp2 ){ + sqlite3OsCloseFree(pTemp2); + } + *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + return rc; +} + +/* +** Advance to the next element in the sorter. +*/ +int sqlite3VdbeSorterNext(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ + VdbeSorter *pSorter = pCsr->pSorter; + int iPrev = pSorter->aTree[1]; /* Index of iterator to advance */ + int i; /* Index of aTree[] to recalculate */ + int rc; /* Return code */ + + rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); + for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ + rc = vdbeSorterDoCompare(pCsr, i); + } + + *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + return rc; +} + +/* +** Copy the current sorter key into the memory cell pOut. +*/ +int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){ + VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorterIter *pIter; + + pIter = &pSorter->aIter[ pSorter->aTree[1] ]; + + /* Coverage testing note: As things are currently, this call will always + ** succeed. This is because the memory cell passed by the VDBE layer + ** happens to be the same one as was used to assemble the keys before they + ** were passed to the sorter - meaning it is always large enough for the + ** largest key. But this could change very easily, so we leave the call + ** to sqlite3VdbeMemGrow() in. */ + if( sqlite3VdbeMemGrow(pOut, pIter->nKey, 0) ){ + return SQLITE_NOMEM; + } + pOut->n = pIter->nKey; + MemSetTypeFlag(pOut, MEM_Blob); + memcpy(pOut->z, pIter->aKey, pIter->nKey); + + return SQLITE_OK; +} + +#endif /* #ifndef SQLITE_OMIT_MERGE_SORT */