- C Remove\san\sunused\sparameter\sfrom\sa\sfunction\sin\svdbesort.c.\sFix\ssome\scomments\sand\sother\sdetails\sin\sthe\ssame\sfile.
- D 2011-08-12T16:11:43.093
-C Make\sthe\sopenDirectory\sroutine\sin\sos_unix.c\soverrideable\sso\sthat\sit\scan\nbe\sturned\sinto\sa\sharmless\sno-op\sfor\sthe\schromium\ssandbox.
-D 2011-08-10T01:52:12.736
++C Merge\slatest\strunk\schanges\sinto\sexperimental\sbranch.
++D 2011-08-12T16:30:30.031
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
- F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e
+ F Makefile.in 1e6988b3c11dee9bd5edc0c804bd4468d74a9cdc
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
- F Makefile.msc 9c76731c20561a33ce1257c70be3f029cedb62a6
+ F Makefile.msc 9fd0401b0134b565a9c7d8ed49dc044cc89f2096
F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151
F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6
- F VERSION 3fcdd7fbe3eb282df3978fe77288544543767961
+ F VERSION f724de7326e87b7f3b0a55f16ef4b4d993680d54
F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
F addopcodes.awk 17dc593f791f874d2c23a0f9360850ded0286531
F art/2005osaward.gif 0d1851b2a7c1c9d0ccce545f3e14bca42d7fd248
F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
- F main.mk df1e47e4bc886f556b39a8cdb9dc3f6fb6810d64
-F main.mk 50adb7368bd0faa888eb4634b874d542d44f3de3
++F main.mk a812efc287a904c3e04098f090de036b264ba779
F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a
F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f
F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac
F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c
F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef
F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
-F src/btree.c 8c46f0ab69ad9549c75a3a91fed87abdaa743e2f
+F src/btree.c a30bdcc27eedc36a38a3a11e1ba83de9a6729f7e
F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce
F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3
- F src/build.c ac39a62c06fbc3d6ef1d5f784e06d806bc821edc
-F src/build.c 19a8957a442d922a0d6ed1a5dd67b63202fc3260
++F src/build.c 77b40abf79e1541356f552c9681b0b431aba672b
F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a
F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
-F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4
+F src/ctime.c 7f283795650dd4122cc07bd02193c40193b32cc6
F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3
F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8
F src/expr.c 4bbdfaf66bc614be9254ce0c26a17429067a3e07
F src/trigger.c 1cfb80e2290ef66ea89cb4e821caae65a02c0d56
F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec
F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0
- F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70
+ F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7
F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e
- F src/vdbe.c 1f5bb8b8a0f24bd93e66958eaebd060ccee272a2
-F src/vdbe.c 49d834f0fe49d305e07f9c212e94007fda2028e9
++F src/vdbe.c 22d0d5bc8e8e467e1e0fa20a0a6b21c297f5601f
F src/vdbe.h 5cf09e7ee8a3f7d93bc51f196a96550786afe7a1
-F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45
+F src/vdbeInt.h f9250326f264ca5f100acc19e9c07096bb889096
F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98
-F src/vdbeaux.c 4d100407e3c72e163854aff8903d19d5ecdf46c0
+F src/vdbeaux.c 8fb978eb73a97b34d352dd3ef3bff35b1b3fa7e9
F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3
F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b
- F src/vdbesort.c a756d20db3ff596dd3dd10f6a13fd832f5715540
++F src/vdbesort.c 3e6b2268326ee075ae5e8cece4e469e5d1d4a859
F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114
F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582
- F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290
+ F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9
F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a
F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
- F src/where.c ce7cce80c5cb07ba40e9bf38a33ef806c61f55f2
- F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199
+ F src/where.c 7d09f4c1512affb60cc1190a4b33d121d4ce039a
+ F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823
F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87
F test/all.test 52fc8dee494092031a556911d404ca30a749a30b
F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401
F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7
F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806
- F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb
+ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16
- F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16
- F test/permutations.test bfe51414b7c355a6dfb8803176808dc2e00e3783
- F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850
- F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47
+ F test/pcache2.test 9f9357bb0f463b87bdf695646024ed2031a0c85a
-F test/permutations.test 2c845b5c8cd2abb32f07076734d10e6a9fd19f26
++F test/permutations.test ad17319066a90e2db71823c3ff104795ffc71b31
+ F test/pragma.test c8108e01da04f16e67e5754e610bc62c1b993f6c
+ F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947
F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea
F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301
F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc
F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
- F test/wal.test 5617ad308bfdb8a8885220d8a261a6096a8d7e57
- F test/wal2.test ceba91f0d82c4fd87f342aa5504eb1f8fa6418f3
- F test/wal3.test 5c396cc22497244d627306f4c1d360167353f8dd
- F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30
- F test/wal5.test f06a0427e06db00347e32eb9fa99d6a5c0f2d088
- F test/wal6.test 07aa31ca8892d0527f2c5c5a9a2a87aa421dfaa8
+ F test/wal.test e11da8d5ea8a38a247339455098357e9adf63d76
-F test/wal2.test a2caa432d056596c6ee815d2743b2ced86f67a59
++F test/wal2.test ad6412596815f553cd30f271d291ab003092bc7e
+ F test/wal3.test 18da4e65c30c43c646ad40e145e9a074e4062fc9
+ F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
+ F test/wal5.test 08e145a352b1223930c7f0a1de82a8747a99c322
+ F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3
F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd
F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe
- F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4
+ F test/walbak.test 53be94bccbc3f592d22a5909588ccc81b319ce83
F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0
- F test/walcksum.test a37b36375c595e61bdb7e1ec49b5f0979b6fc7ce
- F test/walcrash.test e763841551d6b23677ccb419797c1589dcbdbaf5
+ F test/walcksum.test f5447800a157c9e2234fbb8e80243f0813941bde
+ F test/walcrash.test 4fcb661faf71db91214156d52d43ee327f52bde1
F test/walcrash2.test 019d60b89d96c1937adb2b30b850ac7e86e5a142
- F test/walfault.test 58fce626359c9376fe35101b5c0f2df8040aa839
+ F test/walfault.test efb0d5724893133e71b8d9d90abdb781845a6bb0
F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483
- F test/walmode.test 22ddccd073c817ac9ead62b88ac446e8dedc7d2c
- F test/walnoshm.test a074428046408f4eb5c6a00e09df8cc97ff93317
+ F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c
+ F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496
+ F test/walpersist.test 45fb0c94fb63908e2d66b1d99ce4645bfce0fa1e
F test/walro.test 2d5d69e2e99da19ce6faab340330234fc4ca0720
F test/walshared.test 6dda2293880c300baf5d791c307f653094585761
- F test/walslow.test d21625e2e99e11c032ce949e8a94661576548933
- F test/walthread.test a25a393c068a2b42b44333fa3fdaae9072f1617c
+ F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a
+ F test/walthread.test a2ed5270eb695284d4ad27d252517bdc3317ee2a
F test/where.test de337a3fe0a459ec7c93db16a519657a90552330
F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554
F test/where3.test 8e1175c7ef710c70502858fc4fb08d784b3620b9
F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309
F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
-F tool/mksqlite3c.tcl 1fa0ed9cfdc768bf5de7e65fda8d97a46dd2a7e6
+F tool/mksqlite3c.tcl d8c0d3065bc23fd9e27d59bfebd34df203fe6b08
F tool/mksqlite3h.tcl 78013ad79a5e492e5f764f3c7a8ef834255061f8
F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87
+ F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091
F tool/omittest.tcl 8086c014cbae90f1f2b564d59d05a5e4ac1783c9
F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c
F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262
- P 4ced2394b10d0a4f86422ff893bcdf3cf32591e3
- R ced8374c50fa0d83b7bc754994eec7c3
-P dfa22ed4387f9526b74d5265503c7c8e9d559903
-R bf22196e9aa98f18724e5d2624f7dcbf
-U drh
-Z 2ec35109792cc0dc5d4f7cebd4d85034
++P 1a8498d8037a1b93e56951bbdbb76291bd5a4f87 6b236069e1ea3c99ff0a007a790d4baebda70b13
++R 31c6a782e4904c59693691553f2ae0bd
+U dan
- Z 2a841cd45d02eb2ca25a3f8996849530
++Z d620fa181c3729f4f9367beda130e667
--- /dev/null
- return (pCsr->pSorter ? SQLITE_NOMEM : SQLITE_OK);
+/*
+** 2011 July 9
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file contains code for the VdbeSorter object, used in concert with
+** a VdbeCursor to sort large numbers of keys (as may be required, for
+** example, by CREATE INDEX statements on tables too large to fit in main
+** memory).
+*/
+
+#include "sqliteInt.h"
+#include "vdbeInt.h"
+
+#ifndef SQLITE_OMIT_MERGE_SORT
+
+typedef struct VdbeSorterIter VdbeSorterIter;
+
+/*
+** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES:
+**
+** As keys are added to the sorter, they are written to disk in a series
+** of sorted packed-memory-arrays (PMAs). The size of each PMA is roughly
+** the same as the cache-size allowed for temporary databases. In order
+** to allow the caller to extract keys from the sorter in sorted order,
+** all PMAs currently stored on disk must be merged together. This comment
+** describes the data structure used to do so. The structure supports
+** merging any number of arrays in a single pass with no redundant comparison
+** operations.
+**
+** The aIter[] array contains an iterator for each of the PMAs being merged.
+** An aIter[] iterator either points to a valid key or else is at EOF. For
+** the purposes of the paragraphs below, we assume that the array is actually
+** N elements in size, where N is the smallest power of 2 greater to or equal
+** to the number of iterators being merged. The extra aIter[] elements are
+** treated as if they are empty (always at EOF).
+**
+** The aTree[] array is also N elements in size. The value of N is stored in
+** the VdbeSorter.nTree variable.
+**
+** The final (N/2) elements of aTree[] contain the results of comparing
+** pairs of iterator keys together. Element i contains the result of
+** comparing aIter[2*i-N] and aIter[2*i-N+1]. Whichever key is smaller, the
+** aTree element is set to the index of it.
+**
+** For the purposes of this comparison, EOF is considered greater than any
+** other key value. If the keys are equal (only possible with two EOF
+** values), it doesn't matter which index is stored.
+**
+** The (N/4) elements of aTree[] that preceed the final (N/2) described
+** above contains the index of the smallest of each block of 4 iterators.
+** And so on. So that aTree[1] contains the index of the iterator that
+** currently points to the smallest key value. aTree[0] is unused.
+**
+** Example:
+**
+** aIter[0] -> Banana
+** aIter[1] -> Feijoa
+** aIter[2] -> Elderberry
+** aIter[3] -> Currant
+** aIter[4] -> Grapefruit
+** aIter[5] -> Apple
+** aIter[6] -> Durian
+** aIter[7] -> EOF
+**
+** aTree[] = { X, 5 0, 5 0, 3, 5, 6 }
+**
+** The current element is "Apple" (the value of the key indicated by
+** iterator 5). When the Next() operation is invoked, iterator 5 will
+** be advanced to the next key in its segment. Say the next key is
+** "Eggplant":
+**
+** aIter[5] -> Eggplant
+**
+** The contents of aTree[] are updated first by comparing the new iterator
+** 5 key to the current key of iterator 4 (still "Grapefruit"). The iterator
+** 5 value is still smaller, so aTree[6] is set to 5. And so on up the tree.
+** The value of iterator 6 - "Durian" - is now smaller than that of iterator
+** 5, so aTree[3] is set to 6. Key 0 is smaller than key 6 (Banana<Durian),
+** so the value written into element 1 of the array is 0. As follows:
+**
+** aTree[] = { X, 0 0, 6 0, 3, 5, 6 }
+**
+** In other words, each time we advance to the next sorter element, log2(N)
+** key comparison operations are required, where N is the number of segments
+** being merged (rounded up to the next power of 2).
+*/
+struct VdbeSorter {
+ int nWorking; /* Start a new b-tree after this many pages */
+ int nBtree; /* Current size of b-tree contents as PMA */
+ int nTree; /* Used size of aTree/aIter (power of 2) */
+ VdbeSorterIter *aIter; /* Array of iterators to merge */
+ int *aTree; /* Current state of incremental merge */
+ i64 iWriteOff; /* Current write offset within file pTemp1 */
+ i64 iReadOff; /* Current read offset within file pTemp1 */
+ sqlite3_file *pTemp1; /* PMA file 1 */
+ int nPMA; /* Number of PMAs stored in pTemp1 */
+};
+
+/*
+** The following type is an iterator for a PMA. It caches the current key in
+** variables nKey/aKey. If the iterator is at EOF, pFile==0.
+*/
+struct VdbeSorterIter {
+ i64 iReadOff; /* Current read offset */
+ i64 iEof; /* 1 byte past EOF for this iterator */
+ sqlite3_file *pFile; /* File iterator is reading from */
+ int nAlloc; /* Bytes of space at aAlloc */
+ u8 *aAlloc; /* Allocated space */
+ int nKey; /* Number of bytes in key */
+ u8 *aKey; /* Pointer to current key */
+};
+
+/* Minimum allowable value for the VdbeSorter.nWorking variable */
+#define SORTER_MIN_WORKING 10
+
+/* Maximum number of segments to merge in a single pass. */
+#define SORTER_MAX_MERGE_COUNT 16
+
+/*
+** Free all memory belonging to the VdbeSorterIter object passed as the second
+** argument. All structure fields are set to zero before returning.
+*/
+static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){
+ sqlite3DbFree(db, pIter->aAlloc);
+ memset(pIter, 0, sizeof(VdbeSorterIter));
+}
+
+/*
+** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if
+** no error occurs, or an SQLite error code if one does.
+*/
+static int vdbeSorterIterNext(
+ sqlite3 *db, /* Database handle (for sqlite3DbMalloc() ) */
+ VdbeSorterIter *pIter /* Iterator to advance */
+){
+ int rc; /* Return Code */
+ int nRead; /* Number of bytes read */
+ int nRec; /* Size of record in bytes */
+ int iOff; /* Size of serialized size varint in bytes */
+
+ nRead = pIter->iEof - pIter->iReadOff;
+ if( nRead>5 ) nRead = 5;
+ if( nRead<=0 ){
+ /* This is an EOF condition */
+ vdbeSorterIterZero(db, pIter);
+ return SQLITE_OK;
+ }
+
+ rc = sqlite3OsRead(pIter->pFile, pIter->aAlloc, nRead, pIter->iReadOff);
+ iOff = getVarint32(pIter->aAlloc, nRec);
+
+ if( rc==SQLITE_OK && (iOff+nRec)>nRead ){
+ int nRead2; /* Number of extra bytes to read */
+ if( (iOff+nRec)>pIter->nAlloc ){
+ int nNew = pIter->nAlloc*2;
+ while( (iOff+nRec)>nNew ) nNew = nNew*2;
+ pIter->aAlloc = sqlite3DbReallocOrFree(db, pIter->aAlloc, nNew);
+ if( !pIter->aAlloc ) return SQLITE_NOMEM;
+ pIter->nAlloc = nNew;
+ }
+
+ nRead2 = iOff + nRec - nRead;
+ rc = sqlite3OsRead(
+ pIter->pFile, &pIter->aAlloc[nRead], nRead2, pIter->iReadOff+nRead
+ );
+ }
+
+ assert( nRec>0 || rc!=SQLITE_OK );
+ pIter->iReadOff += iOff+nRec;
+ pIter->nKey = nRec;
+ pIter->aKey = &pIter->aAlloc[iOff];
+ return rc;
+}
+
+/*
+** Write a single varint, value iVal, to file-descriptor pFile. Return
+** SQLITE_OK if successful, or an SQLite error code if some error occurs.
+**
+** The value of *piOffset when this function is called is used as the byte
+** offset in file pFile to write to. Before returning, *piOffset is
+** incremented by the number of bytes written.
+*/
+static int vdbeSorterWriteVarint(
+ sqlite3_file *pFile, /* File to write to */
+ i64 iVal, /* Value to write as a varint */
+ i64 *piOffset /* IN/OUT: Write offset in file pFile */
+){
+ u8 aVarint[9]; /* Buffer large enough for a varint */
+ int nVarint; /* Number of used bytes in varint */
+ int rc; /* Result of write() call */
+
+ nVarint = sqlite3PutVarint(aVarint, iVal);
+ rc = sqlite3OsWrite(pFile, aVarint, nVarint, *piOffset);
+ *piOffset += nVarint;
+
+ return rc;
+}
+
+/*
+** Read a single varint from file-descriptor pFile. Return SQLITE_OK if
+** successful, or an SQLite error code if some error occurs.
+**
+** The value of *piOffset when this function is called is used as the
+** byte offset in file pFile from whence to read the varint. If successful
+** (i.e. if no IO error occurs), then *piOffset is set to the offset of
+** the first byte past the end of the varint before returning. *piVal is
+** set to the integer value read. If an error occurs, the final values of
+** both *piOffset and *piVal are undefined.
+*/
+static int vdbeSorterReadVarint(
+ sqlite3_file *pFile, /* File to read from */
+ i64 iEof, /* Total number of bytes in file */
+ i64 *piOffset, /* IN/OUT: Read offset in pFile */
+ i64 *piVal /* OUT: Value read from file */
+){
+ u8 aVarint[9]; /* Buffer large enough for a varint */
+ i64 iOff = *piOffset; /* Offset in file to read from */
+ int nRead = 9; /* Number of bytes to read from file */
+ int rc; /* Return code */
+
+ assert( iEof>iOff );
+ if( (iEof-iOff)<nRead ){
+ nRead = iEof-iOff;
+ }
+
+ rc = sqlite3OsRead(pFile, aVarint, nRead, iOff);
+ if( rc==SQLITE_OK ){
+ *piOffset += getVarint(aVarint, (u64 *)piVal);
+ }
+
+ return rc;
+}
+
+/*
+** Initialize iterator pIter to scan through the PMA stored in file pFile
+** starting at offset iStart and ending at offset iEof-1. This function
+** leaves the iterator pointing to the first key in the PMA (or EOF if the
+** PMA is empty).
+*/
+static int vdbeSorterIterInit(
+ sqlite3 *db, /* Database handle */
+ VdbeSorter *pSorter, /* Sorter object */
+ i64 iStart, /* Start offset in pFile */
+ VdbeSorterIter *pIter, /* Iterator to populate */
+ i64 *pnByte /* IN/OUT: Increment this value by PMA size */
+){
+ int rc;
+
+ assert( pSorter->iWriteOff>iStart );
+ assert( pIter->aAlloc==0 );
+ pIter->pFile = pSorter->pTemp1;
+ pIter->iReadOff = iStart;
+ pIter->nAlloc = 128;
+ pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc);
+ if( !pIter->aAlloc ){
+ rc = SQLITE_NOMEM;
+ }else{
+ i64 iEof = pSorter->iWriteOff; /* EOF of file pSorter->pTemp1 */
+ i64 nByte; /* Total size of PMA in bytes */
+ rc = vdbeSorterReadVarint(pSorter->pTemp1, iEof, &pIter->iReadOff, &nByte);
+ *pnByte += nByte;
+ pIter->iEof = pIter->iReadOff + nByte;
+ }
+ if( rc==SQLITE_OK ){
+ rc = vdbeSorterIterNext(db, pIter);
+ }
+ return rc;
+}
+
+/*
+** This function is called to compare two iterator keys when merging
+** multiple b-tree segments. Parameter iOut is the index of the aTree[]
+** value to recalculate.
+*/
+static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){
+ VdbeSorter *pSorter = pCsr->pSorter;
+ int i1;
+ int i2;
+ int iRes;
+ VdbeSorterIter *p1;
+ VdbeSorterIter *p2;
+
+ assert( iOut<pSorter->nTree && iOut>0 );
+
+ if( iOut>=(pSorter->nTree/2) ){
+ i1 = (iOut - pSorter->nTree/2) * 2;
+ i2 = i1 + 1;
+ }else{
+ i1 = pSorter->aTree[iOut*2];
+ i2 = pSorter->aTree[iOut*2+1];
+ }
+
+ p1 = &pSorter->aIter[i1];
+ p2 = &pSorter->aIter[i2];
+
+ if( p1->pFile==0 ){
+ iRes = i2;
+ }else if( p2->pFile==0 ){
+ iRes = i1;
+ }else{
+ char aSpace[150];
+ UnpackedRecord *r1;
+
+ r1 = sqlite3VdbeRecordUnpack(
+ pCsr->pKeyInfo, p1->nKey, p1->aKey, aSpace, sizeof(aSpace)
+ );
+ if( r1==0 ) return SQLITE_NOMEM;
+
+ if( sqlite3VdbeRecordCompare(p2->nKey, p2->aKey, r1)>=0 ){
+ iRes = i1;
+ }else{
+ iRes = i2;
+ }
+ sqlite3VdbeDeleteUnpackedRecord(r1);
+ }
+
+ pSorter->aTree[iOut] = iRes;
+ return SQLITE_OK;
+}
+
+/*
+** Initialize the temporary index cursor just opened as a sorter cursor.
+*/
+int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){
+ assert( pCsr->pKeyInfo && pCsr->pBt );
+ pCsr->pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter));
++ return (pCsr->pSorter ? SQLITE_OK : SQLITE_NOMEM);
+}
+
+/*
+** Free any cursor components allocated by sqlite3VdbeSorterXXX routines.
+*/
+void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){
+ VdbeSorter *pSorter = pCsr->pSorter;
+ if( pSorter ){
+ if( pSorter->aIter ){
+ int i;
+ for(i=0; i<pSorter->nTree; i++){
+ vdbeSorterIterZero(db, &pSorter->aIter[i]);
+ }
+ sqlite3DbFree(db, pSorter->aIter);
+ }
+ if( pSorter->pTemp1 ){
+ sqlite3OsCloseFree(pSorter->pTemp1);
+ }
+ sqlite3DbFree(db, pSorter);
+ pCsr->pSorter = 0;
+ }
+}
+
+/*
+** Allocate space for a file-handle and open a temporary file. If successful,
+** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK.
+** Otherwise, set *ppFile to 0 and return an SQLite error code.
+*/
+static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){
+ int dummy;
+ return sqlite3OsOpenMalloc(db->pVfs, 0, ppFile,
+ SQLITE_OPEN_TEMP_DB |
+ SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE |
+ SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &dummy
+ );
+}
+
+
+/*
+** Write the current contents of the b-tree to a PMA. Return SQLITE_OK
+** if successful, or an SQLite error code otherwise.
+**
+** The format of a PMA is:
+**
+** * A varint. This varint contains the total number of bytes of content
+** in the PMA (not including the varint itself).
+**
+** * One or more records packed end-to-end in order of ascending keys.
+** Each record consists of a varint followed by a blob of data (the
+** key). The varint is the number of bytes in the blob of data.
+*/
+static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){
+ int rc = SQLITE_OK; /* Return code */
+ VdbeSorter *pSorter = pCsr->pSorter;
+ int res = 0;
+
+ rc = sqlite3BtreeFirst(pCsr->pCursor, &res);
+ if( rc!=SQLITE_OK || res ) return rc;
+ assert( pSorter->nBtree>0 );
+
+ /* If the first temporary PMA file has not been opened, open it now. */
+ if( pSorter->pTemp1==0 ){
+ rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1);
+ assert( rc!=SQLITE_OK || pSorter->pTemp1 );
+ assert( pSorter->iWriteOff==0 );
+ assert( pSorter->nPMA==0 );
+ }
+
+ if( rc==SQLITE_OK ){
+ i64 iWriteOff = pSorter->iWriteOff;
+ void *aMalloc = 0; /* Array used to hold a single record */
+ int nMalloc = 0; /* Allocated size of aMalloc[] in bytes */
+
+ pSorter->nPMA++;
+ for(
+ rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nBtree, &iWriteOff);
+ rc==SQLITE_OK && res==0;
+ rc = sqlite3BtreeNext(pCsr->pCursor, &res)
+ ){
+ i64 nKey; /* Size of this key in bytes */
+
+ /* Write the size of the record in bytes to the output file */
+ (void)sqlite3BtreeKeySize(pCsr->pCursor, &nKey);
+ rc = vdbeSorterWriteVarint(pSorter->pTemp1, nKey, &iWriteOff);
+
+ /* Make sure the aMalloc[] buffer is large enough for the record */
+ if( rc==SQLITE_OK && nKey>nMalloc ){
+ aMalloc = sqlite3DbReallocOrFree(db, aMalloc, nKey);
+ if( !aMalloc ){
+ rc = SQLITE_NOMEM;
+ }else{
+ nMalloc = nKey;
+ }
+ }
+
+ /* Write the record itself to the output file */
+ if( rc==SQLITE_OK ){
+ rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc);
+ if( rc==SQLITE_OK ){
+ rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff);
+ iWriteOff += nKey;
+ }
+ }
+
+ if( rc!=SQLITE_OK ) break;
+ }
+
+ /* This assert verifies that unless an error has occurred, the size of
+ ** the PMA on disk is the same as the expected size stored in
+ ** pSorter->nBtree. */
+ assert( rc!=SQLITE_OK || pSorter->nBtree==(
+ iWriteOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nBtree)
+ ));
+
+ pSorter->iWriteOff = iWriteOff;
+ sqlite3DbFree(db, aMalloc);
+ }
+
+ pSorter->nBtree = 0;
+ return rc;
+}
+
+/*
+** This function is called on a sorter cursor by the VDBE before each row
+** is inserted into VdbeCursor.pCsr. Argument nKey is the size of the key, in
+** bytes, about to be inserted.
+**
+** If it is determined that the temporary b-tree accessed via VdbeCursor.pCsr
+** is large enough, its contents are written to a sorted PMA on disk and the
+** tree emptied. This prevents the b-tree (which must be small enough to
+** fit entirely in the cache in order to support efficient inserts) from
+** growing too large.
+**
+** An SQLite error code is returned if an error occurs. Otherwise, SQLITE_OK.
+*/
+int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){
+ int rc = SQLITE_OK; /* Return code */
+ VdbeSorter *pSorter = pCsr->pSorter;
+ if( pSorter ){
+ Pager *pPager = sqlite3BtreePager(pCsr->pBt);
+ int nPage; /* Current size of temporary file in pages */
+
+ /* Determine how many pages the temporary b-tree has grown to */
+ sqlite3PagerPagecount(pPager, &nPage);
+
+ /* If pSorter->nWorking is still zero, but the temporary file has been
+ ** created in the file-system, then the most recent insert into the
+ ** current b-tree segment probably caused the cache to overflow (it is
+ ** also possible that sqlite3_release_memory() was called). So set the
+ ** size of the working set to a little less than the current size of the
+ ** file in pages. */
+ if( pSorter->nWorking==0 && sqlite3PagerFile(pPager)->pMethods ){
+ pSorter->nWorking = nPage-5;
+ if( pSorter->nWorking<SORTER_MIN_WORKING ){
+ pSorter->nWorking = SORTER_MIN_WORKING;
+ }
+ }
+
+ /* If the number of pages used by the current b-tree segment is greater
+ ** than the size of the working set (VdbeSorter.nWorking), start a new
+ ** segment b-tree. */
+ if( pSorter->nWorking && nPage>=pSorter->nWorking ){
+ BtCursor *p = pCsr->pCursor;/* Cursor structure to close and reopen */
+ int iRoot; /* Root page of new tree */
+
+ /* Copy the current contents of the b-tree into a PMA in sorted order.
+ ** Close the currently open b-tree cursor. */
+ rc = vdbeSorterBtreeToPMA(db, pCsr);
+ sqlite3BtreeCloseCursor(p);
+
+ if( rc==SQLITE_OK ){
+ rc = sqlite3BtreeDropTable(pCsr->pBt, 2, 0);
+#ifdef SQLITE_DEBUG
+ sqlite3PagerPagecount(pPager, &nPage);
+ assert( rc!=SQLITE_OK || nPage==1 );
+#endif
+ }
+ if( rc==SQLITE_OK ){
+ rc = sqlite3BtreeCreateTable(pCsr->pBt, &iRoot, BTREE_BLOBKEY);
+ }
+ if( rc==SQLITE_OK ){
+ assert( iRoot==2 );
+ rc = sqlite3BtreeCursor(pCsr->pBt, iRoot, 1, pCsr->pKeyInfo, p);
+ }
+ }
+
+ pSorter->nBtree += sqlite3VarintLen(nKey) + nKey;
+ }
+ return rc;
+}
+
+/*
+** Helper function for sqlite3VdbeSorterRewind().
+*/
+static int vdbeSorterInitMerge(
+ sqlite3 *db, /* Database handle */
+ VdbeCursor *pCsr, /* Cursor handle for this sorter */
+ i64 *pnByte /* Sum of bytes in all opened PMAs */
+){
+ VdbeSorter *pSorter = pCsr->pSorter;
+ int rc = SQLITE_OK; /* Return code */
+ int i; /* Used to iterator through aIter[] */
+ i64 nByte = 0; /* Total bytes in all opened PMAs */
+
+ /* Initialize the iterators. */
+ for(i=0; rc==SQLITE_OK && i<SORTER_MAX_MERGE_COUNT; i++){
+ VdbeSorterIter *pIter = &pSorter->aIter[i];
+ rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte);
+ pSorter->iReadOff = pIter->iEof;
+ assert( pSorter->iReadOff<=pSorter->iWriteOff );
+ if( pSorter->iReadOff>=pSorter->iWriteOff ) break;
+ }
+
+ /* Initialize the aTree[] array. */
+ for(i=pSorter->nTree-1; rc==SQLITE_OK && i>0; i--){
+ rc = vdbeSorterDoCompare(pCsr, i);
+ }
+
+ *pnByte = nByte;
+ return rc;
+}
+
+/*
+** Once the sorter has been populated, this function is called to prepare
+** for iterating through its contents in sorted order.
+*/
+int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){
+ VdbeSorter *pSorter = pCsr->pSorter;
+ int rc; /* Return code */
+ sqlite3_file *pTemp2 = 0; /* Second temp file to use */
+ i64 iWrite2 = 0; /* Write offset for pTemp2 */
+ int nIter; /* Number of iterators used */
+ int nByte; /* Bytes of space required for aIter/aTree */
+ int N = 2; /* Power of 2 >= nIter */
+
+ assert( pSorter );
+
+ /* Write the current b-tree to a PMA. Close the b-tree cursor. */
+ rc = vdbeSorterBtreeToPMA(db, pCsr);
+ sqlite3BtreeCloseCursor(pCsr->pCursor);
+ if( rc!=SQLITE_OK ) return rc;
+ if( pSorter->nPMA==0 ){
+ *pbEof = 1;
+ return SQLITE_OK;
+ }
+
+ /* Allocate space for aIter[] and aTree[]. */
+ nIter = pSorter->nPMA;
+ if( nIter>SORTER_MAX_MERGE_COUNT ) nIter = SORTER_MAX_MERGE_COUNT;
+ assert( nIter>0 );
+ while( N<nIter ) N += N;
+ nByte = N * (sizeof(int) + sizeof(VdbeSorterIter));
+ pSorter->aIter = (VdbeSorterIter *)sqlite3DbMallocZero(db, nByte);
+ if( !pSorter->aIter ) return SQLITE_NOMEM;
+ pSorter->aTree = (int *)&pSorter->aIter[N];
+ pSorter->nTree = N;
+
+ do {
+ int iNew; /* Index of new, merged, PMA */
+
+ for(iNew=0;
+ rc==SQLITE_OK && iNew*SORTER_MAX_MERGE_COUNT<pSorter->nPMA;
+ iNew++
+ ){
+ i64 nWrite; /* Number of bytes in new PMA */
+
+ /* If there are SORTER_MAX_MERGE_COUNT or less PMAs in file pTemp1,
+ ** initialize an iterator for each of them and break out of the loop.
+ ** These iterators will be incrementally merged as the VDBE layer calls
+ ** sqlite3VdbeSorterNext().
+ **
+ ** Otherwise, if pTemp1 contains more than SORTER_MAX_MERGE_COUNT PMAs,
+ ** initialize interators for SORTER_MAX_MERGE_COUNT of them. These PMAs
+ ** are merged into a single PMA that is written to file pTemp2.
+ */
+ rc = vdbeSorterInitMerge(db, pCsr, &nWrite);
+ assert( rc!=SQLITE_OK || pSorter->aIter[ pSorter->aTree[1] ].pFile );
+ if( rc!=SQLITE_OK || pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){
+ break;
+ }
+
+ /* Open the second temp file, if it is not already open. */
+ if( pTemp2==0 ){
+ assert( iWrite2==0 );
+ rc = vdbeSorterOpenTempFile(db, &pTemp2);
+ }
+
+ if( rc==SQLITE_OK ){
+ rc = vdbeSorterWriteVarint(pTemp2, nWrite, &iWrite2);
+ }
+
+ if( rc==SQLITE_OK ){
+ int bEof = 0;
+ while( rc==SQLITE_OK && bEof==0 ){
+ int nByte;
+ VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ];
+ assert( pIter->pFile );
+ nByte = pIter->nKey + sqlite3VarintLen(pIter->nKey);
+ rc = sqlite3OsWrite(pTemp2, pIter->aAlloc, nByte, iWrite2);
+ iWrite2 += nByte;
+ if( rc==SQLITE_OK ){
+ rc = sqlite3VdbeSorterNext(db, pCsr, &bEof);
+ }
+ }
+ }
+ }
+
+ if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){
+ break;
+ }else{
+ sqlite3_file *pTmp = pSorter->pTemp1;
+ pSorter->nPMA = iNew;
+ pSorter->pTemp1 = pTemp2;
+ pTemp2 = pTmp;
+ pSorter->iWriteOff = iWrite2;
+ pSorter->iReadOff = 0;
+ iWrite2 = 0;
+ }
+ }while( rc==SQLITE_OK );
+
+ if( pTemp2 ){
+ sqlite3OsCloseFree(pTemp2);
+ }
+ *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0);
+ return rc;
+}
+
+/*
+** Advance to the next element in the sorter.
+*/
+int sqlite3VdbeSorterNext(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){
+ VdbeSorter *pSorter = pCsr->pSorter;
+ int iPrev = pSorter->aTree[1]; /* Index of iterator to advance */
+ int i; /* Index of aTree[] to recalculate */
+ int rc; /* Return code */
+
+ rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]);
+ for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){
+ rc = vdbeSorterDoCompare(pCsr, i);
+ }
+
+ *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0);
+ return rc;
+}
+
+/*
+** Copy the current sorter key into the memory cell pOut.
+*/
+int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){
+ VdbeSorter *pSorter = pCsr->pSorter;
+ VdbeSorterIter *pIter;
+
+ pIter = &pSorter->aIter[ pSorter->aTree[1] ];
+
+ /* Coverage testing note: As things are currently, this call will always
+ ** succeed. This is because the memory cell passed by the VDBE layer
+ ** happens to be the same one as was used to assemble the keys before they
+ ** were passed to the sorter - meaning it is always large enough for the
+ ** largest key. But this could change very easily, so we leave the call
+ ** to sqlite3VdbeMemGrow() in. */
+ if( sqlite3VdbeMemGrow(pOut, pIter->nKey, 0) ){
+ return SQLITE_NOMEM;
+ }
+ pOut->n = pIter->nKey;
+ MemSetTypeFlag(pOut, MEM_Blob);
+ memcpy(pOut->z, pIter->aKey, pIter->nKey);
+
+ return SQLITE_OK;
+}
+
+#endif /* #ifndef SQLITE_OMIT_MERGE_SORT */
projects / thirdparty / sqlite.git / commitdiff
