Add the "sorter-reference" optimization, allowing SQLite to be configured so

that some required values may be loaded from the database after external
sorting occurs for SELECT statements with ORDER BY clauses that are not
satisfied by database indexes.

FossilOrigin-Name: ef74090a40ceaef2fd93a7613ec99a191ce986811c852e96f4a19719f18af4f0

diff --cc manifest
index f791b128ae0643e04128245a266576af3ab74c53,b1c72a522d3ad7817e19a6e6089f5304147efa8e..07aaffea965d2815fff872740ae7bf2edeb536fd
--- 1/manifest
--- 2/manifest
+++ b/manifest
@@@ -1,12 -1,12 +1,12 @@@
- C Minor\schanges\sto\stest\sscript\supsert4.test.
- D 2018-04-18T19:45:14.101
 -C Avoid\sa\sNULL-pointer\sderef\sfollowing\sOOM.
 -D 2018-04-18T19:08:44.210
++C Add\sthe\s"sorter-reference"\soptimization,\sallowing\sSQLite\sto\sbe\sconfigured\sso\nthat\ssome\srequired\svalues\smay\sbe\sloaded\sfrom\sthe\sdatabase\safter\sexternal\nsorting\soccurs\sfor\sSELECT\sstatements\swith\sORDER\sBY\sclauses\sthat\sare\snot\nsatisfied\sby\sdatabase\sindexes.
++D 2018-04-18T19:56:14.234
  F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
  F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 -F Makefile.in 7016fc56c6b9bfe5daac4f34be8be38d8c0b5fab79ccbfb764d3b23bf1c6fff3
 +F Makefile.in 5ce9343cba9c189046f1afe6d2bcc1f68079439febc05267b98aec6ecc752439
  F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 -F Makefile.msc 43dd6ae2e5a2bb8df7bfc9ed85935129caffeafb8c1803d24c5d038b1e74c8ca
 +F Makefile.msc 59179295f6a9b433e3f59a6dc2fcf6db6fcac35d92015294beb5d27f2924ebb9
  F README.md 7764d56778d567913ef11c82da9ab94aefa0826f7c243351e4e2d7adaef6f373
 -F VERSION 7169eb6959db9ad1b7004ae3b754ef6e703eb7d8dde3b07d2e63103413eb25fb
 +F VERSION b7c9d1d11cb70ef8e90cfcf3c944aa58a9f801cc2ad487eebb0a110c16dfc2df
  F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
  F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
  F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
@@@ -443,7 -443,7 +443,7 @@@ F src/date.c ebe1dc7c8a347117bb02570f1a
  F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
  F src/dbstat.c edabb82611143727511a45ca0859b8cd037851ebe756ae3db289859dd18b6f91
  F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
- F src/expr.c 7c08754d68897488cf4d916d15680f7d8ef6ea3c7fd32a9c7d739b0b4dda9cf7
 -F src/expr.c 2448a255ce627c4e772bd68cf5529877c2bdfb6b580803d5fadc8528bdf7c1ef
++F src/expr.c 53df437b3a4a404f039645919b2d6a56a2f59b9883e858940cd2a8858a25cd3d
  F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
  F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
  F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51
@@@ -452,10 -452,10 +452,10 @@@ F src/hash.c a12580e143f10301ed5166ea49
  F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
  F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
  F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 -F src/insert.c 752740e4619416d4262f6e9e51cdb6af5965eb0c8e943832a5af77d41e2839c7
 +F src/insert.c 5fa74146492f5da33e42c35f5f58fb8d56e047c42746fdbd52c8ebdb21160e27
  F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
  F src/loadext.c f6e4e416a736369f9e80eba609f0acda97148a8b0453784d670c78d3eed2f302
- F src/main.c 1648fc7a9bcfdbfd9a9a04af96ff2796c3164b3f3c7e56ed63a3c51cd11d198d
+ F src/main.c 10e3897f5d78cef6bcbd1eedc8ccc3fe9e9783d07e052d9d70e57364ded19274
  F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18
  F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
  F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@@ -489,14 -489,14 +489,14 @@@ F src/pragma.h bb83728944b42f6d409c77f5
  F src/prepare.c b086fea6a1952db88beca31fdd621201ee5e4ce3f02905248cc3035a8174aa89
  F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660
  F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 -F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6
 +F src/resolve.c 6415381a0e9d22c0e7cba33ca4a53f81474190862f5d4838190f5eb5b0b47bc9
  F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
- F src/select.c 3e84cb869930aa8fcacd3acbb1a0ec2d82736c8479d6a4367a5f1a926fb8a763
- F src/shell.c.in bcde676be8aef998449e4fc8076c671257f41cbfe0cf39c87464b4558e6abf9a
- F src/sqlite.h.in e0be726ea6e4e6571724d39d242472ecd8bd1ba6f84ade88e1641bde98a6d02b
 -F src/select.c 9257a7f26e6c9da27deae819098d1005f4e1c37e19b8f029292bd2b1aab35721
 -F src/shell.c.in 6c9e2c1136f3697eb75f5dce010e7af005f62b4e1fda6d1066c3a473eb922889
++F src/select.c 04ae9b4a30992bf428ec27027972ea9e8b31c59e4cbead8adebda1512be2e450
++F src/shell.c.in 8ab4687da814ddc4adf6ea0fcd43ea1eb2784ee6915674dd690759241b7a24b3
+ F src/sqlite.h.in aa9bd3ae4a077c7002059cb418271abe52214b0227b2a734bc44736b24cbcc40
  F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
  F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
- F src/sqliteInt.h 1d4317da12c4a555edf1630c02336708ae57bd6acc6effec73bd6ceca6c50afc
 -F src/sqliteInt.h ad14bfeab6c1ada3aa1181c6cc14a3e6a8f24b35ad96a2daa3d11100a5267236
++F src/sqliteInt.h a5c534cda7ec61f7180eee99804a63ee2e4a6412a9da6c4e2a77dee7f8caafb5
  F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
  F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
  F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@@ -1584,7 -1580,7 +1585,7 @@@ F test/where4.test 4a371bfcc607f41d2337
  F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2
  F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b
  F test/where7.test f520bcec2c3d12dc4615623b06b2aec7c2d67e94
--F test/where8.test 98eedca0d375fb400b8377269c4b4686582dfb45
++F test/where8.test 461ca40265ed996a6305da99bb024b0e41602bb586acf544c08f95922358e49f
  F test/where9.test 729c3ba9b47e8f9f1aab96bae7dad2a524f1d1a2
  F test/whereA.test 6c6a420ca7d313242f9b1bd471dc80e4d0f8323700ba9c78df0bb843d4daa3b4
  F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5
@@@ -1722,7 -1718,7 +1723,8 @@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9
  F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
  F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
  F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
- P 61cb8a391a0c709340ac60ffd0c58f950567892a8404c2bec7b9b1f64b3cc5cf
- R 6c33138f3c02d47a81dcde27f15e792f
 -P 902a40897f74ac8a3bc72ef84c2161ab308b5601381cc9eea18147bfefa978ce
 -R 4bf2a8a4f284f188f9cc406a1fead38f
 -U drh
 -Z 857937520ddb8fedd9d522f195cdf4dc
++P 0cb83c84d10b89ef7a5504862566a609307c63e7571dd711d8b9f995d29e5a8c 413015c029d850d4ce7e66be1f59b57f291254240a958856378a62f5ac4a5092
++R 4678faec743cd6b53d517fb16a4da20a
++T +closed 413015c029d850d4ce7e66be1f59b57f291254240a958856378a62f5ac4a5092
 +U dan
- Z 1740208f0524dae2e22fe373b2efc4e7
++Z 97f82718fcc75c5ef29e50fdc72006e4

diff --cc manifest.uuid
index b4c6aa04e4145f2f7aaf8ec2de6d4d9db8078366,8036b78b15f1d08e10cdd0586503129b6bf9e059..e1de99ec007b2816b54a1b766a28834b19cddc34
--- 1/manifest.uuid
--- 2/manifest.uuid
+++ b/manifest.uuid
@@@ -1,1 -1,1 +1,1 @@@
- 0cb83c84d10b89ef7a5504862566a609307c63e7571dd711d8b9f995d29e5a8c
 -413015c029d850d4ce7e66be1f59b57f291254240a958856378a62f5ac4a5092
++ef74090a40ceaef2fd93a7613ec99a191ce986811c852e96f4a19719f18af4f0

diff --cc src/expr.c
Simple merge

diff --cc src/select.c
Simple merge

diff --cc src/shell.c.in
index 87087ed45f9a6d5a16046a77b014870ca2788d57,9750cf28037132b9353d4c66cb3dce581f45ae4d..38e5c5cbe267cd60de1d4ac439989a98723ffd06
--- 1/src/shell.c.in
--- 2/src/shell.c.in
+++ b/src/shell.c.in
@@@ -8343,8 -8334,19 +8346,13 @@@ int SQLITE_CDECL wmain(int argc, wchar_
      }else if( strcmp(z,"-mmap")==0 ){
        sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i));
        sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz);
+ #ifdef SQLITE_ENABLE_SORTER_REFERENCES
+     }else if( strcmp(z,"-sorterref")==0 ){
+       sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i));
+       sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz);
+ #endif
      }else if( strcmp(z,"-vfs")==0 ){
 -      sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i));
 -      if( pVfs ){
 -        sqlite3_vfs_register(pVfs, 1);
 -      }else{
 -        utf8_printf(stderr, "no such VFS: \"%s\"\n", argv[i]);
 -        exit(1);
 -      }
 +      zVfs = cmdline_option_value(argc, argv, ++i);
  #ifdef SQLITE_HAVE_ZLIB
      }else if( strcmp(z,"-zip")==0 ){
        data.openMode = SHELL_OPEN_ZIPFILE;

diff --cc src/sqliteInt.h
Simple merge

diff --cc test/where8.test
index 38214bc895d3877989e6d64e2192e96f859be9c5,38214bc895d3877989e6d64e2192e96f859be9c5..a8dbcfd9fed705fd5d2751c709346e109183820c
--- 1/test/where8.test
--- 2/test/where8.test
+++ b/test/where8.test
@@@ -16,6 -16,6 +16,11 @@@
  set testdir [file dirname $argv0]
  source $testdir/tester.tcl
  
++if {[permutation]=="sorterref"} {
++  finish_test
++  return
++}
++
  # Test organization:
  #
  #   where8-1.*: Tests to demonstrate simple cases work with a single table