From: dan Date: Sat, 3 May 2014 14:28:14 +0000 (+0000) Subject: Fix a problem in the sorter causing it to return spurious SQLITE_NOMEM errors when... X-Git-Tag: version-3.8.7~132^2~55 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=0d51def29e2734627d8ea4009dcfbbf837de30b7;p=thirdparty%2Fsqlite.git Fix a problem in the sorter causing it to return spurious SQLITE_NOMEM errors when configured to use memsys3 or memsys5. FossilOrigin-Name: 3a66c4e1bf311d38668dfcdcd77867feff6db7bd --- diff --git a/manifest b/manifest index a2cac08c7a..53d89eafaf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sSQLITE_MUTEX_STATIC_APP1\sand\s_APP2\sworking\sfor\sthe\sdebugMutex\nimplementation. -D 2014-05-03T13:53:37.085 +C Fix\sa\sproblem\sin\sthe\ssorter\scausing\sit\sto\sreturn\sspurious\sSQLITE_NOMEM\serrors\swhen\sconfigured\sto\suse\smemsys3\sor\smemsys5. +D 2014-05-03T14:28:14.676 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd -F src/test1.c 0cd73ae82fdf7add76ca603e3575380ae7539ae2 +F src/test1.c bd88cc00bff2f15279d808e84501f06148c144f9 F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -241,7 +241,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c ebd0a42983b696f2b121515d577753cf2afdc9b0 +F src/test_config.c bf2e0bf49ebd8fe3dccabb4542157f9571fe48fa F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c d205b56d0a1c2cbd8f6c8c4f513337ab0096d0b3 +F src/vdbesort.c 3e8827bb9d12465556357c24641f8805a7e2bba0 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -741,7 +741,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test a214a42b4767bbbc7cd0fd965ea6198044ab414d +F test/permutations.test 46a18489379943fcbd2ef07faa3858a88adb30cb F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -821,10 +821,10 @@ F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a -F test/sort2.test 04e99d0d028b469c6cfab2c647c6c28755504063 -F test/sort3.test c3f88d233452a129de519de311d109a0ad0da0af -F test/sortfault.test 2e2337aa5db6ab5cd546368cf2410676c11cb577 +F test/sort.test 8330b31b160483b52bb502a3ac4013f3f9028d73 +F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 +F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 +F test/sortfault.test 7fdc4a9bd76280a659c5782cdc6d95806d62d512 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 13686035dd1cf67ad9c6d282ab13c3259e7273d1 -R 7c657c53542f045bb02fc7656a8fcff4 -U drh -Z af16da93d438f6495f6a372d4f94c5d4 +P f49ba1c926c63ee1c4609930138389fca182c845 +R ecf8f8bb6befc88e6cfdbe14a65100aa +U dan +Z 4d5ad6cf671998327bccef4dcd098843 diff --git a/manifest.uuid b/manifest.uuid index 1fae5ede0f..500fa40c2b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f49ba1c926c63ee1c4609930138389fca182c845 \ No newline at end of file +3a66c4e1bf311d38668dfcdcd77867feff6db7bd \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 44e96c2c1f..403a7fafa7 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6336,6 +6336,40 @@ static int tclLoadStaticExtensionCmd( return TCL_OK; } +/* +** sorter_test_fakeheap BOOL +** +*/ +static int sorter_test_fakeheap( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int bArg; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "BOOL"); + return TCL_ERROR; + } + + if( Tcl_GetBooleanFromObj(interp, objv[1], &bArg) ){ + return TCL_ERROR; + } + + if( bArg ){ + if( sqlite3GlobalConfig.pHeap==0 ){ + sqlite3GlobalConfig.pHeap = SQLITE_INT_TO_PTR(-1); + } + }else{ + if( sqlite3GlobalConfig.pHeap==SQLITE_INT_TO_PTR(-1) ){ + sqlite3GlobalConfig.pHeap = 0; + } + } + + Tcl_ResetResult(interp); + return TCL_OK; +} + /* ** Register commands with the TCL interpreter. @@ -6569,6 +6603,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "getrusage", test_getrusage }, #endif { "load_static_extension", tclLoadStaticExtensionCmd }, + { "sorter_test_fakeheap", sorter_test_fakeheap }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; diff --git a/src/test_config.c b/src/test_config.c index 2201fba007..33898a759c 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -99,11 +99,9 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); #endif -#if SQLITE_MAX_WORKER_THREADS>0 - Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "1", TCL_GLOBAL_ONLY); -#else - Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "0", TCL_GLOBAL_ONLY); -#endif + Tcl_SetVar2(interp, "sqlite_options", "worker_threads", + STRINGVALUE(SQLITE_MAX_WORKER_THREADS), TCL_GLOBAL_ONLY + ); #if 1 /* def SQLITE_MEMDEBUG */ Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY); diff --git a/src/vdbesort.c b/src/vdbesort.c index 413e85c3b5..5efd6f112c 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1556,7 +1556,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ if( aMem ){ pSorter->list.aMemory = aMem; pSorter->nMemory = sqlite3MallocSize(aMem); - }else{ + }else if( pSorter->list.aMemory ){ pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory); if( !pSorter->list.aMemory ) return SQLITE_NOMEM; } diff --git a/test/permutations.test b/test/permutations.test index 4487af055b..b547e7b1bc 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -332,6 +332,12 @@ test_suite "coverage-analyze" -description { analyze.test analyzeB.test mallocA.test } +test_suite "coverage-sorter" -description { + Coverage tests for file vdbesort.c. +} -files { + sort.test sortfault.test +} + lappend ::testsuitelist xxx #------------------------------------------------------------------------- diff --git a/test/sort.test b/test/sort.test index ccbfdda2b3..c6c7fc6e41 100644 --- a/test/sort.test +++ b/test/sort.test @@ -8,10 +8,10 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# # This file implements regression tests for SQLite library. The -# focus of this file is testing the CREATE TABLE statement. +# focus of this file is testing the sorter (code in vdbesort.c). # -# $Id: sort.test,v 1.25 2005/11/14 22:29:06 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -485,5 +485,125 @@ do_execsql_test sort-13.3 { SELECT a, b FROM t10 ORDER BY a; } [db eval {SELECT a, b FROM t10 ORDER BY a, b}] +#------------------------------------------------------------------------- +# Sort some large ( > 4KiB) records. +# +proc cksum {x} { + set i1 1 + set i2 2 + binary scan $x c* L + foreach {a b} $L { + set i1 [expr (($i2<<3) + $a) & 0x7FFFFFFF] + set i2 [expr (($i1<<3) + $b) & 0x7FFFFFFF] + } + list $i1 $i2 +} +db func cksum cksum + +do_execsql_test sort-14.0 { + PRAGMA cache_size = 5; + CREATE TABLE t11(a, b); + INSERT INTO t11 VALUES(randomblob(5000), NULL); + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --2 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --3 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --4 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --5 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --6 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --7 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --8 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --9 + UPDATE t11 SET b = cksum(a); +} + +foreach {tn mmap_limit} { + 1 0 + 2 1000000 +} { + do_test sort-14.$tn { + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit + set prev "" + db eval { SELECT * FROM t11 ORDER BY b } { + if {$b != [cksum $a]} {error "checksum failed"} + if {[string compare $b $prev] < 0} {error "sort failed"} + set prev $b + } + set {} {} + } {} +} + +#------------------------------------------------------------------------- +# +foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { + 1 0 3 file true false + 2 0 3 file true true + 3 0 0 file true false + 4 1000000 3 file true false + 5 0 0 memory false true +} { + db close + + sqlite3_shutdown + sqlite3_config_worker_threads $nWorker + if {$coremutex} { + sqlite3_config multithread + } else { + sqlite3_config singlethread + } + sqlite3_initialize + + sorter_test_fakeheap $fakeheap + + reset_db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit + execsql "PRAGMA temp_store = $tmpstore" + + set ten [string repeat X 10300] + set one [string repeat y 200] + + do_execsql_test 15.$tn.1 { + PRAGMA cache_size = 5; + WITH rr AS ( + SELECT 4, $ten UNION ALL + SELECT 2, $one UNION ALL + SELECT 1, $ten UNION ALL + SELECT 3, $one + ) + SELECT * FROM rr ORDER BY 1; + } [list 1 $ten 2 $one 3 $one 4 $ten] + + do_execsql_test 15.$tn.2 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(4); + INSERT INTO t1 VALUES(5); + INSERT INTO t1 VALUES(3); + INSERT INTO t1 VALUES(2); + INSERT INTO t1 VALUES(6); + INSERT INTO t1 VALUES(1); + CREATE INDEX i1 ON t1(a); + SELECT * FROM t1 ORDER BY a; + } {1 2 3 4 5 6} + + do_execsql_test 15.$tn.3 { + PRAGMA cache_size = 5; + WITH rr AS ( + SELECT 4, $ten UNION ALL + SELECT 2, $one + ) + SELECT * FROM rr ORDER BY 1; + } [list 2 $one 4 $ten] + + sorter_test_fakeheap 0 +} + +db close +sqlite3_shutdown +#sqlite3_config_worker_threads $sqlite_options(worker_threads) +sqlite3_config_worker_threads 0 +set t(0) singlethread +set t(1) multithread +set t(2) serialized +sqlite3_config $t($sqlite_options(threadsafe)) +sqlite3_initialize finish_test + diff --git a/test/sort2.test b/test/sort2.test index 4fb6a9462b..e4e40dab74 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -10,6 +10,9 @@ #*********************************************************************** # This file implements regression tests for SQLite library. # +# Specifically, the tests in this file attempt to verify that +# multi-threaded sorting works. +# set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/sort3.test b/test/sort3.test index 9963dcccd3..80d8bbca3f 100644 --- a/test/sort3.test +++ b/test/sort3.test @@ -10,6 +10,10 @@ #*********************************************************************** # This file implements regression tests for SQLite library. # +# The tests in this file verify that sorting works when the library is +# configured to use mmap(), but the temporary files generated by the +# sorter are too large to be completely mapped. +# set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/sortfault.test b/test/sortfault.test index bdf57b45ba..abe9af6854 100644 --- a/test/sortfault.test +++ b/test/sortfault.test @@ -10,6 +10,9 @@ #*********************************************************************** # This file implements regression tests for SQLite library. # +# Specifically, it tests the effects of fault injection on the sorter +# module (code in vdbesort.c). +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -20,20 +23,26 @@ do_execsql_test 1.0 { PRAGMA cache_size = 5; } -do_faultsim_test 1 -prep { - sqlite3 db test.db -} -body { - execsql { - WITH r(x,y) AS ( - SELECT 1, randomblob(1000) - UNION ALL - SELECT x+1, randomblob(1000) FROM r - LIMIT 500 - ) - SELECT count(x), length(y) FROM r GROUP BY (x%5) - } -} -test { - faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}} +foreach {tn mmap_limit} { + 1 0 + 2 100000 +} { + do_faultsim_test 1.$tn -prep { + sqlite3 db test.db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit + } -body { + execsql { + WITH r(x,y) AS ( + SELECT 1, randomblob(1000) + UNION ALL + SELECT x+1, randomblob(1000) FROM r + LIMIT 500 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } + } -test { + faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}} + } } finish_test