From: dan Date: Thu, 14 Jul 2022 18:09:56 +0000 (+0000) Subject: Update testrunner.tcl so that it can run the test suites defined in permutation.test. X-Git-Tag: version-3.40.0~302 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ca0720a9d80b232b3b52481c0ae081e78daf4133;p=thirdparty%2Fsqlite.git Update testrunner.tcl so that it can run the test suites defined in permutation.test. FossilOrigin-Name: 15ce937ef42491c503cb91f5bb3ce4dc5cdb3a927ff2f2b873fb6bf96808f3aa --- diff --git a/manifest b/manifest index a3df518eea..6074bc96bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sapplying\sthe\somit-ORDER-BY\soptimization,\sdefer\sdeleting\sthe\sAST\sof\nthe\sdeleted\sORDER\sBY\sclause\suntil\safter\scode\sgeneration\sends. -D 2022-07-14T01:48:27.069 +C Update\stestrunner.tcl\sso\sthat\sit\scan\srun\sthe\stest\ssuites\sdefined\sin\spermutation.test. +D 2022-07-14T18:09:56.687 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1507,8 +1507,7 @@ F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 76771269dcc20b2c2d1d6f1175dd50d1eebddc004aebac865483f1829a5cd398 -F test/testrunner.tcl da340a87fd8b5c94e0d2196b086b96a90de0ce16bfdb71c1254d543bcee1614a -F test/testset.tcl 27a6bbbc93cbbcf442c57e3c023e6b5d7304dc415e09eb0e9ac61edd0e6c1fbe +F test/testrunner.tcl 711ccfb6309e63877b20910f76e8b5c8c9e45176cce0efd6f8a40b4e4d44d0e9 F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1981,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 14918f28221a3124b78a490fbb483279551ccc5a0032ea854ff0ac365684cc60 -R d2c6af8b43cf81f08214974216426964 -U drh -Z 663440ae7ea06bca4d8f20ca5b3783d1 +P f22f95b838873f1d2a320afe3d0f4e4847948fcd343097b93a9f684a6f66d6ba +R f0a96981c3415711d35db8d014ac10fa +U dan +Z 26ecdc9b9c72535e31216f99411f29ef # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4924411865..57333817f3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f22f95b838873f1d2a320afe3d0f4e4847948fcd343097b93a9f684a6f66d6ba \ No newline at end of file +15ce937ef42491c503cb91f5bb3ce4dc5cdb3a927ff2f2b873fb6bf96808f3aa \ No newline at end of file diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 568c599136..51cb9bfa25 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1,21 +1,47 @@ - #------------------------------------------------------------------------- # Usage: # proc usage {} { set a0 testrunner.tcl - puts stderr "Usage: $a0 ?SWITCHES? ?all|veryquick? ?PATTERNS?" + set ::argv [list] + uplevel [list source $::testdir/permutations.test] + + puts stderr "Usage: $a0 ?SWITCHES? ?PERMUTATION? ?PATTERNS?" puts stderr "" puts stderr "where SWITCHES are:" puts stderr " --jobs NUMBER-OF-JOBS" puts stderr "" + puts stderr "available PERMUTATION values are:" + set ii 0 + foreach name [lsort [array names ::testspec]] { + if {($ii % 3)==0} { puts -nonewline stderr " " } + puts -nonewline stderr [format "% -22s" $name] + if {($ii % 3)==2} { puts stderr "" } + incr ii + } + puts stderr "" + puts stderr "" puts stderr "Examples:" - puts stderr " $a0 # Run veryquick.test tests" - puts stderr " $a0 all # Run all tests" - puts stderr " $a0 veryquick rtree% # Run all test scripts from veryquick.test that match 'rtree%'" - puts stderr " $a0 alter% fts5% # Run all test scripts that match 'alter%' or 'rtree%'" + puts stderr " 1) Run the veryquick tests:" + puts stderr " $a0" + puts stderr " 2) Run all test scripts in the source tree:" + puts stderr " $a0 full" + puts stderr " 2) Run the 'memsubsys1' permutation:" + puts stderr " $a0 memsubsys1" + puts stderr " 3) Run all permutations usually run by \[make fulltest\]" + puts stderr " $a0 release" + puts stderr " 4) Run all scripts that match the pattern 'select%':" + puts stderr " $a0 select%" + puts stderr " $a0 all select%" + puts stderr " $a0 full select%" + puts stderr " 5) Run all scripts that are part of the veryquick permutation and match the pattern 'select%':" + puts stderr " $a0 veryquick select%" + puts stderr " 6) Run the 'memsubsys1' permutation, but just those scripts that match 'window%':" + puts stderr " $a0 memsubsys1 window%" + puts stderr " 7) Run all the permutations, but only the scripts that match either 'fts5%' or 'rtree%':" + puts stderr " $a0 release fts5% rtree%" exit 1 } @@ -30,13 +56,15 @@ set R(schema) { DROP TABLE IF EXISTS malloc; CREATE TABLE script( - filename TEXT PRIMARY KEY, -- full path to test script + config TEXT, + filename TEXT, -- full path to test script state TEXT CHECK( state IN ('ready', 'running', 'done') ), testfixtureid, -- Id of process that ran script time INTEGER, -- Time in ms nerr INTEGER, -- if 'done', the number of errors ntest INTEGER, -- if 'done', the number of tests - output TEXT -- full output of test script + output TEXT, -- full output of test script + PRIMARY KEY(config, filename) ); CREATE TABLE malloc( @@ -95,7 +123,6 @@ set R(leaker) "" ;# Name of first script to leak memory set R(patternlist) [list] set testdir [file dirname $argv0] -source $testdir/testset.tcl # Parse the command line options. There are two ways to invoke this # script - to create a helper or coordinator process. If there are @@ -140,15 +167,86 @@ if {$R(helper)==0} { usage } } else { - lappend R(patternlist) [string map {% * _ .} $a] + lappend R(patternlist) [string map {% *} $a] } } set argv [list] } +source $testdir/permutations.test + +#------------------------------------------------------------------------- +# Return a list of tests to run. Each element of the list is itself a +# list of two elements - the name of a permuations.test configuration +# followed by the full path to a test script. i.e.: +# +# {CONFIG FILENAME} {CONFIG FILENAME} ... +# +proc testset_patternlist {patternlist} { + + set first [lindex $patternlist 0] + if {$first=="all"} { set first "full" } + + if {$first=="release"} { + + # The following mirrors the set of test suites invoked by "all.test". + # + set clist { + full + no_optimization memsubsys1 memsubsys2 singlethread + multithread onefile utf16 exclusive persistent_journal + persistent_journal_error no_journal no_journal_error + autovacuum_ioerr no_mutex_try fullmutex journaltest + inmemory_journal pcache0 pcache10 pcache50 pcache90 + pcache100 prepare mmap + } + ifcapable rbu { lappend clist rbu } + if {$::tcl_platform(platform)=="unix"} { + ifcapable !default_autovacuum { + lappend clist autovacuum_crash + } + } + set patternlist [lrange $patternlist 1 end] + + } elseif {[info exists ::testspec($first)]} { + set clist $first + set patternlist [lrange $patternlist 1 end] + } elseif { [llength $patternlist]==0 } { + set clist veryquick + } else { + set clist full + } + + set testset [list] + + foreach config $clist { + catch { array unset O } + array set O $::testspec($config) + foreach f $O(-files) { + if {[file pathtype $f]!="absolute"} { + set f [file join $::testdir $f] + } + lappend testset [list $config [file normalize $f]] + } + } + + if {[llength $patternlist]>0} { + foreach t $testset { + set tail [file tail [lindex $t 1]] + foreach p $patternlist { + if {[string match $p $tail]} { + lappend ret $t + break; + } + } + } + } else { + set ret $testset + } -source $testdir/tester.tcl -db close + set ret +} +#-------------------------------------------------------------------------- proc r_write_db {tcl} { @@ -166,37 +264,63 @@ proc r_write_db {tcl} { proc make_new_testset {} { global R - set scripts [testset_patternlist $R(patternlist)] + set tests [testset_patternlist $R(patternlist)] r_write_db { db eval $R(schema) - foreach s $scripts { - db eval { INSERT INTO script(filename, state) VALUES ($s, 'ready') } + foreach t $tests { + foreach {c s} $t {} + db eval { + INSERT INTO script(config, filename, state) VALUES ($c, $s, 'ready') + } } } } +# Find the next job in the database and mark it as 'running'. Then return +# a list consisting of the +# +# CONFIG FILENAME +# +# pair for the test. +# proc get_next_test {} { global R set myid $R(helper_id) r_write_db { - set f [db one { - SELECT filename FROM script WHERE state='ready' ORDER BY 1 LIMIT 1 - }] + set f "" + set c "" + db eval { + SELECT config, filename FROM script WHERE state='ready' + ORDER BY config!='full', config, filename LIMIT 1 + } { + set c $config + set f $filename + } if {$f!=""} { db eval { - UPDATE script SET state='running', testfixtureid=$myid WHERE filename=$f + UPDATE script SET state='running', testfixtureid=$myid + WHERE (config, filename) = ($c, $f) } } } - return $f + if {$f==""} { return "" } + list $c $f +} + +proc r_testname {config filename} { + set name [file tail $filename] + if {$config!="" && $config!="full" && $config!="veryquick"} { + set name "$config-$name" + } + return $name } -proc r_set_test_result {filename ms nerr ntest output} { +proc r_set_test_result {config filename ms nerr ntest output} { global R - set f [file tail $filename] + set f [r_testname $config $filename] if {$nerr==0} { set msg "$f... Ok" } else { @@ -211,14 +335,14 @@ proc r_set_test_result {filename ms nerr ntest output} { set nMalloc [lindex [sqlite3_status SQLITE_STATUS_MALLOC_COUNT 0] 1] set nByte [sqlite3_memory_used] if {($nByte>0 || $nMalloc>0) && $R(leaker)==""} { - set R(leaker) $filename + set R(leaker) $f } r_write_db { db eval { UPDATE script SET state='done', output=$output, nerr=$nerr, ntest=$ntest, time=$ms - WHERE filename=$filename; + WHERE (config, filename)=($config, $filename); INSERT INTO msg(msg) VALUES ($msg); } @@ -408,7 +532,7 @@ proc r_final_report {} { # Create the text log file. This is just the concatenation of the # 'output' column of the database for every script that was run. set fd [open $R(logname) w] - db eval {SELECT output FROM script ORDER BY filename} { + db eval {SELECT output FROM script ORDER BY config!='full',config,filename} { puts $fd $output } close $fd @@ -421,8 +545,8 @@ proc r_final_report {} { db eval { SELECT sum(nerr) AS nerr, sum(ntest) AS ntest FROM script } { } puts "$nerr errors from $ntest tests." if {$nerr>0} { - db eval { SELECT filename FROM script WHERE nerr>0 } { - lappend errlist [file tail $filename] + db eval { SELECT config, filename FROM script WHERE nerr>0 } { + lappend errlist [r_testname $config $filename] } puts "Errors in: $errlist" set errcode 1 @@ -431,8 +555,8 @@ proc r_final_report {} { # Check if any scripts were not run or did not finish. Print out a # line identifying them if there are any. set errlist [list] - db eval { SELECT filename FROM script WHERE state!='done' } { - lappend errlist [file tail $filename] + db eval { SELECT config, filename FROM script WHERE state!='done' } { + lappend errlist [r_testname $config $filename] } if {$errlist!=[list]} { puts "Tests DID NOT FINISH (crashed?): $errlist" @@ -498,13 +622,31 @@ proc r_helper_readable {id chan} { } if {$R(nHelperRunning)==0} { - while { ""!=[set f [get_next_test]] } { + while { ""!=[set t [get_next_test]] } { set R(output) "" set TC(count) 0 set TC(errors) 0 - set ms [slave_test_file $f] - r_set_test_result $f $ms $TC(errors) $TC(count) $R(output) + foreach {config filename} $t {} + + array set O $::testspec($config) + set ::G(perm:name) $config + set ::G(perm:prefix) $O(-prefix) + set ::G(isquick) 1 + set ::G(perm:dbconfig) $O(-dbconfig) + set ::G(perm:presql) $O(-presql) + + eval $O(-initialize) + set ms [slave_test_file $filename] + eval $O(-shutdown) + + unset -nocomplain ::G(perm:sqlite3_args) + unset ::G(perm:name) + unset ::G(perm:prefix) + unset ::G(perm:dbconfig) + unset ::G(perm:presql) + + r_set_test_result $config $filename $ms $TC(errors) $TC(count) $R(output) if {$R(helper)==0} { foreach msg [r_get_messages] { puts $msg } diff --git a/test/testset.tcl b/test/testset.tcl deleted file mode 100644 index 8f761d686a..0000000000 --- a/test/testset.tcl +++ /dev/null @@ -1,121 +0,0 @@ - - -# Commands in this file: -# -# testset_all -# Return a list of all test scripts designed to be run individually. -# -# testset_veryquick -# The subset of [testset_all] meant to run as veryquick.test. -# - -set D(testdir) [file dir [file normalize [info script]]] - -proc testset_all {} { - global D - set ret [list] - - # The following tests are driver scripts that themselves run lots of other - # test scripts. They should be ignored here. - set drivers { - all.test async.test quick.test veryquick.test - memleak.test permutations.test soak.test fts3.test - mallocAll.test rtree.test full.test extraquick.test - session.test rbu.test - } - - set srcdir [file dirname $D(testdir)] - set ret [glob -nocomplain \ - $srcdir/test/*.test \ - $srcdir/ext/rtree/*.test \ - $srcdir/ext/fts5/test/*.test \ - $srcdir/ext/expert/*.test \ - $srcdir/ext/session/*.test \ - ] - set ret [ts_filter $ret $drivers] - return $ret -} - -proc testset_veryquick {} { - set ret [testset_all] - - set ret [ts_filter $ret { - async2.test async3.test backup_ioerr.test corrupt.test - corruptC.test crash.test crash2.test crash3.test crash4.test crash5.test - crash6.test crash7.test delete3.test e_fts3.test fts3rnd.test - fkey_malloc.test fuzz.test fuzz3.test fuzz_malloc.test in2.test loadext.test - misc7.test mutex2.test notify2.test onefile.test pagerfault2.test - savepoint4.test savepoint6.test select9.test - speed1.test speed1p.test speed2.test speed3.test speed4.test - speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test - thread003.test thread004.test thread005.test trans2.test vacuum3.test - incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test - vtab_err.test walslow.test walcrash.test walcrash3.test - walthread.test rtree3.test indexfault.test securedel2.test - sort3.test sort4.test fts4growth.test fts4growth2.test - bigsort.test walprotocol.test mmap4.test fuzzer2.test - walcrash2.test e_fkey.test backup.test - - fts4merge.test fts4merge2.test fts4merge4.test fts4check.test - fts4merge5.test - fts3cov.test fts3snippet.test fts3corrupt2.test fts3an.test - fts3defer.test fts4langid.test fts3sort.test fts5unicode.test - - rtree4.test - sessionbig.test - }] - - set ret [ts_filter $ret { - *malloc* *ioerr* *fault* *bigfile* *_err* *fts5corrupt* *fts5big* *fts5aj* - }] - - return $ret -} - -proc ts_filter {input exlist} { - foreach f $input { set a($f) 1 } - foreach e $exlist { array unset a */$e } - array names a -} - -proc testset_patternlist {patternlist} { - set nPat [llength $patternlist] - - if {$nPat==0} { - set scripts [testset_veryquick] - } else { - set ii 0 - set p0 [lindex $patternlist 0] - - if {$p0=="veryquick"} { - set scripts [testset_veryquick] - incr ii - } elseif {$p0=="all"} { - set scripts [testset_all] - incr ii - } else { - set scripts [testset_all] - } - - if {$nPat>$ii} { - array set S [list] - foreach f $scripts { set a([file tail $f]) $f } - - foreach p [lrange $patternlist $ii end] { - set nList [llength [array names a $p]] - if {$nList==0} { - puts stderr "Argument $p matches no scripts (typo?)" - exit 1 - } - foreach n [array names a $p] { set S($a($n)) 1 } - } - - set scripts [lsort [array names S]] - } - - } - - set scripts -} - -