From: dan Date: Tue, 12 Jul 2022 20:31:16 +0000 (+0000) Subject: Add test/testrunner.tcl, an experimental script for distributing the work of veryquic... X-Git-Tag: version-3.40.0~306^2~3 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=0505851451d7495feecc8f0386c80c3e7863c03e;p=thirdparty%2Fsqlite.git Add test/testrunner.tcl, an experimental script for distributing the work of veryquick.test between multiple processes. FossilOrigin-Name: ef229cbb7ffbeb8c8877dff70e9d6d43050d2297dee582a37df3a0caaebd2a41 --- diff --git a/manifest b/manifest index 67c01b4f0b..6bf93332c7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\stest\scase\serror\smessage\ssimilar\sto\sthose\sfixed\sin\s[b3d6b3c3]. -D 2022-07-12T15:17:50.749 +C Add\stest/testrunner.tcl,\san\sexperimental\sscript\sfor\sdistributing\sthe\swork\sof\sveryquick.test\sbetween\smultiple\sprocesses. +D 2022-07-12T20:31:16.301 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1507,6 +1507,7 @@ F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 76771269dcc20b2c2d1d6f1175dd50d1eebddc004aebac865483f1829a5cd398 +F test/testrunner.tcl bfaaddd58df6176af83159e3b27767c53abd87db68619a300234ad7fbf9aeed1 F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1979,8 +1980,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9c4a9d09b6b22d7d95420b495dc7d7a42a0638be5824f6af6630539fe787cd4 -R 7e19ab5805b123d32a175181853b886b +P 6d0f677291d2b5ec68c86292da240c5557422aae1290c0844223974449ce539b +R 8212f2952574d93e95a3664f57ee334d +T *branch * testrunner +T *sym-testrunner * +T -sym-trunk * U dan -Z 02ab1b6108a49a8c83f37e6697731f43 +Z a723c28ab5e45345241e356c4c65467f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b5e23fb90a..0e8e50979b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d0f677291d2b5ec68c86292da240c5557422aae1290c0844223974449ce539b \ No newline at end of file +ef229cbb7ffbeb8c8877dff70e9d6d43050d2297dee582a37df3a0caaebd2a41 \ No newline at end of file diff --git a/test/testrunner.tcl b/test/testrunner.tcl new file mode 100644 index 0000000000..7ebbbb712f --- /dev/null +++ b/test/testrunner.tcl @@ -0,0 +1,456 @@ + + +#------------------------------------------------------------------------- +# Usage: +# +proc usage {} { + puts stderr "Usage: $::argv0 ?SWITCHES?" + puts stderr "" + puts stderr "where SWITCHES are:" + puts stderr " --jobs NUMBER-OF-JOBS" + exit 1 +} +#------------------------------------------------------------------------- + + +switch -- [lindex $argv 0] { + helper { + set R(helper) 1 + set R(helper_id) [lindex $argv 1] + set argv [list --testdir=testdir$R(helper_id)] + } + + default { + set R(helper) 0 + set R(helper_id) 0 + } +} + + +set R(dbname) [file normalize testrunner.db] +set R(logname) [file normalize testrunner.log] +set R(timeout) 10000 +set R(nHelper) 4 +set R(info_script) [file normalize [info script]] + +if {$R(helper)==0} { + for {set ii 0} {$ii < [llength $argv]} {incr ii} { + set a [lindex $argv $ii] + set n [string length $a] + + if {($n>2 && [string match "$a*" --jobs]) || $a=="-j"} { + incr ii + set R(nHelper) [lindex $argv $ii] + } else { + usage + } + } + + set argv [list] +} + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +db close + +# The database schema used by the testset database. +# +set R(schema) { + DROP TABLE IF EXISTS script; + DROP TABLE IF EXISTS msg; + + CREATE TABLE script( + filename TEXT PRIMARY KEY, -- full path to test script + state TEXT CHECK( state IN ('ready', 'running', 'done') ), + testfixtureid, -- Id of process that ran script + nerr INTEGER, -- if 'done', the number of errors + ntest INTEGER, -- if 'done', the number of tests + output TEXT -- full output of test script + ); + + CREATE TABLE msg( + id INTEGER PRIMARY KEY, + msg TEXT + ); +} + + +#-------------------------------------------------------------------- +# This is temporary! +# +# Return a list of all scripts in the "veryquick" test. +# +proc all_veryquick_scripts {} { + set OMIT { + async2.test async3.test backup_ioerr.test corrupt.test + corruptC.test crash.test crash2.test crash3.test crash4.test crash5.test + crash6.test crash7.test delete3.test e_fts3.test fts3rnd.test + fkey_malloc.test fuzz.test fuzz3.test fuzz_malloc.test in2.test loadext.test + misc7.test mutex2.test notify2.test onefile.test pagerfault2.test + savepoint4.test savepoint6.test select9.test + speed1.test speed1p.test speed2.test speed3.test speed4.test + speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test + thread003.test thread004.test thread005.test trans2.test vacuum3.test + incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test + vtab_err.test walslow.test walcrash.test walcrash3.test + walthread.test rtree3.test indexfault.test securedel2.test + sort3.test sort4.test fts4growth.test fts4growth2.test + bigsort.test walprotocol.test mmap4.test fuzzer2.test + walcrash2.test e_fkey.test backup.test + writecrash.test + + fts4merge.test fts4merge2.test fts4merge4.test fts4check.test + fts4merge5.test + fts3cov.test fts3snippet.test fts3corrupt2.test fts3an.test + fts3defer.test fts4langid.test fts3sort.test fts5unicode.test + rtree4.test sessionbig.test + + all.test async.test quick.test veryquick.test + memleak.test permutations.test soak.test fts3.test + mallocAll.test rtree.test full.test extraquick.test + session.test rbu.test + } + + set testdir [file normalize $::testdir] + set ret [list] + + foreach f [glob -nocomplain $testdir/*.test] { + if {[lsearch $OMIT [file tail $f]]<0 + && [string match *malloc* $f]==0 + && [string match *ioerr* $f]==0 + && [string match *fault* $f]==0 + && [string match *bigfile* $f]==0 + && [string match *_err* $f]==0 + && [string match *fts5corrupt* $f]==0 + && [string match *fts5big* $f]==0 + && [string match *fts5aj* $f]==0 + } { + lappend ret $f + } + } + + set ret +} +#proc all_veryquick_scripts {} { +# set testdir [file normalize $::testdir] +# glob -nocomplain $testdir/select*.test +#} +#-------------------------------------------------------------------- + + +proc make_new_testset {} { + global R + + sqlite3 db $R(dbname) + db eval $R(schema) + foreach s [all_veryquick_scripts] { + db eval { INSERT INTO script(filename, state) VALUES ($s, 'ready') } + } + + # db eval { SELECT filename FROM Script ORDER BY 1 } { puts $filename } + # exit + + + db close +} + +proc get_next_test {} { + global R + set myid $R(helper_id) + + sqlite3 db $R(dbname) + db timeout $R(timeout) + db eval { BEGIN EXCLUSIVE } + set f [db one { + SELECT filename FROM script WHERE state='ready' ORDER BY 1 LIMIT 1 + }] + if {$f!=""} { + db eval { + UPDATE script SET state='running', testfixtureid=$myid WHERE filename=$f + } + } + db eval { COMMIT } + db close + + return $f +} + +proc r_write_db {tcl} { + global R + sqlite3 db $R(dbname) + db timeout $R(timeout) + db eval { BEGIN EXCLUSIVE } + uplevel $tcl + db eval { COMMIT } + db close +} + +proc r_read_db {tcl} { + global R + sqlite3 db $R(dbname) + db timeout $R(timeout) + db eval { BEGIN } + uplevel $tcl + db eval { COMMIT } + db close +} + +proc r_set_test_result {filename ms nerr ntest output} { + global R + + set f [file tail $filename] + if {$nerr==0} { + set msg "$f... Ok" + } else { + set msg "$f... FAILED - $nerr errors of $ntest tests" + } + append msg " (${ms}ms)" + if {$R(helper)} { + append msg " (helper $R(helper_id))" + } + + r_write_db { + db eval { + UPDATE script + SET state='done', output=$output, nerr=$nerr, ntest=$ntest + WHERE filename=$filename; + + INSERT INTO msg(msg) VALUES ($msg); + } + } +} + +set R(iNextMsg) 1 +proc r_get_messages {{db ""}} { + global R + + if {$db==""} { + sqlite3 rgmhandle $R(dbname) + set dbhandle rgmhandle + $dbhandle timeout $R(timeout) + } else { + set dbhandle $db + } + + $dbhandle transaction { + set next $R(iNextMsg) + set ret [$dbhandle eval {SELECT msg FROM msg WHERE id>=$next}] + set R(iNextMsg) [$dbhandle one {SELECT COALESCE(max(id), 0)+1 FROM msg}] + } + + if {$db==""} { + rgmhandle close + } + + set ret +} + +#-------------------------------------------------------------------------- +# + + +set ::R_INSTALL_PUTS_WRAPPER { + proc puts_sts_wrapper {args} { + set n [llength $args] + if {$n==1 || ($n==2 && [string first [lindex $args 0] -nonewline]==0)} { + uplevel puts_into_caller $args + } else { + # A channel was explicitly specified. + uplevel puts_sts_original $args + } + } + rename puts puts_sts_original + proc puts {args} { uplevel puts_sts_wrapper $args } +} + +proc r_install_puts_wrapper {} $::R_INSTALL_PUTS_WRAPPER +proc r_uninstall_puts_wrapper {} { + rename puts "" + rename puts_sts_original puts +} + +proc slave_test_script {script} { + + # Create the interpreter used to run the test script. + interp create tinterp + + # Populate some global variables that tester.tcl expects to see. + foreach {var value} [list \ + ::argv0 $::argv0 \ + ::argv {} \ + ::SLAVE 1 \ + ] { + interp eval tinterp [list set $var $value] + } + + # The alias used to access the global test counters. + tinterp alias set_test_counter set_test_counter + + # Set up an empty ::cmdlinearg array in the slave. + interp eval tinterp [list array set ::cmdlinearg [array get ::cmdlinearg]] + + # Set up the ::G array in the slave. + interp eval tinterp [list array set ::G [array get ::G]] + interp eval tinterp [list set ::G(runner.tcl) 1] + + interp eval tinterp $::R_INSTALL_PUTS_WRAPPER + tinterp alias puts_into_caller puts_into_caller + + # Load the various test interfaces implemented in C. + load_testfixture_extensions tinterp + + # Run the test script. + set rc [catch { interp eval tinterp $script } msg opt] + if {$rc} { + puts_into_caller $msg + puts_into_caller [dict get $opt -errorinfo] + incr ::TC(errors) + } + + # Check if the interpreter call [run_thread_tests] + if { [interp eval tinterp {info exists ::run_thread_tests_called}] } { + set ::run_thread_tests_called 1 + } + + # Delete the interpreter used to run the test script. + interp delete tinterp +} + +proc slave_test_file {zFile} { + set tail [file tail $zFile] + + # Remember the value of the shared-cache setting. So that it is possible + # to check afterwards that it was not modified by the test script. + # + ifcapable shared_cache { set scs [sqlite3_enable_shared_cache] } + + # Run the test script in a slave interpreter. + # + unset -nocomplain ::run_thread_tests_called + reset_prng_state + set ::sqlite_open_file_count 0 + set time [time { slave_test_script [list source $zFile] }] + set ms [expr [lindex $time 0] / 1000] + + r_install_puts_wrapper + + # Test that all files opened by the test script were closed. Omit this + # if the test script has "thread" in its name. The open file counter + # is not thread-safe. + # + if {[info exists ::run_thread_tests_called]==0} { + do_test ${tail}-closeallfiles { expr {$::sqlite_open_file_count>0} } {0} + } + set ::sqlite_open_file_count 0 + + # Test that the global "shared-cache" setting was not altered by + # the test script. + # + ifcapable shared_cache { + set res [expr {[sqlite3_enable_shared_cache] == $scs}] + do_test ${tail}-sharedcachesetting [list set {} $res] 1 + } + + # Add some info to the output. + # + output2 "Time: $tail $ms ms" + show_memstats + + r_uninstall_puts_wrapper + return $ms +} + +proc puts_into_caller {args} { + global R + if {[llength $args]==1} { + append R(output) [lindex $args 0] + append R(output) "\n" + } else { + append R(output) [lindex $args 1] + } +} + +set R(nHelperRunning) 0 +if {$R(helper)==0} { + cd $cmdlinearg(TESTFIXTURE_HOME) + make_new_testset + for {set ii 1} {$ii <= $R(nHelper)} {incr ii} { + set cmd "[info nameofexec] $R(info_script) helper $ii 2>@1" + puts "Launching helper $ii ($cmd)" + set chan [open "|$cmd" r] + fconfigure $chan -blocking false + fileevent $chan readable [list r_helper_readable $ii $chan] + incr R(nHelperRunning) + } + cd $cmdlinearg(testdir) +} + +proc r_helper_readable {id chan} { + puts "helper $id:[gets $chan]" + if {[eof $chan]} { + puts "helper $id is FINISHED" + incr ::R(nHelperRunning) -1 + close $chan + } +} + +if {$R(helper) || $R(nHelper)<4} { + while { ""!=[set f [get_next_test]] } { + set R(output) "" + set TC(count) 0 + set TC(errors) 0 + set ms [slave_test_file $f] + r_set_test_result $f $ms $TC(errors) $TC(count) $R(output) + + if {$R(helper)==0} { + foreach msg [r_get_messages] { puts $msg } + } + } +} + +set TTT 0 +sqlite3 db $R(dbname) +db timeout $R(timeout) +while {$R(nHelperRunning)>0} { + after 250 { incr TTT } + vwait TTT + foreach msg [r_get_messages db] { puts $msg } +} +db close + +set errcode 0 +if {$R(helper)==0} { + sqlite3 db $R(dbname) + db timeout $R(timeout) + db eval { SELECT sum(nerr) AS nerr, sum(ntest) AS ntest FROM script } { + puts "$nerr errors from $ntest tests." + } + if {$nerr>0} { + db eval { SELECT filename FROM script WHERE nerr>0 } { + lappend errlist [file tail $filename] + } + puts "Errors in: $errlist" + set errcode 1 + } + + set errlist [list] + db eval { SELECT filename FROM script WHERE state!='done' } { + lappend errlist [file tail $filename] + } + if {$errlist!=[list]} { + puts "Tests DID NOT FINISH (crashed?): $errlist" + set errcode 1 + } + + set fd [open $R(logname) w] + db eval {SELECT output FROM script ORDER BY filename} { + puts $fd $output + } + close $fd + + puts "Test database is $R(dbname)" + puts "Test log file is $R(logname)" +} + +exit $errcode +