From: drh Date: Tue, 26 May 2015 18:15:08 +0000 (+0000) Subject: The "make fuzztest" target now uses fuzzcheck instead of fuzzershell. X-Git-Tag: version-3.8.11~208^2~2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ea93c7005db216a2372d213ca3a17cf4668036c3;p=thirdparty%2Fsqlite.git The "make fuzztest" target now uses fuzzcheck instead of fuzzershell. FossilOrigin-Name: 4a5f6f1f0128657fd8d4d99d0682edd5bac2a19e --- diff --git a/Makefile.in b/Makefile.in index bc8e7a148d..cc994488b0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -522,6 +522,12 @@ TESTPROGS = \ sqlite3_analyzer$(TEXE) \ sqldiff$(TEXE) +# Databases containing fuzzer test cases +# +FUZZDATA = \ + $(TOP)/test/fuzzdata1.db \ + $(TOP)/test/fuzzdata2.db \ + $(TOP)/test/fuzzdata3.db # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. @@ -975,7 +981,7 @@ fulltest: $(TESTPROGS) fuzztest ./testfixture$(TEXE) $(TOP)/test/all.test # Really really long testing -soaktest: $(TESTPROGS) fuzzoomtest +soaktest: $(TESTPROGS) ./testfixture$(TEXE) $(TOP)/test/all.test -soak=1 # Do extra testing but not everything. @@ -983,11 +989,8 @@ fulltestonly: $(TESTPROGS) ./testfixture$(TEXE) $(TOP)/test/full.test # Fuzz testing -fuzztest: fuzzershell$(TEXE) - ./fuzzershell$(TEXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt - -fuzzoomtest: fuzzershell$(TEXE) - ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt --oom +fuzztest: fuzzcheck$(TEXE) + ./fuzzcheck$(TEXE) $(FUZZDATA) # This is the common case. Run many tests but not those that take # a really long time. @@ -998,15 +1001,15 @@ test: $(TESTPROGS) fuzztest # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # -valgrindtest: $(TESTPROGS) fuzzershell$(TEXE) - valgrind -v ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt +valgrindtest: $(TESTPROGS) fuzzcheck$(TEXE) + valgrind -v ./fuzzcheck$(TEXE) $(FUZZDATA) OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind # A very fast test that checks basic sanity. The name comes from # the 60s-era electronics testing: "Turn it on and see if smoke # comes out." # -smoketest: $(TESTPROGS) fuzzershell$(TEXE) +smoketest: $(TESTPROGS) fuzzcheck$(TEXE) ./testfixture$(TEXE) $(TOP)/test/main.test sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl diff --git a/Makefile.msc b/Makefile.msc index 6c9cd7ecc4..642616e935 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1190,6 +1190,13 @@ TESTPROGS = \ sqlite3_analyzer.exe \ sqldiff.exe +# Databases containing fuzzer test cases +# +FUZZDATA = \ + $(TOP)\test\fuzzdata1.db \ + $(TOP)\test\fuzzdata2.db \ + $(TOP)\test\fuzzdata3.db + # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. @@ -1660,7 +1667,7 @@ extensiontest: testfixture.exe testloadext.dll fulltest: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\all.test -soaktest: $(TESTPROGS) fuzzoomtest +soaktest: $(TESTPROGS) .\testfixture.exe $(TOP)\test\all.test -soak=1 fulltestonly: $(TESTPROGS) fuzztest @@ -1669,16 +1676,13 @@ fulltestonly: $(TESTPROGS) fuzztest queryplantest: testfixture.exe sqlite3.exe .\testfixture.exe $(TOP)\test\permutations.test queryplanner -fuzztest: fuzzershell.exe - .\fuzzershell.exe $(TOP)\test\fuzzdata1.txt $(TOP)\test\fuzzdata2.txt - -fuzzoomtest: fuzzershell.exe - .\fuzzershell.exe -f $(TOP)\test\fuzzdata1.txt --oom +fuzztest: fuzzcheck.exe + .\fuzzcheck.exe $(FUZZDATA) test: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\veryquick.test -smoketest: $(TESTPROGS) fuzzershell.exe +smoketest: $(TESTPROGS) .\testfixture.exe $(TOP)\test\main.test sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl diff --git a/main.mk b/main.mk index df1f0c7265..bb17a17d80 100644 --- a/main.mk +++ b/main.mk @@ -403,6 +403,13 @@ TESTPROGS = \ sqlite3_analyzer$(EXE) \ sqldiff$(EXE) +# Databases containing fuzzer test cases +# +FUZZDATA = \ + $(TOP)/test/fuzzdata1.db \ + $(TOP)/test/fuzzdata2.db \ + $(TOP)/test/fuzzdata3.db + # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # @@ -657,7 +664,7 @@ fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c fulltest: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/all.test -soaktest: $(TESTPROGS) fuzzoomtest +soaktest: $(TESTPROGS) ./testfixture$(EXE) $(TOP)/test/all.test -soak=1 fulltestonly: $(TESTPROGS) fuzztest @@ -666,11 +673,8 @@ fulltestonly: $(TESTPROGS) fuzztest queryplantest: testfixture$(EXE) sqlite3$(EXE) ./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner -fuzztest: fuzzershell$(EXE) - ./fuzzershell$(EXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt - -fuzzoomtest: fuzzershell$(EXE) - ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt --oom +fuzztest: fuzzcheck$(EXE) $(FUZZDATA) + ./fuzzcheck$(EXE) $(FUZZDATA) test: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/veryquick.test @@ -678,15 +682,15 @@ test: $(TESTPROGS) fuzztest # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # -valgrindtest: $(TESTPROGS) fuzzershell$(EXE) - valgrind -v ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt +valgrindtest: $(TESTPROGS) fuzzcheck$(EXE) $(FUZZDATA) + valgrind -v ./fuzzcheck$(EXE) $(FUZZDATA) OMIT_MISUSE=1 valgrind -v ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind # A very fast test that checks basic sanity. The name comes from # the 60s-era electronics testing: "Turn it on and see if smoke # comes out." # -smoketest: $(TESTPROGS) fuzzershell$(EXE) +smoketest: $(TESTPROGS) fuzzcheck$(EXE) ./testfixture$(EXE) $(TOP)/test/main.test # The next two rules are used to support the "threadtest" target. Building diff --git a/manifest b/manifest index 7a107d9d01..46ac057018 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Enhance\sfuzzcheck\swith\sthe\sability\sto\sstore\sdescriptions\sin\seach\ssource\ndatabase\sand\sto\srun\smultiple\ssource\sdatabases\sin\sa\ssingle\sinvocation. -D 2015-05-26T17:57:56.164 +C The\s"make\sfuzztest"\starget\snow\suses\sfuzzcheck\sinstead\sof\sfuzzershell. +D 2015-05-26T18:15:08.927 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 69b596e4be75a16f4e55587f29db66497b207fad +F Makefile.in 3feb7cbdad8898fe7a8a24355b4a753029c3ec3b F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 6c5a92060e8e5fe465186ea7facfc80c258bcf9e +F Makefile.msc d37d2c2323df3acae6e24c71a478889421c17264 F Makefile.vxworks e1b65dea203f054e71653415bd8f96dcaed47858 F README.md 0bfccb18927349653c09137a458b961fa8ab4cb9 F VERSION ce0ae95abd7121c534f6917c1c8f2b70d9acd4db @@ -171,7 +171,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk ed889716a9ef6887d8732e3a6fd8774e85199643 +F main.mk 031e36b22e2be9b9fdab086fd3115fc88c2e37bd F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk d5e22023b5238985bb54a72d33e0ac71fe4f8a32 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -652,8 +652,9 @@ F test/fuzz3.test efd384b896c647b61a2c1848ba70d42aad60a7b3 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 F test/fuzzcheck.c 69998e6430903af8f1904dfa8c7fc21c94d59ebc -F test/fuzzdata1.txt 9fceb50868e0b798160e83742bd7e44e457176a0 -F test/fuzzdata2.txt ba9b4467d7ec46cc85d32c0d031540cd727ae6ad +F test/fuzzdata1.db b60254eeb6bc11474071b883059662a73c48da7f +F test/fuzzdata2.db f03a420d3b822cc82e4f894ca957618fbe9c4973 +F test/fuzzdata3.db 57ae47d2a1c837fb8ddc10ce4ab56a04ad044cb4 F test/fuzzer1.test d4c52aaf3ef923da293a2653cfab33d02f718a36 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -776,7 +777,6 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 F test/misc8.test fc2754d38892f7dac30c22db3616c2764f117d66 F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 -F test/mkfuzzdata1.tcl 5f9c33fadc64b078bb4a2c04c18b6dd3da075bec F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 F test/mmap3.test c92273e16eb8d23c1d55c9815b446bb72ef0512e @@ -1279,7 +1279,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5b46d2a78a45de4176b666bfc89c2aaa6f083662 -R 0ae8e5db841e94a0cd60c9e4742373e1 +P 193364c81c301a41c16835108d23ad2ab84d9dd7 +R 72bebf492c55de7ece7f6fdbb9a7dd3f +T *branch * test-using-fuzzcheck +T *sym-test-using-fuzzcheck * +T -sym-trunk * U drh -Z f62c751ed8a012eb7d841a949af555ac +Z 66f2916d8728b836edfc9ebde30c4bfc diff --git a/manifest.uuid b/manifest.uuid index cf359c332f..0f9ca58d31 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -193364c81c301a41c16835108d23ad2ab84d9dd7 \ No newline at end of file +4a5f6f1f0128657fd8d4d99d0682edd5bac2a19e \ No newline at end of file diff --git a/test/fuzzdata1.db b/test/fuzzdata1.db new file mode 100644 index 0000000000..222e598a44 Binary files /dev/null and b/test/fuzzdata1.db differ diff --git a/test/fuzzdata1.txt b/test/fuzzdata1.txt deleted file mode 100644 index 6d3b20c131..0000000000 Binary files a/test/fuzzdata1.txt and /dev/null differ diff --git a/test/fuzzdata2.txt b/test/fuzzdata2.db similarity index 75% rename from test/fuzzdata2.txt rename to test/fuzzdata2.db index b1d4bb7bb1..598814622c 100644 Binary files a/test/fuzzdata2.txt and b/test/fuzzdata2.db differ diff --git a/test/fuzzdata3.db b/test/fuzzdata3.db new file mode 100644 index 0000000000..f06a607412 Binary files /dev/null and b/test/fuzzdata3.db differ diff --git a/test/mkfuzzdata1.tcl b/test/mkfuzzdata1.tcl deleted file mode 100644 index 8af45f6254..0000000000 --- a/test/mkfuzzdata1.tcl +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/tclsh -# -# Run this script in order to rebuild the fuzzdata1.txt file containing -# fuzzer data for the fuzzershell utility that is create by afl-fuzz. -# -# This script gathers all of the test cases identified by afl-fuzz and -# runs afl-cmin and afl-tmin over them all to try to generate a mimimum -# set of tests that cover all observed behavior. -# -# Options: -# -# --afl-bin DIR1 DIR1 contains the AFL binaries -# --fuzzershell PATH Full pathname of instrumented fuzzershell -# --afl-data DIR3 DIR3 is the "-o" directory from afl-fuzz -# -o FILE Write results into FILE -# -set AFLBIN {} -set FUZZERSHELL {} -set AFLDATA {} -set OUTFILE {} - -proc usage {} { - puts stderr "Usage: $::argv0 --afl-bin DIR --fuzzershell PATH\ - --afl-data DIR -o FILE" - exit 1 -} -proc cmdlineerr {msg} { - puts stderr $msg - usage -} - -for {set i 0} {$i<[llength $argv]} {incr i} { - set x [lindex $argv $i] - if {[string index $x 0]!="-"} {cmdlineerr "illegal argument: $x"} - set x [string trimleft $x -] - incr i - if {$i>=[llength $argv]} {cmdlineerr "no argument on --$x"} - set a [lindex $argv $i] - switch -- $x { - afl-bin {set AFLBIN $a} - afl-data {set AFLDATA $a} - fuzzershell {set FUZZERSHELL $a} - o {set OUTFILE $a} - default {cmdlineerr "unknown option: --$x"} - } -} -proc checkarg {varname option} { - set val [set ::$varname] - if {$val==""} {cmdlineerr "required option missing: --$option"} -} -checkarg AFLBIN afl-bin -checkarg AFLDATA afl-data -checkarg FUZZERSHELL fuzzershell -checkarg OUTFILE o -proc checkexec {x} { - if {![file exec $x]} {cmdlineerr "cannot find $x"} -} -checkexec $AFLBIN/afl-cmin -checkexec $AFLBIN/afl-tmin -checkexec $FUZZERSHELL -proc checkdir {x} { - if {![file isdir $x]} {cmdlineerr "no such directory: $x"} -} -checkdir $AFLDATA/queue - -proc progress {msg} { - puts "******** $msg" - flush stdout -} -progress "mkdir tmp1 tmp2" -file mkdir tmp1 tmp2 -progress "copying test cases from $AFLDATA into tmp1..." -set n 0 -foreach file [glob -nocomplain $AFLDATA/queue/id:*] { - incr n - file copy $file tmp1/$n -} -foreach file [glob -nocomplain $AFLDATA/crash*/id:*] { - incr n - file copy $file tmp1/$n -} -progress "total $n files copied." -progress "running: $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL" -exec $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL >&@ stdout -progress "afl-cmin complete." -# -# Experiments show that running afl-tmin is too slow for this application. -# And it doesn't really make the test cases that much smaller. So let's -# just skip it. -# -# foreach file [glob tmp2/*] { -# progress "$AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] $FUZZERSHELL" -# exec $AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] \ -# $FUZZERSHELL >&@ stdout -# } -progress "generating final output into $OUTFILE" -set out [open $OUTFILE wb] -puts $out "# Test data for use with fuzzershell. Automatically -# generated using $argv0. This file contains binary data -#" -set n 0 -foreach file [glob tmp2/*] { - incr n - puts -nonewline $out "/****<$n>****/" - set in [open $file rb] - puts -nonewline $out [read $in] - close $in -} -close $out -progress "done. $n test cases written to $OUTFILE" -progress "clean-up..." -file delete -force tmp1 -progress "culled test cases left in the tmp2 directory"