t/perf: add iteration setup mechanism to perf-lib

Tests that affect the repo in stateful ways are easier to write if we
can run setup steps outside of the measured portion of perf iteration.

This change adds a "--setup 'setup-script'" parameter to test_perf. To
make invocations easier to understand, I also moved the prerequisites to
a new --prereq parameter.

The setup facility will be used in the upcoming perf tests for batch
mode, but it already helps in some existing tests, like t5302 and t7820.

Signed-off-by: Neeraj Singh <neerajsi@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/t/perf/p4220-log-grep-engines.sh b/t/perf/p4220-log-grep-engines.sh
index 2bc47ded4d11f95a1711b442dbb8a363c1fb531f..03fbfbb85d34142713787997009be48b97ec03fc 100755 (executable)
--- a/t/perf/p4220-log-grep-engines.sh
+++ b/t/perf/p4220-log-grep-engines.sh
@@ -36,7 +36,8 @@ do
                else
                        prereq=""
                fi
-               test_perf $prereq "$engine log$GIT_PERF_4220_LOG_OPTS --grep='$pattern'" "
+               test_perf "$engine log$GIT_PERF_4220_LOG_OPTS --grep='$pattern'" \
+                       --prereq "$prereq" "
                        git -c grep.patternType=$engine log --pretty=format:%h$GIT_PERF_4220_LOG_OPTS --grep='$pattern' >'out.$engine' || :
                "
        done

diff --git a/t/perf/p4221-log-grep-engines-fixed.sh b/t/perf/p4221-log-grep-engines-fixed.sh
index 060971265a9c852c608d97d5059172f14a7c3395..0a6d6dfc2198f7bbf0fec141575ff25306eda423 100755 (executable)
--- a/t/perf/p4221-log-grep-engines-fixed.sh
+++ b/t/perf/p4221-log-grep-engines-fixed.sh
@@ -26,7 +26,8 @@ do
                else
                        prereq=""
                fi
-               test_perf $prereq "$engine log$GIT_PERF_4221_LOG_OPTS --grep='$pattern'" "
+               test_perf "$engine log$GIT_PERF_4221_LOG_OPTS --grep='$pattern'" \
+                       --prereq "$prereq" "
                        git -c grep.patternType=$engine log --pretty=format:%h$GIT_PERF_4221_LOG_OPTS --grep='$pattern' >'out.$engine' || :
                "
        done

diff --git a/t/perf/p5302-pack-index.sh b/t/perf/p5302-pack-index.sh
index c16f6a3ff698c088db427ae9d7b0a8f16f59a8c6..14c601bbf860aa67087e5b9a0bb421f938f9ff3c 100755 (executable)
--- a/t/perf/p5302-pack-index.sh
+++ b/t/perf/p5302-pack-index.sh
@@ -26,9 +26,8 @@ test_expect_success 'set up thread-counting tests' '
        done
 '
 
-test_perf PERF_EXTRA 'index-pack 0 threads' '
-       rm -rf repo.git &&
-       git init --bare repo.git &&
+test_perf 'index-pack 0 threads' --prereq PERF_EXTRA \
+       --setup 'rm -rf repo.git && git init --bare repo.git' '
        GIT_DIR=repo.git git index-pack --threads=1 --stdin < $PACK
 '
 
@@ -36,17 +35,15 @@ for t in $threads
 do
        THREADS=$t
        export THREADS
-       test_perf PERF_EXTRA "index-pack $t threads" '
-               rm -rf repo.git &&
-               git init --bare repo.git &&
+       test_perf "index-pack $t threads" --prereq PERF_EXTRA \
+               --setup 'rm -rf repo.git && git init --bare repo.git' '
                GIT_DIR=repo.git GIT_FORCE_THREADS=1 \
                git index-pack --threads=$THREADS --stdin <$PACK
        '
 done
 
-test_perf 'index-pack default number of threads' '
-       rm -rf repo.git &&
-       git init --bare repo.git &&
+test_perf 'index-pack default number of threads' \
+       --setup 'rm -rf repo.git && git init --bare repo.git' '
        GIT_DIR=repo.git git index-pack --stdin < $PACK
 '
 

diff --git a/t/perf/p7519-fsmonitor.sh b/t/perf/p7519-fsmonitor.sh
index c8be58f3c763beb7b890925251300494365b3900..5b489c968b884d12a313c506c044e545cbdfcf36 100755 (executable)
--- a/t/perf/p7519-fsmonitor.sh
+++ b/t/perf/p7519-fsmonitor.sh
@@ -60,18 +60,6 @@ then
        esac
 fi
 
-if test -n "$GIT_PERF_7519_DROP_CACHE"
-then
-       # When using GIT_PERF_7519_DROP_CACHE, GIT_PERF_REPEAT_COUNT must be 1 to
-       # generate valid results. Otherwise the caching that happens for the nth
-       # run will negate the validity of the comparisons.
-       if test "$GIT_PERF_REPEAT_COUNT" -ne 1
-       then
-               echo "warning: Setting GIT_PERF_REPEAT_COUNT=1" >&2
-               GIT_PERF_REPEAT_COUNT=1
-       fi
-fi
-
 trace_start() {
        if test -n "$GIT_PERF_7519_TRACE"
        then
@@ -167,10 +155,10 @@ setup_for_fsmonitor() {
 
 test_perf_w_drop_caches () {
        if test -n "$GIT_PERF_7519_DROP_CACHE"; then
-               test-tool drop-caches
+               test_perf "$1" --setup "test-tool drop-caches" "$2"
+       else
+               test_perf "$@"
        fi
-
-       test_perf "$@"
 }
 
 test_fsmonitor_suite() {

diff --git a/t/perf/p7820-grep-engines.sh b/t/perf/p7820-grep-engines.sh
index 8b09c5bf328b880b7bbe10f37c7356b785473384..9bfb86842a90df812c7a9260e153830668408483 100755 (executable)
--- a/t/perf/p7820-grep-engines.sh
+++ b/t/perf/p7820-grep-engines.sh
@@ -49,13 +49,15 @@ do
                fi
                if ! test_have_prereq PERF_GREP_ENGINES_THREADS
                then
-                       test_perf $prereq "$engine grep$GIT_PERF_7820_GREP_OPTS '$pattern'" "
+                       test_perf "$engine grep$GIT_PERF_7820_GREP_OPTS '$pattern'" \
+                               --prereq "$prereq" "
                                git -c grep.patternType=$engine grep$GIT_PERF_7820_GREP_OPTS -- '$pattern' >'out.$engine' || :
                        "
                else
                        for threads in $GIT_PERF_GREP_THREADS
                        do
-                               test_perf PTHREADS,$prereq "$engine grep$GIT_PERF_7820_GREP_OPTS '$pattern' with $threads threads" "
+                               test_perf "$engine grep$GIT_PERF_7820_GREP_OPTS '$pattern' with $threads threads"
+                                       --prereq PTHREADS,$prereq "
                                        git -c grep.patternType=$engine -c grep.threads=$threads grep$GIT_PERF_7820_GREP_OPTS -- '$pattern' >'out.$engine.$threads' || :
                                "
                        done

diff --git a/t/perf/perf-lib.sh b/t/perf/perf-lib.sh
index 407252bac70fbfc7b3a30fe29833f5944b3de8a5..df5b1f1c37c59393f96df8fb6477ff8836b50bc5 100644 (file)
--- a/t/perf/perf-lib.sh
+++ b/t/perf/perf-lib.sh
@@ -189,19 +189,39 @@ exit $ret' >&3 2>&4
 }
 
 test_wrapper_ () {
-       test_wrapper_func_=$1; shift
+       local test_wrapper_func_="$1"; shift
+       local test_title_="$1"; shift
        test_start_
-       test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
-       test "$#" = 2 ||
-       BUG "not 2 or 3 parameters to test-expect-success"
+       test_prereq=
+       test_perf_setup_=
+       while test $# != 0
+       do
+               case $1 in
+               --prereq)
+                       test_prereq=$2
+                       shift
+                       ;;
+               --setup)
+                       test_perf_setup_=$2
+                       shift
+                       ;;
+               *)
+                       break
+                       ;;
+               esac
+               shift
+       done
+       test "$#" = 1 || BUG "test_wrapper_ needs 2 positional parameters"
        export test_prereq
-       if ! test_skip "$@"
+       export test_perf_setup_
+
+       if ! test_skip "$test_title_" "$@"
        then
                base=$(basename "$0" .sh)
                echo "$test_count" >>"$perf_results_dir"/$base.subtests
                echo "$1" >"$perf_results_dir"/$base.$test_count.descr
                base="$perf_results_dir"/"$PERF_RESULTS_PREFIX$(basename "$0" .sh)"."$test_count"
-               "$test_wrapper_func_" "$@"
+               "$test_wrapper_func_" "$test_title_" "$@"
        fi
 
        test_finish_
@@ -214,6 +234,16 @@ test_perf_ () {
                echo "perf $test_count - $1:"
        fi
        for i in $(test_seq 1 $GIT_PERF_REPEAT_COUNT); do
+               if test -n "$test_perf_setup_"
+               then
+                       say >&3 "setup: $test_perf_setup_"
+                       if ! test_eval_ $test_perf_setup_
+                       then
+                               test_failure_ "$test_perf_setup_"
+                               break
+                       fi
+
+               fi
                say >&3 "running: $2"
                if test_run_perf_ "$2"
                then
@@ -237,11 +267,24 @@ test_perf_ () {
        rm test_time.*
 }
 
+# Usage: test_perf 'title' [options] 'perf-test'
+#      Run the performance test script specified in perf-test with
+#      optional prerequisite and setup steps.
+# Options:
+#      --prereq prerequisites: Skip the test if prequisites aren't met
+#      --setup "setup-steps": Run setup steps prior to each measured iteration
+#
 test_perf () {
        test_wrapper_ test_perf_ "$@"
 }
 
 test_size_ () {
+       if test -n "$test_perf_setup_"
+       then
+               say >&3 "setup: $test_perf_setup_"
+               test_eval_ $test_perf_setup_
+       fi
+
        say >&3 "running: $2"
        if test_eval_ "$2" 3>"$base".result; then
                test_ok_ "$1"
@@ -250,6 +293,14 @@ test_size_ () {
        fi
 }
 
+# Usage: test_size 'title' [options] 'size-test'
+#      Run the size test script specified in size-test with optional
+#      prerequisites and setup steps. Returns the numeric value
+#      returned by size-test.
+# Options:
+#      --prereq prerequisites: Skip the test if prequisites aren't met
+#      --setup "setup-steps": Run setup steps prior to the size measurement
+
 test_size () {
        test_wrapper_ test_size_ "$@"
 }