benchtests: Enable BENCHSET to run subset of tests

This patch adds BENCHSET variable to benchtests/Makefile in order to
provide the capability to run a list of subsets of benchmark tests, ie;

    make bench BENCHSET="bench-pthread bench-math malloc-thread"

This helps users to benchmark specific glibc area

ChangeLog:

        * benchtests/Makefile:Add BENCHSET to allow subsets of
        benchmarks to be run.
        * benchtests/README: Add documentation for: Running subsets of
        benchmarks.

Signed-off-by: Victor Rodriguez <victor.rodriguez.bahena@intel.com>
Signed-off-by: Icarus Sparry <icarus.w.sparry@intel.com>
Reviewed-By: Siddhesh Poyarekar <siddhesh@sourceware.org>

diff --git a/ChangeLog b/ChangeLog
index 9685aec392d57512165e10f88277c5b2126d1893..f7616dcb2b5d315d31e18bf5c062fb105e50d4de 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2017-11-28  Victor Rodriguez  <victor.rodriguez.bahena@intel.com>
+           Icarus Sparry  <icarus.w.sparry@intel.com>
+
+       * benchtests/Makefile:Add BENCHSET to allow subsets of
+       benchmarks to be run.
+       * benchtests/README: Add documentation for: Running subsets of
+       benchmarks.
+
 2017-11-28  Victor Rodriguez  <victor.rodriguez.bahena@intel.com>
 
        * benchtests/scripts/benchout.schema.json: Fix regex to accept a wider

diff --git a/benchtests/Makefile b/benchtests/Makefile
index 4157f8ce22e7d62e69eb0e0ee029d772a08e9133..74b3821ccfea6912e68578ad2598d68a9e38223c 100644 (file)
--- a/benchtests/Makefile
+++ b/benchtests/Makefile
@@ -30,7 +30,11 @@ bench-pthread := pthread_once thread_create
 
 bench-string := ffs ffsll
 
+ifeq (${BENCHSET},)
 bench := $(bench-math) $(bench-pthread) $(bench-string)
+else
+bench := $(foreach B,$(filter bench-%,${BENCHSET}), ${${B}})
+endif
 
 # String function benchmarks.
 string-benchset := bcopy bzero memccpy memchr memcmp memcpy memmem memmove \
@@ -68,8 +72,12 @@ stdio-common-benchset := sprintf
 
 math-benchset := math-inlines
 
+ifeq (${BENCHSET},)
 benchset := $(string-benchset-all) $(stdlib-benchset) $(stdio-common-benchset) \
            $(math-benchset)
+else
+benchset := $(foreach B,$(filter %-benchset,${BENCHSET}), ${${B}})
+endif
 
 CFLAGS-bench-ffs.c += -fno-builtin
 CFLAGS-bench-ffsll.c += -fno-builtin
@@ -81,7 +89,11 @@ CFLAGS-bench-fmaxf.c += -fno-builtin
 CFLAGS-bench-trunc.c += -fno-builtin
 CFLAGS-bench-truncf.c += -fno-builtin
 
+ifeq (${BENCHSET},)
 bench-malloc := malloc-thread
+else
+bench-malloc := $(filter malloc-%,${BENCHSET})
+endif
 
 $(addprefix $(objpfx)bench-,$(bench-math)): $(libm)
 $(addprefix $(objpfx)bench-,$(math-benchset)): $(libm)
@@ -149,6 +161,19 @@ bench-clean:
        rm -f $(timing-type) $(addsuffix .o,$(timing-type))
        rm -f $(addprefix $(objpfx),$(bench-extra-objs))
 
+# Validate the passed in BENCHSET
+ifneq ($(strip ${BENCHSET}),)
+VALIDBENCHSETNAMES := bench-pthread bench-math bench-string string-benchset \
+   wcsmbs-benchset stdlib-benchset stdio-common-benchset math-benchset \
+   malloc-thread
+INVALIDBENCHSETNAMES := $(filter-out ${VALIDBENCHSETNAMES},${BENCHSET})
+ifneq (${INVALIDBENCHSETNAMES},)
+$(info The following values in BENCHSET are invalid: ${INVALIDBENCHSETNAMES})
+$(info The valid ones are: ${VALIDBENCHSETNAMES})
+$(error Invalid BENCHSET value)
+endif
+endif
+
 # Define the bench target only if the target has a usable python installation.
 ifdef PYTHON
 bench: bench-build bench-set bench-func bench-malloc
@@ -175,10 +200,11 @@ bench-set: $(binaries-benchset)
        done
 
 bench-malloc: $(binaries-bench-malloc)
-       run=$(objpfx)bench-malloc-thread; \
-       for thr in 1 8 16 32; do \
-         echo "Running $${run} $${thr}"; \
+       for run in $^; do \
+               for thr in 1 8 16 32; do \
+                       echo "Running $${run} $${thr}"; \
          $(run-bench) $${thr} > $${run}-$${thr}.out; \
+         done;\
        done
 
 # Build and execute the benchmark functions.  This target generates JSON
@@ -186,25 +212,28 @@ bench-malloc: $(binaries-bench-malloc)
 # so one could even execute them individually and process it using any JSON
 # capable language or tool.
 bench-func: $(binaries-bench)
+       if [ -n '$^' ] ; then \
        { timing_type=$$($(timing-type)); \
-       echo "{\"timing_type\": \"$${timing_type}\","; \
-       echo " \"functions\": {"; \
-       for run in $^; do \
-         if ! [ "x$${run}" = "x$<" ]; then \
-           echo ","; \
+         echo "{\"timing_type\": \"$${timing_type}\","; \
+         echo " \"functions\": {"; \
+         for run in $^; do \
+           if ! [ "x$${run}" = "x$<" ]; then \
+             echo ","; \
+           fi; \
+           echo "Running $${run}" >&2; \
+           $(run-bench) $(DETAILED_OPT); \
+         done; \
+         echo; \
+         echo " }"; \
+         echo "}"; \
+         } > $(objpfx)bench.out-tmp; \
+         if [ -f $(objpfx)bench.out ]; then \
+           mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
          fi; \
-         echo "Running $${run}" >&2; \
-         $(run-bench) $(DETAILED_OPT); \
-       done; \
-       echo; \
-       echo " }"; \
-       echo "}"; } > $(objpfx)bench.out-tmp; \
-       if [ -f $(objpfx)bench.out ]; then \
-         mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
-       fi; \
-       mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
-       $(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
-               scripts/benchout.schema.json
+         mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out; \
+         $(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
+         scripts/benchout.schema.json; \
+       fi
 
 $(timing-type) $(binaries-bench) $(binaries-benchset) \
        $(binaries-bench-malloc): %: %.o $(objpfx)json-lib.o \

diff --git a/benchtests/README b/benchtests/README
index 9aa750a519a18fccf688a0ab41fe7d8798ed54bf..4ddff794d136f65fefb90c23f0fd6a4f5babaea2 100644 (file)
--- a/benchtests/README
+++ b/benchtests/README
@@ -53,6 +53,25 @@ otherwise the above command may try to build the benchmark again.  Benchmarks
 that require generated code to be executed during the build are skipped when
 cross-building.
 
+Running subsets of benchmarks:
+==============================
+
+To run only a subset of benchmarks, one may invoke make as follows
+
+  $ make bench BENCHSET="bench-pthread bench-math malloc-thread"
+
+where BENCHSET may be a space-separated list of the following values:
+
+    bench-math
+    bench-pthread
+    bench-string
+    string-benchset
+    wcsmbs-benchset
+    stdlib-benchset
+    stdio-common-benchset
+    math-benchset
+    malloc-thread
+
 Adding a function to benchtests:
 ===============================