Add "make test-network-all" to verify multiple test networks

make test-network-all is Makefile target which verifies a series
of test networks generated using test-network.sh and chutney.

It runs IPv6 and mixed version test networks if the prerequisites are
available.

Each test network reports PASS, FAIL, or SKIP.
Closes ticket 16953. Patch by "teor".

Also adds "--hs-multi-client 1" option to TEST_NETWORK_FLAGS.
This resolves #17012.

Larger networks, such as bridges+hs, may fail until #16952 is merged.

diff --git a/Makefile.am b/Makefile.am
index b5e71aad1e3d66afe8e0ea05af7e0ba7f6d675ed..68695ede0aa2def255267c93657b96a5642e7896 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -36,13 +36,16 @@ EXTRA_DIST+= \
 if COVERAGE_ENABLED
 TEST_CFLAGS=-fno-inline -fprofile-arcs -ftest-coverage
 TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE
-TEST_NETWORK_FLAGS="--coverage"
+TEST_NETWORK_FLAGS=--coverage --hs-multi-client 1
 else
 TEST_CFLAGS=
 TEST_CPPFLAGS=-DTOR_UNIT_TESTS
-TEST_NETWORK_FLAGS=
+TEST_NETWORK_FLAGS=--hs-multi-client 1
 endif
 
+TEST_NETWORK_ALL_LOG_DIR=$(top_builddir)/test_network_log
+TEST_NETWORK_ALL_DRIVER_FLAGS=--color-tests yes
+
 #install-data-local:
 #      $(INSTALL) -m 755 -d $(LOCALSTATEDIR)/lib/tor
 
@@ -88,6 +91,36 @@ need-chutney-path:
 test-network: need-chutney-path all
        $(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_FLAGS)
 
+# Run all available tests using automake's test-driver
+# only run IPv6 tests if we can ping6 ::1 (localhost)
+# some IPv6 tests will fail without an IPv6 DNS server (see #16971 and #17011)
+# only run mixed tests if we have a tor-stable binary
+# see #17015 for autodetection of different tor versions
+test-network-all: need-chutney-path all test-driver
+       mkdir -p $(TEST_NETWORK_ALL_LOG_DIR)
+       @flavors="$(TEST_CHUTNEY_FLAVORS)"; \
+       if ping6 -q -o ::1 >/dev/null 2>&1; then \
+               echo "ping6 ::1 succeeded, running IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
+               flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
+       else \
+               echo "ping6 ::1 failed, skipping IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
+               skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
+       fi; \
+       if command -v tor-stable >/dev/null 2>&1; then \
+               echo "tor-stable found, running mixed flavors: $(TEST_CHUTNEY_FLAVORS_MIXED)."; \
+               flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_MIXED)"; \
+       else \
+               echo "tor-stable not found, skipping mixed flavors: $(TEST_CHUTNEY_FLAVORS_MIXED)."; \
+               skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_MIXED)"; \
+       fi; \
+       for f in $$skip_flavors; do \
+               echo "SKIP: $$f"; \
+       done; \
+       for f in $$flavors; do \
+               ./test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \
+       done
+       @echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."
+
 need-stem-path:
        @if test ! -d "$$STEM_SOURCE_DIR"; then \
                echo '$$STEM_SOURCE_DIR was not set.'; echo; \
@@ -173,3 +206,4 @@ mostlyclean-local:
        rm -f $(top_builddir)/src/*/*.gc{da,no} $(top_builddir)/src/*/*/*.gc{da,no}
        rm -rf $(HTML_COVER_DIR)
        rm -rf $(top_builddir)/doc/doxygen
+       rm -rf $(TEST_NETWORK_ALL_LOG_DIR)

diff --git a/src/test/include.am b/src/test/include.am
index ef2fd4fb8eac247883927af6fadf5d5b119d5924..f7c0204832d604b97c2b084aaa3ce64a0d0c2fb5 100644 (file)
--- a/src/test/include.am
+++ b/src/test/include.am
@@ -8,6 +8,13 @@ endif
 TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
        src/test/test_workqueue src/test/test_keygen.sh $(TESTSCRIPTS)
 
+# These flavors are run using automake's test-driver and test-network.sh
+TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-min bridges+hs
+# only run if we can ping6 ::1 (localhost)
+TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min
+# only run if we can find a stable (or simply another) version of tor
+TEST_CHUTNEY_FLAVORS_MIXED = mixed
+
 ### This is a lovely feature, but it requires automake >= 1.12, and Tor
 ###  doesn't require that yet.  Below is a kludge to work around.
 ###