From fa461b4eff52b413f88debf543b5350a6cef4724 Mon Sep 17 00:00:00 2001 From: Aki <75532970+AkiSakurai@users.noreply.github.com> Date: Wed, 7 Aug 2024 22:57:43 +0800 Subject: [PATCH] GHA/macos: enable HTTPS tests with stunnel - Install stunnel. - Regenerate certificates (as SecureTransport requires a validity period less than 398 days). - Restart server if it is unresponsive. - Do not hardcode the SHA-256 base64 public pinned key. - Ignore test 313 as SecureTransport does not support crl file. - Ignore tests 1631 and 1632 as SecureTransport is not yet able to shut down FTP over HTTPS gracefully. - Add a CMake target for generating certificates. Closes #14486 --- .github/workflows/macos.yml | 24 +++++++- tests/CMakeLists.txt | 1 + tests/certs/CMakeLists.txt | 48 +++++++++++++++ tests/certs/Makefile.am | 88 ++-------------------------- tests/certs/Makefile.inc | 104 +++++++++++++++++++++++++++++++++ tests/certs/scripts/genserv.sh | 2 +- tests/data/test2041 | 2 +- tests/data/test2087 | 2 +- tests/data/test417 | 26 +-------- tests/runner.pm | 53 +++++++++++------ tests/runtests.pl | 2 + tests/servers.pm | 3 +- tests/testutil.pm | 40 +++++++++++++ 13 files changed, 262 insertions(+), 133 deletions(-) create mode 100644 tests/certs/CMakeLists.txt create mode 100644 tests/certs/Makefile.inc diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 52edbc33a1..8cde2dae3d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -152,7 +152,7 @@ jobs: # while running the tests, for example # https://github.com/curl/curl/runs/4095721123?check_suite_focus=true run: | - echo libtool autoconf automake pkg-config libpsl libssh2 nghttp2 openssl ${{ matrix.install }} | xargs -Ix -n1 echo brew '"x"' > /tmp/Brewfile + echo libtool autoconf automake pkg-config libpsl libssh2 nghttp2 openssl stunnel ${{ matrix.install }} | xargs -Ix -n1 echo brew '"x"' > /tmp/Brewfile while [[ $? == 0 ]]; do for i in 1 2 3; do brew update && brew bundle install --no-lock --file /tmp/Brewfile && break 2 || { echo Error: wait to try again; sleep 10; } done; false Too many retries; done - name: 'brew unlink openssl' @@ -234,6 +234,12 @@ jobs: echo '::group::raw'; cat bld/lib/curl_config.h || true; echo '::endgroup::' cat bld/lib/curl_config.h | grep -F '#define' | sort || true + - name: 'build-cert' + if: contains(matrix.configure, '--with-secure-transport') + run: | + make -C bld/tests/certs clean-certs + make -C bld/tests/certs build-certs -j1 + - name: 'make' run: make -C bld V=1 @@ -259,6 +265,10 @@ jobs: TFLAGS+=' ~2402 ~2404' # non-SecureTransport + nghttp2 fi fi + if [[ '${{ matrix.configure }}' = *'--with-secure-transport'* ]]; then + TFLAGS+=' ~313' # SecureTransport does not support crl file + TFLAGS+=' ~1631 ~1632' # SecureTransport is not able to shutdown ftp over https gracefully yet + fi rm -f $HOME/.curlrc make -C bld V=1 test-ci @@ -303,7 +313,7 @@ jobs: steps: - name: 'brew install' run: | - echo libtool autoconf automake pkg-config libpsl libssh2 nghttp2 openssl ${{ matrix.build.install }} | xargs -Ix -n1 echo brew '"x"' > /tmp/Brewfile + echo libtool autoconf automake pkg-config libpsl libssh2 nghttp2 openssl stunnel ${{ matrix.build.install }} | xargs -Ix -n1 echo brew '"x"' > /tmp/Brewfile while [[ $? == 0 ]]; do for i in 1 2 3; do brew update && brew bundle install --no-lock --file /tmp/Brewfile && break 2 || { echo Error: wait to try again; sleep 10; } done; false Too many retries; done - name: 'brew unlink openssl' @@ -358,6 +368,12 @@ jobs: echo '::group::raw'; cat bld/lib/curl_config.h || true; echo '::endgroup::' cat bld/lib/curl_config.h | grep -F '#define' | sort || true + - name: 'build-cert' + if: contains(matrix.build.generate, '-DCURL_USE_SECTRANSP=ON') + run: | + make -C bld/tests/certs clean-certs + make -C bld/tests/certs build-certs -j1 + - name: 'cmake build' run: make -C bld VERBOSE=1 @@ -380,6 +396,10 @@ jobs: TFLAGS+=' ~2402 ~2404' # non-SecureTransport + nghttp2 fi fi + if [[ '${{ matrix.build.generate }}' = *'-DCURL_USE_SECTRANSP=ON'* ]]; then + TFLAGS+=' ~313' # SecureTransport does not support crl file + TFLAGS+=' ~1631 ~1632' # SecureTransport is not able to shutdown ftp over https gracefully yet + fi rm -f $HOME/.curlrc make -C bld test-ci diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 22a58c2927..87200fa312 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -37,6 +37,7 @@ add_subdirectory(http/clients) add_subdirectory(server) add_subdirectory(libtest) add_subdirectory(unit) +add_subdirectory(certs EXCLUDE_FROM_ALL) function(add_runtests _targetname _test_flags) # Use a special '$TFLAGS' placeholder as last argument which will be diff --git a/tests/certs/CMakeLists.txt b/tests/certs/CMakeLists.txt new file mode 100644 index 0000000000..f9011b13f6 --- /dev/null +++ b/tests/certs/CMakeLists.txt @@ -0,0 +1,48 @@ +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) Daniel Stenberg, , et al. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +# SPDX-License-Identifier: curl +# +########################################################################### +find_program(SH_EXECUTABLE "sh") +mark_as_advanced(SH_EXECUTABLE) +if(SH_EXECUTABLE) + # Get 'CERTCONFIGS', 'GENERATEDCERTS', 'SRPFILES' variables + transform_makefile_inc("Makefile.inc" "${CMAKE_CURRENT_BINARY_DIR}/Makefile.inc.cmake") + include("${CMAKE_CURRENT_BINARY_DIR}/Makefile.inc.cmake") + + add_custom_target(clean-certs + COMMAND ${CMAKE_COMMAND} -E remove ${GENERATEDCERTS} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) + + add_custom_target(build-certs + DEPENDS ${CERTCONFIGS} ${SRPFILES} + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/genroot.sh" EdelCurlRoot + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/genserv.sh" Server-localhost EdelCurlRoot + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/genserv.sh" Server-localhost.nn EdelCurlRoot + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/genserv.sh" Server-localhost0h EdelCurlRoot + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/genserv.sh" Server-localhost-firstSAN EdelCurlRoot + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/genserv.sh" Server-localhost-lastSAN EdelCurlRoot + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/genserv.sh" stunnel EdelCurlRoot + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/stunnel-sv.pem" "${CMAKE_CURRENT_SOURCE_DIR}/../stunnel.pem" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) +endif() diff --git a/tests/certs/Makefile.am b/tests/certs/Makefile.am index dc7deb818d..a2b45885af 100644 --- a/tests/certs/Makefile.am +++ b/tests/certs/Makefile.am @@ -25,89 +25,9 @@ AUTOMAKE_OPTIONS = foreign SUBDIRS = scripts -CERTCONFIGS = \ - EdelCurlRoot-ca.prm \ - EdelCurlRoot-ca.cnf \ - Server-localhost-sv.prm \ - Server-localhost.nn-sv.prm \ - Server-localhost0h-sv.prm \ - Server-localhost-firstSAN-sv.prm \ - Server-localhost-lastSAN-sv.prm \ - stunnel-sv.prm +include Makefile.inc -GENERATEDCERTS = \ - EdelCurlRoot-ca.cacert \ - EdelCurlRoot-ca.crt \ - EdelCurlRoot-ca.csr \ - EdelCurlRoot-ca.der \ - EdelCurlRoot-ca.key \ - Server-localhost-sv.crl \ - Server-localhost-sv.crt \ - Server-localhost-sv.csr \ - Server-localhost-sv.der \ - Server-localhost-sv.dhp \ - Server-localhost-sv.key \ - Server-localhost-sv.pem \ - Server-localhost-sv.pub.der \ - Server-localhost-sv.pub.pem \ - Server-localhost-sv.pubkey-pinned \ - Server-localhost.nn-sv.crl \ - Server-localhost.nn-sv.crt \ - Server-localhost.nn-sv.csr \ - Server-localhost.nn-sv.der \ - Server-localhost.nn-sv.dhp \ - Server-localhost.nn-sv.key \ - Server-localhost.nn-sv.pem \ - Server-localhost.nn-sv.pub.der \ - Server-localhost.nn-sv.pub.pem \ - Server-localhost.nn-sv.pubkey-pinned \ - Server-localhost0h-sv.crl \ - Server-localhost0h-sv.crt \ - Server-localhost0h-sv.csr \ - Server-localhost0h-sv.der \ - Server-localhost0h-sv.dhp \ - Server-localhost0h-sv.key \ - Server-localhost0h-sv.pem \ - Server-localhost0h-sv.pub.der \ - Server-localhost0h-sv.pub.pem \ - Server-localhost0h-sv.pubkey-pinned \ - Server-localhost-firstSAN-sv.crl \ - Server-localhost-firstSAN-sv.crt \ - Server-localhost-firstSAN-sv.csr \ - Server-localhost-firstSAN-sv.der \ - Server-localhost-firstSAN-sv.dhp \ - Server-localhost-firstSAN-sv.key \ - Server-localhost-firstSAN-sv.pem \ - Server-localhost-firstSAN-sv.pub.der \ - Server-localhost-firstSAN-sv.pub.pem \ - Server-localhost-firstSAN-sv.pubkey-pinned \ - Server-localhost-lastSAN-sv.crl \ - Server-localhost-lastSAN-sv.crt \ - Server-localhost-lastSAN-sv.csr \ - Server-localhost-lastSAN-sv.der \ - Server-localhost-lastSAN-sv.dhp \ - Server-localhost-lastSAN-sv.key \ - Server-localhost-lastSAN-sv.pem \ - Server-localhost-lastSAN-sv.pub.der \ - Server-localhost-lastSAN-sv.pub.pem \ - Server-localhost-lastSAN-sv.pubkey-pinned \ - stunnel-sv.crl \ - stunnel-sv.crt \ - stunnel-sv.csr \ - stunnel-sv.der \ - stunnel-sv.dhp \ - stunnel-sv.key \ - stunnel-sv.pem \ - stunnel-sv.der \ - stunnel-sv.pub.der \ - stunnel-sv.pub.pem \ - stunnel-sv.pubkey-pinned - -SRPFILES = \ - srp-verifier-conf \ - srp-verifier-db - -EXTRA_DIST = $(CERTCONFIGS) $(GENERATEDCERTS) $(SRPFILES) +EXTRA_DIST = $(CERTCONFIGS) $(GENERATEDCERTS) $(SRPFILES) CMakeLists.txt # Rebuild the certificates @@ -117,7 +37,7 @@ clean-certs: build-certs: $(srcdir)/EdelCurlRoot-ca.cacert $(srcdir)/Server-localhost-sv.pem \ $(srcdir)/Server-localhost.nn-sv.pem $(srcdir)/Server-localhost0h-sv.pem \ $(srcdir)/Server-localhost-firstSAN-sv.pem $(srcdir)/Server-localhost-lastSAN-sv.pem \ - $(srcdir)/stunnel-sv.pem ../stunnel.pem + $(srcdir)/stunnel-sv.pem $(srcdir)/../stunnel.pem $(srcdir)/EdelCurlRoot-ca.cacert: cd $(srcdir); scripts/genroot.sh EdelCurlRoot @@ -140,5 +60,5 @@ $(srcdir)/Server-localhost-lastSAN-sv.pem: $(srcdir)/EdelCurlRoot-ca.cacert $(srcdir)/stunnel-sv.pem: $(srcdir)/EdelCurlRoot-ca.cacert cd $(srcdir); scripts/genserv.sh stunnel EdelCurlRoot -../stunnel.pem: $(srcdir)/stunnel-sv.pem + $(srcdir)/../stunnel.pem: $(srcdir)/stunnel-sv.pem cp $< $@ diff --git a/tests/certs/Makefile.inc b/tests/certs/Makefile.inc new file mode 100644 index 0000000000..032baec249 --- /dev/null +++ b/tests/certs/Makefile.inc @@ -0,0 +1,104 @@ +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) Daniel Stenberg, , et al. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +# SPDX-License-Identifier: curl +# +########################################################################### +CERTCONFIGS = \ + EdelCurlRoot-ca.prm \ + EdelCurlRoot-ca.cnf \ + Server-localhost-sv.prm \ + Server-localhost.nn-sv.prm \ + Server-localhost0h-sv.prm \ + Server-localhost-firstSAN-sv.prm \ + Server-localhost-lastSAN-sv.prm \ + stunnel-sv.prm + +GENERATEDCERTS = \ + EdelCurlRoot-ca.cacert \ + EdelCurlRoot-ca.crt \ + EdelCurlRoot-ca.csr \ + EdelCurlRoot-ca.der \ + EdelCurlRoot-ca.key \ + Server-localhost-sv.crl \ + Server-localhost-sv.crt \ + Server-localhost-sv.csr \ + Server-localhost-sv.der \ + Server-localhost-sv.dhp \ + Server-localhost-sv.key \ + Server-localhost-sv.pem \ + Server-localhost-sv.pub.der \ + Server-localhost-sv.pub.pem \ + Server-localhost-sv.pubkey-pinned \ + Server-localhost.nn-sv.crl \ + Server-localhost.nn-sv.crt \ + Server-localhost.nn-sv.csr \ + Server-localhost.nn-sv.der \ + Server-localhost.nn-sv.dhp \ + Server-localhost.nn-sv.key \ + Server-localhost.nn-sv.pem \ + Server-localhost.nn-sv.pub.der \ + Server-localhost.nn-sv.pub.pem \ + Server-localhost.nn-sv.pubkey-pinned \ + Server-localhost0h-sv.crl \ + Server-localhost0h-sv.crt \ + Server-localhost0h-sv.csr \ + Server-localhost0h-sv.der \ + Server-localhost0h-sv.dhp \ + Server-localhost0h-sv.key \ + Server-localhost0h-sv.pem \ + Server-localhost0h-sv.pub.der \ + Server-localhost0h-sv.pub.pem \ + Server-localhost0h-sv.pubkey-pinned \ + Server-localhost-firstSAN-sv.crl \ + Server-localhost-firstSAN-sv.crt \ + Server-localhost-firstSAN-sv.csr \ + Server-localhost-firstSAN-sv.der \ + Server-localhost-firstSAN-sv.dhp \ + Server-localhost-firstSAN-sv.key \ + Server-localhost-firstSAN-sv.pem \ + Server-localhost-firstSAN-sv.pub.der \ + Server-localhost-firstSAN-sv.pub.pem \ + Server-localhost-firstSAN-sv.pubkey-pinned \ + Server-localhost-lastSAN-sv.crl \ + Server-localhost-lastSAN-sv.crt \ + Server-localhost-lastSAN-sv.csr \ + Server-localhost-lastSAN-sv.der \ + Server-localhost-lastSAN-sv.dhp \ + Server-localhost-lastSAN-sv.key \ + Server-localhost-lastSAN-sv.pem \ + Server-localhost-lastSAN-sv.pub.der \ + Server-localhost-lastSAN-sv.pub.pem \ + Server-localhost-lastSAN-sv.pubkey-pinned \ + stunnel-sv.crl \ + stunnel-sv.crt \ + stunnel-sv.csr \ + stunnel-sv.der \ + stunnel-sv.dhp \ + stunnel-sv.key \ + stunnel-sv.pem \ + stunnel-sv.der \ + stunnel-sv.pub.der \ + stunnel-sv.pub.pem \ + stunnel-sv.pubkey-pinned + +SRPFILES = \ + srp-verifier-conf \ + srp-verifier-db diff --git a/tests/certs/scripts/genserv.sh b/tests/certs/scripts/genserv.sh index ce184e937d..d1303e2feb 100755 --- a/tests/certs/scripts/genserv.sh +++ b/tests/certs/scripts/genserv.sh @@ -37,7 +37,7 @@ HOME=$(pwd) cd "$HOME" KEYSIZE=2048 -DURATION=3000 +DURATION=300 # The -sha256 option was introduced in OpenSSL 1.0.1 DIGESTALGO=-sha256 diff --git a/tests/data/test2041 b/tests/data/test2041 index b7b461e0d0..f7d0600645 100644 --- a/tests/data/test2041 +++ b/tests/data/test2041 @@ -35,7 +35,7 @@ https Server-localhost-sv.pem simple HTTPS GET with base64-sha256 public key pinning ---cacert %SRCDIR/certs/EdelCurlRoot-ca.crt --pinnedpubkey sha256//AAUDLk4c98xcFUDvA9i/MnA9HuO03IPi15r+Cx9OXnc= https://localhost:%HTTPSPORT/%TESTNUMBER +--cacert %SRCDIR/certs/EdelCurlRoot-ca.crt --pinnedpubkey sha256//%sha256b64file[%SRCDIR/certs/Server-localhost-sv.pub.der]sha256b64file% https://localhost:%HTTPSPORT/%TESTNUMBER # Ensure that we're running on localhost because we're checking the host name diff --git a/tests/data/test2087 b/tests/data/test2087 index eb8d3bad89..13efe48b65 100644 --- a/tests/data/test2087 +++ b/tests/data/test2087 @@ -39,7 +39,7 @@ simple HTTPS GET with base64-sha256 public key pinning (Schannel variant) CURL_SSL_BACKEND=schannel ---cacert %SRCDIR/certs/EdelCurlRoot-ca.crt --pinnedpubkey sha256//AAUDLk4c98xcFUDvA9i/MnA9HuO03IPi15r+Cx9OXnc= --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER +--cacert %SRCDIR/certs/EdelCurlRoot-ca.crt --pinnedpubkey sha256//%sha256b64file[%SRCDIR/certs/Server-localhost-sv.pub.der]sha256b64file% --ssl-revoke-best-effort https://localhost:%HTTPSPORT/%TESTNUMBER # Ensure that we're running on localhost because we're checking the host name diff --git a/tests/data/test417 b/tests/data/test417 index 9061f8332b..50f4b479b4 100644 --- a/tests/data/test417 +++ b/tests/data/test417 @@ -51,31 +51,7 @@ s/^(.*):(.*)[\r\n]*// nomnom ------BEGIN CERTIFICATE----- -MIIERDCCAyygAwIBAgIGDzR1UZ/TMA0GCSqGSIb3DQEBCwUAMGgxCzAJBgNVBAYT -Ak5OMTEwLwYDVQQKDChFZGVsIEN1cmwgQXJjdGljIElsbHVkaXVtIFJlc2VhcmNo -IENsb3VkMSYwJAYDVQQDDB1Ob3J0aGVybiBOb3doZXJlIFRydXN0IEFuY2hvcjAe -Fw0yMjEyMjMxMjIxMzlaFw0zMTAzMTExMjIxMzlaMFQxCzAJBgNVBAYTAk5OMTEw -LwYDVQQKDChFZGVsIEN1cmwgQXJjdGljIElsbHVkaXVtIFJlc2VhcmNoIENsb3Vk -MRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK -AoIBAQCrCrAD0Hb+Xs4V3mHV45FvfNa7yiaOeL4mNdGmWfHVPFU+CSzsoNSvDjxa -orWweFGVYoCAcchOn1lZk0ASsqnOss0Xi58n8+PPI3gG0gYjX5sg7EJ3Zq2kXoK0 -TZRy6hNkcvzLgyzXoYv1LkzTwYiyyJgZX++Y/GKAs2fMHyP8XzjNgm4tltk1k/4p -omllwN9Fqz+sFxgAgEq3ybq4Xym7xKwWl8xXNBDJNmVsPtiJRcilQoR8Xs0a6PE+ -VbMhD9A2E/LEL7lzQfqHqtxE1mSW5FpQ+Uqf4KLnafStWs86IOWnCeLP6BmhAK6o -uyICNFyzz7UkTHa/renxuNOGun2TAgMBAAGjggEGMIIBAjAUBgNVHREEDTALggls -b2NhbGhvc3QwCwYDVR0PBAQDAgOoMBMGA1UdJQQMMAoGCCsGAQUFBwMBMB0GA1Ud -DgQWBBScl7A9s1Cx9tRx4uvLgOqTfJjMcjAfBgNVHSMEGDAWgBSHy7EzLsFnfnHj -5StMTaSzbtJbqTAJBgNVHRMEAjAAMEMGCCsGAQUFBwEBBDcwNTAzBggrBgEFBQcw -AoYnaHR0cDovL3Rlc3QuY3VybC5zZS9jYS9FZGVsQ3VybFJvb3QuY2VyMDgGA1Ud -HwQxMC8wLaAroCmGJ2h0dHA6Ly90ZXN0LmN1cmwuc2UvY2EvRWRlbEN1cmxSb290 -LmNybDANBgkqhkiG9w0BAQsFAAOCAQEAC4rtaof6cRWIJViFG0oJv0MANZN4DXIU -MFHik4Oh2hsvqTGut8dMcsJeMiTxlpNw1T+1hYATdTLPDvhdxKIphEMsdYEmEmqg -y3tXwZJ4hQj6ZFDCe4MCTXkTvGFkTbhr1fGEaxJcaZCtQEfA7d3qimZ+h4UZqonT -PAhyCKFNY2BbmxeeABKhAFLKeAGIGMftW8fk2eu9P6+SUz/+WFcN/PR7e6JP6blc -taRSULRWWkSO2dDt3o9+rBxYdluoecmVq4Ud20wTgkqlQRsp9dOW34DRHgB9ujWU -V4HhCCqBaxwwUDcBGg4mT2vtyVAXNyszP2j+xvAhjOeyeVXyQr0vsA== ------END CERTIFICATE----- +%strippemfile[%SRCDIR/certs/stunnel-sv.crt]strippemfile% diff --git a/tests/runner.pm b/tests/runner.pm index eeb84bcad4..105009d990 100644 --- a/tests/runner.pm +++ b/tests/runner.pm @@ -103,6 +103,8 @@ use testutil qw( runclient shell_quote subbase64 + subsha256base64file + substrippemfile subnewlines ); use valgrind; @@ -364,6 +366,8 @@ sub prepro { } subvariables(\$s, $testnum, "%"); subbase64(\$s); + subsha256base64file(\$s); + substrippemfile(\$s); subnewlines(0, \$s) if($data_crlf); push @out, $s; } @@ -401,6 +405,32 @@ sub logslocked { return @locks; } +####################################################################### +# Wait log locks to be unlocked +# +sub waitlockunlock { + # If a server logs advisor read lock file exists, it is an indication + # that the server has not yet finished writing out all its log files, + # including server request log files used for protocol verification. + # So, if the lock file exists the script waits here a certain amount + # of time until the server removes it, or the given time expires. + my $serverlogslocktimeout = shift; + + if($serverlogslocktimeout) { + my $lockretry = $serverlogslocktimeout * 20; + my @locks; + while((@locks = logslocked()) && $lockretry--) { + portable_sleep(0.05); + } + if(($lockretry < 0) && + ($serverlogslocktimeout >= $defserverlogslocktimeout)) { + logmsg "Warning: server logs lock timeout ", + "($serverlogslocktimeout seconds) expired (locks: " . + join(", ", @locks) . ")\n"; + } + } +} + ####################################################################### # Memory allocation test and failure torture testing. # @@ -1027,11 +1057,6 @@ sub singletest_clean { } } - # If a server logs advisor read lock file exists, it is an indication - # that the server has not yet finished writing out all its log files, - # including server request log files used for protocol verification. - # So, if the lock file exists the script waits here a certain amount - # of time until the server removes it, or the given time expires. my $serverlogslocktimeout = $defserverlogslocktimeout; my %cmdhash = getpartattr("client", "command"); if($cmdhash{'timeout'}) { @@ -1040,19 +1065,8 @@ sub singletest_clean { $serverlogslocktimeout = $1 if($1 >= 0); } } - if($serverlogslocktimeout) { - my $lockretry = $serverlogslocktimeout * 20; - my @locks; - while((@locks = logslocked()) && $lockretry--) { - portable_sleep(0.05); - } - if(($lockretry < 0) && - ($serverlogslocktimeout >= $defserverlogslocktimeout)) { - logmsg "Warning: server logs lock timeout ", - "($serverlogslocktimeout seconds) expired (locks: " . - join(", ", @locks) . ")\n"; - } - } + + waitlockunlock($serverlogslocktimeout); # Test harness ssh server does not have this synchronization mechanism, # this implies that some ssh server based tests might need a small delay @@ -1148,6 +1162,9 @@ sub runner_test_preprocess { # Start the servers needed to run this test case my ($why, $error) = singletest_startservers($testnum, \%testtimings); + # make sure no locks left for responsive test + waitlockunlock($defserverlogslocktimeout); + if(!$why) { ############################################################### diff --git a/tests/runtests.pl b/tests/runtests.pl index 64f6f941a7..c0c7188112 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -1477,6 +1477,8 @@ sub singletest_check { } for my $line (@upload) { subbase64(\$line); + subsha256base64file(\$line); + substrippemfile(\$line); } # verify uploaded data diff --git a/tests/servers.pm b/tests/servers.pm index 5ef0d2fa00..dcf1d51587 100644 --- a/tests/servers.pm +++ b/tests/servers.pm @@ -2432,8 +2432,9 @@ sub startservers { } } elsif($what eq "http") { - if($torture && $run{'http'} && + if($run{'http'} && !responsive_http_server("http", $verbose, 0, protoport('http'))) { + logmsg "* restarting unresponsive HTTP server\n"; if(stopserver('http')) { return ("failed stopping unresponsive HTTP server", 3); } diff --git a/tests/testutil.pm b/tests/testutil.pm index 36bbe513cd..4d68c5d280 100644 --- a/tests/testutil.pm +++ b/tests/testutil.pm @@ -40,6 +40,8 @@ BEGIN { shell_quote subbase64 subnewlines + subsha256base64file + substrippemfile ); our @EXPORT_OK = qw( @@ -48,6 +50,7 @@ BEGIN { ); } +use Digest::SHA qw(sha256); use MIME::Base64; use globalconfig qw( @@ -215,4 +218,41 @@ sub shell_quote { return $s; } +sub get_sha256_base64 { + my ($file_path) = @_; + return encode_base64(sha256(do { local $/; open my $fh, '<:raw', $file_path or die $!; <$fh> }), ""); +} + +sub subsha256base64file { + my ($thing) = @_; + + # SHA-256 base64 + while ($$thing =~ s/%sha256b64file\[(.*?)\]sha256b64file%/%%SHA256B64FILE%%/i) { + my $file_path = $1; + $file_path =~ s/%([0-9A-Fa-f]{2})/chr(hex($1))/eg; + my $hash_b64 = get_sha256_base64($file_path); + $$thing =~ s/%%SHA256B64FILE%%/$hash_b64/; + } +} + +sub get_file_content { + my ($file_path) = @_; + my $content = do { local $/; open my $fh, '<', $file_path or die $!; <$fh> }; + $content =~ s/(^|-----END .*?-----[\r\n]?)(.*?)(-----BEGIN .*?-----|$)/$1$3/gs; + $content =~ s/\r\n/\n/g; + chomp($content); + return $content; +} + +sub substrippemfile { + my ($thing) = @_; + + # File content substitution + while ($$thing =~ s/%strippemfile\[(.*?)\]strippemfile%/%%FILE%%/i) { + my $file_path = $1; + $file_path =~ s/%([0-9A-Fa-f]{2})/chr(hex($1))/eg; + my $file_content = get_file_content($file_path); + $$thing =~ s/%%FILE%%/$file_content/; + } +} 1; -- 2.47.3