CI/runtests.pl: add param for dedicated curl to talk to APIs

This should make it possible to also report test failures
if our freshly build curl binary is not fully functional.

Reviewed-by: Daniel Stenberg
Closes #9360

diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml
index 8d1e5d7bcba1b12ec78636f0f5eebaa9a07da5c9..c21d49f724c810a879fd15a7755bfef4ca1cdaa1 100644 (file)
--- a/.azure-pipelines.yml
+++ b/.azure-pipelines.yml
@@ -95,7 +95,7 @@ stages:
       displayName: 'test'
       env:
         AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
-        TFLAGS: "-r $(tests)"
+        TFLAGS: "-ac /usr/bin/curl -r $(tests)"
 
 - stage: distcheck
   dependsOn: []
@@ -294,4 +294,4 @@ stages:
       displayName: 'test'
       env:
         AZURE_ACCESS_TOKEN: "$(System.AccessToken)"
-        TFLAGS: "!IDN !SCP ~612 ~1056 $(tests)"
+        TFLAGS: "-ac /usr/bin/curl.exe !IDN !SCP ~612 ~1056 $(tests)"

diff --git a/appveyor.yml b/appveyor.yml
index 522dcaf50d5a9a8205fc8d56d9f4a9820bdffed9..e04ca0fffe22024317337165c66995faa6c868a0 100644 (file)
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -306,15 +306,19 @@ build_script:
       ))
 
 test_script:
+    - if exist C:/msys64/usr/bin/curl.exe (
+        set ACURL=-ac %POSIX_PATH_PREFIX%/c/msys64/usr/bin/curl.exe )
+    - if exist C:/Windows/System32/curl.exe (
+        set ACURL=-ac %POSIX_PATH_PREFIX%/c/Windows/System32/curl.exe )
     - if %TESTING%==ON (
         if %BUILD_SYSTEM%==CMake (
-          set TFLAGS=%DISABLED_TESTS% &&
+          set TFLAGS=%ACURL% %DISABLED_TESTS% &&
           cmake --build . --config %PRJ_CFG% --target test-ci
         ) else (
         if %BUILD_SYSTEM%==autotools (
-          bash.exe -e -l -c "cd %POSIX_PATH_PREFIX%/c/projects/curl && make V=1 TFLAGS='%DISABLED_TESTS%' test-ci"
+          bash.exe -e -l -c "cd %POSIX_PATH_PREFIX%/c/projects/curl && make V=1 TFLAGS='%ACURL% %DISABLED_TESTS%' test-ci"
         ) else (
-          bash.exe -e -l -c "cd %POSIX_PATH_PREFIX%/c/projects/curl/tests && ./runtests.pl -a -p !flaky -r -rm %DISABLED_TESTS%"
+          bash.exe -e -l -c "cd %POSIX_PATH_PREFIX%/c/projects/curl/tests && ./runtests.pl -a -p !flaky -r -rm %ACURL% %DISABLED_TESTS%"
         ))
       )
 

diff --git a/tests/runtests.pl b/tests/runtests.pl
index 80370ba1ab4238a3bffdf367da866dc35a865dbd..7eeabfc3807694dbf29dc85d95d5966c12045f6f 100755 (executable)
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -175,10 +175,12 @@ my $SSHSRVSHA256 = "[uninitialized]"; # SHA256 of ssh server public key
 my $VERSION="";          # curl's reported version number
 
 my $srcdir = $ENV{'srcdir'} || '.';
-my $CURL="../src/curl".exe_ext('TOOL'); # what curl executable to run on the tests
+my $CURL="../src/curl".exe_ext('TOOL'); # what curl binary to run on the tests
 my $VCURL=$CURL;   # what curl binary to use to verify the servers with
                    # VCURL is handy to set to the system one when the one you
                    # just built hangs or crashes and thus prevent verification
+my $ACURL=$VCURL;  # what curl binary to use to talk to APIs (relevant for CI)
+                   # ACURL is handy to set to the system one for reliability
 my $DBGCURL=$CURL; #"../src/.libs/curl";  # alternative for debugging
 my $LOGDIR="log";
 my $TESTDIR="$srcdir/data";
@@ -3789,10 +3791,10 @@ sub singletest {
 
     # create test result in CI services
     if(azure_check_environment() && $AZURE_RUN_ID) {
-        $AZURE_RESULT_ID = azure_create_test_result($VCURL, $AZURE_RUN_ID, $testnum, $testname);
+        $AZURE_RESULT_ID = azure_create_test_result($ACURL, $AZURE_RUN_ID, $testnum, $testname);
     }
     elsif(appveyor_check_environment()) {
-        appveyor_create_test_result($VCURL, $testnum, $testname);
+        appveyor_create_test_result($ACURL, $testnum, $testname);
     }
 
     # remove test server commands file before servers are started/verified
@@ -5598,6 +5600,11 @@ while(@ARGV) {
         $VCURL="\"$ARGV[1]\"";
         shift @ARGV;
     }
+    elsif ($ARGV[0] eq "-ac") {
+        # use this curl only to talk to APIs (currently only CI test APIs)
+        $ACURL="\"$ARGV[1]\"";
+        shift @ARGV;
+    }
     elsif ($ARGV[0] eq "-d") {
         # have the servers display protocol output
         $debugprotocol=1;
@@ -5756,6 +5763,7 @@ while(@ARGV) {
         print <<EOHELP
 Usage: runtests.pl [options] [test selection(s)]
   -a       continue even if a test fails
+  -ac path use this curl only to talk to APIs (currently only CI test APIs)
   -am      automake style output PASS/FAIL: [number] [name]
   -c path  use this curl executable
   -d       display server debug info
@@ -6140,7 +6148,7 @@ sub displaylogs {
 #
 
 if(azure_check_environment()) {
-    $AZURE_RUN_ID = azure_create_test_run($VCURL);
+    $AZURE_RUN_ID = azure_create_test_run($ACURL);
     logmsg "Azure Run ID: $AZURE_RUN_ID\n" if ($verbose);
 }
 
@@ -6169,11 +6177,11 @@ foreach $testnum (@at) {
 
     # update test result in CI services
     if(azure_check_environment() && $AZURE_RUN_ID && $AZURE_RESULT_ID) {
-        $AZURE_RESULT_ID = azure_update_test_result($VCURL, $AZURE_RUN_ID, $AZURE_RESULT_ID, $testnum, $error,
+        $AZURE_RESULT_ID = azure_update_test_result($ACURL, $AZURE_RUN_ID, $AZURE_RESULT_ID, $testnum, $error,
                                                     $timeprepini{$testnum}, $timevrfyend{$testnum});
     }
     elsif(appveyor_check_environment()) {
-        appveyor_update_test_result($VCURL, $testnum, $error, $timeprepini{$testnum}, $timevrfyend{$testnum});
+        appveyor_update_test_result($ACURL, $testnum, $error, $timeprepini{$testnum}, $timevrfyend{$testnum});
     }
 
     if($error < 0) {
@@ -6218,7 +6226,7 @@ my $sofar = time() - $start;
 #
 
 if(azure_check_environment() && $AZURE_RUN_ID) {
-    $AZURE_RUN_ID = azure_update_test_run($VCURL, $AZURE_RUN_ID);
+    $AZURE_RUN_ID = azure_update_test_run($ACURL, $AZURE_RUN_ID);
 }
 
 # Tests done, stop the servers