tool_writeout: add URL component variables

Output specific components from the used URL. The following variables
are added for this purpose:

  url.scheme, url.user, url.password, url.options, url.host, url.port,
  url.path, url.query, url.fragment, url.zoneid

Add the following for outputting parts of the "effective URL":

 urle.scheme, urle.user, urle.password, urle.options, urle.host, urle.port,
 urle.path, urle.query, urle.fragment, urle.zoneid

Added test 423 and 424 to verify.

Closes #10853

diff --git a/docs/cmdline-opts/write-out.d b/docs/cmdline-opts/write-out.d
index bb26528b5d1d00a386e3b30ce75f83de6f1abab5..5a872007e733d1e48c4276e9161faec6e19dae7b 100644 (file)
--- a/docs/cmdline-opts/write-out.d
+++ b/docs/cmdline-opts/write-out.d
@@ -209,6 +209,70 @@ The total time, in seconds, that the full operation lasted.
 .B url
 The URL that was fetched. (Added in 7.75.0)
 .TP
+.B url.scheme
+The scheme part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.user
+The user part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.password
+The password part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.options
+The options part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.host
+The host part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.port
+The port number of the URL that was fetched. If no port number was specified,
+but the URL scheme is known, that scheme's default port number is
+shown. (Added in 8.1.0)
+.TP
+.B url.path
+The path part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.query
+The query part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.fragment
+The fragment part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B url.zoneid
+The zoneid part of the URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.scheme
+The scheme part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.user
+The user part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.password
+The password part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.options
+The options part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.host
+The host part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.port
+The port number of the effective (last) URL that was fetched. If no port
+number was specified, but the URL scheme is known, that scheme's default port
+number is shown. (Added in 8.1.0)
+.TP
+.B urle.path
+The path part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.query
+The query part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.fragment
+The fragment part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
+.B urle.zoneid
+The zoneid part of the effective (last) URL that was fetched. (Added in 8.1.0)
+.TP
 .B urlnum
 The URL index number of this transfer, 0-indexed. De-globbed URLs share the
 same index number as the origin globbed URL. (Added in 7.75.0)

diff --git a/src/tool_writeout.c b/src/tool_writeout.c
index a2e3c39bbf5c435ed561df95f00575cc65f61073..ddf7deaa7ea54df73065fa8a7cf61442271e76d3 100644 (file)
--- a/src/tool_writeout.c
+++ b/src/tool_writeout.c
@@ -123,6 +123,26 @@ static const struct writeoutvar variables[] = {
    writeTime},
   {"time_total", VAR_TOTAL_TIME, CURLINFO_TOTAL_TIME_T, writeTime},
   {"url", VAR_INPUT_URL, CURLINFO_NONE, writeString},
+  {"url.scheme", VAR_INPUT_URLSCHEME, CURLINFO_NONE, writeString},
+  {"url.user", VAR_INPUT_URLUSER, CURLINFO_NONE, writeString},
+  {"url.password", VAR_INPUT_URLPASSWORD, CURLINFO_NONE, writeString},
+  {"url.options", VAR_INPUT_URLOPTIONS, CURLINFO_NONE, writeString},
+  {"url.host", VAR_INPUT_URLHOST, CURLINFO_NONE, writeString},
+  {"url.port", VAR_INPUT_URLPORT, CURLINFO_NONE, writeString},
+  {"url.path", VAR_INPUT_URLPATH, CURLINFO_NONE, writeString},
+  {"url.query", VAR_INPUT_URLQUERY, CURLINFO_NONE, writeString},
+  {"url.fragment", VAR_INPUT_URLFRAGMENT, CURLINFO_NONE, writeString},
+  {"url.zoneid", VAR_INPUT_URLZONEID, CURLINFO_NONE, writeString},
+  {"urle.scheme", VAR_INPUT_URLESCHEME, CURLINFO_NONE, writeString},
+  {"urle.user", VAR_INPUT_URLEUSER, CURLINFO_NONE, writeString},
+  {"urle.password", VAR_INPUT_URLEPASSWORD, CURLINFO_NONE, writeString},
+  {"urle.options", VAR_INPUT_URLEOPTIONS, CURLINFO_NONE, writeString},
+  {"urle.host", VAR_INPUT_URLEHOST, CURLINFO_NONE, writeString},
+  {"urle.port", VAR_INPUT_URLEPORT, CURLINFO_NONE, writeString},
+  {"urle.path", VAR_INPUT_URLEPATH, CURLINFO_NONE, writeString},
+  {"urle.query", VAR_INPUT_URLEQUERY, CURLINFO_NONE, writeString},
+  {"urle.fragment", VAR_INPUT_URLEFRAGMENT, CURLINFO_NONE, writeString},
+  {"urle.zoneid", VAR_INPUT_URLEZONEID, CURLINFO_NONE, writeString},
   {"url_effective", VAR_EFFECTIVE_URL, CURLINFO_EFFECTIVE_URL, writeString},
   {"urlnum", VAR_URLNUM, CURLINFO_NONE, writeLong},
   {NULL, VAR_NONE, CURLINFO_NONE, NULL}
@@ -165,12 +185,94 @@ static int writeTime(FILE *stream, const struct writeoutvar *wovar,
   return 1; /* return 1 if anything was written */
 }
 
+static int urlpart(struct per_transfer *per, writeoutid vid,
+                   const char **contentp)
+{
+  CURLU *uh = curl_url();
+  int rc = 0;
+  if(uh) {
+    CURLUPart cpart = CURLUPART_HOST;
+    char *part = NULL;
+    const char *url = NULL;
+
+    if(vid >= VAR_INPUT_URLEHOST) {
+      if(curl_easy_getinfo(per->curl, CURLINFO_EFFECTIVE_URL, &url))
+        rc = 5;
+    }
+    else
+      url = per->this_url;
+
+    if(!rc) {
+      switch(vid) {
+      case VAR_INPUT_URLSCHEME:
+      case VAR_INPUT_URLESCHEME:
+        cpart = CURLUPART_SCHEME;
+        break;
+      case VAR_INPUT_URLUSER:
+      case VAR_INPUT_URLEUSER:
+        cpart = CURLUPART_USER;
+        break;
+      case VAR_INPUT_URLPASSWORD:
+      case VAR_INPUT_URLEPASSWORD:
+        cpart = CURLUPART_PASSWORD;
+        break;
+      case VAR_INPUT_URLOPTIONS:
+      case VAR_INPUT_URLEOPTIONS:
+        cpart = CURLUPART_OPTIONS;
+        break;
+      case VAR_INPUT_URLHOST:
+      case VAR_INPUT_URLEHOST:
+        cpart = CURLUPART_HOST;
+        break;
+      case VAR_INPUT_URLPORT:
+      case VAR_INPUT_URLEPORT:
+        cpart = CURLUPART_PORT;
+        break;
+      case VAR_INPUT_URLPATH:
+      case VAR_INPUT_URLEPATH:
+        cpart = CURLUPART_PATH;
+        break;
+      case VAR_INPUT_URLQUERY:
+      case VAR_INPUT_URLEQUERY:
+        cpart = CURLUPART_QUERY;
+        break;
+      case VAR_INPUT_URLFRAGMENT:
+      case VAR_INPUT_URLEFRAGMENT:
+        cpart = CURLUPART_FRAGMENT;
+        break;
+      case VAR_INPUT_URLZONEID:
+      case VAR_INPUT_URLEZONEID:
+        cpart = CURLUPART_ZONEID;
+        break;
+      default:
+        /* not implemented */
+        rc = 4;
+        break;
+      }
+    }
+    if(!rc && curl_url_set(uh, CURLUPART_URL, url,
+                           CURLU_GUESS_SCHEME|CURLU_NON_SUPPORT_SCHEME))
+      rc = 2;
+
+    if(!rc && curl_url_get(uh, cpart, &part, CURLU_DEFAULT_PORT))
+      rc = 3;
+
+    if(!rc && part)
+      *contentp = part;
+    curl_url_cleanup(uh);
+  }
+  else
+    return 1;
+  return rc;
+}
+
 static int writeString(FILE *stream, const struct writeoutvar *wovar,
                        struct per_transfer *per, CURLcode per_result,
                        bool use_json)
 {
   bool valid = false;
   const char *strinfo = NULL;
+  const char *freestr = NULL;
   struct dynbuf buf;
   curlx_dyn_init(&buf, 256*1024);
 
@@ -262,6 +364,33 @@ static int writeString(FILE *stream, const struct writeoutvar *wovar,
         valid = true;
       }
       break;
+    case VAR_INPUT_URLSCHEME:
+    case VAR_INPUT_URLUSER:
+    case VAR_INPUT_URLPASSWORD:
+    case VAR_INPUT_URLOPTIONS:
+    case VAR_INPUT_URLHOST:
+    case VAR_INPUT_URLPORT:
+    case VAR_INPUT_URLPATH:
+    case VAR_INPUT_URLQUERY:
+    case VAR_INPUT_URLFRAGMENT:
+    case VAR_INPUT_URLZONEID:
+    case VAR_INPUT_URLESCHEME:
+    case VAR_INPUT_URLEUSER:
+    case VAR_INPUT_URLEPASSWORD:
+    case VAR_INPUT_URLEOPTIONS:
+    case VAR_INPUT_URLEHOST:
+    case VAR_INPUT_URLEPORT:
+    case VAR_INPUT_URLEPATH:
+    case VAR_INPUT_URLEQUERY:
+    case VAR_INPUT_URLEFRAGMENT:
+    case VAR_INPUT_URLEZONEID:
+      if(per->this_url) {
+        if(!urlpart(per, wovar->id, &strinfo)) {
+          freestr = strinfo;
+          valid = true;
+        }
+      }
+      break;
     default:
       DEBUGASSERT(0);
       break;
@@ -281,6 +410,7 @@ static int writeString(FILE *stream, const struct writeoutvar *wovar,
     if(use_json)
       fprintf(stream, "\"%s\":null", wovar->name);
   }
+  curl_free((char *)freestr);
 
   curlx_dyn_free(&buf);
   return 1; /* return 1 if anything was written */

diff --git a/src/tool_writeout.h b/src/tool_writeout.h
index fac7bdd19690fe71387abdf9e4de21dc6327e3a2..25895bb440465e398ebda84f15d4f271b1da98e7 100644 (file)
--- a/src/tool_writeout.h
+++ b/src/tool_writeout.h
@@ -44,6 +44,27 @@ typedef enum {
   VAR_HTTP_CODE_PROXY,
   VAR_HTTP_VERSION,
   VAR_INPUT_URL,
+  VAR_INPUT_URLSCHEME,
+  VAR_INPUT_URLUSER,
+  VAR_INPUT_URLPASSWORD,
+  VAR_INPUT_URLOPTIONS,
+  VAR_INPUT_URLHOST,
+  VAR_INPUT_URLPORT,
+  VAR_INPUT_URLPATH,
+  VAR_INPUT_URLQUERY,
+  VAR_INPUT_URLFRAGMENT,
+  VAR_INPUT_URLZONEID,
+  /* the same ones again for url *effective* */
+  VAR_INPUT_URLESCHEME,
+  VAR_INPUT_URLEUSER,
+  VAR_INPUT_URLEPASSWORD,
+  VAR_INPUT_URLEOPTIONS,
+  VAR_INPUT_URLEHOST,
+  VAR_INPUT_URLEPORT,
+  VAR_INPUT_URLEPATH,
+  VAR_INPUT_URLEQUERY,
+  VAR_INPUT_URLEFRAGMENT,
+  VAR_INPUT_URLEZONEID,
   VAR_JSON,
   VAR_LOCAL_IP,
   VAR_LOCAL_PORT,

diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index ec51f2b1dc3cc79a51ba452a71ff2a760ae87e90..ca5903fe7d73f6f6566d4dcdffcca05c4d1c1512 100644 (file)
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -68,7 +68,7 @@ test380 test381 test383 test384 test385 test386 test387 test388 test389 \
 test390 test391 test392 test393 test394 test395 test396 test397 test398 \
 test399 test400 test401 test402 test403 test404 test405 test406 test407 \
 test408 test409 test410 test411 test412 test413 test414 test415 test416 \
-test417 test418 test419 test420 test421 test422 \
+test417 test418 test419 test420 test421 test422 test423 test424 \
 \
 test430 test431 test432 test433 test434 test435 test436 \
 \

diff --git a/tests/data/test423 b/tests/data/test423
new file mode 100644 (file)
index 0000000..6c6e725
--- /dev/null
+++ b/tests/data/test423
@@ -0,0 +1,51 @@
+<testcase>
+<info>
+<keywords>
+-w
+--write-out
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes" nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 0
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+-w with url.* variables
+</name>
+<command option="no-include">
+"http://uuuu:pppp@%HOSTIP:%HTTPPORT/%TESTNUMBER?qqqq#ffff" "h55p://hello2000:1/%TESTNUMBER?qqqq#ffff" "local host" "http://u22u:p22p@%HOSTIP:%HTTPPORT/%TESTNUMBER?qqqq#ffff" -w '%{url.host}+%{url.path}+%{url.scheme}+%{url.user}+%{url.password}+%{url.port}+%{url.query}+%{url.fragment}\n'
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<stdout>
+%HOSTIP+/%TESTNUMBER+http+uuuu+pppp+%HTTPPORT+qqqq+ffff
+hello2000+/%TESTNUMBER+h55p+++1+qqqq+ffff
++++++++
+%HOSTIP+/%TESTNUMBER+http+u22u+p22p+%HTTPPORT+qqqq+ffff
+</stdout>
+</verify>
+</testcase>

diff --git a/tests/data/test424 b/tests/data/test424
new file mode 100644 (file)
index 0000000..36ba8c4
--- /dev/null
+++ b/tests/data/test424
@@ -0,0 +1,68 @@
+<testcase>
+<info>
+<keywords>
+-w
+--write-out
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes" nocheck="yes">
+HTTP/1.1 301 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 0
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+Location: http://anotherhost.example:2023/%TESTNUMBER0002?moo.html
+
+</data>
+<data2 crlf="yes" nocheck="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 0
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+</data2>
+</reply>
+
+#
+# Client-side
+<client>
+<features>
+proxy
+</features>
+<server>
+http
+</server>
+<name>
+-w with urle.* variables
+</name>
+<command option="no-include">
+"http://uuuu:pppp@%HOSTIP:%HTTPPORT/%TESTNUMBER?qqqq#ffff" "h55p://hello2000:1/%TESTNUMBER?qqqq#ffff" "local host" "http://u22u:p22p@%HOSTIP:%HTTPPORT/%TESTNUMBER?qqqq#ffff" -w '%{urle.host}+%{urle.path}+%{urle.scheme}+%{urle.user}+%{urle.password}+%{urle.port}+%{urle.query}+%{urle.fragment}\n' -x http://%HOSTIP:%HTTPPORT/ -L
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<stdout>
+anotherhost.example+/%TESTNUMBER0002+http+uuuu+pppp+2023+moo.html+
+hello2000+/%TESTNUMBER+h55p+++1+qqqq+ffff
++++++++
+anotherhost.example+/%TESTNUMBER0002+http+u22u+p22p+2023+moo.html+
+</stdout>
+</verify>
+</testcase>

diff --git a/tests/data/test970 b/tests/data/test970
index 74c7cb3d6d2150a70ee95012869a814c2897f3c2..3ff3a635595585e50fae9cdf7206b4c29316a9dc 100644 (file)
--- a/tests/data/test970
+++ b/tests/data/test970
@@ -59,7 +59,7 @@ Accept: */*
 \r
 </protocol>
 <stdout nonewline="yes">
-{"certs":"","content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"%LOGDIR/out%TESTNUMBER","ftp_entry_path":null,"http_code":200,"http_connect":0,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_certs":0,"num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","url_effective":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","urlnum":0,"curl_version":"curl-unit-test-fake-version"}
+{"certs":"","content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"%LOGDIR/out%TESTNUMBER","ftp_entry_path":null,"http_code":200,"http_connect":0,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_certs":0,"num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","url.scheme":"http","url.user":null,"url.password":null,"url.options":null,"url.host":"%HOSTIP","url.port":"%HTTPPORT","url.path":"/%TESTNUMBER","url.query":null,"url.fragment":null,"url.zoneid":null,"urle.scheme":"http","urle.user":null,"urle.password":null,"urle.options":null,"urle.host":"%HOSTIP","urle.port":"%HTTPPORT","urle.path":"/%TESTNUMBER","urle.query":null,"urle.fragment":null,"urle.zoneid":null,"url_effective":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","urlnum":0,"curl_version":"curl-unit-test-fake-version"}
 </stdout>
 </verify>
 </testcase>

diff --git a/tests/data/test972 b/tests/data/test972
index 7ab134806ae94bab6d6677f7d4a692638393e905..fe8ee6f3181b673f8b2c4cf84f07f6c7b2ee82cb 100644 (file)
--- a/tests/data/test972
+++ b/tests/data/test972
@@ -60,7 +60,7 @@ Accept: */*
 \r
 </protocol>
 <stdout mode="text">
-{"certs":"","content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"%LOGDIR/out972","ftp_entry_path":null,"http_code":200,"http_connect":0,"http_version":"1.1","local_ip":"%HOSTIP","local_port":13,"method":"GET","num_certs":0,"num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","url_effective":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","urlnum":0,"curl_version":"curl-unit-test-fake-version"}
+{"certs":"","content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"%LOGDIR/out972","ftp_entry_path":null,"http_code":200,"http_connect":0,"http_version":"1.1","local_ip":"%HOSTIP","local_port":13,"method":"GET","num_certs":0,"num_connects":1,"num_headers":9,"num_redirects":0,"proxy_ssl_verify_result":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"HTTP","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_pretransfer":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"url":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","url.scheme":"http","url.user":null,"url.password":null,"url.options":null,"url.host":"%HOSTIP","url.port":"%HTTPPORT","url.path":"/%TESTNUMBER","url.query":null,"url.fragment":null,"url.zoneid":null,"urle.scheme":"http","urle.user":null,"urle.password":null,"urle.options":null,"urle.host":"%HOSTIP","urle.port":"%HTTPPORT","urle.path":"/%TESTNUMBER","urle.query":null,"urle.fragment":null,"urle.zoneid":null,"url_effective":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","urlnum":0,"curl_version":"curl-unit-test-fake-version"}
 </stdout>
 </verify>
 </testcase>