curl: add byte range support to --variable reading from file

Allowing --variable read a portion of provided files, makes curl work on
partial files for any options that accepts strings. Like --data and others.

The byte offset is provided within brackets, with a semicolon separator
like: --variable name@file;[100-200]"

Inspired by #14479
Assisted-by: Manuel Einfalt
Test 784 - 789. Documentation update provided.

Closes #15739

diff --git a/docs/cmdline-opts/variable.md b/docs/cmdline-opts/variable.md
index 9ecc3f0e7147d60c12c068d409ac56a67b814bd4..a84078ab8ebbc015cca4dfbd8c214e57905437b5 100644 (file)
--- a/docs/cmdline-opts/variable.md
+++ b/docs/cmdline-opts/variable.md
@@ -36,6 +36,15 @@ the environment variable is not set, use --variable %name=content or
 --variable %name@content. Note that on some systems - but not all -
 environment variables are case insensitive.
 
+Added in curl 8.12.0: when getting contents from a file, you can request to
+get a byte range from it by appending ";[start-end]" to the filename, where
+*start* and *end* are byte offsets to include from the file. For example,
+asking for offset "2-10" means offset two to offset ten, including the byte
+offset 10, meaning 9 bytes in total. "2-2" means a single byte at offset 2.
+Not providing a second number implies to the end of the file. The start offset
+cannot be larger than the end offset. Asking for a range that is outside of
+the file size makes the variable contents empty.
+
 To assign a variable using contents from another variable, use
 --expand-variable. Like for example assigning a new variable using contents
 from two other:

diff --git a/src/tool_getparam.h b/src/tool_getparam.h
index 7d9abbd1417cd2e126c876412151e9d4fb576ff0..90708e001c0ebee1eb974b91ca9b10591a1c415d 100644 (file)
--- a/src/tool_getparam.h
+++ b/src/tool_getparam.h
@@ -348,6 +348,7 @@ typedef enum {
   PARAM_READ_ERROR,
   PARAM_EXPAND_ERROR, /* --expand problem */
   PARAM_BLANK_STRING,
+  PARAM_VAR_SYNTAX, /* --variable syntax error */
   PARAM_LAST
 } ParameterError;
 

diff --git a/src/tool_helpers.c b/src/tool_helpers.c
index 02193c3e5151aabf944cc652368be255c55f7497..aa315880ee05a17ac4a7dbfeb00c1bfb19ec6bac 100644 (file)
--- a/src/tool_helpers.c
+++ b/src/tool_helpers.c
@@ -75,6 +75,8 @@ const char *param2text(ParameterError error)
     return "variable expansion failure";
   case PARAM_BLANK_STRING:
     return "blank argument where content is expected";
+  case PARAM_VAR_SYNTAX:
+    return "syntax error in --variable argument";
   default:
     return "unknown error";
   }

diff --git a/src/tool_paramhlp.c b/src/tool_paramhlp.c
index d4024e13401a02e445b353825505ec7f274c5f04..5abf49d342691425620da512e64078b93ed5cbb1 100644 (file)
--- a/src/tool_paramhlp.c
+++ b/src/tool_paramhlp.c
@@ -124,15 +124,45 @@ ParameterError file2string(char **bufp, FILE *file)
   return PARAM_OK;
 }
 
-ParameterError file2memory(char **bufp, size_t *size, FILE *file)
+static int myfseek(void *stream, curl_off_t offset, int whence)
+{
+#if defined(_WIN32) && defined(USE_WIN32_LARGE_FILES)
+  return _fseeki64(stream, (__int64)offset, whence);
+#elif defined(HAVE_FSEEKO) && defined(HAVE_DECL_FSEEKO)
+  return fseeko(stream, (off_t)offset, whence);
+#else
+  if(offset > LONG_MAX)
+    return -1;
+  return fseek(stream, (long)offset, whence);
+#endif
+}
+
+ParameterError file2memory_range(char **bufp, size_t *size, FILE *file,
+                                 curl_off_t starto, curl_off_t endo)
 {
   if(file) {
     size_t nread;
     struct curlx_dynbuf dyn;
+    curl_off_t offset = 0;
+    curl_off_t throwaway = 0;
+
+    if(starto) {
+      if(file != stdin) {
+        if(myfseek(file, starto, SEEK_SET))
+          return PARAM_READ_ERROR;
+        offset = starto;
+      }
+      else
+        /* we can't seek stdin, read 'starto' bytes and throw them away */
+        throwaway = starto;
+    }
+
     /* The size needs to fit in an int later */
     curlx_dyn_init(&dyn, MAX_FILE2MEMORY);
     do {
       char buffer[4096];
+      size_t n_add;
+      char *ptr_add;
       nread = fread(buffer, 1, sizeof(buffer), file);
       if(ferror(file)) {
         curlx_dyn_free(&dyn);
@@ -140,9 +170,35 @@ ParameterError file2memory(char **bufp, size_t *size, FILE *file)
         *bufp = NULL;
         return PARAM_READ_ERROR;
       }
-      if(nread)
-        if(curlx_dyn_addn(&dyn, buffer, nread))
-          return PARAM_NO_MEM;
+      n_add = nread;
+      ptr_add = buffer;
+      if(nread) {
+        if(throwaway) {
+          if(throwaway >= (curl_off_t)nread) {
+            throwaway -= nread;
+            offset += nread;
+            n_add = 0; /* nothing to add */
+          }
+          else {
+            /* append the trailing piece */
+            n_add = (size_t)(nread - throwaway);
+            ptr_add = &buffer[throwaway];
+            offset += throwaway;
+            throwaway = 0;
+          }
+        }
+        if(n_add) {
+          if((curl_off_t)(n_add + offset) > endo)
+            n_add = (size_t)(endo - offset + 1);
+
+          if(curlx_dyn_addn(&dyn, ptr_add, n_add))
+            return PARAM_NO_MEM;
+
+          offset += n_add;
+          if(offset > endo)
+            break;
+        }
+      }
     } while(!feof(file));
     *size = curlx_dyn_len(&dyn);
     *bufp = curlx_dyn_ptr(&dyn);
@@ -154,6 +210,11 @@ ParameterError file2memory(char **bufp, size_t *size, FILE *file)
   return PARAM_OK;
 }
 
+ParameterError file2memory(char **bufp, size_t *size, FILE *file)
+{
+  return file2memory_range(bufp, size, file, 0, CURL_OFF_T_MAX);
+}
+
 /*
  * Parse the string and write the long in the given address. Return PARAM_OK
  * on success, otherwise a parameter specific error enum.

diff --git a/src/tool_paramhlp.h b/src/tool_paramhlp.h
index bd703afc8ca759936c95dd12b635e1928cb59609..136214bb203674f8915d60d9bd4e11963a99a8c7 100644 (file)
--- a/src/tool_paramhlp.h
+++ b/src/tool_paramhlp.h
@@ -37,6 +37,8 @@ ParameterError file2string(char **bufp, FILE *file);
 #endif
 
 ParameterError file2memory(char **bufp, size_t *size, FILE *file);
+ParameterError file2memory_range(char **bufp, size_t *size, FILE *file,
+                                 curl_off_t starto, curl_off_t endo);
 
 ParameterError str2num(long *val, const char *str);
 ParameterError str2unum(long *val, const char *str);

diff --git a/src/var.c b/src/var.c
index 348c7707d64cc458e6ad053c645df9091c880526..e42b5b2c40d8e59484e342fc647456e42a539160 100644 (file)
--- a/src/var.c
+++ b/src/var.c
@@ -374,6 +374,8 @@ static ParameterError addvariable(struct GlobalConfig *global,
   return PARAM_NO_MEM;
 }
 
+#define MAX_FILENAME 10000
+
 ParameterError setvariable(struct GlobalConfig *global,
                            const char *input)
 {
@@ -427,21 +429,56 @@ ParameterError setvariable(struct GlobalConfig *global,
     /* read from file or stdin */
     FILE *file;
     bool use_stdin;
+    char *range;
+    struct dynbuf fname;
+    curl_off_t startoffset = 0;
+    curl_off_t endoffset = CURL_OFF_T_MAX;
     line++;
+
+    Curl_dyn_init(&fname, MAX_FILENAME);
+
+    /* is there a byte range specified? ;[num-num] */
+    range = strstr(line, ";[");
+    if(range && ISDIGIT(range[2])) {
+      char *p = range;
+      char *endp;
+      if(curlx_strtoofft(&p[2], &endp, 10, &startoffset) || (*endp != '-'))
+        return PARAM_VAR_SYNTAX;
+      else {
+        p = endp + 1; /* pass the '-' */
+        if(*p != ']') {
+          if(curlx_strtoofft(p, &endp, 10, &endoffset) || (*endp != ']'))
+            return PARAM_VAR_SYNTAX;
+        }
+      }
+      if(startoffset > endoffset)
+        return PARAM_VAR_SYNTAX;
+      /* create a dynbuf for the filename without the range */
+      if(Curl_dyn_addn(&fname, line, (range - line)))
+        return PARAM_NO_MEM;
+      /* point to the new file name buffer */
+      line = Curl_dyn_ptr(&fname);
+    }
+
     use_stdin = !strcmp(line, "-");
     if(use_stdin)
       file = stdin;
     else {
       file = fopen(line, "rb");
       if(!file) {
-        errorf(global, "Failed to open %s", line);
-        return PARAM_READ_ERROR;
+        errorf(global, "Failed to open %s: %s", line,
+               strerror(errno));
+        err = PARAM_READ_ERROR;
       }
     }
-    err = file2memory(&content, &clen, file);
-    /* in case of out of memory, this should fail the entire operation */
-    contalloc = TRUE;
-    if(!use_stdin)
+    if(!err) {
+      err = file2memory_range(&content, &clen, file, startoffset, endoffset);
+      /* in case of out of memory, this should fail the entire operation */
+      if(clen)
+        contalloc = TRUE;
+    }
+    Curl_dyn_free(&fname);
+    if(!use_stdin && file)
       fclose(file);
     if(err)
       return err;

diff --git a/tests/data/Makefile.am b/tests/data/Makefile.am
index 105108309d52e5c2d4dad55f0ae82e901c40e104..6d4ddd29aee3aad0cebdc7d4f5b578eee624861e 100644 (file)
--- a/tests/data/Makefile.am
+++ b/tests/data/Makefile.am
@@ -109,7 +109,8 @@ test718 test719 test720 test721 test722 test723 test724 test725 test726 \
 test727 test728 test729 test730 test731 test732 test733 test734 test735 \
 test736 test737 test738 test739 test740 test741 test742 \
 \
-test780 test781 test782 test783 \
+test780 test781 test782 test783 test784 test785 test786 test787 test788 \
+test789 \
 \
 test799 test800 test801 test802 test803 test804 test805 test806 test807 \
 test808 test809 test810 test811 test812 test813 test814 test815 test816 \

diff --git a/tests/data/test784 b/tests/data/test784
new file mode 100644 (file)
index 0000000..0c6acc6
--- /dev/null
+++ b/tests/data/test784
@@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+--variable
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--variable with a file byte range
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@%LOGDIR/in%TESTNUMBER;[5-15]" --expand-data '{{name}}'
+</command>
+<file name="%LOGDIR/in%TESTNUMBER">
+On the first Monday of the month of April, 1625, the market town of Meung
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes" nonewline="yes">
+POST /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+Content-Length: 11
+Content-Type: application/x-www-form-urlencoded
+
+e first Mon
+</protocol>
+</verify>
+</testcase>

diff --git a/tests/data/test785 b/tests/data/test785
new file mode 100644 (file)
index 0000000..c46104a
--- /dev/null
+++ b/tests/data/test785
@@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+--variable
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--variable with a file byte range without end
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@%LOGDIR/in%TESTNUMBER;[5-]" --expand-data '{{name}}'
+</command>
+<file name="%LOGDIR/in%TESTNUMBER">
+On the first Monday of the month of April, 1625, the market town of Meung
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+POST /%TESTNUMBER HTTP/1.1\r
+Host: %HOSTIP:%HTTPPORT\r
+User-Agent: curl/%VERSION\r
+Accept: */*\r
+Content-Length: 69\r
+Content-Type: application/x-www-form-urlencoded\r
+\r
+e first Monday of the month of April, 1625, the market town of Meung
+</protocol>
+</verify>
+</testcase>

diff --git a/tests/data/test786 b/tests/data/test786
new file mode 100644 (file)
index 0000000..5da450a
--- /dev/null
+++ b/tests/data/test786
@@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+--variable
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--variable with a file byte range, reading from stdin
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@-;[5-15]" --expand-data '{{name}}'
+</command>
+<stdin>
+On the first Monday of the month of April, 1625, the market town of Meung
+</stdin>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol nonewline="yes">
+POST /%TESTNUMBER HTTP/1.1\r
+Host: %HOSTIP:%HTTPPORT\r
+User-Agent: curl/%VERSION\r
+Accept: */*\r
+Content-Length: 11\r
+Content-Type: application/x-www-form-urlencoded\r
+\r
+e first Mon
+</protocol>
+</verify>
+</testcase>

diff --git a/tests/data/test787 b/tests/data/test787
new file mode 100644 (file)
index 0000000..23dbad8
--- /dev/null
+++ b/tests/data/test787
@@ -0,0 +1,35 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+--variable
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--variable with a file byte range, bad range
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@&LOGDIR/fooo;[15-14]" --expand-data '{{name}}'
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<errorcode>
+2
+</errorcode>
+</verify>
+</testcase>

diff --git a/tests/data/test788 b/tests/data/test788
new file mode 100644 (file)
index 0000000..268eeb6
--- /dev/null
+++ b/tests/data/test788
@@ -0,0 +1,59 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+--variable
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--variable with a file and single-byte byte range
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@%LOGDIR/in%TESTNUMBER;[15-15]" --expand-data '{{name}}'
+</command>
+<file name="%LOGDIR/in%TESTNUMBER">
+On the first Monday of the month of April, 1625, the market town of Meung
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes" nonewline="yes">
+POST /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+Content-Length: 1
+Content-Type: application/x-www-form-urlencoded
+
+n
+</protocol>
+</verify>
+</testcase>

diff --git a/tests/data/test789 b/tests/data/test789
new file mode 100644 (file)
index 0000000..dd36303
--- /dev/null
+++ b/tests/data/test789
@@ -0,0 +1,58 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+--variable
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data crlf="yes">
+HTTP/1.1 200 OK
+Date: Tue, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+--variable with a file and byte range out of file
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --variable name"@%LOGDIR/in%TESTNUMBER;[75-85]" --expand-data '{{name}}'
+</command>
+<file name="%LOGDIR/in%TESTNUMBER">
+On the first Monday of the month of April, 1625, the market town of Meung
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol crlf="yes">
+POST /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+Content-Length: 0
+Content-Type: application/x-www-form-urlencoded
+
+</protocol>
+</verify>
+</testcase>