content_encoding: Transfer-Encoding parser improvements

- allow and ignore "identity" as an encoding

- fail if any other encoder than chunked follows after chunked

- fail on unsolicited encodings - when the server encodes but curl did
  not ask for it

Add test 1493 to 1496 to verify.

Disable test 319 as that is now broken: issue #16974

Reported-by: Jonathan Rosa
Fixes #16956
Closes #16959

diff --git a/lib/content_encoding.c b/lib/content_encoding.c
index e9039a984d8eb1d97ccaa6ebe109cd54d3c8281a..3ca24607a162d6098082bb262001baf86df7cf62 100644 (file)
--- a/lib/content_encoding.c
+++ b/lib/content_encoding.c
@@ -737,6 +737,7 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
   Curl_cwriter_phase phase = is_transfer ?
     CURL_CW_TRANSFER_DECODE : CURL_CW_CONTENT_DECODE;
   CURLcode result;
+  bool has_chunked = FALSE;
 
   do {
     const char *name;
@@ -765,9 +766,21 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
        * Exception is "chunked" transfer-encoding which always must happen */
       if((is_transfer && !data->set.http_transfer_encoding && !is_chunked) ||
          (!is_transfer && data->set.http_ce_skip)) {
+        bool is_identity = strncasecompare(name, "identity", 8);
         /* not requested, ignore */
         CURL_TRC_WRITE(data, "decoder not requested, ignored: %.*s",
                        (int)namelen, name);
+        if(is_transfer) {
+          if(has_chunked)
+            failf(data, "A Transfer-Encoding (%.*s) was listed after chunked",
+                  (int)namelen, name);
+          else if(is_identity)
+            continue;
+          else
+            failf(data, "Unsolicited Transfer-Encoding (%.*s) found",
+                  (int)namelen, name);
+          return CURLE_BAD_CONTENT_ENCODING;
+        }
         return CURLE_OK;
       }
 
@@ -818,6 +831,8 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
         Curl_cwriter_free(data, writer);
         return result;
       }
+      if(is_chunked)
+        has_chunked = TRUE;
     }
   } while(*enclist);
 

diff --git a/tests/data/DISABLED b/tests/data/DISABLED
index 5b27569ce8be39515591035cdf4f7be0e253a17c..9f5974233c90317d811c07a59e2c2fc70d4334a0 100644 (file)
--- a/tests/data/DISABLED
+++ b/tests/data/DISABLED
@@ -27,6 +27,8 @@
 # per line.
 # Lines starting with '#' letters are treated as comments.
 #
+# PR #16959 makes Transfer-Encoding stricer and thus --raw broke
+319
 # Uses SRP to "a server not supporting it" but modern stunnel versions
 # will silently accept it and remain happy
 323

diff --git a/tests/data/Makefile.am b/tests/data/Makefile.am
index f17ad4be339ed119a850ec899954375a565d03d0..02c3cbe5d5f1dc08b948928d0964c6e68b071f1a 100644 (file)
--- a/tests/data/Makefile.am
+++ b/tests/data/Makefile.am
@@ -196,7 +196,7 @@ test1460 test1461 test1462 test1463 test1464 test1465 test1466 test1467 \
 test1468 test1469 test1470 test1471 test1472 test1473 test1474 test1475 \
 test1476 test1477 test1478 test1479 test1480 test1481 test1482 test1483 \
 test1484 test1485 test1486 test1487 test1488 test1489 test1490 test1491 \
-test1492 \
+test1492 test1493 test1494 test1495 test1496 \
 \
 test1500 test1501 test1502 test1503 test1504 test1505 test1506 test1507 \
 test1508 test1509 test1510 test1511 test1512 test1513 test1514 test1515 \

diff --git a/tests/data/test1493 b/tests/data/test1493
new file mode 100644 (file)
index 0000000..f1d1fba
--- /dev/null
+++ b/tests/data/test1493
@@ -0,0 +1,78 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+chunked Transfer-Encoding
+DELAY
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 funky chunky!\r
+Server: fakeit/0.9 fakeitbad/1.0\r
+Transfer-Encoding: identity, chunked\r
+Connection: mooo\r
+\r
+40\r
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r
+30\r
+bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\r
+21;heresatest=moooo\r
+cccccccccccccccccccccccccccccccc
+\r
+0\r
+chunky-trailer: header data\r
+another-header: yes\r
+\r
+</data>
+<datacheck>
+HTTP/1.1 200 funky chunky!\r
+Server: fakeit/0.9 fakeitbad/1.0\r
+Transfer-Encoding: identity, chunked\r
+Connection: mooo\r
+\r
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccc
+chunky-trailer: header data\r
+another-header: yes\r
+</datacheck>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+HTTP GET with identity + chunked in TE header
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER -D %LOGDIR/heads%TESTNUMBER
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /%TESTNUMBER HTTP/1.1\r
+Host: %HOSTIP:%HTTPPORT\r
+User-Agent: curl/%VERSION\r
+Accept: */*\r
+\r
+</protocol>
+<file name="%LOGDIR/heads%TESTNUMBER">
+HTTP/1.1 200 funky chunky!\r
+Server: fakeit/0.9 fakeitbad/1.0\r
+Transfer-Encoding: identity, chunked\r
+Connection: mooo\r
+\r
+chunky-trailer: header data\r
+another-header: yes\r
+</file>
+</verify>
+
+</testcase>

diff --git a/tests/data/test1494 b/tests/data/test1494
new file mode 100644 (file)
index 0000000..5546831
--- /dev/null
+++ b/tests/data/test1494
@@ -0,0 +1,50 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+chunked Transfer-Encoding
+DELAY
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data crlf="yes">
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: identity, identity
+Content-Length: 19
+
+stuff server sends
+</data>
+
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+HTTP GET with identity twice in TE header
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER -D %LOGDIR/heads%TESTNUMBER
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /%TESTNUMBER HTTP/1.1\r
+Host: %HOSTIP:%HTTPPORT\r
+User-Agent: curl/%VERSION\r
+Accept: */*\r
+\r
+</protocol>
+</verify>
+
+</testcase>

diff --git a/tests/data/test1495 b/tests/data/test1495
new file mode 100644 (file)
index 0000000..2c0f33f
--- /dev/null
+++ b/tests/data/test1495
@@ -0,0 +1,53 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+chunked Transfer-Encoding
+DELAY
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data crlf="yes" nocheck="yes">
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: chunked, identity
+Content-Length: 19
+
+stuff server sends
+</data>
+
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+HTTP GET with chunked + identity in TE header
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER -D %LOGDIR/heads%TESTNUMBER
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /%TESTNUMBER HTTP/1.1\r
+Host: %HOSTIP:%HTTPPORT\r
+User-Agent: curl/%VERSION\r
+Accept: */*\r
+\r
+</protocol>
+<errorcode>
+61
+</errorcode>
+</verify>
+
+</testcase>

diff --git a/tests/data/test1496 b/tests/data/test1496
new file mode 100644 (file)
index 0000000..dda3260
--- /dev/null
+++ b/tests/data/test1496
@@ -0,0 +1,53 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+chunked Transfer-Encoding
+DELAY
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data crlf="yes" nocheck="yes">
+HTTP/1.1 200 funky chunky!
+Server: fakeit/0.9 fakeitbad/1.0
+Transfer-Encoding: gzip, chunked
+Content-Length: 19
+
+stuff server sends
+</data>
+
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+HTTP GET with gzip + chunked transfer-encoding without being asked
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER -D %LOGDIR/heads%TESTNUMBER
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /%TESTNUMBER HTTP/1.1\r
+Host: %HOSTIP:%HTTPPORT\r
+User-Agent: curl/%VERSION\r
+Accept: */*\r
+\r
+</protocol>
+<errorcode>
+61
+</errorcode>
+</verify>
+
+</testcase>

diff --git a/tests/data/test319 b/tests/data/test319
index a7e5edda4197ce22e82613371bb0036489b65daf..d6ad42e9b1c104c93dbe6eea6bc89df5164c0b3a 100644 (file)
--- a/tests/data/test319
+++ b/tests/data/test319
@@ -3,8 +3,8 @@
 <keywords>
 HTTP
 HTTP GET
-compressed
 Transfer-Encoding
+--raw
 </keywords>
 </info>
 #
@@ -15,7 +15,7 @@ HTTP/1.1 200 OK swsclose
 Date: Mon, 29 Nov 2004 21:56:53 GMT\r
 Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29\r
 Content-Type: text/html; charset=ISO-8859-1\r
-Transfer-Encoding: gzip\r
+Transfer-Encoding: gobbledigook\r
 Content-Length: 44\r
 \r
 %hex[%1f%8b%08%08%79%9e%ab%41%00%03%6c%61%6c%61%6c%61%00%cb%c9%cc%4b%55%30%e4%52%c8%01%d1%46%5c]hex%
@@ -34,14 +34,11 @@ Content-Length: 44
 #
 # Client-side
 <client>
-<features>
-libz
-</features>
 <server>
 http
 </server>
 <name>
-HTTP GET gzip transfer-encoded data in raw mode
+HTTP GET gobbledigook transfer-encoded data in raw mode
 </name>
  <command option="no-include">
 http://%HOSTIP:%HTTPPORT/%TESTNUMBER --raw