http: decode transfer encoding first

The unencoding stack is added to as Transfer-Encoding and
Content-Encoding fields are encountered with no distinction between the
two, meaning the stack will be incorrect if, e.g., the message has both
fields and a non-chunked Transfer-Encoding comes first. This commit
fixes this by ordering the stack with transfer encodings first.

Reviewed-by: Patrick Monnerat
Closes #10187

diff --git a/lib/content_encoding.c b/lib/content_encoding.c
index bfc13e254db9cfa060e93849346a0d1e3fb29e88..1a90e40ea4cd31d9021fb037b352945c7bc8635b 100644 (file)
--- a/lib/content_encoding.c
+++ b/lib/content_encoding.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2023, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -977,7 +977,8 @@ static const struct content_encoding error_encoding = {
 static struct contenc_writer *
 new_unencoding_writer(struct Curl_easy *data,
                       const struct content_encoding *handler,
-                      struct contenc_writer *downstream)
+                      struct contenc_writer *downstream,
+                      int order)
 {
   struct contenc_writer *writer;
 
@@ -987,6 +988,7 @@ new_unencoding_writer(struct Curl_easy *data,
   if(writer) {
     writer->handler = handler;
     writer->downstream = downstream;
+    writer->order = order;
     if(handler->init_writer(data, writer)) {
       free(writer);
       writer = NULL;
@@ -1042,10 +1044,11 @@ static const struct content_encoding *find_encoding(const char *name,
 /* Set-up the unencoding stack from the Content-Encoding header value.
  * See RFC 7231 section 3.1.2.2. */
 CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
-                                     const char *enclist, int maybechunked)
+                                     const char *enclist, int is_transfer)
 {
   struct SingleRequest *k = &data->req;
   int counter = 0;
+  unsigned int order = is_transfer? 2: 1;
 
   do {
     const char *name;
@@ -1062,7 +1065,7 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
         namelen = enclist - name + 1;
 
     /* Special case: chunked encoding is handled at the reader level. */
-    if(maybechunked && namelen == 7 && strncasecompare(name, "chunked", 7)) {
+    if(is_transfer && namelen == 7 && strncasecompare(name, "chunked", 7)) {
       k->chunk = TRUE;             /* chunks coming our way. */
       Curl_httpchunk_init(data);   /* init our chunky engine. */
     }
@@ -1071,7 +1074,8 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
       struct contenc_writer *writer;
 
       if(!k->writer_stack) {
-        k->writer_stack = new_unencoding_writer(data, &client_encoding, NULL);
+        k->writer_stack = new_unencoding_writer(data, &client_encoding,
+                                                NULL, 0);
 
         if(!k->writer_stack)
           return CURLE_OUT_OF_MEMORY;
@@ -1086,10 +1090,23 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
         return CURLE_BAD_CONTENT_ENCODING;
       }
       /* Stack the unencoding stage. */
-      writer = new_unencoding_writer(data, encoding, k->writer_stack);
-      if(!writer)
-        return CURLE_OUT_OF_MEMORY;
-      k->writer_stack = writer;
+      if(order >= k->writer_stack->order) {
+        writer = new_unencoding_writer(data, encoding,
+                                       k->writer_stack, order);
+        if(!writer)
+          return CURLE_OUT_OF_MEMORY;
+        k->writer_stack = writer;
+      }
+      else {
+        struct contenc_writer *w = k->writer_stack;
+        while(w->downstream && order < w->downstream->order)
+          w = w->downstream;
+        writer = new_unencoding_writer(data, encoding,
+                                       w->downstream, order);
+        if(!writer)
+          return CURLE_OUT_OF_MEMORY;
+        w->downstream = writer;
+      }
     }
   } while(*enclist);
 
@@ -1099,11 +1116,11 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
 #else
 /* Stubs for builds without HTTP. */
 CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
-                                     const char *enclist, int maybechunked)
+                                     const char *enclist, int is_transfer)
 {
   (void) data;
   (void) enclist;
-  (void) maybechunked;
+  (void) is_transfer;
   return CURLE_NOT_BUILT_IN;
 }
 

diff --git a/lib/content_encoding.h b/lib/content_encoding.h
index 3c278cf7271ddb3921a87e185fb861de5fb461a6..4433b48f336b300cc0e302790e22b405459e0a6d 100644 (file)
--- a/lib/content_encoding.h
+++ b/lib/content_encoding.h
@@ -7,7 +7,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2023, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -28,6 +28,7 @@
 struct contenc_writer {
   const struct content_encoding *handler;  /* Encoding handler. */
   struct contenc_writer *downstream;  /* Downstream writer. */
+  unsigned int order; /* Ordering within writer stack. */
 };
 
 /* Content encoding writer. */
@@ -46,7 +47,7 @@ struct content_encoding {
 
 
 CURLcode Curl_build_unencoding_stack(struct Curl_easy *data,
-                                     const char *enclist, int maybechunked);
+                                     const char *enclist, int is_transfer);
 CURLcode Curl_unencode_write(struct Curl_easy *data,
                              struct contenc_writer *writer,
                              const char *buf, size_t nbytes);

diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index c15b0a453007ccbc779a6e35f80be373690e2919..6f62642783748e3974b4fb5a297ca82e7c5bbbca 100644 (file)
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -5,7 +5,7 @@
 #                            | (__| |_| |  _ <| |___
 #                             \___|\___/|_| \_\_____|
 #
-# Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
+# Copyright (C) 1998 - 2023, Daniel Stenberg, <daniel@haxx.se>, et al.
 #
 # This software is licensed as described in the file COPYING, which
 # you should have received as part of this distribution. The terms
@@ -160,7 +160,7 @@ test1239 test1240 test1241 test1242 test1243 test1244 test1245 test1246 \
 test1247 test1248 test1249 test1250 test1251 test1252 test1253 test1254 \
 test1255 test1256 test1257 test1258 test1259 test1260 test1261 test1262 \
 test1263 test1264 test1265 test1266 test1267 test1268 test1269 test1270 \
-test1271 test1272 test1273 test1274 test1275 test1276 \
+test1271 test1272 test1273 test1274 test1275 test1276 test1277 \
 \
 test1280 test1281 test1282 test1283 test1284 test1285 test1286 test1287 \
 test1288 test1289 test1290 test1291 test1292 test1293 test1294 test1295 \

diff --git a/tests/data/test1277 b/tests/data/test1277
new file mode 100644 (file)
index 0000000..b705569
--- /dev/null
+++ b/tests/data/test1277
@@ -0,0 +1,206 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+compressed
+Transfer-Encoding
+Content-Encoding
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data base64="yes">
+SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBNb24sIDI5IE5vdiAyMDA0IDIxOjU2OjUzIEdNVA0KU2Vy
+dmVyOiBBcGFjaGUvMS4zLjMxIChEZWJpYW4gR05VL0xpbnV4KSBtb2RfZ3ppcC8xLjMuMjYuMWEg
+UEhQLzQuMy45LTEgbW9kX3NzbC8yLjguMjAgT3BlblNTTC8wLjkuN2QgbW9kX3BlcmwvMS4yOQ0K
+VmFyeTogQWNjZXB0LUVuY29kaW5nDQpDb250ZW50LVR5cGU6IHRleHQvaHRtbDsgY2hhcnNldD1J
+U08tODg1OS0xDQpUcmFuc2Zlci1FbmNvZGluZzogZ3ppcCwgY2h1bmtlZA0KQ29udGVudC1FbmNv
+ZGluZzogZGVmbGF0ZQ0KDQo1MjINCh+LCAAHgbBjAv8BCwX0+nja3VjbbuM2EH03kH9g/dQCtm62
+EzlwtChy2QTNNsHaBbpPBi3RNhtJFEjKuTz02zuUKImxHK93g74ESGJy5pzD4QxJk5l8ekpitCFc
+UJaedV3L6SKShiyi6eqsezO96/v+aNx3u5+CzuSXi7vz2bf7S5Rx9g8JZT+mQgIQTb9NZ5dfUHct
+ZXZq20tOxDohWFopkfYChw8kjexl0tc80XesoRXJqAuiW1pBB6HKptpNb06jwD12vYltGEpEhCWZ
+4ygigBiP/b7j9z0XOcNTxz31xhPb8BuEPFMfUeA5zrDvOn3XR4536sHPQFMqxKs4UpyQuVgzLoMw
+53EdjmFv45d5HBdwhNMIxXTRphYQHR4RoZY6Z0lSUlKCJGO1AMf8GS0ZR2FMSSr7gkYE/fX1FkmO
+U7GEilowi0bIUN4ZDKIC4XIEpVqpcFXfJY2JOOrkQnXUGOI5lfjJQjcSiTzLQF+g69nsvlf8nfbQ
+FbSPOldF++LmfNZDs8vbPy/h8/bid4Bd3dxe9orhP9/dX19+hTYM8EhiiKmU6mdMyJ5u5tAKGXug
+RBTaKM9ihiMYApZanpComXUPZViIR8Yj1YTI0jxZFPbp9PaoExIu6ZKGUFcw/UE4+JgoQ4FqPMEI
+xbQgGxl7JJxER53Fc5WkHpJr8mbGq7JYR53ZGqgIw++CpmozCUhtnWqVYAabDnnOUSfG6SrHKyKK
+GEi6oZylCQwgQEeX0FgbGypxTOXzXISMk2B04g+t0cnE3rJvgTPCQ5AMBpZ7bGAr8xYa5vMAe21s
+IAuTXtcsy2PMm7GOR2OIYQjLedvTIlQDugPLd18RXodiOMpgnNErtBEOx8W54VsjZ2LrjumZhywH
+Xc+tvNrwClMO4g9qjDGAyBci5DSTcESKYKBOh9emEgalT8P1XO3l4IIscR7LiW0aSxjUf14dYBmU
+Pdh5aFYnpa3Wiz2xW6xGbM0S8rYQLGDKSxV1eNomQ+vWAo2mXL0cLgdgraRojcjixTtcBMBaRNEa
+kReaHS4CYC2iaI1IuIY9RmK2OlyqpmjBRqKR5VlyuCCAtZSiNSIRWRwuAmAtomiNCBNPh4sAWIso
+mlEvEanz8gdqVhKqumm6scxzDmkTJNDrt+oahdmIHyjJRlTF2AhDRN0cDldRaC1TEBudhHLO+OFK
+JV5rabJZ14QFVa2gWXpiCoccpODLzcz+G53D6QEpo3mCbkvHxK4QGq++p+Sck5jUmWuZ5/r+FpxY
+rmfBBekN9xtsda86OXb8UYtYXbDaHHUv2n1x2gUs52LvmswEzlECE14zOZdr+KqGnLVMJRLnEm4y
+og6p7Ffd2qBPYJxSEqOpJCl8w68mtulsUaBKu+v+7wKvIqKKbCBbdM5iEtw9poTXuMJURWqboVY9
+YVzJOE1oiqU5O8nhhlDUBgpad1pezzve5x7s5e5zur6/l+vuHReWxv6o97rd4+E+93gv2R/v9Tp7
+3e7oO3F/J7D9kQ1aY5f3u+3yw5rI4MkEDzG4jtY6tfEZyeeMnHU5CeF1oGzqJVUvygZnbmRHDbXL
+sZOo7y3gPhmqYux07WQaj7MWdfvh9hZXUqk2FNjhzo5+1bep33Zqldh6pzWIt/PGinsbjv+/pDk/
+nbHh4J0ZU++rj58o1xuM3Hdk6gWeYh8/SwPPGZ68I0vXhCYRjj9+oobD0cB7R6LCPjz1xc/kybDo
+s77+r1DQqZvNf8f+A904jtHT7+gYCwUAAA0KMA0KDQo=
+</data>
+
+
+<datacheck>
+HTTP/1.1 200 OK\r
+Date: Mon, 29 Nov 2004 21:56:53 GMT\r
+Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29\r
+Vary: Accept-Encoding\r
+Content-Type: text/html; charset=ISO-8859-1\r
+Transfer-Encoding: gzip, chunked\r
+Content-Encoding: deflate\r
+\r
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<!DOCTYPE project-listing SYSTEM "http://freshmeat.net/backend/fm-projects-0.4.dtd">
+<project-listing>
+  <project>
+    <project_id>1612</project_id>
+    <date_added>1998-08-21 04:01:29</date_added>
+    <date_updated>2004-10-18 02:22:23</date_updated>
+    <projectname_short>curl</projectname_short>
+    <projectname_full>curl and libcurl</projectname_full>
+    <desc_short>Command line tool and library for client-side URL transfers.</desc_short>
+    <desc_full>curl and libcurl is a tool for transferring files\r
+using URL syntax. It supports HTTP, HTTPS, FTP,\r
+FTPS, DICT, TELNET, LDAP, FILE, and GOPHER, as\r
+well as HTTP-post, HTTP-put, cookies, FTP upload,\r
+resumed transfers, passwords, portnumbers, SSL\r
+certificates, Kerberos, and proxies. It is powered\r
+by libcurl, the client-side URL transfer library.\r
+There are bindings to libcurl for over 20\r
+languages and environments.\r
+</desc_full>
+    <vitality_score>5784.57</vitality_score>
+    <vitality_percent>3.16</vitality_percent>
+    <vitality_rank>169</vitality_rank>
+    <popularity_score>6594.54</popularity_score>
+    <popularity_percent>13.81</popularity_percent>
+    <popularity_rank>105</popularity_rank>
+    <rating>8.50</rating>
+    <rating_count>21</rating_count>
+    <rating_rank>183</rating_rank>
+    <subscriptions>323</subscriptions>
+    <branch_name>Default</branch_name>
+    <url_project_page>http://freshmeat.net/projects/curl/</url_project_page>
+    <url_homepage>http://freshmeat.net/redir/curl/1612/url_homepage/</url_homepage>
+    <url_tgz>http://freshmeat.net/redir/curl/1612/url_tgz/</url_tgz>
+    <url_bz2>http://freshmeat.net/redir/curl/1612/url_bz2/</url_bz2>
+    <url_zip>http://freshmeat.net/redir/curl/1612/url_zip/</url_zip>
+    <url_changelog>http://freshmeat.net/redir/curl/1612/url_changelog/</url_changelog>
+    <url_rpm>http://freshmeat.net/redir/curl/1612/url_rpm/</url_rpm>
+    <url_deb>http://freshmeat.net/redir/curl/1612/url_deb/</url_deb>
+    <url_osx>http://freshmeat.net/redir/curl/1612/url_osx/</url_osx>
+    <url_bsdport>http://freshmeat.net/redir/curl/1612/url_bsdport/</url_bsdport>
+    <url_purchase></url_purchase>
+    <url_cvs>http://freshmeat.net/redir/curl/1612/url_cvs/</url_cvs>
+    <url_list>http://freshmeat.net/redir/curl/1612/url_list/</url_list>
+    <url_mirror>http://freshmeat.net/redir/curl/1612/url_mirror/</url_mirror>
+    <url_demo></url_demo>
+    <license>MIT/X Consortium License</license>
+    <latest_release>
+      <latest_release_version>7.12.2</latest_release_version>
+      <latest_release_id>176085</latest_release_id>
+      <latest_release_date>2004-10-18 02:22:23</latest_release_date>
+    </latest_release>
+    <screenshot_thumb></screenshot_thumb>
+    <authors>
+      <author>
+        <author_name>Daniel Stenberg</author_name>
+        <author_url>http://freshmeat.net/~bagder/</author_url>
+        <author_role>Owner</author_role>
+      </author>
+    </authors>
+    <descriminators>
+      <trove_id>12</trove_id>
+      <trove_id>226</trove_id>
+      <trove_id>3</trove_id>
+      <trove_id>2</trove_id>
+      <trove_id>188</trove_id>
+      <trove_id>216</trove_id>
+      <trove_id>200</trove_id>
+      <trove_id>220</trove_id>
+      <trove_id>164</trove_id>
+      <trove_id>90</trove_id>
+      <trove_id>89</trove_id>
+      <trove_id>809</trove_id>
+      <trove_id>150</trove_id>
+      <trove_id>224</trove_id>
+      <trove_id>900</trove_id>
+      <trove_id>839</trove_id>
+    </descriminators>
+    <dependencies>
+      <dependency type="recommended">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>7464</dependency_branch_id>
+        <dependency_project_id>7464</dependency_project_id>
+        <dependency_project_title>OpenSSL (Default)</dependency_project_title>
+      </dependency>
+      <dependency type="optional">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>0</dependency_branch_id>
+        <dependency_project_id>7443</dependency_project_id>
+        <dependency_project_title>OpenLDAP</dependency_project_title>
+      </dependency>
+      <dependency type="optional">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>0</dependency_branch_id>
+        <dependency_project_id>12351</dependency_project_id>
+        <dependency_project_title>zlib</dependency_project_title>
+      </dependency>
+      <dependency type="optional">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>0</dependency_branch_id>
+        <dependency_project_id>32047</dependency_project_id>
+        <dependency_project_title>Heimdal</dependency_project_title>
+      </dependency>
+      <dependency type="optional">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>0</dependency_branch_id>
+        <dependency_project_id>44532</dependency_project_id>
+        <dependency_project_title>c-ares</dependency_project_title>
+      </dependency>
+    </dependencies>
+  </project>
+</project-listing>
+</datacheck>
+
+</reply>
+
+#
+# Client-side
+<client>
+<features>
+libz
+</features>
+<server>
+http
+</server>
+ <name>
+HTTP GET with both content and transfer encoding
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --tr-encoding --compressed
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strippart>
+s/^Accept-Encoding: [a-zA-Z, ]*/Accept-Encoding: xxx/
+</strippart>
+<protocol>
+GET /%TESTNUMBER HTTP/1.1\r
+Host: %HOSTIP:%HTTPPORT\r
+User-Agent: curl/%VERSION\r
+Accept: */*\r
+Connection: TE\r
+TE: gzip\r
+Accept-Encoding: xxx\r
+\r
+</protocol>
+</verify>
+</testcase>