From: Josh Brobst Date: Sat, 31 Dec 2022 19:41:44 +0000 (-0500) Subject: http: decode transfer encoding first X-Git-Tag: curl-7_88_0~195 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=aa6e7a1f45d3d82ceaf58534bcc5ac409abbd0a0;p=thirdparty%2Fcurl.git http: decode transfer encoding first The unencoding stack is added to as Transfer-Encoding and Content-Encoding fields are encountered with no distinction between the two, meaning the stack will be incorrect if, e.g., the message has both fields and a non-chunked Transfer-Encoding comes first. This commit fixes this by ordering the stack with transfer encodings first. Reviewed-by: Patrick Monnerat Closes #10187 --- diff --git a/lib/content_encoding.c b/lib/content_encoding.c index bfc13e254d..1a90e40ea4 100644 --- a/lib/content_encoding.c +++ b/lib/content_encoding.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2022, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2023, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -977,7 +977,8 @@ static const struct content_encoding error_encoding = { static struct contenc_writer * new_unencoding_writer(struct Curl_easy *data, const struct content_encoding *handler, - struct contenc_writer *downstream) + struct contenc_writer *downstream, + int order) { struct contenc_writer *writer; @@ -987,6 +988,7 @@ new_unencoding_writer(struct Curl_easy *data, if(writer) { writer->handler = handler; writer->downstream = downstream; + writer->order = order; if(handler->init_writer(data, writer)) { free(writer); writer = NULL; @@ -1042,10 +1044,11 @@ static const struct content_encoding *find_encoding(const char *name, /* Set-up the unencoding stack from the Content-Encoding header value. * See RFC 7231 section 3.1.2.2. */ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data, - const char *enclist, int maybechunked) + const char *enclist, int is_transfer) { struct SingleRequest *k = &data->req; int counter = 0; + unsigned int order = is_transfer? 2: 1; do { const char *name; @@ -1062,7 +1065,7 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data, namelen = enclist - name + 1; /* Special case: chunked encoding is handled at the reader level. */ - if(maybechunked && namelen == 7 && strncasecompare(name, "chunked", 7)) { + if(is_transfer && namelen == 7 && strncasecompare(name, "chunked", 7)) { k->chunk = TRUE; /* chunks coming our way. */ Curl_httpchunk_init(data); /* init our chunky engine. */ } @@ -1071,7 +1074,8 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data, struct contenc_writer *writer; if(!k->writer_stack) { - k->writer_stack = new_unencoding_writer(data, &client_encoding, NULL); + k->writer_stack = new_unencoding_writer(data, &client_encoding, + NULL, 0); if(!k->writer_stack) return CURLE_OUT_OF_MEMORY; @@ -1086,10 +1090,23 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data, return CURLE_BAD_CONTENT_ENCODING; } /* Stack the unencoding stage. */ - writer = new_unencoding_writer(data, encoding, k->writer_stack); - if(!writer) - return CURLE_OUT_OF_MEMORY; - k->writer_stack = writer; + if(order >= k->writer_stack->order) { + writer = new_unencoding_writer(data, encoding, + k->writer_stack, order); + if(!writer) + return CURLE_OUT_OF_MEMORY; + k->writer_stack = writer; + } + else { + struct contenc_writer *w = k->writer_stack; + while(w->downstream && order < w->downstream->order) + w = w->downstream; + writer = new_unencoding_writer(data, encoding, + w->downstream, order); + if(!writer) + return CURLE_OUT_OF_MEMORY; + w->downstream = writer; + } } } while(*enclist); @@ -1099,11 +1116,11 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data, #else /* Stubs for builds without HTTP. */ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data, - const char *enclist, int maybechunked) + const char *enclist, int is_transfer) { (void) data; (void) enclist; - (void) maybechunked; + (void) is_transfer; return CURLE_NOT_BUILT_IN; } diff --git a/lib/content_encoding.h b/lib/content_encoding.h index 3c278cf727..4433b48f33 100644 --- a/lib/content_encoding.h +++ b/lib/content_encoding.h @@ -7,7 +7,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2022, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2023, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -28,6 +28,7 @@ struct contenc_writer { const struct content_encoding *handler; /* Encoding handler. */ struct contenc_writer *downstream; /* Downstream writer. */ + unsigned int order; /* Ordering within writer stack. */ }; /* Content encoding writer. */ @@ -46,7 +47,7 @@ struct content_encoding { CURLcode Curl_build_unencoding_stack(struct Curl_easy *data, - const char *enclist, int maybechunked); + const char *enclist, int is_transfer); CURLcode Curl_unencode_write(struct Curl_easy *data, struct contenc_writer *writer, const char *buf, size_t nbytes); diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc index c15b0a4530..6f62642783 100644 --- a/tests/data/Makefile.inc +++ b/tests/data/Makefile.inc @@ -5,7 +5,7 @@ # | (__| |_| | _ <| |___ # \___|\___/|_| \_\_____| # -# Copyright (C) 1998 - 2022, Daniel Stenberg, , et al. +# Copyright (C) 1998 - 2023, Daniel Stenberg, , et al. # # This software is licensed as described in the file COPYING, which # you should have received as part of this distribution. The terms @@ -160,7 +160,7 @@ test1239 test1240 test1241 test1242 test1243 test1244 test1245 test1246 \ test1247 test1248 test1249 test1250 test1251 test1252 test1253 test1254 \ test1255 test1256 test1257 test1258 test1259 test1260 test1261 test1262 \ test1263 test1264 test1265 test1266 test1267 test1268 test1269 test1270 \ -test1271 test1272 test1273 test1274 test1275 test1276 \ +test1271 test1272 test1273 test1274 test1275 test1276 test1277 \ \ test1280 test1281 test1282 test1283 test1284 test1285 test1286 test1287 \ test1288 test1289 test1290 test1291 test1292 test1293 test1294 test1295 \ diff --git a/tests/data/test1277 b/tests/data/test1277 new file mode 100644 index 0000000000..b705569d06 --- /dev/null +++ b/tests/data/test1277 @@ -0,0 +1,206 @@ + + + +HTTP +HTTP GET +compressed +Transfer-Encoding +Content-Encoding + + +# +# Server-side + + +SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBNb24sIDI5IE5vdiAyMDA0IDIxOjU2OjUzIEdNVA0KU2Vy +dmVyOiBBcGFjaGUvMS4zLjMxIChEZWJpYW4gR05VL0xpbnV4KSBtb2RfZ3ppcC8xLjMuMjYuMWEg +UEhQLzQuMy45LTEgbW9kX3NzbC8yLjguMjAgT3BlblNTTC8wLjkuN2QgbW9kX3BlcmwvMS4yOQ0K +VmFyeTogQWNjZXB0LUVuY29kaW5nDQpDb250ZW50LVR5cGU6IHRleHQvaHRtbDsgY2hhcnNldD1J +U08tODg1OS0xDQpUcmFuc2Zlci1FbmNvZGluZzogZ3ppcCwgY2h1bmtlZA0KQ29udGVudC1FbmNv +ZGluZzogZGVmbGF0ZQ0KDQo1MjINCh+LCAAHgbBjAv8BCwX0+nja3VjbbuM2EH03kH9g/dQCtm62 +EzlwtChy2QTNNsHaBbpPBi3RNhtJFEjKuTz02zuUKImxHK93g74ESGJy5pzD4QxJk5l8ekpitCFc +UJaedV3L6SKShiyi6eqsezO96/v+aNx3u5+CzuSXi7vz2bf7S5Rx9g8JZT+mQgIQTb9NZ5dfUHct +ZXZq20tOxDohWFopkfYChw8kjexl0tc80XesoRXJqAuiW1pBB6HKptpNb06jwD12vYltGEpEhCWZ +4ygigBiP/b7j9z0XOcNTxz31xhPb8BuEPFMfUeA5zrDvOn3XR4536sHPQFMqxKs4UpyQuVgzLoMw +53EdjmFv45d5HBdwhNMIxXTRphYQHR4RoZY6Z0lSUlKCJGO1AMf8GS0ZR2FMSSr7gkYE/fX1FkmO +U7GEilowi0bIUN4ZDKIC4XIEpVqpcFXfJY2JOOrkQnXUGOI5lfjJQjcSiTzLQF+g69nsvlf8nfbQ +FbSPOldF++LmfNZDs8vbPy/h8/bid4Bd3dxe9orhP9/dX19+hTYM8EhiiKmU6mdMyJ5u5tAKGXug +RBTaKM9ihiMYApZanpComXUPZViIR8Yj1YTI0jxZFPbp9PaoExIu6ZKGUFcw/UE4+JgoQ4FqPMEI +xbQgGxl7JJxER53Fc5WkHpJr8mbGq7JYR53ZGqgIw++CpmozCUhtnWqVYAabDnnOUSfG6SrHKyKK +GEi6oZylCQwgQEeX0FgbGypxTOXzXISMk2B04g+t0cnE3rJvgTPCQ5AMBpZ7bGAr8xYa5vMAe21s +IAuTXtcsy2PMm7GOR2OIYQjLedvTIlQDugPLd18RXodiOMpgnNErtBEOx8W54VsjZ2LrjumZhywH +Xc+tvNrwClMO4g9qjDGAyBci5DSTcESKYKBOh9emEgalT8P1XO3l4IIscR7LiW0aSxjUf14dYBmU +Pdh5aFYnpa3Wiz2xW6xGbM0S8rYQLGDKSxV1eNomQ+vWAo2mXL0cLgdgraRojcjixTtcBMBaRNEa +kReaHS4CYC2iaI1IuIY9RmK2OlyqpmjBRqKR5VlyuCCAtZSiNSIRWRwuAmAtomiNCBNPh4sAWIso +mlEvEanz8gdqVhKqumm6scxzDmkTJNDrt+oahdmIHyjJRlTF2AhDRN0cDldRaC1TEBudhHLO+OFK +JV5rabJZ14QFVa2gWXpiCoccpODLzcz+G53D6QEpo3mCbkvHxK4QGq++p+Sck5jUmWuZ5/r+FpxY +rmfBBekN9xtsda86OXb8UYtYXbDaHHUv2n1x2gUs52LvmswEzlECE14zOZdr+KqGnLVMJRLnEm4y +og6p7Ffd2qBPYJxSEqOpJCl8w68mtulsUaBKu+v+7wKvIqKKbCBbdM5iEtw9poTXuMJURWqboVY9 +YVzJOE1oiqU5O8nhhlDUBgpad1pezzve5x7s5e5zur6/l+vuHReWxv6o97rd4+E+93gv2R/v9Tp7 +3e7oO3F/J7D9kQ1aY5f3u+3yw5rI4MkEDzG4jtY6tfEZyeeMnHU5CeF1oGzqJVUvygZnbmRHDbXL +sZOo7y3gPhmqYux07WQaj7MWdfvh9hZXUqk2FNjhzo5+1bep33Zqldh6pzWIt/PGinsbjv+/pDk/ +nbHh4J0ZU++rj58o1xuM3Hdk6gWeYh8/SwPPGZ68I0vXhCYRjj9+oobD0cB7R6LCPjz1xc/kybDo +s77+r1DQqZvNf8f+A904jtHT7+gYCwUAAA0KMA0KDQo= + + + + +HTTP/1.1 200 OK +Date: Mon, 29 Nov 2004 21:56:53 GMT +Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29 +Vary: Accept-Encoding +Content-Type: text/html; charset=ISO-8859-1 +Transfer-Encoding: gzip, chunked +Content-Encoding: deflate + + + + + + 1612 + 1998-08-21 04:01:29 + 2004-10-18 02:22:23 + curl + curl and libcurl + Command line tool and library for client-side URL transfers. + curl and libcurl is a tool for transferring files +using URL syntax. It supports HTTP, HTTPS, FTP, +FTPS, DICT, TELNET, LDAP, FILE, and GOPHER, as +well as HTTP-post, HTTP-put, cookies, FTP upload, +resumed transfers, passwords, portnumbers, SSL +certificates, Kerberos, and proxies. It is powered +by libcurl, the client-side URL transfer library. +There are bindings to libcurl for over 20 +languages and environments. + + 5784.57 + 3.16 + 169 + 6594.54 + 13.81 + 105 + 8.50 + 21 + 183 + 323 + Default + http://freshmeat.net/projects/curl/ + http://freshmeat.net/redir/curl/1612/url_homepage/ + http://freshmeat.net/redir/curl/1612/url_tgz/ + http://freshmeat.net/redir/curl/1612/url_bz2/ + http://freshmeat.net/redir/curl/1612/url_zip/ + http://freshmeat.net/redir/curl/1612/url_changelog/ + http://freshmeat.net/redir/curl/1612/url_rpm/ + http://freshmeat.net/redir/curl/1612/url_deb/ + http://freshmeat.net/redir/curl/1612/url_osx/ + http://freshmeat.net/redir/curl/1612/url_bsdport/ + + http://freshmeat.net/redir/curl/1612/url_cvs/ + http://freshmeat.net/redir/curl/1612/url_list/ + http://freshmeat.net/redir/curl/1612/url_mirror/ + + MIT/X Consortium License + + 7.12.2 + 176085 + 2004-10-18 02:22:23 + + + + + Daniel Stenberg + http://freshmeat.net/~bagder/ + Owner + + + + 12 + 226 + 3 + 2 + 188 + 216 + 200 + 220 + 164 + 90 + 89 + 809 + 150 + 224 + 900 + 839 + + + + 0 + 7464 + 7464 + OpenSSL (Default) + + + 0 + 0 + 7443 + OpenLDAP + + + 0 + 0 + 12351 + zlib + + + 0 + 0 + 32047 + Heimdal + + + 0 + 0 + 44532 + c-ares + + + + + + + + +# +# Client-side + + +libz + + +http + + +HTTP GET with both content and transfer encoding + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER --tr-encoding --compressed + + + +# +# Verify data after the test has been "shot" + + +s/^Accept-Encoding: [a-zA-Z, ]*/Accept-Encoding: xxx/ + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Connection: TE +TE: gzip +Accept-Encoding: xxx + + + +