From: Amos Jeffries Date: Fri, 8 May 2009 01:16:48 +0000 (+1200) Subject: Author: Christos Tsantilas X-Git-Tag: SQUID_3_0_STABLE16_RC1~7 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6750af35fb1a72f09da620d4ba7cfbeee0a8a490;p=thirdparty%2Fsquid.git Author: Christos Tsantilas Bug 1148: Chunked Transfer Encoding not supported Thanks to Alin Nastac of the Gentoo Project for back-porting to 3.0. --- diff --git a/src/ICAP/ChunkedCodingParser.cc b/src/ICAP/ChunkedCodingParser.cc deleted file mode 100644 index c71175143c..0000000000 --- a/src/ICAP/ChunkedCodingParser.cc +++ /dev/null @@ -1,245 +0,0 @@ -#include "squid.h" -#include "Parsing.h" -#include "TextException.h" -#include "ChunkedCodingParser.h" -#include "MemBuf.h" - -ChunkedCodingParser::Step ChunkedCodingParser::psChunkBeg = &ChunkedCodingParser::parseChunkBeg; -ChunkedCodingParser::Step ChunkedCodingParser::psChunkBody = &ChunkedCodingParser::parseChunkBody; -ChunkedCodingParser::Step ChunkedCodingParser::psChunkEnd = &ChunkedCodingParser::parseChunkEnd; -ChunkedCodingParser::Step ChunkedCodingParser::psTrailer = &ChunkedCodingParser::parseTrailer; -ChunkedCodingParser::Step ChunkedCodingParser::psMessageEnd = &ChunkedCodingParser::parseMessageEnd; - -ChunkedCodingParser::ChunkedCodingParser() -{ - reset(); -} - -void ChunkedCodingParser::reset() -{ - theStep = psChunkBeg; - theChunkSize = theLeftBodySize = 0; - doNeedMoreData = false; - sawIeof = false; - theIn = theOut = NULL; -} - -bool ChunkedCodingParser::parse(MemBuf *rawData, MemBuf *parsedContent) -{ - Must(rawData && parsedContent); - theIn = rawData; - theOut = parsedContent; - - // we must reset this all the time so that mayContinue() lets us - // output more content if we stopped due to needsMoreSpace() before - doNeedMoreData = !theIn->hasContent(); - - while (mayContinue()) { - (this->*theStep)(); - } - - return theStep == psMessageEnd; -} - -bool ChunkedCodingParser::needsMoreData() const -{ - return doNeedMoreData; -} - -bool ChunkedCodingParser::needsMoreSpace() const -{ - assert(theOut); - return theStep == psChunkBody && !theOut->hasPotentialSpace(); -} - -bool ChunkedCodingParser::mayContinue() const -{ - return !needsMoreData() && !needsMoreSpace() && theStep != psMessageEnd; -} - -void ChunkedCodingParser::parseChunkBeg() -{ - Must(theChunkSize <= 0); // Should(), really - - size_t crlfBeg = 0; - size_t crlfEnd = 0; - - if (findCrlf(crlfBeg, crlfEnd)) { - debugs(93,7, "found chunk-size end: " << crlfBeg << "-" << crlfEnd); - int64_t size = -1; - const char *p = 0; - - if (StringToInt64(theIn->content(), size, &p, 16)) { - if (size < 0) { - throw TexcHere("negative chunk size"); - return; - } - - // check for ieof chunk extension in the last-chunk - if (size == 0 && p && *p++ == ';') { - const char *e = theIn->content() + crlfBeg; // end of extension - - while (p < e && xisspace(*p)) - ++p; // skip space - - sawIeof = e - p >= 4 && - strncmp(p, "ieof", 4) == 0 && - xisspace(p[4]); - } - - theIn->consume(crlfEnd); - theChunkSize = theLeftBodySize = size; - debugs(93,7, "found chunk: " << theChunkSize); - theStep = theChunkSize == 0 ? psTrailer : psChunkBody; - return; - } - - throw TexcHere("corrupted chunk size"); - } - - doNeedMoreData = true; -} - -void ChunkedCodingParser::parseChunkBody() -{ - Must(theLeftBodySize > 0); // Should, really - - const size_t availSize = XMIN(theLeftBodySize, (uint64_t)theIn->contentSize()); - const size_t safeSize = XMIN(availSize, (size_t)theOut->potentialSpaceSize()); - - doNeedMoreData = availSize < theLeftBodySize; - // and we may also need more space - - theOut->append(theIn->content(), safeSize); - theIn->consume(safeSize); - theLeftBodySize -= safeSize; - - if (theLeftBodySize == 0) - theStep = psChunkEnd; - else - Must(needsMoreData() || needsMoreSpace()); -} - -void ChunkedCodingParser::parseChunkEnd() -{ - Must(theLeftBodySize == 0); // Should(), really - - size_t crlfBeg = 0; - size_t crlfEnd = 0; - - if (findCrlf(crlfBeg, crlfEnd)) { - if (crlfBeg != 0) { - throw TexcHere("found data bewteen chunk end and CRLF"); - return; - } - - theIn->consume(crlfEnd); - theChunkSize = 0; // done with the current chunk - theStep = psChunkBeg; - return; - } - - doNeedMoreData = true; -} - -void ChunkedCodingParser::parseTrailer() -{ - Must(theChunkSize == 0); // Should(), really - - while (mayContinue()) - parseTrailerHeader(); -} - -void ChunkedCodingParser::parseTrailerHeader() -{ - size_t crlfBeg = 0; - size_t crlfEnd = 0; - - if (findCrlf(crlfBeg, crlfEnd)) { - if (crlfBeg > 0) - - ; //theTrailer.append(theIn->content(), crlfEnd); - - theIn->consume(crlfEnd); - - if (crlfBeg == 0) - theStep = psMessageEnd; - - return; - } - - doNeedMoreData = true; -} - -void ChunkedCodingParser::parseMessageEnd() -{ - // termination step, should not be called - Must(false); // Should(), really -} - -// finds next CRLF -bool ChunkedCodingParser::findCrlf(size_t &crlfBeg, size_t &crlfEnd) -{ - // XXX: This code was copied, with permission, from another software. - // There is a similar and probably better code inside httpHeaderParse - // but it seems difficult to isolate due to parsing-unrelated bloat. - // Such isolation should probably be done before this class is used - // for handling of traffic "more external" than ICAP. - - const char *buf = theIn->content(); - size_t size = theIn->contentSize(); - - ssize_t crOff = -1; - bool quoted = false; - bool slashed = false; - - for (size_t i = 0; i < size; ++i) { - if (slashed) { - slashed = false; - continue; - } - - const char c = buf[i]; - - // handle quoted strings - if (quoted) { - if (c == '\\') - slashed = true; - else - if (c == '"') - quoted = false; - - continue; - } else - if (c == '"') { - quoted = true; - crOff = -1; - continue; - } - - if (crOff < 0) { // looking for the first CR or LF - - if (c == '\n') { - crlfBeg = i; - crlfEnd = ++i; - return true; - } - - if (c == '\r') - crOff = i; - } else { // skipping CRs, looking for the first LF - - if (c == '\n') { - crlfBeg = crOff; - crlfEnd = ++i; - return true; - } - - if (c != '\r') - crOff = -1; - } - } - - return false; -} - diff --git a/src/ICAP/ChunkedCodingParser.h b/src/ICAP/ChunkedCodingParser.h deleted file mode 100644 index d6e1a6fba4..0000000000 --- a/src/ICAP/ChunkedCodingParser.h +++ /dev/null @@ -1,90 +0,0 @@ - -/* - * $Id: ChunkedCodingParser.h,v 1.3 2007/08/13 17:20:53 hno Exp $ - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#ifndef SQUID_CHUNKEDCODINGPARSER_H -#define SQUID_CHUNKEDCODINGPARSER_H - -#include "RefCount.h" - -// ChunkedCodingParser is an incremental parser for chunked transfer coding -// used by HTTP and ICAP. The parser shovels content bytes from the raw -// input buffer into the content output buffer, both caller-supplied. -// Ignores chunk extensions except for ICAP's ieof. -// Has a trailer-handling placeholder. - -class ChunkedCodingParser -{ - -public: - ChunkedCodingParser(); - - void reset(); - - // true = complete success; false == needs more data - bool parse(MemBuf *rawData, MemBuf *parsedContent); // throws on error - - bool needsMoreData() const; - bool needsMoreSpace() const; - bool sawIeof; // saw ieof chunk extension after a 0-size chunk - -private: - typedef void (ChunkedCodingParser::*Step)(); - -private: - bool mayContinue() const; - - void parseChunkBeg(); - void parseChunkBody(); - void parseChunkEnd(); - void parseTrailer(); - void parseTrailerHeader(); - void parseMessageEnd(); - - bool findCrlf(size_t &crlfBeg, size_t &crlfEnd); - -private: - static Step psChunkBeg; - static Step psChunkBody; - static Step psChunkEnd; - static Step psTrailer; - static Step psMessageEnd; - - MemBuf *theIn; - MemBuf *theOut; - - Step theStep; - uint64_t theChunkSize; - uint64_t theLeftBodySize; - bool doNeedMoreData; -}; - -#endif /* SQUID_CHUNKEDCODINGPARSER_H */ diff --git a/src/ICAP/TextException.cc b/src/ICAP/TextException.cc deleted file mode 100644 index 3300e20d3a..0000000000 --- a/src/ICAP/TextException.cc +++ /dev/null @@ -1,27 +0,0 @@ -#include "squid.h" -#include "TextException.h" - -TextException::TextException(const char *aMsg, const char *aFileName, int aLineNo): - message(xstrdup(aMsg)), theFileName(aFileName), theLineNo(aLineNo) -{} - -TextException::~TextException() -{ - xfree(message); -} - -void Throw(const char *message, const char *fileName, int lineNo) -{ - - // or should we let the exception recepient print the exception instead? - - if (fileName) { - debugs(0, 3, fileName << ':' << lineNo << ": exception" << - (message ? ": " : ".") << (message ? message : "")); - } else { - debugs(0, 3, "exception" << - (message ? ": " : ".") << (message ? message : "")); - } - - throw TextException(message, fileName, lineNo); -} diff --git a/src/ICAP/TextException.h b/src/ICAP/TextException.h deleted file mode 100644 index 3a78dadf2d..0000000000 --- a/src/ICAP/TextException.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef SQUID__TEXTEXCEPTION_H -#define SQUID__TEXTEXCEPTION_H - -// Origin: xstd/TextException - - -// simple exception to report custom errors -// we may want to change the interface to be able to report system errors - -class TextException -{ - -public: - TextException(const char *aMessage, const char *aFileName = 0, int aLineNo = -1); - ~TextException(); - - // ostream &print(ostream &os) const; - -public: - char *message; // read-only - -protected: - // optional location information - const char *theFileName; - int theLineNo; -}; - -//inline -//ostream &operator <<(ostream &os, const TextException &exx) { -// return exx.print(os); -//} - -#if !defined(TexcHere) -# define TexcHere(msg) TextException((msg), __FILE__, __LINE__) -#endif - -extern void Throw(const char *message, const char *fileName, int lineNo); - -// Must(condition) is like assert(condition) but throws an exception instead -#if !defined(Must) -# define Must(cond) ((cond) ? \ - (void)0 : \ - (void)Throw(#cond, __FILE__, __LINE__)) -#endif - -#endif /* SQUID__TEXTEXCEPTION_H */ diff --git a/src/Makefile.am b/src/Makefile.am index 66aa0eb3f0..8c0b66de6f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -426,6 +426,8 @@ squid_SOURCES = \ carp.cc \ cbdata.cc \ cbdata.h \ + ChunkedCodingParser.cc \ + ChunkedCodingParser.h \ client_db.cc \ client_side.cc \ client_side.h \ @@ -605,6 +607,8 @@ squid_SOURCES = \ structs.h \ SwapDir.cc \ SwapDir.h \ + TextException.cc \ + TextException.h \ time.cc \ tools.cc \ tunnel.cc \ @@ -677,8 +681,6 @@ squid_DEPENDENCIES = $(top_builddir)/lib/libmiscutil.a \ ICAP_libicap_a_SOURCES = \ ICAP/AsyncJob.cc \ ICAP/AsyncJob.h \ - ICAP/ChunkedCodingParser.cc \ - ICAP/ChunkedCodingParser.h \ ICAP/ICAPClient.cc \ ICAP/ICAPClient.h \ ICAP/ICAPInitiator.cc \ @@ -701,9 +703,7 @@ ICAP_libicap_a_SOURCES = \ ICAP/ICAPServiceRep.cc \ ICAP/ICAPServiceRep.h \ ICAP/ICAPXaction.cc \ - ICAP/ICAPXaction.h \ - ICAP/TextException.cc \ - ICAP/TextException.h + ICAP/ICAPXaction.h unlinkd_SOURCES = unlinkd_daemon.cc SquidNew.cc @@ -769,6 +769,8 @@ ufsdump_SOURCES = \ CacheDigest.cc \ carp.cc \ cbdata.cc \ + ChunkedCodingParser.cc \ + ChunkedCodingParser.h \ client_db.cc \ client_side.cc \ client_side_reply.cc \ @@ -873,6 +875,8 @@ ufsdump_SOURCES = \ store_swapout.cc \ structs.h \ SwapDir.cc \ + TextException.cc \ + TextException.h \ tools.cc \ typedefs.h \ $(UNLINKDSOURCE) \ @@ -1305,6 +1309,7 @@ tests_testCacheManager_SOURCES = \ CacheDigest.cc \ carp.cc \ cbdata.cc \ + ChunkedCodingParser.cc \ client_db.cc \ client_side.cc \ client_side_reply.cc \ @@ -1397,6 +1402,7 @@ tests_testCacheManager_SOURCES = \ StoreMetaURL.cc \ StoreMetaVary.cc \ StoreSwapLogData.cc \ + TextException.cc \ tools.cc \ tunnel.cc \ SwapDir.cc \ @@ -1473,6 +1479,7 @@ tests_testEvent_SOURCES = \ CacheDigest.cc \ carp.cc \ cbdata.cc \ + ChunkedCodingParser.cc \ client_db.cc \ client_side.cc \ client_side_reply.cc \ @@ -1564,6 +1571,7 @@ tests_testEvent_SOURCES = \ StoreMetaURL.cc \ StoreMetaVary.cc \ StoreSwapLogData.cc \ + TextException.cc \ tools.cc \ tunnel.cc \ SwapDir.cc \ @@ -1627,6 +1635,7 @@ tests_testEventLoop_SOURCES = \ CacheDigest.cc \ carp.cc \ cbdata.cc \ + ChunkedCodingParser.cc \ client_db.cc \ client_side.cc \ client_side_reply.cc \ @@ -1718,6 +1727,7 @@ tests_testEventLoop_SOURCES = \ StoreMetaURL.cc \ StoreMetaVary.cc \ StoreSwapLogData.cc \ + TextException.cc \ tools.cc \ tunnel.cc \ SwapDir.cc \ @@ -1805,6 +1815,7 @@ tests_test_http_range_SOURCES = \ CacheDigest.cc \ carp.cc \ cbdata.cc \ + ChunkedCodingParser.cc \ client_db.cc \ client_side.cc \ client_side_reply.cc \ @@ -1902,6 +1913,7 @@ tests_test_http_range_SOURCES = \ StoreSwapLogData.cc \ String.cc \ SwapDir.cc \ + TextException.cc \ time.cc \ tools.cc \ tunnel.cc \ @@ -1966,6 +1978,7 @@ tests_testHttpRequest_SOURCES = \ CacheDigest.cc \ carp.cc \ cbdata.cc \ + ChunkedCodingParser.cc \ client_db.cc \ client_side.cc \ client_side_reply.cc \ @@ -2058,6 +2071,7 @@ tests_testHttpRequest_SOURCES = \ StoreMetaURL.cc \ StoreMetaVary.cc \ StoreSwapLogData.cc \ + TextException.cc \ tools.cc \ tunnel.cc \ SwapDir.cc \ @@ -2307,6 +2321,7 @@ tests_testURL_SOURCES = \ CacheDigest.cc \ carp.cc \ cbdata.cc \ + ChunkedCodingParser.cc \ client_db.cc \ client_side.cc \ client_side_reply.cc \ @@ -2398,6 +2413,7 @@ tests_testURL_SOURCES = \ StoreMetaURL.cc \ StoreMetaVary.cc \ StoreSwapLogData.cc \ + TextException.cc \ tools.cc \ tunnel.cc \ SwapDir.cc \ diff --git a/src/http.cc b/src/http.cc index 0e63d9ba4d..4a46903813 100644 --- a/src/http.cc +++ b/src/http.cc @@ -56,6 +56,16 @@ #include "DelayPools.h" #endif #include "SquidTime.h" +#include "TextException.h" + +#define SQUID_ENTER_THROWING_CODE() try { +#define SQUID_EXIT_THROWING_CODE(status) \ + status = true; \ + } \ + catch (const TextException &e) { \ + debugs (11, 1, "Exception error:" << e.message); \ + status = false; \ + } CBDATA_CLASS_INIT(HttpStateData); @@ -68,7 +78,7 @@ static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeader HttpHeader * hdr_out, int we_do_ranges, http_state_flags); HttpStateData::HttpStateData(FwdState *theFwdState) : ServerStateData(theFwdState), - header_bytes_read(0), reply_bytes_read(0) + lastChunk(0), header_bytes_read(0), reply_bytes_read(0), httpChunkDecoder(NULL) { debugs(11,5,HERE << "HttpStateData " << this << " created"); ignoreCacheControl = false; @@ -119,7 +129,6 @@ HttpStateData::HttpStateData(FwdState *theFwdState) : ServerStateData(theFwdStat entry->setNoDelay(_peer->options.no_delay); #endif - } /* @@ -139,6 +148,9 @@ HttpStateData::~HttpStateData() delete readBuf; + if(httpChunkDecoder) + delete httpChunkDecoder; + HTTPMSGUNLOCK(orig_request); debugs(11,5, HERE << "HttpStateData " << this << " destroyed; FD " << fd); @@ -735,6 +747,12 @@ HttpStateData::processReplyHeader() readBuf->consume(header_bytes_read); } + flags.chunked = 0; + if (newrep->header.hasListMember(HDR_TRANSFER_ENCODING, "chunked", ',')) { + flags.chunked = 1; + httpChunkDecoder = new ChunkedCodingParser; + } + HttpReply *vrep = setVirginReply(newrep); flags.headers_parsed = 1; @@ -899,6 +917,13 @@ HttpStateData::persistentConnStatus() const if (eof) // already reached EOF return COMPLETE_NONPERSISTENT_MSG; + /* In chunked responce we do not know the content length but we are absolutelly + * sure about the end of response, so we are calling the statusIfComplete to + * decide if we can be persistant + */ + if (lastChunk && flags.chunked) + return statusIfComplete(); + const int64_t clen = vrep->bodySize(request->method); debugs(11, 5, "persistentConnStatus: clen=" << clen); @@ -1098,10 +1123,31 @@ HttpStateData::writeReplyBody() { const char *data = readBuf->content(); int len = readBuf->contentSize(); - addVirginReplyBody(data, len); readBuf->consume(len); +} +bool +HttpStateData::decodeAndWriteReplyBody() +{ + const char *data = NULL; + int len; + bool status = false; + assert(flags.chunked); + assert(httpChunkDecoder); + SQUID_ENTER_THROWING_CODE(); + MemBuf decodedData; + decodedData.init(); + const bool done = httpChunkDecoder->parse(readBuf,&decodedData); + len = decodedData.contentSize(); + data=decodedData.content(); + addVirginReplyBody(data, len); + if (done) { + lastChunk = 1; + flags.do_next_read = 0; + } + SQUID_EXIT_THROWING_CODE(status); + return status; } /* @@ -1134,7 +1180,15 @@ HttpStateData::processReplyBody() * That means header content has been removed from readBuf and * it contains only body data. */ - writeReplyBody(); + if(flags.chunked){ + if(!decodeAndWriteReplyBody()){ + flags.do_next_read = 0; + serverComplete(); + return; + } + } + else + writeReplyBody(); if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { /* diff --git a/src/http.h b/src/http.h index 3544d530cc..efd413befd 100644 --- a/src/http.h +++ b/src/http.h @@ -1,6 +1,6 @@ /* - * $Id: http.h,v 1.32 2007/08/09 23:30:53 rousskov Exp $ + * $Id: http.h,v 1.33 2007/12/26 23:39:55 hno Exp $ * * * SQUID Web Proxy Cache http://www.squid-cache.org/ @@ -38,6 +38,7 @@ #include "comm.h" #include "forward.h" #include "Server.h" +#include "ChunkedCodingParser.h" class HttpStateData : public ServerStateData { @@ -65,6 +66,7 @@ public: peer *_peer; /* peer request made to */ int eof; /* reached end-of-object? */ + int lastChunk; /* reached last chunk of a chunk-encoded reply */ HttpRequest *orig_request; int fd; http_state_flags flags; @@ -103,6 +105,7 @@ private: virtual void handleRequestBodyProducerAborted(); void writeReplyBody(); + bool decodeAndWriteReplyBody(); void doneSendingRequestBody(); void requestBodyHandler(MemBuf &); virtual void sentRequestBody(int fd, size_t size, comm_err_t errflag); @@ -113,6 +116,7 @@ private: http_state_flags flags); static bool decideIfWeDoRanges (HttpRequest * orig_request); + ChunkedCodingParser *httpChunkDecoder; private: CBDATA_CLASS2(HttpStateData); }; diff --git a/src/structs.h b/src/structs.h index e10eceeab2..d3182ad3fa 100644 --- a/src/structs.h +++ b/src/structs.h @@ -924,6 +924,8 @@ unsigned int do_next_read: unsigned int consume_body_data: 1; + +unsigned int chunked:1; }; struct _ipcache_addrs