2 * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 86 ESI processing */
15 #include "client_side.h"
16 #include "client_side_request.h"
17 #include "esi/Include.h"
18 #include "esi/VarState.h"
20 #include "http/Stream.h"
21 #include "HttpReply.h"
22 #include "log/access_log.h"
24 CBDATA_CLASS_INIT (ESIStreamContext
);
27 static CSCB esiBufferRecipient
;
28 static CSD esiBufferDetach
;
29 /* esiStreamContext */
30 static ESIStreamContext
*ESIStreamContextNew (ESIIncludePtr
);
33 * 1. retry failed upstream requests
36 /* Detach from a buffering stream
39 esiBufferDetach (clientStreamNode
*node
, ClientHttpRequest
*http
)
41 /* Detach ourselves */
42 clientStreamDetach (node
, http
);
46 * Write a chunk of data to a client 'socket'.
47 * If the reply is present, send the reply headers down the wire too.
50 * The request is an internal ESI subrequest.
51 * data context is not NULL
52 * There are no more entries in the stream chain.
53 * The caller is responsible for creation and deletion of the Reply headers.
56 * Bug 975, bug 1566 : delete rep; 2006/09/02: TS, #975
58 * This was causing double-deletes. Its possible that not deleting
59 * it here will cause memory leaks, but if so, this delete should
60 * not be reinstated or it will trigger bug #975 again - RBC 20060903
63 esiBufferRecipient (clientStreamNode
*node
, ClientHttpRequest
*http
, HttpReply
*rep
, StoreIOBuffer receivedData
)
65 /* Test preconditions */
66 assert (node
!= NULL
);
67 /* ESI TODO: handle thisNode rather than asserting
68 * - it should only ever happen if we cause an
69 * abort and the callback chain loops back to
70 * here, so we can simply return. However, that
71 * itself shouldn't happen, so it stays as an
73 assert (cbdataReferenceValid (node
));
74 assert (node
->node
.next
== NULL
);
75 assert (http
->getConn() == NULL
);
77 ESIStreamContext::Pointer esiStream
= dynamic_cast<ESIStreamContext
*>(node
->data
.getRaw());
78 assert (esiStream
.getRaw() != NULL
);
79 /* If segments become more flexible, ignore thisNode */
80 assert (receivedData
.length
<= sizeof(esiStream
->localbuffer
->buf
));
81 assert (!esiStream
->finished
);
83 debugs (86,5, HERE
<< "rep " << rep
<< " body " << receivedData
.data
<< " len " << receivedData
.length
);
84 assert (node
->readBuffer
.offset
== receivedData
.offset
|| receivedData
.length
== 0);
88 if (http
->out
.offset
!= 0) {
92 if (rep
->sline
.status() != Http::scOkay
) {
94 esiStream
->include
->includeFail (esiStream
);
95 esiStream
->finished
= 1;
96 httpRequestFree (http
);
101 /* should be done in the store rather than every recipient? */
102 headersLog(0, 0, http
->request
->method
, rep
);
109 if (receivedData
.data
&& receivedData
.length
) {
110 http
->out
.offset
+= receivedData
.length
;
112 if (receivedData
.data
>= esiStream
->localbuffer
->buf
&&
113 receivedData
.data
< &esiStream
->localbuffer
->buf
[sizeof(esiStream
->localbuffer
->buf
)]) {
114 /* original static buffer */
116 if (receivedData
.data
!= esiStream
->localbuffer
->buf
) {
117 /* But not the start of it */
118 memmove(esiStream
->localbuffer
->buf
, receivedData
.data
, receivedData
.length
);
121 esiStream
->localbuffer
->len
= receivedData
.length
;
123 assert (esiStream
->buffer
.getRaw() != NULL
);
124 esiStream
->buffer
->len
= receivedData
.length
;
128 /* EOF / Read error / aborted entry */
129 if (rep
== NULL
&& receivedData
.data
== NULL
&& receivedData
.length
== 0) {
130 /* TODO: get stream status to test the entry for aborts */
131 debugs(86, 5, HERE
<< "Finished reading upstream data in subrequest");
132 esiStream
->include
->subRequestDone (esiStream
, true);
133 esiStream
->finished
= 1;
134 httpRequestFree (http
);
138 /* after the write to the user occurs, (ie here, or in a callback)
140 if (clientHttpRequestStatus(-1, http
)) {
141 /* TODO: Does thisNode if block leak htto ? */
142 /* XXX when reviewing ESI this is the first place to look */
144 esiStream
->finished
= 1;
145 esiStream
->include
->includeFail (esiStream
);
149 switch (clientStreamStatus (node
, http
)) {
151 case STREAM_UNPLANNED_COMPLETE
: /* fallthru ok */
153 case STREAM_COMPLETE
: /* ok */
154 debugs(86, 3, "ESI subrequest finished OK");
155 esiStream
->include
->subRequestDone (esiStream
, true);
156 esiStream
->finished
= 1;
157 httpRequestFree (http
);
161 debugs(86, DBG_IMPORTANT
, "ESI subrequest failed transfer");
162 esiStream
->include
->includeFail (esiStream
);
163 esiStream
->finished
= 1;
164 httpRequestFree (http
);
168 StoreIOBuffer tempBuffer
;
170 if (!esiStream
->buffer
.getRaw()) {
171 esiStream
->buffer
= esiStream
->localbuffer
;
174 esiStream
->buffer
= esiStream
->buffer
->tail();
176 if (esiStream
->buffer
->len
) {
177 esiStream
->buffer
->next
= new ESISegment
;
178 esiStream
->buffer
= esiStream
->buffer
->next
;
181 tempBuffer
.offset
= http
->out
.offset
;
182 tempBuffer
.length
= sizeof (esiStream
->buffer
->buf
);
183 tempBuffer
.data
= esiStream
->buffer
->buf
;
184 /* now just read into 'buffer' */
185 clientStreamRead (node
, http
, tempBuffer
);
186 debugs(86, 5, HERE
<< "Requested more data for ESI subrequest");
192 fatal ("Hit unreachable code in esiBufferRecipient\n");
197 /* esiStream functions */
198 ESIStreamContext::~ESIStreamContext()
204 ESIStreamContext::freeResources()
206 debugs(86, 5, "Freeing stream context resources.");
213 ESIStreamContextNew (ESIIncludePtr include
)
215 ESIStreamContext
*rv
= new ESIStreamContext
;
216 rv
->include
= include
;
221 ESIInclude::~ESIInclude()
223 debugs(86, 5, "ESIInclude::Free " << this);
224 ESISegmentFreeList (srccontent
);
225 ESISegmentFreeList (altcontent
);
226 cbdataReferenceDone (varState
);
238 ESIInclude::makeCacheable() const
240 return new ESIInclude (*this);
244 ESIInclude::makeUsable(esiTreeParentPtr newParent
, ESIVarState
&newVarState
) const
246 ESIInclude
*resultI
= new ESIInclude (*this);
247 ESIElement::Pointer result
= resultI
;
248 resultI
->parent
= newParent
;
249 resultI
->varState
= cbdataReference (&newVarState
);
252 resultI
->src
= ESIStreamContextNew (resultI
);
255 resultI
->alt
= ESIStreamContextNew (resultI
);
260 ESIInclude::ESIInclude(ESIInclude
const &old
) :
268 memset(&flags
, 0, sizeof(flags
));
269 flags
.onerrorcontinue
= old
.flags
.onerrorcontinue
;
272 srcurl
= xstrdup(old
.srcurl
);
275 alturl
= xstrdup(old
.alturl
);
279 ESIInclude::prepareRequestHeaders(HttpHeader
&tempheaders
, ESIVarState
*vars
)
281 tempheaders
.update(&vars
->header());
282 tempheaders
.removeHopByHopEntries();
286 ESIInclude::Start (ESIStreamContext::Pointer stream
, char const *url
, ESIVarState
*vars
)
288 if (!stream
.getRaw())
291 HttpHeader
tempheaders(hoRequest
);
293 prepareRequestHeaders(tempheaders
, vars
);
295 /* Ensure variable state is clean */
296 vars
->feedData(url
, strlen (url
));
298 /* tempUrl is eaten by the request */
299 char const *tempUrl
= vars
->extractChar ();
301 debugs(86, 5, "ESIIncludeStart: Starting subrequest with url '" << tempUrl
<< "'");
303 if (clientBeginRequest(Http::METHOD_GET
, tempUrl
, esiBufferRecipient
, esiBufferDetach
, stream
.getRaw(), &tempheaders
, stream
->localbuffer
->buf
, HTTP_REQBUF_SZ
)) {
304 debugs(86, DBG_CRITICAL
, "starting new ESI subrequest failed");
310 ESIInclude::ESIInclude(esiTreeParentPtr aParent
, int attrcount
, char const **attr
, ESIContext
*aContext
) :
319 memset(&flags
, 0, sizeof(flags
));
321 for (int i
= 0; i
< attrcount
&& attr
[i
]; i
+= 2) {
322 if (!strcmp(attr
[i
],"src")) {
323 /* Start a request for thisNode url */
324 debugs(86, 5, "ESIIncludeNew: Requesting source '" << attr
[i
+1] << "'");
326 /* TODO: don't assert on thisNode, ignore the duplicate */
327 assert (src
.getRaw() == NULL
);
328 src
= ESIStreamContextNew (this);
329 assert (src
.getRaw() != NULL
);
330 srcurl
= xstrdup(attr
[i
+1]);
331 } else if (!strcmp(attr
[i
],"alt")) {
332 /* Start a secondary request for thisNode url */
333 /* TODO: make a config parameter to wait on requesting alt's
334 * for the src to fail
336 debugs(86, 5, "ESIIncludeNew: Requesting alternate '" << attr
[i
+1] << "'");
338 assert (alt
.getRaw() == NULL
); /* TODO: FIXME */
339 alt
= ESIStreamContextNew (this);
340 assert (alt
.getRaw() != NULL
);
341 alturl
= xstrdup(attr
[i
+1]);
342 } else if (!strcmp(attr
[i
],"onerror")) {
343 if (!strcmp(attr
[i
+1], "continue")) {
344 flags
.onerrorcontinue
= 1;
346 /* ignore mistyped attributes */
347 debugs(86, DBG_IMPORTANT
, "invalid value for onerror='" << attr
[i
+1] << "'");
350 /* ignore mistyped attributes. TODO:? error on these for user feedback - config parameter needed
355 varState
= cbdataReference(aContext
->varState
);
361 /* prevent freeing ourselves */
362 ESIIncludePtr
foo(this);
370 Start (src
, srcurl
, varState
);
371 Start (alt
, alturl
, varState
);
375 debugs(86, DBG_IMPORTANT
, "ESIIncludeNew: esi:include with no src attributes");
382 ESIInclude::render(ESISegment::Pointer output
)
387 ESISegment::Pointer myout
;
389 debugs(86, 5, "ESIIncludeRender: Rendering include " << this);
391 assert (flags
.finished
|| (flags
.failed
&& flags
.onerrorcontinue
));
393 if (flags
.failed
&& flags
.onerrorcontinue
) {
397 /* Render the content */
398 if (srccontent
.getRaw()) {
401 } else if (altcontent
.getRaw()) {
405 fatal ("ESIIncludeRender called with no content, and no failure!\n");
407 assert (output
->next
== NULL
);
409 output
->next
= myout
;
415 ESIInclude::process (int dovars
)
417 /* Prevent refcount race leading to free */
420 debugs(86, 5, "ESIIncludeRender: Processing include " << this);
423 if (flags
.onerrorcontinue
)
424 return ESI_PROCESS_COMPLETE
;
426 return ESI_PROCESS_FAILED
;
429 if (!flags
.finished
) {
430 if (flags
.onerrorcontinue
)
431 return ESI_PROCESS_PENDING_WONTFAIL
;
433 return ESI_PROCESS_PENDING_MAYFAIL
;
436 return ESI_PROCESS_COMPLETE
;
440 ESIInclude::includeFail (ESIStreamContext::Pointer stream
)
442 subRequestDone (stream
, false);
446 ESIInclude::dataNeeded() const
448 return !(flags
.finished
|| flags
.failed
);
452 ESIInclude::subRequestDone (ESIStreamContext::Pointer stream
, bool success
)
458 debugs(86, 3, "ESIInclude::subRequestDone: " << srcurl
);
461 /* copy the lead segment */
462 debugs(86, 3, "ESIIncludeSubRequestDone: Src OK - include PASSED.");
463 assert (!srccontent
.getRaw());
464 ESISegment::ListTransfer (stream
->localbuffer
, srccontent
);
468 /* Fail if there is no alt being retrieved */
469 debugs(86, 3, "ESIIncludeSubRequestDone: Src FAILED");
471 if (!(alt
.getRaw() || altcontent
.getRaw())) {
472 debugs(86, 3, "ESIIncludeSubRequestDone: Include FAILED - No ALT");
474 } else if (altcontent
.getRaw()) {
475 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - ALT already Complete");
476 /* ALT was already retrieved, we are done */
482 } else if (stream
== alt
) {
483 debugs(86, 3, "ESIInclude::subRequestDone: " << alturl
);
486 debugs(86, 3, "ESIIncludeSubRequestDone: ALT OK.");
487 /* copy the lead segment */
488 assert (!altcontent
.getRaw());
489 ESISegment::ListTransfer (stream
->localbuffer
, altcontent
);
492 if (!(src
.getRaw() || srccontent
.getRaw())) {
493 /* src already failed, kick ESI processor */
494 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - SRC already failed.");
498 if (!(src
.getRaw() || srccontent
.getRaw())) {
499 debugs(86, 3, "ESIIncludeSubRequestDone: ALT FAILED, Include FAILED - SRC already failed");
500 /* src already failed */
507 fatal ("ESIIncludeSubRequestDone: non-owned stream found!\n");
510 if (flags
.finished
|| flags
.failed
) {
511 /* Kick ESI Processor */
512 debugs (86, 5, "ESIInclude " << this <<
513 " SubRequest " << stream
.getRaw() <<
514 " completed, kicking processor , status " <<
515 (flags
.finished
? "OK" : "FAILED"));
516 /* There is a race condition - and we have no reproducible test case -
517 * during a subrequest the parent will get set to NULL, which is not
518 * meant to be possible. Rather than killing squid, we let it leak
519 * memory but complain in the log.
521 * Someone wanting to debug this could well start by running squid with
522 * a hardware breakpoint set to this location.
523 * Its probably due to parent being set to null - by a call to
524 * 'this.finish' while the subrequest is still not completed.
526 if (parent
.getRaw() == NULL
) {
527 debugs (86, 0, "ESIInclude::subRequestDone: Sub request completed "
528 "after finish() called and parent unlinked. Unable to "
529 "continue handling the request, and may be memory leaking. "
530 "See http://www.squid-cache.org/bugs/show_bug.cgi?id=951 - we "
531 "are looking for a reproducible test case. This will require "
532 "an ESI template with includes, probably with alt-options, "
533 "and we're likely to need traffic dumps to allow us to "
534 "reconstruct the exact tcp handling sequences to trigger this "
535 "rather elusive bug.");
538 assert (parent
.getRaw());
542 parent
->provideData (srccontent
.getRaw() ? srccontent
:altcontent
,this);
544 if (srccontent
.getRaw())
548 } else if (flags
.onerrorcontinue
) {
549 /* render nothing but inform of completion */
553 parent
->provideData (new ESISegment
, this);
557 parent
->fail(this, "esi:include could not be completed.");
561 #endif /* USE_SQUID_ESI */