src/esi/Include.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 86    ESI processing */
  10
  11 #include "squid.h"
  12
  13 #if USE_SQUID_ESI
  14
  15 #include "client_side.h"
  16 #include "client_side_request.h"
  17 #include "esi/Include.h"
  18 #include "esi/VarState.h"
  19 #include "fatal.h"
  20 #include "HttpReply.h"
  21 #include "log/access_log.h"
  22
  23 CBDATA_CLASS_INIT (ESIStreamContext);
  24
  25 /* other */
  26 static CSCB esiBufferRecipient;
  27 static CSD esiBufferDetach;
  28 /* esiStreamContext */
  29 static ESIStreamContext *ESIStreamContextNew (ESIIncludePtr);
  30
  31 /* ESI TO CONSIDER:
  32  * 1. retry failed upstream requests
  33  */
  34
  35 /* Detach from a buffering stream
  36  */
  37 void
  38 esiBufferDetach (clientStreamNode *node, ClientHttpRequest *http)
  39 {
  40     /* Detach ourselves */
  41     clientStreamDetach (node, http);
  42 }
  43
  44 /**
  45  * Write a chunk of data to a client 'socket'.
  46  * If the reply is present, send the reply headers down the wire too.
  47  *
  48  * Pre-condition:
  49  *   The request is an internal ESI subrequest.
  50  *   data context is not NULL
  51  *   There are no more entries in the stream chain.
  52  *   The caller is responsible for creation and deletion of the Reply headers.
  53  *
  54  \note
  55  * Bug 975, bug 1566 : delete rep; 2006/09/02: TS, #975
  56  *
  57  * This was causing double-deletes. Its possible that not deleting
  58  * it here will cause memory leaks, but if so, this delete should
  59  * not be reinstated or it will trigger bug #975 again - RBC 20060903
  60  */
  61 void
  62 esiBufferRecipient (clientStreamNode *node, ClientHttpRequest *http, HttpReply *rep, StoreIOBuffer receivedData)
  63 {
  64     /* Test preconditions */
  65     assert (node != NULL);
  66     /* ESI TODO: handle thisNode rather than asserting
  67      * - it should only ever happen if we cause an
  68      * abort and the callback chain loops back to
  69      * here, so we can simply return. However, that
  70      * itself shouldn't happen, so it stays as an
  71      * assert for now. */
  72     assert (cbdataReferenceValid (node));
  73     assert (node->node.next == NULL);
  74     assert (http->getConn() == NULL);
  75
  76     ESIStreamContext::Pointer esiStream = dynamic_cast<ESIStreamContext *>(node->data.getRaw());
  77     assert (esiStream.getRaw() != NULL);
  78     /* If segments become more flexible, ignore thisNode */
  79     assert (receivedData.length <= sizeof(esiStream->localbuffer->buf));
  80     assert (!esiStream->finished);
  81
  82     debugs (86,5, HERE << "rep " << rep << " body " << receivedData.data << " len " << receivedData.length);
  83     assert (node->readBuffer.offset == receivedData.offset || receivedData.length == 0);
  84
  85     /* trivial case */
  86
  87     if (http->out.offset != 0) {
  88         assert(rep == NULL);
  89     } else {
  90         if (rep) {
  91             if (rep->sline.status() != Http::scOkay) {
  92                 rep = NULL;
  93                 esiStream->include->includeFail (esiStream);
  94                 esiStream->finished = 1;
  95                 httpRequestFree (http);
  96                 return;
  97             }
  98
  99 #if HEADERS_LOG
 100             /* should be done in the store rather than every recipient?  */
 101             headersLog(0, 0, http->request->method, rep);
 102
 103 #endif
 104             rep = NULL;
 105         }
 106     }
 107
 108     if (receivedData.data && receivedData.length) {
 109         http->out.offset += receivedData.length;
 110
 111         if (receivedData.data >= esiStream->localbuffer->buf &&
 112                 receivedData.data < &esiStream->localbuffer->buf[sizeof(esiStream->localbuffer->buf)]) {
 113             /* original static buffer */
 114
 115             if (receivedData.data != esiStream->localbuffer->buf) {
 116                 /* But not the start of it */
 117                 memmove(esiStream->localbuffer->buf, receivedData.data, receivedData.length);
 118             }
 119
 120             esiStream->localbuffer->len = receivedData.length;
 121         } else {
 122             assert (esiStream->buffer.getRaw() != NULL);
 123             esiStream->buffer->len = receivedData.length;
 124         }
 125     }
 126
 127     /* EOF / Read error /  aborted entry */
 128     if (rep == NULL && receivedData.data == NULL && receivedData.length == 0) {
 129         /* TODO: get stream status to test the entry for aborts */
 130         debugs(86, 5, HERE << "Finished reading upstream data in subrequest");
 131         esiStream->include->subRequestDone (esiStream, true);
 132         esiStream->finished = 1;
 133         httpRequestFree (http);
 134         return;
 135     }
 136
 137     /* after the write to the user occurs, (ie here, or in a callback)
 138      * we call */
 139     if (clientHttpRequestStatus(-1, http)) {
 140         /* TODO: Does thisNode if block leak htto ? */
 141         /* XXX when reviewing ESI this is the first place to look */
 142         node->data = NULL;
 143         esiStream->finished = 1;
 144         esiStream->include->includeFail (esiStream);
 145         return;
 146     };
 147
 148     switch (clientStreamStatus (node, http)) {
 149
 150     case STREAM_UNPLANNED_COMPLETE: /* fallthru ok */
 151
 152     case STREAM_COMPLETE: /* ok */
 153         debugs(86, 3, "ESI subrequest finished OK");
 154         esiStream->include->subRequestDone (esiStream, true);
 155         esiStream->finished = 1;
 156         httpRequestFree (http);
 157         return;
 158
 159     case STREAM_FAILED:
 160         debugs(86, DBG_IMPORTANT, "ESI subrequest failed transfer");
 161         esiStream->include->includeFail (esiStream);
 162         esiStream->finished = 1;
 163         httpRequestFree (http);
 164         return;
 165
 166     case STREAM_NONE: {
 167         StoreIOBuffer tempBuffer;
 168
 169         if (!esiStream->buffer.getRaw()) {
 170             esiStream->buffer = esiStream->localbuffer;
 171         }
 172
 173         esiStream->buffer = esiStream->buffer->tail();
 174
 175         if (esiStream->buffer->len) {
 176             esiStream->buffer->next = new ESISegment;
 177             esiStream->buffer = esiStream->buffer->next;
 178         }
 179
 180         tempBuffer.offset = http->out.offset;
 181         tempBuffer.length = sizeof (esiStream->buffer->buf);
 182         tempBuffer.data = esiStream->buffer->buf;
 183         /* now just read into 'buffer' */
 184         clientStreamRead (node, http, tempBuffer);
 185         debugs(86, 5, HERE << "Requested more data for ESI subrequest");
 186     }
 187
 188     break;
 189
 190     default:
 191         fatal ("Hit unreachable code in esiBufferRecipient\n");
 192     }
 193
 194 }
 195
 196 /* esiStream functions */
 197 ESIStreamContext::~ESIStreamContext()
 198 {
 199     freeResources();
 200 }
 201
 202 void
 203 ESIStreamContext::freeResources()
 204 {
 205     debugs(86, 5, "Freeing stream context resources.");
 206     buffer = NULL;
 207     localbuffer = NULL;
 208     include = NULL;
 209 }
 210
 211 ESIStreamContext *
 212 ESIStreamContextNew (ESIIncludePtr include)
 213 {
 214     ESIStreamContext *rv = new ESIStreamContext;
 215     rv->include = include;
 216     return rv;
 217 }
 218
 219 /* ESIInclude */
 220 ESIInclude::~ESIInclude()
 221 {
 222     debugs(86, 5, "ESIInclude::Free " << this);
 223     ESISegmentFreeList (srccontent);
 224     ESISegmentFreeList (altcontent);
 225     cbdataReferenceDone (varState);
 226     safe_free (srcurl);
 227     safe_free (alturl);
 228 }
 229
 230 void
 231 ESIInclude::finish()
 232 {
 233     parent = NULL;
 234 }
 235
 236 ESIElement::Pointer
 237 ESIInclude::makeCacheable() const
 238 {
 239     return new ESIInclude (*this);
 240 }
 241
 242 ESIElement::Pointer
 243 ESIInclude::makeUsable(esiTreeParentPtr newParent, ESIVarState &newVarState) const
 244 {
 245     ESIInclude *resultI = new ESIInclude (*this);
 246     ESIElement::Pointer result = resultI;
 247     resultI->parent = newParent;
 248     resultI->varState = cbdataReference (&newVarState);
 249
 250     if (resultI->srcurl)
 251         resultI->src = ESIStreamContextNew (resultI);
 252
 253     if (resultI->alturl)
 254         resultI->alt = ESIStreamContextNew (resultI);
 255
 256     return result;
 257 }
 258
 259 ESIInclude::ESIInclude(ESIInclude const &old) :
 260     varState(NULL),
 261     srcurl(NULL),
 262     alturl(NULL),
 263     parent(NULL),
 264     started(false),
 265     sent(false)
 266 {
 267     memset(&flags, 0, sizeof(flags));
 268     flags.onerrorcontinue = old.flags.onerrorcontinue;
 269
 270     if (old.srcurl)
 271         srcurl = xstrdup(old.srcurl);
 272
 273     if (old.alturl)
 274         alturl = xstrdup(old.alturl);
 275 }
 276
 277 void
 278 ESIInclude::prepareRequestHeaders(HttpHeader &tempheaders, ESIVarState *vars)
 279 {
 280     tempheaders.update (&vars->header(), NULL);
 281     tempheaders.removeHopByHopEntries();
 282 }
 283
 284 void
 285 ESIInclude::Start (ESIStreamContext::Pointer stream, char const *url, ESIVarState *vars)
 286 {
 287     if (!stream.getRaw())
 288         return;
 289
 290     HttpHeader tempheaders(hoRequest);
 291
 292     prepareRequestHeaders(tempheaders, vars);
 293
 294     /* Ensure variable state is clean */
 295     vars->feedData(url, strlen (url));
 296
 297     /* tempUrl is eaten by the request */
 298     char const *tempUrl = vars->extractChar ();
 299
 300     debugs(86, 5, "ESIIncludeStart: Starting subrequest with url '" << tempUrl << "'");
 301
 302     if (clientBeginRequest(Http::METHOD_GET, tempUrl, esiBufferRecipient, esiBufferDetach, stream.getRaw(), &tempheaders, stream->localbuffer->buf, HTTP_REQBUF_SZ)) {
 303         debugs(86, DBG_CRITICAL, "starting new ESI subrequest failed");
 304     }
 305
 306     tempheaders.clean();
 307 }
 308
 309 ESIInclude::ESIInclude(esiTreeParentPtr aParent, int attrcount, char const **attr, ESIContext *aContext) :
 310     varState(NULL),
 311     srcurl(NULL),
 312     alturl(NULL),
 313     parent(aParent),
 314     started(false),
 315     sent(false)
 316 {
 317     assert (aContext);
 318     memset(&flags, 0, sizeof(flags));
 319
 320     for (int i = 0; i < attrcount && attr[i]; i += 2) {
 321         if (!strcmp(attr[i],"src")) {
 322             /* Start a request for thisNode url */
 323             debugs(86, 5, "ESIIncludeNew: Requesting source '" << attr[i+1] << "'");
 324
 325             /* TODO: don't assert on thisNode, ignore the duplicate */
 326             assert (src.getRaw() == NULL);
 327             src = ESIStreamContextNew (this);
 328             assert (src.getRaw() != NULL);
 329             srcurl = xstrdup(attr[i+1]);
 330         } else if (!strcmp(attr[i],"alt")) {
 331             /* Start a secondary request for thisNode url */
 332             /* TODO: make a config parameter to wait on requesting alt's
 333              * for the src to fail
 334              */
 335             debugs(86, 5, "ESIIncludeNew: Requesting alternate '" << attr[i+1] << "'");
 336
 337             assert (alt.getRaw() == NULL); /* TODO: FIXME */
 338             alt = ESIStreamContextNew (this);
 339             assert (alt.getRaw() != NULL);
 340             alturl = xstrdup(attr[i+1]);
 341         } else if (!strcmp(attr[i],"onerror")) {
 342             if (!strcmp(attr[i+1], "continue")) {
 343                 flags.onerrorcontinue = 1;
 344             } else {
 345                 /* ignore mistyped attributes */
 346                 debugs(86, DBG_IMPORTANT, "invalid value for onerror='" << attr[i+1] << "'");
 347             }
 348         } else {
 349             /* ignore mistyped attributes. TODO:? error on these for user feedback - config parameter needed
 350              */
 351         }
 352     }
 353
 354     varState = cbdataReference(aContext->varState);
 355 }
 356
 357 void
 358 ESIInclude::start()
 359 {
 360     /* prevent freeing ourselves */
 361     ESIIncludePtr foo(this);
 362
 363     if (started)
 364         return;
 365
 366     started = true;
 367
 368     if (src.getRaw()) {
 369         Start (src, srcurl, varState);
 370         Start (alt, alturl, varState);
 371     } else {
 372         alt = NULL;
 373
 374         debugs(86, DBG_IMPORTANT, "ESIIncludeNew: esi:include with no src attributes");
 375
 376         flags.failed = 1;
 377     }
 378 }
 379
 380 void
 381 ESIInclude::render(ESISegment::Pointer output)
 382 {
 383     if (sent)
 384         return;
 385
 386     ESISegment::Pointer myout;
 387
 388     debugs(86, 5, "ESIIncludeRender: Rendering include " << this);
 389
 390     assert (flags.finished || (flags.failed && flags.onerrorcontinue));
 391
 392     if (flags.failed && flags.onerrorcontinue) {
 393         return;
 394     }
 395
 396     /* Render the content */
 397     if (srccontent.getRaw()) {
 398         myout = srccontent;
 399         srccontent = NULL;
 400     } else if (altcontent.getRaw()) {
 401         myout = altcontent;
 402         altcontent = NULL;
 403     } else
 404         fatal ("ESIIncludeRender called with no content, and no failure!\n");
 405
 406     assert (output->next == NULL);
 407
 408     output->next = myout;
 409
 410     sent = true;
 411 }
 412
 413 esiProcessResult_t
 414 ESIInclude::process (int dovars)
 415 {
 416     /* Prevent refcount race leading to free */
 417     Pointer me (this);
 418     start();
 419     debugs(86, 5, "ESIIncludeRender: Processing include " << this);
 420
 421     if (flags.failed) {
 422         if (flags.onerrorcontinue)
 423             return ESI_PROCESS_COMPLETE;
 424         else
 425             return ESI_PROCESS_FAILED;
 426     }
 427
 428     if (!flags.finished) {
 429         if (flags.onerrorcontinue)
 430             return ESI_PROCESS_PENDING_WONTFAIL;
 431         else
 432             return ESI_PROCESS_PENDING_MAYFAIL;
 433     }
 434
 435     return ESI_PROCESS_COMPLETE;
 436 }
 437
 438 void
 439 ESIInclude::includeFail (ESIStreamContext::Pointer stream)
 440 {
 441     subRequestDone (stream, false);
 442 }
 443
 444 bool
 445 ESIInclude::dataNeeded() const
 446 {
 447     return !(flags.finished || flags.failed);
 448 }
 449
 450 void
 451 ESIInclude::subRequestDone (ESIStreamContext::Pointer stream, bool success)
 452 {
 453     if (!dataNeeded())
 454         return;
 455
 456     if (stream == src) {
 457         debugs(86, 3, "ESIInclude::subRequestDone: " << srcurl);
 458
 459         if (success) {
 460             /* copy the lead segment */
 461             debugs(86, 3, "ESIIncludeSubRequestDone: Src OK - include PASSED.");
 462             assert (!srccontent.getRaw());
 463             ESISegment::ListTransfer (stream->localbuffer, srccontent);
 464             /* we're done! */
 465             flags.finished = 1;
 466         } else {
 467             /* Fail if there is no alt being retrieved */
 468             debugs(86, 3, "ESIIncludeSubRequestDone: Src FAILED");
 469
 470             if (!(alt.getRaw() || altcontent.getRaw())) {
 471                 debugs(86, 3, "ESIIncludeSubRequestDone: Include FAILED - No ALT");
 472                 flags.failed = 1;
 473             } else if (altcontent.getRaw()) {
 474                 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - ALT already Complete");
 475                 /* ALT was already retrieved, we are done */
 476                 flags.finished = 1;
 477             }
 478         }
 479
 480         src = NULL;
 481     } else if (stream == alt) {
 482         debugs(86, 3, "ESIInclude::subRequestDone: " << alturl);
 483
 484         if (success) {
 485             debugs(86, 3, "ESIIncludeSubRequestDone: ALT OK.");
 486             /* copy the lead segment */
 487             assert (!altcontent.getRaw());
 488             ESISegment::ListTransfer (stream->localbuffer, altcontent);
 489             /* we're done! */
 490
 491             if (!(src.getRaw() || srccontent.getRaw())) {
 492                 /* src already failed, kick ESI processor */
 493                 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - SRC already failed.");
 494                 flags.finished = 1;
 495             }
 496         } else {
 497             if (!(src.getRaw() || srccontent.getRaw())) {
 498                 debugs(86, 3, "ESIIncludeSubRequestDone: ALT FAILED, Include FAILED - SRC already failed");
 499                 /* src already failed */
 500                 flags.failed = 1;
 501             }
 502         }
 503
 504         alt = NULL;
 505     } else {
 506         fatal ("ESIIncludeSubRequestDone: non-owned stream found!\n");
 507     }
 508
 509     if (flags.finished || flags.failed) {
 510         /* Kick ESI Processor */
 511         debugs (86, 5, "ESIInclude " << this <<
 512                 " SubRequest " << stream.getRaw() <<
 513                 " completed, kicking processor , status " <<
 514                 (flags.finished ? "OK" : "FAILED"));
 515         /* There is a race condition - and we have no reproducible test case -
 516          * during a subrequest the parent will get set to NULL, which is not
 517          * meant to be possible. Rather than killing squid, we let it leak
 518          * memory but complain in the log.
 519          *
 520          * Someone wanting to debug this could well start by running squid with
 521          * a hardware breakpoint set to this location.
 522          * Its probably due to parent being set to null - by a call to
 523          * 'this.finish' while the subrequest is still not completed.
 524          */
 525         if (parent.getRaw() == NULL) {
 526             debugs (86, 0, "ESIInclude::subRequestDone: Sub request completed "
 527                     "after finish() called and parent unlinked. Unable to "
 528                     "continue handling the request, and may be memory leaking. "
 529                     "See http://www.squid-cache.org/bugs/show_bug.cgi?id=951 - we "
 530                     "are looking for a reproducible test case. This will require "
 531                     "an ESI template with includes, probably with alt-options, "
 532                     "and we're likely to need traffic dumps to allow us to "
 533                     "reconstruct the exact tcp handling sequences to trigger this "
 534                     "rather elusive bug.");
 535             return;
 536         }
 537         assert (parent.getRaw());
 538
 539         if (!flags.failed) {
 540             sent = true;
 541             parent->provideData (srccontent.getRaw() ? srccontent:altcontent,this);
 542
 543             if (srccontent.getRaw())
 544                 srccontent = NULL;
 545             else
 546                 altcontent = NULL;
 547         } else if (flags.onerrorcontinue) {
 548             /* render nothing but inform of completion */
 549
 550             if (!sent) {
 551                 sent = true;
 552                 parent->provideData (new ESISegment, this);
 553             } else
 554                 assert (0);
 555         } else
 556             parent->fail(this, "esi:include could not be completed.");
 557     }
 558 }
 559
 560 #endif /* USE_SQUID_ESI */
 561