src/esi/Include.cc

   1 /*
   2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 86    ESI processing */
  10
  11 #include "squid.h"
  12
  13 #if USE_SQUID_ESI
  14
  15 #include "client_side.h"
  16 #include "client_side_request.h"
  17 #include "esi/Include.h"
  18 #include "esi/VarState.h"
  19 #include "fatal.h"
  20 #include "http/Stream.h"
  21 #include "HttpReply.h"
  22 #include "log/access_log.h"
  23
  24 CBDATA_CLASS_INIT (ESIStreamContext);
  25
  26 /* other */
  27 static CSCB esiBufferRecipient;
  28 static CSD esiBufferDetach;
  29 /* esiStreamContext */
  30 static ESIStreamContext *ESIStreamContextNew (ESIIncludePtr);
  31
  32 /* ESI TO CONSIDER:
  33  * 1. retry failed upstream requests
  34  */
  35
  36 /* Detach from a buffering stream
  37  */
  38 void
  39 esiBufferDetach (clientStreamNode *node, ClientHttpRequest *http)
  40 {
  41     /* Detach ourselves */
  42     clientStreamDetach (node, http);
  43 }
  44
  45 /**
  46  * Write a chunk of data to a client 'socket'.
  47  * If the reply is present, send the reply headers down the wire too.
  48  *
  49  * Pre-condition:
  50  *   The request is an internal ESI subrequest.
  51  *   data context is not NULL
  52  *   There are no more entries in the stream chain.
  53  *   The caller is responsible for creation and deletion of the Reply headers.
  54  *
  55  \note
  56  * Bug 975, bug 1566 : delete rep; 2006/09/02: TS, #975
  57  *
  58  * This was causing double-deletes. Its possible that not deleting
  59  * it here will cause memory leaks, but if so, this delete should
  60  * not be reinstated or it will trigger bug #975 again - RBC 20060903
  61  */
  62 void
  63 esiBufferRecipient (clientStreamNode *node, ClientHttpRequest *http, HttpReply *rep, StoreIOBuffer receivedData)
  64 {
  65     /* Test preconditions */
  66     assert (node != NULL);
  67     /* ESI TODO: handle thisNode rather than asserting
  68      * - it should only ever happen if we cause an
  69      * abort and the callback chain loops back to
  70      * here, so we can simply return. However, that
  71      * itself shouldn't happen, so it stays as an
  72      * assert for now. */
  73     assert (cbdataReferenceValid (node));
  74     assert (node->node.next == NULL);
  75     assert (http->getConn() == NULL);
  76
  77     ESIStreamContext::Pointer esiStream = dynamic_cast<ESIStreamContext *>(node->data.getRaw());
  78     assert (esiStream.getRaw() != NULL);
  79     /* If segments become more flexible, ignore thisNode */
  80     assert (receivedData.length <= sizeof(esiStream->localbuffer->buf));
  81     assert (!esiStream->finished);
  82
  83     debugs (86,5, HERE << "rep " << rep << " body " << receivedData.data << " len " << receivedData.length);
  84     assert (node->readBuffer.offset == receivedData.offset || receivedData.length == 0);
  85
  86     /* trivial case */
  87
  88     if (http->out.offset != 0) {
  89         assert(rep == NULL);
  90     } else {
  91         if (rep) {
  92             if (rep->sline.status() != Http::scOkay) {
  93                 rep = NULL;
  94                 esiStream->include->includeFail (esiStream);
  95                 esiStream->finished = 1;
  96                 httpRequestFree (http);
  97                 return;
  98             }
  99
 100 #if HEADERS_LOG
 101             /* should be done in the store rather than every recipient?  */
 102             headersLog(0, 0, http->request->method, rep);
 103
 104 #endif
 105             rep = NULL;
 106         }
 107     }
 108
 109     if (receivedData.data && receivedData.length) {
 110         http->out.offset += receivedData.length;
 111
 112         if (receivedData.data >= esiStream->localbuffer->buf &&
 113                 receivedData.data < &esiStream->localbuffer->buf[sizeof(esiStream->localbuffer->buf)]) {
 114             /* original static buffer */
 115
 116             if (receivedData.data != esiStream->localbuffer->buf) {
 117                 /* But not the start of it */
 118                 memmove(esiStream->localbuffer->buf, receivedData.data, receivedData.length);
 119             }
 120
 121             esiStream->localbuffer->len = receivedData.length;
 122         } else {
 123             assert (esiStream->buffer.getRaw() != NULL);
 124             esiStream->buffer->len = receivedData.length;
 125         }
 126     }
 127
 128     /* EOF / Read error /  aborted entry */
 129     if (rep == NULL && receivedData.data == NULL && receivedData.length == 0) {
 130         /* TODO: get stream status to test the entry for aborts */
 131         debugs(86, 5, HERE << "Finished reading upstream data in subrequest");
 132         esiStream->include->subRequestDone (esiStream, true);
 133         esiStream->finished = 1;
 134         httpRequestFree (http);
 135         return;
 136     }
 137
 138     /* after the write to the user occurs, (ie here, or in a callback)
 139      * we call */
 140     if (clientHttpRequestStatus(-1, http)) {
 141         /* TODO: Does thisNode if block leak htto ? */
 142         /* XXX when reviewing ESI this is the first place to look */
 143         node->data = NULL;
 144         esiStream->finished = 1;
 145         esiStream->include->includeFail (esiStream);
 146         return;
 147     };
 148
 149     switch (clientStreamStatus (node, http)) {
 150
 151     case STREAM_UNPLANNED_COMPLETE: /* fallthru ok */
 152
 153     case STREAM_COMPLETE: /* ok */
 154         debugs(86, 3, "ESI subrequest finished OK");
 155         esiStream->include->subRequestDone (esiStream, true);
 156         esiStream->finished = 1;
 157         httpRequestFree (http);
 158         return;
 159
 160     case STREAM_FAILED:
 161         debugs(86, DBG_IMPORTANT, "ESI subrequest failed transfer");
 162         esiStream->include->includeFail (esiStream);
 163         esiStream->finished = 1;
 164         httpRequestFree (http);
 165         return;
 166
 167     case STREAM_NONE: {
 168         StoreIOBuffer tempBuffer;
 169
 170         if (!esiStream->buffer.getRaw()) {
 171             esiStream->buffer = esiStream->localbuffer;
 172         }
 173
 174         esiStream->buffer = esiStream->buffer->tail();
 175
 176         if (esiStream->buffer->len) {
 177             esiStream->buffer->next = new ESISegment;
 178             esiStream->buffer = esiStream->buffer->next;
 179         }
 180
 181         tempBuffer.offset = http->out.offset;
 182         tempBuffer.length = sizeof (esiStream->buffer->buf);
 183         tempBuffer.data = esiStream->buffer->buf;
 184         /* now just read into 'buffer' */
 185         clientStreamRead (node, http, tempBuffer);
 186         debugs(86, 5, HERE << "Requested more data for ESI subrequest");
 187     }
 188
 189     break;
 190
 191     default:
 192         fatal ("Hit unreachable code in esiBufferRecipient\n");
 193     }
 194
 195 }
 196
 197 /* esiStream functions */
 198 ESIStreamContext::~ESIStreamContext()
 199 {
 200     freeResources();
 201 }
 202
 203 void
 204 ESIStreamContext::freeResources()
 205 {
 206     debugs(86, 5, "Freeing stream context resources.");
 207     buffer = NULL;
 208     localbuffer = NULL;
 209     include = NULL;
 210 }
 211
 212 ESIStreamContext *
 213 ESIStreamContextNew (ESIIncludePtr include)
 214 {
 215     ESIStreamContext *rv = new ESIStreamContext;
 216     rv->include = include;
 217     return rv;
 218 }
 219
 220 /* ESIInclude */
 221 ESIInclude::~ESIInclude()
 222 {
 223     debugs(86, 5, "ESIInclude::Free " << this);
 224     ESISegmentFreeList (srccontent);
 225     ESISegmentFreeList (altcontent);
 226     cbdataReferenceDone (varState);
 227     safe_free (srcurl);
 228     safe_free (alturl);
 229 }
 230
 231 void
 232 ESIInclude::finish()
 233 {
 234     parent = NULL;
 235 }
 236
 237 ESIElement::Pointer
 238 ESIInclude::makeCacheable() const
 239 {
 240     return new ESIInclude (*this);
 241 }
 242
 243 ESIElement::Pointer
 244 ESIInclude::makeUsable(esiTreeParentPtr newParent, ESIVarState &newVarState) const
 245 {
 246     ESIInclude *resultI = new ESIInclude (*this);
 247     ESIElement::Pointer result = resultI;
 248     resultI->parent = newParent;
 249     resultI->varState = cbdataReference (&newVarState);
 250
 251     if (resultI->srcurl)
 252         resultI->src = ESIStreamContextNew (resultI);
 253
 254     if (resultI->alturl)
 255         resultI->alt = ESIStreamContextNew (resultI);
 256
 257     return result;
 258 }
 259
 260 ESIInclude::ESIInclude(ESIInclude const &old) :
 261     varState(NULL),
 262     srcurl(NULL),
 263     alturl(NULL),
 264     parent(NULL),
 265     started(false),
 266     sent(false)
 267 {
 268     memset(&flags, 0, sizeof(flags));
 269     flags.onerrorcontinue = old.flags.onerrorcontinue;
 270
 271     if (old.srcurl)
 272         srcurl = xstrdup(old.srcurl);
 273
 274     if (old.alturl)
 275         alturl = xstrdup(old.alturl);
 276 }
 277
 278 void
 279 ESIInclude::prepareRequestHeaders(HttpHeader &tempheaders, ESIVarState *vars)
 280 {
 281     tempheaders.update(&vars->header());
 282     tempheaders.removeHopByHopEntries();
 283 }
 284
 285 void
 286 ESIInclude::Start (ESIStreamContext::Pointer stream, char const *url, ESIVarState *vars)
 287 {
 288     if (!stream.getRaw())
 289         return;
 290
 291     HttpHeader tempheaders(hoRequest);
 292
 293     prepareRequestHeaders(tempheaders, vars);
 294
 295     /* Ensure variable state is clean */
 296     vars->feedData(url, strlen (url));
 297
 298     /* tempUrl is eaten by the request */
 299     char const *tempUrl = vars->extractChar ();
 300
 301     debugs(86, 5, "ESIIncludeStart: Starting subrequest with url '" << tempUrl << "'");
 302
 303     if (clientBeginRequest(Http::METHOD_GET, tempUrl, esiBufferRecipient, esiBufferDetach, stream.getRaw(), &tempheaders, stream->localbuffer->buf, HTTP_REQBUF_SZ)) {
 304         debugs(86, DBG_CRITICAL, "starting new ESI subrequest failed");
 305     }
 306
 307     tempheaders.clean();
 308 }
 309
 310 ESIInclude::ESIInclude(esiTreeParentPtr aParent, int attrcount, char const **attr, ESIContext *aContext) :
 311     varState(NULL),
 312     srcurl(NULL),
 313     alturl(NULL),
 314     parent(aParent),
 315     started(false),
 316     sent(false)
 317 {
 318     assert (aContext);
 319     memset(&flags, 0, sizeof(flags));
 320
 321     for (int i = 0; i < attrcount && attr[i]; i += 2) {
 322         if (!strcmp(attr[i],"src")) {
 323             /* Start a request for thisNode url */
 324             debugs(86, 5, "ESIIncludeNew: Requesting source '" << attr[i+1] << "'");
 325
 326             /* TODO: don't assert on thisNode, ignore the duplicate */
 327             assert (src.getRaw() == NULL);
 328             src = ESIStreamContextNew (this);
 329             assert (src.getRaw() != NULL);
 330             srcurl = xstrdup(attr[i+1]);
 331         } else if (!strcmp(attr[i],"alt")) {
 332             /* Start a secondary request for thisNode url */
 333             /* TODO: make a config parameter to wait on requesting alt's
 334              * for the src to fail
 335              */
 336             debugs(86, 5, "ESIIncludeNew: Requesting alternate '" << attr[i+1] << "'");
 337
 338             assert (alt.getRaw() == NULL); /* TODO: FIXME */
 339             alt = ESIStreamContextNew (this);
 340             assert (alt.getRaw() != NULL);
 341             alturl = xstrdup(attr[i+1]);
 342         } else if (!strcmp(attr[i],"onerror")) {
 343             if (!strcmp(attr[i+1], "continue")) {
 344                 flags.onerrorcontinue = 1;
 345             } else {
 346                 /* ignore mistyped attributes */
 347                 debugs(86, DBG_IMPORTANT, "invalid value for onerror='" << attr[i+1] << "'");
 348             }
 349         } else {
 350             /* ignore mistyped attributes. TODO:? error on these for user feedback - config parameter needed
 351              */
 352         }
 353     }
 354
 355     varState = cbdataReference(aContext->varState);
 356 }
 357
 358 void
 359 ESIInclude::start()
 360 {
 361     /* prevent freeing ourselves */
 362     ESIIncludePtr foo(this);
 363
 364     if (started)
 365         return;
 366
 367     started = true;
 368
 369     if (src.getRaw()) {
 370         Start (src, srcurl, varState);
 371         Start (alt, alturl, varState);
 372     } else {
 373         alt = NULL;
 374
 375         debugs(86, DBG_IMPORTANT, "ESIIncludeNew: esi:include with no src attributes");
 376
 377         flags.failed = 1;
 378     }
 379 }
 380
 381 void
 382 ESIInclude::render(ESISegment::Pointer output)
 383 {
 384     if (sent)
 385         return;
 386
 387     ESISegment::Pointer myout;
 388
 389     debugs(86, 5, "ESIIncludeRender: Rendering include " << this);
 390
 391     assert (flags.finished || (flags.failed && flags.onerrorcontinue));
 392
 393     if (flags.failed && flags.onerrorcontinue) {
 394         return;
 395     }
 396
 397     /* Render the content */
 398     if (srccontent.getRaw()) {
 399         myout = srccontent;
 400         srccontent = NULL;
 401     } else if (altcontent.getRaw()) {
 402         myout = altcontent;
 403         altcontent = NULL;
 404     } else
 405         fatal ("ESIIncludeRender called with no content, and no failure!\n");
 406
 407     assert (output->next == NULL);
 408
 409     output->next = myout;
 410
 411     sent = true;
 412 }
 413
 414 esiProcessResult_t
 415 ESIInclude::process (int dovars)
 416 {
 417     /* Prevent refcount race leading to free */
 418     Pointer me (this);
 419     start();
 420     debugs(86, 5, "ESIIncludeRender: Processing include " << this);
 421
 422     if (flags.failed) {
 423         if (flags.onerrorcontinue)
 424             return ESI_PROCESS_COMPLETE;
 425         else
 426             return ESI_PROCESS_FAILED;
 427     }
 428
 429     if (!flags.finished) {
 430         if (flags.onerrorcontinue)
 431             return ESI_PROCESS_PENDING_WONTFAIL;
 432         else
 433             return ESI_PROCESS_PENDING_MAYFAIL;
 434     }
 435
 436     return ESI_PROCESS_COMPLETE;
 437 }
 438
 439 void
 440 ESIInclude::includeFail (ESIStreamContext::Pointer stream)
 441 {
 442     subRequestDone (stream, false);
 443 }
 444
 445 bool
 446 ESIInclude::dataNeeded() const
 447 {
 448     return !(flags.finished || flags.failed);
 449 }
 450
 451 void
 452 ESIInclude::subRequestDone (ESIStreamContext::Pointer stream, bool success)
 453 {
 454     if (!dataNeeded())
 455         return;
 456
 457     if (stream == src) {
 458         debugs(86, 3, "ESIInclude::subRequestDone: " << srcurl);
 459
 460         if (success) {
 461             /* copy the lead segment */
 462             debugs(86, 3, "ESIIncludeSubRequestDone: Src OK - include PASSED.");
 463             assert (!srccontent.getRaw());
 464             ESISegment::ListTransfer (stream->localbuffer, srccontent);
 465             /* we're done! */
 466             flags.finished = 1;
 467         } else {
 468             /* Fail if there is no alt being retrieved */
 469             debugs(86, 3, "ESIIncludeSubRequestDone: Src FAILED");
 470
 471             if (!(alt.getRaw() || altcontent.getRaw())) {
 472                 debugs(86, 3, "ESIIncludeSubRequestDone: Include FAILED - No ALT");
 473                 flags.failed = 1;
 474             } else if (altcontent.getRaw()) {
 475                 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - ALT already Complete");
 476                 /* ALT was already retrieved, we are done */
 477                 flags.finished = 1;
 478             }
 479         }
 480
 481         src = NULL;
 482     } else if (stream == alt) {
 483         debugs(86, 3, "ESIInclude::subRequestDone: " << alturl);
 484
 485         if (success) {
 486             debugs(86, 3, "ESIIncludeSubRequestDone: ALT OK.");
 487             /* copy the lead segment */
 488             assert (!altcontent.getRaw());
 489             ESISegment::ListTransfer (stream->localbuffer, altcontent);
 490             /* we're done! */
 491
 492             if (!(src.getRaw() || srccontent.getRaw())) {
 493                 /* src already failed, kick ESI processor */
 494                 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - SRC already failed.");
 495                 flags.finished = 1;
 496             }
 497         } else {
 498             if (!(src.getRaw() || srccontent.getRaw())) {
 499                 debugs(86, 3, "ESIIncludeSubRequestDone: ALT FAILED, Include FAILED - SRC already failed");
 500                 /* src already failed */
 501                 flags.failed = 1;
 502             }
 503         }
 504
 505         alt = NULL;
 506     } else {
 507         fatal ("ESIIncludeSubRequestDone: non-owned stream found!\n");
 508     }
 509
 510     if (flags.finished || flags.failed) {
 511         /* Kick ESI Processor */
 512         debugs (86, 5, "ESIInclude " << this <<
 513                 " SubRequest " << stream.getRaw() <<
 514                 " completed, kicking processor , status " <<
 515                 (flags.finished ? "OK" : "FAILED"));
 516         /* There is a race condition - and we have no reproducible test case -
 517          * during a subrequest the parent will get set to NULL, which is not
 518          * meant to be possible. Rather than killing squid, we let it leak
 519          * memory but complain in the log.
 520          *
 521          * Someone wanting to debug this could well start by running squid with
 522          * a hardware breakpoint set to this location.
 523          * Its probably due to parent being set to null - by a call to
 524          * 'this.finish' while the subrequest is still not completed.
 525          */
 526         if (parent.getRaw() == NULL) {
 527             debugs (86, 0, "ESIInclude::subRequestDone: Sub request completed "
 528                     "after finish() called and parent unlinked. Unable to "
 529                     "continue handling the request, and may be memory leaking. "
 530                     "See http://www.squid-cache.org/bugs/show_bug.cgi?id=951 - we "
 531                     "are looking for a reproducible test case. This will require "
 532                     "an ESI template with includes, probably with alt-options, "
 533                     "and we're likely to need traffic dumps to allow us to "
 534                     "reconstruct the exact tcp handling sequences to trigger this "
 535                     "rather elusive bug.");
 536             return;
 537         }
 538         assert (parent.getRaw());
 539
 540         if (!flags.failed) {
 541             sent = true;
 542             parent->provideData (srccontent.getRaw() ? srccontent:altcontent,this);
 543
 544             if (srccontent.getRaw())
 545                 srccontent = NULL;
 546             else
 547                 altcontent = NULL;
 548         } else if (flags.onerrorcontinue) {
 549             /* render nothing but inform of completion */
 550
 551             if (!sent) {
 552                 sent = true;
 553                 parent->provideData (new ESISegment, this);
 554             } else
 555                 assert (0);
 556         } else
 557             parent->fail(this, "esi:include could not be completed.");
 558     }
 559 }
 560
 561 #endif /* USE_SQUID_ESI */
 562