src/esi/Include.cc

   1
   2 /*
   3  * DEBUG: section 86    ESI processing
   4  * AUTHOR: Robert Collins
   5  *
   6  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   7  * ----------------------------------------------------------
   8  *
   9  *  Squid is the result of efforts by numerous individuals from
  10  *  the Internet community; see the CONTRIBUTORS file for full
  11  *  details.   Many organizations have provided support for Squid's
  12  *  development; see the SPONSORS file for full details.  Squid is
  13  *  Copyrighted (C) 2001 by the Regents of the University of
  14  *  California; see the COPYRIGHT file for full details.  Squid
  15  *  incorporates software developed and/or copyrighted by other
  16  *  sources; see the CREDITS file for full details.
  17  *
  18  *  This program is free software; you can redistribute it and/or modify
  19  *  it under the terms of the GNU General Public License as published by
  20  *  the Free Software Foundation; either version 2 of the License, or
  21  *  (at your option) any later version.
  22  *
  23  *  This program is distributed in the hope that it will be useful,
  24  ;  but WITHOUT ANY WARRANTY; without even the implied warranty of
  25  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  26  *  GNU General Public License for more details.
  27  *
  28  *  You should have received a copy of the GNU General Public License
  29  *  along with this program; if not, write to the Free Software
  30  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  31  *
  32  * Copyright (c) 2003, Robert Collins <robertc@squid-cache.org>
  33  */
  34
  35 #include "squid.h"
  36
  37 /* MS Visual Studio Projects are monolithic, so we need the following
  38  * #if to exclude the ESI code from compile process when not needed.
  39  */
  40 #if (USE_SQUID_ESI == 1)
  41
  42 #include "client_side.h"
  43 #include "client_side_request.h"
  44 #include "esi/Include.h"
  45 #include "esi/VarState.h"
  46 #include "HttpReply.h"
  47 #include "log/access_log.h"
  48
  49 CBDATA_CLASS_INIT (ESIStreamContext);
  50
  51 /* other */
  52 static CSCB esiBufferRecipient;
  53 static CSD esiBufferDetach;
  54 /* esiStreamContext */
  55 static ESIStreamContext *ESIStreamContextNew (ESIIncludePtr);
  56
  57 /* ESI TO CONSIDER:
  58  * 1. retry failed upstream requests
  59  */
  60
  61 /* Detach from a buffering stream
  62  */
  63 void
  64 esiBufferDetach (clientStreamNode *node, ClientHttpRequest *http)
  65 {
  66     /* Detach ourselves */
  67     clientStreamDetach (node, http);
  68 }
  69
  70 /**
  71  * Write a chunk of data to a client 'socket'.
  72  * If the reply is present, send the reply headers down the wire too.
  73  *
  74  * Pre-condition:
  75  *   The request is an internal ESI subrequest.
  76  *   data context is not NULL
  77  *   There are no more entries in the stream chain.
  78  *   The caller is responsible for creation and deletion of the Reply headers.
  79  *
  80  \note
  81  * Bug 975, bug 1566 : delete rep; 2006/09/02: TS, #975
  82  *
  83  * This was causing double-deletes. Its possible that not deleting
  84  * it here will cause memory leaks, but if so, this delete should
  85  * not be reinstated or it will trigger bug #975 again - RBC 20060903
  86  */
  87 void
  88 esiBufferRecipient (clientStreamNode *node, ClientHttpRequest *http, HttpReply *rep, StoreIOBuffer receivedData)
  89 {
  90     /* Test preconditions */
  91     assert (node != NULL);
  92     /* ESI TODO: handle thisNode rather than asserting
  93      * - it should only ever happen if we cause an
  94      * abort and the callback chain loops back to
  95      * here, so we can simply return. However, that
  96      * itself shouldn't happen, so it stays as an
  97      * assert for now. */
  98     assert (cbdataReferenceValid (node));
  99     assert (node->node.next == NULL);
 100     assert (http->getConn() == NULL);
 101
 102     ESIStreamContext::Pointer esiStream = dynamic_cast<ESIStreamContext *>(node->data.getRaw());
 103     assert (esiStream.getRaw() != NULL);
 104     /* If segments become more flexible, ignore thisNode */
 105     assert (receivedData.length <= sizeof(esiStream->localbuffer->buf));
 106     assert (!esiStream->finished);
 107
 108     debugs (86,5, HERE << "rep " << rep << " body " << receivedData.data << " len " << receivedData.length);
 109     assert (node->readBuffer.offset == receivedData.offset || receivedData.length == 0);
 110
 111     /* trivial case */
 112
 113     if (http->out.offset != 0) {
 114         assert(rep == NULL);
 115     } else {
 116         if (rep) {
 117             if (rep->sline.status() != Http::scOkay) {
 118                 rep = NULL;
 119                 esiStream->include->includeFail (esiStream);
 120                 esiStream->finished = 1;
 121                 httpRequestFree (http);
 122                 return;
 123             }
 124
 125 #if HEADERS_LOG
 126             /* should be done in the store rather than every recipient?  */
 127             headersLog(0, 0, http->request->method, rep);
 128
 129 #endif
 130             rep = NULL;
 131         }
 132     }
 133
 134     if (receivedData.data && receivedData.length) {
 135         http->out.offset += receivedData.length;
 136
 137         if (receivedData.data >= esiStream->localbuffer->buf &&
 138                 receivedData.data < &esiStream->localbuffer->buf[sizeof(esiStream->localbuffer->buf)]) {
 139             /* original static buffer */
 140
 141             if (receivedData.data != esiStream->localbuffer->buf) {
 142                 /* But not the start of it */
 143                 memmove(esiStream->localbuffer->buf, receivedData.data, receivedData.length);
 144             }
 145
 146             esiStream->localbuffer->len = receivedData.length;
 147         } else {
 148             assert (esiStream->buffer.getRaw() != NULL);
 149             esiStream->buffer->len = receivedData.length;
 150         }
 151     }
 152
 153     /* EOF / Read error /  aborted entry */
 154     if (rep == NULL && receivedData.data == NULL && receivedData.length == 0) {
 155         /* TODO: get stream status to test the entry for aborts */
 156         debugs(86, 5, HERE << "Finished reading upstream data in subrequest");
 157         esiStream->include->subRequestDone (esiStream, true);
 158         esiStream->finished = 1;
 159         httpRequestFree (http);
 160         return;
 161     }
 162
 163     /* after the write to the user occurs, (ie here, or in a callback)
 164      * we call */
 165     if (clientHttpRequestStatus(-1, http)) {
 166         /* TODO: Does thisNode if block leak htto ? */
 167         /* XXX when reviewing ESI this is the first place to look */
 168         node->data = NULL;
 169         esiStream->finished = 1;
 170         esiStream->include->includeFail (esiStream);
 171         return;
 172     };
 173
 174     switch (clientStreamStatus (node, http)) {
 175
 176     case STREAM_UNPLANNED_COMPLETE: /* fallthru ok */
 177
 178     case STREAM_COMPLETE: /* ok */
 179         debugs(86, 3, "ESI subrequest finished OK");
 180         esiStream->include->subRequestDone (esiStream, true);
 181         esiStream->finished = 1;
 182         httpRequestFree (http);
 183         return;
 184
 185     case STREAM_FAILED:
 186         debugs(86, DBG_IMPORTANT, "ESI subrequest failed transfer");
 187         esiStream->include->includeFail (esiStream);
 188         esiStream->finished = 1;
 189         httpRequestFree (http);
 190         return;
 191
 192     case STREAM_NONE: {
 193         StoreIOBuffer tempBuffer;
 194
 195         if (!esiStream->buffer.getRaw()) {
 196             esiStream->buffer = esiStream->localbuffer;
 197         }
 198
 199         esiStream->buffer = esiStream->buffer->tail();
 200
 201         if (esiStream->buffer->len) {
 202             esiStream->buffer->next = new ESISegment;
 203             esiStream->buffer = esiStream->buffer->next;
 204         }
 205
 206         tempBuffer.offset = http->out.offset;
 207         tempBuffer.length = sizeof (esiStream->buffer->buf);
 208         tempBuffer.data = esiStream->buffer->buf;
 209         /* now just read into 'buffer' */
 210         clientStreamRead (node, http, tempBuffer);
 211         debugs(86, 5, HERE << "Requested more data for ESI subrequest");
 212     }
 213
 214     break;
 215
 216     default:
 217         fatal ("Hit unreachable code in esiBufferRecipient\n");
 218     }
 219
 220 }
 221
 222 /* esiStream functions */
 223 ESIStreamContext::~ESIStreamContext()
 224 {
 225     assert (this);
 226     freeResources();
 227 }
 228
 229 void
 230 ESIStreamContext::freeResources()
 231 {
 232     debugs(86, 5, "Freeing stream context resources.");
 233     buffer = NULL;
 234     localbuffer = NULL;
 235     include = NULL;
 236 }
 237
 238 ESIStreamContext *
 239 ESIStreamContextNew (ESIIncludePtr include)
 240 {
 241     ESIStreamContext *rv = new ESIStreamContext;
 242     rv->include = include;
 243     return rv;
 244 }
 245
 246 /* ESIInclude */
 247 ESIInclude::~ESIInclude()
 248 {
 249     debugs(86, 5, "ESIInclude::Free " << this);
 250     ESISegmentFreeList (srccontent);
 251     ESISegmentFreeList (altcontent);
 252     cbdataReferenceDone (varState);
 253     safe_free (srcurl);
 254     safe_free (alturl);
 255 }
 256
 257 void
 258 ESIInclude::finish()
 259 {
 260     parent = NULL;
 261 }
 262
 263 ESIElement::Pointer
 264 ESIInclude::makeCacheable() const
 265 {
 266     return new ESIInclude (*this);
 267 }
 268
 269 ESIElement::Pointer
 270 ESIInclude::makeUsable(esiTreeParentPtr newParent, ESIVarState &newVarState) const
 271 {
 272     ESIInclude *resultI = new ESIInclude (*this);
 273     ESIElement::Pointer result = resultI;
 274     resultI->parent = newParent;
 275     resultI->varState = cbdataReference (&newVarState);
 276
 277     if (resultI->srcurl)
 278         resultI->src = ESIStreamContextNew (resultI);
 279
 280     if (resultI->alturl)
 281         resultI->alt = ESIStreamContextNew (resultI);
 282
 283     return result;
 284 }
 285
 286 ESIInclude::ESIInclude(ESIInclude const &old) :
 287         varState(NULL),
 288         srcurl(NULL),
 289         alturl(NULL),
 290         parent(NULL),
 291         started(false),
 292         sent(false)
 293 {
 294     memset(&flags, 0, sizeof(flags));
 295     flags.onerrorcontinue = old.flags.onerrorcontinue;
 296
 297     if (old.srcurl)
 298         srcurl = xstrdup (old.srcurl);
 299
 300     if (old.alturl)
 301         alturl = xstrdup (old.alturl);
 302 }
 303
 304 void
 305 ESIInclude::prepareRequestHeaders(HttpHeader &tempheaders, ESIVarState *vars)
 306 {
 307     tempheaders.update (&vars->header(), NULL);
 308     tempheaders.removeHopByHopEntries();
 309 }
 310
 311 void
 312 ESIInclude::Start (ESIStreamContext::Pointer stream, char const *url, ESIVarState *vars)
 313 {
 314     if (!stream.getRaw())
 315         return;
 316
 317     HttpHeader tempheaders(hoRequest);
 318
 319     prepareRequestHeaders(tempheaders, vars);
 320
 321     /* Ensure variable state is clean */
 322     vars->feedData(url, strlen (url));
 323
 324     /* tempUrl is eaten by the request */
 325     char const *tempUrl = vars->extractChar ();
 326
 327     debugs(86, 5, "ESIIncludeStart: Starting subrequest with url '" << tempUrl << "'");
 328
 329     if (clientBeginRequest(Http::METHOD_GET, tempUrl, esiBufferRecipient, esiBufferDetach, stream.getRaw(), &tempheaders, stream->localbuffer->buf, HTTP_REQBUF_SZ)) {
 330         debugs(86, DBG_CRITICAL, "starting new ESI subrequest failed");
 331     }
 332
 333     tempheaders.clean();
 334 }
 335
 336 ESIInclude::ESIInclude(esiTreeParentPtr aParent, int attrcount, char const **attr, ESIContext *aContext) :
 337         varState(NULL),
 338         srcurl(NULL),
 339         alturl(NULL),
 340         parent(aParent),
 341         started(false),
 342         sent(false)
 343 {
 344     assert (aContext);
 345     memset(&flags, 0, sizeof(flags));
 346
 347     for (int i = 0; i < attrcount && attr[i]; i += 2) {
 348         if (!strcmp(attr[i],"src")) {
 349             /* Start a request for thisNode url */
 350             debugs(86, 5, "ESIIncludeNew: Requesting source '" << attr[i+1] << "'");
 351
 352             /* TODO: don't assert on thisNode, ignore the duplicate */
 353             assert (src.getRaw() == NULL);
 354             src = ESIStreamContextNew (this);
 355             assert (src.getRaw() != NULL);
 356             srcurl = xstrdup ( attr[i+1]);
 357         } else if (!strcmp(attr[i],"alt")) {
 358             /* Start a secondary request for thisNode url */
 359             /* TODO: make a config parameter to wait on requesting alt's
 360              * for the src to fail
 361              */
 362             debugs(86, 5, "ESIIncludeNew: Requesting alternate '" << attr[i+1] << "'");
 363
 364             assert (alt.getRaw() == NULL); /* TODO: FIXME */
 365             alt = ESIStreamContextNew (this);
 366             assert (alt.getRaw() != NULL);
 367             alturl = xstrdup (attr[i+1]);
 368         } else if (!strcmp(attr[i],"onerror")) {
 369             if (!strcmp(attr[i+1], "continue")) {
 370                 flags.onerrorcontinue = 1;
 371             } else {
 372                 /* ignore mistyped attributes */
 373                 debugs(86, DBG_IMPORTANT, "invalid value for onerror='" << attr[i+1] << "'");
 374             }
 375         } else {
 376             /* ignore mistyped attributes. TODO:? error on these for user feedback - config parameter needed
 377              */
 378         }
 379     }
 380
 381     varState = cbdataReference(aContext->varState);
 382 }
 383
 384 void
 385 ESIInclude::start()
 386 {
 387     /* prevent freeing ourselves */
 388     ESIIncludePtr foo(this);
 389
 390     if (started)
 391         return;
 392
 393     started = true;
 394
 395     if (src.getRaw()) {
 396         Start (src, srcurl, varState);
 397         Start (alt, alturl, varState);
 398     } else {
 399         alt = NULL;
 400
 401         debugs(86, DBG_IMPORTANT, "ESIIncludeNew: esi:include with no src attributes");
 402
 403         flags.failed = 1;
 404     }
 405 }
 406
 407 void
 408 ESIInclude::render(ESISegment::Pointer output)
 409 {
 410     if (sent)
 411         return;
 412
 413     ESISegment::Pointer myout;
 414
 415     debugs(86, 5, "ESIIncludeRender: Rendering include " << this);
 416
 417     assert (flags.finished || (flags.failed && flags.onerrorcontinue));
 418
 419     if (flags.failed && flags.onerrorcontinue) {
 420         return;
 421     }
 422
 423     /* Render the content */
 424     if (srccontent.getRaw()) {
 425         myout = srccontent;
 426         srccontent = NULL;
 427     } else if (altcontent.getRaw()) {
 428         myout = altcontent;
 429         altcontent = NULL;
 430     } else
 431         fatal ("ESIIncludeRender called with no content, and no failure!\n");
 432
 433     assert (output->next == NULL);
 434
 435     output->next = myout;
 436
 437     sent = true;
 438 }
 439
 440 esiProcessResult_t
 441 ESIInclude::process (int dovars)
 442 {
 443     /* Prevent refcount race leading to free */
 444     Pointer me (this);
 445     start();
 446     debugs(86, 5, "ESIIncludeRender: Processing include " << this);
 447
 448     if (flags.failed) {
 449         if (flags.onerrorcontinue)
 450             return ESI_PROCESS_COMPLETE;
 451         else
 452             return ESI_PROCESS_FAILED;
 453     }
 454
 455     if (!flags.finished) {
 456         if (flags.onerrorcontinue)
 457             return ESI_PROCESS_PENDING_WONTFAIL;
 458         else
 459             return ESI_PROCESS_PENDING_MAYFAIL;
 460     }
 461
 462     return ESI_PROCESS_COMPLETE;
 463 }
 464
 465 void
 466 ESIInclude::includeFail (ESIStreamContext::Pointer stream)
 467 {
 468     subRequestDone (stream, false);
 469 }
 470
 471 bool
 472 ESIInclude::dataNeeded() const
 473 {
 474     return !(flags.finished || flags.failed);
 475 }
 476
 477 void
 478 ESIInclude::subRequestDone (ESIStreamContext::Pointer stream, bool success)
 479 {
 480     assert (this);
 481
 482     if (!dataNeeded())
 483         return;
 484
 485     if (stream == src) {
 486         debugs(86, 3, "ESIInclude::subRequestDone: " << srcurl);
 487
 488         if (success) {
 489             /* copy the lead segment */
 490             debugs(86, 3, "ESIIncludeSubRequestDone: Src OK - include PASSED.");
 491             assert (!srccontent.getRaw());
 492             ESISegment::ListTransfer (stream->localbuffer, srccontent);
 493             /* we're done! */
 494             flags.finished = 1;
 495         } else {
 496             /* Fail if there is no alt being retrieved */
 497             debugs(86, 3, "ESIIncludeSubRequestDone: Src FAILED");
 498
 499             if (!(alt.getRaw() || altcontent.getRaw())) {
 500                 debugs(86, 3, "ESIIncludeSubRequestDone: Include FAILED - No ALT");
 501                 flags.failed = 1;
 502             } else if (altcontent.getRaw()) {
 503                 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - ALT already Complete");
 504                 /* ALT was already retrieved, we are done */
 505                 flags.finished = 1;
 506             }
 507         }
 508
 509         src = NULL;
 510     } else if (stream == alt) {
 511         debugs(86, 3, "ESIInclude::subRequestDone: " << alturl);
 512
 513         if (success) {
 514             debugs(86, 3, "ESIIncludeSubRequestDone: ALT OK.");
 515             /* copy the lead segment */
 516             assert (!altcontent.getRaw());
 517             ESISegment::ListTransfer (stream->localbuffer, altcontent);
 518             /* we're done! */
 519
 520             if (!(src.getRaw() || srccontent.getRaw())) {
 521                 /* src already failed, kick ESI processor */
 522                 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - SRC already failed.");
 523                 flags.finished = 1;
 524             }
 525         } else {
 526             if (!(src.getRaw() || srccontent.getRaw())) {
 527                 debugs(86, 3, "ESIIncludeSubRequestDone: ALT FAILED, Include FAILED - SRC already failed");
 528                 /* src already failed */
 529                 flags.failed = 1;
 530             }
 531         }
 532
 533         alt = NULL;
 534     } else {
 535         fatal ("ESIIncludeSubRequestDone: non-owned stream found!\n");
 536     }
 537
 538     if (flags.finished || flags.failed) {
 539         /* Kick ESI Processor */
 540         debugs (86, 5, "ESIInclude " << this <<
 541                 " SubRequest " << stream.getRaw() <<
 542                 " completed, kicking processor , status " <<
 543                 (flags.finished ? "OK" : "FAILED"));
 544         /* There is a race condition - and we have no reproducible test case -
 545          * during a subrequest the parent will get set to NULL, which is not
 546          * meant to be possible. Rather than killing squid, we let it leak
 547          * memory but complain in the log.
 548          *
 549          * Someone wanting to debug this could well start by running squid with
 550          * a hardware breakpoint set to this location.
 551          * Its probably due to parent being set to null - by a call to
 552          * 'this.finish' while the subrequest is still not completed.
 553          */
 554         if (parent.getRaw() == NULL) {
 555             debugs (86, 0, "ESIInclude::subRequestDone: Sub request completed "
 556                     "after finish() called and parent unlinked. Unable to "
 557                     "continue handling the request, and may be memory leaking. "
 558                     "See http://www.squid-cache.org/bugs/show_bug.cgi?id=951 - we "
 559                     "are looking for a reproducible test case. This will require "
 560                     "an ESI template with includes, probably with alt-options, "
 561                     "and we're likely to need traffic dumps to allow us to "
 562                     "reconstruct the exact tcp handling sequences to trigger this "
 563                     "rather elusive bug.");
 564             return;
 565         }
 566         assert (parent.getRaw());
 567
 568         if (!flags.failed) {
 569             sent = true;
 570             parent->provideData (srccontent.getRaw() ? srccontent:altcontent,this);
 571
 572             if (srccontent.getRaw())
 573                 srccontent = NULL;
 574             else
 575                 altcontent = NULL;
 576         } else if (flags.onerrorcontinue) {
 577             /* render nothing but inform of completion */
 578
 579             if (!sent) {
 580                 sent = true;
 581                 parent->provideData (new ESISegment, this);
 582             } else
 583                 assert (0);
 584         } else
 585             parent->fail(this, "esi:include could not be completed.");
 586     }
 587 }
 588
 589 #endif /* USE_SQUID_ESI == 1 */