src/esi/Include.cc

   1 /*
   2  * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 86    ESI processing */
  10
  11 #include "squid.h"
  12
  13 #if USE_SQUID_ESI
  14
  15 #include "client_side.h"
  16 #include "client_side_request.h"
  17 #include "esi/Include.h"
  18 #include "esi/VarState.h"
  19 #include "fatal.h"
  20 #include "http/Stream.h"
  21 #include "HttpReply.h"
  22 #include "log/access_log.h"
  23
  24 CBDATA_CLASS_INIT (ESIStreamContext);
  25
  26 /* other */
  27 static CSCB esiBufferRecipient;
  28 static CSD esiBufferDetach;
  29 /* esiStreamContext */
  30 static ESIStreamContext *ESIStreamContextNew (ESIIncludePtr);
  31
  32 /* ESI TO CONSIDER:
  33  * 1. retry failed upstream requests
  34  */
  35
  36 /* Detach from a buffering stream
  37  */
  38 void
  39 esiBufferDetach (clientStreamNode *node, ClientHttpRequest *http)
  40 {
  41     /* Detach ourselves */
  42     clientStreamDetach (node, http);
  43 }
  44
  45 /**
  46  * Write a chunk of data to a client 'socket'.
  47  * If the reply is present, send the reply headers down the wire too.
  48  *
  49  * Pre-condition:
  50  *   The request is an internal ESI subrequest.
  51  *   data context is not NULL
  52  *   There are no more entries in the stream chain.
  53  *   The caller is responsible for creation and deletion of the Reply headers.
  54  *
  55  \note
  56  * Bug 975, bug 1566 : delete rep; 2006/09/02: TS, #975
  57  *
  58  * This was causing double-deletes. Its possible that not deleting
  59  * it here will cause memory leaks, but if so, this delete should
  60  * not be reinstated or it will trigger bug #975 again - RBC 20060903
  61  */
  62 void
  63 esiBufferRecipient (clientStreamNode *node, ClientHttpRequest *http, HttpReply *rep, StoreIOBuffer receivedData)
  64 {
  65     /* Test preconditions */
  66     assert (node != nullptr);
  67     /* ESI TODO: handle thisNode rather than asserting
  68      * - it should only ever happen if we cause an
  69      * abort and the callback chain loops back to
  70      * here, so we can simply return. However, that
  71      * itself shouldn't happen, so it stays as an
  72      * assert for now. */
  73     assert (cbdataReferenceValid (node));
  74     assert (node->node.next == nullptr);
  75     assert (http->getConn() == nullptr);
  76
  77     ESIStreamContext::Pointer esiStream = dynamic_cast<ESIStreamContext *>(node->data.getRaw());
  78     assert (esiStream.getRaw() != nullptr);
  79     /* If segments become more flexible, ignore thisNode */
  80     assert (receivedData.length <= sizeof(esiStream->localbuffer->buf));
  81     assert (!esiStream->finished);
  82
  83     debugs (86,5, "rep " << rep << " body " << receivedData.data << " len " << receivedData.length);
  84     assert (node->readBuffer.offset == receivedData.offset || receivedData.length == 0);
  85
  86     /* trivial case */
  87
  88     if (http->out.offset != 0) {
  89         assert(rep == nullptr);
  90     } else {
  91         if (rep) {
  92             if (rep->sline.status() != Http::scOkay) {
  93                 rep = nullptr;
  94                 esiStream->include->includeFail (esiStream);
  95                 esiStream->finished = 1;
  96                 httpRequestFree (http);
  97                 return;
  98             }
  99
 100             rep = nullptr;
 101         }
 102     }
 103
 104     if (receivedData.data && receivedData.length) {
 105         http->out.offset += receivedData.length;
 106
 107         if (receivedData.data >= esiStream->localbuffer->buf &&
 108                 receivedData.data < &esiStream->localbuffer->buf[sizeof(esiStream->localbuffer->buf)]) {
 109             /* original static buffer */
 110
 111             if (receivedData.data != esiStream->localbuffer->buf) {
 112                 /* But not the start of it */
 113                 memmove(esiStream->localbuffer->buf, receivedData.data, receivedData.length);
 114             }
 115
 116             esiStream->localbuffer->len = receivedData.length;
 117         } else {
 118             assert (esiStream->buffer.getRaw() != nullptr);
 119             esiStream->buffer->len = receivedData.length;
 120         }
 121     }
 122
 123     /* EOF / Read error /  aborted entry */
 124     if (rep == nullptr && receivedData.data == nullptr && receivedData.length == 0) {
 125         /* TODO: get stream status to test the entry for aborts */
 126         debugs(86, 5, "Finished reading upstream data in subrequest");
 127         esiStream->include->subRequestDone (esiStream, true);
 128         esiStream->finished = 1;
 129         httpRequestFree (http);
 130         return;
 131     }
 132
 133     switch (clientStreamStatus (node, http)) {
 134
 135     case STREAM_UNPLANNED_COMPLETE:
 136     case STREAM_COMPLETE: /* ok */
 137         debugs(86, 3, "ESI subrequest finished OK");
 138         esiStream->include->subRequestDone (esiStream, true);
 139         esiStream->finished = 1;
 140         httpRequestFree (http);
 141         return;
 142
 143     case STREAM_FAILED:
 144         debugs(86, DBG_IMPORTANT, "ERROR: ESI subrequest failed transfer");
 145         esiStream->include->includeFail (esiStream);
 146         esiStream->finished = 1;
 147         httpRequestFree (http);
 148         return;
 149
 150     case STREAM_NONE: {
 151         StoreIOBuffer tempBuffer;
 152
 153         if (!esiStream->buffer.getRaw()) {
 154             esiStream->buffer = esiStream->localbuffer;
 155         }
 156
 157         esiStream->buffer = esiStream->buffer->tail();
 158
 159         if (esiStream->buffer->len) {
 160             esiStream->buffer->next = new ESISegment;
 161             esiStream->buffer = esiStream->buffer->next;
 162         }
 163
 164         tempBuffer.offset = http->out.offset;
 165         tempBuffer.length = sizeof (esiStream->buffer->buf);
 166         tempBuffer.data = esiStream->buffer->buf;
 167         /* now just read into 'buffer' */
 168         clientStreamRead (node, http, tempBuffer);
 169         debugs(86, 5, "Requested more data for ESI subrequest");
 170     }
 171
 172     break;
 173
 174     default:
 175         fatal ("Hit unreachable code in esiBufferRecipient\n");
 176     }
 177
 178 }
 179
 180 /* esiStream functions */
 181 ESIStreamContext::~ESIStreamContext()
 182 {
 183     freeResources();
 184 }
 185
 186 void
 187 ESIStreamContext::freeResources()
 188 {
 189     debugs(86, 5, "Freeing stream context resources.");
 190     buffer = nullptr;
 191     localbuffer = nullptr;
 192     include = nullptr;
 193 }
 194
 195 ESIStreamContext *
 196 ESIStreamContextNew (ESIIncludePtr include)
 197 {
 198     ESIStreamContext *rv = new ESIStreamContext;
 199     rv->include = include;
 200     return rv;
 201 }
 202
 203 /* ESIInclude */
 204 ESIInclude::~ESIInclude()
 205 {
 206     debugs(86, 5, "ESIInclude::Free " << this);
 207     ESISegmentFreeList (srccontent);
 208     ESISegmentFreeList (altcontent);
 209     cbdataReferenceDone (varState);
 210     safe_free (srcurl);
 211     safe_free (alturl);
 212 }
 213
 214 void
 215 ESIInclude::finish()
 216 {
 217     parent = nullptr;
 218 }
 219
 220 ESIElement::Pointer
 221 ESIInclude::makeCacheable() const
 222 {
 223     return new ESIInclude (*this);
 224 }
 225
 226 ESIElement::Pointer
 227 ESIInclude::makeUsable(esiTreeParentPtr newParent, ESIVarState &newVarState) const
 228 {
 229     ESIInclude *resultI = new ESIInclude (*this);
 230     ESIElement::Pointer result = resultI;
 231     resultI->parent = newParent;
 232     resultI->varState = cbdataReference (&newVarState);
 233
 234     if (resultI->srcurl)
 235         resultI->src = ESIStreamContextNew (resultI);
 236
 237     if (resultI->alturl)
 238         resultI->alt = ESIStreamContextNew (resultI);
 239
 240     return result;
 241 }
 242
 243 ESIInclude::ESIInclude(ESIInclude const &old) :
 244     varState(nullptr),
 245     srcurl(nullptr),
 246     alturl(nullptr),
 247     parent(nullptr),
 248     started(false),
 249     sent(false)
 250 {
 251     memset(&flags, 0, sizeof(flags));
 252     flags.onerrorcontinue = old.flags.onerrorcontinue;
 253
 254     if (old.srcurl)
 255         srcurl = xstrdup(old.srcurl);
 256
 257     if (old.alturl)
 258         alturl = xstrdup(old.alturl);
 259 }
 260
 261 void
 262 ESIInclude::prepareRequestHeaders(HttpHeader &tempheaders, ESIVarState *vars)
 263 {
 264     tempheaders.update(&vars->header());
 265     tempheaders.removeHopByHopEntries();
 266 }
 267
 268 void
 269 ESIInclude::Start (ESIStreamContext::Pointer stream, char const *url, ESIVarState *vars)
 270 {
 271     if (!stream.getRaw())
 272         return;
 273
 274     HttpHeader tempheaders(hoRequest);
 275
 276     prepareRequestHeaders(tempheaders, vars);
 277
 278     /* Ensure variable state is clean */
 279     vars->feedData(url, strlen (url));
 280
 281     /* tempUrl is eaten by the request */
 282     char const *tempUrl = vars->extractChar ();
 283
 284     debugs(86, 5, "ESIIncludeStart: Starting subrequest with url '" << tempUrl << "'");
 285     const auto mx = MasterXaction::MakePortless<XactionInitiator::initEsi>();
 286     if (clientBeginRequest(Http::METHOD_GET, tempUrl, esiBufferRecipient, esiBufferDetach, stream.getRaw(), &tempheaders, stream->localbuffer->buf, HTTP_REQBUF_SZ, mx)) {
 287         debugs(86, DBG_CRITICAL, "ERROR: starting new ESI subrequest failed");
 288     }
 289
 290     tempheaders.clean();
 291 }
 292
 293 ESIInclude::ESIInclude(esiTreeParentPtr aParent, int attrcount, char const **attr, ESIContext *aContext) :
 294     varState(nullptr),
 295     srcurl(nullptr),
 296     alturl(nullptr),
 297     parent(aParent),
 298     started(false),
 299     sent(false)
 300 {
 301     assert (aContext);
 302     memset(&flags, 0, sizeof(flags));
 303
 304     for (int i = 0; i < attrcount && attr[i]; i += 2) {
 305         if (!strcmp(attr[i],"src")) {
 306             /* Start a request for thisNode url */
 307             debugs(86, 5, "ESIIncludeNew: Requesting source '" << attr[i+1] << "'");
 308
 309             /* TODO: don't assert on thisNode, ignore the duplicate */
 310             assert (src.getRaw() == nullptr);
 311             src = ESIStreamContextNew (this);
 312             assert (src.getRaw() != nullptr);
 313             srcurl = xstrdup(attr[i+1]);
 314         } else if (!strcmp(attr[i],"alt")) {
 315             /* Start a secondary request for thisNode url */
 316             /* TODO: make a config parameter to wait on requesting alt's
 317              * for the src to fail
 318              */
 319             debugs(86, 5, "ESIIncludeNew: Requesting alternate '" << attr[i+1] << "'");
 320
 321             assert (alt.getRaw() == nullptr); /* TODO: fix? */
 322             alt = ESIStreamContextNew (this);
 323             assert (alt.getRaw() != nullptr);
 324             alturl = xstrdup(attr[i+1]);
 325         } else if (!strcmp(attr[i],"onerror")) {
 326             if (!strcmp(attr[i+1], "continue")) {
 327                 flags.onerrorcontinue = 1;
 328             } else {
 329                 /* ignore mistyped attributes */
 330                 debugs(86, DBG_IMPORTANT, "ERROR: invalid value for onerror='" << attr[i+1] << "'");
 331             }
 332         } else {
 333             /* ignore mistyped attributes. TODO:? error on these for user feedback - config parameter needed
 334              */
 335         }
 336     }
 337
 338     varState = cbdataReference(aContext->varState);
 339 }
 340
 341 void
 342 ESIInclude::start()
 343 {
 344     /* prevent freeing ourselves */
 345     ESIIncludePtr foo(this);
 346
 347     if (started)
 348         return;
 349
 350     started = true;
 351
 352     if (src.getRaw()) {
 353         Start (src, srcurl, varState);
 354         Start (alt, alturl, varState);
 355     } else {
 356         alt = nullptr;
 357
 358         debugs(86, DBG_IMPORTANT, "ESIIncludeNew: esi:include with no src attributes");
 359
 360         flags.failed = 1;
 361     }
 362 }
 363
 364 void
 365 ESIInclude::render(ESISegment::Pointer output)
 366 {
 367     if (sent)
 368         return;
 369
 370     ESISegment::Pointer myout;
 371
 372     debugs(86, 5, "ESIIncludeRender: Rendering include " << this);
 373
 374     assert (flags.finished || (flags.failed && flags.onerrorcontinue));
 375
 376     if (flags.failed && flags.onerrorcontinue) {
 377         return;
 378     }
 379
 380     /* Render the content */
 381     if (srccontent.getRaw()) {
 382         myout = srccontent;
 383         srccontent = nullptr;
 384     } else if (altcontent.getRaw()) {
 385         myout = altcontent;
 386         altcontent = nullptr;
 387     } else
 388         fatal ("ESIIncludeRender called with no content, and no failure!\n");
 389
 390     assert (output->next == nullptr);
 391
 392     output->next = myout;
 393
 394     sent = true;
 395 }
 396
 397 esiProcessResult_t
 398 ESIInclude::process(int)
 399 {
 400     /* Prevent refcount race leading to free */
 401     Pointer me (this);
 402     start();
 403     debugs(86, 5, "ESIIncludeRender: Processing include " << this);
 404
 405     if (flags.failed) {
 406         if (flags.onerrorcontinue)
 407             return ESI_PROCESS_COMPLETE;
 408         else
 409             return ESI_PROCESS_FAILED;
 410     }
 411
 412     if (!flags.finished) {
 413         if (flags.onerrorcontinue)
 414             return ESI_PROCESS_PENDING_WONTFAIL;
 415         else
 416             return ESI_PROCESS_PENDING_MAYFAIL;
 417     }
 418
 419     return ESI_PROCESS_COMPLETE;
 420 }
 421
 422 void
 423 ESIInclude::includeFail (ESIStreamContext::Pointer stream)
 424 {
 425     subRequestDone (stream, false);
 426 }
 427
 428 bool
 429 ESIInclude::dataNeeded() const
 430 {
 431     return !(flags.finished || flags.failed);
 432 }
 433
 434 void
 435 ESIInclude::subRequestDone (ESIStreamContext::Pointer stream, bool success)
 436 {
 437     if (!dataNeeded())
 438         return;
 439
 440     if (stream == src) {
 441         debugs(86, 3, "ESIInclude::subRequestDone: " << srcurl);
 442
 443         if (success) {
 444             /* copy the lead segment */
 445             debugs(86, 3, "ESIIncludeSubRequestDone: Src OK - include PASSED.");
 446             assert (!srccontent.getRaw());
 447             ESISegment::ListTransfer (stream->localbuffer, srccontent);
 448             /* we're done! */
 449             flags.finished = 1;
 450         } else {
 451             /* Fail if there is no alt being retrieved */
 452             debugs(86, 3, "ESIIncludeSubRequestDone: Src FAILED");
 453
 454             if (!(alt.getRaw() || altcontent.getRaw())) {
 455                 debugs(86, 3, "ESIIncludeSubRequestDone: Include FAILED - No ALT");
 456                 flags.failed = 1;
 457             } else if (altcontent.getRaw()) {
 458                 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - ALT already Complete");
 459                 /* ALT was already retrieved, we are done */
 460                 flags.finished = 1;
 461             }
 462         }
 463
 464         src = nullptr;
 465     } else if (stream == alt) {
 466         debugs(86, 3, "ESIInclude::subRequestDone: " << alturl);
 467
 468         if (success) {
 469             debugs(86, 3, "ESIIncludeSubRequestDone: ALT OK.");
 470             /* copy the lead segment */
 471             assert (!altcontent.getRaw());
 472             ESISegment::ListTransfer (stream->localbuffer, altcontent);
 473             /* we're done! */
 474
 475             if (!(src.getRaw() || srccontent.getRaw())) {
 476                 /* src already failed, kick ESI processor */
 477                 debugs(86, 3, "ESIIncludeSubRequestDone: Include PASSED - SRC already failed.");
 478                 flags.finished = 1;
 479             }
 480         } else {
 481             if (!(src.getRaw() || srccontent.getRaw())) {
 482                 debugs(86, 3, "ESIIncludeSubRequestDone: ALT FAILED, Include FAILED - SRC already failed");
 483                 /* src already failed */
 484                 flags.failed = 1;
 485             }
 486         }
 487
 488         alt = nullptr;
 489     } else {
 490         fatal ("ESIIncludeSubRequestDone: non-owned stream found!\n");
 491     }
 492
 493     if (flags.finished || flags.failed) {
 494         /* Kick ESI Processor */
 495         debugs (86, 5, "ESIInclude " << this <<
 496                 " SubRequest " << stream.getRaw() <<
 497                 " completed, kicking processor , status " <<
 498                 (flags.finished ? "OK" : "FAILED"));
 499         /* There is a race condition - and we have no reproducible test case -
 500          * during a subrequest the parent will get set to NULL, which is not
 501          * meant to be possible. Rather than killing squid, we let it leak
 502          * memory but complain in the log.
 503          *
 504          * Someone wanting to debug this could well start by running squid with
 505          * a hardware breakpoint set to this location.
 506          * Its probably due to parent being set to null - by a call to
 507          * 'this.finish' while the subrequest is still not completed.
 508          */
 509         if (parent.getRaw() == nullptr) {
 510             debugs(86, DBG_CRITICAL, "ERROR: Squid Bug #951: ESIInclude::subRequestDone: Sub request completed "
 511                    "after finish() called and parent unlinked. Unable to "
 512                    "continue handling the request, and may be memory leaking. "
 513                    "See http://www.squid-cache.org/bugs/show_bug.cgi?id=951 - we "
 514                    "are looking for a reproducible test case. This will require "
 515                    "an ESI template with includes, probably with alt-options, "
 516                    "and we're likely to need traffic dumps to allow us to "
 517                    "reconstruct the exact tcp handling sequences to trigger this "
 518                    "rather elusive bug.");
 519             return;
 520         }
 521         assert (parent.getRaw());
 522
 523         if (!flags.failed) {
 524             sent = true;
 525             parent->provideData (srccontent.getRaw() ? srccontent:altcontent,this);
 526
 527             if (srccontent.getRaw())
 528                 srccontent = nullptr;
 529             else
 530                 altcontent = nullptr;
 531         } else if (flags.onerrorcontinue) {
 532             /* render nothing but inform of completion */
 533
 534             if (!sent) {
 535                 sent = true;
 536                 parent->provideData (new ESISegment, this);
 537             } else
 538                 assert (0);
 539         } else
 540             parent->fail(this, "esi:include could not be completed.");
 541     }
 542 }
 543
 544 #endif /* USE_SQUID_ESI */
 545