src/client_side_reply.cc

   1 /*
   2  * Copyright (C) 1996-2025 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 88    Client-side Reply Routines */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "acl/Gadgets.h"
  14 #include "anyp/PortCfg.h"
  15 #include "client_side_reply.h"
  16 #include "clientStream.h"
  17 #include "errorpage.h"
  18 #include "ETag.h"
  19 #include "fd.h"
  20 #include "fde.h"
  21 #include "format/Token.h"
  22 #include "FwdState.h"
  23 #include "globals.h"
  24 #include "http/Stream.h"
  25 #include "HttpHeaderTools.h"
  26 #include "HttpReply.h"
  27 #include "HttpRequest.h"
  28 #include "ip/QosConfig.h"
  29 #include "ipcache.h"
  30 #include "log/access_log.h"
  31 #include "MemObject.h"
  32 #include "mime_header.h"
  33 #include "neighbors.h"
  34 #include "refresh.h"
  35 #include "RequestFlags.h"
  36 #include "SquidConfig.h"
  37 #include "SquidMath.h"
  38 #include "Store.h"
  39 #include "StrList.h"
  40 #include "tools.h"
  41 #if USE_AUTH
  42 #include "auth/UserRequest.h"
  43 #endif
  44 #if USE_DELAY_POOLS
  45 #include "DelayPools.h"
  46 #endif
  47
  48 #include <memory>
  49
  50 CBDATA_CLASS_INIT(clientReplyContext);
  51
  52 /* Local functions */
  53 CSS clientReplyStatus;
  54 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *, const ConnStateData *, HttpRequest *, const AccessLogEntry::Pointer &);
  55
  56 /* privates */
  57
  58 clientReplyContext::~clientReplyContext()
  59 {
  60     deleting = true;
  61     /* This may trigger a callback back into SendMoreData as the cbdata
  62      * is still valid
  63      */
  64     removeClientStoreReference(&sc, http);
  65     /* old_entry might still be set if we didn't yet get the reply
  66      * code in HandleIMSReply() */
  67     removeStoreReference(&old_sc, &old_entry);
  68     cbdataReferenceDone(http);
  69     HTTPMSGUNLOCK(reply);
  70 }
  71
  72 clientReplyContext::clientReplyContext(ClientHttpRequest *clientContext) :
  73     purgeStatus(Http::scNone),
  74     http(cbdataReference(clientContext)),
  75     sc(nullptr),
  76     ourNode(nullptr),
  77     reply(nullptr),
  78     old_entry(nullptr),
  79     old_sc(nullptr),
  80     old_lastmod(-1),
  81     deleting(false),
  82     collapsedRevalidation(crNone)
  83 {
  84     *tempbuf = 0;
  85 }
  86
  87 /** Create an error in the store awaiting the client side to read it.
  88  *
  89  * This may be better placed in the clientStream logic, but it has not been
  90  * relocated there yet
  91  */
  92 void
  93 clientReplyContext::setReplyToError(
  94     err_type err, Http::StatusCode status, char const *uri,
  95     const ConnStateData *conn, HttpRequest *failedrequest, const char *unparsedrequest,
  96 #if USE_AUTH
  97     Auth::UserRequest::Pointer auth_user_request
  98 #else
  99     void*
 100 #endif
 101 )
 102 {
 103     auto errstate = clientBuildError(err, status, uri, conn, failedrequest, http->al);
 104
 105     if (unparsedrequest)
 106         errstate->request_hdrs = xstrdup(unparsedrequest);
 107
 108 #if USE_AUTH
 109     errstate->auth_user_request = auth_user_request;
 110 #endif
 111     setReplyToError(failedrequest ? failedrequest->method : HttpRequestMethod(Http::METHOD_NONE), errstate);
 112 }
 113
 114 void clientReplyContext::setReplyToError(const HttpRequestMethod& method, ErrorState *errstate)
 115 {
 116     if (errstate->httpStatus == Http::scNotImplemented && http->request)
 117         /* prevent confusion over whether we default to persistent or not */
 118         http->request->flags.proxyKeepalive = false;
 119
 120     http->al->http.code = errstate->httpStatus;
 121
 122     if (http->request)
 123         http->request->ignoreRange("responding with a Squid-generated error");
 124
 125     createStoreEntry(method, RequestFlags());
 126     assert(errstate->callback_data == nullptr);
 127     errorAppendEntry(http->storeEntry(), errstate);
 128     /* Now the caller reads to get this */
 129 }
 130
 131 void
 132 clientReplyContext::setReplyToReply(HttpReply *futureReply)
 133 {
 134     Must(futureReply);
 135     http->al->http.code = futureReply->sline.status();
 136
 137     HttpRequestMethod method;
 138     if (http->request) { // nil on responses to unparsable requests
 139         http->request->ignoreRange("responding with a Squid-generated reply");
 140         method = http->request->method;
 141     }
 142
 143     createStoreEntry(method, RequestFlags());
 144
 145     http->storeEntry()->storeErrorResponse(futureReply);
 146     /* Now the caller reads to get futureReply */
 147 }
 148
 149 // Assumes that the entry contains an error response without Content-Range.
 150 // To use with regular entries, make HTTP Range header removal conditional.
 151 void clientReplyContext::setReplyToStoreEntry(StoreEntry *entry, const char *reason)
 152 {
 153     entry->lock("clientReplyContext::setReplyToStoreEntry"); // removeClientStoreReference() unlocks
 154     sc = storeClientListAdd(entry, this);
 155 #if USE_DELAY_POOLS
 156     sc->setDelayId(DelayId::DelayClient(http));
 157 #endif
 158     if (http->request)
 159         http->request->ignoreRange(reason);
 160     flags.storelogiccomplete = 1;
 161     http->storeEntry(entry);
 162 }
 163
 164 void
 165 clientReplyContext::removeStoreReference(store_client ** scp,
 166         StoreEntry ** ep)
 167 {
 168     StoreEntry *e;
 169     store_client *sc_tmp = *scp;
 170
 171     if ((e = *ep) != nullptr) {
 172         *ep = nullptr;
 173         storeUnregister(sc_tmp, e, this);
 174         *scp = nullptr;
 175         e->unlock("clientReplyContext::removeStoreReference");
 176     }
 177 }
 178
 179 void
 180 clientReplyContext::removeClientStoreReference(store_client **scp, ClientHttpRequest *aHttpRequest)
 181 {
 182     StoreEntry *reference = aHttpRequest->storeEntry();
 183     removeStoreReference(scp, &reference);
 184     aHttpRequest->storeEntry(reference);
 185 }
 186
 187 void
 188 clientReplyContext::saveState()
 189 {
 190     assert(old_sc == nullptr);
 191     debugs(88, 3, "clientReplyContext::saveState: saving store context");
 192     old_entry = http->storeEntry();
 193     old_sc = sc;
 194     old_lastmod = http->request->lastmod;
 195     old_etag = http->request->etag;
 196     /* Prevent accessing the now saved entries */
 197     http->storeEntry(nullptr);
 198     sc = nullptr;
 199 }
 200
 201 void
 202 clientReplyContext::restoreState()
 203 {
 204     assert(old_sc != nullptr);
 205     debugs(88, 3, "clientReplyContext::restoreState: Restoring store context");
 206     removeClientStoreReference(&sc, http);
 207     http->storeEntry(old_entry);
 208     sc = old_sc;
 209     http->request->lastmod = old_lastmod;
 210     http->request->etag = old_etag;
 211     /* Prevent accessed the old saved entries */
 212     old_entry = nullptr;
 213     old_sc = nullptr;
 214     old_lastmod = -1;
 215     old_etag.clean();
 216 }
 217
 218 void
 219 clientReplyContext::startError(ErrorState * err)
 220 {
 221     createStoreEntry(http->request->method, RequestFlags());
 222     triggerInitialStoreRead();
 223     errorAppendEntry(http->storeEntry(), err);
 224 }
 225
 226 clientStreamNode *
 227 clientReplyContext::getNextNode() const
 228 {
 229     return (clientStreamNode *)ourNode->node.next->data;
 230 }
 231
 232 /// Request HTTP response headers from Store, to be sent to the given recipient.
 233 /// That recipient also gets zero, some, or all HTTP response body bytes (into
 234 /// next()->readBuffer).
 235 void
 236 clientReplyContext::triggerInitialStoreRead(STCB recipient)
 237 {
 238     Assure(recipient != HandleIMSReply);
 239     lastStreamBufferedBytes = StoreIOBuffer(); // storeClientCopy(next()->readBuffer) invalidates
 240     StoreIOBuffer localTempBuffer (next()->readBuffer.length, 0, next()->readBuffer.data);
 241     ::storeClientCopy(sc, http->storeEntry(), localTempBuffer, recipient, this);
 242 }
 243
 244 /// Request HTTP response body bytes from Store into next()->readBuffer. This
 245 /// method requests body bytes at readerBuffer.offset and, hence, it should only
 246 /// be called after we triggerInitialStoreRead() and get the requested HTTP
 247 /// response headers (using zero offset).
 248 void
 249 clientReplyContext::requestMoreBodyFromStore()
 250 {
 251     lastStreamBufferedBytes = StoreIOBuffer(); // storeClientCopy(next()->readBuffer) invalidates
 252     ::storeClientCopy(sc, http->storeEntry(), next()->readBuffer, SendMoreData, this);
 253 }
 254
 255 /* there is an expired entry in the store.
 256  * setup a temporary buffer area and perform an IMS to the origin
 257  */
 258 void
 259 clientReplyContext::processExpired()
 260 {
 261     const char *url = storeId();
 262     debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'");
 263     const time_t lastmod = http->storeEntry()->lastModified();
 264     assert(lastmod >= 0);
 265     /*
 266      * check if we are allowed to contact other servers
 267      * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return
 268      *      a stale entry *if* it matches client requirements
 269      */
 270
 271     if (http->onlyIfCached()) {
 272         processOnlyIfCachedMiss();
 273         return;
 274     }
 275
 276     http->updateLoggingTags(LOG_TCP_REFRESH);
 277     http->request->flags.refresh = true;
 278 #if STORE_CLIENT_LIST_DEBUG
 279     /* Prevent a race with the store client memory free routines
 280      */
 281     assert(storeClientIsThisAClient(sc, this));
 282 #endif
 283     /* Prepare to make a new temporary request */
 284     saveState();
 285
 286     // TODO: Consider also allowing regular (non-collapsed) revalidation hits.
 287     // TODO: support collapsed revalidation for Vary-controlled entries
 288     bool collapsingAllowed = Config.onoff.collapsed_forwarding &&
 289                              !Store::Controller::SmpAware() &&
 290                              http->request->vary_headers.isEmpty();
 291
 292     StoreEntry *entry = nullptr;
 293     if (collapsingAllowed) {
 294         if (const auto e = storeGetPublicByRequest(http->request, ksRevalidation)) {
 295             if (e->hittingRequiresCollapsing() && startCollapsingOn(*e, true)) {
 296                 entry = e;
 297                 entry->lock("clientReplyContext::processExpired#alreadyRevalidating");
 298             } else {
 299                 e->abandon(__func__);
 300                 // assume mayInitiateCollapsing() would fail too
 301                 collapsingAllowed = false;
 302             }
 303         }
 304     }
 305
 306     if (entry) {
 307         entry->ensureMemObject(url, http->log_uri, http->request->method);
 308         debugs(88, 5, "collapsed on existing revalidation entry: " << *entry);
 309         collapsedRevalidation = crSlave;
 310     } else {
 311         entry = storeCreateEntry(url,
 312                                  http->log_uri, http->request->flags, http->request->method);
 313         /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */
 314
 315         if (collapsingAllowed && mayInitiateCollapsing() &&
 316                 Store::Root().allowCollapsing(entry, http->request->flags, http->request->method)) {
 317             debugs(88, 5, "allow other revalidation requests to collapse on " << *entry);
 318             collapsedRevalidation = crInitiator;
 319         } else {
 320             collapsedRevalidation = crNone;
 321         }
 322     }
 323
 324     sc = storeClientListAdd(entry, this);
 325 #if USE_DELAY_POOLS
 326     /* delay_id is already set on original store client */
 327     sc->setDelayId(DelayId::DelayClient(http));
 328 #endif
 329
 330     http->request->lastmod = lastmod;
 331
 332     if (!http->request->header.has(Http::HdrType::IF_NONE_MATCH)) {
 333         ETag etag = {nullptr, -1}; // TODO: make that a default ETag constructor
 334         if (old_entry->hasEtag(etag) && !etag.weak)
 335             http->request->etag = etag.str;
 336     }
 337
 338     debugs(88, 5, "lastmod " << entry->lastModified());
 339     http->storeEntry(entry);
 340     assert(http->out.offset == 0);
 341     assert(http->request->clientConnectionManager == http->getConn());
 342
 343     if (collapsedRevalidation != crSlave) {
 344         /*
 345          * A refcounted pointer so that FwdState stays around as long as
 346          * this clientReplyContext does
 347          */
 348         Comm::ConnectionPointer conn = http->getConn() != nullptr ? http->getConn()->clientConnection : nullptr;
 349         FwdState::Start(conn, http->storeEntry(), http->request, http->al);
 350     }
 351     /* Register with storage manager to receive updates when data comes in. */
 352
 353     if (EBIT_TEST(entry->flags, ENTRY_ABORTED))
 354         debugs(88, DBG_CRITICAL, "clientReplyContext::processExpired: Found ENTRY_ABORTED object");
 355
 356     {
 357         /* start counting the length from 0 */
 358         StoreIOBuffer localTempBuffer(HTTP_REQBUF_SZ, 0, tempbuf);
 359         // keep lastStreamBufferedBytes: tempbuf is not a Client Stream buffer
 360         ::storeClientCopy(sc, entry, localTempBuffer, HandleIMSReply, this);
 361     }
 362 }
 363
 364 void
 365 clientReplyContext::sendClientUpstreamResponse(const StoreIOBuffer &upstreamResponse)
 366 {
 367     removeStoreReference(&old_sc, &old_entry);
 368
 369     if (collapsedRevalidation)
 370         http->storeEntry()->clearPublicKeyScope();
 371
 372     /* here the data to send is the data we just received */
 373     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 374     sendMoreData(upstreamResponse);
 375 }
 376
 377 void
 378 clientReplyContext::HandleIMSReply(void *data, StoreIOBuffer result)
 379 {
 380     clientReplyContext *context = (clientReplyContext *)data;
 381     context->handleIMSReply(result);
 382 }
 383
 384 void
 385 clientReplyContext::sendClientOldEntry()
 386 {
 387     /* Get the old request back */
 388     restoreState();
 389
 390     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
 391         debugs(88, 3, "stale entry aborted while we revalidated: " << *http->storeEntry());
 392         http->updateLoggingTags(LOG_TCP_MISS);
 393         processMiss();
 394         return;
 395     }
 396
 397     /* here the data to send is in the next nodes buffers already */
 398     Assure(matchesStreamBodyBuffer(lastStreamBufferedBytes));
 399     Assure(!lastStreamBufferedBytes.offset);
 400     sendMoreData(lastStreamBufferedBytes);
 401 }
 402
 403 /* This is the workhorse of the HandleIMSReply callback.
 404  *
 405  * It is called when we've got data back from the origin following our
 406  * IMS request to revalidate a stale entry.
 407  */
 408 void
 409 clientReplyContext::handleIMSReply(const StoreIOBuffer result)
 410 {
 411     if (deleting)
 412         return;
 413
 414     if (http->storeEntry() == nullptr)
 415         return;
 416
 417     debugs(88, 3, http->storeEntry()->url() << " got " << result);
 418
 419     if (result.flags.error && !EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED))
 420         return;
 421
 422     if (collapsedRevalidation == crSlave && !http->storeEntry()->mayStartHitting()) {
 423         debugs(88, 3, "CF slave hit private non-shareable " << *http->storeEntry() << ". MISS");
 424         // restore context to meet processMiss() expectations
 425         restoreState();
 426         http->updateLoggingTags(LOG_TCP_MISS);
 427         processMiss();
 428         return;
 429     }
 430
 431     // request to origin was aborted
 432     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
 433         debugs(88, 3, "request to origin aborted '" << http->storeEntry()->url() << "', sending old entry to client");
 434         http->updateLoggingTags(LOG_TCP_REFRESH_FAIL_OLD);
 435         sendClientOldEntry();
 436         return;
 437     }
 438
 439     const auto oldStatus = old_entry->mem().freshestReply().sline.status();
 440     const auto &new_rep = http->storeEntry()->mem().freshestReply();
 441     const auto status = new_rep.sline.status();
 442
 443     // XXX: Disregard stale incomplete (i.e. still being written) borrowed (i.e.
 444     // not caused by our request) IMS responses. That new_rep may be very old!
 445
 446     // origin replied 304
 447     if (status == Http::scNotModified) {
 448         // TODO: The update may not be instantaneous. Should we wait for its
 449         // completion to avoid spawning too much client-disassociated work?
 450         if (!Store::Root().updateOnNotModified(old_entry, *http->storeEntry())) {
 451             old_entry->release(true);
 452             restoreState();
 453             http->updateLoggingTags(LOG_TCP_MISS);
 454             processMiss();
 455             return;
 456         }
 457
 458         http->updateLoggingTags(LOG_TCP_REFRESH_UNMODIFIED);
 459         http->request->flags.staleIfHit = false; // old_entry is no longer stale
 460
 461         // if client sent IMS
 462         if (http->request->flags.ims && !old_entry->modifiedSince(http->request->ims, http->request->imslen)) {
 463             // forward the 304 from origin
 464             debugs(88, 3, "origin replied 304, revalidated existing entry and forwarding 304 to client");
 465             sendClientUpstreamResponse(result);
 466             return;
 467         }
 468
 469         // send existing entry, it's still valid
 470         debugs(88, 3, "origin replied 304, revalidated existing entry and sending " << oldStatus << " to client");
 471         sendClientOldEntry();
 472         return;
 473     }
 474
 475     // origin replied with a non-error code
 476     if (status > Http::scNone && status < Http::scInternalServerError) {
 477         // RFC 9111 section 4:
 478         // "When more than one suitable response is stored,
 479         //  a cache MUST use the most recent one
 480         // (as determined by the Date header field)."
 481         if (new_rep.olderThan(&old_entry->mem().freshestReply())) {
 482             http->al->cache.code.err.ignored = true;
 483             debugs(88, 3, "origin replied " << status << " but with an older date header, sending old entry (" << oldStatus << ") to client");
 484             sendClientOldEntry();
 485             return;
 486         }
 487
 488         http->updateLoggingTags(LOG_TCP_REFRESH_MODIFIED);
 489         debugs(88, 3, "origin replied " << status << ", forwarding to client");
 490         sendClientUpstreamResponse(result);
 491         return;
 492     }
 493
 494     // origin replied with an error
 495     if (http->request->flags.failOnValidationError) {
 496         http->updateLoggingTags(LOG_TCP_REFRESH_FAIL_ERR);
 497         debugs(88, 3, "origin replied with error " << status << ", forwarding to client due to fail_on_validation_err");
 498         sendClientUpstreamResponse(result);
 499         return;
 500     }
 501
 502     // ignore and let client have old entry
 503     http->updateLoggingTags(LOG_TCP_REFRESH_FAIL_OLD);
 504     debugs(88, 3, "origin replied with error " << status << ", sending old entry (" << oldStatus << ") to client");
 505     sendClientOldEntry();
 506 }
 507
 508 CSR clientGetMoreData;
 509 CSD clientReplyDetach;
 510
 511 /// \copydoc clientReplyContext::cacheHit()
 512 void
 513 clientReplyContext::CacheHit(void *data, StoreIOBuffer result)
 514 {
 515     clientReplyContext *context = (clientReplyContext *)data;
 516     context->cacheHit(result);
 517 }
 518
 519 /// Processes HTTP response headers received from Store on a suspected cache hit
 520 /// path. May be called several times (e.g., a Vary marker object hit followed
 521 /// by the corresponding variant hit).
 522 void
 523 clientReplyContext::cacheHit(const StoreIOBuffer result)
 524 {
 525     /** Ignore if the HIT object is being deleted. */
 526     if (deleting) {
 527         debugs(88, 3, "HIT object being deleted. Ignore the HIT.");
 528         return;
 529     }
 530
 531     StoreEntry *e = http->storeEntry();
 532
 533     HttpRequest *r = http->request;
 534
 535     debugs(88, 3, http->uri << " got " << result);
 536
 537     if (http->storeEntry() == nullptr) {
 538         debugs(88, 3, "clientCacheHit: request aborted");
 539         return;
 540     } else if (result.flags.error) {
 541         /* swap in failure */
 542         debugs(88, 3, "clientCacheHit: swapin failure for " << http->uri);
 543         http->updateLoggingTags(LOG_TCP_SWAPFAIL_MISS);
 544         removeClientStoreReference(&sc, http);
 545         processMiss();
 546         return;
 547     }
 548
 549     // The previously identified hit suddenly became unshareable!
 550     // This is common for collapsed forwarding slaves but might also
 551     // happen to regular hits because we are called asynchronously.
 552     if (!e->mayStartHitting()) {
 553         debugs(88, 3, "unshareable " << *e << ". MISS");
 554         http->updateLoggingTags(LOG_TCP_MISS);
 555         processMiss();
 556         return;
 557     }
 558
 559     if (EBIT_TEST(e->flags, ENTRY_ABORTED)) {
 560         debugs(88, 3, "refusing aborted " << *e);
 561         http->updateLoggingTags(LOG_TCP_MISS);
 562         processMiss();
 563         return;
 564     }
 565
 566     /*
 567      * Got the headers, now grok them
 568      */
 569     assert(http->loggingTags().oldType == LOG_TCP_HIT);
 570
 571     if (http->request->storeId().cmp(e->mem_obj->storeId()) != 0) {
 572         debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->storeId() << "' != '" << http->request->storeId() << "'");
 573         http->updateLoggingTags(LOG_TCP_MISS); // we lack a more precise LOG_*_MISS code
 574         processMiss();
 575         return;
 576     }
 577
 578     noteStreamBufferredBytes(result);
 579
 580     switch (varyEvaluateMatch(e, r)) {
 581
 582     case VARY_NONE:
 583         /* No variance detected. Continue as normal */
 584         break;
 585
 586     case VARY_MATCH:
 587         /* This is the correct entity for this request. Continue */
 588         debugs(88, 2, "clientProcessHit: Vary MATCH!");
 589         break;
 590
 591     case VARY_OTHER:
 592         /* This is not the correct entity for this request. We need
 593          * to requery the cache.
 594          */
 595         removeClientStoreReference(&sc, http);
 596         e = nullptr;
 597         /* Note: varyEvalyateMatch updates the request with vary information
 598          * so we only get here once. (it also takes care of cancelling loops)
 599          */
 600         debugs(88, 2, "clientProcessHit: Vary detected!");
 601         clientGetMoreData(ourNode, http);
 602         return;
 603
 604     case VARY_CANCEL:
 605         /* varyEvaluateMatch found a object loop. Process as miss */
 606         debugs(88, DBG_IMPORTANT, "clientProcessHit: Vary object loop!");
 607         http->updateLoggingTags(LOG_TCP_MISS); // we lack a more precise LOG_*_MISS code
 608         processMiss();
 609         return;
 610     }
 611
 612     if (r->method == Http::METHOD_PURGE) {
 613         debugs(88, 5, "PURGE gets a HIT");
 614         removeClientStoreReference(&sc, http);
 615         e = nullptr;
 616         purgeRequest();
 617         return;
 618     }
 619
 620     if (e->checkNegativeHit() && !r->flags.noCacheHack()) {
 621         debugs(88, 5, "negative-HIT");
 622         http->updateLoggingTags(LOG_TCP_NEGATIVE_HIT);
 623         sendMoreData(result);
 624         return;
 625     } else if (blockedHit()) {
 626         debugs(88, 5, "send_hit forces a MISS");
 627         http->updateLoggingTags(LOG_TCP_MISS);
 628         processMiss();
 629         return;
 630     } else if (!r->flags.internal && !didCollapse && refreshCheckHTTP(e, r)) {
 631         debugs(88, 5, "clientCacheHit: in refreshCheck() block");
 632         /*
 633          * We hold a stale copy; it needs to be validated
 634          */
 635         /*
 636          * The 'needValidation' flag is used to prevent forwarding
 637          * loops between siblings.  If our copy of the object is stale,
 638          * then we should probably only use parents for the validation
 639          * request.  Otherwise two siblings could generate a loop if
 640          * both have a stale version of the object.
 641          */
 642         r->flags.needValidation = true;
 643
 644         if (e->lastModified() < 0) {
 645             debugs(88, 3, "validate HIT object? NO. Can't calculate entry modification time. Do MISS.");
 646             /*
 647              * We cannot revalidate entries without knowing their
 648              * modification time.
 649              * XXX: BUG 1890 objects without Date do not get one added.
 650              */
 651             http->updateLoggingTags(LOG_TCP_MISS);
 652             processMiss();
 653         } else if (r->flags.noCache) {
 654             debugs(88, 3, "validate HIT object? NO. Client sent CC:no-cache. Do CLIENT_REFRESH_MISS");
 655             /*
 656              * This did not match a refresh pattern that overrides no-cache
 657              * we should honour the client no-cache header.
 658              */
 659             http->updateLoggingTags(LOG_TCP_CLIENT_REFRESH_MISS);
 660             processMiss();
 661         } else if (r->url.getScheme() == AnyP::PROTO_HTTP || r->url.getScheme() == AnyP::PROTO_HTTPS) {
 662             debugs(88, 3, "validate HIT object? YES.");
 663             /*
 664              * Object needs to be revalidated
 665              * XXX This could apply to FTP as well, if Last-Modified is known.
 666              */
 667             processExpired();
 668         } else {
 669             debugs(88, 3, "validate HIT object? NO. Client protocol non-HTTP. Do MISS.");
 670             /*
 671              * We don't know how to re-validate other protocols. Handle
 672              * them as if the object has expired.
 673              */
 674             http->updateLoggingTags(LOG_TCP_MISS);
 675             processMiss();
 676         }
 677         return;
 678     } else if (r->conditional()) {
 679         debugs(88, 5, "conditional HIT");
 680         if (processConditional())
 681             return;
 682     }
 683
 684     /*
 685      * plain ol' cache hit
 686      */
 687     debugs(88, 5, "plain old HIT");
 688
 689 #if USE_DELAY_POOLS
 690     if (e->store_status != STORE_OK)
 691         http->updateLoggingTags(LOG_TCP_MISS);
 692     else
 693 #endif
 694         if (e->mem_status == IN_MEMORY)
 695             http->updateLoggingTags(LOG_TCP_MEM_HIT);
 696         else if (Config.onoff.offline)
 697             http->updateLoggingTags(LOG_TCP_OFFLINE_HIT);
 698
 699     sendMoreData(result);
 700 }
 701
 702 /**
 703  * Prepare to fetch the object as it's a cache miss of some kind.
 704  */
 705 void
 706 clientReplyContext::processMiss()
 707 {
 708     char *url = http->uri;
 709     HttpRequest *r = http->request;
 710     ErrorState *err = nullptr;
 711     debugs(88, 4, r->method << ' ' << url);
 712
 713     /**
 714      * We might have a left-over StoreEntry from a failed cache hit
 715      * or IMS request.
 716      */
 717     if (http->storeEntry()) {
 718         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
 719             debugs(88, DBG_CRITICAL, "clientProcessMiss: miss on a special object (" << url << ").");
 720             debugs(88, DBG_CRITICAL, "\tlog_type = " << http->loggingTags().c_str());
 721             http->storeEntry()->dump(1);
 722         }
 723
 724         removeClientStoreReference(&sc, http);
 725     }
 726
 727     /** Check if its a PURGE request to be actioned. */
 728     if (r->method == Http::METHOD_PURGE) {
 729         purgeRequest();
 730         return;
 731     }
 732
 733     /** Check if its an 'OTHER' request. Purge all cached entries if so and continue. */
 734     if (r->method == Http::METHOD_OTHER) {
 735         purgeAllCached();
 736     }
 737
 738     /** Check if 'only-if-cached' flag is set. Action if so. */
 739     if (http->onlyIfCached()) {
 740         processOnlyIfCachedMiss();
 741         return;
 742     }
 743
 744     /// Deny loops
 745     if (r->flags.loopDetected) {
 746         http->al->http.code = Http::scForbidden;
 747         err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, nullptr, http->getConn(), http->request, http->al);
 748         createStoreEntry(r->method, RequestFlags());
 749         errorAppendEntry(http->storeEntry(), err);
 750         triggerInitialStoreRead();
 751         return;
 752     } else {
 753         assert(http->out.offset == 0);
 754         createStoreEntry(r->method, r->flags);
 755         triggerInitialStoreRead();
 756
 757         if (http->redirect.status) {
 758             const HttpReplyPointer rep(new HttpReply);
 759             http->updateLoggingTags(LOG_TCP_REDIRECT);
 760             http->storeEntry()->releaseRequest();
 761             rep->redirect(http->redirect.status, http->redirect.location);
 762             http->storeEntry()->replaceHttpReply(rep);
 763             http->storeEntry()->complete();
 764             return;
 765         }
 766
 767         assert(r->clientConnectionManager == http->getConn());
 768
 769         Comm::ConnectionPointer conn = http->getConn() != nullptr ? http->getConn()->clientConnection : nullptr;
 770         /** Start forwarding to get the new object from network */
 771         FwdState::Start(conn, http->storeEntry(), r, http->al);
 772     }
 773 }
 774
 775 /**
 776  * client issued a request with an only-if-cached cache-control directive;
 777  * we did not find a cached object that can be returned without
 778  *     contacting other servers;
 779  * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068]
 780  */
 781 void
 782 clientReplyContext::processOnlyIfCachedMiss()
 783 {
 784     debugs(88, 4, http->request->method << ' ' << http->uri);
 785     http->al->http.code = Http::scGatewayTimeout;
 786     ErrorState *err = clientBuildError(ERR_ONLY_IF_CACHED_MISS, Http::scGatewayTimeout, nullptr,
 787                                        http->getConn(), http->request, http->al);
 788     removeClientStoreReference(&sc, http);
 789     startError(err);
 790 }
 791
 792 /// process conditional request from client
 793 bool
 794 clientReplyContext::processConditional()
 795 {
 796     StoreEntry *const e = http->storeEntry();
 797
 798     const auto replyStatusCode = e->mem().baseReply().sline.status();
 799     if (replyStatusCode != Http::scOkay) {
 800         debugs(88, 4, "miss because " << replyStatusCode << " != 200");
 801         http->updateLoggingTags(LOG_TCP_MISS);
 802         processMiss();
 803         return true;
 804     }
 805
 806     HttpRequest &r = *http->request;
 807
 808     if (r.header.has(Http::HdrType::IF_MATCH) && !e->hasIfMatchEtag(r)) {
 809         // RFC 2616: reply with 412 Precondition Failed if If-Match did not match
 810         sendPreconditionFailedError();
 811         return true;
 812     }
 813
 814     if (r.header.has(Http::HdrType::IF_NONE_MATCH)) {
 815         // RFC 7232: If-None-Match recipient MUST ignore IMS
 816         r.flags.ims = false;
 817         r.ims = -1;
 818         r.imslen = 0;
 819         r.header.delById(Http::HdrType::IF_MODIFIED_SINCE);
 820
 821         if (e->hasIfNoneMatchEtag(r)) {
 822             sendNotModifiedOrPreconditionFailedError();
 823             return true;
 824         }
 825
 826         // None-Match is true (no ETag matched); treat as an unconditional hit
 827         return false;
 828     }
 829
 830     if (r.flags.ims) {
 831         // handle If-Modified-Since requests from the client
 832         if (e->modifiedSince(r.ims, r.imslen)) {
 833             // Modified-Since is true; treat as an unconditional hit
 834             return false;
 835
 836         } else {
 837             // otherwise reply with 304 Not Modified
 838             sendNotModified();
 839         }
 840         return true;
 841     }
 842
 843     return false;
 844 }
 845
 846 /// whether squid.conf send_hit prevents us from serving this hit
 847 bool
 848 clientReplyContext::blockedHit() const
 849 {
 850     if (!Config.accessList.sendHit)
 851         return false; // hits are not blocked by default
 852
 853     if (http->request->flags.internal)
 854         return false; // internal content "hits" cannot be blocked
 855
 856     {
 857         ACLFilledChecklist chl(Config.accessList.sendHit, nullptr);
 858         clientAclChecklistFill(chl, http);
 859         chl.updateReply(&http->storeEntry()->mem().freshestReply());
 860         return !chl.fastCheck().allowed(); // when in doubt, block
 861     }
 862 }
 863
 864 // Purges all entries with a given url
 865 // TODO: move to SideAgent parent, when we have one
 866 /*
 867  * We probably cannot purge Vary-affected responses because their MD5
 868  * keys depend on vary headers.
 869  */
 870 void
 871 purgeEntriesByUrl(HttpRequest * req, const char *url)
 872 {
 873     for (HttpRequestMethod m(Http::METHOD_NONE); m != Http::METHOD_ENUM_END; ++m) {
 874         if (m.respMaybeCacheable()) {
 875             const cache_key *key = storeKeyPublic(url, m);
 876             debugs(88, 5, m << ' ' << url << ' ' << storeKeyText(key));
 877 #if USE_HTCP
 878             neighborsHtcpClear(nullptr, req, m, HTCP_CLR_INVALIDATION);
 879 #else
 880             (void)req;
 881 #endif
 882             Store::Root().evictIfFound(key);
 883         }
 884     }
 885 }
 886
 887 void
 888 clientReplyContext::purgeAllCached()
 889 {
 890     // XXX: performance regression, c_str() reallocates
 891     SBuf url(http->request->effectiveRequestUri());
 892     purgeEntriesByUrl(http->request, url.c_str());
 893 }
 894
 895 LogTags *
 896 clientReplyContext::loggingTags() const
 897 {
 898     // XXX: clientReplyContext code assumes that http cbdata is always valid.
 899     // TODO: Either add cbdataReferenceValid(http) checks in all the relevant
 900     // places, like this one, or remove cbdata protection of the http member.
 901     return &http->al->cache.code;
 902 }
 903
 904 void
 905 clientReplyContext::purgeRequest()
 906 {
 907     debugs(88, 3, "Config2.onoff.enable_purge = " <<
 908            Config2.onoff.enable_purge);
 909
 910     if (!Config2.onoff.enable_purge) {
 911         http->updateLoggingTags(LOG_TCP_DENIED);
 912         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, nullptr,
 913                                            http->getConn(), http->request, http->al);
 914         startError(err);
 915         return;
 916     }
 917
 918     /* Release both IP cache */
 919     ipcacheInvalidate(http->request->url.host());
 920
 921     // TODO: can we use purgeAllCached() here instead?
 922     purgeDoPurge();
 923 }
 924
 925 void
 926 clientReplyContext::purgeDoPurge()
 927 {
 928     auto firstFound = false;
 929     if (const auto entry = storeGetPublicByRequestMethod(http->request, Http::METHOD_GET)) {
 930         // special entries are only METHOD_GET entries without variance
 931         if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) {
 932             http->updateLoggingTags(LOG_TCP_DENIED);
 933             const auto err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, nullptr,
 934                                               http->getConn(), http->request, http->al);
 935             startError(err);
 936             entry->abandon(__func__);
 937             return;
 938         }
 939         firstFound = true;
 940         if (!purgeEntry(*entry, Http::METHOD_GET))
 941             return;
 942     }
 943
 944     detailStoreLookup(storeLookupString(firstFound));
 945
 946     if (const auto entry = storeGetPublicByRequestMethod(http->request, Http::METHOD_HEAD)) {
 947         if (!purgeEntry(*entry, Http::METHOD_HEAD))
 948             return;
 949     }
 950
 951     /* And for Vary, release the base URI if none of the headers was included in the request */
 952     if (!http->request->vary_headers.isEmpty()
 953             && http->request->vary_headers.find('=') != SBuf::npos) {
 954         // XXX: performance regression, c_str() reallocates
 955         SBuf tmp(http->request->effectiveRequestUri());
 956
 957         if (const auto entry = storeGetPublic(tmp.c_str(), Http::METHOD_GET)) {
 958             if (!purgeEntry(*entry, Http::METHOD_GET, "Vary "))
 959                 return;
 960         }
 961
 962         if (const auto entry = storeGetPublic(tmp.c_str(), Http::METHOD_HEAD)) {
 963             if (!purgeEntry(*entry, Http::METHOD_HEAD, "Vary "))
 964                 return;
 965         }
 966     }
 967
 968     if (purgeStatus == Http::scNone)
 969         purgeStatus = Http::scNotFound;
 970
 971     /*
 972      * Make a new entry to hold the reply to be written
 973      * to the client.
 974      */
 975     /* TODO: This doesn't need to go through the store. Simply
 976      * push down the client chain
 977      */
 978     createStoreEntry(http->request->method, RequestFlags());
 979
 980     triggerInitialStoreRead();
 981
 982     const HttpReplyPointer rep(new HttpReply);
 983     rep->setHeaders(purgeStatus, nullptr, nullptr, 0, 0, -1);
 984     http->storeEntry()->replaceHttpReply(rep);
 985     http->storeEntry()->complete();
 986 }
 987
 988 bool
 989 clientReplyContext::purgeEntry(StoreEntry &entry, const Http::MethodType methodType, const char *descriptionPrefix)
 990 {
 991     debugs(88, 4, descriptionPrefix << Http::MethodStr(methodType) << " '" << entry.url() << "'" );
 992 #if USE_HTCP
 993     neighborsHtcpClear(&entry, http->request, HttpRequestMethod(methodType), HTCP_CLR_PURGE);
 994 #endif
 995     entry.release(true);
 996     purgeStatus = Http::scOkay;
 997     return true;
 998 }
 999
1000 void
1001 clientReplyContext::traceReply()
1002 {
1003     createStoreEntry(http->request->method, RequestFlags());
1004     triggerInitialStoreRead();
1005     http->storeEntry()->releaseRequest();
1006     http->storeEntry()->buffer();
1007     const HttpReplyPointer rep(new HttpReply);
1008     rep->setHeaders(Http::scOkay, nullptr, "text/plain", http->request->prefixLen(), 0, squid_curtime);
1009     http->storeEntry()->replaceHttpReply(rep);
1010     http->request->swapOut(http->storeEntry());
1011     http->storeEntry()->complete();
1012 }
1013
1014 #define SENDING_BODY 0
1015 #define SENDING_HDRSONLY 1
1016 int
1017 clientReplyContext::checkTransferDone()
1018 {
1019     StoreEntry *entry = http->storeEntry();
1020
1021     if (entry == nullptr)
1022         return 0;
1023
1024     /*
1025      * For now, 'done_copying' is used for special cases like
1026      * Range and HEAD requests.
1027      */
1028     if (http->flags.done_copying)
1029         return 1;
1030
1031     if (http->request->flags.chunkedReply && !flags.complete) {
1032         // last-chunk was not sent
1033         return 0;
1034     }
1035
1036     /*
1037      * Handle STORE_OK objects.
1038      * objectLen(entry) will be set proprely.
1039      * RC: Does objectLen(entry) include the Headers?
1040      * RC: Yes.
1041      */
1042     if (entry->store_status == STORE_OK) {
1043         return storeOKTransferDone();
1044     } else {
1045         return storeNotOKTransferDone();
1046     }
1047 }
1048
1049 int
1050 clientReplyContext::storeOKTransferDone() const
1051 {
1052     assert(http->storeEntry()->objectLen() >= 0);
1053     const auto headers_sz = http->storeEntry()->mem().baseReply().hdr_sz;
1054     assert(http->storeEntry()->objectLen() >= headers_sz);
1055     const auto done = http->out.offset >= http->storeEntry()->objectLen() - headers_sz;
1056     const auto debugLevel = done ? 3 : 5;
1057     debugs(88, debugLevel, done <<
1058            " out.offset=" << http->out.offset <<
1059            " objectLen()=" << http->storeEntry()->objectLen() <<
1060            " headers_sz=" << headers_sz);
1061     return done ? 1 : 0;
1062 }
1063
1064 int
1065 clientReplyContext::storeNotOKTransferDone() const
1066 {
1067     /*
1068      * Now, handle STORE_PENDING objects
1069      */
1070     MemObject *mem = http->storeEntry()->mem_obj;
1071     assert(mem != nullptr);
1072     assert(http->request != nullptr);
1073
1074     if (!http->storeEntry()->hasParsedReplyHeader())
1075         /* haven't found end of headers yet */
1076         return 0;
1077
1078     // TODO: Use MemObject::expectedReplySize(method) after resolving XXX below.
1079     const auto expectedBodySize = mem->baseReply().content_length;
1080
1081     // XXX: The code below talks about sending data, and checks stats about
1082     // bytes written to the client connection, but this method must determine
1083     // whether we are done _receiving_ data from Store. This code should work OK
1084     // when expectedBodySize is unknown or matches written data, but it may
1085     // malfunction when we are writing ranges while receiving a full response.
1086
1087     /*
1088      * Figure out how much data we are supposed to send.
1089      * If we are sending a body and we don't have a content-length,
1090      * then we must wait for the object to become STORE_OK.
1091      */
1092     if (expectedBodySize < 0)
1093         return 0;
1094
1095     const auto done = http->out.offset >= expectedBodySize;
1096     const auto debugLevel = done ? 3 : 5;
1097     debugs(88, debugLevel, done <<
1098            " out.offset=" << http->out.offset <<
1099            " expectedBodySize=" << expectedBodySize);
1100     return done ? 1 : 0;
1101 }
1102
1103 /* Preconditions:
1104  * *http is a valid structure.
1105  * fd is either -1, or an open fd.
1106  *
1107  * TODO: enumify this
1108  *
1109  * This function is used by any http request sink, to determine the status
1110  * of the object.
1111  */
1112 clientStream_status_t
1113 clientReplyStatus(clientStreamNode * aNode, ClientHttpRequest * http)
1114 {
1115     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1116     assert (context);
1117     assert (context->http == http);
1118     return context->replyStatus();
1119 }
1120
1121 clientStream_status_t
1122 clientReplyContext::replyStatus()
1123 {
1124     int done;
1125     /* Here because lower nodes don't need it */
1126
1127     if (http->storeEntry() == nullptr) {
1128         debugs(88, 5, "clientReplyStatus: no storeEntry");
1129         return STREAM_FAILED;   /* yuck, but what can we do? */
1130     }
1131
1132     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
1133         /* TODO: Could upstream read errors (result.flags.error) be
1134          * lost, and result in undersize requests being considered
1135          * complete. Should we tcp reset such connections ?
1136          */
1137         debugs(88, 5, "clientReplyStatus: aborted storeEntry");
1138         return STREAM_FAILED;
1139     }
1140
1141     if ((done = checkTransferDone()) != 0 || flags.complete) {
1142         debugs(88, 5, "clientReplyStatus: transfer is DONE: " << done << flags.complete);
1143         /* Ok we're finished, but how? */
1144
1145         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_BAD_LENGTH)) {
1146             debugs(88, 5, "clientReplyStatus: truncated response body");
1147             return STREAM_UNPLANNED_COMPLETE;
1148         }
1149
1150         if (!done) {
1151             debugs(88, 5, "clientReplyStatus: closing, !done, but read 0 bytes");
1152             return STREAM_FAILED;
1153         }
1154
1155         // TODO: See also (and unify with) storeNotOKTransferDone() checks.
1156         const int64_t expectedBodySize =
1157             http->storeEntry()->mem().baseReply().bodySize(http->request->method);
1158         if (expectedBodySize >= 0 && !http->gotEnough()) {
1159             debugs(88, 5, "clientReplyStatus: client didn't get all it expected");
1160             return STREAM_UNPLANNED_COMPLETE;
1161         }
1162
1163         debugs(88, 5, "clientReplyStatus: stream complete; keepalive=" <<
1164                http->request->flags.proxyKeepalive);
1165         return STREAM_COMPLETE;
1166     }
1167
1168     // XXX: Should this be checked earlier? We could return above w/o checking.
1169     if (reply->receivedBodyTooLarge(*http->request, http->out.offset)) {
1170         debugs(88, 5, "clientReplyStatus: client reply body is too large");
1171         return STREAM_FAILED;
1172     }
1173
1174     return STREAM_NONE;
1175 }
1176
1177 /* Responses with no body will not have a content-type header,
1178  * which breaks the rep_mime_type acl, which
1179  * coincidentally, is the most common acl for reply access lists.
1180  * A better long term fix for this is to allow acl matches on the various
1181  * status codes, and then supply a default ruleset that puts these
1182  * codes before any user defines access entries. That way the user
1183  * can choose to block these responses where appropriate, but won't get
1184  * mysterious breakages.
1185  */
1186 bool
1187 clientReplyContext::alwaysAllowResponse(Http::StatusCode sline) const
1188 {
1189     bool result;
1190
1191     switch (sline) {
1192
1193     case Http::scContinue:
1194
1195     case Http::scSwitchingProtocols:
1196
1197     case Http::scProcessing:
1198
1199     case Http::scNoContent:
1200
1201     case Http::scNotModified:
1202         result = true;
1203         break;
1204
1205     default:
1206         result = false;
1207     }
1208
1209     return result;
1210 }
1211
1212 /**
1213  * Generate the reply headers sent to client.
1214  *
1215  * Filters out unwanted entries and hop-by-hop from original reply header
1216  * then adds extra entries if we have more info than origin server
1217  * then adds Squid specific entries
1218  */
1219 void
1220 clientReplyContext::buildReplyHeader()
1221 {
1222     HttpHeader *hdr = &reply->header;
1223     const bool is_hit = http->loggingTags().isTcpHit();
1224     HttpRequest *request = http->request;
1225
1226     if (is_hit || collapsedRevalidation == crSlave)
1227         hdr->delById(Http::HdrType::SET_COOKIE);
1228     // TODO: RFC 2965 : Must honour Cache-Control: no-cache="set-cookie2" and remove header.
1229
1230     // if there is not configured a peer proxy with login=PASS or login=PASSTHRU option enabled
1231     // remove the Proxy-Authenticate header
1232     if ( !request->peer_login || (strcmp(request->peer_login,"PASS") != 0 && strcmp(request->peer_login,"PASSTHRU") != 0)) {
1233 #if USE_ADAPTATION
1234         // but allow adaptation services to authenticate clients
1235         // via request satisfaction
1236         if (!http->requestSatisfactionMode())
1237 #endif
1238             reply->header.delById(Http::HdrType::PROXY_AUTHENTICATE);
1239     }
1240
1241     reply->header.removeHopByHopEntries();
1242     // paranoid: ContentLengthInterpreter has cleaned non-generated replies
1243     reply->removeIrrelevantContentLength();
1244
1245     //    if (request->range)
1246     //      clientBuildRangeHeader(http, reply);
1247
1248     /*
1249      * Add a estimated Age header on cache hits.
1250      */
1251     if (is_hit) {
1252         /*
1253          * Remove any existing Age header sent by upstream caches
1254          * (note that the existing header is passed along unmodified
1255          * on cache misses)
1256          */
1257         hdr->delById(Http::HdrType::AGE);
1258         /*
1259          * This adds the calculated object age. Note that the details of the
1260          * age calculation is performed by adjusting the timestamp in
1261          * StoreEntry::timestampsSet(), not here.
1262          */
1263         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
1264             hdr->delById(Http::HdrType::DATE);
1265             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1266         } else if (http->getConn() && http->getConn()->port->actAsOrigin) {
1267             // Swap the Date: header to current time if we are simulating an origin
1268             HttpHeaderEntry *h = hdr->findEntry(Http::HdrType::DATE);
1269             if (h)
1270                 hdr->putExt("X-Origin-Date", h->value.termedBuf());
1271             hdr->delById(Http::HdrType::DATE);
1272             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1273             h = hdr->findEntry(Http::HdrType::EXPIRES);
1274             if (h && http->storeEntry()->expires >= 0) {
1275                 hdr->putExt("X-Origin-Expires", h->value.termedBuf());
1276                 hdr->delById(Http::HdrType::EXPIRES);
1277                 hdr->putTime(Http::HdrType::EXPIRES, squid_curtime + http->storeEntry()->expires - http->storeEntry()->timestamp);
1278             }
1279             if (http->storeEntry()->timestamp <= squid_curtime) {
1280                 // put X-Cache-Age: instead of Age:
1281                 char age[64];
1282                 snprintf(age, sizeof(age), "%" PRId64, static_cast<int64_t>(squid_curtime - http->storeEntry()->timestamp));
1283                 hdr->putExt("X-Cache-Age", age);
1284             }
1285         } else if (http->storeEntry()->timestamp <= squid_curtime) {
1286             hdr->putInt(Http::HdrType::AGE,
1287                         squid_curtime - http->storeEntry()->timestamp);
1288         }
1289     }
1290
1291     /* RFC 2616: Section 14.18
1292      *
1293      * Add a Date: header if missing.
1294      * We have access to a clock therefore are required to amend any shortcoming in servers.
1295      *
1296      * NP: done after Age: to prevent ENTRY_SPECIAL double-handling this header.
1297      */
1298     if ( !hdr->has(Http::HdrType::DATE) ) {
1299         if (!http->storeEntry())
1300             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1301         else if (http->storeEntry()->timestamp > 0)
1302             hdr->putTime(Http::HdrType::DATE, http->storeEntry()->timestamp);
1303         else {
1304             debugs(88, DBG_IMPORTANT, "ERROR: Squid BUG #3279: HTTP reply without Date:");
1305             /* dump something useful about the problem */
1306             http->storeEntry()->dump(DBG_IMPORTANT);
1307         }
1308     }
1309
1310     /* Filter unproxyable authentication types */
1311     if (http->loggingTags().oldType != LOG_TCP_DENIED &&
1312             hdr->has(Http::HdrType::WWW_AUTHENTICATE)) {
1313         HttpHeaderPos pos = HttpHeaderInitPos;
1314         HttpHeaderEntry *e;
1315
1316         int connection_auth_blocked = 0;
1317         while ((e = hdr->getEntry(&pos))) {
1318             if (e->id == Http::HdrType::WWW_AUTHENTICATE) {
1319                 const char *value = e->value.rawBuf();
1320
1321                 if ((strncasecmp(value, "NTLM", 4) == 0 &&
1322                         (value[4] == '\0' || value[4] == ' '))
1323                         ||
1324                         (strncasecmp(value, "Negotiate", 9) == 0 &&
1325                          (value[9] == '\0' || value[9] == ' '))
1326                         ||
1327                         (strncasecmp(value, "Kerberos", 8) == 0 &&
1328                          (value[8] == '\0' || value[8] == ' '))) {
1329                     if (request->flags.connectionAuthDisabled) {
1330                         hdr->delAt(pos, connection_auth_blocked);
1331                         continue;
1332                     }
1333                     request->flags.mustKeepalive = true;
1334                     if (!request->flags.accelerated && !request->flags.intercepted) {
1335                         httpHeaderPutStrf(hdr, Http::HdrType::PROXY_SUPPORT, "Session-Based-Authentication");
1336                         /*
1337                           We send "Connection: Proxy-Support" header to mark
1338                           Proxy-Support as a hop-by-hop header for intermediaries that do not
1339                           understand the semantics of this header. The RFC should have included
1340                           this recommendation.
1341                         */
1342                         httpHeaderPutStrf(hdr, Http::HdrType::CONNECTION, "Proxy-support");
1343                     }
1344                     break;
1345                 }
1346             }
1347         }
1348
1349         if (connection_auth_blocked)
1350             hdr->refreshMask();
1351     }
1352
1353 #if USE_AUTH
1354     /* Handle authentication headers */
1355     if (http->loggingTags().oldType == LOG_TCP_DENIED &&
1356             ( reply->sline.status() == Http::scProxyAuthenticationRequired ||
1357               reply->sline.status() == Http::scUnauthorized)
1358        ) {
1359         /* Add authentication header */
1360         /* TODO: alter errorstate to be accel on|off aware. The 0 on the next line
1361          * depends on authenticate behaviour: all schemes to date send no extra
1362          * data on 407/401 responses, and do not check the accel state on 401/407
1363          * responses
1364          */
1365         Auth::UserRequest::AddReplyAuthHeader(reply, request->auth_user_request, request, 0, 1);
1366     } else if (request->auth_user_request != nullptr)
1367         Auth::UserRequest::AddReplyAuthHeader(reply, request->auth_user_request, request, http->flags.accel, 0);
1368 #endif
1369
1370     SBuf cacheStatus(uniqueHostname());
1371     if (const auto hitOrFwd = http->loggingTags().cacheStatusSource())
1372         cacheStatus.append(hitOrFwd);
1373     if (firstStoreLookup_) {
1374         cacheStatus.append(";detail=");
1375         cacheStatus.append(firstStoreLookup_);
1376     }
1377     // TODO: Remove c_str() after converting HttpHeaderEntry::value to SBuf
1378     hdr->putStr(Http::HdrType::CACHE_STATUS, cacheStatus.c_str());
1379
1380     const bool maySendChunkedReply = !request->multipartRangeRequest() &&
1381                                      reply->sline.version.protocol == AnyP::PROTO_HTTP && // response is HTTP
1382                                      (request->http_ver >= Http::ProtocolVersion(1,1));
1383
1384     /* Check whether we should send keep-alive */
1385     if (!Config.onoff.error_pconns && reply->sline.status() >= 400 && !request->flags.mustKeepalive) {
1386         debugs(33, 3, "clientBuildReplyHeader: Error, don't keep-alive");
1387         request->flags.proxyKeepalive = false;
1388     } else if (!Config.onoff.client_pconns && !request->flags.mustKeepalive) {
1389         debugs(33, 2, "clientBuildReplyHeader: Connection Keep-Alive not requested by admin or client");
1390         request->flags.proxyKeepalive = false;
1391     } else if (request->flags.proxyKeepalive && shutting_down) {
1392         debugs(88, 3, "clientBuildReplyHeader: Shutting down, don't keep-alive.");
1393         request->flags.proxyKeepalive = false;
1394     } else if (request->flags.connectionAuth && !reply->keep_alive) {
1395         debugs(33, 2, "clientBuildReplyHeader: Connection oriented auth but server side non-persistent");
1396         request->flags.proxyKeepalive = false;
1397     } else if (reply->bodySize(request->method) < 0 && !maySendChunkedReply) {
1398         debugs(88, 3, "clientBuildReplyHeader: can't keep-alive, unknown body size" );
1399         request->flags.proxyKeepalive = false;
1400     } else if (fdUsageHigh()&& !request->flags.mustKeepalive) {
1401         debugs(88, 3, "clientBuildReplyHeader: Not many unused FDs, can't keep-alive");
1402         request->flags.proxyKeepalive = false;
1403     } else if (request->flags.sslBumped && !reply->persistent()) {
1404         // We do not really have to close, but we pretend we are a tunnel.
1405         debugs(88, 3, "clientBuildReplyHeader: bumped reply forces close");
1406         request->flags.proxyKeepalive = false;
1407     } else if (request->pinnedConnection() && !reply->persistent()) {
1408         // The peer wants to close the pinned connection
1409         debugs(88, 3, "pinned reply forces close");
1410         request->flags.proxyKeepalive = false;
1411     } else if (http->getConn()) {
1412         ConnStateData * conn = http->getConn();
1413         if (!Comm::IsConnOpen(conn->port->listenConn)) {
1414             // The listening port closed because of a reconfigure
1415             debugs(88, 3, "listening port closed");
1416             request->flags.proxyKeepalive = false;
1417         }
1418     }
1419
1420     // Decide if we send chunked reply
1421     if (maySendChunkedReply && reply->bodySize(request->method) < 0) {
1422         debugs(88, 3, "clientBuildReplyHeader: chunked reply");
1423         request->flags.chunkedReply = true;
1424         hdr->putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
1425     }
1426
1427     hdr->addVia(reply->sline.version);
1428
1429     /* Signal keep-alive or close explicitly */
1430     hdr->putStr(Http::HdrType::CONNECTION, request->flags.proxyKeepalive ? "keep-alive" : "close");
1431
1432     /* Surrogate-Control requires Surrogate-Capability from upstream to pass on */
1433     if ( hdr->has(Http::HdrType::SURROGATE_CONTROL) ) {
1434         if (!request->header.has(Http::HdrType::SURROGATE_CAPABILITY)) {
1435             hdr->delById(Http::HdrType::SURROGATE_CONTROL);
1436         }
1437         /* TODO: else case: drop any controls intended specifically for our surrogate ID */
1438     }
1439
1440     httpHdrMangleList(hdr, request, http->al, ROR_REPLY);
1441 }
1442
1443 void
1444 clientReplyContext::cloneReply()
1445 {
1446     assert(reply == nullptr);
1447
1448     reply = http->storeEntry()->mem().freshestReply().clone();
1449     HTTPMSGLOCK(reply);
1450
1451     http->al->reply = reply;
1452
1453     if (reply->sline.version.protocol == AnyP::PROTO_HTTP) {
1454         /* RFC 2616 requires us to advertise our version (but only on real HTTP traffic) */
1455         reply->sline.version = Http::ProtocolVersion();
1456     }
1457
1458     /* do header conversions */
1459     buildReplyHeader();
1460 }
1461
1462 /// Safely disposes of an entry pointing to a cache hit that we do not want.
1463 /// We cannot just ignore the entry because it may be locking or otherwise
1464 /// holding an associated cache resource of some sort.
1465 void
1466 clientReplyContext::forgetHit()
1467 {
1468     StoreEntry *e = http->storeEntry();
1469     assert(e); // or we are not dealing with a hit
1470     // We probably have not locked the entry earlier, unfortunately. We lock it
1471     // now so that we can unlock two lines later (and trigger cleanup).
1472     // Ideally, ClientHttpRequest::storeEntry() should lock/unlock, but it is
1473     // used so inconsistently that simply adding locking there leads to bugs.
1474     e->lock("clientReplyContext::forgetHit");
1475     http->storeEntry(nullptr);
1476     e->unlock("clientReplyContext::forgetHit"); // may delete e
1477 }
1478
1479 void
1480 clientReplyContext::identifyStoreObject()
1481 {
1482     HttpRequest *r = http->request;
1483
1484     // client sent CC:no-cache or some other condition has been
1485     // encountered which prevents delivering a public/cached object.
1486     // XXX: The above text does not match the condition below. It might describe
1487     // the opposite condition, but the condition itself should be adjusted
1488     // (e.g., to honor flags.noCache in cache manager requests).
1489     if (!r->flags.noCache || r->flags.internal) {
1490         const auto e = storeGetPublicByRequest(r);
1491         identifyFoundObject(e, storeLookupString(bool(e)));
1492     } else {
1493         // "external" no-cache requests skip Store lookups
1494         identifyFoundObject(nullptr, "no-cache");
1495     }
1496 }
1497
1498 /**
1499  * Check state of the current StoreEntry object.
1500  * to see if we can determine the final status of the request.
1501  */
1502 void
1503 clientReplyContext::identifyFoundObject(StoreEntry *newEntry, const char *detail)
1504 {
1505     detailStoreLookup(detail);
1506
1507     HttpRequest *r = http->request;
1508     http->storeEntry(newEntry);
1509     const auto e = http->storeEntry();
1510
1511     /* Release IP-cache entries on reload */
1512     /** \li If the request has no-cache flag set or some no_cache HACK in operation we
1513       * 'invalidate' the cached IP entries for this request ???
1514       */
1515     if (r->flags.noCache || r->flags.noCacheHack())
1516         ipcacheInvalidateNegative(r->url.host());
1517
1518     if (!e) {
1519         /** \li If no StoreEntry object is current assume this object isn't in the cache set MISS*/
1520         debugs(85, 3, "StoreEntry is NULL -  MISS");
1521         http->updateLoggingTags(LOG_TCP_MISS);
1522         doGetMoreData();
1523         return;
1524     }
1525
1526     if (Config.onoff.offline) {
1527         /** \li If we are running in offline mode set to HIT */
1528         debugs(85, 3, "offline HIT " << *e);
1529         http->updateLoggingTags(LOG_TCP_HIT);
1530         doGetMoreData();
1531         return;
1532     }
1533
1534     if (http->redirect.status) {
1535         /** \li If redirection status is True force this to be a MISS */
1536         debugs(85, 3, "REDIRECT status forced StoreEntry to NULL (no body on 3XX responses) " << *e);
1537         forgetHit();
1538         http->updateLoggingTags(LOG_TCP_REDIRECT);
1539         doGetMoreData();
1540         return;
1541     }
1542
1543     if (!e->validToSend()) {
1544         debugs(85, 3, "!storeEntryValidToSend MISS " << *e);
1545         forgetHit();
1546         http->updateLoggingTags(LOG_TCP_MISS);
1547         doGetMoreData();
1548         return;
1549     }
1550
1551     if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
1552         /* \li Special entries are always hits, no matter what the client says */
1553         debugs(85, 3, "ENTRY_SPECIAL HIT " << *e);
1554         http->updateLoggingTags(LOG_TCP_HIT);
1555         doGetMoreData();
1556         return;
1557     }
1558
1559     if (r->flags.noCache) {
1560         debugs(85, 3, "no-cache REFRESH MISS " << *e);
1561         forgetHit();
1562         http->updateLoggingTags(LOG_TCP_CLIENT_REFRESH_MISS);
1563         doGetMoreData();
1564         return;
1565     }
1566
1567     if (e->hittingRequiresCollapsing() && !startCollapsingOn(*e, false)) {
1568         debugs(85, 3, "prohibited CF MISS " << *e);
1569         forgetHit();
1570         http->updateLoggingTags(LOG_TCP_MISS);
1571         doGetMoreData();
1572         return;
1573     }
1574
1575     debugs(85, 3, "default HIT " << *e);
1576     http->updateLoggingTags(LOG_TCP_HIT);
1577     doGetMoreData();
1578 }
1579
1580 /// remembers the very first Store lookup classification, ignoring the rest
1581 void
1582 clientReplyContext::detailStoreLookup(const char *detail)
1583 {
1584     if (!firstStoreLookup_) {
1585         debugs(85, 7, detail);
1586         firstStoreLookup_ = detail;
1587     } else {
1588         debugs(85, 7, "ignores " << detail << " after " << firstStoreLookup_);
1589     }
1590 }
1591
1592 /**
1593  * Request more data from the store for the client Stream
1594  * This is *the* entry point to this module.
1595  *
1596  * Preconditions:
1597  *  - This is the head of the list.
1598  *  - There is at least one more node.
1599  *  - Data context is not null
1600  */
1601 void
1602 clientGetMoreData(clientStreamNode * aNode, ClientHttpRequest * http)
1603 {
1604     /* Test preconditions */
1605     assert(aNode != nullptr);
1606     assert(cbdataReferenceValid(aNode));
1607     assert(aNode->node.prev == nullptr);
1608     assert(aNode->node.next != nullptr);
1609     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1610     assert (context);
1611     assert(context->http == http);
1612
1613     if (!context->ourNode)
1614         context->ourNode = aNode;
1615
1616     /* no cbdatareference, this is only used once, and safely */
1617     if (context->flags.storelogiccomplete) {
1618         context->requestMoreBodyFromStore();
1619         return;
1620     }
1621
1622     if (context->http->request->method == Http::METHOD_PURGE) {
1623         context->purgeRequest();
1624         return;
1625     }
1626
1627     // OPTIONS with Max-Forwards:0 handled in clientProcessRequest()
1628
1629     if (context->http->request->method == Http::METHOD_TRACE) {
1630         if (context->http->request->header.getInt64(Http::HdrType::MAX_FORWARDS) == 0) {
1631             context->traceReply();
1632             return;
1633         }
1634
1635         /* continue forwarding, not finished yet. */
1636         http->updateLoggingTags(LOG_TCP_MISS);
1637
1638         context->doGetMoreData();
1639     } else
1640         context->identifyStoreObject();
1641 }
1642
1643 void
1644 clientReplyContext::doGetMoreData()
1645 {
1646     /* We still have to do store logic processing - vary, cache hit etc */
1647     if (http->storeEntry() != nullptr) {
1648         /* someone found the object in the cache for us */
1649         StoreIOBuffer localTempBuffer;
1650
1651         http->storeEntry()->lock("clientReplyContext::doGetMoreData");
1652
1653         http->storeEntry()->ensureMemObject(storeId(), http->log_uri, http->request->method);
1654
1655         sc = storeClientListAdd(http->storeEntry(), this);
1656 #if USE_DELAY_POOLS
1657         sc->setDelayId(DelayId::DelayClient(http));
1658 #endif
1659
1660         assert(http->loggingTags().oldType == LOG_TCP_HIT);
1661         /* guarantee nothing has been sent yet! */
1662         assert(http->out.size == 0);
1663         assert(http->out.offset == 0);
1664
1665         if (ConnStateData *conn = http->getConn()) {
1666             if (Ip::Qos::TheConfig.isHitTosActive()) {
1667                 Ip::Qos::doTosLocalHit(conn->clientConnection);
1668             }
1669
1670             if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
1671                 Ip::Qos::doNfmarkLocalHit(conn->clientConnection);
1672             }
1673         }
1674
1675         triggerInitialStoreRead(CacheHit);
1676     } else {
1677         /* MISS CASE, http->loggingTags() are already set! */
1678         processMiss();
1679     }
1680 }
1681
1682 /** The next node has removed itself from the stream. */
1683 void
1684 clientReplyDetach(clientStreamNode * node, ClientHttpRequest * http)
1685 {
1686     /** detach from the stream */
1687     clientStreamDetach(node, http);
1688 }
1689
1690 /**
1691  * Accepts chunk of a http message in buf, parses prefix, filters headers and
1692  * such, writes processed message to the message recipient
1693  */
1694 void
1695 clientReplyContext::SendMoreData(void *data, StoreIOBuffer result)
1696 {
1697     clientReplyContext *context = static_cast<clientReplyContext *>(data);
1698     context->sendMoreData (result);
1699 }
1700
1701 void
1702 clientReplyContext::makeThisHead()
1703 {
1704     /* At least, I think that's what this does */
1705     dlinkDelete(&http->active, &ClientActiveRequests);
1706     dlinkAdd(http, &http->active, &ClientActiveRequests);
1707 }
1708
1709 bool
1710 clientReplyContext::errorInStream(const StoreIOBuffer &result) const
1711 {
1712     return /* aborted request */
1713         (http->storeEntry() && EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) ||
1714         /* Upstream read error */ (result.flags.error);
1715 }
1716
1717 void
1718 clientReplyContext::sendStreamError(StoreIOBuffer const &result)
1719 {
1720     /** call clientWriteComplete so the client socket gets closed
1721      *
1722      * We call into the stream, because we don't know that there is a
1723      * client socket!
1724      */
1725     debugs(88, 5, "A stream error has occurred, marking as complete and sending no data.");
1726     StoreIOBuffer localTempBuffer;
1727     flags.complete = 1;
1728     http->request->flags.streamError = true;
1729     localTempBuffer.flags.error = result.flags.error;
1730     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, nullptr,
1731                          localTempBuffer);
1732 }
1733
1734 void
1735 clientReplyContext::pushStreamData(const StoreIOBuffer &result)
1736 {
1737     if (result.length == 0) {
1738         debugs(88, 5, "clientReplyContext::pushStreamData: marking request as complete due to 0 length store result");
1739         flags.complete = 1;
1740     }
1741
1742     assert(!result.length || result.offset == next()->readBuffer.offset);
1743     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, nullptr,
1744                          result);
1745 }
1746
1747 clientStreamNode *
1748 clientReplyContext::next() const
1749 {
1750     assert ( (clientStreamNode*)http->client_stream.head->next->data == getNextNode());
1751     return getNextNode();
1752 }
1753
1754 void
1755 clientReplyContext::sendBodyTooLargeError()
1756 {
1757     http->updateLoggingTags(LOG_TCP_DENIED_REPLY);
1758     ErrorState *err = clientBuildError(ERR_TOO_BIG, Http::scForbidden, nullptr,
1759                                        http->getConn(), http->request, http->al);
1760     removeClientStoreReference(&(sc), http);
1761     HTTPMSGUNLOCK(reply);
1762     startError(err);
1763
1764 }
1765
1766 /// send 412 (Precondition Failed) to client
1767 void
1768 clientReplyContext::sendPreconditionFailedError()
1769 {
1770     http->updateLoggingTags(LOG_TCP_HIT);
1771     ErrorState *const err =
1772         clientBuildError(ERR_PRECONDITION_FAILED, Http::scPreconditionFailed,
1773                          nullptr, http->getConn(), http->request, http->al);
1774     removeClientStoreReference(&sc, http);
1775     HTTPMSGUNLOCK(reply);
1776     startError(err);
1777 }
1778
1779 /// send 304 (Not Modified) to client
1780 void
1781 clientReplyContext::sendNotModified()
1782 {
1783     StoreEntry *e = http->storeEntry();
1784     const time_t timestamp = e->timestamp;
1785     const auto temprep = e->mem().freshestReply().make304();
1786     // log as TCP_INM_HIT if code 304 generated for
1787     // If-None-Match request
1788     if (!http->request->flags.ims)
1789         http->updateLoggingTags(LOG_TCP_INM_HIT);
1790     else
1791         http->updateLoggingTags(LOG_TCP_IMS_HIT);
1792     removeClientStoreReference(&sc, http);
1793     createStoreEntry(http->request->method, RequestFlags());
1794     e = http->storeEntry();
1795     // Copy timestamp from the original entry so the 304
1796     // reply has a meaningful Age: header.
1797     e->timestampsSet();
1798     e->timestamp = timestamp;
1799     e->replaceHttpReply(temprep);
1800     e->complete();
1801     /*
1802      * TODO: why put this in the store and then serialise it and
1803      * then parse it again. Simply mark the request complete in
1804      * our context and write the reply struct to the client side.
1805      */
1806     triggerInitialStoreRead();
1807 }
1808
1809 /// send 304 (Not Modified) or 412 (Precondition Failed) to client
1810 /// depending on request method
1811 void
1812 clientReplyContext::sendNotModifiedOrPreconditionFailedError()
1813 {
1814     if (http->request->method == Http::METHOD_GET ||
1815             http->request->method == Http::METHOD_HEAD)
1816         sendNotModified();
1817     else
1818         sendPreconditionFailedError();
1819 }
1820
1821 void
1822 clientReplyContext::processReplyAccess ()
1823 {
1824     /* NP: this should probably soft-fail to a zero-sized-reply error ?? */
1825     assert(reply);
1826
1827     /** Don't block our own responses or HTTP status messages */
1828     if (http->loggingTags().oldType == LOG_TCP_DENIED ||
1829             http->loggingTags().oldType == LOG_TCP_DENIED_REPLY ||
1830             alwaysAllowResponse(reply->sline.status())) {
1831         processReplyAccessResult(ACCESS_ALLOWED);
1832         return;
1833     }
1834
1835     /** Check for reply to big error */
1836     if (reply->expectedBodyTooLarge(*http->request)) {
1837         sendBodyTooLargeError();
1838         return;
1839     }
1840
1841     /** check for absent access controls (permit by default) */
1842     if (!Config.accessList.reply) {
1843         processReplyAccessResult(ACCESS_ALLOWED);
1844         return;
1845     }
1846
1847     /** Process http_reply_access lists */
1848     auto replyChecklist =
1849         clientAclChecklistCreate(Config.accessList.reply, http);
1850     replyChecklist->updateReply(reply);
1851     ACLFilledChecklist::NonBlockingCheck(std::move(replyChecklist), ProcessReplyAccessResult, this);
1852 }
1853
1854 void
1855 clientReplyContext::ProcessReplyAccessResult(Acl::Answer rv, void *voidMe)
1856 {
1857     clientReplyContext *me = static_cast<clientReplyContext *>(voidMe);
1858     me->processReplyAccessResult(rv);
1859 }
1860
1861 void
1862 clientReplyContext::processReplyAccessResult(const Acl::Answer &accessAllowed)
1863 {
1864     debugs(88, 2, "The reply for " << http->request->method
1865            << ' ' << http->uri << " is " << accessAllowed << ", because it matched "
1866            << accessAllowed.lastCheckDescription());
1867
1868     if (!accessAllowed.allowed()) {
1869         ErrorState *err;
1870         auto page_id = FindDenyInfoPage(accessAllowed, true);
1871
1872         http->updateLoggingTags(LOG_TCP_DENIED_REPLY);
1873
1874         if (page_id == ERR_NONE)
1875             page_id = ERR_ACCESS_DENIED;
1876
1877         err = clientBuildError(page_id, Http::scForbidden, nullptr,
1878                                http->getConn(), http->request, http->al);
1879
1880         removeClientStoreReference(&sc, http);
1881
1882         HTTPMSGUNLOCK(reply);
1883
1884         startError(err);
1885
1886         return;
1887     }
1888
1889     /* Ok, the reply is allowed, */
1890     http->loggingEntry(http->storeEntry());
1891
1892     Assure(matchesStreamBodyBuffer(lastStreamBufferedBytes));
1893     Assure(!lastStreamBufferedBytes.offset);
1894     auto body_size = lastStreamBufferedBytes.length; // may be zero
1895
1896     debugs(88, 3, "clientReplyContext::sendMoreData: Appending " <<
1897            (int) body_size << " bytes after " << reply->hdr_sz <<
1898            " bytes of headers");
1899
1900     if (http->request->method == Http::METHOD_HEAD) {
1901         /* do not forward body for HEAD replies */
1902         body_size = 0;
1903         http->flags.done_copying = true;
1904         flags.complete = 1;
1905     }
1906
1907     assert (!flags.headersSent);
1908     flags.headersSent = true;
1909
1910     // next()->readBuffer.offset may be positive for Range requests, but our
1911     // localTempBuffer initialization code assumes that next()->readBuffer.data
1912     // points to the response body at offset 0 because the first
1913     // storeClientCopy() request always has offset 0 (i.e. our first Store
1914     // request ignores next()->readBuffer.offset).
1915     //
1916     // XXX: We cannot fully check that assumption: readBuffer.offset field is
1917     // often out of sync with the buffer content, and if some buggy code updates
1918     // the buffer while we were waiting for the processReplyAccessResult()
1919     // callback, we may not notice.
1920
1921     StoreIOBuffer localTempBuffer;
1922     const auto body_buf = next()->readBuffer.data;
1923
1924     //Server side may disable ranges under some circumstances.
1925
1926     if ((!http->request->range))
1927         next()->readBuffer.offset = 0;
1928
1929     if (next()->readBuffer.offset > 0) {
1930         if (Less(body_size, next()->readBuffer.offset)) {
1931             /* Can't use any of the body we received. send nothing */
1932             localTempBuffer.length = 0;
1933             localTempBuffer.data = nullptr;
1934         } else {
1935             localTempBuffer.length = body_size - next()->readBuffer.offset;
1936             localTempBuffer.data = body_buf + next()->readBuffer.offset;
1937         }
1938     } else {
1939         localTempBuffer.length = body_size;
1940         localTempBuffer.data = body_buf;
1941     }
1942
1943     clientStreamCallback((clientStreamNode *)http->client_stream.head->data,
1944                          http, reply, localTempBuffer);
1945
1946     return;
1947 }
1948
1949 void
1950 clientReplyContext::sendMoreData (StoreIOBuffer result)
1951 {
1952     if (deleting)
1953         return;
1954
1955     debugs(88, 5, http->uri << " got " << result);
1956
1957     StoreEntry *entry = http->storeEntry();
1958
1959     if (ConnStateData * conn = http->getConn()) {
1960         if (!conn->isOpen()) {
1961             debugs(33,3, "not sending more data to closing connection " << conn->clientConnection);
1962             return;
1963         }
1964         if (conn->pinning.zeroReply) {
1965             debugs(33,3, "not sending more data after a pinned zero reply " << conn->clientConnection);
1966             return;
1967         }
1968
1969         if (!flags.headersSent && !http->loggingTags().isTcpHit()) {
1970             // We get here twice if processReplyAccessResult() calls startError().
1971             // TODO: Revise when we check/change QoS markings to reduce syscalls.
1972             if (Ip::Qos::TheConfig.isHitTosActive()) {
1973                 Ip::Qos::doTosLocalMiss(conn->clientConnection, http->request->hier.code);
1974             }
1975             if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
1976                 Ip::Qos::doNfmarkLocalMiss(conn->clientConnection, http->request->hier.code);
1977             }
1978         }
1979
1980         debugs(88, 5, conn->clientConnection <<
1981                " '" << entry->url() << "'" <<
1982                " out.offset=" << http->out.offset);
1983     }
1984
1985     /* We've got the final data to start pushing... */
1986     flags.storelogiccomplete = 1;
1987
1988     assert(http->request != nullptr);
1989
1990     /* ESI TODO: remove this assert once everything is stable */
1991     assert(http->client_stream.head->data
1992            && cbdataReferenceValid(http->client_stream.head->data));
1993
1994     makeThisHead();
1995
1996     if (errorInStream(result)) {
1997         sendStreamError(result);
1998         return;
1999     }
2000
2001     if (!matchesStreamBodyBuffer(result)) {
2002         // Subsequent processing expects response body bytes to be at the start
2003         // of our Client Stream buffer. When given something else (e.g., bytes
2004         // in our tempbuf), we copy and adjust to meet those expectations.
2005         const auto &ourClientStreamsBuffer = next()->readBuffer;
2006         assert(result.length <= ourClientStreamsBuffer.length);
2007         memcpy(ourClientStreamsBuffer.data, result.data, result.length);
2008         result.data = ourClientStreamsBuffer.data;
2009     }
2010
2011     noteStreamBufferredBytes(result);
2012
2013     if (flags.headersSent) {
2014         pushStreamData(result);
2015         return;
2016     }
2017
2018     cloneReply();
2019
2020 #if USE_DELAY_POOLS
2021     if (sc)
2022         sc->setDelayId(DelayId::DelayClient(http,reply));
2023 #endif
2024
2025     processReplyAccess();
2026     return;
2027 }
2028
2029 /// Whether the given body area describes the start of our Client Stream buffer.
2030 /// An empty area does.
2031 bool
2032 clientReplyContext::matchesStreamBodyBuffer(const StoreIOBuffer &their) const
2033 {
2034     // the answer is undefined for errors; they are not really "body buffers"
2035     Assure(!their.flags.error);
2036
2037     if (!their.length)
2038         return true; // an empty body area always matches our body area
2039
2040     if (their.data != next()->readBuffer.data) {
2041         debugs(88, 7, "no: " << their << " vs. " << next()->readBuffer);
2042         return false;
2043     }
2044
2045     return true;
2046 }
2047
2048 void
2049 clientReplyContext::noteStreamBufferredBytes(const StoreIOBuffer &result)
2050 {
2051     Assure(matchesStreamBodyBuffer(result));
2052     lastStreamBufferedBytes = result; // may be unchanged and/or zero-length
2053 }
2054
2055 void
2056 clientReplyContext::fillChecklist(ACLFilledChecklist &checklist) const
2057 {
2058     clientAclChecklistFill(checklist, http);
2059 }
2060
2061 /* Using this breaks the client layering just a little!
2062  */
2063 void
2064 clientReplyContext::createStoreEntry(const HttpRequestMethod& m, RequestFlags reqFlags)
2065 {
2066     assert(http != nullptr);
2067     /*
2068      * For erroneous requests, we might not have a h->request,
2069      * so make a fake one.
2070      */
2071
2072     if (http->request == nullptr) {
2073         const auto connManager = http->getConn();
2074         const auto mx = MasterXaction::MakePortful(connManager ? connManager->port : nullptr);
2075         // XXX: These fake URI parameters shadow the real (or error:...) URI.
2076         // TODO: Either always set the request earlier and assert here OR use
2077         // http->uri (converted to Anyp::Uri) to create this catch-all request.
2078         const_cast<HttpRequest *&>(http->request) =  new HttpRequest(m, AnyP::PROTO_NONE, "http", null_string, mx);
2079         HTTPMSGLOCK(http->request);
2080     }
2081
2082     StoreEntry *e = storeCreateEntry(storeId(), http->log_uri, reqFlags, m);
2083
2084     // Make entry collapsible ASAP, to increase collapsing chances for others,
2085     // TODO: every must-revalidate and similar request MUST reach the origin,
2086     // but do we have to prohibit others from collapsing on that request?
2087     if (reqFlags.cachable &&
2088             !reqFlags.needValidation &&
2089             (m == Http::METHOD_GET || m == Http::METHOD_HEAD) &&
2090             mayInitiateCollapsing()) {
2091         // make the entry available for future requests now
2092         (void)Store::Root().allowCollapsing(e, reqFlags, m);
2093     }
2094
2095     sc = storeClientListAdd(e, this);
2096
2097 #if USE_DELAY_POOLS
2098     sc->setDelayId(DelayId::DelayClient(http));
2099 #endif
2100
2101     /* The next line is illegal because we don't know if the client stream
2102      * buffers have been set up
2103      */
2104     //    storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf,
2105     //        SendMoreData, this);
2106     /* So, we mark the store logic as complete */
2107     flags.storelogiccomplete = 1;
2108
2109     /* and get the caller to request a read, from wherever they are */
2110     /* NOTE: after ANY data flows down the pipe, even one step,
2111      * this function CAN NOT be used to manage errors
2112      */
2113     http->storeEntry(e);
2114 }
2115
2116 ErrorState *
2117 clientBuildError(err_type page_id, Http::StatusCode status, char const *url,
2118                  const ConnStateData *conn, HttpRequest *request, const AccessLogEntry::Pointer &al)
2119 {
2120     const auto err = new ErrorState(page_id, status, request, al);
2121     err->src_addr = conn && conn->clientConnection ? conn->clientConnection->remote : Ip::Address::NoAddr();
2122
2123     if (url)
2124         err->url = xstrdup(url);
2125
2126     return err;
2127 }
2128