src/client_side_reply.cc

   1 /*
   2  * Copyright (C) 1996-2019 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 88    Client-side Reply Routines */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "acl/Gadgets.h"
  14 #include "anyp/PortCfg.h"
  15 #include "client_side_reply.h"
  16 #include "errorpage.h"
  17 #include "ETag.h"
  18 #include "fd.h"
  19 #include "fde.h"
  20 #include "format/Token.h"
  21 #include "FwdState.h"
  22 #include "globals.h"
  23 #include "http/Stream.h"
  24 #include "HttpHeaderTools.h"
  25 #include "HttpReply.h"
  26 #include "HttpRequest.h"
  27 #include "ip/QosConfig.h"
  28 #include "ipcache.h"
  29 #include "log/access_log.h"
  30 #include "MemObject.h"
  31 #include "mime_header.h"
  32 #include "neighbors.h"
  33 #include "refresh.h"
  34 #include "RequestFlags.h"
  35 #include "SquidConfig.h"
  36 #include "SquidTime.h"
  37 #include "Store.h"
  38 #include "StrList.h"
  39 #include "tools.h"
  40 #if USE_AUTH
  41 #include "auth/UserRequest.h"
  42 #endif
  43 #if USE_DELAY_POOLS
  44 #include "DelayPools.h"
  45 #endif
  46 #if USE_SQUID_ESI
  47 #include "esi/Esi.h"
  48 #endif
  49
  50 #include <memory>
  51
  52 CBDATA_CLASS_INIT(clientReplyContext);
  53
  54 /* Local functions */
  55 extern "C" CSS clientReplyStatus;
  56 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *, Ip::Address &, HttpRequest *, const AccessLogEntry::Pointer &);
  57
  58 /* privates */
  59
  60 clientReplyContext::~clientReplyContext()
  61 {
  62     deleting = true;
  63     /* This may trigger a callback back into SendMoreData as the cbdata
  64      * is still valid
  65      */
  66     removeClientStoreReference(&sc, http);
  67     /* old_entry might still be set if we didn't yet get the reply
  68      * code in HandleIMSReply() */
  69     removeStoreReference(&old_sc, &old_entry);
  70     safe_free(tempBuffer.data);
  71     cbdataReferenceDone(http);
  72     HTTPMSGUNLOCK(reply);
  73 }
  74
  75 clientReplyContext::clientReplyContext(ClientHttpRequest *clientContext) :
  76     purgeStatus(Http::scNone),
  77     lookingforstore(0),
  78     http(cbdataReference(clientContext)),
  79     headers_sz(0),
  80     sc(NULL),
  81     old_reqsize(0),
  82     reqsize(0),
  83     reqofs(0),
  84 #if USE_CACHE_DIGESTS
  85     lookup_type(NULL),
  86 #endif
  87     ourNode(NULL),
  88     reply(NULL),
  89     old_entry(NULL),
  90     old_sc(NULL),
  91     old_lastmod(-1),
  92     deleting(false),
  93     collapsedRevalidation(crNone)
  94 {
  95     *tempbuf = 0;
  96 }
  97
  98 /** Create an error in the store awaiting the client side to read it.
  99  *
 100  * This may be better placed in the clientStream logic, but it has not been
 101  * relocated there yet
 102  */
 103 void
 104 clientReplyContext::setReplyToError(
 105     err_type err, Http::StatusCode status, const HttpRequestMethod& method, char const *uri,
 106     Ip::Address &addr, HttpRequest * failedrequest, const char *unparsedrequest,
 107 #if USE_AUTH
 108     Auth::UserRequest::Pointer auth_user_request
 109 #else
 110     void*
 111 #endif
 112 )
 113 {
 114     auto errstate = clientBuildError(err, status, uri, addr, failedrequest, http->al);
 115
 116     if (unparsedrequest)
 117         errstate->request_hdrs = xstrdup(unparsedrequest);
 118
 119 #if USE_AUTH
 120     errstate->auth_user_request = auth_user_request;
 121 #endif
 122     setReplyToError(method, errstate);
 123 }
 124
 125 void clientReplyContext::setReplyToError(const HttpRequestMethod& method, ErrorState *errstate)
 126 {
 127     if (errstate->httpStatus == Http::scNotImplemented && http->request)
 128         /* prevent confusion over whether we default to persistent or not */
 129         http->request->flags.proxyKeepalive = false;
 130
 131     http->al->http.code = errstate->httpStatus;
 132
 133     if (http->request)
 134         http->request->ignoreRange("responding with a Squid-generated error");
 135
 136     createStoreEntry(method, RequestFlags());
 137     assert(errstate->callback_data == NULL);
 138     errorAppendEntry(http->storeEntry(), errstate);
 139     /* Now the caller reads to get this */
 140 }
 141
 142 void
 143 clientReplyContext::setReplyToReply(HttpReply *futureReply)
 144 {
 145     Must(futureReply);
 146     http->al->http.code = futureReply->sline.status();
 147
 148     HttpRequestMethod method;
 149     if (http->request) { // nil on responses to unparsable requests
 150         http->request->ignoreRange("responding with a Squid-generated reply");
 151         method = http->request->method;
 152     }
 153
 154     createStoreEntry(method, RequestFlags());
 155
 156     http->storeEntry()->storeErrorResponse(futureReply);
 157     /* Now the caller reads to get futureReply */
 158 }
 159
 160 // Assumes that the entry contains an error response without Content-Range.
 161 // To use with regular entries, make HTTP Range header removal conditional.
 162 void clientReplyContext::setReplyToStoreEntry(StoreEntry *entry, const char *reason)
 163 {
 164     entry->lock("clientReplyContext::setReplyToStoreEntry"); // removeClientStoreReference() unlocks
 165     sc = storeClientListAdd(entry, this);
 166 #if USE_DELAY_POOLS
 167     sc->setDelayId(DelayId::DelayClient(http));
 168 #endif
 169     reqofs = 0;
 170     reqsize = 0;
 171     if (http->request)
 172         http->request->ignoreRange(reason);
 173     flags.storelogiccomplete = 1;
 174     http->storeEntry(entry);
 175 }
 176
 177 void
 178 clientReplyContext::removeStoreReference(store_client ** scp,
 179         StoreEntry ** ep)
 180 {
 181     StoreEntry *e;
 182     store_client *sc_tmp = *scp;
 183
 184     if ((e = *ep) != NULL) {
 185         *ep = NULL;
 186         storeUnregister(sc_tmp, e, this);
 187         *scp = NULL;
 188         e->unlock("clientReplyContext::removeStoreReference");
 189     }
 190 }
 191
 192 void
 193 clientReplyContext::removeClientStoreReference(store_client **scp, ClientHttpRequest *aHttpRequest)
 194 {
 195     StoreEntry *reference = aHttpRequest->storeEntry();
 196     removeStoreReference(scp, &reference);
 197     aHttpRequest->storeEntry(reference);
 198 }
 199
 200 void
 201 clientReplyContext::saveState()
 202 {
 203     assert(old_sc == NULL);
 204     debugs(88, 3, "clientReplyContext::saveState: saving store context");
 205     old_entry = http->storeEntry();
 206     old_sc = sc;
 207     old_lastmod = http->request->lastmod;
 208     old_etag = http->request->etag;
 209     old_reqsize = reqsize;
 210     tempBuffer.offset = reqofs;
 211     /* Prevent accessing the now saved entries */
 212     http->storeEntry(NULL);
 213     sc = NULL;
 214     reqsize = 0;
 215     reqofs = 0;
 216 }
 217
 218 void
 219 clientReplyContext::restoreState()
 220 {
 221     assert(old_sc != NULL);
 222     debugs(88, 3, "clientReplyContext::restoreState: Restoring store context");
 223     removeClientStoreReference(&sc, http);
 224     http->storeEntry(old_entry);
 225     sc = old_sc;
 226     reqsize = old_reqsize;
 227     reqofs = tempBuffer.offset;
 228     http->request->lastmod = old_lastmod;
 229     http->request->etag = old_etag;
 230     /* Prevent accessed the old saved entries */
 231     old_entry = NULL;
 232     old_sc = NULL;
 233     old_lastmod = -1;
 234     old_etag.clean();
 235     old_reqsize = 0;
 236     tempBuffer.offset = 0;
 237 }
 238
 239 void
 240 clientReplyContext::startError(ErrorState * err)
 241 {
 242     createStoreEntry(http->request->method, RequestFlags());
 243     triggerInitialStoreRead();
 244     errorAppendEntry(http->storeEntry(), err);
 245 }
 246
 247 clientStreamNode *
 248 clientReplyContext::getNextNode() const
 249 {
 250     return (clientStreamNode *)ourNode->node.next->data;
 251 }
 252
 253 /* This function is wrong - the client parameters don't include the
 254  * header offset
 255  */
 256 void
 257 clientReplyContext::triggerInitialStoreRead()
 258 {
 259     /* when confident, 0 becomes reqofs, and then this factors into
 260      * startSendProcess
 261      */
 262     assert(reqofs == 0);
 263     StoreIOBuffer localTempBuffer (next()->readBuffer.length, 0, next()->readBuffer.data);
 264     storeClientCopy(sc, http->storeEntry(), localTempBuffer, SendMoreData, this);
 265 }
 266
 267 /* there is an expired entry in the store.
 268  * setup a temporary buffer area and perform an IMS to the origin
 269  */
 270 void
 271 clientReplyContext::processExpired()
 272 {
 273     const char *url = storeId();
 274     debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'");
 275     const time_t lastmod = http->storeEntry()->lastModified();
 276     assert(lastmod >= 0);
 277     /*
 278      * check if we are allowed to contact other servers
 279      * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return
 280      *      a stale entry *if* it matches client requirements
 281      */
 282
 283     if (http->onlyIfCached()) {
 284         processOnlyIfCachedMiss();
 285         return;
 286     }
 287
 288     http->logType.update(LOG_TCP_REFRESH);
 289     http->request->flags.refresh = true;
 290 #if STORE_CLIENT_LIST_DEBUG
 291     /* Prevent a race with the store client memory free routines
 292      */
 293     assert(storeClientIsThisAClient(sc, this));
 294 #endif
 295     /* Prepare to make a new temporary request */
 296     saveState();
 297
 298     // TODO: Consider also allowing regular (non-collapsed) revalidation hits.
 299     // TODO: support collapsed revalidation for Vary-controlled entries
 300     bool collapsingAllowed = Config.onoff.collapsed_forwarding &&
 301                              !Store::Controller::SmpAware() &&
 302                              http->request->vary_headers.isEmpty();
 303
 304     StoreEntry *entry = nullptr;
 305     if (collapsingAllowed) {
 306         if (const auto e = storeGetPublicByRequest(http->request, ksRevalidation)) {
 307             if (e->hittingRequiresCollapsing() && startCollapsingOn(*e, true)) {
 308                 entry = e;
 309                 entry->lock("clientReplyContext::processExpired#alreadyRevalidating");
 310             } else {
 311                 e->abandon(__FUNCTION__);
 312                 // assume mayInitiateCollapsing() would fail too
 313                 collapsingAllowed = false;
 314             }
 315         }
 316     }
 317
 318     if (entry) {
 319         entry->ensureMemObject(url, http->log_uri, http->request->method);
 320         debugs(88, 5, "collapsed on existing revalidation entry: " << *entry);
 321         collapsedRevalidation = crSlave;
 322     } else {
 323         entry = storeCreateEntry(url,
 324                                  http->log_uri, http->request->flags, http->request->method);
 325         /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */
 326
 327         if (collapsingAllowed && mayInitiateCollapsing() &&
 328                 Store::Root().allowCollapsing(entry, http->request->flags, http->request->method)) {
 329             debugs(88, 5, "allow other revalidation requests to collapse on " << *entry);
 330             collapsedRevalidation = crInitiator;
 331         } else {
 332             collapsedRevalidation = crNone;
 333         }
 334     }
 335
 336     sc = storeClientListAdd(entry, this);
 337 #if USE_DELAY_POOLS
 338     /* delay_id is already set on original store client */
 339     sc->setDelayId(DelayId::DelayClient(http));
 340 #endif
 341
 342     http->request->lastmod = lastmod;
 343
 344     if (!http->request->header.has(Http::HdrType::IF_NONE_MATCH)) {
 345         ETag etag = {NULL, -1}; // TODO: make that a default ETag constructor
 346         if (old_entry->hasEtag(etag) && !etag.weak)
 347             http->request->etag = etag.str;
 348     }
 349
 350     debugs(88, 5, "lastmod " << entry->lastModified());
 351     http->storeEntry(entry);
 352     assert(http->out.offset == 0);
 353     assert(http->request->clientConnectionManager == http->getConn());
 354
 355     if (collapsedRevalidation != crSlave) {
 356         /*
 357          * A refcounted pointer so that FwdState stays around as long as
 358          * this clientReplyContext does
 359          */
 360         Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 361         FwdState::Start(conn, http->storeEntry(), http->request, http->al);
 362     }
 363     /* Register with storage manager to receive updates when data comes in. */
 364
 365     if (EBIT_TEST(entry->flags, ENTRY_ABORTED))
 366         debugs(88, DBG_CRITICAL, "clientReplyContext::processExpired: Found ENTRY_ABORTED object");
 367
 368     {
 369         /* start counting the length from 0 */
 370         StoreIOBuffer localTempBuffer(HTTP_REQBUF_SZ, 0, tempbuf);
 371         storeClientCopy(sc, entry, localTempBuffer, HandleIMSReply, this);
 372     }
 373 }
 374
 375 void
 376 clientReplyContext::sendClientUpstreamResponse()
 377 {
 378     StoreIOBuffer tempresult;
 379     removeStoreReference(&old_sc, &old_entry);
 380     /* here the data to send is the data we just received */
 381     tempBuffer.offset = 0;
 382     old_reqsize = 0;
 383     /* sendMoreData tracks the offset as well.
 384      * Force it back to zero */
 385     reqofs = 0;
 386     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 387     /* TODO: provide sendMoreData with the ready parsed reply */
 388     tempresult.length = reqsize;
 389     tempresult.data = tempbuf;
 390     sendMoreData(tempresult);
 391 }
 392
 393 void
 394 clientReplyContext::HandleIMSReply(void *data, StoreIOBuffer result)
 395 {
 396     clientReplyContext *context = (clientReplyContext *)data;
 397     context->handleIMSReply(result);
 398 }
 399
 400 void
 401 clientReplyContext::sendClientOldEntry()
 402 {
 403     /* Get the old request back */
 404     restoreState();
 405     /* here the data to send is in the next nodes buffers already */
 406     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 407     /* sendMoreData tracks the offset as well.
 408      * Force it back to zero */
 409     reqofs = 0;
 410     StoreIOBuffer tempresult (reqsize, reqofs, next()->readBuffer.data);
 411     sendMoreData(tempresult);
 412 }
 413
 414 /* This is the workhorse of the HandleIMSReply callback.
 415  *
 416  * It is called when we've got data back from the origin following our
 417  * IMS request to revalidate a stale entry.
 418  */
 419 void
 420 clientReplyContext::handleIMSReply(StoreIOBuffer result)
 421 {
 422     if (deleting)
 423         return;
 424
 425     debugs(88, 3, http->storeEntry()->url() << ", " << (long unsigned) result.length << " bytes");
 426
 427     if (http->storeEntry() == NULL)
 428         return;
 429
 430     if (result.flags.error && !EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED))
 431         return;
 432
 433     if (collapsedRevalidation == crSlave && !http->storeEntry()->mayStartHitting()) {
 434         debugs(88, 3, "CF slave hit private non-shareable " << *http->storeEntry() << ". MISS");
 435         // restore context to meet processMiss() expectations
 436         restoreState();
 437         http->logType.update(LOG_TCP_MISS);
 438         processMiss();
 439         return;
 440     }
 441
 442     /* update size of the request */
 443     reqsize = result.length + reqofs;
 444
 445     const Http::StatusCode status = http->storeEntry()->getReply()->sline.status();
 446
 447     // request to origin was aborted
 448     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
 449         debugs(88, 3, "request to origin aborted '" << http->storeEntry()->url() << "', sending old entry to client");
 450         http->logType.update(LOG_TCP_REFRESH_FAIL_OLD);
 451         sendClientOldEntry();
 452     }
 453
 454     const HttpReply *old_rep = old_entry->getReply();
 455
 456     // origin replied 304
 457     if (status == Http::scNotModified) {
 458         http->logType.update(LOG_TCP_REFRESH_UNMODIFIED);
 459         http->request->flags.staleIfHit = false; // old_entry is no longer stale
 460
 461         // update headers on existing entry
 462         Store::Root().updateOnNotModified(old_entry, *http->storeEntry());
 463
 464         // if client sent IMS
 465
 466         if (http->request->flags.ims && !old_entry->modifiedSince(http->request->ims, http->request->imslen)) {
 467             // forward the 304 from origin
 468             debugs(88, 3, "origin replied 304, revalidating existing entry and forwarding 304 to client");
 469             sendClientUpstreamResponse();
 470         } else {
 471             // send existing entry, it's still valid
 472             debugs(88, 3, "origin replied 304, revalidating existing entry and sending " <<
 473                    old_rep->sline.status() << " to client");
 474             sendClientOldEntry();
 475         }
 476     }
 477
 478     // origin replied with a non-error code
 479     else if (status > Http::scNone && status < Http::scInternalServerError) {
 480         const HttpReply *new_rep = http->storeEntry()->getReply();
 481         // RFC 7234 section 4: a cache MUST use the most recent response
 482         // (as determined by the Date header field)
 483         if (new_rep->olderThan(old_rep)) {
 484             http->logType.err.ignored = true;
 485             debugs(88, 3, "origin replied " << status <<
 486                    " but with an older date header, sending old entry (" <<
 487                    old_rep->sline.status() << ") to client");
 488             sendClientOldEntry();
 489         } else {
 490             http->logType.update(LOG_TCP_REFRESH_MODIFIED);
 491             debugs(88, 3, "origin replied " << status <<
 492                    ", replacing existing entry and forwarding to client");
 493
 494             if (collapsedRevalidation)
 495                 http->storeEntry()->clearPublicKeyScope();
 496
 497             sendClientUpstreamResponse();
 498         }
 499     }
 500
 501     // origin replied with an error
 502     else if (http->request->flags.failOnValidationError) {
 503         http->logType.update(LOG_TCP_REFRESH_FAIL_ERR);
 504         debugs(88, 3, "origin replied with error " << status <<
 505                ", forwarding to client due to fail_on_validation_err");
 506         sendClientUpstreamResponse();
 507     } else {
 508         // ignore and let client have old entry
 509         http->logType.update(LOG_TCP_REFRESH_FAIL_OLD);
 510         debugs(88, 3, "origin replied with error " <<
 511                status << ", sending old entry (" << old_rep->sline.status() << ") to client");
 512         sendClientOldEntry();
 513     }
 514 }
 515
 516 SQUIDCEXTERN CSR clientGetMoreData;
 517 SQUIDCEXTERN CSD clientReplyDetach;
 518
 519 /**
 520  * clientReplyContext::cacheHit Should only be called until the HTTP reply headers
 521  * have been parsed.  Normally this should be a single call, but
 522  * it might take more than one.  As soon as we have the headers,
 523  * we hand off to clientSendMoreData, processExpired, or
 524  * processMiss.
 525  */
 526 void
 527 clientReplyContext::CacheHit(void *data, StoreIOBuffer result)
 528 {
 529     clientReplyContext *context = (clientReplyContext *)data;
 530     context->cacheHit(result);
 531 }
 532
 533 /**
 534  * Process a possible cache HIT.
 535  */
 536 void
 537 clientReplyContext::cacheHit(StoreIOBuffer result)
 538 {
 539     /** Ignore if the HIT object is being deleted. */
 540     if (deleting) {
 541         debugs(88, 3, "HIT object being deleted. Ignore the HIT.");
 542         return;
 543     }
 544
 545     StoreEntry *e = http->storeEntry();
 546
 547     HttpRequest *r = http->request;
 548
 549     debugs(88, 3, "clientCacheHit: " << http->uri << ", " << result.length << " bytes");
 550
 551     if (http->storeEntry() == NULL) {
 552         debugs(88, 3, "clientCacheHit: request aborted");
 553         return;
 554     } else if (result.flags.error) {
 555         /* swap in failure */
 556         debugs(88, 3, "clientCacheHit: swapin failure for " << http->uri);
 557         http->logType.update(LOG_TCP_SWAPFAIL_MISS);
 558         removeClientStoreReference(&sc, http);
 559         processMiss();
 560         return;
 561     }
 562
 563     // The previously identified hit suddenly became unsharable!
 564     // This is common for collapsed forwarding slaves but might also
 565     // happen to regular hits because we are called asynchronously.
 566     if (!e->mayStartHitting()) {
 567         debugs(88, 3, "unsharable " << *e << ". MISS");
 568         http->logType.update(LOG_TCP_MISS);
 569         processMiss();
 570         return;
 571     }
 572
 573     if (result.length == 0) {
 574         debugs(88, 5, "store IO buffer has no content. MISS");
 575         /* the store couldn't get enough data from the file for us to id the
 576          * object
 577          */
 578         /* treat as a miss */
 579         http->logType.update(LOG_TCP_MISS);
 580         processMiss();
 581         return;
 582     }
 583
 584     assert(!EBIT_TEST(e->flags, ENTRY_ABORTED));
 585     /* update size of the request */
 586     reqsize = result.length + reqofs;
 587
 588     /*
 589      * Got the headers, now grok them
 590      */
 591     assert(http->logType.oldType == LOG_TCP_HIT);
 592
 593     if (http->request->storeId().cmp(e->mem_obj->storeId()) != 0) {
 594         debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->storeId() << "' != '" << http->request->storeId() << "'");
 595         http->logType.update(LOG_TCP_MISS); // we lack a more precise LOG_*_MISS code
 596         processMiss();
 597         return;
 598     }
 599
 600     switch (varyEvaluateMatch(e, r)) {
 601
 602     case VARY_NONE:
 603         /* No variance detected. Continue as normal */
 604         break;
 605
 606     case VARY_MATCH:
 607         /* This is the correct entity for this request. Continue */
 608         debugs(88, 2, "clientProcessHit: Vary MATCH!");
 609         break;
 610
 611     case VARY_OTHER:
 612         /* This is not the correct entity for this request. We need
 613          * to requery the cache.
 614          */
 615         removeClientStoreReference(&sc, http);
 616         e = NULL;
 617         /* Note: varyEvalyateMatch updates the request with vary information
 618          * so we only get here once. (it also takes care of cancelling loops)
 619          */
 620         debugs(88, 2, "clientProcessHit: Vary detected!");
 621         clientGetMoreData(ourNode, http);
 622         return;
 623
 624     case VARY_CANCEL:
 625         /* varyEvaluateMatch found a object loop. Process as miss */
 626         debugs(88, DBG_IMPORTANT, "clientProcessHit: Vary object loop!");
 627         http->logType.update(LOG_TCP_MISS); // we lack a more precise LOG_*_MISS code
 628         processMiss();
 629         return;
 630     }
 631
 632     if (r->method == Http::METHOD_PURGE) {
 633         debugs(88, 5, "PURGE gets a HIT");
 634         removeClientStoreReference(&sc, http);
 635         e = NULL;
 636         purgeRequest();
 637         return;
 638     }
 639
 640     if (e->checkNegativeHit() && !r->flags.noCacheHack()) {
 641         debugs(88, 5, "negative-HIT");
 642         http->logType.update(LOG_TCP_NEGATIVE_HIT);
 643         sendMoreData(result);
 644         return;
 645     } else if (blockedHit()) {
 646         debugs(88, 5, "send_hit forces a MISS");
 647         http->logType.update(LOG_TCP_MISS);
 648         processMiss();
 649         return;
 650     } else if (!http->flags.internal && refreshCheckHTTP(e, r)) {
 651         debugs(88, 5, "clientCacheHit: in refreshCheck() block");
 652         /*
 653          * We hold a stale copy; it needs to be validated
 654          */
 655         /*
 656          * The 'needValidation' flag is used to prevent forwarding
 657          * loops between siblings.  If our copy of the object is stale,
 658          * then we should probably only use parents for the validation
 659          * request.  Otherwise two siblings could generate a loop if
 660          * both have a stale version of the object.
 661          */
 662         r->flags.needValidation = true;
 663
 664         if (e->lastModified() < 0) {
 665             debugs(88, 3, "validate HIT object? NO. Can't calculate entry modification time. Do MISS.");
 666             /*
 667              * We cannot revalidate entries without knowing their
 668              * modification time.
 669              * XXX: BUG 1890 objects without Date do not get one added.
 670              */
 671             http->logType.update(LOG_TCP_MISS);
 672             processMiss();
 673         } else if (r->flags.noCache) {
 674             debugs(88, 3, "validate HIT object? NO. Client sent CC:no-cache. Do CLIENT_REFRESH_MISS");
 675             /*
 676              * This did not match a refresh pattern that overrides no-cache
 677              * we should honour the client no-cache header.
 678              */
 679             http->logType.update(LOG_TCP_CLIENT_REFRESH_MISS);
 680             processMiss();
 681         } else if (r->url.getScheme() == AnyP::PROTO_HTTP || r->url.getScheme() == AnyP::PROTO_HTTPS) {
 682             debugs(88, 3, "validate HIT object? YES.");
 683             /*
 684              * Object needs to be revalidated
 685              * XXX This could apply to FTP as well, if Last-Modified is known.
 686              */
 687             processExpired();
 688         } else {
 689             debugs(88, 3, "validate HIT object? NO. Client protocol non-HTTP. Do MISS.");
 690             /*
 691              * We don't know how to re-validate other protocols. Handle
 692              * them as if the object has expired.
 693              */
 694             http->logType.update(LOG_TCP_MISS);
 695             processMiss();
 696         }
 697         return;
 698     } else if (r->conditional()) {
 699         debugs(88, 5, "conditional HIT");
 700         if (processConditional(result))
 701             return;
 702     }
 703
 704     /*
 705      * plain ol' cache hit
 706      */
 707     debugs(88, 5, "plain old HIT");
 708
 709 #if USE_DELAY_POOLS
 710     if (e->store_status != STORE_OK)
 711         http->logType.update(LOG_TCP_MISS);
 712     else
 713 #endif
 714         if (e->mem_status == IN_MEMORY)
 715             http->logType.update(LOG_TCP_MEM_HIT);
 716         else if (Config.onoff.offline)
 717             http->logType.update(LOG_TCP_OFFLINE_HIT);
 718
 719     sendMoreData(result);
 720 }
 721
 722 /**
 723  * Prepare to fetch the object as it's a cache miss of some kind.
 724  */
 725 void
 726 clientReplyContext::processMiss()
 727 {
 728     char *url = http->uri;
 729     HttpRequest *r = http->request;
 730     ErrorState *err = NULL;
 731     debugs(88, 4, r->method << ' ' << url);
 732
 733     /**
 734      * We might have a left-over StoreEntry from a failed cache hit
 735      * or IMS request.
 736      */
 737     if (http->storeEntry()) {
 738         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
 739             debugs(88, DBG_CRITICAL, "clientProcessMiss: miss on a special object (" << url << ").");
 740             debugs(88, DBG_CRITICAL, "\tlog_type = " << http->logType.c_str());
 741             http->storeEntry()->dump(1);
 742         }
 743
 744         removeClientStoreReference(&sc, http);
 745     }
 746
 747     /** Check if its a PURGE request to be actioned. */
 748     if (r->method == Http::METHOD_PURGE) {
 749         purgeRequest();
 750         return;
 751     }
 752
 753     /** Check if its an 'OTHER' request. Purge all cached entries if so and continue. */
 754     if (r->method == Http::METHOD_OTHER) {
 755         purgeAllCached();
 756     }
 757
 758     /** Check if 'only-if-cached' flag is set. Action if so. */
 759     if (http->onlyIfCached()) {
 760         processOnlyIfCachedMiss();
 761         return;
 762     }
 763
 764     Comm::ConnectionPointer conn = http->getConn() != nullptr ? http->getConn()->clientConnection : nullptr;
 765     /// Deny loops
 766     if (r->flags.loopDetected) {
 767         http->al->http.code = Http::scForbidden;
 768         Ip::Address tmp_noaddr;
 769         tmp_noaddr.setNoAddr();
 770         err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, nullptr, conn ? conn->remote : tmp_noaddr, http->request, http->al);
 771         createStoreEntry(r->method, RequestFlags());
 772         errorAppendEntry(http->storeEntry(), err);
 773         triggerInitialStoreRead();
 774         return;
 775     } else {
 776         assert(http->out.offset == 0);
 777         createStoreEntry(r->method, r->flags);
 778         triggerInitialStoreRead();
 779
 780         if (http->redirect.status) {
 781             HttpReply *rep = new HttpReply;
 782             http->logType.update(LOG_TCP_REDIRECT);
 783             http->storeEntry()->releaseRequest();
 784             rep->redirect(http->redirect.status, http->redirect.location);
 785             http->storeEntry()->replaceHttpReply(rep);
 786             http->storeEntry()->complete();
 787             return;
 788         }
 789
 790         assert(r->clientConnectionManager == http->getConn());
 791
 792         /** Start forwarding to get the new object from network */
 793         FwdState::Start(conn, http->storeEntry(), r, http->al);
 794     }
 795 }
 796
 797 /**
 798  * client issued a request with an only-if-cached cache-control directive;
 799  * we did not find a cached object that can be returned without
 800  *     contacting other servers;
 801  * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068]
 802  */
 803 void
 804 clientReplyContext::processOnlyIfCachedMiss()
 805 {
 806     debugs(88, 4, http->request->method << ' ' << http->uri);
 807     http->al->http.code = Http::scGatewayTimeout;
 808     Ip::Address tmp_noaddr;
 809     tmp_noaddr.setNoAddr();
 810     ErrorState *err = clientBuildError(ERR_ONLY_IF_CACHED_MISS, Http::scGatewayTimeout, NULL,
 811                                        http->getConn() ? http->getConn()->clientConnection->remote : tmp_noaddr,
 812                                        http->request, http->al);
 813     removeClientStoreReference(&sc, http);
 814     startError(err);
 815 }
 816
 817 /// process conditional request from client
 818 bool
 819 clientReplyContext::processConditional(StoreIOBuffer &result)
 820 {
 821     StoreEntry *const e = http->storeEntry();
 822
 823     if (e->getReply()->sline.status() != Http::scOkay) {
 824         debugs(88, 4, "Reply code " << e->getReply()->sline.status() << " != 200");
 825         http->logType.update(LOG_TCP_MISS);
 826         processMiss();
 827         return true;
 828     }
 829
 830     HttpRequest &r = *http->request;
 831
 832     if (r.header.has(Http::HdrType::IF_MATCH) && !e->hasIfMatchEtag(r)) {
 833         // RFC 2616: reply with 412 Precondition Failed if If-Match did not match
 834         sendPreconditionFailedError();
 835         return true;
 836     }
 837
 838     if (r.header.has(Http::HdrType::IF_NONE_MATCH)) {
 839         // RFC 7232: If-None-Match recipient MUST ignore IMS
 840         r.flags.ims = false;
 841         r.ims = -1;
 842         r.imslen = 0;
 843         r.header.delById(Http::HdrType::IF_MODIFIED_SINCE);
 844
 845         if (e->hasIfNoneMatchEtag(r)) {
 846             sendNotModifiedOrPreconditionFailedError();
 847             return true;
 848         }
 849
 850         // None-Match is true (no ETag matched); treat as an unconditional hit
 851         return false;
 852     }
 853
 854     if (r.flags.ims) {
 855         // handle If-Modified-Since requests from the client
 856         if (e->modifiedSince(r.ims, r.imslen)) {
 857             // Modified-Since is true; treat as an unconditional hit
 858             return false;
 859
 860         } else {
 861             // otherwise reply with 304 Not Modified
 862             sendNotModified();
 863         }
 864         return true;
 865     }
 866
 867     return false;
 868 }
 869
 870 /// whether squid.conf send_hit prevents us from serving this hit
 871 bool
 872 clientReplyContext::blockedHit() const
 873 {
 874     if (!Config.accessList.sendHit)
 875         return false; // hits are not blocked by default
 876
 877     if (http->flags.internal)
 878         return false; // internal content "hits" cannot be blocked
 879
 880     if (const HttpReply *rep = http->storeEntry()->getReply()) {
 881         std::unique_ptr<ACLFilledChecklist> chl(clientAclChecklistCreate(Config.accessList.sendHit, http));
 882         chl->reply = const_cast<HttpReply*>(rep); // ACLChecklist API bug
 883         HTTPMSGLOCK(chl->reply);
 884         return !chl->fastCheck().allowed(); // when in doubt, block
 885     }
 886
 887     // This does not happen, I hope, because we are called from CacheHit, which
 888     // is called via a storeClientCopy() callback, and store should initialize
 889     // the reply before calling that callback.
 890     debugs(88, 3, "Missing reply!");
 891     return false;
 892 }
 893
 894 void
 895 clientReplyContext::purgeRequestFindObjectToPurge()
 896 {
 897     /* Try to find a base entry */
 898     http->flags.purging = true;
 899     lookingforstore = 1;
 900
 901     // TODO: can we use purgeAllCached() here instead of doing the
 902     // getPublicByRequestMethod() dance?
 903     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_GET);
 904 }
 905
 906 // Purges all entries with a given url
 907 // TODO: move to SideAgent parent, when we have one
 908 /*
 909  * We probably cannot purge Vary-affected responses because their MD5
 910  * keys depend on vary headers.
 911  */
 912 void
 913 purgeEntriesByUrl(HttpRequest * req, const char *url)
 914 {
 915     for (HttpRequestMethod m(Http::METHOD_NONE); m != Http::METHOD_ENUM_END; ++m) {
 916         if (m.respMaybeCacheable()) {
 917             const cache_key *key = storeKeyPublic(url, m);
 918             debugs(88, 5, m << ' ' << url << ' ' << storeKeyText(key));
 919 #if USE_HTCP
 920             neighborsHtcpClear(nullptr, url, req, m, HTCP_CLR_INVALIDATION);
 921 #endif
 922             Store::Root().evictIfFound(key);
 923         }
 924     }
 925 }
 926
 927 void
 928 clientReplyContext::purgeAllCached()
 929 {
 930     // XXX: performance regression, c_str() reallocates
 931     SBuf url(http->request->effectiveRequestUri());
 932     purgeEntriesByUrl(http->request, url.c_str());
 933 }
 934
 935 void
 936 clientReplyContext::created(StoreEntry *newEntry)
 937 {
 938     if (lookingforstore == 1)
 939         purgeFoundGet(newEntry);
 940     else if (lookingforstore == 2)
 941         purgeFoundHead(newEntry);
 942     else if (lookingforstore == 3)
 943         purgeDoPurgeGet(newEntry);
 944     else if (lookingforstore == 4)
 945         purgeDoPurgeHead(newEntry);
 946     else if (lookingforstore == 5)
 947         identifyFoundObject(newEntry);
 948 }
 949
 950 LogTags *
 951 clientReplyContext::loggingTags()
 952 {
 953     // XXX: clientReplyContext code assumes that http cbdata is always valid.
 954     // TODO: Either add cbdataReferenceValid(http) checks in all the relevant
 955     // places, like this one, or remove cbdata protection of the http member.
 956     return &http->logType;
 957 }
 958
 959 void
 960 clientReplyContext::purgeFoundGet(StoreEntry *newEntry)
 961 {
 962     if (!newEntry) {
 963         lookingforstore = 2;
 964         StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
 965     } else
 966         purgeFoundObject (newEntry);
 967 }
 968
 969 void
 970 clientReplyContext::purgeFoundHead(StoreEntry *newEntry)
 971 {
 972     if (!newEntry)
 973         purgeDoMissPurge();
 974     else
 975         purgeFoundObject (newEntry);
 976 }
 977
 978 void
 979 clientReplyContext::purgeFoundObject(StoreEntry *entry)
 980 {
 981     assert (entry);
 982
 983     if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) {
 984         http->logType.update(LOG_TCP_DENIED);
 985         Ip::Address tmp_noaddr;
 986         tmp_noaddr.setNoAddr(); // TODO: make a global const
 987         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL,
 988                                            http->getConn() ? http->getConn()->clientConnection->remote : tmp_noaddr,
 989                                            http->request, http->al);
 990         startError(err);
 991         return; // XXX: leaking unused entry if some store does not keep it
 992     }
 993
 994     StoreIOBuffer localTempBuffer;
 995     /* Swap in the metadata */
 996     http->storeEntry(entry);
 997
 998     http->storeEntry()->lock("clientReplyContext::purgeFoundObject");
 999     http->storeEntry()->ensureMemObject(storeId(), http->log_uri,
1000                                         http->request->method);
1001
1002     sc = storeClientListAdd(http->storeEntry(), this);
1003
1004     http->logType.update(LOG_TCP_HIT);
1005
1006     reqofs = 0;
1007
1008     localTempBuffer.offset = http->out.offset;
1009
1010     localTempBuffer.length = next()->readBuffer.length;
1011
1012     localTempBuffer.data = next()->readBuffer.data;
1013
1014     storeClientCopy(sc, http->storeEntry(),
1015                     localTempBuffer, CacheHit, this);
1016 }
1017
1018 void
1019 clientReplyContext::purgeRequest()
1020 {
1021     debugs(88, 3, "Config2.onoff.enable_purge = " <<
1022            Config2.onoff.enable_purge);
1023
1024     if (!Config2.onoff.enable_purge) {
1025         http->logType.update(LOG_TCP_DENIED);
1026         Ip::Address tmp_noaddr;
1027         tmp_noaddr.setNoAddr();
1028         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL,
1029                                            http->getConn() ? http->getConn()->clientConnection->remote : tmp_noaddr, http->request, http->al);
1030         startError(err);
1031         return;
1032     }
1033
1034     /* Release both IP cache */
1035     ipcacheInvalidate(http->request->url.host());
1036
1037     if (!http->flags.purging)
1038         purgeRequestFindObjectToPurge();
1039     else
1040         purgeDoMissPurge();
1041 }
1042
1043 void
1044 clientReplyContext::purgeDoMissPurge()
1045 {
1046     http->logType.update(LOG_TCP_MISS);
1047     lookingforstore = 3;
1048     StoreEntry::getPublicByRequestMethod(this,http->request, Http::METHOD_GET);
1049 }
1050
1051 void
1052 clientReplyContext::purgeDoPurgeGet(StoreEntry *newEntry)
1053 {
1054     if (newEntry) {
1055         /* Move to new() when that is created */
1056         purgeStatus = Http::scNotFound;
1057         /* Release the cached URI */
1058         debugs(88, 4, "clientPurgeRequest: GET '" << newEntry->url() << "'" );
1059 #if USE_HTCP
1060         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1061 #endif
1062         newEntry->release(true);
1063         purgeStatus = Http::scOkay;
1064     }
1065
1066     lookingforstore = 4;
1067     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
1068 }
1069
1070 void
1071 clientReplyContext::purgeDoPurgeHead(StoreEntry *newEntry)
1072 {
1073     if (newEntry) {
1074         debugs(88, 4, "HEAD " << newEntry->url());
1075 #if USE_HTCP
1076         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1077 #endif
1078         newEntry->release(true);
1079         purgeStatus = Http::scOkay;
1080     }
1081
1082     /* And for Vary, release the base URI if none of the headers was included in the request */
1083     if (!http->request->vary_headers.isEmpty()
1084             && http->request->vary_headers.find('=') != SBuf::npos) {
1085         // XXX: performance regression, c_str() reallocates
1086         SBuf tmp(http->request->effectiveRequestUri());
1087         StoreEntry *entry = storeGetPublic(tmp.c_str(), Http::METHOD_GET);
1088
1089         if (entry) {
1090             debugs(88, 4, "Vary GET " << entry->url());
1091 #if USE_HTCP
1092             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1093 #endif
1094             entry->release(true);
1095             purgeStatus = Http::scOkay;
1096         }
1097
1098         entry = storeGetPublic(tmp.c_str(), Http::METHOD_HEAD);
1099
1100         if (entry) {
1101             debugs(88, 4, "Vary HEAD " << entry->url());
1102 #if USE_HTCP
1103             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1104 #endif
1105             entry->release(true);
1106             purgeStatus = Http::scOkay;
1107         }
1108     }
1109
1110     /*
1111      * Make a new entry to hold the reply to be written
1112      * to the client.
1113      */
1114     /* FIXME: This doesn't need to go through the store. Simply
1115      * push down the client chain
1116      */
1117     createStoreEntry(http->request->method, RequestFlags());
1118
1119     triggerInitialStoreRead();
1120
1121     HttpReply *rep = new HttpReply;
1122     rep->setHeaders(purgeStatus, NULL, NULL, 0, 0, -1);
1123     http->storeEntry()->replaceHttpReply(rep);
1124     http->storeEntry()->complete();
1125 }
1126
1127 void
1128 clientReplyContext::traceReply(clientStreamNode * node)
1129 {
1130     clientStreamNode *nextNode = (clientStreamNode *)node->node.next->data;
1131     StoreIOBuffer localTempBuffer;
1132     createStoreEntry(http->request->method, RequestFlags());
1133     localTempBuffer.offset = nextNode->readBuffer.offset + headers_sz;
1134     localTempBuffer.length = nextNode->readBuffer.length;
1135     localTempBuffer.data = nextNode->readBuffer.data;
1136     storeClientCopy(sc, http->storeEntry(),
1137                     localTempBuffer, SendMoreData, this);
1138     http->storeEntry()->releaseRequest();
1139     http->storeEntry()->buffer();
1140     HttpReply *rep = new HttpReply;
1141     rep->setHeaders(Http::scOkay, NULL, "text/plain", http->request->prefixLen(), 0, squid_curtime);
1142     http->storeEntry()->replaceHttpReply(rep);
1143     http->request->swapOut(http->storeEntry());
1144     http->storeEntry()->complete();
1145 }
1146
1147 #define SENDING_BODY 0
1148 #define SENDING_HDRSONLY 1
1149 int
1150 clientReplyContext::checkTransferDone()
1151 {
1152     StoreEntry *entry = http->storeEntry();
1153
1154     if (entry == NULL)
1155         return 0;
1156
1157     /*
1158      * For now, 'done_copying' is used for special cases like
1159      * Range and HEAD requests.
1160      */
1161     if (http->flags.done_copying)
1162         return 1;
1163
1164     if (http->request->flags.chunkedReply && !flags.complete) {
1165         // last-chunk was not sent
1166         return 0;
1167     }
1168
1169     /*
1170      * Handle STORE_OK objects.
1171      * objectLen(entry) will be set proprely.
1172      * RC: Does objectLen(entry) include the Headers?
1173      * RC: Yes.
1174      */
1175     if (entry->store_status == STORE_OK) {
1176         return storeOKTransferDone();
1177     } else {
1178         return storeNotOKTransferDone();
1179     }
1180 }
1181
1182 int
1183 clientReplyContext::storeOKTransferDone() const
1184 {
1185     assert(http->storeEntry()->objectLen() >= 0);
1186     assert(http->storeEntry()->objectLen() >= headers_sz);
1187     if (http->out.offset >= http->storeEntry()->objectLen() - headers_sz) {
1188         debugs(88,3,HERE << "storeOKTransferDone " <<
1189                " out.offset=" << http->out.offset <<
1190                " objectLen()=" << http->storeEntry()->objectLen() <<
1191                " headers_sz=" << headers_sz);
1192         return 1;
1193     }
1194
1195     return 0;
1196 }
1197
1198 int
1199 clientReplyContext::storeNotOKTransferDone() const
1200 {
1201     /*
1202      * Now, handle STORE_PENDING objects
1203      */
1204     MemObject *mem = http->storeEntry()->mem_obj;
1205     assert(mem != NULL);
1206     assert(http->request != NULL);
1207
1208     /* mem->reply was wrong because it uses the UPSTREAM header length!!! */
1209     if (headers_sz == 0)
1210         /* haven't found end of headers yet */
1211         return 0;
1212
1213     const HttpReplyPointer curReply(mem->getReply());
1214
1215     /*
1216      * Figure out how much data we are supposed to send.
1217      * If we are sending a body and we don't have a content-length,
1218      * then we must wait for the object to become STORE_OK.
1219      */
1220     if (curReply->content_length < 0)
1221         return 0;
1222
1223     uint64_t expectedLength = curReply->content_length + http->out.headers_sz;
1224
1225     if (http->out.size < expectedLength)
1226         return 0;
1227     else {
1228         debugs(88,3,HERE << "storeNotOKTransferDone " <<
1229                " out.size=" << http->out.size <<
1230                " expectedLength=" << expectedLength);
1231         return 1;
1232     }
1233 }
1234
1235 /* A write has completed, what is the next status based on the
1236  * canonical request data?
1237  * 1 something is wrong
1238  * 0 nothing is wrong.
1239  *
1240  */
1241 int
1242 clientHttpRequestStatus(int fd, ClientHttpRequest const *http)
1243 {
1244 #if SIZEOF_INT64_T == 4
1245     if (http->out.size > 0x7FFF0000) {
1246         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd << " to prevent out.size counter overflow");
1247         if (http->getConn())
1248             debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1249         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes");
1250         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1251         return 1;
1252     }
1253
1254     if (http->out.offset > 0x7FFF0000) {
1255         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd < " to prevent out.offset counter overflow");
1256         if (http->getConn())
1257             debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1258         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes, offset " << http->out.offset);
1259         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1260         return 1;
1261     }
1262
1263 #endif
1264     return 0;
1265 }
1266
1267 /* Preconditions:
1268  * *http is a valid structure.
1269  * fd is either -1, or an open fd.
1270  *
1271  * TODO: enumify this
1272  *
1273  * This function is used by any http request sink, to determine the status
1274  * of the object.
1275  */
1276 clientStream_status_t
1277 clientReplyStatus(clientStreamNode * aNode, ClientHttpRequest * http)
1278 {
1279     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1280     assert (context);
1281     assert (context->http == http);
1282     return context->replyStatus();
1283 }
1284
1285 clientStream_status_t
1286 clientReplyContext::replyStatus()
1287 {
1288     int done;
1289     /* Here because lower nodes don't need it */
1290
1291     if (http->storeEntry() == NULL) {
1292         debugs(88, 5, "clientReplyStatus: no storeEntry");
1293         return STREAM_FAILED;   /* yuck, but what can we do? */
1294     }
1295
1296     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
1297         /* TODO: Could upstream read errors (result.flags.error) be
1298          * lost, and result in undersize requests being considered
1299          * complete. Should we tcp reset such connections ?
1300          */
1301         debugs(88, 5, "clientReplyStatus: aborted storeEntry");
1302         return STREAM_FAILED;
1303     }
1304
1305     if ((done = checkTransferDone()) != 0 || flags.complete) {
1306         debugs(88, 5, "clientReplyStatus: transfer is DONE: " << done << flags.complete);
1307         /* Ok we're finished, but how? */
1308
1309         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_BAD_LENGTH)) {
1310             debugs(88, 5, "clientReplyStatus: truncated response body");
1311             return STREAM_UNPLANNED_COMPLETE;
1312         }
1313
1314         if (!done) {
1315             debugs(88, 5, "clientReplyStatus: closing, !done, but read 0 bytes");
1316             return STREAM_FAILED;
1317         }
1318
1319         const int64_t expectedBodySize =
1320             http->storeEntry()->getReply()->bodySize(http->request->method);
1321         if (expectedBodySize >= 0 && !http->gotEnough()) {
1322             debugs(88, 5, "clientReplyStatus: client didn't get all it expected");
1323             return STREAM_UNPLANNED_COMPLETE;
1324         }
1325
1326         debugs(88, 5, "clientReplyStatus: stream complete; keepalive=" <<
1327                http->request->flags.proxyKeepalive);
1328         return STREAM_COMPLETE;
1329     }
1330
1331     // XXX: Should this be checked earlier? We could return above w/o checking.
1332     if (reply->receivedBodyTooLarge(*http->request, http->out.offset - 4096)) {
1333         /* 4096 is a margin for the HTTP headers included in out.offset */
1334         debugs(88, 5, "clientReplyStatus: client reply body is too large");
1335         return STREAM_FAILED;
1336     }
1337
1338     return STREAM_NONE;
1339 }
1340
1341 /* Responses with no body will not have a content-type header,
1342  * which breaks the rep_mime_type acl, which
1343  * coincidentally, is the most common acl for reply access lists.
1344  * A better long term fix for this is to allow acl matchs on the various
1345  * status codes, and then supply a default ruleset that puts these
1346  * codes before any user defines access entries. That way the user
1347  * can choose to block these responses where appropriate, but won't get
1348  * mysterious breakages.
1349  */
1350 bool
1351 clientReplyContext::alwaysAllowResponse(Http::StatusCode sline) const
1352 {
1353     bool result;
1354
1355     switch (sline) {
1356
1357     case Http::scContinue:
1358
1359     case Http::scSwitchingProtocols:
1360
1361     case Http::scProcessing:
1362
1363     case Http::scNoContent:
1364
1365     case Http::scNotModified:
1366         result = true;
1367         break;
1368
1369     default:
1370         result = false;
1371     }
1372
1373     return result;
1374 }
1375
1376 /**
1377  * Generate the reply headers sent to client.
1378  *
1379  * Filters out unwanted entries and hop-by-hop from original reply header
1380  * then adds extra entries if we have more info than origin server
1381  * then adds Squid specific entries
1382  */
1383 void
1384 clientReplyContext::buildReplyHeader()
1385 {
1386     HttpHeader *hdr = &reply->header;
1387     const bool is_hit = http->logType.isTcpHit();
1388     HttpRequest *request = http->request;
1389 #if DONT_FILTER_THESE
1390     /* but you might want to if you run Squid as an HTTP accelerator */
1391     /* hdr->delById(HDR_ACCEPT_RANGES); */
1392     hdr->delById(HDR_ETAG);
1393 #endif
1394
1395     if (is_hit || collapsedRevalidation == crSlave)
1396         hdr->delById(Http::HdrType::SET_COOKIE);
1397     // TODO: RFC 2965 : Must honour Cache-Control: no-cache="set-cookie2" and remove header.
1398
1399     // if there is not configured a peer proxy with login=PASS or login=PASSTHRU option enabled
1400     // remove the Proxy-Authenticate header
1401     if ( !request->peer_login || (strcmp(request->peer_login,"PASS") != 0 && strcmp(request->peer_login,"PASSTHRU") != 0)) {
1402 #if USE_ADAPTATION
1403         // but allow adaptation services to authenticate clients
1404         // via request satisfaction
1405         if (!http->requestSatisfactionMode())
1406 #endif
1407             reply->header.delById(Http::HdrType::PROXY_AUTHENTICATE);
1408     }
1409
1410     reply->header.removeHopByHopEntries();
1411     // paranoid: ContentLengthInterpreter has cleaned non-generated replies
1412     reply->removeIrrelevantContentLength();
1413
1414     //    if (request->range)
1415     //      clientBuildRangeHeader(http, reply);
1416
1417     /*
1418      * Add a estimated Age header on cache hits.
1419      */
1420     if (is_hit) {
1421         /*
1422          * Remove any existing Age header sent by upstream caches
1423          * (note that the existing header is passed along unmodified
1424          * on cache misses)
1425          */
1426         hdr->delById(Http::HdrType::AGE);
1427         /*
1428          * This adds the calculated object age. Note that the details of the
1429          * age calculation is performed by adjusting the timestamp in
1430          * StoreEntry::timestampsSet(), not here.
1431          */
1432         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
1433             hdr->delById(Http::HdrType::DATE);
1434             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1435         } else if (http->getConn() && http->getConn()->port->actAsOrigin) {
1436             // Swap the Date: header to current time if we are simulating an origin
1437             HttpHeaderEntry *h = hdr->findEntry(Http::HdrType::DATE);
1438             if (h)
1439                 hdr->putExt("X-Origin-Date", h->value.termedBuf());
1440             hdr->delById(Http::HdrType::DATE);
1441             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1442             h = hdr->findEntry(Http::HdrType::EXPIRES);
1443             if (h && http->storeEntry()->expires >= 0) {
1444                 hdr->putExt("X-Origin-Expires", h->value.termedBuf());
1445                 hdr->delById(Http::HdrType::EXPIRES);
1446                 hdr->putTime(Http::HdrType::EXPIRES, squid_curtime + http->storeEntry()->expires - http->storeEntry()->timestamp);
1447             }
1448             if (http->storeEntry()->timestamp <= squid_curtime) {
1449                 // put X-Cache-Age: instead of Age:
1450                 char age[64];
1451                 snprintf(age, sizeof(age), "%" PRId64, static_cast<int64_t>(squid_curtime - http->storeEntry()->timestamp));
1452                 hdr->putExt("X-Cache-Age", age);
1453             }
1454         } else if (http->storeEntry()->timestamp <= squid_curtime) {
1455             hdr->putInt(Http::HdrType::AGE,
1456                         squid_curtime - http->storeEntry()->timestamp);
1457             /* Signal old objects.  NB: rfc 2616 is not clear,
1458              * by implication, on whether we should do this to all
1459              * responses, or only cache hits.
1460              * 14.46 states it ONLY applys for heuristically caclulated
1461              * freshness values, 13.2.4 doesn't specify the same limitation.
1462              * We interpret RFC 2616 under the combination.
1463              */
1464             /* TODO: if maxage or s-maxage is present, don't do this */
1465
1466             if (squid_curtime - http->storeEntry()->timestamp >= 86400)
1467                 hdr->putWarning(113, "This cache hit is still fresh and more than 1 day old");
1468         }
1469     }
1470
1471     /* RFC 2616: Section 14.18
1472      *
1473      * Add a Date: header if missing.
1474      * We have access to a clock therefore are required to amend any shortcoming in servers.
1475      *
1476      * NP: done after Age: to prevent ENTRY_SPECIAL double-handling this header.
1477      */
1478     if ( !hdr->has(Http::HdrType::DATE) ) {
1479         if (!http->storeEntry())
1480             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1481         else if (http->storeEntry()->timestamp > 0)
1482             hdr->putTime(Http::HdrType::DATE, http->storeEntry()->timestamp);
1483         else {
1484             debugs(88,DBG_IMPORTANT,"BUG 3279: HTTP reply without Date:");
1485             /* dump something useful about the problem */
1486             http->storeEntry()->dump(DBG_IMPORTANT);
1487         }
1488     }
1489
1490     // add Warnings required by RFC 2616 if serving a stale hit
1491     if (http->request->flags.staleIfHit && http->logType.isTcpHit()) {
1492         hdr->putWarning(110, "Response is stale");
1493         if (http->request->flags.needValidation)
1494             hdr->putWarning(111, "Revalidation failed");
1495     }
1496
1497     /* Filter unproxyable authentication types */
1498     if (http->logType.oldType != LOG_TCP_DENIED &&
1499             hdr->has(Http::HdrType::WWW_AUTHENTICATE)) {
1500         HttpHeaderPos pos = HttpHeaderInitPos;
1501         HttpHeaderEntry *e;
1502
1503         int connection_auth_blocked = 0;
1504         while ((e = hdr->getEntry(&pos))) {
1505             if (e->id == Http::HdrType::WWW_AUTHENTICATE) {
1506                 const char *value = e->value.rawBuf();
1507
1508                 if ((strncasecmp(value, "NTLM", 4) == 0 &&
1509                         (value[4] == '\0' || value[4] == ' '))
1510                         ||
1511                         (strncasecmp(value, "Negotiate", 9) == 0 &&
1512                          (value[9] == '\0' || value[9] == ' '))
1513                         ||
1514                         (strncasecmp(value, "Kerberos", 8) == 0 &&
1515                          (value[8] == '\0' || value[8] == ' '))) {
1516                     if (request->flags.connectionAuthDisabled) {
1517                         hdr->delAt(pos, connection_auth_blocked);
1518                         continue;
1519                     }
1520                     request->flags.mustKeepalive = true;
1521                     if (!request->flags.accelerated && !request->flags.intercepted) {
1522                         httpHeaderPutStrf(hdr, Http::HdrType::PROXY_SUPPORT, "Session-Based-Authentication");
1523                         /*
1524                           We send "Connection: Proxy-Support" header to mark
1525                           Proxy-Support as a hop-by-hop header for intermediaries that do not
1526                           understand the semantics of this header. The RFC should have included
1527                           this recommendation.
1528                         */
1529                         httpHeaderPutStrf(hdr, Http::HdrType::CONNECTION, "Proxy-support");
1530                     }
1531                     break;
1532                 }
1533             }
1534         }
1535
1536         if (connection_auth_blocked)
1537             hdr->refreshMask();
1538     }
1539
1540 #if USE_AUTH
1541     /* Handle authentication headers */
1542     if (http->logType.oldType == LOG_TCP_DENIED &&
1543             ( reply->sline.status() == Http::scProxyAuthenticationRequired ||
1544               reply->sline.status() == Http::scUnauthorized)
1545        ) {
1546         /* Add authentication header */
1547         /*! \todo alter errorstate to be accel on|off aware. The 0 on the next line
1548          * depends on authenticate behaviour: all schemes to date send no extra
1549          * data on 407/401 responses, and do not check the accel state on 401/407
1550          * responses
1551          */
1552         Auth::UserRequest::AddReplyAuthHeader(reply, request->auth_user_request, request, 0, 1);
1553     } else if (request->auth_user_request != NULL)
1554         Auth::UserRequest::AddReplyAuthHeader(reply, request->auth_user_request, request, http->flags.accel, 0);
1555 #endif
1556
1557     /* Append X-Cache */
1558     httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE, "%s from %s",
1559                       is_hit ? "HIT" : "MISS", getMyHostname());
1560
1561 #if USE_CACHE_DIGESTS
1562     /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */
1563     httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE_LOOKUP, "%s from %s:%d",
1564                       lookup_type ? lookup_type : "NONE",
1565                       getMyHostname(), getMyPort());
1566
1567 #endif
1568
1569     const bool maySendChunkedReply = !request->multipartRangeRequest() &&
1570                                      reply->sline.protocol == AnyP::PROTO_HTTP && // response is HTTP
1571                                      (request->http_ver >= Http::ProtocolVersion(1,1));
1572
1573     /* Check whether we should send keep-alive */
1574     if (!Config.onoff.error_pconns && reply->sline.status() >= 400 && !request->flags.mustKeepalive) {
1575         debugs(33, 3, "clientBuildReplyHeader: Error, don't keep-alive");
1576         request->flags.proxyKeepalive = false;
1577     } else if (!Config.onoff.client_pconns && !request->flags.mustKeepalive) {
1578         debugs(33, 2, "clientBuildReplyHeader: Connection Keep-Alive not requested by admin or client");
1579         request->flags.proxyKeepalive = false;
1580     } else if (request->flags.proxyKeepalive && shutting_down) {
1581         debugs(88, 3, "clientBuildReplyHeader: Shutting down, don't keep-alive.");
1582         request->flags.proxyKeepalive = false;
1583     } else if (request->flags.connectionAuth && !reply->keep_alive) {
1584         debugs(33, 2, "clientBuildReplyHeader: Connection oriented auth but server side non-persistent");
1585         request->flags.proxyKeepalive = false;
1586     } else if (reply->bodySize(request->method) < 0 && !maySendChunkedReply) {
1587         debugs(88, 3, "clientBuildReplyHeader: can't keep-alive, unknown body size" );
1588         request->flags.proxyKeepalive = false;
1589     } else if (fdUsageHigh()&& !request->flags.mustKeepalive) {
1590         debugs(88, 3, "clientBuildReplyHeader: Not many unused FDs, can't keep-alive");
1591         request->flags.proxyKeepalive = false;
1592     } else if (request->flags.sslBumped && !reply->persistent()) {
1593         // We do not really have to close, but we pretend we are a tunnel.
1594         debugs(88, 3, "clientBuildReplyHeader: bumped reply forces close");
1595         request->flags.proxyKeepalive = false;
1596     } else if (request->pinnedConnection() && !reply->persistent()) {
1597         // The peer wants to close the pinned connection
1598         debugs(88, 3, "pinned reply forces close");
1599         request->flags.proxyKeepalive = false;
1600     } else if (http->getConn()) {
1601         ConnStateData * conn = http->getConn();
1602         if (!Comm::IsConnOpen(conn->port->listenConn)) {
1603             // The listening port closed because of a reconfigure
1604             debugs(88, 3, "listening port closed");
1605             request->flags.proxyKeepalive = false;
1606         }
1607     }
1608
1609     // Decide if we send chunked reply
1610     if (maySendChunkedReply && reply->bodySize(request->method) < 0) {
1611         debugs(88, 3, "clientBuildReplyHeader: chunked reply");
1612         request->flags.chunkedReply = true;
1613         hdr->putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
1614     }
1615
1616     hdr->addVia(reply->sline.version);
1617
1618     /* Signal keep-alive or close explicitly */
1619     hdr->putStr(Http::HdrType::CONNECTION, request->flags.proxyKeepalive ? "keep-alive" : "close");
1620
1621 #if ADD_X_REQUEST_URI
1622     /*
1623      * Knowing the URI of the request is useful when debugging persistent
1624      * connections in a client; we cannot guarantee the order of http headers,
1625      * but X-Request-URI is likely to be the very last header to ease use from a
1626      * debugger [hdr->entries.count-1].
1627      */
1628     hdr->putStr(Http::HdrType::X_REQUEST_URI,
1629                 http->memOjbect()->url ? http->memObject()->url : http->uri);
1630
1631 #endif
1632
1633     /* Surrogate-Control requires Surrogate-Capability from upstream to pass on */
1634     if ( hdr->has(Http::HdrType::SURROGATE_CONTROL) ) {
1635         if (!request->header.has(Http::HdrType::SURROGATE_CAPABILITY)) {
1636             hdr->delById(Http::HdrType::SURROGATE_CONTROL);
1637         }
1638         /* TODO: else case: drop any controls intended specifically for our surrogate ID */
1639     }
1640
1641     httpHdrMangleList(hdr, request, http->al, ROR_REPLY);
1642 }
1643
1644 void
1645 clientReplyContext::cloneReply()
1646 {
1647     assert(reply == NULL);
1648
1649     reply = http->storeEntry()->getReply()->clone();
1650     HTTPMSGLOCK(reply);
1651
1652     http->al->reply = reply;
1653
1654     if (reply->sline.protocol == AnyP::PROTO_HTTP) {
1655         /* RFC 2616 requires us to advertise our version (but only on real HTTP traffic) */
1656         reply->sline.version = Http::ProtocolVersion();
1657     }
1658
1659     /* do header conversions */
1660     buildReplyHeader();
1661 }
1662
1663 /// Safely disposes of an entry pointing to a cache hit that we do not want.
1664 /// We cannot just ignore the entry because it may be locking or otherwise
1665 /// holding an associated cache resource of some sort.
1666 void
1667 clientReplyContext::forgetHit()
1668 {
1669     StoreEntry *e = http->storeEntry();
1670     assert(e); // or we are not dealing with a hit
1671     // We probably have not locked the entry earlier, unfortunately. We lock it
1672     // now so that we can unlock two lines later (and trigger cleanup).
1673     // Ideally, ClientHttpRequest::storeEntry() should lock/unlock, but it is
1674     // used so inconsistently that simply adding locking there leads to bugs.
1675     e->lock("clientReplyContext::forgetHit");
1676     http->storeEntry(NULL);
1677     e->unlock("clientReplyContext::forgetHit"); // may delete e
1678 }
1679
1680 void
1681 clientReplyContext::identifyStoreObject()
1682 {
1683     HttpRequest *r = http->request;
1684
1685     // client sent CC:no-cache or some other condition has been
1686     // encountered which prevents delivering a public/cached object.
1687     if (!r->flags.noCache || r->flags.internal) {
1688         lookingforstore = 5;
1689         StoreEntry::getPublicByRequest (this, r);
1690     } else {
1691         identifyFoundObject(nullptr);
1692     }
1693 }
1694
1695 /**
1696  * Check state of the current StoreEntry object.
1697  * to see if we can determine the final status of the request.
1698  */
1699 void
1700 clientReplyContext::identifyFoundObject(StoreEntry *newEntry)
1701 {
1702     HttpRequest *r = http->request;
1703     http->storeEntry(newEntry);
1704     const auto e = http->storeEntry();
1705
1706     /* Release IP-cache entries on reload */
1707     /** \li If the request has no-cache flag set or some no_cache HACK in operation we
1708       * 'invalidate' the cached IP entries for this request ???
1709       */
1710     if (r->flags.noCache || r->flags.noCacheHack())
1711         ipcacheInvalidateNegative(r->url.host());
1712
1713 #if USE_CACHE_DIGESTS
1714     lookup_type = e ? "HIT" : "MISS";
1715 #endif
1716
1717     if (!e) {
1718         /** \li If no StoreEntry object is current assume this object isn't in the cache set MISS*/
1719         debugs(85, 3, "StoreEntry is NULL -  MISS");
1720         http->logType.update(LOG_TCP_MISS);
1721         doGetMoreData();
1722         return;
1723     }
1724
1725     if (Config.onoff.offline) {
1726         /** \li If we are running in offline mode set to HIT */
1727         debugs(85, 3, "offline HIT " << *e);
1728         http->logType.update(LOG_TCP_HIT);
1729         doGetMoreData();
1730         return;
1731     }
1732
1733     if (http->redirect.status) {
1734         /** \li If redirection status is True force this to be a MISS */
1735         debugs(85, 3, "REDIRECT status forced StoreEntry to NULL (no body on 3XX responses) " << *e);
1736         forgetHit();
1737         http->logType.update(LOG_TCP_REDIRECT);
1738         doGetMoreData();
1739         return;
1740     }
1741
1742     if (!e->validToSend()) {
1743         debugs(85, 3, "!storeEntryValidToSend MISS " << *e);
1744         forgetHit();
1745         http->logType.update(LOG_TCP_MISS);
1746         doGetMoreData();
1747         return;
1748     }
1749
1750     if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
1751         /* \li Special entries are always hits, no matter what the client says */
1752         debugs(85, 3, "ENTRY_SPECIAL HIT " << *e);
1753         http->logType.update(LOG_TCP_HIT);
1754         doGetMoreData();
1755         return;
1756     }
1757
1758     if (r->flags.noCache) {
1759         debugs(85, 3, "no-cache REFRESH MISS " << *e);
1760         forgetHit();
1761         http->logType.update(LOG_TCP_CLIENT_REFRESH_MISS);
1762         doGetMoreData();
1763         return;
1764     }
1765
1766     if (e->hittingRequiresCollapsing() && !startCollapsingOn(*e, false)) {
1767         debugs(85, 3, "prohibited CF MISS " << *e);
1768         forgetHit();
1769         http->logType.update(LOG_TCP_MISS);
1770         doGetMoreData();
1771         return;
1772     }
1773
1774     debugs(85, 3, "default HIT " << *e);
1775     http->logType.update(LOG_TCP_HIT);
1776     doGetMoreData();
1777 }
1778
1779 /**
1780  * Request more data from the store for the client Stream
1781  * This is *the* entry point to this module.
1782  *
1783  * Preconditions:
1784  *  - This is the head of the list.
1785  *  - There is at least one more node.
1786  *  - Data context is not null
1787  */
1788 void
1789 clientGetMoreData(clientStreamNode * aNode, ClientHttpRequest * http)
1790 {
1791     /* Test preconditions */
1792     assert(aNode != NULL);
1793     assert(cbdataReferenceValid(aNode));
1794     assert(aNode->node.prev == NULL);
1795     assert(aNode->node.next != NULL);
1796     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1797     assert (context);
1798     assert(context->http == http);
1799
1800     clientStreamNode *next = ( clientStreamNode *)aNode->node.next->data;
1801
1802     if (!context->ourNode)
1803         context->ourNode = aNode;
1804
1805     /* no cbdatareference, this is only used once, and safely */
1806     if (context->flags.storelogiccomplete) {
1807         StoreIOBuffer tempBuffer;
1808         tempBuffer.offset = next->readBuffer.offset + context->headers_sz;
1809         tempBuffer.length = next->readBuffer.length;
1810         tempBuffer.data = next->readBuffer.data;
1811
1812         storeClientCopy(context->sc, http->storeEntry(),
1813                         tempBuffer, clientReplyContext::SendMoreData, context);
1814         return;
1815     }
1816
1817     if (context->http->request->method == Http::METHOD_PURGE) {
1818         context->purgeRequest();
1819         return;
1820     }
1821
1822     // OPTIONS with Max-Forwards:0 handled in clientProcessRequest()
1823
1824     if (context->http->request->method == Http::METHOD_TRACE) {
1825         if (context->http->request->header.getInt64(Http::HdrType::MAX_FORWARDS) == 0) {
1826             context->traceReply(aNode);
1827             return;
1828         }
1829
1830         /* continue forwarding, not finished yet. */
1831         http->logType.update(LOG_TCP_MISS);
1832
1833         context->doGetMoreData();
1834     } else
1835         context->identifyStoreObject();
1836 }
1837
1838 void
1839 clientReplyContext::doGetMoreData()
1840 {
1841     /* We still have to do store logic processing - vary, cache hit etc */
1842     if (http->storeEntry() != NULL) {
1843         /* someone found the object in the cache for us */
1844         StoreIOBuffer localTempBuffer;
1845
1846         http->storeEntry()->lock("clientReplyContext::doGetMoreData");
1847
1848         http->storeEntry()->ensureMemObject(storeId(), http->log_uri, http->request->method);
1849
1850         sc = storeClientListAdd(http->storeEntry(), this);
1851 #if USE_DELAY_POOLS
1852         sc->setDelayId(DelayId::DelayClient(http));
1853 #endif
1854
1855         assert(http->logType.oldType == LOG_TCP_HIT);
1856         reqofs = 0;
1857         /* guarantee nothing has been sent yet! */
1858         assert(http->out.size == 0);
1859         assert(http->out.offset == 0);
1860
1861         if (ConnStateData *conn = http->getConn()) {
1862             if (Ip::Qos::TheConfig.isHitTosActive()) {
1863                 Ip::Qos::doTosLocalHit(conn->clientConnection);
1864             }
1865
1866             if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
1867                 Ip::Qos::doNfmarkLocalHit(conn->clientConnection);
1868             }
1869         }
1870
1871         localTempBuffer.offset = reqofs;
1872         localTempBuffer.length = getNextNode()->readBuffer.length;
1873         localTempBuffer.data = getNextNode()->readBuffer.data;
1874         storeClientCopy(sc, http->storeEntry(), localTempBuffer, CacheHit, this);
1875     } else {
1876         /* MISS CASE, http->logType is already set! */
1877         processMiss();
1878     }
1879 }
1880
1881 /** The next node has removed itself from the stream. */
1882 void
1883 clientReplyDetach(clientStreamNode * node, ClientHttpRequest * http)
1884 {
1885     /** detach from the stream */
1886     clientStreamDetach(node, http);
1887 }
1888
1889 /**
1890  * Accepts chunk of a http message in buf, parses prefix, filters headers and
1891  * such, writes processed message to the message recipient
1892  */
1893 void
1894 clientReplyContext::SendMoreData(void *data, StoreIOBuffer result)
1895 {
1896     clientReplyContext *context = static_cast<clientReplyContext *>(data);
1897     context->sendMoreData (result);
1898 }
1899
1900 void
1901 clientReplyContext::makeThisHead()
1902 {
1903     /* At least, I think thats what this does */
1904     dlinkDelete(&http->active, &ClientActiveRequests);
1905     dlinkAdd(http, &http->active, &ClientActiveRequests);
1906 }
1907
1908 bool
1909 clientReplyContext::errorInStream(StoreIOBuffer const &result, size_t const &sizeToProcess)const
1910 {
1911     return /* aborted request */
1912         (http->storeEntry() && EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) ||
1913         /* Upstream read error */ (result.flags.error) ||
1914         /* Upstream EOF */ (sizeToProcess == 0);
1915 }
1916
1917 void
1918 clientReplyContext::sendStreamError(StoreIOBuffer const &result)
1919 {
1920     /** call clientWriteComplete so the client socket gets closed
1921      *
1922      * We call into the stream, because we don't know that there is a
1923      * client socket!
1924      */
1925     debugs(88, 5, "A stream error has occurred, marking as complete and sending no data.");
1926     StoreIOBuffer localTempBuffer;
1927     flags.complete = 1;
1928     http->request->flags.streamError = true;
1929     localTempBuffer.flags.error = result.flags.error;
1930     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1931                          localTempBuffer);
1932 }
1933
1934 void
1935 clientReplyContext::pushStreamData(StoreIOBuffer const &result, char *source)
1936 {
1937     StoreIOBuffer localTempBuffer;
1938
1939     if (result.length == 0) {
1940         debugs(88, 5, "clientReplyContext::pushStreamData: marking request as complete due to 0 length store result");
1941         flags.complete = 1;
1942     }
1943
1944     assert(result.offset - headers_sz == next()->readBuffer.offset);
1945     localTempBuffer.offset = result.offset - headers_sz;
1946     localTempBuffer.length = result.length;
1947
1948     if (localTempBuffer.length)
1949         localTempBuffer.data = source;
1950
1951     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1952                          localTempBuffer);
1953 }
1954
1955 clientStreamNode *
1956 clientReplyContext::next() const
1957 {
1958     assert ( (clientStreamNode*)http->client_stream.head->next->data == getNextNode());
1959     return getNextNode();
1960 }
1961
1962 void
1963 clientReplyContext::sendBodyTooLargeError()
1964 {
1965     Ip::Address tmp_noaddr;
1966     tmp_noaddr.setNoAddr(); // TODO: make a global const
1967     http->logType.update(LOG_TCP_DENIED_REPLY);
1968     ErrorState *err = clientBuildError(ERR_TOO_BIG, Http::scForbidden, NULL,
1969                                        http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
1970                                        http->request, http->al);
1971     removeClientStoreReference(&(sc), http);
1972     HTTPMSGUNLOCK(reply);
1973     startError(err);
1974
1975 }
1976
1977 /// send 412 (Precondition Failed) to client
1978 void
1979 clientReplyContext::sendPreconditionFailedError()
1980 {
1981     http->logType.update(LOG_TCP_HIT);
1982     Ip::Address tmp_noaddr;
1983     tmp_noaddr.setNoAddr();
1984     ErrorState *const err =
1985         clientBuildError(ERR_PRECONDITION_FAILED, Http::scPreconditionFailed,
1986                          NULL, http->getConn() ? http->getConn()->clientConnection->remote : tmp_noaddr, http->request, http->al);
1987     removeClientStoreReference(&sc, http);
1988     HTTPMSGUNLOCK(reply);
1989     startError(err);
1990 }
1991
1992 /// send 304 (Not Modified) to client
1993 void
1994 clientReplyContext::sendNotModified()
1995 {
1996     StoreEntry *e = http->storeEntry();
1997     const time_t timestamp = e->timestamp;
1998     HttpReply *const temprep = e->getReply()->make304();
1999     // log as TCP_INM_HIT if code 304 generated for
2000     // If-None-Match request
2001     if (!http->request->flags.ims)
2002         http->logType.update(LOG_TCP_INM_HIT);
2003     else
2004         http->logType.update(LOG_TCP_IMS_HIT);
2005     removeClientStoreReference(&sc, http);
2006     createStoreEntry(http->request->method, RequestFlags());
2007     e = http->storeEntry();
2008     // Copy timestamp from the original entry so the 304
2009     // reply has a meaningful Age: header.
2010     e->timestampsSet();
2011     e->timestamp = timestamp;
2012     e->replaceHttpReply(temprep);
2013     e->complete();
2014     /*
2015      * TODO: why put this in the store and then serialise it and
2016      * then parse it again. Simply mark the request complete in
2017      * our context and write the reply struct to the client side.
2018      */
2019     triggerInitialStoreRead();
2020 }
2021
2022 /// send 304 (Not Modified) or 412 (Precondition Failed) to client
2023 /// depending on request method
2024 void
2025 clientReplyContext::sendNotModifiedOrPreconditionFailedError()
2026 {
2027     if (http->request->method == Http::METHOD_GET ||
2028             http->request->method == Http::METHOD_HEAD)
2029         sendNotModified();
2030     else
2031         sendPreconditionFailedError();
2032 }
2033
2034 void
2035 clientReplyContext::processReplyAccess ()
2036 {
2037     /* NP: this should probably soft-fail to a zero-sized-reply error ?? */
2038     assert(reply);
2039
2040     /** Don't block our own responses or HTTP status messages */
2041     if (http->logType.oldType == LOG_TCP_DENIED ||
2042             http->logType.oldType == LOG_TCP_DENIED_REPLY ||
2043             alwaysAllowResponse(reply->sline.status())) {
2044         headers_sz = reply->hdr_sz;
2045         processReplyAccessResult(ACCESS_ALLOWED);
2046         return;
2047     }
2048
2049     /** Check for reply to big error */
2050     if (reply->expectedBodyTooLarge(*http->request)) {
2051         sendBodyTooLargeError();
2052         return;
2053     }
2054
2055     headers_sz = reply->hdr_sz;
2056
2057     /** check for absent access controls (permit by default) */
2058     if (!Config.accessList.reply) {
2059         processReplyAccessResult(ACCESS_ALLOWED);
2060         return;
2061     }
2062
2063     /** Process http_reply_access lists */
2064     ACLFilledChecklist *replyChecklist =
2065         clientAclChecklistCreate(Config.accessList.reply, http);
2066     replyChecklist->reply = reply;
2067     HTTPMSGLOCK(replyChecklist->reply);
2068     replyChecklist->nonBlockingCheck(ProcessReplyAccessResult, this);
2069 }
2070
2071 void
2072 clientReplyContext::ProcessReplyAccessResult(Acl::Answer rv, void *voidMe)
2073 {
2074     clientReplyContext *me = static_cast<clientReplyContext *>(voidMe);
2075     me->processReplyAccessResult(rv);
2076 }
2077
2078 void
2079 clientReplyContext::processReplyAccessResult(const Acl::Answer &accessAllowed)
2080 {
2081     debugs(88, 2, "The reply for " << http->request->method
2082            << ' ' << http->uri << " is " << accessAllowed << ", because it matched "
2083            << (AclMatchedName ? AclMatchedName : "NO ACL's"));
2084
2085     if (!accessAllowed.allowed()) {
2086         ErrorState *err;
2087         err_type page_id;
2088         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1);
2089
2090         http->logType.update(LOG_TCP_DENIED_REPLY);
2091
2092         if (page_id == ERR_NONE)
2093             page_id = ERR_ACCESS_DENIED;
2094
2095         Ip::Address tmp_noaddr;
2096         tmp_noaddr.setNoAddr();
2097         err = clientBuildError(page_id, Http::scForbidden, NULL,
2098                                http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
2099                                http->request, http->al);
2100
2101         removeClientStoreReference(&sc, http);
2102
2103         HTTPMSGUNLOCK(reply);
2104
2105         startError(err);
2106
2107         return;
2108     }
2109
2110     /* Ok, the reply is allowed, */
2111     http->loggingEntry(http->storeEntry());
2112
2113     ssize_t body_size = reqofs - reply->hdr_sz;
2114     if (body_size < 0) {
2115         reqofs = reply->hdr_sz;
2116         body_size = 0;
2117     }
2118
2119     debugs(88, 3, "clientReplyContext::sendMoreData: Appending " <<
2120            (int) body_size << " bytes after " << reply->hdr_sz <<
2121            " bytes of headers");
2122
2123 #if USE_SQUID_ESI
2124
2125     if (http->flags.accel && reply->sline.status() != Http::scForbidden &&
2126             !alwaysAllowResponse(reply->sline.status()) &&
2127             esiEnableProcessing(reply)) {
2128         debugs(88, 2, "Enabling ESI processing for " << http->uri);
2129         clientStreamInsertHead(&http->client_stream, esiStreamRead,
2130                                esiProcessStream, esiStreamDetach, esiStreamStatus, NULL);
2131     }
2132
2133 #endif
2134
2135     if (http->request->method == Http::METHOD_HEAD) {
2136         /* do not forward body for HEAD replies */
2137         body_size = 0;
2138         http->flags.done_copying = true;
2139         flags.complete = 1;
2140     }
2141
2142     assert (!flags.headersSent);
2143     flags.headersSent = true;
2144
2145     StoreIOBuffer localTempBuffer;
2146     char *buf = next()->readBuffer.data;
2147     char *body_buf = buf + reply->hdr_sz;
2148
2149     //Server side may disable ranges under some circumstances.
2150
2151     if ((!http->request->range))
2152         next()->readBuffer.offset = 0;
2153
2154     body_buf -= next()->readBuffer.offset;
2155
2156     if (next()->readBuffer.offset != 0) {
2157         if (next()->readBuffer.offset > body_size) {
2158             /* Can't use any of the body we received. send nothing */
2159             localTempBuffer.length = 0;
2160             localTempBuffer.data = NULL;
2161         } else {
2162             localTempBuffer.length = body_size - next()->readBuffer.offset;
2163             localTempBuffer.data = body_buf + next()->readBuffer.offset;
2164         }
2165     } else {
2166         localTempBuffer.length = body_size;
2167         localTempBuffer.data = body_buf;
2168     }
2169
2170     /* TODO??: move the data in the buffer back by the request header size */
2171     clientStreamCallback((clientStreamNode *)http->client_stream.head->data,
2172                          http, reply, localTempBuffer);
2173
2174     return;
2175 }
2176
2177 void
2178 clientReplyContext::sendMoreData (StoreIOBuffer result)
2179 {
2180     if (deleting)
2181         return;
2182
2183     StoreEntry *entry = http->storeEntry();
2184
2185     if (ConnStateData * conn = http->getConn()) {
2186         if (!conn->isOpen()) {
2187             debugs(33,3, "not sending more data to closing connection " << conn->clientConnection);
2188             return;
2189         }
2190         if (conn->pinning.zeroReply) {
2191             debugs(33,3, "not sending more data after a pinned zero reply " << conn->clientConnection);
2192             return;
2193         }
2194
2195         if (reqofs==0 && !http->logType.isTcpHit()) {
2196             if (Ip::Qos::TheConfig.isHitTosActive()) {
2197                 Ip::Qos::doTosLocalMiss(conn->clientConnection, http->request->hier.code);
2198             }
2199             if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
2200                 Ip::Qos::doNfmarkLocalMiss(conn->clientConnection, http->request->hier.code);
2201             }
2202         }
2203
2204         debugs(88, 5, conn->clientConnection <<
2205                " '" << entry->url() << "'" <<
2206                " out.offset=" << http->out.offset);
2207     }
2208
2209     char *buf = next()->readBuffer.data;
2210
2211     if (buf != result.data) {
2212         /* we've got to copy some data */
2213         assert(result.length <= next()->readBuffer.length);
2214         memcpy(buf, result.data, result.length);
2215     }
2216
2217     /* We've got the final data to start pushing... */
2218     flags.storelogiccomplete = 1;
2219
2220     reqofs += result.length;
2221
2222     assert(reqofs <= HTTP_REQBUF_SZ || flags.headersSent);
2223
2224     assert(http->request != NULL);
2225
2226     /* ESI TODO: remove this assert once everything is stable */
2227     assert(http->client_stream.head->data
2228            && cbdataReferenceValid(http->client_stream.head->data));
2229
2230     makeThisHead();
2231
2232     debugs(88, 5, "clientReplyContext::sendMoreData: " << http->uri << ", " <<
2233            reqofs << " bytes (" << result.length <<
2234            " new bytes)");
2235
2236     /* update size of the request */
2237     reqsize = reqofs;
2238
2239     if (errorInStream(result, reqofs)) {
2240         sendStreamError(result);
2241         return;
2242     }
2243
2244     if (flags.headersSent) {
2245         pushStreamData (result, buf);
2246         return;
2247     }
2248
2249     cloneReply();
2250
2251 #if USE_DELAY_POOLS
2252     if (sc)
2253         sc->setDelayId(DelayId::DelayClient(http,reply));
2254 #endif
2255
2256     holdingBuffer = result;
2257     processReplyAccess();
2258     return;
2259 }
2260
2261 void
2262 clientReplyContext::fillChecklist(ACLFilledChecklist &checklist) const
2263 {
2264     clientAclChecklistFill(checklist, http);
2265 }
2266
2267 /* Using this breaks the client layering just a little!
2268  */
2269 void
2270 clientReplyContext::createStoreEntry(const HttpRequestMethod& m, RequestFlags reqFlags)
2271 {
2272     assert(http != NULL);
2273     /*
2274      * For erroneous requests, we might not have a h->request,
2275      * so make a fake one.
2276      */
2277
2278     if (http->request == NULL) {
2279         const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initClient);
2280         // XXX: These fake URI parameters shadow the real (or error:...) URI.
2281         // TODO: Either always set the request earlier and assert here OR use
2282         // http->uri (converted to Anyp::Uri) to create this catch-all request.
2283         const_cast<HttpRequest *&>(http->request) =  new HttpRequest(m, AnyP::PROTO_NONE, "http", null_string, mx);
2284         HTTPMSGLOCK(http->request);
2285     }
2286
2287     StoreEntry *e = storeCreateEntry(storeId(), http->log_uri, reqFlags, m);
2288
2289     // Make entry collapsable ASAP, to increase collapsing chances for others,
2290     // TODO: every must-revalidate and similar request MUST reach the origin,
2291     // but do we have to prohibit others from collapsing on that request?
2292     if (reqFlags.cachable &&
2293             !reqFlags.needValidation &&
2294             (m == Http::METHOD_GET || m == Http::METHOD_HEAD) &&
2295             mayInitiateCollapsing()) {
2296         // make the entry available for future requests now
2297         (void)Store::Root().allowCollapsing(e, reqFlags, m);
2298     }
2299
2300     sc = storeClientListAdd(e, this);
2301
2302 #if USE_DELAY_POOLS
2303     sc->setDelayId(DelayId::DelayClient(http));
2304 #endif
2305
2306     reqofs = 0;
2307
2308     reqsize = 0;
2309
2310     /* I don't think this is actually needed! -- adrian */
2311     /* http->reqbuf = http->norm_reqbuf; */
2312     //    assert(http->reqbuf == http->norm_reqbuf);
2313     /* The next line is illegal because we don't know if the client stream
2314      * buffers have been set up
2315      */
2316     //    storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf,
2317     //        SendMoreData, this);
2318     /* So, we mark the store logic as complete */
2319     flags.storelogiccomplete = 1;
2320
2321     /* and get the caller to request a read, from whereever they are */
2322     /* NOTE: after ANY data flows down the pipe, even one step,
2323      * this function CAN NOT be used to manage errors
2324      */
2325     http->storeEntry(e);
2326 }
2327
2328 ErrorState *
2329 clientBuildError(err_type page_id, Http::StatusCode status, char const *url,
2330                  Ip::Address &src_addr, HttpRequest * request, const AccessLogEntry::Pointer &al)
2331 {
2332     const auto err = new ErrorState(page_id, status, request, al);
2333     err->src_addr = src_addr;
2334
2335     if (url)
2336         err->url = xstrdup(url);
2337
2338     return err;
2339 }
2340