src/client_side_reply.cc

   1 /*
   2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 88    Client-side Reply Routines */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "acl/Gadgets.h"
  14 #include "anyp/PortCfg.h"
  15 #include "client_side_reply.h"
  16 #include "errorpage.h"
  17 #include "ETag.h"
  18 #include "fd.h"
  19 #include "fde.h"
  20 #include "format/Token.h"
  21 #include "FwdState.h"
  22 #include "globals.h"
  23 #include "http/Stream.h"
  24 #include "HttpHeaderTools.h"
  25 #include "HttpReply.h"
  26 #include "HttpRequest.h"
  27 #include "ip/QosConfig.h"
  28 #include "ipcache.h"
  29 #include "log/access_log.h"
  30 #include "MemObject.h"
  31 #include "mime_header.h"
  32 #include "neighbors.h"
  33 #include "refresh.h"
  34 #include "RequestFlags.h"
  35 #include "SquidConfig.h"
  36 #include "SquidTime.h"
  37 #include "Store.h"
  38 #include "StrList.h"
  39 #include "tools.h"
  40 #include "URL.h"
  41 #if USE_AUTH
  42 #include "auth/UserRequest.h"
  43 #endif
  44 #if USE_DELAY_POOLS
  45 #include "DelayPools.h"
  46 #endif
  47 #if USE_SQUID_ESI
  48 #include "esi/Esi.h"
  49 #endif
  50
  51 #include <memory>
  52
  53 CBDATA_CLASS_INIT(clientReplyContext);
  54
  55 /* Local functions */
  56 extern "C" CSS clientReplyStatus;
  57 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *, Ip::Address &, HttpRequest *);
  58
  59 /* privates */
  60
  61 clientReplyContext::~clientReplyContext()
  62 {
  63     deleting = true;
  64     /* This may trigger a callback back into SendMoreData as the cbdata
  65      * is still valid
  66      */
  67     removeClientStoreReference(&sc, http);
  68     /* old_entry might still be set if we didn't yet get the reply
  69      * code in HandleIMSReply() */
  70     removeStoreReference(&old_sc, &old_entry);
  71     safe_free(tempBuffer.data);
  72     cbdataReferenceDone(http);
  73     HTTPMSGUNLOCK(reply);
  74 }
  75
  76 clientReplyContext::clientReplyContext(ClientHttpRequest *clientContext) :
  77     purgeStatus(Http::scNone),
  78     lookingforstore(0),
  79     http(cbdataReference(clientContext)),
  80     headers_sz(0),
  81     sc(NULL),
  82     old_reqsize(0),
  83     reqsize(0),
  84     reqofs(0),
  85 #if USE_CACHE_DIGESTS
  86     lookup_type(NULL),
  87 #endif
  88     ourNode(NULL),
  89     reply(NULL),
  90     old_entry(NULL),
  91     old_sc(NULL),
  92     old_lastmod(-1),
  93     deleting(false),
  94     collapsedRevalidation(crNone)
  95 {
  96     *tempbuf = 0;
  97 }
  98
  99 /** Create an error in the store awaiting the client side to read it.
 100  *
 101  * This may be better placed in the clientStream logic, but it has not been
 102  * relocated there yet
 103  */
 104 void
 105 clientReplyContext::setReplyToError(
 106     err_type err, Http::StatusCode status, const HttpRequestMethod& method, char const *uri,
 107     Ip::Address &addr, HttpRequest * failedrequest, const char *unparsedrequest,
 108 #if USE_AUTH
 109     Auth::UserRequest::Pointer auth_user_request
 110 #else
 111     void*
 112 #endif
 113 )
 114 {
 115     ErrorState *errstate = clientBuildError(err, status, uri, addr, failedrequest);
 116
 117     if (unparsedrequest)
 118         errstate->request_hdrs = xstrdup(unparsedrequest);
 119
 120 #if USE_AUTH
 121     errstate->auth_user_request = auth_user_request;
 122 #endif
 123     setReplyToError(method, errstate);
 124 }
 125
 126 void clientReplyContext::setReplyToError(const HttpRequestMethod& method, ErrorState *errstate)
 127 {
 128     if (errstate->httpStatus == Http::scNotImplemented && http->request)
 129         /* prevent confusion over whether we default to persistent or not */
 130         http->request->flags.proxyKeepalive = false;
 131
 132     http->al->http.code = errstate->httpStatus;
 133
 134     if (http->request)
 135         http->request->ignoreRange("responding with a Squid-generated error");
 136
 137     createStoreEntry(method, RequestFlags());
 138     assert(errstate->callback_data == NULL);
 139     errorAppendEntry(http->storeEntry(), errstate);
 140     /* Now the caller reads to get this */
 141 }
 142
 143 void
 144 clientReplyContext::setReplyToReply(HttpReply *futureReply)
 145 {
 146     Must(futureReply);
 147     http->al->http.code = futureReply->sline.status();
 148
 149     HttpRequestMethod method;
 150     if (http->request) { // nil on responses to unparsable requests
 151         http->request->ignoreRange("responding with a Squid-generated reply");
 152         method = http->request->method;
 153     }
 154
 155     createStoreEntry(method, RequestFlags());
 156
 157     http->storeEntry()->storeErrorResponse(futureReply);
 158     /* Now the caller reads to get futureReply */
 159 }
 160
 161 // Assumes that the entry contains an error response without Content-Range.
 162 // To use with regular entries, make HTTP Range header removal conditional.
 163 void clientReplyContext::setReplyToStoreEntry(StoreEntry *entry, const char *reason)
 164 {
 165     entry->lock("clientReplyContext::setReplyToStoreEntry"); // removeClientStoreReference() unlocks
 166     sc = storeClientListAdd(entry, this);
 167 #if USE_DELAY_POOLS
 168     sc->setDelayId(DelayId::DelayClient(http));
 169 #endif
 170     reqofs = 0;
 171     reqsize = 0;
 172     if (http->request)
 173         http->request->ignoreRange(reason);
 174     flags.storelogiccomplete = 1;
 175     http->storeEntry(entry);
 176 }
 177
 178 void
 179 clientReplyContext::removeStoreReference(store_client ** scp,
 180         StoreEntry ** ep)
 181 {
 182     StoreEntry *e;
 183     store_client *sc_tmp = *scp;
 184
 185     if ((e = *ep) != NULL) {
 186         *ep = NULL;
 187         storeUnregister(sc_tmp, e, this);
 188         *scp = NULL;
 189         e->unlock("clientReplyContext::removeStoreReference");
 190     }
 191 }
 192
 193 void
 194 clientReplyContext::removeClientStoreReference(store_client **scp, ClientHttpRequest *aHttpRequest)
 195 {
 196     StoreEntry *reference = aHttpRequest->storeEntry();
 197     removeStoreReference(scp, &reference);
 198     aHttpRequest->storeEntry(reference);
 199 }
 200
 201 void
 202 clientReplyContext::saveState()
 203 {
 204     assert(old_sc == NULL);
 205     debugs(88, 3, "clientReplyContext::saveState: saving store context");
 206     old_entry = http->storeEntry();
 207     old_sc = sc;
 208     old_lastmod = http->request->lastmod;
 209     old_etag = http->request->etag;
 210     old_reqsize = reqsize;
 211     tempBuffer.offset = reqofs;
 212     /* Prevent accessing the now saved entries */
 213     http->storeEntry(NULL);
 214     sc = NULL;
 215     reqsize = 0;
 216     reqofs = 0;
 217 }
 218
 219 void
 220 clientReplyContext::restoreState()
 221 {
 222     assert(old_sc != NULL);
 223     debugs(88, 3, "clientReplyContext::restoreState: Restoring store context");
 224     removeClientStoreReference(&sc, http);
 225     http->storeEntry(old_entry);
 226     sc = old_sc;
 227     reqsize = old_reqsize;
 228     reqofs = tempBuffer.offset;
 229     http->request->lastmod = old_lastmod;
 230     http->request->etag = old_etag;
 231     /* Prevent accessed the old saved entries */
 232     old_entry = NULL;
 233     old_sc = NULL;
 234     old_lastmod = -1;
 235     old_etag.clean();
 236     old_reqsize = 0;
 237     tempBuffer.offset = 0;
 238 }
 239
 240 void
 241 clientReplyContext::startError(ErrorState * err)
 242 {
 243     createStoreEntry(http->request->method, RequestFlags());
 244     triggerInitialStoreRead();
 245     errorAppendEntry(http->storeEntry(), err);
 246 }
 247
 248 clientStreamNode *
 249 clientReplyContext::getNextNode() const
 250 {
 251     return (clientStreamNode *)ourNode->node.next->data;
 252 }
 253
 254 /* This function is wrong - the client parameters don't include the
 255  * header offset
 256  */
 257 void
 258 clientReplyContext::triggerInitialStoreRead()
 259 {
 260     /* when confident, 0 becomes reqofs, and then this factors into
 261      * startSendProcess
 262      */
 263     assert(reqofs == 0);
 264     StoreIOBuffer localTempBuffer (next()->readBuffer.length, 0, next()->readBuffer.data);
 265     storeClientCopy(sc, http->storeEntry(), localTempBuffer, SendMoreData, this);
 266 }
 267
 268 /* there is an expired entry in the store.
 269  * setup a temporary buffer area and perform an IMS to the origin
 270  */
 271 void
 272 clientReplyContext::processExpired()
 273 {
 274     const char *url = storeId();
 275     debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'");
 276     const time_t lastmod = http->storeEntry()->lastModified();
 277     assert(lastmod >= 0);
 278     /*
 279      * check if we are allowed to contact other servers
 280      * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return
 281      *      a stale entry *if* it matches client requirements
 282      */
 283
 284     if (http->onlyIfCached()) {
 285         processOnlyIfCachedMiss();
 286         return;
 287     }
 288
 289     http->logType = LOG_TCP_REFRESH;
 290     http->request->flags.refresh = true;
 291 #if STORE_CLIENT_LIST_DEBUG
 292     /* Prevent a race with the store client memory free routines
 293      */
 294     assert(storeClientIsThisAClient(sc, this));
 295 #endif
 296     /* Prepare to make a new temporary request */
 297     saveState();
 298
 299     // TODO: support collapsed revalidation for Vary-controlled entries
 300     const bool collapsingAllowed = Config.onoff.collapsed_forwarding &&
 301                                    !Store::Root().smpAware() &&
 302                                    http->request->vary_headers.isEmpty();
 303
 304     StoreEntry *entry = nullptr;
 305     if (collapsingAllowed) {
 306         if ((entry = storeGetPublicByRequest(http->request, ksRevalidation)))
 307             entry->lock("clientReplyContext::processExpired#alreadyRevalidating");
 308     }
 309
 310     if (entry) {
 311         entry->ensureMemObject(url, http->log_uri, http->request->method);
 312         debugs(88, 5, "collapsed on existing revalidation entry: " << *entry);
 313         collapsedRevalidation = crSlave;
 314     } else {
 315         entry = storeCreateEntry(url,
 316                                  http->log_uri, http->request->flags, http->request->method);
 317         /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */
 318
 319         if (collapsingAllowed) {
 320             debugs(88, 5, "allow other revalidation requests to collapse on " << *entry);
 321             Store::Root().allowCollapsing(entry, http->request->flags,
 322                                           http->request->method);
 323             collapsedRevalidation = crInitiator;
 324         } else {
 325             collapsedRevalidation = crNone;
 326         }
 327     }
 328
 329     sc = storeClientListAdd(entry, this);
 330 #if USE_DELAY_POOLS
 331     /* delay_id is already set on original store client */
 332     sc->setDelayId(DelayId::DelayClient(http));
 333 #endif
 334
 335     http->request->lastmod = lastmod;
 336
 337     if (!http->request->header.has(Http::HdrType::IF_NONE_MATCH)) {
 338         ETag etag = {NULL, -1}; // TODO: make that a default ETag constructor
 339         if (old_entry->hasEtag(etag) && !etag.weak)
 340             http->request->etag = etag.str;
 341     }
 342
 343     debugs(88, 5, "lastmod " << entry->lastModified());
 344     http->storeEntry(entry);
 345     assert(http->out.offset == 0);
 346     assert(http->request->clientConnectionManager == http->getConn());
 347
 348     if (collapsedRevalidation != crSlave) {
 349         /*
 350          * A refcounted pointer so that FwdState stays around as long as
 351          * this clientReplyContext does
 352          */
 353         Comm::ConnectionPointer conn = http->getConn() != NULL ? http->getConn()->clientConnection : NULL;
 354         FwdState::Start(conn, http->storeEntry(), http->request, http->al);
 355     }
 356     /* Register with storage manager to receive updates when data comes in. */
 357
 358     if (EBIT_TEST(entry->flags, ENTRY_ABORTED))
 359         debugs(88, DBG_CRITICAL, "clientReplyContext::processExpired: Found ENTRY_ABORTED object");
 360
 361     {
 362         /* start counting the length from 0 */
 363         StoreIOBuffer localTempBuffer(HTTP_REQBUF_SZ, 0, tempbuf);
 364         storeClientCopy(sc, entry, localTempBuffer, HandleIMSReply, this);
 365     }
 366 }
 367
 368 void
 369 clientReplyContext::sendClientUpstreamResponse()
 370 {
 371     StoreIOBuffer tempresult;
 372     removeStoreReference(&old_sc, &old_entry);
 373     /* here the data to send is the data we just received */
 374     tempBuffer.offset = 0;
 375     old_reqsize = 0;
 376     /* sendMoreData tracks the offset as well.
 377      * Force it back to zero */
 378     reqofs = 0;
 379     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 380     /* TODO: provide sendMoreData with the ready parsed reply */
 381     tempresult.length = reqsize;
 382     tempresult.data = tempbuf;
 383     sendMoreData(tempresult);
 384 }
 385
 386 void
 387 clientReplyContext::HandleIMSReply(void *data, StoreIOBuffer result)
 388 {
 389     clientReplyContext *context = (clientReplyContext *)data;
 390     context->handleIMSReply(result);
 391 }
 392
 393 void
 394 clientReplyContext::sendClientOldEntry()
 395 {
 396     /* Get the old request back */
 397     restoreState();
 398     /* here the data to send is in the next nodes buffers already */
 399     assert(!EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED));
 400     /* sendMoreData tracks the offset as well.
 401      * Force it back to zero */
 402     reqofs = 0;
 403     StoreIOBuffer tempresult (reqsize, reqofs, next()->readBuffer.data);
 404     sendMoreData(tempresult);
 405 }
 406
 407 /* This is the workhorse of the HandleIMSReply callback.
 408  *
 409  * It is called when we've got data back from the origin following our
 410  * IMS request to revalidate a stale entry.
 411  */
 412 void
 413 clientReplyContext::handleIMSReply(StoreIOBuffer result)
 414 {
 415     if (deleting)
 416         return;
 417
 418     debugs(88, 3, http->storeEntry()->url() << ", " << (long unsigned) result.length << " bytes");
 419
 420     if (http->storeEntry() == NULL)
 421         return;
 422
 423     if (result.flags.error && !EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED))
 424         return;
 425
 426     if (collapsedRevalidation == crSlave && !http->storeEntry()->mayStartHitting()) {
 427         debugs(88, 3, "CF slave hit private non-shareable " << *http->storeEntry() << ". MISS");
 428         // restore context to meet processMiss() expectations
 429         restoreState();
 430         http->logType = LOG_TCP_MISS;
 431         processMiss();
 432         return;
 433     }
 434
 435     /* update size of the request */
 436     reqsize = result.length + reqofs;
 437
 438     const Http::StatusCode status = http->storeEntry()->getReply()->sline.status();
 439
 440     // request to origin was aborted
 441     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
 442         debugs(88, 3, "request to origin aborted '" << http->storeEntry()->url() << "', sending old entry to client");
 443         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 444         sendClientOldEntry();
 445     }
 446
 447     const HttpReply *old_rep = old_entry->getReply();
 448
 449     // origin replied 304
 450     if (status == Http::scNotModified) {
 451         http->logType = LOG_TCP_REFRESH_UNMODIFIED;
 452         http->request->flags.staleIfHit = false; // old_entry is no longer stale
 453
 454         // update headers on existing entry
 455         Store::Root().updateOnNotModified(old_entry, *http->storeEntry());
 456
 457         // if client sent IMS
 458
 459         if (http->request->flags.ims && !old_entry->modifiedSince(http->request->ims, http->request->imslen)) {
 460             // forward the 304 from origin
 461             debugs(88, 3, "origin replied 304, revalidating existing entry and forwarding 304 to client");
 462             sendClientUpstreamResponse();
 463         } else {
 464             // send existing entry, it's still valid
 465             debugs(88, 3, "origin replied 304, revalidating existing entry and sending " <<
 466                    old_rep->sline.status() << " to client");
 467             sendClientOldEntry();
 468         }
 469     }
 470
 471     // origin replied with a non-error code
 472     else if (status > Http::scNone && status < Http::scInternalServerError) {
 473         const HttpReply *new_rep = http->storeEntry()->getReply();
 474         // RFC 7234 section 4: a cache MUST use the most recent response
 475         // (as determined by the Date header field)
 476         if (new_rep->olderThan(old_rep)) {
 477             http->logType.err.ignored = true;
 478             debugs(88, 3, "origin replied " << status <<
 479                    " but with an older date header, sending old entry (" <<
 480                    old_rep->sline.status() << ") to client");
 481             sendClientOldEntry();
 482         } else {
 483             http->logType = LOG_TCP_REFRESH_MODIFIED;
 484             debugs(88, 3, "origin replied " << status <<
 485                    ", replacing existing entry and forwarding to client");
 486
 487             if (collapsedRevalidation)
 488                 http->storeEntry()->clearPublicKeyScope();
 489
 490             sendClientUpstreamResponse();
 491         }
 492     }
 493
 494     // origin replied with an error
 495     else if (http->request->flags.failOnValidationError) {
 496         http->logType = LOG_TCP_REFRESH_FAIL_ERR;
 497         debugs(88, 3, "origin replied with error " << status <<
 498                ", forwarding to client due to fail_on_validation_err");
 499         sendClientUpstreamResponse();
 500     } else {
 501         // ignore and let client have old entry
 502         http->logType = LOG_TCP_REFRESH_FAIL_OLD;
 503         debugs(88, 3, "origin replied with error " <<
 504                status << ", sending old entry (" << old_rep->sline.status() << ") to client");
 505         sendClientOldEntry();
 506     }
 507 }
 508
 509 SQUIDCEXTERN CSR clientGetMoreData;
 510 SQUIDCEXTERN CSD clientReplyDetach;
 511
 512 /**
 513  * clientReplyContext::cacheHit Should only be called until the HTTP reply headers
 514  * have been parsed.  Normally this should be a single call, but
 515  * it might take more than one.  As soon as we have the headers,
 516  * we hand off to clientSendMoreData, processExpired, or
 517  * processMiss.
 518  */
 519 void
 520 clientReplyContext::CacheHit(void *data, StoreIOBuffer result)
 521 {
 522     clientReplyContext *context = (clientReplyContext *)data;
 523     context->cacheHit(result);
 524 }
 525
 526 /**
 527  * Process a possible cache HIT.
 528  */
 529 void
 530 clientReplyContext::cacheHit(StoreIOBuffer result)
 531 {
 532     /** Ignore if the HIT object is being deleted. */
 533     if (deleting) {
 534         debugs(88, 3, "HIT object being deleted. Ignore the HIT.");
 535         return;
 536     }
 537
 538     StoreEntry *e = http->storeEntry();
 539
 540     HttpRequest *r = http->request;
 541
 542     debugs(88, 3, "clientCacheHit: " << http->uri << ", " << result.length << " bytes");
 543
 544     if (http->storeEntry() == NULL) {
 545         debugs(88, 3, "clientCacheHit: request aborted");
 546         return;
 547     } else if (result.flags.error) {
 548         /* swap in failure */
 549         debugs(88, 3, "clientCacheHit: swapin failure for " << http->uri);
 550         http->logType = LOG_TCP_SWAPFAIL_MISS;
 551         removeClientStoreReference(&sc, http);
 552         processMiss();
 553         return;
 554     }
 555
 556     // The previously identified hit suddenly became unsharable!
 557     // This is common for collapsed forwarding slaves but might also
 558     // happen to regular hits because we are called asynchronously.
 559     if (!e->mayStartHitting()) {
 560         debugs(88, 3, "unsharable " << *e << ". MISS");
 561         http->logType = LOG_TCP_MISS;
 562         processMiss();
 563         return;
 564     }
 565
 566     if (result.length == 0) {
 567         debugs(88, 5, "store IO buffer has no content. MISS");
 568         /* the store couldn't get enough data from the file for us to id the
 569          * object
 570          */
 571         /* treat as a miss */
 572         http->logType = LOG_TCP_MISS;
 573         processMiss();
 574         return;
 575     }
 576
 577     assert(!EBIT_TEST(e->flags, ENTRY_ABORTED));
 578     /* update size of the request */
 579     reqsize = result.length + reqofs;
 580
 581     /*
 582      * Got the headers, now grok them
 583      */
 584     assert(http->logType.oldType == LOG_TCP_HIT);
 585
 586     if (http->request->storeId().cmp(e->mem_obj->storeId()) != 0) {
 587         debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->storeId() << "' != '" << http->request->storeId() << "'");
 588         http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
 589         processMiss();
 590         return;
 591     }
 592
 593     switch (varyEvaluateMatch(e, r)) {
 594
 595     case VARY_NONE:
 596         /* No variance detected. Continue as normal */
 597         break;
 598
 599     case VARY_MATCH:
 600         /* This is the correct entity for this request. Continue */
 601         debugs(88, 2, "clientProcessHit: Vary MATCH!");
 602         break;
 603
 604     case VARY_OTHER:
 605         /* This is not the correct entity for this request. We need
 606          * to requery the cache.
 607          */
 608         removeClientStoreReference(&sc, http);
 609         e = NULL;
 610         /* Note: varyEvalyateMatch updates the request with vary information
 611          * so we only get here once. (it also takes care of cancelling loops)
 612          */
 613         debugs(88, 2, "clientProcessHit: Vary detected!");
 614         clientGetMoreData(ourNode, http);
 615         return;
 616
 617     case VARY_CANCEL:
 618         /* varyEvaluateMatch found a object loop. Process as miss */
 619         debugs(88, DBG_IMPORTANT, "clientProcessHit: Vary object loop!");
 620         http->logType = LOG_TCP_MISS; // we lack a more precise LOG_*_MISS code
 621         processMiss();
 622         return;
 623     }
 624
 625     if (r->method == Http::METHOD_PURGE) {
 626         debugs(88, 5, "PURGE gets a HIT");
 627         removeClientStoreReference(&sc, http);
 628         e = NULL;
 629         purgeRequest();
 630         return;
 631     }
 632
 633     if (e->checkNegativeHit() && !r->flags.noCacheHack()) {
 634         debugs(88, 5, "negative-HIT");
 635         http->logType = LOG_TCP_NEGATIVE_HIT;
 636         sendMoreData(result);
 637         return;
 638     } else if (blockedHit()) {
 639         debugs(88, 5, "send_hit forces a MISS");
 640         http->logType = LOG_TCP_MISS;
 641         processMiss();
 642         return;
 643     } else if (!http->flags.internal && refreshCheckHTTP(e, r)) {
 644         debugs(88, 5, "clientCacheHit: in refreshCheck() block");
 645         /*
 646          * We hold a stale copy; it needs to be validated
 647          */
 648         /*
 649          * The 'needValidation' flag is used to prevent forwarding
 650          * loops between siblings.  If our copy of the object is stale,
 651          * then we should probably only use parents for the validation
 652          * request.  Otherwise two siblings could generate a loop if
 653          * both have a stale version of the object.
 654          */
 655         r->flags.needValidation = true;
 656
 657         if (e->lastModified() < 0) {
 658             debugs(88, 3, "validate HIT object? NO. Can't calculate entry modification time. Do MISS.");
 659             /*
 660              * We cannot revalidate entries without knowing their
 661              * modification time.
 662              * XXX: BUG 1890 objects without Date do not get one added.
 663              */
 664             http->logType = LOG_TCP_MISS;
 665             processMiss();
 666         } else if (r->flags.noCache) {
 667             debugs(88, 3, "validate HIT object? NO. Client sent CC:no-cache. Do CLIENT_REFRESH_MISS");
 668             /*
 669              * This did not match a refresh pattern that overrides no-cache
 670              * we should honour the client no-cache header.
 671              */
 672             http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
 673             processMiss();
 674         } else if (r->url.getScheme() == AnyP::PROTO_HTTP || r->url.getScheme() == AnyP::PROTO_HTTPS) {
 675             debugs(88, 3, "validate HIT object? YES.");
 676             /*
 677              * Object needs to be revalidated
 678              * XXX This could apply to FTP as well, if Last-Modified is known.
 679              */
 680             processExpired();
 681         } else {
 682             debugs(88, 3, "validate HIT object? NO. Client protocol non-HTTP. Do MISS.");
 683             /*
 684              * We don't know how to re-validate other protocols. Handle
 685              * them as if the object has expired.
 686              */
 687             http->logType = LOG_TCP_MISS;
 688             processMiss();
 689         }
 690         return;
 691     } else if (r->conditional()) {
 692         debugs(88, 5, "conditional HIT");
 693         if (processConditional(result))
 694             return;
 695     }
 696
 697     /*
 698      * plain ol' cache hit
 699      */
 700     debugs(88, 5, "plain old HIT");
 701
 702 #if USE_DELAY_POOLS
 703     if (e->store_status != STORE_OK)
 704         http->logType = LOG_TCP_MISS;
 705     else
 706 #endif
 707         if (e->mem_status == IN_MEMORY)
 708             http->logType = LOG_TCP_MEM_HIT;
 709         else if (Config.onoff.offline)
 710             http->logType = LOG_TCP_OFFLINE_HIT;
 711
 712     sendMoreData(result);
 713 }
 714
 715 /**
 716  * Prepare to fetch the object as it's a cache miss of some kind.
 717  */
 718 void
 719 clientReplyContext::processMiss()
 720 {
 721     char *url = http->uri;
 722     HttpRequest *r = http->request;
 723     ErrorState *err = NULL;
 724     debugs(88, 4, r->method << ' ' << url);
 725
 726     /**
 727      * We might have a left-over StoreEntry from a failed cache hit
 728      * or IMS request.
 729      */
 730     if (http->storeEntry()) {
 731         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
 732             debugs(88, DBG_CRITICAL, "clientProcessMiss: miss on a special object (" << url << ").");
 733             debugs(88, DBG_CRITICAL, "\tlog_type = " << http->logType.c_str());
 734             http->storeEntry()->dump(1);
 735         }
 736
 737         removeClientStoreReference(&sc, http);
 738     }
 739
 740     /** Check if its a PURGE request to be actioned. */
 741     if (r->method == Http::METHOD_PURGE) {
 742         purgeRequest();
 743         return;
 744     }
 745
 746     /** Check if its an 'OTHER' request. Purge all cached entries if so and continue. */
 747     if (r->method == Http::METHOD_OTHER) {
 748         purgeAllCached();
 749     }
 750
 751     /** Check if 'only-if-cached' flag is set. Action if so. */
 752     if (http->onlyIfCached()) {
 753         processOnlyIfCachedMiss();
 754         return;
 755     }
 756
 757     Comm::ConnectionPointer conn = http->getConn() != nullptr ? http->getConn()->clientConnection : nullptr;
 758     /// Deny loops
 759     if (r->flags.loopDetected) {
 760         http->al->http.code = Http::scForbidden;
 761         Ip::Address tmp_noaddr;
 762         tmp_noaddr.setNoAddr();
 763         err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, nullptr, conn ? conn->remote : tmp_noaddr, http->request);
 764         createStoreEntry(r->method, RequestFlags());
 765         errorAppendEntry(http->storeEntry(), err);
 766         triggerInitialStoreRead();
 767         return;
 768     } else {
 769         assert(http->out.offset == 0);
 770         createStoreEntry(r->method, r->flags);
 771         triggerInitialStoreRead();
 772
 773         if (http->redirect.status) {
 774             HttpReply *rep = new HttpReply;
 775             http->logType = LOG_TCP_REDIRECT;
 776             http->storeEntry()->releaseRequest();
 777             rep->redirect(http->redirect.status, http->redirect.location);
 778             http->storeEntry()->replaceHttpReply(rep);
 779             http->storeEntry()->complete();
 780             return;
 781         }
 782
 783         assert(r->clientConnectionManager == http->getConn());
 784
 785         /** Start forwarding to get the new object from network */
 786         FwdState::Start(conn, http->storeEntry(), r, http->al);
 787     }
 788 }
 789
 790 /**
 791  * client issued a request with an only-if-cached cache-control directive;
 792  * we did not find a cached object that can be returned without
 793  *     contacting other servers;
 794  * respond with a 504 (Gateway Timeout) as suggested in [RFC 2068]
 795  */
 796 void
 797 clientReplyContext::processOnlyIfCachedMiss()
 798 {
 799     debugs(88, 4, http->request->method << ' ' << http->uri);
 800     http->al->http.code = Http::scGatewayTimeout;
 801     Ip::Address tmp_noaddr;
 802     tmp_noaddr.setNoAddr();
 803     ErrorState *err = clientBuildError(ERR_ONLY_IF_CACHED_MISS, Http::scGatewayTimeout, NULL,
 804                                        http->getConn() ? http->getConn()->clientConnection->remote : tmp_noaddr,
 805                                        http->request);
 806     removeClientStoreReference(&sc, http);
 807     startError(err);
 808 }
 809
 810 /// process conditional request from client
 811 bool
 812 clientReplyContext::processConditional(StoreIOBuffer &result)
 813 {
 814     StoreEntry *const e = http->storeEntry();
 815
 816     if (e->getReply()->sline.status() != Http::scOkay) {
 817         debugs(88, 4, "Reply code " << e->getReply()->sline.status() << " != 200");
 818         http->logType = LOG_TCP_MISS;
 819         processMiss();
 820         return true;
 821     }
 822
 823     HttpRequest &r = *http->request;
 824
 825     if (r.header.has(Http::HdrType::IF_MATCH) && !e->hasIfMatchEtag(r)) {
 826         // RFC 2616: reply with 412 Precondition Failed if If-Match did not match
 827         sendPreconditionFailedError();
 828         return true;
 829     }
 830
 831     if (r.header.has(Http::HdrType::IF_NONE_MATCH)) {
 832         // RFC 7232: If-None-Match recipient MUST ignore IMS
 833         r.flags.ims = false;
 834         r.ims = -1;
 835         r.imslen = 0;
 836         r.header.delById(Http::HdrType::IF_MODIFIED_SINCE);
 837
 838         if (e->hasIfNoneMatchEtag(r)) {
 839             sendNotModifiedOrPreconditionFailedError();
 840             return true;
 841         }
 842
 843         // None-Match is true (no ETag matched); treat as an unconditional hit
 844         return false;
 845     }
 846
 847     if (r.flags.ims) {
 848         // handle If-Modified-Since requests from the client
 849         if (e->modifiedSince(r.ims, r.imslen)) {
 850             // Modified-Since is true; treat as an unconditional hit
 851             return false;
 852
 853         } else {
 854             // otherwise reply with 304 Not Modified
 855             sendNotModified();
 856         }
 857         return true;
 858     }
 859
 860     return false;
 861 }
 862
 863 /// whether squid.conf send_hit prevents us from serving this hit
 864 bool
 865 clientReplyContext::blockedHit() const
 866 {
 867     if (!Config.accessList.sendHit)
 868         return false; // hits are not blocked by default
 869
 870     if (http->flags.internal)
 871         return false; // internal content "hits" cannot be blocked
 872
 873     if (const HttpReply *rep = http->storeEntry()->getReply()) {
 874         std::unique_ptr<ACLFilledChecklist> chl(clientAclChecklistCreate(Config.accessList.sendHit, http));
 875         chl->reply = const_cast<HttpReply*>(rep); // ACLChecklist API bug
 876         HTTPMSGLOCK(chl->reply);
 877         return !chl->fastCheck().allowed(); // when in doubt, block
 878     }
 879
 880     // This does not happen, I hope, because we are called from CacheHit, which
 881     // is called via a storeClientCopy() callback, and store should initialize
 882     // the reply before calling that callback.
 883     debugs(88, 3, "Missing reply!");
 884     return false;
 885 }
 886
 887 void
 888 clientReplyContext::purgeRequestFindObjectToPurge()
 889 {
 890     /* Try to find a base entry */
 891     http->flags.purging = true;
 892     lookingforstore = 1;
 893
 894     // TODO: can we use purgeAllCached() here instead of doing the
 895     // getPublicByRequestMethod() dance?
 896     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_GET);
 897 }
 898
 899 // Purges all entries with a given url
 900 // TODO: move to SideAgent parent, when we have one
 901 /*
 902  * We probably cannot purge Vary-affected responses because their MD5
 903  * keys depend on vary headers.
 904  */
 905 void
 906 purgeEntriesByUrl(HttpRequest * req, const char *url)
 907 {
 908 #if USE_HTCP
 909     bool get_or_head_sent = false;
 910 #endif
 911
 912     for (HttpRequestMethod m(Http::METHOD_NONE); m != Http::METHOD_ENUM_END; ++m) {
 913         if (m.respMaybeCacheable()) {
 914             if (StoreEntry *entry = storeGetPublic(url, m)) {
 915                 debugs(88, 5, "purging " << *entry << ' ' << m << ' ' << url);
 916 #if USE_HTCP
 917                 neighborsHtcpClear(entry, url, req, m, HTCP_CLR_INVALIDATION);
 918                 if (m == Http::METHOD_GET || m == Http::METHOD_HEAD) {
 919                     get_or_head_sent = true;
 920                 }
 921 #endif
 922                 entry->release();
 923             }
 924         }
 925     }
 926
 927 #if USE_HTCP
 928     if (!get_or_head_sent) {
 929         neighborsHtcpClear(NULL, url, req, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_INVALIDATION);
 930     }
 931 #endif
 932 }
 933
 934 void
 935 clientReplyContext::purgeAllCached()
 936 {
 937     // XXX: performance regression, c_str() reallocates
 938     SBuf url(http->request->effectiveRequestUri());
 939     purgeEntriesByUrl(http->request, url.c_str());
 940 }
 941
 942 void
 943 clientReplyContext::created(StoreEntry *newEntry)
 944 {
 945     if (lookingforstore == 1)
 946         purgeFoundGet(newEntry);
 947     else if (lookingforstore == 2)
 948         purgeFoundHead(newEntry);
 949     else if (lookingforstore == 3)
 950         purgeDoPurgeGet(newEntry);
 951     else if (lookingforstore == 4)
 952         purgeDoPurgeHead(newEntry);
 953     else if (lookingforstore == 5)
 954         identifyFoundObject(newEntry);
 955 }
 956
 957 void
 958 clientReplyContext::purgeFoundGet(StoreEntry *newEntry)
 959 {
 960     if (newEntry->isNull()) {
 961         lookingforstore = 2;
 962         StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
 963     } else
 964         purgeFoundObject (newEntry);
 965 }
 966
 967 void
 968 clientReplyContext::purgeFoundHead(StoreEntry *newEntry)
 969 {
 970     if (newEntry->isNull())
 971         purgeDoMissPurge();
 972     else
 973         purgeFoundObject (newEntry);
 974 }
 975
 976 void
 977 clientReplyContext::purgeFoundObject(StoreEntry *entry)
 978 {
 979     assert (entry && !entry->isNull());
 980
 981     if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) {
 982         http->logType = LOG_TCP_DENIED;
 983         Ip::Address tmp_noaddr;
 984         tmp_noaddr.setNoAddr(); // TODO: make a global const
 985         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL,
 986                                            http->getConn() ? http->getConn()->clientConnection->remote : tmp_noaddr,
 987                                            http->request);
 988         startError(err);
 989         return; // XXX: leaking unused entry if some store does not keep it
 990     }
 991
 992     StoreIOBuffer localTempBuffer;
 993     /* Swap in the metadata */
 994     http->storeEntry(entry);
 995
 996     http->storeEntry()->lock("clientReplyContext::purgeFoundObject");
 997     http->storeEntry()->ensureMemObject(storeId(), http->log_uri,
 998                                         http->request->method);
 999
1000     sc = storeClientListAdd(http->storeEntry(), this);
1001
1002     http->logType = LOG_TCP_HIT;
1003
1004     reqofs = 0;
1005
1006     localTempBuffer.offset = http->out.offset;
1007
1008     localTempBuffer.length = next()->readBuffer.length;
1009
1010     localTempBuffer.data = next()->readBuffer.data;
1011
1012     storeClientCopy(sc, http->storeEntry(),
1013                     localTempBuffer, CacheHit, this);
1014 }
1015
1016 void
1017 clientReplyContext::purgeRequest()
1018 {
1019     debugs(88, 3, "Config2.onoff.enable_purge = " <<
1020            Config2.onoff.enable_purge);
1021
1022     if (!Config2.onoff.enable_purge) {
1023         http->logType = LOG_TCP_DENIED;
1024         Ip::Address tmp_noaddr;
1025         tmp_noaddr.setNoAddr();
1026         ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, Http::scForbidden, NULL,
1027                                            http->getConn() ? http->getConn()->clientConnection->remote : tmp_noaddr, http->request);
1028         startError(err);
1029         return;
1030     }
1031
1032     /* Release both IP cache */
1033     ipcacheInvalidate(http->request->url.host());
1034
1035     if (!http->flags.purging)
1036         purgeRequestFindObjectToPurge();
1037     else
1038         purgeDoMissPurge();
1039 }
1040
1041 void
1042 clientReplyContext::purgeDoMissPurge()
1043 {
1044     http->logType = LOG_TCP_MISS;
1045     lookingforstore = 3;
1046     StoreEntry::getPublicByRequestMethod(this,http->request, Http::METHOD_GET);
1047 }
1048
1049 void
1050 clientReplyContext::purgeDoPurgeGet(StoreEntry *newEntry)
1051 {
1052     assert (newEntry);
1053     /* Move to new() when that is created */
1054     purgeStatus = Http::scNotFound;
1055
1056     if (!newEntry->isNull()) {
1057         /* Release the cached URI */
1058         debugs(88, 4, "clientPurgeRequest: GET '" << newEntry->url() << "'" );
1059 #if USE_HTCP
1060         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1061 #endif
1062         newEntry->release();
1063         purgeStatus = Http::scOkay;
1064     }
1065
1066     lookingforstore = 4;
1067     StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_HEAD);
1068 }
1069
1070 void
1071 clientReplyContext::purgeDoPurgeHead(StoreEntry *newEntry)
1072 {
1073     if (newEntry && !newEntry->isNull()) {
1074         debugs(88, 4, "HEAD " << newEntry->url());
1075 #if USE_HTCP
1076         neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1077 #endif
1078         newEntry->release();
1079         purgeStatus = Http::scOkay;
1080     }
1081
1082     /* And for Vary, release the base URI if none of the headers was included in the request */
1083     if (!http->request->vary_headers.isEmpty()
1084             && http->request->vary_headers.find('=') != SBuf::npos) {
1085         // XXX: performance regression, c_str() reallocates
1086         SBuf tmp(http->request->effectiveRequestUri());
1087         StoreEntry *entry = storeGetPublic(tmp.c_str(), Http::METHOD_GET);
1088
1089         if (entry) {
1090             debugs(88, 4, "Vary GET " << entry->url());
1091 #if USE_HTCP
1092             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE);
1093 #endif
1094             entry->release();
1095             purgeStatus = Http::scOkay;
1096         }
1097
1098         entry = storeGetPublic(tmp.c_str(), Http::METHOD_HEAD);
1099
1100         if (entry) {
1101             debugs(88, 4, "Vary HEAD " << entry->url());
1102 #if USE_HTCP
1103             neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE);
1104 #endif
1105             entry->release();
1106             purgeStatus = Http::scOkay;
1107         }
1108     }
1109
1110     /*
1111      * Make a new entry to hold the reply to be written
1112      * to the client.
1113      */
1114     /* FIXME: This doesn't need to go through the store. Simply
1115      * push down the client chain
1116      */
1117     createStoreEntry(http->request->method, RequestFlags());
1118
1119     triggerInitialStoreRead();
1120
1121     HttpReply *rep = new HttpReply;
1122     rep->setHeaders(purgeStatus, NULL, NULL, 0, 0, -1);
1123     http->storeEntry()->replaceHttpReply(rep);
1124     http->storeEntry()->complete();
1125 }
1126
1127 void
1128 clientReplyContext::traceReply(clientStreamNode * node)
1129 {
1130     clientStreamNode *nextNode = (clientStreamNode *)node->node.next->data;
1131     StoreIOBuffer localTempBuffer;
1132     createStoreEntry(http->request->method, RequestFlags());
1133     localTempBuffer.offset = nextNode->readBuffer.offset + headers_sz;
1134     localTempBuffer.length = nextNode->readBuffer.length;
1135     localTempBuffer.data = nextNode->readBuffer.data;
1136     storeClientCopy(sc, http->storeEntry(),
1137                     localTempBuffer, SendMoreData, this);
1138     http->storeEntry()->releaseRequest();
1139     http->storeEntry()->buffer();
1140     HttpReply *rep = new HttpReply;
1141     rep->setHeaders(Http::scOkay, NULL, "text/plain", http->request->prefixLen(), 0, squid_curtime);
1142     http->storeEntry()->replaceHttpReply(rep);
1143     http->request->swapOut(http->storeEntry());
1144     http->storeEntry()->complete();
1145 }
1146
1147 #define SENDING_BODY 0
1148 #define SENDING_HDRSONLY 1
1149 int
1150 clientReplyContext::checkTransferDone()
1151 {
1152     StoreEntry *entry = http->storeEntry();
1153
1154     if (entry == NULL)
1155         return 0;
1156
1157     /*
1158      * For now, 'done_copying' is used for special cases like
1159      * Range and HEAD requests.
1160      */
1161     if (http->flags.done_copying)
1162         return 1;
1163
1164     if (http->request->flags.chunkedReply && !flags.complete) {
1165         // last-chunk was not sent
1166         return 0;
1167     }
1168
1169     /*
1170      * Handle STORE_OK objects.
1171      * objectLen(entry) will be set proprely.
1172      * RC: Does objectLen(entry) include the Headers?
1173      * RC: Yes.
1174      */
1175     if (entry->store_status == STORE_OK) {
1176         return storeOKTransferDone();
1177     } else {
1178         return storeNotOKTransferDone();
1179     }
1180 }
1181
1182 int
1183 clientReplyContext::storeOKTransferDone() const
1184 {
1185     assert(http->storeEntry()->objectLen() >= 0);
1186     assert(http->storeEntry()->objectLen() >= headers_sz);
1187     if (http->out.offset >= http->storeEntry()->objectLen() - headers_sz) {
1188         debugs(88,3,HERE << "storeOKTransferDone " <<
1189                " out.offset=" << http->out.offset <<
1190                " objectLen()=" << http->storeEntry()->objectLen() <<
1191                " headers_sz=" << headers_sz);
1192         return 1;
1193     }
1194
1195     return 0;
1196 }
1197
1198 int
1199 clientReplyContext::storeNotOKTransferDone() const
1200 {
1201     /*
1202      * Now, handle STORE_PENDING objects
1203      */
1204     MemObject *mem = http->storeEntry()->mem_obj;
1205     assert(mem != NULL);
1206     assert(http->request != NULL);
1207
1208     /* mem->reply was wrong because it uses the UPSTREAM header length!!! */
1209     if (headers_sz == 0)
1210         /* haven't found end of headers yet */
1211         return 0;
1212
1213     const HttpReplyPointer curReply(mem->getReply());
1214
1215     /*
1216      * Figure out how much data we are supposed to send.
1217      * If we are sending a body and we don't have a content-length,
1218      * then we must wait for the object to become STORE_OK.
1219      */
1220     if (curReply->content_length < 0)
1221         return 0;
1222
1223     uint64_t expectedLength = curReply->content_length + http->out.headers_sz;
1224
1225     if (http->out.size < expectedLength)
1226         return 0;
1227     else {
1228         debugs(88,3,HERE << "storeNotOKTransferDone " <<
1229                " out.size=" << http->out.size <<
1230                " expectedLength=" << expectedLength);
1231         return 1;
1232     }
1233 }
1234
1235 /* A write has completed, what is the next status based on the
1236  * canonical request data?
1237  * 1 something is wrong
1238  * 0 nothing is wrong.
1239  *
1240  */
1241 int
1242 clientHttpRequestStatus(int fd, ClientHttpRequest const *http)
1243 {
1244 #if SIZEOF_INT64_T == 4
1245     if (http->out.size > 0x7FFF0000) {
1246         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd << " to prevent out.size counter overflow");
1247         if (http->getConn())
1248             debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1249         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes");
1250         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1251         return 1;
1252     }
1253
1254     if (http->out.offset > 0x7FFF0000) {
1255         debugs(88, DBG_IMPORTANT, "WARNING: closing FD " << fd < " to prevent out.offset counter overflow");
1256         if (http->getConn())
1257             debugs(88, DBG_IMPORTANT, "\tclient " << http->getConn()->peer);
1258         debugs(88, DBG_IMPORTANT, "\treceived " << http->out.size << " bytes, offset " << http->out.offset);
1259         debugs(88, DBG_IMPORTANT, "\tURI " << http->log_uri);
1260         return 1;
1261     }
1262
1263 #endif
1264     return 0;
1265 }
1266
1267 /* Preconditions:
1268  * *http is a valid structure.
1269  * fd is either -1, or an open fd.
1270  *
1271  * TODO: enumify this
1272  *
1273  * This function is used by any http request sink, to determine the status
1274  * of the object.
1275  */
1276 clientStream_status_t
1277 clientReplyStatus(clientStreamNode * aNode, ClientHttpRequest * http)
1278 {
1279     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1280     assert (context);
1281     assert (context->http == http);
1282     return context->replyStatus();
1283 }
1284
1285 clientStream_status_t
1286 clientReplyContext::replyStatus()
1287 {
1288     int done;
1289     /* Here because lower nodes don't need it */
1290
1291     if (http->storeEntry() == NULL) {
1292         debugs(88, 5, "clientReplyStatus: no storeEntry");
1293         return STREAM_FAILED;   /* yuck, but what can we do? */
1294     }
1295
1296     if (EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) {
1297         /* TODO: Could upstream read errors (result.flags.error) be
1298          * lost, and result in undersize requests being considered
1299          * complete. Should we tcp reset such connections ?
1300          */
1301         debugs(88, 5, "clientReplyStatus: aborted storeEntry");
1302         return STREAM_FAILED;
1303     }
1304
1305     if ((done = checkTransferDone()) != 0 || flags.complete) {
1306         debugs(88, 5, "clientReplyStatus: transfer is DONE: " << done << flags.complete);
1307         /* Ok we're finished, but how? */
1308
1309         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_BAD_LENGTH)) {
1310             debugs(88, 5, "clientReplyStatus: truncated response body");
1311             return STREAM_UNPLANNED_COMPLETE;
1312         }
1313
1314         if (!done) {
1315             debugs(88, 5, "clientReplyStatus: closing, !done, but read 0 bytes");
1316             return STREAM_FAILED;
1317         }
1318
1319         const int64_t expectedBodySize =
1320             http->storeEntry()->getReply()->bodySize(http->request->method);
1321         if (expectedBodySize >= 0 && !http->gotEnough()) {
1322             debugs(88, 5, "clientReplyStatus: client didn't get all it expected");
1323             return STREAM_UNPLANNED_COMPLETE;
1324         }
1325
1326         debugs(88, 5, "clientReplyStatus: stream complete; keepalive=" <<
1327                http->request->flags.proxyKeepalive);
1328         return STREAM_COMPLETE;
1329     }
1330
1331     // XXX: Should this be checked earlier? We could return above w/o checking.
1332     if (reply->receivedBodyTooLarge(*http->request, http->out.offset - 4096)) {
1333         /* 4096 is a margin for the HTTP headers included in out.offset */
1334         debugs(88, 5, "clientReplyStatus: client reply body is too large");
1335         return STREAM_FAILED;
1336     }
1337
1338     return STREAM_NONE;
1339 }
1340
1341 /* Responses with no body will not have a content-type header,
1342  * which breaks the rep_mime_type acl, which
1343  * coincidentally, is the most common acl for reply access lists.
1344  * A better long term fix for this is to allow acl matchs on the various
1345  * status codes, and then supply a default ruleset that puts these
1346  * codes before any user defines access entries. That way the user
1347  * can choose to block these responses where appropriate, but won't get
1348  * mysterious breakages.
1349  */
1350 bool
1351 clientReplyContext::alwaysAllowResponse(Http::StatusCode sline) const
1352 {
1353     bool result;
1354
1355     switch (sline) {
1356
1357     case Http::scContinue:
1358
1359     case Http::scSwitchingProtocols:
1360
1361     case Http::scProcessing:
1362
1363     case Http::scNoContent:
1364
1365     case Http::scNotModified:
1366         result = true;
1367         break;
1368
1369     default:
1370         result = false;
1371     }
1372
1373     return result;
1374 }
1375
1376 /**
1377  * Generate the reply headers sent to client.
1378  *
1379  * Filters out unwanted entries and hop-by-hop from original reply header
1380  * then adds extra entries if we have more info than origin server
1381  * then adds Squid specific entries
1382  */
1383 void
1384 clientReplyContext::buildReplyHeader()
1385 {
1386     HttpHeader *hdr = &reply->header;
1387     const bool is_hit = http->logType.isTcpHit();
1388     HttpRequest *request = http->request;
1389 #if DONT_FILTER_THESE
1390     /* but you might want to if you run Squid as an HTTP accelerator */
1391     /* hdr->delById(HDR_ACCEPT_RANGES); */
1392     hdr->delById(HDR_ETAG);
1393 #endif
1394
1395     if (is_hit || collapsedRevalidation == crSlave)
1396         hdr->delById(Http::HdrType::SET_COOKIE);
1397     // TODO: RFC 2965 : Must honour Cache-Control: no-cache="set-cookie2" and remove header.
1398
1399     // if there is not configured a peer proxy with login=PASS or login=PASSTHRU option enabled
1400     // remove the Proxy-Authenticate header
1401     if ( !request->peer_login || (strcmp(request->peer_login,"PASS") != 0 && strcmp(request->peer_login,"PASSTHRU") != 0)) {
1402 #if USE_ADAPTATION
1403         // but allow adaptation services to authenticate clients
1404         // via request satisfaction
1405         if (!http->requestSatisfactionMode())
1406 #endif
1407             reply->header.delById(Http::HdrType::PROXY_AUTHENTICATE);
1408     }
1409
1410     reply->header.removeHopByHopEntries();
1411
1412     //    if (request->range)
1413     //      clientBuildRangeHeader(http, reply);
1414
1415     /*
1416      * Add a estimated Age header on cache hits.
1417      */
1418     if (is_hit) {
1419         /*
1420          * Remove any existing Age header sent by upstream caches
1421          * (note that the existing header is passed along unmodified
1422          * on cache misses)
1423          */
1424         hdr->delById(Http::HdrType::AGE);
1425         /*
1426          * This adds the calculated object age. Note that the details of the
1427          * age calculation is performed by adjusting the timestamp in
1428          * StoreEntry::timestampsSet(), not here.
1429          */
1430         if (EBIT_TEST(http->storeEntry()->flags, ENTRY_SPECIAL)) {
1431             hdr->delById(Http::HdrType::DATE);
1432             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1433         } else if (http->getConn() && http->getConn()->port->actAsOrigin) {
1434             // Swap the Date: header to current time if we are simulating an origin
1435             HttpHeaderEntry *h = hdr->findEntry(Http::HdrType::DATE);
1436             if (h)
1437                 hdr->putExt("X-Origin-Date", h->value.termedBuf());
1438             hdr->delById(Http::HdrType::DATE);
1439             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1440             h = hdr->findEntry(Http::HdrType::EXPIRES);
1441             if (h && http->storeEntry()->expires >= 0) {
1442                 hdr->putExt("X-Origin-Expires", h->value.termedBuf());
1443                 hdr->delById(Http::HdrType::EXPIRES);
1444                 hdr->putTime(Http::HdrType::EXPIRES, squid_curtime + http->storeEntry()->expires - http->storeEntry()->timestamp);
1445             }
1446             if (http->storeEntry()->timestamp <= squid_curtime) {
1447                 // put X-Cache-Age: instead of Age:
1448                 char age[64];
1449                 snprintf(age, sizeof(age), "%" PRId64, static_cast<int64_t>(squid_curtime - http->storeEntry()->timestamp));
1450                 hdr->putExt("X-Cache-Age", age);
1451             }
1452         } else if (http->storeEntry()->timestamp <= squid_curtime) {
1453             hdr->putInt(Http::HdrType::AGE,
1454                         squid_curtime - http->storeEntry()->timestamp);
1455             /* Signal old objects.  NB: rfc 2616 is not clear,
1456              * by implication, on whether we should do this to all
1457              * responses, or only cache hits.
1458              * 14.46 states it ONLY applys for heuristically caclulated
1459              * freshness values, 13.2.4 doesn't specify the same limitation.
1460              * We interpret RFC 2616 under the combination.
1461              */
1462             /* TODO: if maxage or s-maxage is present, don't do this */
1463
1464             if (squid_curtime - http->storeEntry()->timestamp >= 86400) {
1465                 char tbuf[512];
1466                 snprintf (tbuf, sizeof(tbuf), "%s %s %s",
1467                           "113", ThisCache,
1468                           "This cache hit is still fresh and more than 1 day old");
1469                 hdr->putStr(Http::HdrType::WARNING, tbuf);
1470             }
1471         }
1472     }
1473
1474     /* RFC 2616: Section 14.18
1475      *
1476      * Add a Date: header if missing.
1477      * We have access to a clock therefore are required to amend any shortcoming in servers.
1478      *
1479      * NP: done after Age: to prevent ENTRY_SPECIAL double-handling this header.
1480      */
1481     if ( !hdr->has(Http::HdrType::DATE) ) {
1482         if (!http->storeEntry())
1483             hdr->putTime(Http::HdrType::DATE, squid_curtime);
1484         else if (http->storeEntry()->timestamp > 0)
1485             hdr->putTime(Http::HdrType::DATE, http->storeEntry()->timestamp);
1486         else {
1487             debugs(88,DBG_IMPORTANT,"BUG 3279: HTTP reply without Date:");
1488             /* dump something useful about the problem */
1489             http->storeEntry()->dump(DBG_IMPORTANT);
1490         }
1491     }
1492
1493     // add Warnings required by RFC 2616 if serving a stale hit
1494     if (http->request->flags.staleIfHit && http->logType.isTcpHit()) {
1495         hdr->putWarning(110, "Response is stale");
1496         if (http->request->flags.needValidation)
1497             hdr->putWarning(111, "Revalidation failed");
1498     }
1499
1500     /* Filter unproxyable authentication types */
1501     if (http->logType.oldType != LOG_TCP_DENIED &&
1502             hdr->has(Http::HdrType::WWW_AUTHENTICATE)) {
1503         HttpHeaderPos pos = HttpHeaderInitPos;
1504         HttpHeaderEntry *e;
1505
1506         int connection_auth_blocked = 0;
1507         while ((e = hdr->getEntry(&pos))) {
1508             if (e->id == Http::HdrType::WWW_AUTHENTICATE) {
1509                 const char *value = e->value.rawBuf();
1510
1511                 if ((strncasecmp(value, "NTLM", 4) == 0 &&
1512                         (value[4] == '\0' || value[4] == ' '))
1513                         ||
1514                         (strncasecmp(value, "Negotiate", 9) == 0 &&
1515                          (value[9] == '\0' || value[9] == ' '))
1516                         ||
1517                         (strncasecmp(value, "Kerberos", 8) == 0 &&
1518                          (value[8] == '\0' || value[8] == ' '))) {
1519                     if (request->flags.connectionAuthDisabled) {
1520                         hdr->delAt(pos, connection_auth_blocked);
1521                         continue;
1522                     }
1523                     request->flags.mustKeepalive = true;
1524                     if (!request->flags.accelerated && !request->flags.intercepted) {
1525                         httpHeaderPutStrf(hdr, Http::HdrType::PROXY_SUPPORT, "Session-Based-Authentication");
1526                         /*
1527                           We send "Connection: Proxy-Support" header to mark
1528                           Proxy-Support as a hop-by-hop header for intermediaries that do not
1529                           understand the semantics of this header. The RFC should have included
1530                           this recommendation.
1531                         */
1532                         httpHeaderPutStrf(hdr, Http::HdrType::CONNECTION, "Proxy-support");
1533                     }
1534                     break;
1535                 }
1536             }
1537         }
1538
1539         if (connection_auth_blocked)
1540             hdr->refreshMask();
1541     }
1542
1543 #if USE_AUTH
1544     /* Handle authentication headers */
1545     if (http->logType.oldType == LOG_TCP_DENIED &&
1546             ( reply->sline.status() == Http::scProxyAuthenticationRequired ||
1547               reply->sline.status() == Http::scUnauthorized)
1548        ) {
1549         /* Add authentication header */
1550         /*! \todo alter errorstate to be accel on|off aware. The 0 on the next line
1551          * depends on authenticate behaviour: all schemes to date send no extra
1552          * data on 407/401 responses, and do not check the accel state on 401/407
1553          * responses
1554          */
1555         Auth::UserRequest::AddReplyAuthHeader(reply, request->auth_user_request, request, 0, 1);
1556     } else if (request->auth_user_request != NULL)
1557         Auth::UserRequest::AddReplyAuthHeader(reply, request->auth_user_request, request, http->flags.accel, 0);
1558 #endif
1559
1560     /* Append X-Cache */
1561     httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE, "%s from %s",
1562                       is_hit ? "HIT" : "MISS", getMyHostname());
1563
1564 #if USE_CACHE_DIGESTS
1565     /* Append X-Cache-Lookup: -- temporary hack, to be removed @?@ @?@ */
1566     httpHeaderPutStrf(hdr, Http::HdrType::X_CACHE_LOOKUP, "%s from %s:%d",
1567                       lookup_type ? lookup_type : "NONE",
1568                       getMyHostname(), getMyPort());
1569
1570 #endif
1571
1572     const bool maySendChunkedReply = !request->multipartRangeRequest() &&
1573                                      reply->sline.protocol == AnyP::PROTO_HTTP && // response is HTTP
1574                                      (request->http_ver >= Http::ProtocolVersion(1,1));
1575
1576     /* Check whether we should send keep-alive */
1577     if (!Config.onoff.error_pconns && reply->sline.status() >= 400 && !request->flags.mustKeepalive) {
1578         debugs(33, 3, "clientBuildReplyHeader: Error, don't keep-alive");
1579         request->flags.proxyKeepalive = false;
1580     } else if (!Config.onoff.client_pconns && !request->flags.mustKeepalive) {
1581         debugs(33, 2, "clientBuildReplyHeader: Connection Keep-Alive not requested by admin or client");
1582         request->flags.proxyKeepalive = false;
1583     } else if (request->flags.proxyKeepalive && shutting_down) {
1584         debugs(88, 3, "clientBuildReplyHeader: Shutting down, don't keep-alive.");
1585         request->flags.proxyKeepalive = false;
1586     } else if (request->flags.connectionAuth && !reply->keep_alive) {
1587         debugs(33, 2, "clientBuildReplyHeader: Connection oriented auth but server side non-persistent");
1588         request->flags.proxyKeepalive = false;
1589     } else if (reply->bodySize(request->method) < 0 && !maySendChunkedReply) {
1590         debugs(88, 3, "clientBuildReplyHeader: can't keep-alive, unknown body size" );
1591         request->flags.proxyKeepalive = false;
1592     } else if (fdUsageHigh()&& !request->flags.mustKeepalive) {
1593         debugs(88, 3, "clientBuildReplyHeader: Not many unused FDs, can't keep-alive");
1594         request->flags.proxyKeepalive = false;
1595     } else if (request->flags.sslBumped && !reply->persistent()) {
1596         // We do not really have to close, but we pretend we are a tunnel.
1597         debugs(88, 3, "clientBuildReplyHeader: bumped reply forces close");
1598         request->flags.proxyKeepalive = false;
1599     } else if (request->pinnedConnection() && !reply->persistent()) {
1600         // The peer wants to close the pinned connection
1601         debugs(88, 3, "pinned reply forces close");
1602         request->flags.proxyKeepalive = false;
1603     } else if (http->getConn()) {
1604         ConnStateData * conn = http->getConn();
1605         if (!Comm::IsConnOpen(conn->port->listenConn)) {
1606             // The listening port closed because of a reconfigure
1607             debugs(88, 3, "listening port closed");
1608             request->flags.proxyKeepalive = false;
1609         }
1610     }
1611
1612     // Decide if we send chunked reply
1613     if (maySendChunkedReply && reply->bodySize(request->method) < 0) {
1614         debugs(88, 3, "clientBuildReplyHeader: chunked reply");
1615         request->flags.chunkedReply = true;
1616         hdr->putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
1617     }
1618
1619     hdr->addVia(reply->sline.version);
1620
1621     /* Signal keep-alive or close explicitly */
1622     hdr->putStr(Http::HdrType::CONNECTION, request->flags.proxyKeepalive ? "keep-alive" : "close");
1623
1624 #if ADD_X_REQUEST_URI
1625     /*
1626      * Knowing the URI of the request is useful when debugging persistent
1627      * connections in a client; we cannot guarantee the order of http headers,
1628      * but X-Request-URI is likely to be the very last header to ease use from a
1629      * debugger [hdr->entries.count-1].
1630      */
1631     hdr->putStr(Http::HdrType::X_REQUEST_URI,
1632                 http->memOjbect()->url ? http->memObject()->url : http->uri);
1633
1634 #endif
1635
1636     /* Surrogate-Control requires Surrogate-Capability from upstream to pass on */
1637     if ( hdr->has(Http::HdrType::SURROGATE_CONTROL) ) {
1638         if (!request->header.has(Http::HdrType::SURROGATE_CAPABILITY)) {
1639             hdr->delById(Http::HdrType::SURROGATE_CONTROL);
1640         }
1641         /* TODO: else case: drop any controls intended specifically for our surrogate ID */
1642     }
1643
1644     httpHdrMangleList(hdr, request, http->al, ROR_REPLY);
1645 }
1646
1647 void
1648 clientReplyContext::cloneReply()
1649 {
1650     assert(reply == NULL);
1651
1652     reply = http->storeEntry()->getReply()->clone();
1653     HTTPMSGLOCK(reply);
1654
1655     if (reply->sline.protocol == AnyP::PROTO_HTTP) {
1656         /* RFC 2616 requires us to advertise our version (but only on real HTTP traffic) */
1657         reply->sline.version = Http::ProtocolVersion();
1658     }
1659
1660     /* do header conversions */
1661     buildReplyHeader();
1662 }
1663
1664 /// Safely disposes of an entry pointing to a cache hit that we do not want.
1665 /// We cannot just ignore the entry because it may be locking or otherwise
1666 /// holding an associated cache resource of some sort.
1667 void
1668 clientReplyContext::forgetHit()
1669 {
1670     StoreEntry *e = http->storeEntry();
1671     assert(e); // or we are not dealing with a hit
1672     // We probably have not locked the entry earlier, unfortunately. We lock it
1673     // now so that we can unlock two lines later (and trigger cleanup).
1674     // Ideally, ClientHttpRequest::storeEntry() should lock/unlock, but it is
1675     // used so inconsistently that simply adding locking there leads to bugs.
1676     e->lock("clientReplyContext::forgetHit");
1677     http->storeEntry(NULL);
1678     e->unlock("clientReplyContext::forgetHit"); // may delete e
1679 }
1680
1681 void
1682 clientReplyContext::identifyStoreObject()
1683 {
1684     HttpRequest *r = http->request;
1685
1686     // client sent CC:no-cache or some other condition has been
1687     // encountered which prevents delivering a public/cached object.
1688     if (!r->flags.noCache || r->flags.internal) {
1689         lookingforstore = 5;
1690         StoreEntry::getPublicByRequest (this, r);
1691     } else {
1692         identifyFoundObject (NullStoreEntry::getInstance());
1693     }
1694 }
1695
1696 /**
1697  * Check state of the current StoreEntry object.
1698  * to see if we can determine the final status of the request.
1699  */
1700 void
1701 clientReplyContext::identifyFoundObject(StoreEntry *newEntry)
1702 {
1703     StoreEntry *e = newEntry;
1704     HttpRequest *r = http->request;
1705
1706     /** \li If the entry received isNull() then we ignore it. */
1707     if (e->isNull()) {
1708         http->storeEntry(NULL);
1709     } else {
1710         http->storeEntry(e);
1711     }
1712
1713     e = http->storeEntry();
1714
1715     /* Release IP-cache entries on reload */
1716     /** \li If the request has no-cache flag set or some no_cache HACK in operation we
1717       * 'invalidate' the cached IP entries for this request ???
1718       */
1719     if (r->flags.noCache || r->flags.noCacheHack())
1720         ipcacheInvalidateNegative(r->url.host());
1721
1722 #if USE_CACHE_DIGESTS
1723     lookup_type = http->storeEntry() ? "HIT" : "MISS";
1724 #endif
1725
1726     if (NULL == http->storeEntry()) {
1727         /** \li If no StoreEntry object is current assume this object isn't in the cache set MISS*/
1728         debugs(85, 3, "StoreEntry is NULL -  MISS");
1729         http->logType = LOG_TCP_MISS;
1730         doGetMoreData();
1731         return;
1732     }
1733
1734     if (Config.onoff.offline) {
1735         /** \li If we are running in offline mode set to HIT */
1736         debugs(85, 3, "offline HIT " << *e);
1737         http->logType = LOG_TCP_HIT;
1738         doGetMoreData();
1739         return;
1740     }
1741
1742     if (http->redirect.status) {
1743         /** \li If redirection status is True force this to be a MISS */
1744         debugs(85, 3, "REDIRECT status forced StoreEntry to NULL (no body on 3XX responses) " << *e);
1745         forgetHit();
1746         http->logType = LOG_TCP_REDIRECT;
1747         doGetMoreData();
1748         return;
1749     }
1750
1751     if (!e->validToSend()) {
1752         debugs(85, 3, "!storeEntryValidToSend MISS " << *e);
1753         forgetHit();
1754         http->logType = LOG_TCP_MISS;
1755         doGetMoreData();
1756         return;
1757     }
1758
1759     if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
1760         /* \li Special entries are always hits, no matter what the client says */
1761         debugs(85, 3, "ENTRY_SPECIAL HIT " << *e);
1762         http->logType = LOG_TCP_HIT;
1763         doGetMoreData();
1764         return;
1765     }
1766
1767     if (r->flags.noCache) {
1768         debugs(85, 3, "no-cache REFRESH MISS " << *e);
1769         forgetHit();
1770         http->logType = LOG_TCP_CLIENT_REFRESH_MISS;
1771         doGetMoreData();
1772         return;
1773     }
1774
1775     debugs(85, 3, "default HIT " << *e);
1776     http->logType = LOG_TCP_HIT;
1777     doGetMoreData();
1778 }
1779
1780 /**
1781  * Request more data from the store for the client Stream
1782  * This is *the* entry point to this module.
1783  *
1784  * Preconditions:
1785  *  - This is the head of the list.
1786  *  - There is at least one more node.
1787  *  - Data context is not null
1788  */
1789 void
1790 clientGetMoreData(clientStreamNode * aNode, ClientHttpRequest * http)
1791 {
1792     /* Test preconditions */
1793     assert(aNode != NULL);
1794     assert(cbdataReferenceValid(aNode));
1795     assert(aNode->node.prev == NULL);
1796     assert(aNode->node.next != NULL);
1797     clientReplyContext *context = dynamic_cast<clientReplyContext *>(aNode->data.getRaw());
1798     assert (context);
1799     assert(context->http == http);
1800
1801     clientStreamNode *next = ( clientStreamNode *)aNode->node.next->data;
1802
1803     if (!context->ourNode)
1804         context->ourNode = aNode;
1805
1806     /* no cbdatareference, this is only used once, and safely */
1807     if (context->flags.storelogiccomplete) {
1808         StoreIOBuffer tempBuffer;
1809         tempBuffer.offset = next->readBuffer.offset + context->headers_sz;
1810         tempBuffer.length = next->readBuffer.length;
1811         tempBuffer.data = next->readBuffer.data;
1812
1813         storeClientCopy(context->sc, http->storeEntry(),
1814                         tempBuffer, clientReplyContext::SendMoreData, context);
1815         return;
1816     }
1817
1818     if (context->http->request->method == Http::METHOD_PURGE) {
1819         context->purgeRequest();
1820         return;
1821     }
1822
1823     // OPTIONS with Max-Forwards:0 handled in clientProcessRequest()
1824
1825     if (context->http->request->method == Http::METHOD_TRACE) {
1826         if (context->http->request->header.getInt64(Http::HdrType::MAX_FORWARDS) == 0) {
1827             context->traceReply(aNode);
1828             return;
1829         }
1830
1831         /* continue forwarding, not finished yet. */
1832         http->logType = LOG_TCP_MISS;
1833
1834         context->doGetMoreData();
1835     } else
1836         context->identifyStoreObject();
1837 }
1838
1839 void
1840 clientReplyContext::doGetMoreData()
1841 {
1842     /* We still have to do store logic processing - vary, cache hit etc */
1843     if (http->storeEntry() != NULL) {
1844         /* someone found the object in the cache for us */
1845         StoreIOBuffer localTempBuffer;
1846
1847         http->storeEntry()->lock("clientReplyContext::doGetMoreData");
1848
1849         http->storeEntry()->ensureMemObject(storeId(), http->log_uri, http->request->method);
1850
1851         sc = storeClientListAdd(http->storeEntry(), this);
1852 #if USE_DELAY_POOLS
1853         sc->setDelayId(DelayId::DelayClient(http));
1854 #endif
1855
1856         assert(http->logType.oldType == LOG_TCP_HIT);
1857         reqofs = 0;
1858         /* guarantee nothing has been sent yet! */
1859         assert(http->out.size == 0);
1860         assert(http->out.offset == 0);
1861
1862         if (ConnStateData *conn = http->getConn()) {
1863             if (Ip::Qos::TheConfig.isHitTosActive()) {
1864                 Ip::Qos::doTosLocalHit(conn->clientConnection);
1865             }
1866
1867             if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
1868                 Ip::Qos::doNfmarkLocalHit(conn->clientConnection);
1869             }
1870         }
1871
1872         localTempBuffer.offset = reqofs;
1873         localTempBuffer.length = getNextNode()->readBuffer.length;
1874         localTempBuffer.data = getNextNode()->readBuffer.data;
1875         storeClientCopy(sc, http->storeEntry(), localTempBuffer, CacheHit, this);
1876     } else {
1877         /* MISS CASE, http->logType is already set! */
1878         processMiss();
1879     }
1880 }
1881
1882 /** The next node has removed itself from the stream. */
1883 void
1884 clientReplyDetach(clientStreamNode * node, ClientHttpRequest * http)
1885 {
1886     /** detach from the stream */
1887     clientStreamDetach(node, http);
1888 }
1889
1890 /**
1891  * Accepts chunk of a http message in buf, parses prefix, filters headers and
1892  * such, writes processed message to the message recipient
1893  */
1894 void
1895 clientReplyContext::SendMoreData(void *data, StoreIOBuffer result)
1896 {
1897     clientReplyContext *context = static_cast<clientReplyContext *>(data);
1898     context->sendMoreData (result);
1899 }
1900
1901 void
1902 clientReplyContext::makeThisHead()
1903 {
1904     /* At least, I think thats what this does */
1905     dlinkDelete(&http->active, &ClientActiveRequests);
1906     dlinkAdd(http, &http->active, &ClientActiveRequests);
1907 }
1908
1909 bool
1910 clientReplyContext::errorInStream(StoreIOBuffer const &result, size_t const &sizeToProcess)const
1911 {
1912     return /* aborted request */
1913         (http->storeEntry() && EBIT_TEST(http->storeEntry()->flags, ENTRY_ABORTED)) ||
1914         /* Upstream read error */ (result.flags.error) ||
1915         /* Upstream EOF */ (sizeToProcess == 0);
1916 }
1917
1918 void
1919 clientReplyContext::sendStreamError(StoreIOBuffer const &result)
1920 {
1921     /** call clientWriteComplete so the client socket gets closed
1922      *
1923      * We call into the stream, because we don't know that there is a
1924      * client socket!
1925      */
1926     debugs(88, 5, "clientReplyContext::sendStreamError: A stream error has occured, marking as complete and sending no data.");
1927     StoreIOBuffer localTempBuffer;
1928     flags.complete = 1;
1929     http->request->flags.streamError = true;
1930     localTempBuffer.flags.error = result.flags.error;
1931     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1932                          localTempBuffer);
1933 }
1934
1935 void
1936 clientReplyContext::pushStreamData(StoreIOBuffer const &result, char *source)
1937 {
1938     StoreIOBuffer localTempBuffer;
1939
1940     if (result.length == 0) {
1941         debugs(88, 5, "clientReplyContext::pushStreamData: marking request as complete due to 0 length store result");
1942         flags.complete = 1;
1943     }
1944
1945     assert(result.offset - headers_sz == next()->readBuffer.offset);
1946     localTempBuffer.offset = result.offset - headers_sz;
1947     localTempBuffer.length = result.length;
1948
1949     if (localTempBuffer.length)
1950         localTempBuffer.data = source;
1951
1952     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
1953                          localTempBuffer);
1954 }
1955
1956 clientStreamNode *
1957 clientReplyContext::next() const
1958 {
1959     assert ( (clientStreamNode*)http->client_stream.head->next->data == getNextNode());
1960     return getNextNode();
1961 }
1962
1963 void
1964 clientReplyContext::sendBodyTooLargeError()
1965 {
1966     Ip::Address tmp_noaddr;
1967     tmp_noaddr.setNoAddr(); // TODO: make a global const
1968     http->logType = LOG_TCP_DENIED_REPLY;
1969     ErrorState *err = clientBuildError(ERR_TOO_BIG, Http::scForbidden, NULL,
1970                                        http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
1971                                        http->request);
1972     removeClientStoreReference(&(sc), http);
1973     HTTPMSGUNLOCK(reply);
1974     startError(err);
1975
1976 }
1977
1978 /// send 412 (Precondition Failed) to client
1979 void
1980 clientReplyContext::sendPreconditionFailedError()
1981 {
1982     http->logType = LOG_TCP_HIT;
1983     Ip::Address tmp_noaddr;
1984     tmp_noaddr.setNoAddr();
1985     ErrorState *const err =
1986         clientBuildError(ERR_PRECONDITION_FAILED, Http::scPreconditionFailed,
1987                          NULL, http->getConn() ? http->getConn()->clientConnection->remote : tmp_noaddr, http->request);
1988     removeClientStoreReference(&sc, http);
1989     HTTPMSGUNLOCK(reply);
1990     startError(err);
1991 }
1992
1993 /// send 304 (Not Modified) to client
1994 void
1995 clientReplyContext::sendNotModified()
1996 {
1997     StoreEntry *e = http->storeEntry();
1998     const time_t timestamp = e->timestamp;
1999     HttpReply *const temprep = e->getReply()->make304();
2000     // log as TCP_INM_HIT if code 304 generated for
2001     // If-None-Match request
2002     if (!http->request->flags.ims)
2003         http->logType = LOG_TCP_INM_HIT;
2004     else
2005         http->logType = LOG_TCP_IMS_HIT;
2006     removeClientStoreReference(&sc, http);
2007     createStoreEntry(http->request->method, RequestFlags());
2008     e = http->storeEntry();
2009     // Copy timestamp from the original entry so the 304
2010     // reply has a meaningful Age: header.
2011     e->timestampsSet();
2012     e->timestamp = timestamp;
2013     e->replaceHttpReply(temprep);
2014     e->complete();
2015     /*
2016      * TODO: why put this in the store and then serialise it and
2017      * then parse it again. Simply mark the request complete in
2018      * our context and write the reply struct to the client side.
2019      */
2020     triggerInitialStoreRead();
2021 }
2022
2023 /// send 304 (Not Modified) or 412 (Precondition Failed) to client
2024 /// depending on request method
2025 void
2026 clientReplyContext::sendNotModifiedOrPreconditionFailedError()
2027 {
2028     if (http->request->method == Http::METHOD_GET ||
2029             http->request->method == Http::METHOD_HEAD)
2030         sendNotModified();
2031     else
2032         sendPreconditionFailedError();
2033 }
2034
2035 void
2036 clientReplyContext::processReplyAccess ()
2037 {
2038     /* NP: this should probably soft-fail to a zero-sized-reply error ?? */
2039     assert(reply);
2040
2041     /** Don't block our own responses or HTTP status messages */
2042     if (http->logType.oldType == LOG_TCP_DENIED ||
2043             http->logType.oldType == LOG_TCP_DENIED_REPLY ||
2044             alwaysAllowResponse(reply->sline.status())) {
2045         headers_sz = reply->hdr_sz;
2046         processReplyAccessResult(ACCESS_ALLOWED);
2047         return;
2048     }
2049
2050     /** Check for reply to big error */
2051     if (reply->expectedBodyTooLarge(*http->request)) {
2052         sendBodyTooLargeError();
2053         return;
2054     }
2055
2056     headers_sz = reply->hdr_sz;
2057
2058     /** check for absent access controls (permit by default) */
2059     if (!Config.accessList.reply) {
2060         processReplyAccessResult(ACCESS_ALLOWED);
2061         return;
2062     }
2063
2064     /** Process http_reply_access lists */
2065     ACLFilledChecklist *replyChecklist =
2066         clientAclChecklistCreate(Config.accessList.reply, http);
2067     replyChecklist->reply = reply;
2068     HTTPMSGLOCK(replyChecklist->reply);
2069     replyChecklist->nonBlockingCheck(ProcessReplyAccessResult, this);
2070 }
2071
2072 void
2073 clientReplyContext::ProcessReplyAccessResult(allow_t rv, void *voidMe)
2074 {
2075     clientReplyContext *me = static_cast<clientReplyContext *>(voidMe);
2076     me->processReplyAccessResult(rv);
2077 }
2078
2079 void
2080 clientReplyContext::processReplyAccessResult(const allow_t &accessAllowed)
2081 {
2082     debugs(88, 2, "The reply for " << http->request->method
2083            << ' ' << http->uri << " is " << accessAllowed << ", because it matched "
2084            << (AclMatchedName ? AclMatchedName : "NO ACL's"));
2085
2086     if (!accessAllowed.allowed()) {
2087         ErrorState *err;
2088         err_type page_id;
2089         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1);
2090
2091         http->logType = LOG_TCP_DENIED_REPLY;
2092
2093         if (page_id == ERR_NONE)
2094             page_id = ERR_ACCESS_DENIED;
2095
2096         Ip::Address tmp_noaddr;
2097         tmp_noaddr.setNoAddr();
2098         err = clientBuildError(page_id, Http::scForbidden, NULL,
2099                                http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmp_noaddr,
2100                                http->request);
2101
2102         removeClientStoreReference(&sc, http);
2103
2104         HTTPMSGUNLOCK(reply);
2105
2106         startError(err);
2107
2108         return;
2109     }
2110
2111     /* Ok, the reply is allowed, */
2112     http->loggingEntry(http->storeEntry());
2113
2114     ssize_t body_size = reqofs - reply->hdr_sz;
2115     if (body_size < 0) {
2116         reqofs = reply->hdr_sz;
2117         body_size = 0;
2118     }
2119
2120     debugs(88, 3, "clientReplyContext::sendMoreData: Appending " <<
2121            (int) body_size << " bytes after " << reply->hdr_sz <<
2122            " bytes of headers");
2123
2124 #if USE_SQUID_ESI
2125
2126     if (http->flags.accel && reply->sline.status() != Http::scForbidden &&
2127             !alwaysAllowResponse(reply->sline.status()) &&
2128             esiEnableProcessing(reply)) {
2129         debugs(88, 2, "Enabling ESI processing for " << http->uri);
2130         clientStreamInsertHead(&http->client_stream, esiStreamRead,
2131                                esiProcessStream, esiStreamDetach, esiStreamStatus, NULL);
2132     }
2133
2134 #endif
2135
2136     if (http->request->method == Http::METHOD_HEAD) {
2137         /* do not forward body for HEAD replies */
2138         body_size = 0;
2139         http->flags.done_copying = true;
2140         flags.complete = 1;
2141     }
2142
2143     assert (!flags.headersSent);
2144     flags.headersSent = true;
2145
2146     StoreIOBuffer localTempBuffer;
2147     char *buf = next()->readBuffer.data;
2148     char *body_buf = buf + reply->hdr_sz;
2149
2150     //Server side may disable ranges under some circumstances.
2151
2152     if ((!http->request->range))
2153         next()->readBuffer.offset = 0;
2154
2155     body_buf -= next()->readBuffer.offset;
2156
2157     if (next()->readBuffer.offset != 0) {
2158         if (next()->readBuffer.offset > body_size) {
2159             /* Can't use any of the body we received. send nothing */
2160             localTempBuffer.length = 0;
2161             localTempBuffer.data = NULL;
2162         } else {
2163             localTempBuffer.length = body_size - next()->readBuffer.offset;
2164             localTempBuffer.data = body_buf + next()->readBuffer.offset;
2165         }
2166     } else {
2167         localTempBuffer.length = body_size;
2168         localTempBuffer.data = body_buf;
2169     }
2170
2171     /* TODO??: move the data in the buffer back by the request header size */
2172     clientStreamCallback((clientStreamNode *)http->client_stream.head->data,
2173                          http, reply, localTempBuffer);
2174
2175     return;
2176 }
2177
2178 void
2179 clientReplyContext::sendMoreData (StoreIOBuffer result)
2180 {
2181     if (deleting)
2182         return;
2183
2184     StoreEntry *entry = http->storeEntry();
2185
2186     if (ConnStateData * conn = http->getConn()) {
2187         if (!conn->isOpen()) {
2188             debugs(33,3, "not sending more data to closing connection " << conn->clientConnection);
2189             return;
2190         }
2191         if (conn->pinning.zeroReply) {
2192             debugs(33,3, "not sending more data after a pinned zero reply " << conn->clientConnection);
2193             return;
2194         }
2195
2196         if (reqofs==0 && !http->logType.isTcpHit()) {
2197             if (Ip::Qos::TheConfig.isHitTosActive()) {
2198                 Ip::Qos::doTosLocalMiss(conn->clientConnection, http->request->hier.code);
2199             }
2200             if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
2201                 Ip::Qos::doNfmarkLocalMiss(conn->clientConnection, http->request->hier.code);
2202             }
2203         }
2204
2205         debugs(88, 5, conn->clientConnection <<
2206                " '" << entry->url() << "'" <<
2207                " out.offset=" << http->out.offset);
2208     }
2209
2210     char *buf = next()->readBuffer.data;
2211
2212     if (buf != result.data) {
2213         /* we've got to copy some data */
2214         assert(result.length <= next()->readBuffer.length);
2215         memcpy(buf, result.data, result.length);
2216     }
2217
2218     /* We've got the final data to start pushing... */
2219     flags.storelogiccomplete = 1;
2220
2221     reqofs += result.length;
2222
2223     assert(reqofs <= HTTP_REQBUF_SZ || flags.headersSent);
2224
2225     assert(http->request != NULL);
2226
2227     /* ESI TODO: remove this assert once everything is stable */
2228     assert(http->client_stream.head->data
2229            && cbdataReferenceValid(http->client_stream.head->data));
2230
2231     makeThisHead();
2232
2233     debugs(88, 5, "clientReplyContext::sendMoreData: " << http->uri << ", " <<
2234            reqofs << " bytes (" << result.length <<
2235            " new bytes)");
2236
2237     /* update size of the request */
2238     reqsize = reqofs;
2239
2240     if (errorInStream(result, reqofs)) {
2241         sendStreamError(result);
2242         return;
2243     }
2244
2245     if (flags.headersSent) {
2246         pushStreamData (result, buf);
2247         return;
2248     }
2249
2250     cloneReply();
2251
2252 #if USE_DELAY_POOLS
2253     if (sc)
2254         sc->setDelayId(DelayId::DelayClient(http,reply));
2255 #endif
2256
2257     /* handle headers */
2258
2259     if (Config.onoff.log_mime_hdrs) {
2260         size_t k;
2261
2262         if ((k = headersEnd(buf, reqofs))) {
2263             safe_free(http->al->headers.reply);
2264             http->al->headers.reply = (char *)xcalloc(k + 1, 1);
2265             xstrncpy(http->al->headers.reply, buf, k);
2266         }
2267     }
2268
2269     holdingBuffer = result;
2270     processReplyAccess();
2271     return;
2272 }
2273
2274 /* Using this breaks the client layering just a little!
2275  */
2276 void
2277 clientReplyContext::createStoreEntry(const HttpRequestMethod& m, RequestFlags reqFlags)
2278 {
2279     assert(http != NULL);
2280     /*
2281      * For erroneous requests, we might not have a h->request,
2282      * so make a fake one.
2283      */
2284
2285     if (http->request == NULL) {
2286         const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initClient);
2287         http->request = new HttpRequest(m, AnyP::PROTO_NONE, "http", null_string, mx);
2288         HTTPMSGLOCK(http->request);
2289     }
2290
2291     StoreEntry *e = storeCreateEntry(storeId(), http->log_uri, reqFlags, m);
2292
2293     // Make entry collapsable ASAP, to increase collapsing chances for others,
2294     // TODO: every must-revalidate and similar request MUST reach the origin,
2295     // but do we have to prohibit others from collapsing on that request?
2296     if (Config.onoff.collapsed_forwarding && reqFlags.cachable &&
2297             !reqFlags.needValidation &&
2298             (m == Http::METHOD_GET || m == Http::METHOD_HEAD)) {
2299         // make the entry available for future requests now
2300         Store::Root().allowCollapsing(e, reqFlags, m);
2301     }
2302
2303     sc = storeClientListAdd(e, this);
2304
2305 #if USE_DELAY_POOLS
2306     sc->setDelayId(DelayId::DelayClient(http));
2307 #endif
2308
2309     reqofs = 0;
2310
2311     reqsize = 0;
2312
2313     /* I don't think this is actually needed! -- adrian */
2314     /* http->reqbuf = http->norm_reqbuf; */
2315     //    assert(http->reqbuf == http->norm_reqbuf);
2316     /* The next line is illegal because we don't know if the client stream
2317      * buffers have been set up
2318      */
2319     //    storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf,
2320     //        SendMoreData, this);
2321     /* So, we mark the store logic as complete */
2322     flags.storelogiccomplete = 1;
2323
2324     /* and get the caller to request a read, from whereever they are */
2325     /* NOTE: after ANY data flows down the pipe, even one step,
2326      * this function CAN NOT be used to manage errors
2327      */
2328     http->storeEntry(e);
2329 }
2330
2331 ErrorState *
2332 clientBuildError(err_type page_id, Http::StatusCode status, char const *url,
2333                  Ip::Address &src_addr, HttpRequest * request)
2334 {
2335     ErrorState *err = new ErrorState(page_id, status, request);
2336     err->src_addr = src_addr;
2337
2338     if (url)
2339         err->url = xstrdup(url);
2340
2341     return err;
2342 }
2343